From 584e8b5c52db95a92f818a0fc1b2a64f8a42e524 Mon Sep 17 00:00:00 2001 From: kkoniuszy <120419423+kkoniuszy@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:21:22 +0200 Subject: [PATCH 1/4] host_memory: merge adjacent placeholder mappings on Linux Track the private anonymous placeholder mappings created by Unmap() and wherever possible, replace existing placeholders with larger ones instead of creating many small ones. This helps with the buildup of mappings in /proc/YUZU_PID/maps after a longer gaming session, improving stability without having to increase vm.max_map_count to a ridiculous value. The amount of placeholder mappings will no longer outgrow the amount of actual memfd mappings in cases of high memory fragmentation. --- src/common/host_memory.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 8e4f1f97a3..01457d8c60 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -14,6 +14,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif +#include #include #include #include @@ -415,6 +416,7 @@ public: madvise(virtual_base, virtual_size, MADV_HUGEPAGE); #endif + placeholders.add({0, virtual_size}); good = true; } @@ -423,6 +425,10 @@ public: } void Map(size_t virtual_offset, size_t host_offset, size_t length) { + { + std::scoped_lock lock{placeholder_mutex}; + placeholders.subtract({virtual_offset, virtual_offset + length}); + } void* ret = mmap(virtual_base + virtual_offset, length, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, host_offset); @@ -433,6 +439,19 @@ public: // The method name is wrong. We're still talking about the virtual range. // We don't want to unmap, we want to reserve this memory. + { + std::scoped_lock lock{placeholder_mutex}; + auto it = placeholders.find({virtual_offset - 1, virtual_offset + length + 1}); + + if (it != placeholders.end()) { + size_t prev_upper = virtual_offset + length; + virtual_offset = std::min(virtual_offset, it->lower()); + length = std::max(it->upper(), prev_upper) - virtual_offset; + } + + placeholders.add({virtual_offset, virtual_offset + length}); + } + void* ret = mmap(virtual_base + virtual_offset, length, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno)); @@ -476,6 +495,9 @@ private: } int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create + + boost::icl::interval_set placeholders; ///< Mapped placeholders + std::mutex placeholder_mutex; ///< Mutex for placeholders }; #else // ^^^ Linux ^^^ vvv Generic vvv From 790f91fcc57458ef63db7bf1baccc9d9ca2d57ce Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 3 Jun 2023 08:42:52 -0400 Subject: [PATCH 2/4] vfs: add vfs_cached for romfs build --- src/core/CMakeLists.txt | 2 + src/core/file_sys/patch_manager.cpp | 5 ++- src/core/file_sys/vfs_cached.cpp | 63 +++++++++++++++++++++++++++++ src/core/file_sys/vfs_cached.h | 31 ++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 src/core/file_sys/vfs_cached.cpp create mode 100644 src/core/file_sys/vfs_cached.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 45328158f6..e8bf68866e 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -106,6 +106,8 @@ add_library(core STATIC file_sys/system_archive/time_zone_binary.h file_sys/vfs.cpp file_sys/vfs.h + file_sys/vfs_cached.cpp + file_sys/vfs_cached.h file_sys/vfs_concat.cpp file_sys/vfs_concat.h file_sys/vfs_layered.cpp diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 4c80e13a92..f786f2add9 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -21,6 +21,7 @@ #include "core/file_sys/patch_manager.h" #include "core/file_sys/registered_cache.h" #include "core/file_sys/romfs.h" +#include "core/file_sys/vfs_cached.h" #include "core/file_sys/vfs_layered.h" #include "core/file_sys/vfs_vector.h" #include "core/hle/service/filesystem/filesystem.h" @@ -380,11 +381,11 @@ static void ApplyLayeredFS(VirtualFile& romfs, u64 title_id, ContentRecordType t auto romfs_dir = FindSubdirectoryCaseless(subdir, "romfs"); if (romfs_dir != nullptr) - layers.push_back(std::move(romfs_dir)); + layers.push_back(std::make_shared(romfs_dir)); auto ext_dir = FindSubdirectoryCaseless(subdir, "romfs_ext"); if (ext_dir != nullptr) - layers_ext.push_back(std::move(ext_dir)); + layers_ext.push_back(std::make_shared(ext_dir)); } // When there are no layers to apply, return early as there is no need to rebuild the RomFS diff --git a/src/core/file_sys/vfs_cached.cpp b/src/core/file_sys/vfs_cached.cpp new file mode 100644 index 0000000000..c3154ee819 --- /dev/null +++ b/src/core/file_sys/vfs_cached.cpp @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "core/file_sys/vfs_cached.h" +#include "core/file_sys/vfs_types.h" + +namespace FileSys { + +CachedVfsDirectory::CachedVfsDirectory(VirtualDir& source_dir) + : name(source_dir->GetName()), parent(source_dir->GetParentDirectory()) { + for (auto& dir : source_dir->GetSubdirectories()) { + dirs.emplace(dir->GetName(), std::make_shared(dir)); + } + for (auto& file : source_dir->GetFiles()) { + files.emplace(file->GetName(), file); + } +} + +CachedVfsDirectory::~CachedVfsDirectory() = default; + +VirtualFile CachedVfsDirectory::GetFile(std::string_view file_name) const { + auto it = files.find(file_name); + if (it != files.end()) { + return it->second; + } + + return nullptr; +} + +VirtualDir CachedVfsDirectory::GetSubdirectory(std::string_view dir_name) const { + auto it = dirs.find(dir_name); + if (it != dirs.end()) { + return it->second; + } + + return nullptr; +} + +std::vector CachedVfsDirectory::GetFiles() const { + std::vector out; + for (auto& [file_name, file] : files) { + out.push_back(file); + } + return out; +} + +std::vector CachedVfsDirectory::GetSubdirectories() const { + std::vector out; + for (auto& [dir_name, dir] : dirs) { + out.push_back(dir); + } + return out; +} + +std::string CachedVfsDirectory::GetName() const { + return name; +} + +VirtualDir CachedVfsDirectory::GetParentDirectory() const { + return parent; +} + +} // namespace FileSys diff --git a/src/core/file_sys/vfs_cached.h b/src/core/file_sys/vfs_cached.h new file mode 100644 index 0000000000..113acac12c --- /dev/null +++ b/src/core/file_sys/vfs_cached.h @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include "core/file_sys/vfs.h" + +namespace FileSys { + +class CachedVfsDirectory : public ReadOnlyVfsDirectory { +public: + CachedVfsDirectory(VirtualDir& source_directory); + + ~CachedVfsDirectory() override; + VirtualFile GetFile(std::string_view file_name) const override; + VirtualDir GetSubdirectory(std::string_view dir_name) const override; + std::vector GetFiles() const override; + std::vector GetSubdirectories() const override; + std::string GetName() const override; + VirtualDir GetParentDirectory() const override; + +private: + std::string name; + VirtualDir parent; + std::map> dirs; + std::map> files; +}; + +} // namespace FileSys From 6e23c84669d4d6a2cf9ed2ee5cfc0332c6d738db Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 3 Jun 2023 08:56:59 -0400 Subject: [PATCH 3/4] romfs: use vfs_cached for romfs output --- src/core/file_sys/romfs.cpp | 3 ++- src/core/file_sys/vfs_vector.cpp | 19 ------------------- src/core/file_sys/vfs_vector.h | 4 ---- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/core/file_sys/romfs.cpp b/src/core/file_sys/romfs.cpp index fb5683a6bc..614da21307 100644 --- a/src/core/file_sys/romfs.cpp +++ b/src/core/file_sys/romfs.cpp @@ -9,6 +9,7 @@ #include "core/file_sys/fsmitm_romfsbuild.h" #include "core/file_sys/romfs.h" #include "core/file_sys/vfs.h" +#include "core/file_sys/vfs_cached.h" #include "core/file_sys/vfs_concat.h" #include "core/file_sys/vfs_offset.h" #include "core/file_sys/vfs_vector.h" @@ -132,7 +133,7 @@ VirtualDir ExtractRomFS(VirtualFile file, RomFSExtractionType type) { out = out->GetSubdirectories().front(); } - return out; + return std::make_shared(out); } VirtualFile CreateRomFS(VirtualDir dir, VirtualDir ext) { diff --git a/src/core/file_sys/vfs_vector.cpp b/src/core/file_sys/vfs_vector.cpp index af1df4c514..251d9d7c9f 100644 --- a/src/core/file_sys/vfs_vector.cpp +++ b/src/core/file_sys/vfs_vector.cpp @@ -67,23 +67,6 @@ VectorVfsDirectory::VectorVfsDirectory(std::vector files_, VectorVfsDirectory::~VectorVfsDirectory() = default; -VirtualFile VectorVfsDirectory::GetFile(std::string_view file_name) const { - if (!optimized_file_index_built) { - optimized_file_index.clear(); - for (size_t i = 0; i < files.size(); i++) { - optimized_file_index.emplace(files[i]->GetName(), i); - } - optimized_file_index_built = true; - } - - const auto it = optimized_file_index.find(file_name); - if (it != optimized_file_index.end()) { - return files[it->second]; - } - - return nullptr; -} - std::vector VectorVfsDirectory::GetFiles() const { return files; } @@ -124,7 +107,6 @@ bool VectorVfsDirectory::DeleteSubdirectory(std::string_view subdir_name) { } bool VectorVfsDirectory::DeleteFile(std::string_view file_name) { - optimized_file_index_built = false; return FindAndRemoveVectorElement(files, file_name); } @@ -142,7 +124,6 @@ VirtualFile VectorVfsDirectory::CreateFile(std::string_view file_name) { } void VectorVfsDirectory::AddFile(VirtualFile file) { - optimized_file_index_built = false; files.push_back(std::move(file)); } diff --git a/src/core/file_sys/vfs_vector.h b/src/core/file_sys/vfs_vector.h index c9955755bf..bfedb6e42a 100644 --- a/src/core/file_sys/vfs_vector.h +++ b/src/core/file_sys/vfs_vector.h @@ -105,7 +105,6 @@ public: VirtualDir parent = nullptr); ~VectorVfsDirectory() override; - VirtualFile GetFile(std::string_view file_name) const override; std::vector GetFiles() const override; std::vector GetSubdirectories() const override; bool IsWritable() const override; @@ -127,9 +126,6 @@ private: VirtualDir parent; std::string name; - - mutable std::map> optimized_file_index; - mutable bool optimized_file_index_built{}; }; } // namespace FileSys From 58c54aecb6a80e8f8efeae9ae84871fd543dee53 Mon Sep 17 00:00:00 2001 From: Minionguyjpro Date: Sat, 3 Jun 2023 16:29:26 +0200 Subject: [PATCH 4/4] Update README.md (Add Android at builds description) (#10586) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd705ad707..1d5b4626fb 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ SPDX-License-Identifier: GPL-2.0-or-later

yuzu is the world's most popular, open-source, Nintendo Switch emulator — started by the creators of Citra.
-It is written in C++ with portability in mind, and we actively maintain builds for Windows and Linux. +It is written in C++ with portability in mind, and we actively maintain builds for Windows, Linux and Android.