VERY experimental Nintendo Switch 2 emulator
C++
vfs_real: add file LRU cache for open file limits
Liamefaaa6e0bf08
diff --git a/src/core/file_sys/vfs_real.cpp b/src/core/file_sys/vfs_real.cppindex cc00762385..a63629b80f 100644--- a/src/core/file_sys/vfs_real.cpp+++ b/src/core/file_sys/vfs_real.cpp@@ -25,6 +25,8 @@ namespace FS = Common::FS;namespace {+constexpr size_t MaxOpenFiles = 512;+constexpr FS::FileAccessMode ModeFlagsToFileAccessMode(Mode mode) {switch (mode) {case Mode::Read:@@ -73,24 +75,17 @@ VfsEntryType RealVfsFilesystem::GetEntryType(std::string_view path_) const {VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) {const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);- if (const auto weak_iter = cache.find(path); weak_iter != cache.cend()) {- const auto& weak = weak_iter->second;-- if (!weak.expired()) {- return std::shared_ptr<RealVfsFile>(new RealVfsFile(*this, weak.lock(), path, perms));- }- }+ this->EvictSingleReference();auto backing = FS::FileOpen(path, ModeFlagsToFileAccessMode(perms), FS::FileType::BinaryFile);-if (!backing) {return nullptr;}- cache.insert_or_assign(path, std::move(backing));-- // Cannot use make_shared as RealVfsFile constructor is private- return std::shared_ptr<RealVfsFile>(new RealVfsFile(*this, backing, path, perms));+ num_open_files++;+ auto reference = std::make_unique<FileReference>(std::move(backing));+ this->InsertReferenceIntoList(*reference);+ return std::shared_ptr<RealVfsFile>(new RealVfsFile(*this, std::move(reference), path, perms));}VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) {@@ -123,51 +118,19 @@ VirtualFile RealVfsFilesystem::CopyFile(std::string_view old_path_, std::string_VirtualFile RealVfsFilesystem::MoveFile(std::string_view old_path_, std::string_view new_path_) {const auto old_path = FS::SanitizePath(old_path_, FS::DirectorySeparator::PlatformDefault);const auto new_path = FS::SanitizePath(new_path_, FS::DirectorySeparator::PlatformDefault);- const auto cached_file_iter = cache.find(old_path);-- if (cached_file_iter != cache.cend()) {- auto file = cached_file_iter->second.lock();-- if (!cached_file_iter->second.expired()) {- file->Close();- }-- if (!FS::RenameFile(old_path, new_path)) {- return nullptr;- }-- cache.erase(old_path);- file->Open(new_path, FS::FileAccessMode::Read, FS::FileType::BinaryFile);- if (file->IsOpen()) {- cache.insert_or_assign(new_path, std::move(file));- } else {- LOG_ERROR(Service_FS, "Failed to open path {} in order to re-cache it", new_path);- }- } else {- ASSERT(false);+ if (!FS::RenameFile(old_path, new_path)) {return nullptr;}-return OpenFile(new_path, Mode::ReadWrite);}bool RealVfsFilesystem::DeleteFile(std::string_view path_) {const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);- const auto cached_iter = cache.find(path);-- if (cached_iter != cache.cend()) {- if (!cached_iter->second.expired()) {- cached_iter->second.lock()->Close();- }- cache.erase(path);- }-return FS::RemoveFile(path);}VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms) {const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);- // Cannot use make_shared as RealVfsDirectory constructor is privatereturn std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));}@@ -176,7 +139,6 @@ VirtualDir RealVfsFilesystem::CreateDirectory(std::string_view path_, Mode permsif (!FS::CreateDirs(path)) {return nullptr;}- // Cannot use make_shared as RealVfsDirectory constructor is privatereturn std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));}@@ -194,73 +156,102 @@ VirtualDir RealVfsFilesystem::MoveDirectory(std::string_view old_path_,if (!FS::RenameDir(old_path, new_path)) {return nullptr;}+ return OpenDirectory(new_path, Mode::ReadWrite);+}- for (auto& kv : cache) {- // If the path in the cache doesn't start with old_path, then bail on this file.- if (kv.first.rfind(old_path, 0) != 0) {- continue;- }+bool RealVfsFilesystem::DeleteDirectory(std::string_view path_) {+ const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);+ return FS::RemoveDirRecursively(path);+}- const auto file_old_path =- FS::SanitizePath(kv.first, FS::DirectorySeparator::PlatformDefault);- auto file_new_path = FS::SanitizePath(new_path + '/' + kv.first.substr(old_path.size()),- FS::DirectorySeparator::PlatformDefault);- const auto& cached = cache[file_old_path];+void RealVfsFilesystem::RefreshReference(const std::string& path, Mode perms,+ FileReference& reference) {+ // Temporarily remove from list.+ this->RemoveReferenceFromList(reference);- if (cached.expired()) {- continue;- }+ // Restore file if needed.+ if (!reference.file) {+ this->EvictSingleReference();- auto file = cached.lock();- cache.erase(file_old_path);- file->Open(file_new_path, FS::FileAccessMode::Read, FS::FileType::BinaryFile);- if (file->IsOpen()) {- cache.insert_or_assign(std::move(file_new_path), std::move(file));- } else {- LOG_ERROR(Service_FS, "Failed to open path {} in order to re-cache it", file_new_path);+ reference.file =+ FS::FileOpen(path, ModeFlagsToFileAccessMode(perms), FS::FileType::BinaryFile);+ if (reference.file) {+ num_open_files++;}}- return OpenDirectory(new_path, Mode::ReadWrite);+ // Reinsert into list.+ this->InsertReferenceIntoList(reference);}-bool RealVfsFilesystem::DeleteDirectory(std::string_view path_) {- const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault);+void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference) {+ // Remove from list.+ this->RemoveReferenceFromList(*reference);- for (auto& kv : cache) {- // If the path in the cache doesn't start with path, then bail on this file.- if (kv.first.rfind(path, 0) != 0) {- continue;- }+ // Close the file.+ if (reference->file) {+ reference->file.reset();+ num_open_files--;+ }+}- const auto& entry = cache[kv.first];- if (!entry.expired()) {- entry.lock()->Close();- }+void RealVfsFilesystem::EvictSingleReference() {+ if (num_open_files < MaxOpenFiles || open_references.empty()) {+ return;+ }++ // Get and remove from list.+ auto& reference = open_references.back();+ this->RemoveReferenceFromList(reference);- cache.erase(kv.first);+ // Close the file.+ if (reference.file) {+ reference.file.reset();+ num_open_files--;}- return FS::RemoveDirRecursively(path);+ // Reinsert into closed list.+ this->InsertReferenceIntoList(reference);+}++void RealVfsFilesystem::InsertReferenceIntoList(FileReference& reference) {+ if (reference.file) {+ open_references.push_front(reference);+ } else {+ closed_references.push_front(reference);+ }+}++void RealVfsFilesystem::RemoveReferenceFromList(FileReference& reference) {+ if (reference.file) {+ open_references.erase(open_references.iterator_to(reference));+ } else {+ closed_references.erase(closed_references.iterator_to(reference));+ }}-RealVfsFile::RealVfsFile(RealVfsFilesystem& base_, std::shared_ptr<FS::IOFile> backing_,+RealVfsFile::RealVfsFile(RealVfsFilesystem& base_, std::unique_ptr<FileReference> reference_,const std::string& path_, Mode perms_)- : base(base_), backing(std::move(backing_)), path(path_), parent_path(FS::GetParentPath(path_)),- path_components(FS::SplitPathComponents(path_)), perms(perms_) {}+ : base(base_), reference(std::move(reference_)), path(path_),+ parent_path(FS::GetParentPath(path_)), path_components(FS::SplitPathComponents(path_)),+ perms(perms_) {}-RealVfsFile::~RealVfsFile() = default;+RealVfsFile::~RealVfsFile() {+ base.DropReference(std::move(reference));+}std::string RealVfsFile::GetName() const {return path_components.back();}std::size_t RealVfsFile::GetSize() const {- return backing->GetSize();+ base.RefreshReference(path, perms, *reference);+ return reference->file ? reference->file->GetSize() : 0;}bool RealVfsFile::Resize(std::size_t new_size) {- return backing->SetSize(new_size);+ base.RefreshReference(path, perms, *reference);+ return reference->file ? reference->file->SetSize(new_size) : false;}VirtualDir RealVfsFile::GetContainingDirectory() const {@@ -276,27 +267,25 @@ bool RealVfsFile::IsReadable() const {}std::size_t RealVfsFile::Read(u8* data, std::size_t length, std::size_t offset) const {- if (!backing->Seek(static_cast<s64>(offset))) {+ base.RefreshReference(path, perms, *reference);+ if (!reference->file || !reference->file->Seek(static_cast<s64>(offset))) {return 0;}- return backing->ReadSpan(std::span{data, length});+ return reference->file->ReadSpan(std::span{data, length});}std::size_t RealVfsFile::Write(const u8* data, std::size_t length, std::size_t offset) {- if (!backing->Seek(static_cast<s64>(offset))) {+ base.RefreshReference(path, perms, *reference);+ if (!reference->file || !reference->file->Seek(static_cast<s64>(offset))) {return 0;}- return backing->WriteSpan(std::span{data, length});+ return reference->file->WriteSpan(std::span{data, length});}bool RealVfsFile::Rename(std::string_view name) {return base.MoveFile(path, parent_path + '/' + std::string(name)) != nullptr;}-void RealVfsFile::Close() {- backing->Close();-}-// TODO(DarkLordZach): MSVC would not let me combine the following two functions using 'if// constexpr' because there is a compile error in the branch not used.diff --git a/src/core/file_sys/vfs_real.h b/src/core/file_sys/vfs_real.hindex b92c84316e..f29c69fbd2 100644--- a/src/core/file_sys/vfs_real.h+++ b/src/core/file_sys/vfs_real.h@@ -4,7 +4,7 @@#pragma once#include <string_view>-#include <boost/container/flat_map.hpp>+#include "common/intrusive_list.h"#include "core/file_sys/mode.h"#include "core/file_sys/vfs.h"@@ -14,6 +14,12 @@ class IOFile;namespace FileSys {+struct FileReference : public Common::IntrusiveListBaseNode<FileReference> {+ FileReference(std::shared_ptr<Common::FS::IOFile>&& f) : file(f) {}+ std::shared_ptr<Common::FS::IOFile> file;+};++class RealVfsFile;class RealVfsFilesystem : public VfsFilesystem {public:RealVfsFilesystem();@@ -35,7 +41,20 @@ public:bool DeleteDirectory(std::string_view path) override;private:- boost::container::flat_map<std::string, std::weak_ptr<Common::FS::IOFile>> cache;+ using ReferenceListType = Common::IntrusiveListBaseTraits<FileReference>::ListType;+ ReferenceListType open_references;+ ReferenceListType closed_references;+ size_t num_open_files{};++private:+ friend class RealVfsFile;+ void RefreshReference(const std::string& path, Mode perms, FileReference& reference);+ void DropReference(std::unique_ptr<FileReference>&& reference);+ void EvictSingleReference();++private:+ void InsertReferenceIntoList(FileReference& reference);+ void RemoveReferenceFromList(FileReference& reference);};// An implementation of VfsFile that represents a file on the user's computer.@@ -57,13 +76,11 @@ public:bool Rename(std::string_view name) override;private:- RealVfsFile(RealVfsFilesystem& base, std::shared_ptr<Common::FS::IOFile> backing,+ RealVfsFile(RealVfsFilesystem& base, std::unique_ptr<FileReference> reference,const std::string& path, Mode perms = Mode::Read);- void Close();-RealVfsFilesystem& base;- std::shared_ptr<Common::FS::IOFile> backing;+ std::unique_ptr<FileReference> reference;std::string path;std::string parent_path;std::vector<std::string> path_components;