From 6817173e03217c085a661b00cbb9679caac8135e Mon Sep 17 00:00:00 2001 From: lizzie Date: Sat, 6 Dec 2025 03:09:05 +0000 Subject: [PATCH 01/14] [fs] use mmap() to read files off the mmap system for higher throughput Signed-off-by: lizzie --- src/common/fs/file.cpp | 273 ++++++++++++++++++++++------------------- src/common/fs/file.h | 71 +++++------ 2 files changed, 181 insertions(+), 163 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index 854090e7b2..10c80c4d26 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -19,6 +19,9 @@ #include #else #include +#include +#include +#include #endif #ifdef _MSC_VER @@ -246,13 +249,10 @@ FileType IOFile::GetType() const { void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, FileShareFlag flag) { Close(); - file_path = path; file_access_mode = mode; file_type = type; - errno = 0; - #ifdef _WIN32 if (flag != FileShareFlag::ShareNone) { file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); @@ -262,51 +262,63 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File #elif ANDROID if (Android::IsContentUri(path)) { ASSERT_MSG(mode == FileAccessMode::Read, "Content URI file access is for read-only!"); - const auto fd = Android::OpenContentUri(path, Android::OpenMode::Read); + auto const fd = Android::OpenContentUri(path, Android::OpenMode::Read); if (fd != -1) { file = fdopen(fd, "r"); - const auto error_num = errno; - if (error_num != 0 && file == nullptr) { - LOG_ERROR(Common_Filesystem, "Error opening file: {}, error: {}", path.c_str(), - strerror(error_num)); - } + if (errno != 0 && file == nullptr) + LOG_ERROR(Common_Filesystem, "Error opening file: {}, error: {}", path.c_str(), strerror(errno)); } else { LOG_ERROR(Common_Filesystem, "Error opening file: {}", path.c_str()); } } else { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } +#elif defined(__unix__) && !defined(__HAIKU__) && !defined(__managarm__) + if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { + struct stat st; + mmap_fd = open(path.c_str(), O_RDONLY); + fstat(mmap_fd, &st); + mmap_size = st.st_size; + mmap_base = (u8*)mmap(nullptr, mmap_size, PROT_READ, MAP_PRIVATE, mmap_fd, 0); + } else { + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); + } #else file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); #endif - if (!IsOpen()) { const auto ec = std::error_code{errno, std::generic_category()}; LOG_ERROR(Common_Filesystem, "Failed to open the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + PathToUTF8String(file_path), ec.message()); } } void IOFile::Close() { - if (!IsOpen()) { - return; +#ifdef __unix__ + if (mmap_fd != -1) { + munmap(mmap_base, mmap_size); + close(mmap_fd); + mmap_fd = -1; } - - errno = 0; - - const auto close_result = std::fclose(file) == 0; - - if (!close_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to close the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); +#endif + if (file) { + errno = 0; + const auto close_result = std::fclose(file) == 0; + if (!close_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to close the file at path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + } + file = nullptr; } - - file = nullptr; } bool IOFile::IsOpen() const { +#ifdef __unix__ + return file != nullptr || mmap_fd != -1; +#else return file != nullptr; +#endif } std::string IOFile::ReadString(size_t length) const { @@ -323,137 +335,148 @@ size_t IOFile::WriteString(std::span string) const { } bool IOFile::Flush() const { - if (!IsOpen()) { - return false; - } - - errno = 0; - -#ifdef _WIN32 - const auto flush_result = std::fflush(file) == 0; -#else - const auto flush_result = std::fflush(file) == 0; +#ifdef __unix__ + ASSERT(mmap_fd == -1); #endif - - if (!flush_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to flush the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + if (file) { + errno = 0; +#ifdef _WIN32 + const auto flush_result = std::fflush(file) == 0; +#else + const auto flush_result = std::fflush(file) == 0; +#endif + if (!flush_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to flush the file at path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + } + return flush_result; } - - return flush_result; + return false; } bool IOFile::Commit() const { - if (!IsOpen()) { - return false; - } - - errno = 0; - -#ifdef _WIN32 - const auto commit_result = std::fflush(file) == 0 && _commit(fileno(file)) == 0; -#else - const auto commit_result = std::fflush(file) == 0 && fsync(fileno(file)) == 0; +#ifdef __unix__ + ASSERT(mmap_fd == -1); #endif - - if (!commit_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to commit the file at path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + if (file) { + errno = 0; +#ifdef _WIN32 + const auto commit_result = std::fflush(file) == 0 && _commit(fileno(file)) == 0; +#else + const auto commit_result = std::fflush(file) == 0 && fsync(fileno(file)) == 0; +#endif + if (!commit_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to commit the file at path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + } + return commit_result; } - - return commit_result; + return false; } bool IOFile::SetSize(u64 size) const { - if (!IsOpen()) { - return false; - } - - errno = 0; - -#ifdef _WIN32 - const auto set_size_result = _chsize_s(fileno(file), static_cast(size)) == 0; -#else - const auto set_size_result = ftruncate(fileno(file), static_cast(size)) == 0; +#ifdef __unix__ + ASSERT(mmap_fd == -1); #endif - - if (!set_size_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, "Failed to resize the file at path={}, size={}, ec_message={}", - PathToUTF8String(file_path), size, ec.message()); + if (file) { + errno = 0; +#ifdef _WIN32 + const auto set_size_result = _chsize_s(fileno(file), s64(size)) == 0; +#else + const auto set_size_result = ftruncate(fileno(file), s64(size)) == 0; +#endif + if (!set_size_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to resize the file at path={}, size={}, ec_message={}", + PathToUTF8String(file_path), size, ec.message()); + } + return set_size_result; } - - return set_size_result; + return false; } u64 IOFile::GetSize() const { - if (!IsOpen()) { - return 0; - } - - // Flush any unwritten buffered data into the file prior to retrieving the file size. - std::fflush(file); - +#ifdef __unix__ + if (mmap_fd != -1) + return mmap_size; +#endif + if (file) { + // Flush any unwritten buffered data into the file prior to retrieving the file mmap_size. + std::fflush(file); #if ANDROID - u64 file_size = 0; - if (Android::IsContentUri(file_path)) { - file_size = Android::GetSize(file_path); - } else { + u64 file_size = 0; + if (Android::IsContentUri(file_path)) { + file_size = Android::GetSize(file_path); + } else { + std::error_code ec; + + file_size = fs::file_size(file_path, ec); + + if (ec) { + LOG_ERROR(Common_Filesystem, "Failed to retrieve the file mmap_size of path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); + return 0; + } + } +#else std::error_code ec; - - file_size = fs::file_size(file_path, ec); - + auto const file_size = fs::file_size(file_path, ec); if (ec) { - LOG_ERROR(Common_Filesystem, - "Failed to retrieve the file size of path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); + LOG_ERROR(Common_Filesystem, "Failed to retrieve the file mmap_size of path={}, ec_message={}", + PathToUTF8String(file_path), ec.message()); return 0; } - } -#else - std::error_code ec; - - const auto file_size = fs::file_size(file_path, ec); - - if (ec) { - LOG_ERROR(Common_Filesystem, "Failed to retrieve the file size of path={}, ec_message={}", - PathToUTF8String(file_path), ec.message()); - return 0; - } #endif - - return file_size; + return file_size; + } + return 0; } bool IOFile::Seek(s64 offset, SeekOrigin origin) const { - if (!IsOpen()) { - return false; +#ifdef __unix__ + if (mmap_fd != -1) { + // fuck you to whoever made this method const + switch (origin) { + case SeekOrigin::SetOrigin: + mmap_offset = off_t(offset); + break; + case SeekOrigin::CurrentPosition: + mmap_offset += off_t(offset); + break; + case SeekOrigin::End: + mmap_offset = off_t(mmap_size) + off_t(offset); + break; + } + return true; } - - errno = 0; - - const auto seek_result = fseeko(file, offset, ToSeekOrigin(origin)) == 0; - - if (!seek_result) { - const auto ec = std::error_code{errno, std::generic_category()}; - LOG_ERROR(Common_Filesystem, - "Failed to seek the file at path={}, offset={}, origin={}, ec_message={}", - PathToUTF8String(file_path), offset, origin, ec.message()); +#endif + if (file) { + errno = 0; + const auto seek_result = fseeko(file, offset, ToSeekOrigin(origin)) == 0; + if (!seek_result) { + const auto ec = std::error_code{errno, std::generic_category()}; + LOG_ERROR(Common_Filesystem, "Failed to seek the file at path={}, offset={}, origin={}, ec_message={}", + PathToUTF8String(file_path), offset, origin, ec.message()); + } + return seek_result; } - - return seek_result; + return false; } s64 IOFile::Tell() const { - if (!IsOpen()) { - return 0; +#ifdef __unix__ + if (mmap_fd != -1) { + errno = 0; + return s64(mmap_offset); } - - errno = 0; - - return ftello(file); +#endif + if (file) { + errno = 0; + return ftello(file); + } + return 0; } } // namespace Common::FS diff --git a/src/common/fs/file.h b/src/common/fs/file.h index 246c3db5d6..ef79b79858 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -183,19 +183,6 @@ public: FileType type = FileType::BinaryFile, FileShareFlag flag = FileShareFlag::ShareReadOnly); -// #ifdef _WIN32 -// template -// void Open(const Path& path, FileAccessMode mode, FileType type = FileType::BinaryFile, -// FileShareFlag flag = FileShareFlag::ShareReadOnly) { -// using ValueType = typename Path::value_type; -// if constexpr (IsChar) { -// Open(ToU8String(path), mode, type, flag); -// } else { -// Open(std::filesystem::path{path}, mode, type, flag); -// } -// } -// #endif - /// Closes the file if it is opened. void Close(); @@ -225,8 +212,7 @@ public: [[nodiscard]] size_t Read(T& data) const { if constexpr (IsContiguousContainer) { using ContiguousType = typename T::value_type; - static_assert(std::is_trivially_copyable_v, - "Data type must be trivially copyable."); + static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); return ReadSpan(data); } else { return ReadObject(data) ? 1 : 0; @@ -251,8 +237,7 @@ public: [[nodiscard]] size_t Write(const T& data) const { if constexpr (IsContiguousContainer) { using ContiguousType = typename T::value_type; - static_assert(std::is_trivially_copyable_v, - "Data type must be trivially copyable."); + static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); return WriteSpan(data); } else { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); @@ -279,12 +264,13 @@ public: template [[nodiscard]] size_t ReadSpan(std::span data) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); - - if (!IsOpen()) { - return 0; +#ifdef __unix__ + if (mmap_fd != -1) { + std::memcpy(data.data(), mmap_base + mmap_offset, sizeof(T) * data.size()); + return data.size(); } - - return std::fread(data.data(), sizeof(T), data.size(), file); +#endif + return IsOpen() ? std::fread(data.data(), sizeof(T), data.size(), file) : 0; } /** @@ -305,12 +291,13 @@ public: template [[nodiscard]] size_t WriteSpan(std::span data) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); - - if (!IsOpen()) { - return 0; +#ifdef __unix__ + if (mmap_fd != -1) { + std::memcpy(mmap_base + mmap_offset, data.data(), sizeof(T) * data.size()); + return data.size(); } - - return std::fwrite(data.data(), sizeof(T), data.size(), file); +#endif + return IsOpen() ? std::fwrite(data.data(), sizeof(T), data.size(), file) : 0; } /** @@ -333,12 +320,13 @@ public: [[nodiscard]] bool ReadObject(T& object) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); - - if (!IsOpen()) { - return false; +#ifdef __unix__ + if (mmap_fd != -1) { + std::memcpy(&object, mmap_base + mmap_offset, sizeof(T)); + return sizeof(T); } - - return std::fread(&object, sizeof(T), 1, file) == 1; +#endif + return IsOpen() ? std::fread(&object, sizeof(T), 1, file) == 1 : false; } /** @@ -360,12 +348,13 @@ public: [[nodiscard]] bool WriteObject(const T& object) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); - - if (!IsOpen()) { - return false; +#ifdef __unix__ + if (mmap_fd != -1) { + std::memcpy(mmap_base + mmap_offset, &object, sizeof(T)); + return sizeof(T); } - - return std::fwrite(&object, sizeof(T), 1, file) == 1; +#endif + return IsOpen() ? std::fwrite(&object, sizeof(T), 1, file) == 1 : false; } /** @@ -452,8 +441,14 @@ private: std::filesystem::path file_path; FileAccessMode file_access_mode{}; FileType file_type{}; - std::FILE* file = nullptr; +#ifdef __unix__ + int mmap_fd = -1; + u8* mmap_base = nullptr; + size_t mmap_size = 0; + // fuck you + mutable off_t mmap_offset = 0; +#endif }; } // namespace Common::FS From 68833b492d70cbb7dba9f3df0e6d67436cb34554 Mon Sep 17 00:00:00 2001 From: lizzie Date: Sat, 6 Dec 2025 03:13:11 +0000 Subject: [PATCH 02/14] fucking license --- src/common/fs/file.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/common/fs/file.h b/src/common/fs/file.h index ef79b79858..7e343beda5 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later From 32f43c69562879d7f69d5fedae6a5a0530debbf0 Mon Sep 17 00:00:00 2001 From: lizzie Date: Sat, 6 Dec 2025 04:08:35 +0000 Subject: [PATCH 03/14] add cstring 4 std::memcpy --- src/common/fs/file.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/common/fs/file.h b/src/common/fs/file.h index 7e343beda5..04d1ceabab 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include From 7b6a05076e5b8bc26f25dae2b4cf258ca4f05126 Mon Sep 17 00:00:00 2001 From: lizzie Date: Sat, 6 Dec 2025 23:51:50 +0000 Subject: [PATCH 04/14] super align + nosync opts --- src/common/fs/file.cpp | 58 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index 10c80c4d26..8a5d03f053 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -13,6 +13,7 @@ #include "common/fs/fs_android.h" #endif #include "common/logging.h" +#include "common/literals.h" #ifdef _WIN32 #include @@ -273,18 +274,59 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File } else { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } -#elif defined(__unix__) && !defined(__HAIKU__) && !defined(__managarm__) +#elif defined(__HAIKU__) || defined(__managarm__) || defined(__OPENORBIS__) + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); +#else if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { struct stat st; mmap_fd = open(path.c_str(), O_RDONLY); fstat(mmap_fd, &st); mmap_size = st.st_size; - mmap_base = (u8*)mmap(nullptr, mmap_size, PROT_READ, MAP_PRIVATE, mmap_fd, 0); - } else { + + int map_flags = MAP_PRIVATE; +#ifdef MAP_PREFAULT_READ + // Prefaults reads so the final resulting pagetable from this big stupid mmap() + // isn't comically lazily loaded, we just coalesce everything in-place for our + // lovely mmap flags; if we didn't prefault the reads the page table will be + // constructed in-place (i.e on a read-by-read basis) causing lovely soft-faults + // which would nuke any performance gains. + // + // This of course incurs a cost in the initial mmap(2) call, but that is fine. + map_flags |= MAP_PREFAULT_READ; +#endif +#ifdef MAP_NOSYNC + // This causes physical media to not be synched to our file/memory + // This means that if the read-only file is written to, we won't see changes + // or we may see changes which are just funnily scattered, in any case + // this presumes the files won't be changed during execution + // + // Do not ever use this on write files (if we ever support that); this will create + // a fun amount of fragmentation on the disk. + map_flags |= MAP_NOSYNC; +#endif +#ifdef MAP_ALIGNED_SUPER + // File must be big enough that it's worth to super align. We can't just super-align every + // file otherwise we will run out of alignments for actually important files :) + // System doesn't guarantee a super alignment, but if it's available it will delete + // about 3 layers(?) of the TLB tree for each read/write. + // Again the cost of faults may make this negligible gains, but hey, we gotta work + // what we gotta work with. + using namespace Common::Literals; + u64 big_file_threshold = 512_MiB; + map_flags |= u64(st.st_size) >= big_file_threshold ? MAP_ALIGNED_SUPER : 0; +#endif + mmap_base = (u8*)mmap(nullptr, mmap_size, PROT_READ, map_flags, mmap_fd, 0); + if (mmap_base == MAP_FAILED) { + close(mmap_fd); + mmap_fd = -1; + } else { + posix_madvise(mmap_base, mmap_size, POSIX_MADV_WILLNEED); + } + } + // mmap(2) failed or simply we can't use it + if (mmap_fd == -1) { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } -#else - file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); #endif if (!IsOpen()) { const auto ec = std::error_code{errno, std::generic_category()}; @@ -340,11 +382,7 @@ bool IOFile::Flush() const { #endif if (file) { errno = 0; -#ifdef _WIN32 - const auto flush_result = std::fflush(file) == 0; -#else - const auto flush_result = std::fflush(file) == 0; -#endif + auto const flush_result = std::fflush(file) == 0; if (!flush_result) { const auto ec = std::error_code{errno, std::generic_category()}; LOG_ERROR(Common_Filesystem, "Failed to flush the file at path={}, ec_message={}", From c8c02af1d6ece26ccc143e6274e0e00795e79873 Mon Sep 17 00:00:00 2001 From: lizzie Date: Sun, 7 Dec 2025 00:17:47 +0000 Subject: [PATCH 05/14] fuck macos --- src/common/fs/file.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index 8a5d03f053..55daa2db8a 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -274,9 +274,9 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File } else { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } -#elif defined(__HAIKU__) || defined(__managarm__) || defined(__OPENORBIS__) +#elif defined(__HAIKU__) || defined(__managarm__) || defined(__OPENORBIS__) || defined(__APPLE__) file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); -#else +#elif defined(__unix__) if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { struct stat st; mmap_fd = open(path.c_str(), O_RDONLY); @@ -327,6 +327,9 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File if (mmap_fd == -1) { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } +#else + // Some other fancy OS (ahem fucking Darwin/Mac OSX) + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); #endif if (!IsOpen()) { const auto ec = std::error_code{errno, std::generic_category()}; From 943f0f6592324de32f9c923a19d15bb29173df29 Mon Sep 17 00:00:00 2001 From: lizzie Date: Fri, 26 Dec 2025 11:30:25 +0000 Subject: [PATCH 06/14] Add for windows and Android --- src/common/fs/file.cpp | 227 ++++++++++++--------- src/common/fs/file.h | 452 ++++++++++++++++++++--------------------- 2 files changed, 355 insertions(+), 324 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index 55daa2db8a..eb23aea5e9 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -18,6 +18,7 @@ #ifdef _WIN32 #include #include +#include #else #include #include @@ -40,13 +41,13 @@ namespace { #ifdef _WIN32 /** - * Converts the file access mode and file type enums to a file access mode wide string. - * - * @param mode File access mode - * @param type File type - * - * @returns A pointer to a wide string representing the file access mode. - */ +* Converts the file access mode and file type enums to a file access mode wide string. +* +* @param mode File access mode +* @param type File type +* +* @returns A pointer to a wide string representing the file access mode. +*/ [[nodiscard]] constexpr const wchar_t* AccessModeToWStr(FileAccessMode mode, FileType type) { switch (type) { case FileType::BinaryFile: @@ -83,12 +84,12 @@ namespace { } /** - * Converts the file-share access flag enum to a Windows defined file-share access flag. - * - * @param flag File-share access flag - * - * @returns Windows defined file-share access flag. - */ +* Converts the file-share access flag enum to a Windows defined file-share access flag. +* +* @param flag File-share access flag +* +* @returns Windows defined file-share access flag. +*/ [[nodiscard]] constexpr int ToWindowsFileShareFlag(FileShareFlag flag) { switch (flag) { case FileShareFlag::ShareNone: @@ -106,13 +107,13 @@ namespace { #else /** - * Converts the file access mode and file type enums to a file access mode string. - * - * @param mode File access mode - * @param type File type - * - * @returns A pointer to a string representing the file access mode. - */ +* Converts the file access mode and file type enums to a file access mode string. +* +* @param mode File access mode +* @param type File type +* +* @returns A pointer to a string representing the file access mode. +*/ [[nodiscard]] constexpr const char* AccessModeToStr(FileAccessMode mode, FileType type) { switch (type) { case FileType::BinaryFile: @@ -151,12 +152,12 @@ namespace { #endif /** - * Converts the seek origin enum to a seek origin integer. - * - * @param origin Seek origin - * - * @returns Seek origin integer. - */ +* Converts the seek origin enum to a seek origin integer. +* +* @param origin Seek origin +* +* @returns Seek origin integer. +*/ [[nodiscard]] constexpr int ToSeekOrigin(SeekOrigin origin) { switch (origin) { case SeekOrigin::SetOrigin: @@ -182,7 +183,7 @@ std::string ReadStringFromFile(const std::filesystem::path& path, FileType type) } size_t WriteStringToFile(const std::filesystem::path& path, FileType type, - std::string_view string) { + std::string_view string) { if (Exists(path) && !IsFile(path)) { return 0; } @@ -193,7 +194,7 @@ size_t WriteStringToFile(const std::filesystem::path& path, FileType type, } size_t AppendStringToFile(const std::filesystem::path& path, FileType type, - std::string_view string) { + std::string_view string) { if (Exists(path) && !IsFile(path)) { return 0; } @@ -248,40 +249,13 @@ FileType IOFile::GetType() const { return file_type; } -void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, FileShareFlag flag) { - Close(); - file_path = path; - file_access_mode = mode; - file_type = type; - errno = 0; -#ifdef _WIN32 - if (flag != FileShareFlag::ShareNone) { - file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); - } else { - _wfopen_s(&file, path.c_str(), AccessModeToWStr(mode, type)); - } -#elif ANDROID - if (Android::IsContentUri(path)) { - ASSERT_MSG(mode == FileAccessMode::Read, "Content URI file access is for read-only!"); - auto const fd = Android::OpenContentUri(path, Android::OpenMode::Read); - if (fd != -1) { - file = fdopen(fd, "r"); - if (errno != 0 && file == nullptr) - LOG_ERROR(Common_Filesystem, "Error opening file: {}, error: {}", path.c_str(), strerror(errno)); - } else { - LOG_ERROR(Common_Filesystem, "Error opening file: {}", path.c_str()); - } - } else { - file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); - } -#elif defined(__HAIKU__) || defined(__managarm__) || defined(__OPENORBIS__) || defined(__APPLE__) - file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); -#elif defined(__unix__) - if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { +#ifdef __unix__ +int PlatformMapReadOnly(IOFile& io, const char* path) { + io.mmap_fd = open(path., O_RDONLY); + if (io.mmap_fd > 0) { struct stat st; - mmap_fd = open(path.c_str(), O_RDONLY); - fstat(mmap_fd, &st); - mmap_size = st.st_size; + fstat(io.mmap_fd, &st); + io.mmap_size = st.st_size; int map_flags = MAP_PRIVATE; #ifdef MAP_PREFAULT_READ @@ -315,18 +289,97 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File u64 big_file_threshold = 512_MiB; map_flags |= u64(st.st_size) >= big_file_threshold ? MAP_ALIGNED_SUPER : 0; #endif - mmap_base = (u8*)mmap(nullptr, mmap_size, PROT_READ, map_flags, mmap_fd, 0); - if (mmap_base == MAP_FAILED) { - close(mmap_fd); - mmap_fd = -1; + io.mmap_base = (u8*)mmap(nullptr, io.mmap_size, PROT_READ, map_flags, io.mmap_fd, 0); + if (io.mmap_base == MAP_FAILED) { + close(io.mmap_fd); + io.mmap_fd = -1; } else { - posix_madvise(mmap_base, mmap_size, POSIX_MADV_WILLNEED); + posix_madvise(mmap_base, io.mmap_size, POSIX_MADV_WILLNEED); } } - // mmap(2) failed or simply we can't use it - if (mmap_fd == -1) { + return io.mmap_fd; +} +void PlatformUnmap(IOFile& io) { + if (io.mmap_fd != -1) { + munmap(io.mmap_base, io.mmap_size); + close(io.mmap_fd); + io.mmap_fd = -1; + } +} +#else +int PlatformMapReadOnly(IOFile& io, const char* path) { + io.file_handle = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr); + if (HANDLE(io.file_handle) != INVALID_HANDLE_VALUE) { + io.mapping_handle = CreateFileMappingW(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (io.mapping_handle) { + io.mmap_base = (char const*)MapViewOfFile(HANDLE(io.mapping_handle), FILE_MAP_READ, 0, 0, 0); + if (io.mmap_base) { + _LARGE_INTEGER pvalue; + GetFileSizeEx(file_handle, &pvalue); + io.mmap_size = uint32_t(pvalue.QuadPart); + } else { + CloseHandle(io.mapping_handle); + CloseHandle(io.file_handle); + return -1; + } + } else { + CloseHandle(io.file_handle); + return -1; + } + } + return 0; +} +void PlatformUnmap(IOFile& io) { + if(io.mapping_handle) { + if(io.mmap_base) + UnmapViewOfFile(HANDLE(io.mmap_base)); + CloseHandle(HANDLE(io.mapping_handle)); + } + if(io.file_handle != INVALID_HANDLE_VALUE) + CloseHandle(HANDLE(io.file_handle)); +} +#endif + +void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, FileShareFlag flag) { + Close(); + file_path = path; + file_access_mode = mode; + file_type = type; + errno = 0; +#ifdef _WIN32 + if (flag != FileShareFlag::ShareNone) { + file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); + } else { + _wfopen_s(&file, path.c_str(), AccessModeToWStr(mode, type)); + } +#elif ANDROID + if (Android::IsContentUri(path)) { + ASSERT_MSG(mode == FileAccessMode::Read, "Content URI file access is for read-only!"); + if (PlatformMapReadOnly(*this, path.c_str()) == -1) { + LOG_ERROR(Common_Filesystem, "Error mmap'ing file: {}", path.c_str()); + int const fd = Android::OpenContentUri(path, Android::OpenMode::Read); + if (fd != -1) { + file = fdopen(fd, "r"); + if (errno != 0 && file == nullptr) + LOG_ERROR(Common_Filesystem, "Error opening file: {}, error: {}", path.c_str(), strerror(errno)); + } else { + LOG_ERROR(Common_Filesystem, "Error opening file: {}", path.c_str()); + } + } + } else { file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); } +#elif defined(__HAIKU__) || defined(__managarm__) || defined(__OPENORBIS__) || defined(__APPLE__) + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); +#elif defined(__unix__) + if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { + if (PlatformMapReadOnly(*this, path.c_str()) == -1) { + LOG_ERROR(Common_Filesystem, "Error mmap'ing file: {}", path.c_str()); + } + } + if (mmap_fd == -1) { + file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); // mmap(2) failed or simply we can't use it + } #else // Some other fancy OS (ahem fucking Darwin/Mac OSX) file = std::fopen(path.c_str(), AccessModeToStr(mode, type)); @@ -339,13 +392,7 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File } void IOFile::Close() { -#ifdef __unix__ - if (mmap_fd != -1) { - munmap(mmap_base, mmap_size); - close(mmap_fd); - mmap_fd = -1; - } -#endif + PlatformUnmap(*this); if (file) { errno = 0; const auto close_result = std::fclose(file) == 0; @@ -359,11 +406,7 @@ void IOFile::Close() { } bool IOFile::IsOpen() const { -#ifdef __unix__ - return file != nullptr || mmap_fd != -1; -#else - return file != nullptr; -#endif + return file != nullptr || IsMappedFile(); } std::string IOFile::ReadString(size_t length) const { @@ -380,9 +423,7 @@ size_t IOFile::WriteString(std::span string) const { } bool IOFile::Flush() const { -#ifdef __unix__ - ASSERT(mmap_fd == -1); -#endif + ASSERT(!IsMappedFile()); if (file) { errno = 0; auto const flush_result = std::fflush(file) == 0; @@ -397,9 +438,7 @@ bool IOFile::Flush() const { } bool IOFile::Commit() const { -#ifdef __unix__ - ASSERT(mmap_fd == -1); -#endif + ASSERT(!IsMappedFile()); if (file) { errno = 0; #ifdef _WIN32 @@ -418,9 +457,7 @@ bool IOFile::Commit() const { } bool IOFile::SetSize(u64 size) const { -#ifdef __unix__ - ASSERT(mmap_fd == -1); -#endif + ASSERT(!IsMappedFile()); if (file) { errno = 0; #ifdef _WIN32 @@ -439,10 +476,8 @@ bool IOFile::SetSize(u64 size) const { } u64 IOFile::GetSize() const { -#ifdef __unix__ - if (mmap_fd != -1) + if (IsMappedFile()) return mmap_size; -#endif if (file) { // Flush any unwritten buffered data into the file prior to retrieving the file mmap_size. std::fflush(file); @@ -476,8 +511,7 @@ u64 IOFile::GetSize() const { } bool IOFile::Seek(s64 offset, SeekOrigin origin) const { -#ifdef __unix__ - if (mmap_fd != -1) { + if (IsMappedFile()) { // fuck you to whoever made this method const switch (origin) { case SeekOrigin::SetOrigin: @@ -492,7 +526,6 @@ bool IOFile::Seek(s64 offset, SeekOrigin origin) const { } return true; } -#endif if (file) { errno = 0; const auto seek_result = fseeko(file, offset, ToSeekOrigin(origin)) == 0; @@ -507,12 +540,10 @@ bool IOFile::Seek(s64 offset, SeekOrigin origin) const { } s64 IOFile::Tell() const { -#ifdef __unix__ - if (mmap_fd != -1) { + if (IsMappedFile()) { errno = 0; return s64(mmap_offset); } -#endif if (file) { errno = 0; return ftello(file); diff --git a/src/common/fs/file.h b/src/common/fs/file.h index 04d1ceabab..937c7d6054 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -25,12 +25,12 @@ enum class SeekOrigin { }; /** - * Opens a file stream at path with the specified open mode. - * - * @param file_stream Reference to file stream - * @param path Filesystem path - * @param open_mode File stream open mode - */ +* Opens a file stream at path with the specified open mode. +* +* @param file_stream Reference to file stream +* @param path Filesystem path +* @param open_mode File stream open mode +*/ template void OpenFileStream(FileStream& file_stream, const std::filesystem::path& path, std::ios_base::openmode open_mode) { @@ -49,14 +49,14 @@ void OpenFileStream(FileStream& file_stream, const Path& path, std::ios_base::op #endif /** - * Reads an entire file at path and returns a string of the contents read from the file. - * If the filesystem object at path is not a regular file, this function returns an empty string. - * - * @param path Filesystem path - * @param type File type - * - * @returns A string of the contents read from the file. - */ +* Reads an entire file at path and returns a string of the contents read from the file. +* If the filesystem object at path is not a regular file, this function returns an empty string. +* +* @param path Filesystem path +* @param type File type +* +* @returns A string of the contents read from the file. +*/ [[nodiscard]] std::string ReadStringFromFile(const std::filesystem::path& path, FileType type); #ifdef _WIN32 @@ -71,18 +71,18 @@ template #endif /** - * Writes a string to a file at path and returns the number of characters successfully written. - * If a file already exists at path, its contents will be erased. - * If a file does not exist at path, it creates and opens a new empty file for writing. - * If the filesystem object at path exists and is not a regular file, this function returns 0. - * - * @param path Filesystem path - * @param type File type - * - * @returns Number of characters successfully written. - */ +* Writes a string to a file at path and returns the number of characters successfully written. +* If a file already exists at path, its contents will be erased. +* If a file does not exist at path, it creates and opens a new empty file for writing. +* If the filesystem object at path exists and is not a regular file, this function returns 0. +* +* @param path Filesystem path +* @param type File type +* +* @returns Number of characters successfully written. +*/ [[nodiscard]] size_t WriteStringToFile(const std::filesystem::path& path, FileType type, - std::string_view string); + std::string_view string); #ifdef _WIN32 template @@ -96,15 +96,15 @@ template #endif /** - * Appends a string to a file at path and returns the number of characters successfully written. - * If a file does not exist at path, it creates and opens a new empty file for appending. - * If the filesystem object at path exists and is not a regular file, this function returns 0. - * - * @param path Filesystem path - * @param type File type - * - * @returns Number of characters successfully written. - */ +* Appends a string to a file at path and returns the number of characters successfully written. +* If a file does not exist at path, it creates and opens a new empty file for appending. +* If the filesystem object at path exists and is not a regular file, this function returns 0. +* +* @param path Filesystem path +* @param type File type +* +* @returns Number of characters successfully written. +*/ [[nodiscard]] size_t AppendStringToFile(const std::filesystem::path& path, FileType type, std::string_view string); @@ -132,14 +132,14 @@ public: FileShareFlag flag = FileShareFlag::ShareReadOnly); /** - * An IOFile is a lightweight wrapper on C Library file operations. - * Automatically closes an open file on the destruction of an IOFile object. - * - * @param path Filesystem path - * @param mode File access mode - * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file - * @param flag (Windows only) File-share access flag, default is ShareReadOnly - */ + * An IOFile is a lightweight wrapper on C Library file operations. + * Automatically closes an open file on the destruction of an IOFile object. + * + * @param path Filesystem path + * @param mode File access mode + * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file + * @param flag (Windows only) File-share access flag, default is ShareReadOnly + */ explicit IOFile(const std::filesystem::path& path, FileAccessMode mode, FileType type = FileType::BinaryFile, FileShareFlag flag = FileShareFlag::ShareReadOnly); @@ -153,65 +153,65 @@ public: IOFile& operator=(IOFile&& other) noexcept; /** - * Gets the path of the file. - * - * @returns The path of the file. - */ + * Gets the path of the file. + * + * @returns The path of the file. + */ [[nodiscard]] std::filesystem::path GetPath() const; /** - * Gets the access mode of the file. - * - * @returns The access mode of the file. - */ + * Gets the access mode of the file. + * + * @returns The access mode of the file. + */ [[nodiscard]] FileAccessMode GetAccessMode() const; /** - * Gets the type of the file. - * - * @returns The type of the file. - */ + * Gets the type of the file. + * + * @returns The type of the file. + */ [[nodiscard]] FileType GetType() const; /** - * Opens a file at path with the specified file access mode. - * This function behaves differently depending on the FileAccessMode. - * These behaviors are documented in each enum value of FileAccessMode. - * - * @param path Filesystem path - * @param mode File access mode - * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file - * @param flag (Windows only) File-share access flag, default is ShareReadOnly - */ + * Opens a file at path with the specified file access mode. + * This function behaves differently depending on the FileAccessMode. + * These behaviors are documented in each enum value of FileAccessMode. + * + * @param path Filesystem path + * @param mode File access mode + * @param type File type, default is BinaryFile. Use TextFile to open the file as a text file + * @param flag (Windows only) File-share access flag, default is ShareReadOnly + */ void Open(const std::filesystem::path& path, FileAccessMode mode, - FileType type = FileType::BinaryFile, - FileShareFlag flag = FileShareFlag::ShareReadOnly); + FileType type = FileType::BinaryFile, + FileShareFlag flag = FileShareFlag::ShareReadOnly); /// Closes the file if it is opened. void Close(); /** - * Checks whether the file is open. - * Use this to check whether the calls to Open() or Close() succeeded. - * - * @returns True if the file is open, false otherwise. - */ + * Checks whether the file is open. + * Use this to check whether the calls to Open() or Close() succeeded. + * + * @returns True if the file is open, false otherwise. + */ [[nodiscard]] bool IsOpen() const; /** - * Helper function which deduces the value type of a contiguous STL container used in ReadSpan. - * If T is not a contiguous container as defined by the concept IsContiguousContainer, this - * calls ReadObject and T must be a trivially copyable object. - * - * See ReadSpan for more details if T is a contiguous container. - * See ReadObject for more details if T is a trivially copyable object. - * - * @tparam T Contiguous container or trivially copyable object - * - * @param data Container of T::value_type data or reference to object - * - * @returns Count of T::value_type data or objects successfully read. - */ + * Helper function which deduces the value type of a contiguous STL container used in ReadSpan. + * If T is not a contiguous container as defined by the concept IsContiguousContainer, this + * calls ReadObject and T must be a trivially copyable object. + * + * See ReadSpan for more details if T is a contiguous container. + * See ReadObject for more details if T is a trivially copyable object. + * + * @tparam T Contiguous container or trivially copyable object + * + * @param data Container of T::value_type data or reference to object + * + * @returns Count of T::value_type data or objects successfully read. + */ template [[nodiscard]] size_t Read(T& data) const { if constexpr (IsContiguousContainer) { @@ -224,19 +224,19 @@ public: } /** - * Helper function which deduces the value type of a contiguous STL container used in WriteSpan. - * If T is not a contiguous STL container as defined by the concept IsContiguousContainer, this - * calls WriteObject and T must be a trivially copyable object. - * - * See WriteSpan for more details if T is a contiguous container. - * See WriteObject for more details if T is a trivially copyable object. - * - * @tparam T Contiguous container or trivially copyable object - * - * @param data Container of T::value_type data or const reference to object - * - * @returns Count of T::value_type data or objects successfully written. - */ + * Helper function which deduces the value type of a contiguous STL container used in WriteSpan. + * If T is not a contiguous STL container as defined by the concept IsContiguousContainer, this + * calls WriteObject and T must be a trivially copyable object. + * + * See WriteSpan for more details if T is a contiguous container. + * See WriteObject for more details if T is a trivially copyable object. + * + * @tparam T Contiguous container or trivially copyable object + * + * @param data Container of T::value_type data or const reference to object + * + * @returns Count of T::value_type data or objects successfully written. + */ template [[nodiscard]] size_t Write(const T& data) const { if constexpr (IsContiguousContainer) { @@ -250,195 +250,187 @@ public: } /** - * Reads a span of T data from a file sequentially. - * This function reads from the current position of the file pointer and - * advances it by the (count of T * sizeof(T)) bytes successfully read. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks read permissions - * - Attempting to read beyond the end-of-file - * - * @tparam T Data type - * - * @param data Span of T data - * - * @returns Count of T data successfully read. - */ + * Reads a span of T data from a file sequentially. + * This function reads from the current position of the file pointer and + * advances it by the (count of T * sizeof(T)) bytes successfully read. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks read permissions + * - Attempting to read beyond the end-of-file + * + * @tparam T Data type + * + * @param data Span of T data + * + * @returns Count of T data successfully read. + */ template [[nodiscard]] size_t ReadSpan(std::span data) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); -#ifdef __unix__ - if (mmap_fd != -1) { + if (IsMappedFile()) { std::memcpy(data.data(), mmap_base + mmap_offset, sizeof(T) * data.size()); return data.size(); } -#endif return IsOpen() ? std::fread(data.data(), sizeof(T), data.size(), file) : 0; } /** - * Writes a span of T data to a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the (count of T * sizeof(T)) bytes successfully written. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks write permissions - * - * @tparam T Data type - * - * @param data Span of T data - * - * @returns Count of T data successfully written. - */ + * Writes a span of T data to a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the (count of T * sizeof(T)) bytes successfully written. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks write permissions + * + * @tparam T Data type + * + * @param data Span of T data + * + * @returns Count of T data successfully written. + */ template [[nodiscard]] size_t WriteSpan(std::span data) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); -#ifdef __unix__ - if (mmap_fd != -1) { + if (IsMappedFile()) { std::memcpy(mmap_base + mmap_offset, data.data(), sizeof(T) * data.size()); return data.size(); } -#endif return IsOpen() ? std::fwrite(data.data(), sizeof(T), data.size(), file) : 0; } /** - * Reads a T object from a file sequentially. - * This function reads from the current position of the file pointer and - * advances it by the sizeof(T) bytes successfully read. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks read permissions - * - Attempting to read beyond the end-of-file - * - * @tparam T Data type - * - * @param object Reference to object - * - * @returns True if the object is successfully read from the file, false otherwise. - */ + * Reads a T object from a file sequentially. + * This function reads from the current position of the file pointer and + * advances it by the sizeof(T) bytes successfully read. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks read permissions + * - Attempting to read beyond the end-of-file + * + * @tparam T Data type + * + * @param object Reference to object + * + * @returns True if the object is successfully read from the file, false otherwise. + */ template [[nodiscard]] bool ReadObject(T& object) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); -#ifdef __unix__ - if (mmap_fd != -1) { + if (IsMappedFile()) { std::memcpy(&object, mmap_base + mmap_offset, sizeof(T)); return sizeof(T); } -#endif return IsOpen() ? std::fread(&object, sizeof(T), 1, file) == 1 : false; } /** - * Writes a T object to a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the sizeof(T) bytes successfully written. - * - * Failures occur when: - * - The file is not open - * - The opened file lacks write permissions - * - * @tparam T Data type - * - * @param object Const reference to object - * - * @returns True if the object is successfully written to the file, false otherwise. - */ + * Writes a T object to a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the sizeof(T) bytes successfully written. + * + * Failures occur when: + * - The file is not open + * - The opened file lacks write permissions + * + * @tparam T Data type + * + * @param object Const reference to object + * + * @returns True if the object is successfully written to the file, false otherwise. + */ template [[nodiscard]] bool WriteObject(const T& object) const { static_assert(std::is_trivially_copyable_v, "Data type must be trivially copyable."); static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); -#ifdef __unix__ - if (mmap_fd != -1) { + if (IsMappedFile()) { std::memcpy(mmap_base + mmap_offset, &object, sizeof(T)); return sizeof(T); } -#endif return IsOpen() ? std::fwrite(&object, sizeof(T), 1, file) == 1 : false; } /** - * Specialized function to read a string of a given length from a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the number of characters successfully read. - * The size of the returned string may not match length if not all bytes are successfully read. - * - * @param length Length of the string - * - * @returns A string read from the file. - */ + * Specialized function to read a string of a given length from a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the number of characters successfully read. + * The size of the returned string may not match length if not all bytes are successfully read. + * + * @param length Length of the string + * + * @returns A string read from the file. + */ [[nodiscard]] std::string ReadString(size_t length) const; /** - * Specialized function to write a string to a file sequentially. - * This function writes from the current position of the file pointer and - * advances it by the number of characters successfully written. - * - * @param string Span of const char backed std::string or std::string_view - * - * @returns Number of characters successfully written. - */ + * Specialized function to write a string to a file sequentially. + * This function writes from the current position of the file pointer and + * advances it by the number of characters successfully written. + * + * @param string Span of const char backed std::string or std::string_view + * + * @returns Number of characters successfully written. + */ [[nodiscard]] size_t WriteString(std::span string) const; /** - * Attempts to flush any unwritten buffered data into the file. - * - * @returns True if the flush was successful, false otherwise. - */ + * Attempts to flush any unwritten buffered data into the file. + * + * @returns True if the flush was successful, false otherwise. + */ bool Flush() const; /** - * Attempts to commit the file into the disk. - * Note that this is an expensive operation as this forces the operating system to write - * the contents of the file associated with the file descriptor into the disk. - * - * @returns True if the commit was successful, false otherwise. - */ + * Attempts to commit the file into the disk. + * Note that this is an expensive operation as this forces the operating system to write + * the contents of the file associated with the file descriptor into the disk. + * + * @returns True if the commit was successful, false otherwise. + */ bool Commit() const; /** - * Resizes the file to a given size. - * If the file is resized to a smaller size, the remainder of the file is discarded. - * If the file is resized to a larger size, the new area appears as if zero-filled. - * - * Failures occur when: - * - The file is not open - * - * @param size File size in bytes - * - * @returns True if the file resize succeeded, false otherwise. - */ + * Resizes the file to a given size. + * If the file is resized to a smaller size, the remainder of the file is discarded. + * If the file is resized to a larger size, the new area appears as if zero-filled. + * + * Failures occur when: + * - The file is not open + * + * @param size File size in bytes + * + * @returns True if the file resize succeeded, false otherwise. + */ [[nodiscard]] bool SetSize(u64 size) const; /** - * Gets the size of the file. - * - * Failures occur when: - * - The file is not open - * - * @returns The file size in bytes of the file. Returns 0 on failure. - */ + * Gets the size of the file. + * + * Failures occur when: + * - The file is not open + * + * @returns The file size in bytes of the file. Returns 0 on failure. + */ [[nodiscard]] u64 GetSize() const; /** - * Moves the current position of the file pointer with the specified offset and seek origin. - * - * @param offset Offset from seek origin - * @param origin Seek origin - * - * @returns True if the file pointer has moved to the specified offset, false otherwise. - */ + * Moves the current position of the file pointer with the specified offset and seek origin. + * + * @param offset Offset from seek origin + * @param origin Seek origin + * + * @returns True if the file pointer has moved to the specified offset, false otherwise. + */ [[nodiscard]] bool Seek(s64 offset, SeekOrigin origin = SeekOrigin::SetOrigin) const; /** - * Gets the current position of the file pointer. - * - * @returns The current position of the file pointer. - */ + * Gets the current position of the file pointer. + * + * @returns The current position of the file pointer. + */ [[nodiscard]] s64 Tell() const; private: @@ -446,13 +438,21 @@ private: FileAccessMode file_access_mode{}; FileType file_type{}; std::FILE* file = nullptr; -#ifdef __unix__ + + // Any decent system should have mmap() for files + // Systems with artifical mmap() limitations should simply change the logic within file.cpp + // and reduce the threshold for which the mmap() is set to +#ifdef _WIN32 + void *mapping_handle = nullptr; + void *file_handle = nullptr; + bool IsMappedFile() { return file_handle != nullptr; } +#else // POSIX int mmap_fd = -1; + bool IsMappedFile() { return mmap_fd != -1; } +#endif u8* mmap_base = nullptr; size_t mmap_size = 0; - // fuck you - mutable off_t mmap_offset = 0; -#endif + mutable off_t mmap_offset = 0; // fuck you }; } // namespace Common::FS From bab6ea59101ce16e66c553f62bf72b1f90a918bb Mon Sep 17 00:00:00 2001 From: lizzie Date: Fri, 26 Dec 2025 21:27:26 +0000 Subject: [PATCH 07/14] [chore] Fix windows building --- src/common/fs/file.cpp | 41 ++++++++++++++++++++++++++--------------- src/common/fs/file.h | 10 ++++++---- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index eb23aea5e9..a76f48a365 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -249,9 +249,9 @@ FileType IOFile::GetType() const { return file_type; } -#ifdef __unix__ -int PlatformMapReadOnly(IOFile& io, const char* path) { - io.mmap_fd = open(path., O_RDONLY); +#if defined(__unix__) +static int PlatformMapReadOnly(IOFile& io, const char* path) { + io.mmap_fd = open(path, O_RDONLY); if (io.mmap_fd > 0) { struct stat st; fstat(io.mmap_fd, &st); @@ -294,28 +294,30 @@ int PlatformMapReadOnly(IOFile& io, const char* path) { close(io.mmap_fd); io.mmap_fd = -1; } else { - posix_madvise(mmap_base, io.mmap_size, POSIX_MADV_WILLNEED); + posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_WILLNEED); } } return io.mmap_fd; } -void PlatformUnmap(IOFile& io) { +static void PlatformUnmap(IOFile& io) { if (io.mmap_fd != -1) { munmap(io.mmap_base, io.mmap_size); close(io.mmap_fd); io.mmap_fd = -1; } } +#elif defined(__APPLE__) +// NO IMPLEMENTATION YET #else -int PlatformMapReadOnly(IOFile& io, const char* path) { - io.file_handle = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr); +static int PlatformMapReadOnly(IOFile& io, const wchar_t* path) { + io.file_handle = CreateFileW(path, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr); if (HANDLE(io.file_handle) != INVALID_HANDLE_VALUE) { - io.mapping_handle = CreateFileMappingW(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr); + io.mapping_handle = CreateFileMappingW(HANDLE(io.file_handle), nullptr, PAGE_READONLY, 0, 0, nullptr); if (io.mapping_handle) { - io.mmap_base = (char const*)MapViewOfFile(HANDLE(io.mapping_handle), FILE_MAP_READ, 0, 0, 0); + io.mmap_base = (u8*)MapViewOfFile(HANDLE(io.mapping_handle), FILE_MAP_READ, 0, 0, 0); if (io.mmap_base) { _LARGE_INTEGER pvalue; - GetFileSizeEx(file_handle, &pvalue); + GetFileSizeEx(io.file_handle, &pvalue); io.mmap_size = uint32_t(pvalue.QuadPart); } else { CloseHandle(io.mapping_handle); @@ -329,7 +331,7 @@ int PlatformMapReadOnly(IOFile& io, const char* path) { } return 0; } -void PlatformUnmap(IOFile& io) { +static void PlatformUnmap(IOFile& io) { if(io.mapping_handle) { if(io.mmap_base) UnmapViewOfFile(HANDLE(io.mmap_base)); @@ -347,10 +349,19 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File file_type = type; errno = 0; #ifdef _WIN32 - if (flag != FileShareFlag::ShareNone) { - file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); - } else { - _wfopen_s(&file, path.c_str(), AccessModeToWStr(mode, type)); + // TODO: this probably can use better logic but oh well I'm not a windowser + file_handle = nullptr; + if (type == FileType::BinaryFile && mode == FileAccessMode::Read) { + if (PlatformMapReadOnly(*this, path.c_str()) == -1) { + LOG_ERROR(Common_Filesystem, "Error mmap'ing file"); //: {}", path.c_str()); + } + } + if (file_handle == nullptr) { + if (flag != FileShareFlag::ShareNone) { + file = _wfsopen(path.c_str(), AccessModeToWStr(mode, type), ToWindowsFileShareFlag(flag)); + } else { + _wfopen_s(&file, path.c_str(), AccessModeToWStr(mode, type)); + } } #elif ANDROID if (Android::IsContentUri(path)) { diff --git a/src/common/fs/file.h b/src/common/fs/file.h index 937c7d6054..1f46e96f0d 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -433,22 +433,24 @@ public: */ [[nodiscard]] s64 Tell() const; -private: +#ifdef _WIN32 + inline bool IsMappedFile() const noexcept { return file_handle != nullptr; } +#else // POSIX + inline bool IsMappedFile() const noexcept { return mmap_fd != -1; } +#endif + std::filesystem::path file_path; FileAccessMode file_access_mode{}; FileType file_type{}; std::FILE* file = nullptr; - // Any decent system should have mmap() for files // Systems with artifical mmap() limitations should simply change the logic within file.cpp // and reduce the threshold for which the mmap() is set to #ifdef _WIN32 void *mapping_handle = nullptr; void *file_handle = nullptr; - bool IsMappedFile() { return file_handle != nullptr; } #else // POSIX int mmap_fd = -1; - bool IsMappedFile() { return mmap_fd != -1; } #endif u8* mmap_base = nullptr; size_t mmap_size = 0; From 250e377a89ab4edeba917621d2a25fb873ab6966 Mon Sep 17 00:00:00 2001 From: Caio Oliveira Date: Wed, 7 Jan 2026 19:37:21 -0300 Subject: [PATCH 08/14] [chore] It can possible kill someone Signed-off-by: Caio Oliveira --- src/common/fs/file.cpp | 4 ++++ src/common/fs/file.h | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index a76f48a365..a090ccee03 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -403,7 +403,11 @@ void IOFile::Open(const fs::path& path, FileAccessMode mode, FileType type, File } void IOFile::Close() { +#if defined(__APPLE__) + // NO IMPLEMENTATION YET +#else PlatformUnmap(*this); +#endif if (file) { errno = 0; const auto close_result = std::fclose(file) == 0; diff --git a/src/common/fs/file.h b/src/common/fs/file.h index 1f46e96f0d..b8f3810277 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -322,7 +322,11 @@ public: static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); if (IsMappedFile()) { std::memcpy(&object, mmap_base + mmap_offset, sizeof(T)); +#ifdef _WIN32 + return sizeof(T) != 0; +#else return sizeof(T); +#endif } return IsOpen() ? std::fread(&object, sizeof(T), 1, file) == 1 : false; } @@ -348,7 +352,11 @@ public: static_assert(!std::is_pointer_v, "T must not be a pointer to an object."); if (IsMappedFile()) { std::memcpy(mmap_base + mmap_offset, &object, sizeof(T)); +#ifdef _WIN32 + return sizeof(T) != 0; +#else return sizeof(T); +#endif } return IsOpen() ? std::fwrite(&object, sizeof(T), 1, file) == 1 : false; } From beaf989a85785fab39c52d843f5107d501546048 Mon Sep 17 00:00:00 2001 From: Caio Oliveira Date: Fri, 9 Jan 2026 00:35:19 -0300 Subject: [PATCH 09/14] f you msvc --- src/common/fs/file.cpp | 6 +++--- src/common/fs/file.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index a090ccee03..d34e1475e4 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -530,13 +530,13 @@ bool IOFile::Seek(s64 offset, SeekOrigin origin) const { // fuck you to whoever made this method const switch (origin) { case SeekOrigin::SetOrigin: - mmap_offset = off_t(offset); + mmap_offset = s64(offset); break; case SeekOrigin::CurrentPosition: - mmap_offset += off_t(offset); + mmap_offset += s64(offset); break; case SeekOrigin::End: - mmap_offset = off_t(mmap_size) + off_t(offset); + mmap_offset = s64(mmap_size) + s64(offset); break; } return true; diff --git a/src/common/fs/file.h b/src/common/fs/file.h index b8f3810277..dd3eedbe93 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -323,7 +323,7 @@ public: if (IsMappedFile()) { std::memcpy(&object, mmap_base + mmap_offset, sizeof(T)); #ifdef _WIN32 - return sizeof(T) != 0; + return bool(sizeof(T)); #else return sizeof(T); #endif @@ -442,7 +442,7 @@ public: [[nodiscard]] s64 Tell() const; #ifdef _WIN32 - inline bool IsMappedFile() const noexcept { return file_handle != nullptr; } + inline bool IsMappedFile() const noexcept { return mapping_handle != nullptr; } #else // POSIX inline bool IsMappedFile() const noexcept { return mmap_fd != -1; } #endif @@ -462,7 +462,7 @@ public: #endif u8* mmap_base = nullptr; size_t mmap_size = 0; - mutable off_t mmap_offset = 0; // fuck you + mutable s64 mmap_offset = 0; // fuck you }; } // namespace Common::FS From f9df1c88dc7569af6ac9ad0bb88221823b170912 Mon Sep 17 00:00:00 2001 From: lizzie Date: Thu, 22 Jan 2026 17:47:41 +0000 Subject: [PATCH 10/14] license --- src/common/fs/file.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/fs/file.h b/src/common/fs/file.h index dd3eedbe93..66a16c11d1 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project From 329b38a6fec554ae3fd72624588c4d5ffc52fc4f Mon Sep 17 00:00:00 2001 From: lizzie Date: Thu, 5 Feb 2026 22:32:03 +0000 Subject: [PATCH 11/14] alignment with log2 --- src/common/fs/file.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index d34e1475e4..bad7710ecc 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -7,6 +7,7 @@ #include #include "common/assert.h" +#include "common/bit_util.h" #include "common/fs/file.h" #include "common/fs/fs.h" #ifdef ANDROID @@ -278,13 +279,19 @@ static int PlatformMapReadOnly(IOFile& io, const char* path) { // a fun amount of fragmentation on the disk. map_flags |= MAP_NOSYNC; #endif -#ifdef MAP_ALIGNED_SUPER +#if defined(NDEBUG) && defined(MAP_NOCORE) + map_flags |= MAP_NOCORE; +#endif +#ifdef MAP_ALIGNED // File must be big enough that it's worth to super align. We can't just super-align every // file otherwise we will run out of alignments for actually important files :) // System doesn't guarantee a super alignment, but if it's available it will delete // about 3 layers(?) of the TLB tree for each read/write. // Again the cost of faults may make this negligible gains, but hey, we gotta work // what we gotta work with. + auto const align_log = Common::Log2Ceil64(u64(st.st_size)); + map_flags |= align_log > 12 ? MAP_ALIGNED(align_log) : 0; +#elif defined(MAP_ALIGNED_SUPER) using namespace Common::Literals; u64 big_file_threshold = 512_MiB; map_flags |= u64(st.st_size) >= big_file_threshold ? MAP_ALIGNED_SUPER : 0; From 9afeb563a8071e8032c399581eb2e1b7575ad9d1 Mon Sep 17 00:00:00 2001 From: lizzie Date: Tue, 14 Apr 2026 20:42:06 +0000 Subject: [PATCH 12/14] fix linux readahead --- src/common/fs/file.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index bad7710ecc..db45214b98 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -10,6 +10,7 @@ #include "common/bit_util.h" #include "common/fs/file.h" #include "common/fs/fs.h" +#include "common/fs_types.h" #ifdef ANDROID #include "common/fs/fs_android.h" #endif @@ -301,7 +302,13 @@ static int PlatformMapReadOnly(IOFile& io, const char* path) { close(io.mmap_fd); io.mmap_fd = -1; } else { - posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_WILLNEED); + // For small files it is acceptable to use a full readahead + // See https://github.com/torvalds/linux/blob/e80d033851b3bc94c3d254ac66660ddd0a49d72c/include/linux/pagemap.h#L1392 + if (st.st_size >= 256_MiB) { + posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_RANDOM); + } else { + posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_SEQUENTIAL); + } } } return io.mmap_fd; From 4afed9789ea8e19c406175459ff1cad75b791ec2 Mon Sep 17 00:00:00 2001 From: lizzie Date: Tue, 14 Apr 2026 20:55:57 +0000 Subject: [PATCH 13/14] oopsie --- src/common/fs/file.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index db45214b98..f75af2f6f2 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -10,7 +10,7 @@ #include "common/bit_util.h" #include "common/fs/file.h" #include "common/fs/fs.h" -#include "common/fs_types.h" +#include "common/fs/fs_types.h" #ifdef ANDROID #include "common/fs/fs_android.h" #endif @@ -302,9 +302,10 @@ static int PlatformMapReadOnly(IOFile& io, const char* path) { close(io.mmap_fd); io.mmap_fd = -1; } else { + using namespace Common::Literals; // For small files it is acceptable to use a full readahead // See https://github.com/torvalds/linux/blob/e80d033851b3bc94c3d254ac66660ddd0a49d72c/include/linux/pagemap.h#L1392 - if (st.st_size >= 256_MiB) { + if (u64(st.st_size) >= 256_MiB) { posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_RANDOM); } else { posix_madvise(io.mmap_base, io.mmap_size, POSIX_MADV_SEQUENTIAL); From 606a3b4ea50d67e4374d8f3ff962802d0293173b Mon Sep 17 00:00:00 2001 From: lizzie Date: Wed, 29 Apr 2026 21:04:29 +0000 Subject: [PATCH 14/14] use 2mb pages --- src/common/fs/file.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/common/fs/file.cpp b/src/common/fs/file.cpp index f75af2f6f2..847da16b5e 100644 --- a/src/common/fs/file.cpp +++ b/src/common/fs/file.cpp @@ -283,19 +283,16 @@ static int PlatformMapReadOnly(IOFile& io, const char* path) { #if defined(NDEBUG) && defined(MAP_NOCORE) map_flags |= MAP_NOCORE; #endif -#ifdef MAP_ALIGNED +#ifdef MAP_HUGE_2MB + map_flags |= MAP_HUGE_2MB; //2mb pages +#elif defined(MAP_ALIGNED) // File must be big enough that it's worth to super align. We can't just super-align every // file otherwise we will run out of alignments for actually important files :) // System doesn't guarantee a super alignment, but if it's available it will delete // about 3 layers(?) of the TLB tree for each read/write. // Again the cost of faults may make this negligible gains, but hey, we gotta work // what we gotta work with. - auto const align_log = Common::Log2Ceil64(u64(st.st_size)); - map_flags |= align_log > 12 ? MAP_ALIGNED(align_log) : 0; -#elif defined(MAP_ALIGNED_SUPER) - using namespace Common::Literals; - u64 big_file_threshold = 512_MiB; - map_flags |= u64(st.st_size) >= big_file_threshold ? MAP_ALIGNED_SUPER : 0; + map_flags |= MAP_ALIGNED(21); //2^21 = 2mb use 2MB pages #endif io.mmap_base = (u8*)mmap(nullptr, io.mmap_size, PROT_READ, map_flags, io.mmap_fd, 0); if (io.mmap_base == MAP_FAILED) {