VERY experimental Nintendo Switch 2 emulator
C++
[file_sys] robust-er IPSwitch compiler (#3911)
lizzie672bcbae0121
diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cppindex 9981c1865d..2022c9c2e7 100644--- a/src/core/file_sys/ips_layer.cpp+++ b/src/core/file_sys/ips_layer.cpp@@ -3,10 +3,12 @@#include <algorithm>#include <cstring>-#include <map>#include <sstream>#include <string>#include <utility>+#include <span>+#include <cctype>+#include <ankerl/unordered_dense.h>#include "common/hex_util.h"#include "common/logging.h"@@ -22,60 +24,29 @@ enum class IPSFileType {Error,};-constexpr std::array<std::pair<const char*, const char*>, 11> ESCAPE_CHARACTER_MAP{{- {"\\a", "\a"},- {"\\b", "\b"},- {"\\f", "\f"},- {"\\n", "\n"},- {"\\r", "\r"},- {"\\t", "\t"},- {"\\v", "\v"},- {"\\\\", "\\"},- {"\\\'", "\'"},- {"\\\"", "\""},- {"\\\?", "\?"},-}};--static IPSFileType IdentifyMagic(const std::vector<u8>& magic) {- if (magic.size() != 5) {- return IPSFileType::Error;- }-- static constexpr std::array<u8, 5> patch_magic{{'P', 'A', 'T', 'C', 'H'}};- if (std::equal(magic.begin(), magic.end(), patch_magic.begin())) {- return IPSFileType::IPS;- }-- static constexpr std::array<u8, 5> ips32_magic{{'I', 'P', 'S', '3', '2'}};- if (std::equal(magic.begin(), magic.end(), ips32_magic.begin())) {- return IPSFileType::IPS32;+static IPSFileType IdentifyMagic(std::span<const u8> magic) {+ if (magic.size() >= 5) {+ if (std::memcmp(magic.data(), "PATCH", 5) == 0)+ return IPSFileType::IPS;+ if (std::memcmp(magic.data(), "IPS32", 5) == 0)+ return IPSFileType::IPS32;}-return IPSFileType::Error;}-static bool IsEOF(IPSFileType type, const std::vector<u8>& data) {- static constexpr std::array<u8, 3> eof{{'E', 'O', 'F'}};- if (type == IPSFileType::IPS && std::equal(data.begin(), data.end(), eof.begin())) {- return true;- }-- static constexpr std::array<u8, 4> eeof{{'E', 'E', 'O', 'F'}};- return type == IPSFileType::IPS32 && std::equal(data.begin(), data.end(), eeof.begin());+static bool IsEOF(IPSFileType type, std::span<const u8> magic) {+ return (type == IPSFileType::IPS && magic.size() > 3 && std::memcmp(magic.data(), "EOF", 3) == 0)+ || (type == IPSFileType::IPS32 && magic.size() > 4 && std::memcmp(magic.data(), "EEOF", 4) == 0);}VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {if (in == nullptr || ips == nullptr)return nullptr;- const auto type = IdentifyMagic(ips->ReadBytes(0x5));- if (type == IPSFileType::Error)- return nullptr;-auto in_data = in->ReadAllBytes();- if (in_data.size() == 0) {+ auto const type = IdentifyMagic(in_data);+ if (type == IPSFileType::Error)return nullptr;- }std::vector<u8> temp(type == IPSFileType::IPS ? 3 : 4);u64 offset = 5; // After header@@ -85,12 +56,9 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {break;}- u32 real_offset{};- if (type == IPSFileType::IPS32)- real_offset = (temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3];- else- real_offset = (temp[0] << 16) | (temp[1] << 8) | temp[2];-+ u32 real_offset = (type == IPSFileType::IPS32)+ ? ((temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3])+ : ((temp[0] << 16) | (temp[1] << 8) | temp[2]);if (real_offset > in_data.size()) {return nullptr;}@@ -113,34 +81,35 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) {return nullptr;if (real_offset + rle_size > in_data.size())- rle_size = static_cast<u16>(in_data.size() - real_offset);+ rle_size = u16(in_data.size() - real_offset);std::memset(in_data.data() + real_offset, *data, rle_size);} else { // Standard Patchauto read = data_size;if (real_offset + read > in_data.size())- read = static_cast<u16>(in_data.size() - real_offset);+ read = u16(in_data.size() - real_offset);if (ips->Read(in_data.data() + real_offset, read, offset) != data_size)return nullptr;offset += data_size;}}-- if (!IsEOF(type, temp)) {- return nullptr;+ if (IsEOF(type, temp)) {+ return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(), in->GetContainingDirectory());}-- return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(),- in->GetContainingDirectory());+ return nullptr;}++struct IPSwitchRecord {+ std::array<uint8_t, 256 - sizeof(size_t)> data;+ size_t count;+};struct IPSwitchCompiler::IPSwitchPatch {- std::string name;+ ankerl::unordered_dense::map<u32, IPSwitchRecord> records;bool enabled;- std::map<u32, std::vector<u8>> records;};IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) {- Parse();+ Parse(patch_text->ReadAllBytes());}IPSwitchCompiler::~IPSwitchCompiler() = default;@@ -149,201 +118,159 @@ std::array<u8, 32> IPSwitchCompiler::GetBuildID() const {return nso_build_id;}-bool IPSwitchCompiler::IsValid() const {- return valid;-}--static bool StartsWith(std::string_view base, std::string_view check) {- return base.size() >= check.size() && base.substr(0, check.size()) == check;-}--static std::string EscapeStringSequences(std::string in) {- for (const auto& seq : ESCAPE_CHARACTER_MAP) {- for (auto index = in.find(seq.first); index != std::string::npos;- index = in.find(seq.first, index)) {- in.replace(index, std::strlen(seq.first), seq.second);- index += std::strlen(seq.second);+static IPSwitchRecord EscapeStringSequences(std::string_view sv) {+ IPSwitchRecord r{};+ for (auto it = sv.cbegin(); it != sv.cend(); ) {+ if (*it == '\\' && it + 1 < sv.cend()) {+ switch (it[1]) {+ case 'n': r.data[r.count] = '\n'; break;+ case 't': r.data[r.count] = '\t'; break;+ case 'b': r.data[r.count] = '\b'; break;+ case 'r': r.data[r.count] = '\r'; break;+ case 'e': r.data[r.count] = '\e'; break;+ case 'v': r.data[r.count] = '\v'; break;+ case '?': r.data[r.count] = '\?'; break;+ default: r.data[r.count] = it[1]; break;+ }+ ++r.count;+ it += 2;+ } else {+ ++r.count;+ ++it;}}-- return in;+ return r;}-void IPSwitchCompiler::ParseFlag(const std::string& line) {- if (StartsWith(line, "@flag offset_shift ")) {- // Offset Shift Flag- offset_shift = std::strtoll(line.substr(19).c_str(), nullptr, 0);- } else if (StartsWith(line, "@little-endian")) {- // Set values to read as little endian- is_little_endian = true;- } else if (StartsWith(line, "@big-endian")) {- // Set values to read as big endian- is_little_endian = false;- } else if (StartsWith(line, "@flag print_values")) {- // Force printing of applied values- print_values = true;- }+[[nodiscard]] static inline std::array<u8, 32> ReadNSOBuildId(std::string_view const s) {+ std::array<u8, 32> r{};+ for (std::size_t i = 0; i < s.size(); ++i)+ r[i / 2] |= u8(u8(Common::ToHexNibble(s[i])) << u8((i % 2) * 4));+ return r;}-void IPSwitchCompiler::Parse() {- const auto bytes = patch_text->ReadAllBytes();- std::stringstream s;- s.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());-- std::vector<std::string> lines;- std::string stream_line;- while (std::getline(s, stream_line)) {- // Remove a trailing \r- if (!stream_line.empty() && stream_line.back() == '\r')- stream_line.pop_back();- lines.push_back(std::move(stream_line));- }-- for (std::size_t i = 0; i < lines.size(); ++i) {- auto line = lines[i];-- // Remove midline comments- std::size_t comment_index = std::string::npos;- bool within_string = false;- for (std::size_t k = 0; k < line.size(); ++k) {- if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) {- within_string = !within_string;- } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) {- comment_index = k;- break;- }- }-- if (!StartsWith(line, "//") && comment_index != std::string::npos) {- last_comment = line.substr(comment_index + 2);- line = line.substr(0, comment_index);- }-- if (StartsWith(line, "@stop")) {- // Force stop- break;- } else if (StartsWith(line, "@nsobid-")) {- // NSO Build ID Specifier- const auto raw_build_id = fmt::format("{:0<64}", line.substr(8));- nso_build_id = Common::HexStringToArray<0x20>(raw_build_id);- } else if (StartsWith(line, "#")) {- // Mandatory Comment- LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}",- patch_text->GetName(), line.substr(1));- } else if (StartsWith(line, "//")) {- // Normal Comment- last_comment = line.substr(2);- if (last_comment.find_first_not_of(' ') == std::string::npos)- continue;- if (last_comment.find_first_not_of(' ') != 0)- last_comment = last_comment.substr(last_comment.find_first_not_of(' '));- } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) {- // Start of patch- const auto enabled = StartsWith(line, "@enabled");- if (i == 0)- return;- LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})",- patch_text->GetName(), last_comment, line.substr(1));-- IPSwitchPatch patch{last_comment, enabled, {}};-- // Read rest of patch- while (true) {- if (i + 1 >= lines.size()) {- break;- }-- const auto& patch_line = lines[++i];-- // Patch line may contain comments- if (StartsWith(patch_line, "//") || StartsWith(patch_line, "#")) {- continue;- }-- // Start of new patch- if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) {- --i;- break;+void IPSwitchCompiler::Parse(std::span<u8 const> bytes) {+ LOG_INFO(Loader, "IPSwitchCompiler: '{}'", patch_text->GetName());+ bool is_little_endian = false;+ s64 offset_shift = 0;+ //bool print_values = false;++ auto const parse_line = [&](std::string_view const line) {+ // Keep in mind lines have trimmed spaces (at the end & start)!+ LOG_INFO(Loader, "<{}>", line);+ if (line.starts_with("@stop")) {+ return false; // Force stop+ } else if (line.starts_with("@nsobid-")) { // NSO Build ID Specifier+ nso_build_id = ReadNSOBuildId(line.substr(8));+ } else if (line.starts_with("@enabled")) {+ patches.push_back({{}, true}); //enabled patch+ } else if (line.starts_with("@disabled")) {+ patches.push_back({{}, false}); //disabled patch+ } else if (line.starts_with("@flag offset_shift ")) {+ offset_shift = std::strtoll(line.data() + 19, nullptr, 0); // Offset Shift Flag+ } else if (line.starts_with("@little-endian")) {+ is_little_endian = true; // Set values to read as little endian+ } else if (line.starts_with("@big-endian")) {+ is_little_endian = false; // Set values to read as big endian+ } else if (line.starts_with("@flag print_values")) {+ //print_values = true; // Force printing of applied values+ } else if (line.starts_with("@")) {+ LOG_WARNING(Loader, "Unknown flag {}", line);+ } else {+ size_t offset = size_t(std::strtoul(line.data(), nullptr, 16));+ offset += size_t(offset_shift);+ if (auto const first_quote = line.find_first_of("\"\'"); first_quote != std::string::npos) {+ // string replacement+ char quote = line[first_quote];+ auto const start = line.cbegin() + first_quote + 1;+ auto end = start;+ for (; end < line.cend() && *end != quote; )+ end += (*end == '\\') ? 2 : 1;+ if (start <= line.cend() && end <= line.cend()) {+ LOG_INFO(Loader, "[S] value @ {:#08X} ", offset);+ patches.back().records.insert_or_assign(u32(offset), EscapeStringSequences({start, end}));+ } else {+ LOG_WARNING(Loader, "invalid string");}-- // Check for a flag- if (StartsWith(patch_line, "@")) {- ParseFlag(patch_line);- continue;+ } else if (auto const first_space = line.find_last_of(" /\t\r\n"); first_space != std::string::npos) {+ IPSwitchRecord r{}; // hex replacement+ auto const start = line.cbegin() + first_space + 1;+ auto const end = line.cend();+ if (start <= line.cend() && end <= line.cend()) {+ auto const hs = Common::HexStringToVector({start, end}, is_little_endian);+ std::memcpy(r.data.data(), hs.data(), hs.size());+ r.count = hs.size();+ LOG_INFO(Loader, "[H] value @ {:#08X}", offset);+ patches.back().records.insert_or_assign(u32(offset), std::move(r));+ } else {+ LOG_WARNING(Loader, "invalid line");}-- // 11 - 8 hex digit offset + space + minimum two digit overwrite val- if (patch_line.length() < 11)+ } else {+ LOG_WARNING(Loader, "unhandled line!");+ }+ }+ return true; //continue+ };++ for (auto it = bytes.begin(); it < bytes.end(); ) {+ auto const start = it;+ auto end = start;+ for (; end < bytes.end() && *end != '\n' && *end != '\r'; ++end)+ ;+ it = end + 1; //prepare for next line+ std::string_view const sline{+ reinterpret_cast<const char*>(bytes.data() + std::distance(bytes.begin(), start)),+ size_t(std::distance(start, end))+ };+ if (sline.size() > 0) {+ auto p = sline.cbegin();+ // skip space off line+ for (; p < sline.cend() && std::isspace(*p); ++p)+ ;+ // now make a nominal preprocessed line: remove comments+ char quote = '\0';+ auto const sline_start = p;+ for (; p < sline.cend(); ) {+ if ((!quote && p + 1 < sline.cend() && p[0] == '/' && p[1] == '/')+ || (!quote && p[0] == '#')) {break;- auto offset = std::strtoul(patch_line.substr(0, 8).c_str(), nullptr, 16);- offset += static_cast<unsigned long>(offset_shift);-- std::vector<u8> replace;- // 9 - first char of replacement val- if (patch_line[9] == '\"') {- // string replacement- auto end_index = patch_line.find('\"', 10);- if (end_index == std::string::npos || end_index < 10)- return;- while (patch_line[end_index - 1] == '\\') {- end_index = patch_line.find('\"', end_index + 1);- if (end_index == std::string::npos || end_index < 10)- return;- }-- auto value = patch_line.substr(10, end_index - 10);- value = EscapeStringSequences(value);- replace.reserve(value.size());- std::copy(value.begin(), value.end(), std::back_inserter(replace));+ } else if (p[0] == '\"' || p[0] == '\'') {+ quote = (p[0] == quote) ? '\0' : p[0];+ ++p;+ } else if (p + 1 < sline.cend() && p[0] == '\\') {+ p += 2;} else {- // hex replacement- const auto value =- patch_line.substr(9, patch_line.find_first_of(" /\r\n", 9) - 9);- replace = Common::HexStringToVector(value, is_little_endian);- }-- if (print_values) {- LOG_INFO(Loader,- "[IPSwitchCompiler ('{}')] - Patching value at offset {:#08x} "- "with byte string '{}'",- patch_text->GetName(), offset, Common::HexToString(replace));+ ++p;}-- patch.records.insert_or_assign(static_cast<u32>(offset), std::move(replace));}-- patches.push_back(std::move(patch));- } else if (StartsWith(line, "@")) {- ParseFlag(line);+ // now we have the preprocessed string ;)+ std::string_view pp_str(sline_start, p);+ if (pp_str.size() > 0 && !parse_line(pp_str)) {+ break;+ }}}-- valid = true;}VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const {- if (in == nullptr || !valid)+ if (in == nullptr)return nullptr;auto in_data = in->ReadAllBytes();for (const auto& patch : patches) {- if (!patch.enabled)- continue;-- for (const auto& record : patch.records) {- if (record.first >= in_data.size())- continue;- auto replace_size = record.second.size();- if (record.first + replace_size > in_data.size())- replace_size = in_data.size() - record.first;- for (std::size_t i = 0; i < replace_size; ++i)- in_data[i + record.first] = record.second[i];+ if (patch.enabled) {+ for (const auto& record : patch.records) {+ if (record.first < in_data.size()) {+ auto replace_size = record.second.count;+ if (record.first + replace_size > in_data.size())+ replace_size = in_data.size() - record.first;+ std::memcpy(in_data.data() + record.first, record.second.data.data(), replace_size);+ }+ }}}-- return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(),- in->GetContainingDirectory());+ return std::make_shared<VectorVfsFile>(std::move(in_data), in->GetName(), in->GetContainingDirectory());}} // namespace FileSysdiff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.hindex d81378e8ad..e3c61f161b 100644--- a/src/core/file_sys/ips_layer.h+++ b/src/core/file_sys/ips_layer.h@@ -1,11 +1,14 @@+// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project+// SPDX-License-Identifier: GPL-3.0-or-later+// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project// SPDX-License-Identifier: GPL-2.0-or-later#pragma once#include <array>-#include <memory>#include <vector>+#include <span>#include "common/common_types.h"#include "core/file_sys/vfs/vfs.h"@@ -20,24 +23,17 @@ public:~IPSwitchCompiler();std::array<u8, 0x20> GetBuildID() const;- bool IsValid() const;VirtualFile Apply(const VirtualFile& in) const;private:struct IPSwitchPatch;void ParseFlag(const std::string& flag);- void Parse();-- bool valid = false;+ void Parse(std::span<u8 const> bytes);VirtualFile patch_text;std::vector<IPSwitchPatch> patches;std::array<u8, 0x20> nso_build_id{};- bool is_little_endian = false;- s64 offset_shift = 0;- bool print_values = false;- std::string last_comment = "";};} // namespace FileSysdiff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cppindex 6ae4679054..ada8c44e78 100644--- a/src/core/file_sys/patch_manager.cpp+++ b/src/core/file_sys/patch_manager.cpp@@ -345,8 +345,7 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const {return exefs;}-std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualDir>& patch_dirs,- const std::string& build_id) const {+std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualDir>& patch_dirs, const std::string& build_id) const {const auto& disabled = Settings::values.disabled_addons[title_id];const auto nso_build_id = fmt::format("{:0<64}", build_id);@@ -361,16 +360,11 @@ std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualDfor (const auto& file : exefs_dir->GetFiles()) {if (file->GetExtension() == "ips") {auto name = file->GetName();-- const auto this_build_id =- fmt::format("{:0<64}", name.substr(0, name.find('.')));+ const auto this_build_id = fmt::format("{:0<64}", name.substr(0, name.find('.')));if (nso_build_id == this_build_id)out.push_back(file);} else if (file->GetExtension() == "pchtxt") {IPSwitchCompiler compiler{file};- if (!compiler.IsValid())- continue;-const auto this_build_id = Common::HexToString(compiler.GetBuildID());if (nso_build_id == this_build_id)out.push_back(file);@@ -378,7 +372,6 @@ std::vector<VirtualFile> PatchManager::CollectPatches(const std::vector<VirtualD}}}-return out;}diff --git a/src/qt_common/util/mod.cpp b/src/qt_common/util/mod.cppindex c09b39cf82..9256d4e142 100644--- a/src/qt_common/util/mod.cpp+++ b/src/qt_common/util/mod.cpp@@ -2,6 +2,7 @@// SPDX-License-Identifier: GPL-3.0-or-later#include <filesystem>+#include <system_error>#include <JlCompress.h>#include "frontend_common/mod_manager.h"#include "mod.h"@@ -124,8 +125,8 @@ const QString ExtractMod(const QString& path) {fs::remove_all(tmp, ec);if (!fs::create_directories(tmp, ec)) {QtCommon::Frontend::Critical(tr("Mod Extract Failed"),- tr("Failed to create temporary directory %1")- .arg(QString::fromStdString(tmp.string())));+ tr("Failed to create temporary directory %1")+ .arg(QString::fromStdString(tmp.string())));return QString();}