Play your own Switch games on iPhone/iPad — one-click macOS app, no terminal. | 把 Switch 游戏装进 iPhone:图形界面一键构建,不用命令行。Bring your own game.
Swift apple-siliconcustom-formathomebrewiosipaipadipadosiphonemetalnative-portnintendo-switchnxnativeromfsswitch-games
patch_nso.py: every rewrite stopped with a NameError; the NRO header skip as seedds/nxnative#397
v0lumehi2ed54afa9441
diff --git a/dm/tools/patch_nso.py b/dm/tools/patch_nso.pyindex e2072200..1a235db2 100644--- a/dm/tools/patch_nso.py+++ b/dm/tools/patch_nso.py@@ -20,8 +20,21 @@ from capstone.arm64 import ARM64_OP_IMM, ARM64_OP_MEM, ARM64_OP_REGimport nso_asmfrom nso_asm import * # the instruction encoders (B, ADRP, LDRX, ...), shared with the per-build patch files+def code_start(d):+ """Where the code in the text segment starts: 0x80 in an NRO, 0 in an NSO. An NRO's text segment begins at file and memory+ offset 0 with the file's own header in it (the start word, the MOD0 offset, NRO0 at +0x10, the segment table at +0x20, the+ build ID at +0x40), so scanning it from 0 read the header as instructions: the build ID decoded as "ldr x18, #…" and stopped+ Final Fantasy X/X-2 HD Remaster's rewrite (GitHub issue LNPANDY/NX2Apple#14). An NSO's text holds code from 0."""+ return 0x80 if d[0x10:0x14] == b'NRO0' else 0++def code_ranges(data_blocks, text_end, start=0):+ """The text's code ranges: the gaps between its data blocks (nso_code.data_blocks), none of them before `start` (code_start)"""+ gaps = zip([0] + [e for _, e in data_blocks], [s for s, _ in data_blocks] + [text_end])+ return [(max(s, start), e) for s, e in gaps if e > max(s, start)]+src, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3]d = open(src, 'rb').read()+CODE_START = code_start(d)if d[0x10:0x14] == b'NRO0':# An NRO (a module the game loads at run time with nn::ro, Super Smash Bros. Ultimate's fighter code): the file is the image itself, each segment stored at its memory offset# and uncompressed; +0x20 text, +0x28 ro, +0x30 data as {offset, size}, +0x38 bss size, +0x40 the build ID. Rewritten as an NSO of the same layout would be@@ -463,21 +476,14 @@ for _ in range(nfde):a, s = rd(o, tenc); o += s_, s = rd(o, tenc); o += sstarts.add(a)-starts = sorted(x for x in starts if x < TEXT_LO + TEXT_END)+starts = sorted(x for x in starts if CODE_START <= x < TEXT_END)# ---- Linear scan to find all rewrite points (not relying on register read/write reports) ----# Data blocks in the text (nso_code.py: Absolum's main starts with 1.7 MB of them) are not code: words there that read as rewrite sites are left alonesys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))import nso_code-DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_LO + TEXT_END, md, lo=TEXT_LO)-# The text segment starts at its memory offset: 0 for an NSO (the image is rebuilt from 0) and-# 0x80 for an NRO (the file is the image preceded by its header, so text sits at 0x80). Scanning-# an NRO from 0 read the header's build ID as instructions and reported-# "44 ldr x18, #0xfffffffffff40370" on Final Fantasy X/X-2 HD Remaster (GitHub issue #14).-# TEXT_LO is where the code really begins in IMG; data_blocks() gets the same lower bound so its-# returned offsets are already IMG-absolute and CODE_RANGES needs no further shift.-TEXT_LO = segs[0][1]-CODE_RANGES = list(zip([TEXT_LO] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_LO + TEXT_END]))+DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_END, md)+CODE_RANGES = code_ranges(DATA_BLOCKS, TEXT_END, CODE_START)md_lite = Cs(CS_ARCH_ARM64, CS_MODE_ARM) # no details: the x18 scan reads only the operand textR18 = re.compile(r'\b[xw]18\b')def x18_words(lo, hi):@@ -976,7 +982,7 @@ def scratch_dead(off):mov x0, x19 before one) or a ret (none of them carries a result under AAPCS64; Pikmin 4's gate sits on the ldp x29, x30 of an epilogue). Only straight-line code is followed: a branchor a blr before the bl or ret answers no"""written = set()- for a in range(off, min(off + 4 * 16, TEXT_LO + TEXT_END), 4):+ for a in range(off, min(off + 4 * 16, TEXT_END), 4):ins = dis(struct.unpack_from('<I', IMG, a)[0], a)if ins.mnemonic in ('bl', 'ret'): return Trueif ins.mnemonic in ('b', 'br', 'blr', 'cbz', 'cbnz', 'tbz', 'tbnz') or ins.mnemonic.startswith('b.'): return False@@ -993,7 +999,7 @@ for off, oid, ow, why in obs:# and right after a call returns the callee was free to clobber them anyway (x16 / x17 are the veneer registers). Elsewhere, where the code ahead reads none of them before its next blafter_bl = off >= 4 and struct.unpack_from('<I', IMG, off - 4)[0] >> 26 == 0x25assert off in set(starts) or after_bl or scratch_dead(off), f'observation gate neither at a function entry, after a bl nor before a call with x9–x17 unread {off:#x}'- assert off < TEXT_LO + TEXT_END and off not in kinds and not any(lo <= off < hi for wseg in windows for lo, hi in [(wseg[0].address, wseg[-1].address + 4)]), f'observation gate conflicts with a rewrite site {off:#x}'+ assert off < TEXT_END and off not in kinds and not any(lo <= off < hi for wseg in windows for lo, hi in [(wseg[0].address, wseg[-1].address + 4)]), f'observation gate conflicts with a rewrite site {off:#x}'put_branch(off, site_obs(off, oid, w)); stat('obs')# ---- Frame step in whole vsyncs (the mains whose patch file has FRAME_STEP: The Legend of Zelda: Tears of the Kingdom 1.4.3, Breath of the Wild 1.9.0, Hyrule Warriors: Age of Calamity 1.3.0,diff --git a/dm/tools/tests/test_nro_header.py b/dm/tools/tests/test_nro_header.pyindex c5f07b46..6c7ab421 100644--- a/dm/tools/tests/test_nro_header.py+++ b/dm/tools/tests/test_nro_header.py@@ -1,67 +1,59 @@-"""The text scan must start at the text segment's offset, not at 0 (GitHub issue #14).--An NRO file is the image itself preceded by a header: its text segment starts at 0x80, while an-NSO's text starts at 0 after decompression. Scanning an NRO from 0 read the header's build ID as-instructions and reported `44 ldr x18, #0xfffffffffff40370` on Final Fantasy X/X-2 HD Remaster.--Testing the real code path end to end needs a structurally complete NRO (MOD0 block, FDE tables),-which is a lot of fixture for the question "does the lower bound hold". These tests check the two-things that were wrong instead:--1. `nso_code.data_blocks(..., lo=N)` never returns a block that starts below N.-2. `patch_nso.py` passes the text segment's own offset as that bound, which is 0x80 for an NRO and- 0 for an NSO — asserted on the source, since the value comes from the file being rewritten.+"""An NRO's header is not code (GitHub issue #14).++An NRO's text segment starts at offset 0 with the file's own header in it: Hades' EOSSDK-Switch-Shipping.nro has text+{0x0, 0xa41000} in its segment table and NRO0 at +0x10. Scanning it from 0 read the header's build ID as instructions and+reported `44 ldr x18, #0xfffffffffff40370` on Final Fantasy X/X-2 HD Remaster. The segment table cannot give the bound+(it says 0): the earlier TEXT_LO = segs[0][1] form kept scanning the header, and used TEXT_LO before defining it, which+stopped every rewrite with a NameError.++patch_nso.py's code_start() is 0x80 for an NRO and 0 for an NSO, and code_ranges() starts no code range before it; both+scans (the x18 words and the rewrite sites) walk those ranges. This is seedds/nxnative#397, carried by patch_patch_nso in+scripts/sync_upstream.py until it is upstream (its own test there is test_nro_code_start.py). The functions are taken+from patch_nso.py itself, which runs as a script."""-import sys-import unittest+import astfrom pathlib import Path+import struct+import unittest-HERE = Path(__file__).resolve()-sys.path.insert(0, str(HERE.parents[1]))--import nso_code # noqa: E402+TOOLS = Path(__file__).resolve().parents[1]+SOURCE = (TOOLS / 'patch_nso.py').read_text(encoding='utf-8')+scope = {}+exec(compile(ast.Module(body=[n for n in ast.parse(SOURCE).body if isinstance(n, ast.FunctionDef) and n.name in ('code_start', 'code_ranges')],+ type_ignores=[]), str(TOOLS / 'patch_nso.py'), 'exec'), scope)+code_start, code_ranges = scope['code_start'], scope['code_ranges']-TOOLS = HERE.parents[1]+LDR_X18 = 0x58000000 | (0x7ff40370 >> 2 & 0x7ffff) << 5 | 18 # a literal load into x18: what the build ID's bytes read as-class DataBlockLowerBound(unittest.TestCase):- def test_gaps_start_at_lo(self):- """The first gap must begin at `lo`, not at 0.+def nro(text_size=0x1000):+ """A synthetic NRO's first 0x80 bytes, laid out as Hades' EOS NRO is: text at offset 0 with the header inside it"""+ h = bytearray(0x80)+ h[0x10:0x14] = b'NRO0'+ struct.pack_into('<II', h, 0x20, 0, text_size) # text {offset, size}+ struct.pack_into('<I', h, 0x44, LDR_X18) # inside the build ID+ return bytes(h)- data_blocks() derives its gaps from the FDE table, which needs a structurally complete- NRO to exercise. The bound itself is one line — `gaps, last = [], lo` — so it is checked- by patching fde_ranges to report one covered range and reading the gaps back.- """- real = nso_code.fde_ranges- try:- nso_code.fde_ranges = lambda img: [(0x100, 0x200)]- from capstone import Cs, CS_ARCH_ARM64, CS_MODE_ARM- md = Cs(CS_ARCH_ARM64, CS_MODE_ARM)- img = bytes(0x400)- # text_end 0x300, lo 0x80: the gaps are [0x80,0x100) and [0x200,0x300), never below 0x80- blocks = nso_code.data_blocks(img, 0x300, md, lo=0x80)- self.assertTrue(all(s >= 0x80 for s, _ in blocks),- f'a block starts below the text segment: {blocks}')- blocks0 = nso_code.data_blocks(img, 0x300, md)- self.assertTrue(all(s >= 0 for s, _ in blocks0), 'default lo must not break NSOs')- finally:- nso_code.fde_ranges = real- def test_lo_defaults_to_zero(self):- """An NSO keeps the old behaviour: no header, so the scan starts at 0."""- self.assertEqual(nso_code.data_blocks.__defaults__, (0,),- 'data_blocks must default lo to 0 so the NSO path is unchanged')+class CodeStart(unittest.TestCase):+ def test_an_nro_s_header_is_outside_every_code_range(self):+ d = nro()+ self.assertEqual(struct.unpack_from('<II', d, 0x20)[0], 0, 'the segment table says 0, so it cannot bound the scan')+ self.assertEqual(code_start(d), 0x80)+ ranges = code_ranges([(0x400, 0x500)], 0x1000, code_start(d))+ self.assertEqual(ranges, [(0x80, 0x400), (0x500, 0x1000)])+ self.assertFalse(any(s <= 0x44 < e for s, e in ranges))+ def test_an_nso_keeps_its_ranges(self):+ self.assertEqual(code_start(b'NSO0' + bytes(0x7c)), 0)+ self.assertEqual(code_ranges([(0x400, 0x500)], 0x1000), [(0, 0x400), (0x500, 0x1000)])-class PatchNsoPassesTheOffset(unittest.TestCase):- def test_text_lo_comes_from_the_segment(self):- src = (TOOLS / 'patch_nso.py').read_text(encoding='utf-8')- # TEXT_LO must be the text segment's memory offset, and both the scan and CODE_RANGES must- # use it. Hardcoding 0x80 would break NSOs; dropping it would bring the NRO bug back.- self.assertIn('TEXT_LO = segs[0][1]', src)- self.assertIn('data_blocks(IMG, TEXT_LO + TEXT_END, md, lo=TEXT_LO)', src)- self.assertRegex(src, r'CODE_RANGES = list\(zip\(\[TEXT_LO\]')+ def test_the_bounds_are_defined_before_the_scan_uses_them(self):+ """The NameError: each name the scan reads is assigned above its first use"""+ for name, use in (('CODE_START', 'starts = sorted('), ('CODE_RANGES', 'x18 = set().union(')):+ self.assertLess(SOURCE.index(f'{name} = '), SOURCE.index(use), name)+ self.assertNotIn('TEXT_LO', SOURCE)if __name__ == '__main__':- unittest.main(verbosity=2)+ unittest.main()diff --git a/scripts/sync_upstream.py b/scripts/sync_upstream.pyindex 974e653a..681a8c4d 100644--- a/scripts/sync_upstream.py+++ b/scripts/sync_upstream.py@@ -63,7 +63,7 @@ OURS = {'dm/host/romfs_arc.c', 'dm/host/romfs_arc.h', 'dm/tools/romfs_arc.py',PATCHED = ('dm/host/hle_fs.c', 'dm/host/Makefile', 'dm/host/tests/Makefile', 'dm/ios/tools/build_ipa.sh','scripts/build_ipa_from_nsp.py', 'dm/gfx/tools/shader_split.py','dm/ios/Sources/main.m', 'dm/gfx/nvn/nvn_mtl.m', 'dm/host/dm.h',- 'dm/ios/tests/test_romfs_startup.py', 'dm/ios/tests/test_profile_pack.py', 'docs/env_switches.tsv')+ 'dm/ios/tests/test_romfs_startup.py', 'dm/ios/tests/test_profile_pack.py', 'docs/env_switches.tsv', 'dm/host/front_mac.m', 'dm/tools/patch_nso.py', 'dm/games/0100A00019DE0000/game.tsv')def git(*a):r = subprocess.run(['git', *a], cwd=REPO, capture_output=True, text=True)@@ -939,38 +939,23 @@ def patch_main_m(t):return tdef patch_patch_nso(t):- """The NRO header skip in dm/tools/patch_nso.py (issue #14).-- An NRO file is the image itself preceded by a 0x80-byte header, so its text segment starts at- 0x80; an NSO's starts at 0 after decompression. The scan and CODE_RANGES assumed 0, so on an- NRO the header's build ID was decoded as instructions and reported as a site- ("44 ldr x18, #0xfffffffffff40370" on Final Fantasy X/X-2 HD Remaster), and the last 0x80- bytes of text were never scanned.-- This was fixed once (4b863234) and then silently reverted: patch_nso.py was not a carrier, so- the next sync that rewrote it took upstream's version, and the fork's test kept failing from- then on. It is a carrier now.+ """The NRO header skip in dm/tools/patch_nso.py (issue #14), as sent upstream in seedds/nxnative#397.++ An NRO's text segment starts at offset 0 with the file's own header in it (Hades' EOSSDK-Switch-Shipping.nro: text+ {0x0, 0xa41000}), so scanning it from 0 read the header's build ID as instructions ("44 ldr x18, #0xfffffffffff40370" on+ Final Fantasy X/X-2 HD Remaster). code_start() is 0x80 for an NRO and 0 for an NSO; code_ranges() starts no code range+ before it. The segment table cannot give the bound (it says 0): the TEXT_LO = segs[0][1] form this patcher applied+ before kept scanning the header, and it used TEXT_LO in the site bound 13 lines before defining it, so every rewrite+ stopped with a NameError. When #397 is merged upstream, every hunk is found applied and this patcher can go."""- if 'TEXT_LO = segs[0][1]' in t: return skip('patch_nso.py NRO header (already patched)')+ if 'def code_start(d):' in t and 'TEXT_LO' not in t: return skip('patch_nso.py NRO header (already patched)')P = [- ('data blocks and code ranges',- 'DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_END, md)\n'- 'CODE_RANGES = list(zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END]))\n',- 'DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_LO + TEXT_END, md, lo=TEXT_LO)\n'- '# The text segment starts at its memory offset: 0 for an NSO (the image is rebuilt from 0) and\n'- '# 0x80 for an NRO (the file is the image preceded by its header, so text sits at 0x80). Scanning\n'- '# an NRO from 0 read the header\'s build ID as instructions and reported\n'- '# "44 ldr x18, #0xfffffffffff40370" on Final Fantasy X/X-2 HD Remaster (GitHub issue #14).\n'- '# TEXT_LO is where the code really begins in IMG; data_blocks() gets the same lower bound so its\n'- '# returned offsets are already IMG-absolute and CODE_RANGES needs no further shift.\n'- 'TEXT_LO = segs[0][1]\n'- 'CODE_RANGES = list(zip([TEXT_LO] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_LO + TEXT_END]))\n'),+ ('code_start and code_ranges', "src, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3]\nd = open(src, 'rb').read()\n",+ 'def code_start(d):\n """Where the code in the text segment starts: 0x80 in an NRO, 0 in an NSO. An NRO\'s text segment begins at file and memory\n offset 0 with the file\'s own header in it (the start word, the MOD0 offset, NRO0 at +0x10, the segment table at +0x20, the\n build ID at +0x40), so scanning it from 0 read the header as instructions: the build ID decoded as "ldr x18, #…" and stopped\n Final Fantasy X/X-2 HD Remaster\'s rewrite (GitHub issue LNPANDY/NX2Apple#14). An NSO\'s text holds code from 0."""\n return 0x80 if d[0x10:0x14] == b\'NRO0\' else 0\n\ndef code_ranges(data_blocks, text_end, start=0):\n """The text\'s code ranges: the gaps between its data blocks (nso_code.data_blocks), none of them before `start` (code_start)"""\n gaps = zip([0] + [e for _, e in data_blocks], [s for s, _ in data_blocks] + [text_end])\n return [(max(s, start), e) for s, e in gaps if e > max(s, start)]\n\nsrc, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3]\nd = open(src, \'rb\').read()\nCODE_START = code_start(d)\n'),('site starts bound', 'starts = sorted(x for x in starts if x < TEXT_END)',- 'starts = sorted(x for x in starts if x < TEXT_LO + TEXT_END)'),- ('observation gate window', ' for a in range(off, min(off + 4 * 16, TEXT_END), 4):',- ' for a in range(off, min(off + 4 * 16, TEXT_LO + TEXT_END), 4):'),- ('observation gate assert', ' assert off < TEXT_END and off not in kinds',- ' assert off < TEXT_LO + TEXT_END and off not in kinds'),+ 'starts = sorted(x for x in starts if CODE_START <= x < TEXT_END)'),+ ('code ranges', 'CODE_RANGES = list(zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END]))',+ 'CODE_RANGES = code_ranges(DATA_BLOCKS, TEXT_END, CODE_START)'),]for name, old, new in P:if new in t:@@ -978,7 +963,8 @@ def patch_patch_nso(t):continueif old not in t:fail(f'dm/tools/patch_nso.py: patch "{name}" anchor missing (upstream reworked the text '- 'scan; merge the NRO header skip by hand)')+ 'scan; merge the NRO header skip by hand, or take seedds/nxnative#397 if it is merged)')+ continuet = t.replace(old, new, 1)done(f'patch_nso.py {name}')return t@@ -1158,7 +1144,7 @@ MARKERS = {'dm/games/0100A00019DE0000/game.tsv': 'build v1 1.0.137848','dm/host/dm.h': 'hle_profile.h','dm/host/front_mac.m': 'DM_KEYMAP',- 'dm/tools/patch_nso.py': 'TEXT_LO',+ 'dm/tools/patch_nso.py': 'def code_start(d):','dm/ios/tests/test_romfs_startup.py': 'fixture_profile','dm/ios/tests/test_profile_pack.py': 'slim package','docs/env_switches.tsv': 'DM_FULL_ROMFS\t',