Skip to content

Play your own Switch games on iPhone/iPad — one-click macOS app, no terminal. | 把 Switch 游戏装进 iPhone:图形界面一键构建,不用命令行。Bring your own game.

patch_nso.py: every rewrite stopped with a NameError; the NRO header skip as seedds/nxnative#397

v0lumehi2ed54afa9441
diff --git a/dm/tools/patch_nso.py b/dm/tools/patch_nso.py
index e2072200..1a235db2 100644
--- a/dm/tools/patch_nso.py
+++ b/dm/tools/patch_nso.py
@@ -20,8 +20,21 @@ from capstone.arm64 import ARM64_OP_IMM, ARM64_OP_MEM, ARM64_OP_REG
import nso_asm
from nso_asm import * # the instruction encoders (B, ADRP, LDRX, ...), shared with the per-build patch files
+def code_start(d):
+ """Where the code in the text segment starts: 0x80 in an NRO, 0 in an NSO. An NRO's text segment begins at file and memory
+ offset 0 with the file's own header in it (the start word, the MOD0 offset, NRO0 at +0x10, the segment table at +0x20, the
+ build ID at +0x40), so scanning it from 0 read the header as instructions: the build ID decoded as "ldr x18, #…" and stopped
+ Final Fantasy X/X-2 HD Remaster's rewrite (GitHub issue LNPANDY/NX2Apple#14). An NSO's text holds code from 0."""
+ return 0x80 if d[0x10:0x14] == b'NRO0' else 0
+
+def code_ranges(data_blocks, text_end, start=0):
+ """The text's code ranges: the gaps between its data blocks (nso_code.data_blocks), none of them before `start` (code_start)"""
+ gaps = zip([0] + [e for _, e in data_blocks], [s for s, _ in data_blocks] + [text_end])
+ return [(max(s, start), e) for s, e in gaps if e > max(s, start)]
+
src, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3]
d = open(src, 'rb').read()
+CODE_START = code_start(d)
if d[0x10:0x14] == b'NRO0':
# An NRO (a module the game loads at run time with nn::ro, Super Smash Bros. Ultimate's fighter code): the file is the image itself, each segment stored at its memory offset
# and uncompressed; +0x20 text, +0x28 ro, +0x30 data as {offset, size}, +0x38 bss size, +0x40 the build ID. Rewritten as an NSO of the same layout would be
@@ -463,21 +476,14 @@ for _ in range(nfde):
a, s = rd(o, tenc); o += s
_, s = rd(o, tenc); o += s
starts.add(a)
-starts = sorted(x for x in starts if x < TEXT_LO + TEXT_END)
+starts = sorted(x for x in starts if CODE_START <= x < TEXT_END)
# ---- Linear scan to find all rewrite points (not relying on register read/write reports) ----
# Data blocks in the text (nso_code.py: Absolum's main starts with 1.7 MB of them) are not code: words there that read as rewrite sites are left alone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import nso_code
-DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_LO + TEXT_END, md, lo=TEXT_LO)
-# The text segment starts at its memory offset: 0 for an NSO (the image is rebuilt from 0) and
-# 0x80 for an NRO (the file is the image preceded by its header, so text sits at 0x80). Scanning
-# an NRO from 0 read the header's build ID as instructions and reported
-# "44 ldr x18, #0xfffffffffff40370" on Final Fantasy X/X-2 HD Remaster (GitHub issue #14).
-# TEXT_LO is where the code really begins in IMG; data_blocks() gets the same lower bound so its
-# returned offsets are already IMG-absolute and CODE_RANGES needs no further shift.
-TEXT_LO = segs[0][1]
-CODE_RANGES = list(zip([TEXT_LO] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_LO + TEXT_END]))
+DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_END, md)
+CODE_RANGES = code_ranges(DATA_BLOCKS, TEXT_END, CODE_START)
md_lite = Cs(CS_ARCH_ARM64, CS_MODE_ARM) # no details: the x18 scan reads only the operand text
R18 = re.compile(r'\b[xw]18\b')
def x18_words(lo, hi):
@@ -976,7 +982,7 @@ def scratch_dead(off):
mov x0, x19 before one) or a ret (none of them carries a result under AAPCS64; Pikmin 4's gate sits on the ldp x29, x30 of an epilogue). Only straight-line code is followed: a branch
or a blr before the bl or ret answers no"""
written = set()
- for a in range(off, min(off + 4 * 16, TEXT_LO + TEXT_END), 4):
+ for a in range(off, min(off + 4 * 16, TEXT_END), 4):
ins = dis(struct.unpack_from('<I', IMG, a)[0], a)
if ins.mnemonic in ('bl', 'ret'): return True
if ins.mnemonic in ('b', 'br', 'blr', 'cbz', 'cbnz', 'tbz', 'tbnz') or ins.mnemonic.startswith('b.'): return False
@@ -993,7 +999,7 @@ for off, oid, ow, why in obs:
# and right after a call returns the callee was free to clobber them anyway (x16 / x17 are the veneer registers). Elsewhere, where the code ahead reads none of them before its next bl
after_bl = off >= 4 and struct.unpack_from('<I', IMG, off - 4)[0] >> 26 == 0x25
assert off in set(starts) or after_bl or scratch_dead(off), f'observation gate neither at a function entry, after a bl nor before a call with x9–x17 unread {off:#x}'
- assert off < TEXT_LO + TEXT_END and off not in kinds and not any(lo <= off < hi for wseg in windows for lo, hi in [(wseg[0].address, wseg[-1].address + 4)]), f'observation gate conflicts with a rewrite site {off:#x}'
+ assert off < TEXT_END and off not in kinds and not any(lo <= off < hi for wseg in windows for lo, hi in [(wseg[0].address, wseg[-1].address + 4)]), f'observation gate conflicts with a rewrite site {off:#x}'
put_branch(off, site_obs(off, oid, w)); stat('obs')
# ---- Frame step in whole vsyncs (the mains whose patch file has FRAME_STEP: The Legend of Zelda: Tears of the Kingdom 1.4.3, Breath of the Wild 1.9.0, Hyrule Warriors: Age of Calamity 1.3.0,
diff --git a/dm/tools/tests/test_nro_header.py b/dm/tools/tests/test_nro_header.py
index c5f07b46..6c7ab421 100644
--- a/dm/tools/tests/test_nro_header.py
+++ b/dm/tools/tests/test_nro_header.py
@@ -1,67 +1,59 @@
-"""The text scan must start at the text segment's offset, not at 0 (GitHub issue #14).
-
-An NRO file is the image itself preceded by a header: its text segment starts at 0x80, while an
-NSO's text starts at 0 after decompression. Scanning an NRO from 0 read the header's build ID as
-instructions and reported `44 ldr x18, #0xfffffffffff40370` on Final Fantasy X/X-2 HD Remaster.
-
-Testing the real code path end to end needs a structurally complete NRO (MOD0 block, FDE tables),
-which is a lot of fixture for the question "does the lower bound hold". These tests check the two
-things that were wrong instead:
-
-1. `nso_code.data_blocks(..., lo=N)` never returns a block that starts below N.
-2. `patch_nso.py` passes the text segment's own offset as that bound, which is 0x80 for an NRO and
- 0 for an NSO — asserted on the source, since the value comes from the file being rewritten.
+"""An NRO's header is not code (GitHub issue #14).
+
+An NRO's text segment starts at offset 0 with the file's own header in it: Hades' EOSSDK-Switch-Shipping.nro has text
+{0x0, 0xa41000} in its segment table and NRO0 at +0x10. Scanning it from 0 read the header's build ID as instructions and
+reported `44 ldr x18, #0xfffffffffff40370` on Final Fantasy X/X-2 HD Remaster. The segment table cannot give the bound
+(it says 0): the earlier TEXT_LO = segs[0][1] form kept scanning the header, and used TEXT_LO before defining it, which
+stopped every rewrite with a NameError.
+
+patch_nso.py's code_start() is 0x80 for an NRO and 0 for an NSO, and code_ranges() starts no code range before it; both
+scans (the x18 words and the rewrite sites) walk those ranges. This is seedds/nxnative#397, carried by patch_patch_nso in
+scripts/sync_upstream.py until it is upstream (its own test there is test_nro_code_start.py). The functions are taken
+from patch_nso.py itself, which runs as a script.
"""
-import sys
-import unittest
+import ast
from pathlib import Path
+import struct
+import unittest
-HERE = Path(__file__).resolve()
-sys.path.insert(0, str(HERE.parents[1]))
-
-import nso_code # noqa: E402
+TOOLS = Path(__file__).resolve().parents[1]
+SOURCE = (TOOLS / 'patch_nso.py').read_text(encoding='utf-8')
+scope = {}
+exec(compile(ast.Module(body=[n for n in ast.parse(SOURCE).body if isinstance(n, ast.FunctionDef) and n.name in ('code_start', 'code_ranges')],
+ type_ignores=[]), str(TOOLS / 'patch_nso.py'), 'exec'), scope)
+code_start, code_ranges = scope['code_start'], scope['code_ranges']
-TOOLS = HERE.parents[1]
+LDR_X18 = 0x58000000 | (0x7ff40370 >> 2 & 0x7ffff) << 5 | 18 # a literal load into x18: what the build ID's bytes read as
-class DataBlockLowerBound(unittest.TestCase):
- def test_gaps_start_at_lo(self):
- """The first gap must begin at `lo`, not at 0.
+def nro(text_size=0x1000):
+ """A synthetic NRO's first 0x80 bytes, laid out as Hades' EOS NRO is: text at offset 0 with the header inside it"""
+ h = bytearray(0x80)
+ h[0x10:0x14] = b'NRO0'
+ struct.pack_into('<II', h, 0x20, 0, text_size) # text {offset, size}
+ struct.pack_into('<I', h, 0x44, LDR_X18) # inside the build ID
+ return bytes(h)
- data_blocks() derives its gaps from the FDE table, which needs a structurally complete
- NRO to exercise. The bound itself is one line — `gaps, last = [], lo` — so it is checked
- by patching fde_ranges to report one covered range and reading the gaps back.
- """
- real = nso_code.fde_ranges
- try:
- nso_code.fde_ranges = lambda img: [(0x100, 0x200)]
- from capstone import Cs, CS_ARCH_ARM64, CS_MODE_ARM
- md = Cs(CS_ARCH_ARM64, CS_MODE_ARM)
- img = bytes(0x400)
- # text_end 0x300, lo 0x80: the gaps are [0x80,0x100) and [0x200,0x300), never below 0x80
- blocks = nso_code.data_blocks(img, 0x300, md, lo=0x80)
- self.assertTrue(all(s >= 0x80 for s, _ in blocks),
- f'a block starts below the text segment: {blocks}')
- blocks0 = nso_code.data_blocks(img, 0x300, md)
- self.assertTrue(all(s >= 0 for s, _ in blocks0), 'default lo must not break NSOs')
- finally:
- nso_code.fde_ranges = real
- def test_lo_defaults_to_zero(self):
- """An NSO keeps the old behaviour: no header, so the scan starts at 0."""
- self.assertEqual(nso_code.data_blocks.__defaults__, (0,),
- 'data_blocks must default lo to 0 so the NSO path is unchanged')
+class CodeStart(unittest.TestCase):
+ def test_an_nro_s_header_is_outside_every_code_range(self):
+ d = nro()
+ self.assertEqual(struct.unpack_from('<II', d, 0x20)[0], 0, 'the segment table says 0, so it cannot bound the scan')
+ self.assertEqual(code_start(d), 0x80)
+ ranges = code_ranges([(0x400, 0x500)], 0x1000, code_start(d))
+ self.assertEqual(ranges, [(0x80, 0x400), (0x500, 0x1000)])
+ self.assertFalse(any(s <= 0x44 < e for s, e in ranges))
+ def test_an_nso_keeps_its_ranges(self):
+ self.assertEqual(code_start(b'NSO0' + bytes(0x7c)), 0)
+ self.assertEqual(code_ranges([(0x400, 0x500)], 0x1000), [(0, 0x400), (0x500, 0x1000)])
-class PatchNsoPassesTheOffset(unittest.TestCase):
- def test_text_lo_comes_from_the_segment(self):
- src = (TOOLS / 'patch_nso.py').read_text(encoding='utf-8')
- # TEXT_LO must be the text segment's memory offset, and both the scan and CODE_RANGES must
- # use it. Hardcoding 0x80 would break NSOs; dropping it would bring the NRO bug back.
- self.assertIn('TEXT_LO = segs[0][1]', src)
- self.assertIn('data_blocks(IMG, TEXT_LO + TEXT_END, md, lo=TEXT_LO)', src)
- self.assertRegex(src, r'CODE_RANGES = list\(zip\(\[TEXT_LO\]')
+ def test_the_bounds_are_defined_before_the_scan_uses_them(self):
+ """The NameError: each name the scan reads is assigned above its first use"""
+ for name, use in (('CODE_START', 'starts = sorted('), ('CODE_RANGES', 'x18 = set().union(')):
+ self.assertLess(SOURCE.index(f'{name} = '), SOURCE.index(use), name)
+ self.assertNotIn('TEXT_LO', SOURCE)
if __name__ == '__main__':
- unittest.main(verbosity=2)
+ unittest.main()
diff --git a/scripts/sync_upstream.py b/scripts/sync_upstream.py
index 974e653a..681a8c4d 100644
--- a/scripts/sync_upstream.py
+++ b/scripts/sync_upstream.py
@@ -63,7 +63,7 @@ OURS = {'dm/host/romfs_arc.c', 'dm/host/romfs_arc.h', 'dm/tools/romfs_arc.py',
PATCHED = ('dm/host/hle_fs.c', 'dm/host/Makefile', 'dm/host/tests/Makefile', 'dm/ios/tools/build_ipa.sh',
'scripts/build_ipa_from_nsp.py', 'dm/gfx/tools/shader_split.py',
'dm/ios/Sources/main.m', 'dm/gfx/nvn/nvn_mtl.m', 'dm/host/dm.h',
- 'dm/ios/tests/test_romfs_startup.py', 'dm/ios/tests/test_profile_pack.py', 'docs/env_switches.tsv')
+ 'dm/ios/tests/test_romfs_startup.py', 'dm/ios/tests/test_profile_pack.py', 'docs/env_switches.tsv', 'dm/host/front_mac.m', 'dm/tools/patch_nso.py', 'dm/games/0100A00019DE0000/game.tsv')
def git(*a):
r = subprocess.run(['git', *a], cwd=REPO, capture_output=True, text=True)
@@ -939,38 +939,23 @@ def patch_main_m(t):
return t
def patch_patch_nso(t):
- """The NRO header skip in dm/tools/patch_nso.py (issue #14).
-
- An NRO file is the image itself preceded by a 0x80-byte header, so its text segment starts at
- 0x80; an NSO's starts at 0 after decompression. The scan and CODE_RANGES assumed 0, so on an
- NRO the header's build ID was decoded as instructions and reported as a site
- ("44 ldr x18, #0xfffffffffff40370" on Final Fantasy X/X-2 HD Remaster), and the last 0x80
- bytes of text were never scanned.
-
- This was fixed once (4b863234) and then silently reverted: patch_nso.py was not a carrier, so
- the next sync that rewrote it took upstream's version, and the fork's test kept failing from
- then on. It is a carrier now.
+ """The NRO header skip in dm/tools/patch_nso.py (issue #14), as sent upstream in seedds/nxnative#397.
+
+ An NRO's text segment starts at offset 0 with the file's own header in it (Hades' EOSSDK-Switch-Shipping.nro: text
+ {0x0, 0xa41000}), so scanning it from 0 read the header's build ID as instructions ("44 ldr x18, #0xfffffffffff40370" on
+ Final Fantasy X/X-2 HD Remaster). code_start() is 0x80 for an NRO and 0 for an NSO; code_ranges() starts no code range
+ before it. The segment table cannot give the bound (it says 0): the TEXT_LO = segs[0][1] form this patcher applied
+ before kept scanning the header, and it used TEXT_LO in the site bound 13 lines before defining it, so every rewrite
+ stopped with a NameError. When #397 is merged upstream, every hunk is found applied and this patcher can go.
"""
- if 'TEXT_LO = segs[0][1]' in t: return skip('patch_nso.py NRO header (already patched)')
+ if 'def code_start(d):' in t and 'TEXT_LO' not in t: return skip('patch_nso.py NRO header (already patched)')
P = [
- ('data blocks and code ranges',
- 'DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_END, md)\n'
- 'CODE_RANGES = list(zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END]))\n',
- 'DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_LO + TEXT_END, md, lo=TEXT_LO)\n'
- '# The text segment starts at its memory offset: 0 for an NSO (the image is rebuilt from 0) and\n'
- '# 0x80 for an NRO (the file is the image preceded by its header, so text sits at 0x80). Scanning\n'
- '# an NRO from 0 read the header\'s build ID as instructions and reported\n'
- '# "44 ldr x18, #0xfffffffffff40370" on Final Fantasy X/X-2 HD Remaster (GitHub issue #14).\n'
- '# TEXT_LO is where the code really begins in IMG; data_blocks() gets the same lower bound so its\n'
- '# returned offsets are already IMG-absolute and CODE_RANGES needs no further shift.\n'
- 'TEXT_LO = segs[0][1]\n'
- 'CODE_RANGES = list(zip([TEXT_LO] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_LO + TEXT_END]))\n'),
+ ('code_start and code_ranges', "src, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3]\nd = open(src, 'rb').read()\n",
+ 'def code_start(d):\n """Where the code in the text segment starts: 0x80 in an NRO, 0 in an NSO. An NRO\'s text segment begins at file and memory\n offset 0 with the file\'s own header in it (the start word, the MOD0 offset, NRO0 at +0x10, the segment table at +0x20, the\n build ID at +0x40), so scanning it from 0 read the header as instructions: the build ID decoded as "ldr x18, #…" and stopped\n Final Fantasy X/X-2 HD Remaster\'s rewrite (GitHub issue LNPANDY/NX2Apple#14). An NSO\'s text holds code from 0."""\n return 0x80 if d[0x10:0x14] == b\'NRO0\' else 0\n\ndef code_ranges(data_blocks, text_end, start=0):\n """The text\'s code ranges: the gaps between its data blocks (nso_code.data_blocks), none of them before `start` (code_start)"""\n gaps = zip([0] + [e for _, e in data_blocks], [s for s, _ in data_blocks] + [text_end])\n return [(max(s, start), e) for s, e in gaps if e > max(s, start)]\n\nsrc, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3]\nd = open(src, \'rb\').read()\nCODE_START = code_start(d)\n'),
('site starts bound', 'starts = sorted(x for x in starts if x < TEXT_END)',
- 'starts = sorted(x for x in starts if x < TEXT_LO + TEXT_END)'),
- ('observation gate window', ' for a in range(off, min(off + 4 * 16, TEXT_END), 4):',
- ' for a in range(off, min(off + 4 * 16, TEXT_LO + TEXT_END), 4):'),
- ('observation gate assert', ' assert off < TEXT_END and off not in kinds',
- ' assert off < TEXT_LO + TEXT_END and off not in kinds'),
+ 'starts = sorted(x for x in starts if CODE_START <= x < TEXT_END)'),
+ ('code ranges', 'CODE_RANGES = list(zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END]))',
+ 'CODE_RANGES = code_ranges(DATA_BLOCKS, TEXT_END, CODE_START)'),
]
for name, old, new in P:
if new in t:
@@ -978,7 +963,8 @@ def patch_patch_nso(t):
continue
if old not in t:
fail(f'dm/tools/patch_nso.py: patch "{name}" anchor missing (upstream reworked the text '
- 'scan; merge the NRO header skip by hand)')
+ 'scan; merge the NRO header skip by hand, or take seedds/nxnative#397 if it is merged)')
+ continue
t = t.replace(old, new, 1)
done(f'patch_nso.py {name}')
return t
@@ -1158,7 +1144,7 @@ MARKERS = {
'dm/games/0100A00019DE0000/game.tsv': 'build v1 1.0.137848',
'dm/host/dm.h': 'hle_profile.h',
'dm/host/front_mac.m': 'DM_KEYMAP',
- 'dm/tools/patch_nso.py': 'TEXT_LO',
+ 'dm/tools/patch_nso.py': 'def code_start(d):',
'dm/ios/tests/test_romfs_startup.py': 'fixture_profile',
'dm/ios/tests/test_profile_pack.py': 'slim package',
'docs/env_switches.tsv': 'DM_FULL_ROMFS\t',