Play your own Switch games on iPhone/iPad — one-click macOS app, no terminal. | 把 Switch 游戏装进 iPhone:图形界面一键构建,不用命令行。Bring your own game.
Swift apple-siliconcustom-formathomebrewiosipaipadipadosiphonemetalnative-portnintendo-switchnxnativeromfsswitch-games
frontend-qt: stage space-containing paths for metal.exe and enable multi-core NSO rewrite
LNPANDY7d5729260ba7
diff --git a/frontend-qt/nspb_qt/win/shim.py b/frontend-qt/nspb_qt/win/shim.pyindex 75d918c9..832914f8 100644--- a/frontend-qt/nspb_qt/win/shim.py+++ b/frontend-qt/nspb_qt/win/shim.py@@ -58,11 +58,12 @@ if os.name == 'nt' and not os.environ.get('NSPB_SHIM_OFF'):if _p and _p not in sys.path:sys.path.insert(0, _p)- # ---- POSIX signal & os.execve stubs for patch_nso.py- # patch_nso.py's in_parallel() uses fork() + signal.pthread_sigmask when WORKERS >= 2;- # rewrite() already runs one patch_nso.py process per NSO in parallel, so default- # DM_REWRITE_J=1 on Windows and provide pthread_sigmask stubs in case it is called.- os.environ.setdefault('DM_REWRITE_J', '1')+ # ---- POSIX signal, os.execve & multi-core fork Pool adapter for patch_nso.py / nro_collect.py+ # patch_nso.py's in_parallel() uses multiprocessing.get_context('fork').Pool(WORKERS) ++ # signal.pthread_sigmask; nro_collect.py calls multiprocessing.Pool() at top level without+ # an `if __name__ == '__main__':` guard. On Windows there is no fork(), and Capstone's C+ # disassembler (`cs_disasm`) releases the GIL, so a thread pool with per-thread Capstone+ # instances gives full multi-core speedup without fork() or spawn recursion.import signal as _signalfor _sig_attr, _sig_val in (('SIG_BLOCK', 1), ('SIG_UNBLOCK', 2), ('SIG_SETMASK', 3)):if not hasattr(_signal, _sig_attr):@@ -76,6 +77,104 @@ if os.name == 'nt' and not os.environ.get('NSPB_SHIM_OFF'):os.execve = _win_execve+ import concurrent.futures as _cf, multiprocessing as _mp, threading as _mp_threading, types as _mp_types+ _mp_tls = _mp_threading.local()++ class _WinForkPool:+ def __init__(self, processes=None, initializer=None, initargs=(), *a, **k):+ self._workers = max(1, int(processes or os.cpu_count() or 4))+ self._ex = _cf.ThreadPoolExecutor(max_workers=self._workers)++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ self._ex.shutdown(wait=True)+ return False++ def close(self):+ pass++ def join(self):+ self._ex.shutdown(wait=True)++ def terminate(self):+ self._ex.shutdown(wait=False, cancel_futures=True)++ def map(self, fn, iterable, chunksize=1):+ return list(self._ex.map(fn, iterable))++ def imap_unordered(self, fn, iterable, chunksize=1):+ return self._ex.map(fn, iterable)++ def starmap(self, fn, jobs, chunksize=1):+ g = getattr(fn, '__globals__', None)+ fname = getattr(fn, '__name__', '')+ if fname == 'x18_words' and isinstance(g, dict) and 'Cs' in g:+ Cs, ARCH, MODE = g['Cs'], g['CS_ARCH_ARM64'], g['CS_MODE_ARM']+ IMG, R18 = g['IMG'], g['R18']++ def _x18_job(job):+ lo, hi = job+ md = getattr(_mp_tls, 'md_lite', None)+ if md is None:+ md = _mp_tls.md_lite = Cs(ARCH, MODE)+ out, a = set(), lo+ while a < hi:+ end = min(hi, a + 0x4000); last = a - 4+ for addr, _sz, _mn, op in md.disasm_lite(bytes(IMG[a:end]), a):+ last = addr+ if R18.search(op):+ out.add(addr)+ a = last + 4 if last + 4 >= end else last + 8+ return out++ return list(self._ex.map(_x18_job, jobs))+ if fname == 'spin_heads_with_region' and isinstance(g, dict) and 'Cs' in g:+ Cs, ARCH, MODE = g['Cs'], g['CS_ARCH_ARM64'], g['CS_MODE_ARM']+ IMG = g['IMG']++ def _spin_job(job):+ t_fn = getattr(_mp_tls, 'spin_fn', None)+ if t_fn is None or getattr(_mp_tls, 'spin_src', None) is not fn:+ md_d = Cs(ARCH, MODE)+ md_d.detail = True+ dec = {}++ def _tls_spin_ins(a, _dec=dec, _md=md_d, _img=IMG):+ if a not in _dec:+ _dec[a] = next(_md.disasm(bytes(_img[a:a + 4]), a), None)+ return _dec[a]++ tg = dict(g)+ tg['md'] = md_d+ tg['spin_dec'] = dec+ tg['spin_ins'] = _tls_spin_ins+ for _k in ('spin_rw', 'spin_kind', 'spin_rmw', 'spin_lock', 'spin_store',+ 'spin_region1', 'spin_region', 'spin_heads_with_region'):+ if _k in g and isinstance(g[_k], _mp_types.FunctionType):+ tg[_k] = _mp_types.FunctionType(g[_k].__code__, tg, _k,+ g[_k].__defaults__, g[_k].__closure__)+ t_fn = _mp_tls.spin_fn = tg['spin_heads_with_region']+ _mp_tls.spin_src = fn+ return t_fn(*job)++ return list(self._ex.map(_spin_job, jobs))+ return list(self._ex.map(lambda j: fn(*j), jobs))++ class _WinForkContext:+ Pool = staticmethod(_WinForkPool)++ _orig_get_context = _mp.get_context++ def _win_get_context(method=None):+ if method == 'fork':+ return _WinForkContext()+ return _orig_get_context(method)++ _mp.get_context = _win_get_context+ _mp.Pool = _WinForkPool+# ---- text files written the way the pipeline expects them# The host compares lines of the pipeline's tables byte for byte; an inputs.tsv line ending# in "\r" did not match on the device (found by the reference implementation the hard way).@@ -337,6 +436,8 @@ if os.name == 'nt' and not os.environ.get('NSPB_SHIM_OFF'):return Noneif rest[2] == 'metal' and '-c' not in rest[3:] and '-o' in rest[3:]:return None # compile+link in one call (host_msl.py): handled by xcrun_shim.py+ if any(' ' in str(a) for a in rest[3:]):+ return None # paths with spaces break metal.exe; xcrun_shim.py stages them via cwdif 'modules' not in _CACHE:if _PKG not in sys.path:sys.path.insert(0, _PKG)@@ -515,7 +616,6 @@ def build_env(base: dict[str, str] | None = None, python_exe: str | None = None)env['NSPB_PYTHON'] = str(python_exe)env.setdefault('DM_ALLOW_SYSTEM_SPIRV_CROSS', '1')env.setdefault('DM_ALLOW_OTHER_METAL', '1')- env.setdefault('DM_REWRITE_J', '1')# tc.dotnet_dir() first: it knows about the SDK the app installed for itself, which is not on# PATH and not in Program Files. `which` alone missed it, so `dotnet build` and the shader# translator failed on a machine that had just installed .NET through this app.diff --git a/frontend-qt/nspb_qt/win/xcrun_shim.py b/frontend-qt/nspb_qt/win/xcrun_shim.pyindex f1588015..28358d90 100644--- a/frontend-qt/nspb_qt/win/xcrun_shim.py+++ b/frontend-qt/nspb_qt/win/xcrun_shim.py@@ -24,8 +24,10 @@ works would produce a half-written pipeline archive."""from __future__ import annotations+import shutilimport subprocessimport sys+import tempfilefrom pathlib import Path# Importable both as `nspb_qt.win.xcrun_shim` and as a plain script (the `.cmd` stand-in runs it@@ -136,6 +138,66 @@ def _die(msg: str, code: int = 127) -> None:raise SystemExit(code)+def _run_metal_staged(sdk: str, tool: str, exe: Path, rest: list[str]) -> int:+ """Run `metal.exe` or `metallib.exe` in a staging directory using space-free relative names.++ Apple's Windows `metal.exe` splits input paths on spaces when forwarding them to its internal+ frontend (e.g. `.../NS Port Builder/engine/dm/gfx/msl/builtin.metal` becomes three arguments+ `.../NS`, `Port`, `Builder/...`). Staging inputs and outputs in `cwd` with plain filenames+ (`in_0.metal`, `out.air`, `out.metallib`) keeps every argument passed to `metal.exe` and+ `metallib.exe` completely free of spaces, even when the app folder, temp folder or user+ profile path contains spaces.+ """+ idx = rest.index('-o')+ out_target = Path(rest[idx + 1]).resolve()+ compile_and_link = (tool == 'metal' and '-c' not in rest)+ with tempfile.TemporaryDirectory() as td:+ td_path = Path(td)+ staged: list[str] = []+ in_idx = 0+ i = 0+ while i < len(rest):+ if i == idx:+ i += 2 # skip '-o' and its output path; added back below with a local name+ continue+ a = rest[i]+ if not a.startswith('-') and (a.endswith(('.metal', '.air', '.dia', '.pch'))+ or (' ' in a and Path(a).is_file())):+ src_p = Path(a)+ local_name = f'in_{in_idx}{src_p.suffix or ".in"}'+ in_idx += 1+ if src_p.is_file():+ shutil.copyfile(src_p, td_path / local_name)+ staged.append(local_name)+ else:+ staged.append(a)+ i += 1+ try:+ if compile_and_link and out_target.suffix == '.metallib':+ ml_exe = find(sdk, 'metallib')+ if not ml_exe:+ _die(f'no Windows tool for `xcrun -sdk {sdk} metallib`')+ r1 = subprocess.run([str(exe), *staged, '-c', '-o', 'out.air'], cwd=td)+ if r1.returncode != 0:+ return r1.returncode+ r2 = subprocess.run([str(ml_exe), 'out.air', '-o', 'out.metallib'], cwd=td)+ if r2.returncode != 0:+ return r2.returncode+ out_local = td_path / 'out.metallib'+ else:+ local_out_name = 'out' + (out_target.suffix or '.out')+ r = subprocess.run([str(exe), *staged, '-o', local_out_name], cwd=td)+ if r.returncode != 0:+ return r.returncode+ out_local = td_path / local_out_name+ if out_local.is_file():+ out_target.parent.mkdir(parents=True, exist_ok=True)+ shutil.copyfile(out_local, out_target)+ return 0+ except OSError as e:+ _die(f'cannot start {exe}: {e}')++def main(argv: list[str] | None = None) -> int:sdk = 'macosx'a = list(sys.argv[1:] if argv is None else argv)@@ -171,27 +233,11 @@ def main(argv: list[str] | None = None) -> int:' (see the Windows build section of frontend-qt/README.md)'))if tool == 'metal':rest = metal_args(sdk, rest)- if '-c' not in rest and '-o' in rest:- idx = rest.index('-o')- if idx + 1 < len(rest) and rest[idx + 1].endswith('.metallib'):- ml_out = rest[idx + 1]- air_tmp = ml_out + '.air'- compile_rest = rest[:idx] + ['-c', '-o', air_tmp] + rest[idx + 2:]- ml_exe = find(sdk, 'metallib')- if ml_exe:- try:- r1 = subprocess.run([str(exe)] + compile_rest)- if r1.returncode != 0:- return r1.returncode- r2 = subprocess.run([str(ml_exe), air_tmp, '-o', ml_out])- return r2.returncode- except OSError as e:- _die(f'cannot start {exe}: {e}')- finally:- try:- Path(air_tmp).unlink(missing_ok=True)- except OSError:- pass+ if '-o' in rest and ('-c' not in rest or any(' ' in x for x in rest)):+ return _run_metal_staged(sdk, tool, exe, rest)+ elif tool == 'metallib':+ if '-o' in rest and any(' ' in x for x in rest):+ return _run_metal_staged(sdk, tool, exe, rest)elif tool == 'clang':rest = clang_args(rest)try:diff --git a/frontend-qt/tests/test_win.py b/frontend-qt/tests/test_win.pyindex ae9df182..3603cf76 100644--- a/frontend-qt/tests/test_win.py+++ b/frontend-qt/tests/test_win.py@@ -2106,7 +2106,7 @@ class WindowsBuildFixesTest(unittest.TestCase):self.assertEqual(env['NSPB_PYTHON'], 'C:\\venv\\python.exe')self.assertEqual(env['DM_ALLOW_SYSTEM_SPIRV_CROSS'], '1')self.assertEqual(env['DM_ALLOW_OTHER_METAL'], '1')- self.assertEqual(env['DM_REWRITE_J'], '1')+ self.assertNotIn('DM_REWRITE_J', env)pkg_root = str(QT)self.assertIn(pkg_root, env['PYTHONPATH'].split(os.pathsep))@@ -2298,7 +2298,7 @@ class WindowsBuildFixesTest(unittest.TestCase):def test_shim_patches_signal_pthread_sigmask_and_tool_lock_spirv_cross(self) -> None:src = shim._SITECUSTOMIZE_SRC- self.assertIn("os.environ.setdefault('DM_REWRITE_J', '1')", src)+ self.assertIn("class _WinForkPool:", src)self.assertIn("pthread_sigmask", src)self.assertIn("os.execve = _win_execve", src)@@ -2405,25 +2405,96 @@ class WindowsBuildFixesTest(unittest.TestCase):build_app_src = (QT / 'nspb_qt/win/build_app.py').read_text(encoding='utf-8')self.assertIn('host_msl_gen.h', build_app_src)- def test_xcrun_shim_splits_compile_and_link_for_metal_without_c_flag(self) -> None:+ def test_xcrun_shim_splits_compile_and_link_and_stages_paths_with_spaces(self) -> None:from nspb_qt.win import xcrun_shim- calls: list[list[str]] = []-- def fake_run(cmd, *a, **k):- calls.append([str(c) for c in cmd])- return subprocess.CompletedProcess(cmd, 0)-- with mock.patch.object(xcrun_shim, 'find', side_effect=lambda sdk, tool: Path(f'C:/metal/{tool}.exe')), \- mock.patch.object(subprocess, 'run', side_effect=fake_run):- rc = xcrun_shim.main(['-sdk', 'iphoneos', 'metal', '-mios-version-min=18.0',- '-o', 'C:/tmp/clear_depth.metallib', 'C:/src/clear_depth.metal'])- self.assertEqual(rc, 0)- self.assertEqual(len(calls), 2)- self.assertEqual(calls[0][0], 'C:/metal/metal.exe')- self.assertIn('-c', calls[0])- self.assertIn('C:/tmp/clear_depth.metallib.air', calls[0])- self.assertEqual(calls[1], ['C:/metal/metallib.exe', 'C:/tmp/clear_depth.metallib.air',- '-o', 'C:/tmp/clear_depth.metallib'])+ calls: list[tuple[list[str], str | None]] = []++ with tempfile.TemporaryDirectory() as td:+ src_dir = Path(td) / 'NS Port Builder' / 'engine' / 'dm' / 'gfx' / 'msl'+ src_dir.mkdir(parents=True)+ src_metal = src_dir / 'builtin.metal'+ src_metal.write_text('kernel void k() {}\n', encoding='utf-8')+ out_dir = Path(td) / 'Out Folder'+ out_ml = out_dir / 'builtin.metallib'++ def fake_run(cmd, *a, **k):+ cwd = k.get('cwd')+ calls.append(([str(c) for c in cmd], cwd))+ if cwd and 'out.metallib' in cmd:+ (Path(cwd) / 'out.metallib').write_bytes(b'MTLB_STAGED')+ return subprocess.CompletedProcess(cmd, 0)++ with mock.patch.object(xcrun_shim, 'find', side_effect=lambda sdk, tool: Path(f'C:/metal/{tool}.exe')), \+ mock.patch.object(subprocess, 'run', side_effect=fake_run):+ rc = xcrun_shim.main(['-sdk', 'iphoneos', 'metal', '-mios-version-min=18.0',+ '-o', str(out_ml), str(src_metal)])+ self.assertEqual(rc, 0)+ self.assertEqual(len(calls), 2)+ # Every argument after the executable must be free of spaces so metal.exe does not split it+ for args, cwd in calls:+ self.assertIsNotNone(cwd)+ for arg in args[1:]:+ self.assertNotIn(' ', arg, f'argument passed to metal/metallib contains space: {arg!r}')+ self.assertEqual(calls[0][0], ['C:/metal/metal.exe', '-target', 'air64-apple-ios18.0',+ 'in_0.metal', '-c', '-o', 'out.air'])+ self.assertEqual(calls[1][0], ['C:/metal/metallib.exe', 'out.air', '-o', 'out.metallib'])+ self.assertEqual(out_ml.read_bytes(), b'MTLB_STAGED')++ def test_win_fork_pool_rewrites_nso_byte_identical_to_single_core(self) -> None:+ import struct+ try:+ import lz4.block+ import capstone # noqa: F401+ except ImportError:+ self.skipTest('lz4 and capstone required')+ patch_nso = QT.parent / 'engine/dm/tools/patch_nso.py'+ if not patch_nso.is_file():+ self.skipTest('engine/ is missing')++ ins_words = [0x910003FD] * (300 * 1024)+ ins_words[100] = 0xF9400252 # ldr x18, [x18] in chunk 0+ ins_words[270000] = 0x91000652 # add x18, x18, #1 in chunk 1+ ins_words[200] = 0xB9400109 # ldr w9, [x8]+ ins_words[201] = 0x7100053F # cmp w9, #1+ ins_words[202] = 0x54FFFFC0 # b.eq -8 (spin wait!)+ ins_words[203] = 0xD65F03C0 # ret+ ins_words[0] = 0x14000010+ ins_words[1] = 8+ text_bytes = bytearray(struct.pack(f'<{len(ins_words)}I', *ins_words))+ struct.pack_into('<4siiiiii', text_bytes, 8, b'MOD0', 0, 0, 0, 0x38, 0x44, 0)+ struct.pack_into('<BBBBII', text_bytes, 0x40, 1, 0x03, 0x03, 0x03, 0, 0)+ ro_bytes = b'\x00' * 0x1000+ data_bytes = b'\x00' * 0x1000+ tc_b = lz4.block.compress(bytes(text_bytes), store_size=False)+ rc_b = lz4.block.compress(ro_bytes, store_size=False)+ dc_b = lz4.block.compress(data_bytes, store_size=False)+ hdr = bytearray(0x100)+ hdr[:4] = b'NSO0'+ struct.pack_into('<I', hdr, 0x0C, 7)+ struct.pack_into('<III', hdr, 0x10, 0x100, 0, len(text_bytes))+ struct.pack_into('<III', hdr, 0x20, 0x100 + len(tc_b), len(text_bytes), len(ro_bytes))+ struct.pack_into('<III', hdr, 0x30, 0x100 + len(tc_b) + len(rc_b), len(text_bytes) + len(ro_bytes), len(data_bytes))+ hdr[0x40:0x50] = bytes.fromhex('0123456789abcdef0123456789abcdef')+ struct.pack_into('<III', hdr, 0x60, len(tc_b), len(rc_b), len(dc_b))++ with tempfile.TemporaryDirectory() as td:+ nso_p = Path(td) / 'main'+ nso_p.write_bytes(bytes(hdr) + tc_b + rc_b + dc_b)+ out1, out2 = Path(td) / 'out1', Path(td) / 'out2'+ r1 = subprocess.run([sys.executable, str(patch_nso), str(nso_p), str(out1), 'main'],+ env={**os.environ, 'DM_REWRITE_J': '1'}, capture_output=True, text=True)+ self.assertEqual(r1.returncode, 0, r1.stderr)+ sd = Path(td) / 'shim'+ sd.mkdir()+ sc = shim._SITECUSTOMIZE_SRC.replace("if os.name == 'nt'", "if True")+ (sd / 'sitecustomize.py').write_text(sc, encoding='utf-8')+ r2 = subprocess.run([sys.executable, str(patch_nso), str(nso_p), str(out2), 'main'],+ env={**os.environ, 'DM_REWRITE_J': '4', 'PYTHONPATH': str(sd)},+ capture_output=True, text=True)+ self.assertEqual(r2.returncode, 0, r2.stderr)+ for ext in ('patched.bin', 'fixups.tsv', 'gates.tsv', 'sites.tsv'):+ self.assertEqual((out1 / f'main.{ext}').read_bytes(),+ (out2 / f'main.{ext}').read_bytes())if __name__ == '__main__':