Skip to content

Play your own Switch games on iPhone/iPad — one-click macOS app, no terminal. | 把 Switch 游戏装进 iPhone:图形界面一键构建,不用命令行。Bring your own game.

frontend-qt: stage space-containing paths for metal.exe and enable multi-core NSO rewrite

LNPANDY7d5729260ba7
diff --git a/frontend-qt/nspb_qt/win/shim.py b/frontend-qt/nspb_qt/win/shim.py
index 75d918c9..832914f8 100644
--- a/frontend-qt/nspb_qt/win/shim.py
+++ b/frontend-qt/nspb_qt/win/shim.py
@@ -58,11 +58,12 @@ if os.name == 'nt' and not os.environ.get('NSPB_SHIM_OFF'):
if _p and _p not in sys.path:
sys.path.insert(0, _p)
- # ---- POSIX signal & os.execve stubs for patch_nso.py
- # patch_nso.py's in_parallel() uses fork() + signal.pthread_sigmask when WORKERS >= 2;
- # rewrite() already runs one patch_nso.py process per NSO in parallel, so default
- # DM_REWRITE_J=1 on Windows and provide pthread_sigmask stubs in case it is called.
- os.environ.setdefault('DM_REWRITE_J', '1')
+ # ---- POSIX signal, os.execve & multi-core fork Pool adapter for patch_nso.py / nro_collect.py
+ # patch_nso.py's in_parallel() uses multiprocessing.get_context('fork').Pool(WORKERS) +
+ # signal.pthread_sigmask; nro_collect.py calls multiprocessing.Pool() at top level without
+ # an `if __name__ == '__main__':` guard. On Windows there is no fork(), and Capstone's C
+ # disassembler (`cs_disasm`) releases the GIL, so a thread pool with per-thread Capstone
+ # instances gives full multi-core speedup without fork() or spawn recursion.
import signal as _signal
for _sig_attr, _sig_val in (('SIG_BLOCK', 1), ('SIG_UNBLOCK', 2), ('SIG_SETMASK', 3)):
if not hasattr(_signal, _sig_attr):
@@ -76,6 +77,104 @@ if os.name == 'nt' and not os.environ.get('NSPB_SHIM_OFF'):
os.execve = _win_execve
+ import concurrent.futures as _cf, multiprocessing as _mp, threading as _mp_threading, types as _mp_types
+ _mp_tls = _mp_threading.local()
+
+ class _WinForkPool:
+ def __init__(self, processes=None, initializer=None, initargs=(), *a, **k):
+ self._workers = max(1, int(processes or os.cpu_count() or 4))
+ self._ex = _cf.ThreadPoolExecutor(max_workers=self._workers)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ self._ex.shutdown(wait=True)
+ return False
+
+ def close(self):
+ pass
+
+ def join(self):
+ self._ex.shutdown(wait=True)
+
+ def terminate(self):
+ self._ex.shutdown(wait=False, cancel_futures=True)
+
+ def map(self, fn, iterable, chunksize=1):
+ return list(self._ex.map(fn, iterable))
+
+ def imap_unordered(self, fn, iterable, chunksize=1):
+ return self._ex.map(fn, iterable)
+
+ def starmap(self, fn, jobs, chunksize=1):
+ g = getattr(fn, '__globals__', None)
+ fname = getattr(fn, '__name__', '')
+ if fname == 'x18_words' and isinstance(g, dict) and 'Cs' in g:
+ Cs, ARCH, MODE = g['Cs'], g['CS_ARCH_ARM64'], g['CS_MODE_ARM']
+ IMG, R18 = g['IMG'], g['R18']
+
+ def _x18_job(job):
+ lo, hi = job
+ md = getattr(_mp_tls, 'md_lite', None)
+ if md is None:
+ md = _mp_tls.md_lite = Cs(ARCH, MODE)
+ out, a = set(), lo
+ while a < hi:
+ end = min(hi, a + 0x4000); last = a - 4
+ for addr, _sz, _mn, op in md.disasm_lite(bytes(IMG[a:end]), a):
+ last = addr
+ if R18.search(op):
+ out.add(addr)
+ a = last + 4 if last + 4 >= end else last + 8
+ return out
+
+ return list(self._ex.map(_x18_job, jobs))
+ if fname == 'spin_heads_with_region' and isinstance(g, dict) and 'Cs' in g:
+ Cs, ARCH, MODE = g['Cs'], g['CS_ARCH_ARM64'], g['CS_MODE_ARM']
+ IMG = g['IMG']
+
+ def _spin_job(job):
+ t_fn = getattr(_mp_tls, 'spin_fn', None)
+ if t_fn is None or getattr(_mp_tls, 'spin_src', None) is not fn:
+ md_d = Cs(ARCH, MODE)
+ md_d.detail = True
+ dec = {}
+
+ def _tls_spin_ins(a, _dec=dec, _md=md_d, _img=IMG):
+ if a not in _dec:
+ _dec[a] = next(_md.disasm(bytes(_img[a:a + 4]), a), None)
+ return _dec[a]
+
+ tg = dict(g)
+ tg['md'] = md_d
+ tg['spin_dec'] = dec
+ tg['spin_ins'] = _tls_spin_ins
+ for _k in ('spin_rw', 'spin_kind', 'spin_rmw', 'spin_lock', 'spin_store',
+ 'spin_region1', 'spin_region', 'spin_heads_with_region'):
+ if _k in g and isinstance(g[_k], _mp_types.FunctionType):
+ tg[_k] = _mp_types.FunctionType(g[_k].__code__, tg, _k,
+ g[_k].__defaults__, g[_k].__closure__)
+ t_fn = _mp_tls.spin_fn = tg['spin_heads_with_region']
+ _mp_tls.spin_src = fn
+ return t_fn(*job)
+
+ return list(self._ex.map(_spin_job, jobs))
+ return list(self._ex.map(lambda j: fn(*j), jobs))
+
+ class _WinForkContext:
+ Pool = staticmethod(_WinForkPool)
+
+ _orig_get_context = _mp.get_context
+
+ def _win_get_context(method=None):
+ if method == 'fork':
+ return _WinForkContext()
+ return _orig_get_context(method)
+
+ _mp.get_context = _win_get_context
+ _mp.Pool = _WinForkPool
+
# ---- text files written the way the pipeline expects them
# The host compares lines of the pipeline's tables byte for byte; an inputs.tsv line ending
# in "\r" did not match on the device (found by the reference implementation the hard way).
@@ -337,6 +436,8 @@ if os.name == 'nt' and not os.environ.get('NSPB_SHIM_OFF'):
return None
if rest[2] == 'metal' and '-c' not in rest[3:] and '-o' in rest[3:]:
return None # compile+link in one call (host_msl.py): handled by xcrun_shim.py
+ if any(' ' in str(a) for a in rest[3:]):
+ return None # paths with spaces break metal.exe; xcrun_shim.py stages them via cwd
if 'modules' not in _CACHE:
if _PKG not in sys.path:
sys.path.insert(0, _PKG)
@@ -515,7 +616,6 @@ def build_env(base: dict[str, str] | None = None, python_exe: str | None = None)
env['NSPB_PYTHON'] = str(python_exe)
env.setdefault('DM_ALLOW_SYSTEM_SPIRV_CROSS', '1')
env.setdefault('DM_ALLOW_OTHER_METAL', '1')
- env.setdefault('DM_REWRITE_J', '1')
# tc.dotnet_dir() first: it knows about the SDK the app installed for itself, which is not on
# PATH and not in Program Files. `which` alone missed it, so `dotnet build` and the shader
# translator failed on a machine that had just installed .NET through this app.
diff --git a/frontend-qt/nspb_qt/win/xcrun_shim.py b/frontend-qt/nspb_qt/win/xcrun_shim.py
index f1588015..28358d90 100644
--- a/frontend-qt/nspb_qt/win/xcrun_shim.py
+++ b/frontend-qt/nspb_qt/win/xcrun_shim.py
@@ -24,8 +24,10 @@ works would produce a half-written pipeline archive.
"""
from __future__ import annotations
+import shutil
import subprocess
import sys
+import tempfile
from pathlib import Path
# Importable both as `nspb_qt.win.xcrun_shim` and as a plain script (the `.cmd` stand-in runs it
@@ -136,6 +138,66 @@ def _die(msg: str, code: int = 127) -> None:
raise SystemExit(code)
+def _run_metal_staged(sdk: str, tool: str, exe: Path, rest: list[str]) -> int:
+ """Run `metal.exe` or `metallib.exe` in a staging directory using space-free relative names.
+
+ Apple's Windows `metal.exe` splits input paths on spaces when forwarding them to its internal
+ frontend (e.g. `.../NS Port Builder/engine/dm/gfx/msl/builtin.metal` becomes three arguments
+ `.../NS`, `Port`, `Builder/...`). Staging inputs and outputs in `cwd` with plain filenames
+ (`in_0.metal`, `out.air`, `out.metallib`) keeps every argument passed to `metal.exe` and
+ `metallib.exe` completely free of spaces, even when the app folder, temp folder or user
+ profile path contains spaces.
+ """
+ idx = rest.index('-o')
+ out_target = Path(rest[idx + 1]).resolve()
+ compile_and_link = (tool == 'metal' and '-c' not in rest)
+ with tempfile.TemporaryDirectory() as td:
+ td_path = Path(td)
+ staged: list[str] = []
+ in_idx = 0
+ i = 0
+ while i < len(rest):
+ if i == idx:
+ i += 2 # skip '-o' and its output path; added back below with a local name
+ continue
+ a = rest[i]
+ if not a.startswith('-') and (a.endswith(('.metal', '.air', '.dia', '.pch'))
+ or (' ' in a and Path(a).is_file())):
+ src_p = Path(a)
+ local_name = f'in_{in_idx}{src_p.suffix or ".in"}'
+ in_idx += 1
+ if src_p.is_file():
+ shutil.copyfile(src_p, td_path / local_name)
+ staged.append(local_name)
+ else:
+ staged.append(a)
+ i += 1
+ try:
+ if compile_and_link and out_target.suffix == '.metallib':
+ ml_exe = find(sdk, 'metallib')
+ if not ml_exe:
+ _die(f'no Windows tool for `xcrun -sdk {sdk} metallib`')
+ r1 = subprocess.run([str(exe), *staged, '-c', '-o', 'out.air'], cwd=td)
+ if r1.returncode != 0:
+ return r1.returncode
+ r2 = subprocess.run([str(ml_exe), 'out.air', '-o', 'out.metallib'], cwd=td)
+ if r2.returncode != 0:
+ return r2.returncode
+ out_local = td_path / 'out.metallib'
+ else:
+ local_out_name = 'out' + (out_target.suffix or '.out')
+ r = subprocess.run([str(exe), *staged, '-o', local_out_name], cwd=td)
+ if r.returncode != 0:
+ return r.returncode
+ out_local = td_path / local_out_name
+ if out_local.is_file():
+ out_target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(out_local, out_target)
+ return 0
+ except OSError as e:
+ _die(f'cannot start {exe}: {e}')
+
+
def main(argv: list[str] | None = None) -> int:
sdk = 'macosx'
a = list(sys.argv[1:] if argv is None else argv)
@@ -171,27 +233,11 @@ def main(argv: list[str] | None = None) -> int:
' (see the Windows build section of frontend-qt/README.md)'))
if tool == 'metal':
rest = metal_args(sdk, rest)
- if '-c' not in rest and '-o' in rest:
- idx = rest.index('-o')
- if idx + 1 < len(rest) and rest[idx + 1].endswith('.metallib'):
- ml_out = rest[idx + 1]
- air_tmp = ml_out + '.air'
- compile_rest = rest[:idx] + ['-c', '-o', air_tmp] + rest[idx + 2:]
- ml_exe = find(sdk, 'metallib')
- if ml_exe:
- try:
- r1 = subprocess.run([str(exe)] + compile_rest)
- if r1.returncode != 0:
- return r1.returncode
- r2 = subprocess.run([str(ml_exe), air_tmp, '-o', ml_out])
- return r2.returncode
- except OSError as e:
- _die(f'cannot start {exe}: {e}')
- finally:
- try:
- Path(air_tmp).unlink(missing_ok=True)
- except OSError:
- pass
+ if '-o' in rest and ('-c' not in rest or any(' ' in x for x in rest)):
+ return _run_metal_staged(sdk, tool, exe, rest)
+ elif tool == 'metallib':
+ if '-o' in rest and any(' ' in x for x in rest):
+ return _run_metal_staged(sdk, tool, exe, rest)
elif tool == 'clang':
rest = clang_args(rest)
try:
diff --git a/frontend-qt/tests/test_win.py b/frontend-qt/tests/test_win.py
index ae9df182..3603cf76 100644
--- a/frontend-qt/tests/test_win.py
+++ b/frontend-qt/tests/test_win.py
@@ -2106,7 +2106,7 @@ class WindowsBuildFixesTest(unittest.TestCase):
self.assertEqual(env['NSPB_PYTHON'], 'C:\\venv\\python.exe')
self.assertEqual(env['DM_ALLOW_SYSTEM_SPIRV_CROSS'], '1')
self.assertEqual(env['DM_ALLOW_OTHER_METAL'], '1')
- self.assertEqual(env['DM_REWRITE_J'], '1')
+ self.assertNotIn('DM_REWRITE_J', env)
pkg_root = str(QT)
self.assertIn(pkg_root, env['PYTHONPATH'].split(os.pathsep))
@@ -2298,7 +2298,7 @@ class WindowsBuildFixesTest(unittest.TestCase):
def test_shim_patches_signal_pthread_sigmask_and_tool_lock_spirv_cross(self) -> None:
src = shim._SITECUSTOMIZE_SRC
- self.assertIn("os.environ.setdefault('DM_REWRITE_J', '1')", src)
+ self.assertIn("class _WinForkPool:", src)
self.assertIn("pthread_sigmask", src)
self.assertIn("os.execve = _win_execve", src)
@@ -2405,25 +2405,96 @@ class WindowsBuildFixesTest(unittest.TestCase):
build_app_src = (QT / 'nspb_qt/win/build_app.py').read_text(encoding='utf-8')
self.assertIn('host_msl_gen.h', build_app_src)
- def test_xcrun_shim_splits_compile_and_link_for_metal_without_c_flag(self) -> None:
+ def test_xcrun_shim_splits_compile_and_link_and_stages_paths_with_spaces(self) -> None:
from nspb_qt.win import xcrun_shim
- calls: list[list[str]] = []
-
- def fake_run(cmd, *a, **k):
- calls.append([str(c) for c in cmd])
- return subprocess.CompletedProcess(cmd, 0)
-
- with mock.patch.object(xcrun_shim, 'find', side_effect=lambda sdk, tool: Path(f'C:/metal/{tool}.exe')), \
- mock.patch.object(subprocess, 'run', side_effect=fake_run):
- rc = xcrun_shim.main(['-sdk', 'iphoneos', 'metal', '-mios-version-min=18.0',
- '-o', 'C:/tmp/clear_depth.metallib', 'C:/src/clear_depth.metal'])
- self.assertEqual(rc, 0)
- self.assertEqual(len(calls), 2)
- self.assertEqual(calls[0][0], 'C:/metal/metal.exe')
- self.assertIn('-c', calls[0])
- self.assertIn('C:/tmp/clear_depth.metallib.air', calls[0])
- self.assertEqual(calls[1], ['C:/metal/metallib.exe', 'C:/tmp/clear_depth.metallib.air',
- '-o', 'C:/tmp/clear_depth.metallib'])
+ calls: list[tuple[list[str], str | None]] = []
+
+ with tempfile.TemporaryDirectory() as td:
+ src_dir = Path(td) / 'NS Port Builder' / 'engine' / 'dm' / 'gfx' / 'msl'
+ src_dir.mkdir(parents=True)
+ src_metal = src_dir / 'builtin.metal'
+ src_metal.write_text('kernel void k() {}\n', encoding='utf-8')
+ out_dir = Path(td) / 'Out Folder'
+ out_ml = out_dir / 'builtin.metallib'
+
+ def fake_run(cmd, *a, **k):
+ cwd = k.get('cwd')
+ calls.append(([str(c) for c in cmd], cwd))
+ if cwd and 'out.metallib' in cmd:
+ (Path(cwd) / 'out.metallib').write_bytes(b'MTLB_STAGED')
+ return subprocess.CompletedProcess(cmd, 0)
+
+ with mock.patch.object(xcrun_shim, 'find', side_effect=lambda sdk, tool: Path(f'C:/metal/{tool}.exe')), \
+ mock.patch.object(subprocess, 'run', side_effect=fake_run):
+ rc = xcrun_shim.main(['-sdk', 'iphoneos', 'metal', '-mios-version-min=18.0',
+ '-o', str(out_ml), str(src_metal)])
+ self.assertEqual(rc, 0)
+ self.assertEqual(len(calls), 2)
+ # Every argument after the executable must be free of spaces so metal.exe does not split it
+ for args, cwd in calls:
+ self.assertIsNotNone(cwd)
+ for arg in args[1:]:
+ self.assertNotIn(' ', arg, f'argument passed to metal/metallib contains space: {arg!r}')
+ self.assertEqual(calls[0][0], ['C:/metal/metal.exe', '-target', 'air64-apple-ios18.0',
+ 'in_0.metal', '-c', '-o', 'out.air'])
+ self.assertEqual(calls[1][0], ['C:/metal/metallib.exe', 'out.air', '-o', 'out.metallib'])
+ self.assertEqual(out_ml.read_bytes(), b'MTLB_STAGED')
+
+ def test_win_fork_pool_rewrites_nso_byte_identical_to_single_core(self) -> None:
+ import struct
+ try:
+ import lz4.block
+ import capstone # noqa: F401
+ except ImportError:
+ self.skipTest('lz4 and capstone required')
+ patch_nso = QT.parent / 'engine/dm/tools/patch_nso.py'
+ if not patch_nso.is_file():
+ self.skipTest('engine/ is missing')
+
+ ins_words = [0x910003FD] * (300 * 1024)
+ ins_words[100] = 0xF9400252 # ldr x18, [x18] in chunk 0
+ ins_words[270000] = 0x91000652 # add x18, x18, #1 in chunk 1
+ ins_words[200] = 0xB9400109 # ldr w9, [x8]
+ ins_words[201] = 0x7100053F # cmp w9, #1
+ ins_words[202] = 0x54FFFFC0 # b.eq -8 (spin wait!)
+ ins_words[203] = 0xD65F03C0 # ret
+ ins_words[0] = 0x14000010
+ ins_words[1] = 8
+ text_bytes = bytearray(struct.pack(f'<{len(ins_words)}I', *ins_words))
+ struct.pack_into('<4siiiiii', text_bytes, 8, b'MOD0', 0, 0, 0, 0x38, 0x44, 0)
+ struct.pack_into('<BBBBII', text_bytes, 0x40, 1, 0x03, 0x03, 0x03, 0, 0)
+ ro_bytes = b'\x00' * 0x1000
+ data_bytes = b'\x00' * 0x1000
+ tc_b = lz4.block.compress(bytes(text_bytes), store_size=False)
+ rc_b = lz4.block.compress(ro_bytes, store_size=False)
+ dc_b = lz4.block.compress(data_bytes, store_size=False)
+ hdr = bytearray(0x100)
+ hdr[:4] = b'NSO0'
+ struct.pack_into('<I', hdr, 0x0C, 7)
+ struct.pack_into('<III', hdr, 0x10, 0x100, 0, len(text_bytes))
+ struct.pack_into('<III', hdr, 0x20, 0x100 + len(tc_b), len(text_bytes), len(ro_bytes))
+ struct.pack_into('<III', hdr, 0x30, 0x100 + len(tc_b) + len(rc_b), len(text_bytes) + len(ro_bytes), len(data_bytes))
+ hdr[0x40:0x50] = bytes.fromhex('0123456789abcdef0123456789abcdef')
+ struct.pack_into('<III', hdr, 0x60, len(tc_b), len(rc_b), len(dc_b))
+
+ with tempfile.TemporaryDirectory() as td:
+ nso_p = Path(td) / 'main'
+ nso_p.write_bytes(bytes(hdr) + tc_b + rc_b + dc_b)
+ out1, out2 = Path(td) / 'out1', Path(td) / 'out2'
+ r1 = subprocess.run([sys.executable, str(patch_nso), str(nso_p), str(out1), 'main'],
+ env={**os.environ, 'DM_REWRITE_J': '1'}, capture_output=True, text=True)
+ self.assertEqual(r1.returncode, 0, r1.stderr)
+ sd = Path(td) / 'shim'
+ sd.mkdir()
+ sc = shim._SITECUSTOMIZE_SRC.replace("if os.name == 'nt'", "if True")
+ (sd / 'sitecustomize.py').write_text(sc, encoding='utf-8')
+ r2 = subprocess.run([sys.executable, str(patch_nso), str(nso_p), str(out2), 'main'],
+ env={**os.environ, 'DM_REWRITE_J': '4', 'PYTHONPATH': str(sd)},
+ capture_output=True, text=True)
+ self.assertEqual(r2.returncode, 0, r2.stderr)
+ for ext in ('patched.bin', 'fixups.tsv', 'gates.tsv', 'sites.tsv'):
+ self.assertEqual((out1 / f'main.{ext}').read_bytes(),
+ (out2 / f'main.{ext}').read_bytes())
if __name__ == '__main__':