Skip to content

Play your own Switch games on iPhone/iPad — one-click macOS app, no terminal. | 把 Switch 游戏装进 iPhone:图形界面一键构建,不用命令行。Bring your own game.

Launchers: build from engine/ when it is synced, sync it when it is stale

v0lumehi3d35e182fc3f
diff --git a/frontend-qt/nspb_qt/engine.py b/frontend-qt/nspb_qt/engine.py
new file mode 100644
index 00000000..e956ef41
--- /dev/null
+++ b/frontend-qt/nspb_qt/engine.py
@@ -0,0 +1,105 @@
+"""Where the engine of a checkout is, and whether it needs `tools/engine.py sync` (LNPANDY/NX2Apple#214).
+
+Since #214 the engine is built into `<checkout>/engine/` (seedds/nxnative at `.upstream-ref` plus `patches/`). Until the
+committed copy goes (step 2), a checkout has both, and one from before the switch has only the committed copy. The rule, the
+same as the Mac launcher's (frontend/Sources/NSPortBuilder/Engine.swift):
+
+- engine/ when it is synced and current,
+- else the committed tree (dm/, scripts/ at the checkout's root) when the checkout has one,
+- else engine/ as it is (stale or missing: the caller syncs first).
+
+`stamp_key` is tools/engine.py's: a digest of the pin and the series, compared with the stamp sync wrote into engine/.git, so
+the launcher can tell a stale engine without starting Python. Pure stdlib, no Qt.
+"""
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+MARKER = ('dm', 'tools', 'nx_extract', 'Program.cs') # the file that tells an engine tree (as BuildModel.is_repo always did)
+STAMP = Path('.git', 'nx2apple-stamp')
+
+
+def _series(root: Path) -> list[str]:
+ out = []
+ for line in (root / 'patches' / 'series').read_text(encoding='utf-8').splitlines():
+ line = line.split('#', 1)[0].strip()
+ if line:
+ out.append(line)
+ return out
+
+
+def stamp_key(root: str | Path) -> str:
+ root = Path(root)
+ h = hashlib.sha256()
+ h.update(b'pin ' + (root / '.upstream-ref').read_text(encoding='utf-8').strip().encode() + b'\n')
+ for name in _series(root):
+ data = (root / 'patches' / name).read_bytes()
+ h.update(b'patch ' + name.encode() + b'\n' + hashlib.sha256(data).hexdigest().encode() + b'\n')
+ return h.hexdigest()
+
+
+def has_engine_tool(repo: str) -> bool:
+ """A checkout with tools/engine.py: it can make its engine/ (the layout of #214)."""
+ return bool(repo) and Path(repo, 'tools', 'engine.py').is_file() and Path(repo, '.upstream-ref').is_file()
+
+
+def _head(eng: Path) -> str:
+ """HEAD of engine/ read from its files (no git process): the detached id, or the id its branch points to."""
+ git = eng / '.git'
+ try:
+ head = (git / 'HEAD').read_text(encoding='utf-8').strip()
+ except OSError:
+ return ''
+ if not head.startswith('ref: '):
+ return head
+ ref = head[5:]
+ try:
+ return (git / ref).read_text(encoding='utf-8').strip()
+ except OSError:
+ pass
+ try:
+ for line in (git / 'packed-refs').read_text(encoding='utf-8').splitlines():
+ if line.endswith(' ' + ref):
+ return line.split(' ', 1)[0]
+ except OSError:
+ pass
+ return ''
+
+
+def is_stale(repo: str) -> bool:
+ """True when the checkout can make engine/ and engine/ is missing or not built from the current pin and series."""
+ if not has_engine_tool(repo):
+ return False
+ eng = Path(repo, 'engine')
+ try:
+ stamp = dict(l.split(' ', 1) for l in (eng / STAMP).read_text(encoding='utf-8').splitlines() if ' ' in l)
+ return stamp.get('key') != stamp_key(repo) or stamp.get('head') != _head(eng)
+ except (OSError, ValueError):
+ return True
+
+
+def engine_root(repo: str) -> str:
+ """The folder holding dm/ and scripts/ for this checkout (see the module's rule)."""
+ if not repo:
+ return repo
+ eng = Path(repo, 'engine')
+ if eng.joinpath(*MARKER).is_file() and not is_stale(repo):
+ return str(eng)
+ if Path(repo, *MARKER).is_file():
+ return repo
+ return str(eng) if has_engine_tool(repo) else repo
+
+
+def is_checkout(d: str) -> bool:
+ """A checkout the launcher can build from: the committed tree, a synced engine/, or tools/engine.py to make one."""
+ return bool(d) and (Path(d, *MARKER).is_file() or Path(d, 'engine', *MARKER).is_file() or has_engine_tool(d))
+
+
+def needs_sync_before_build(repo: str) -> bool:
+ """engine/ is the only engine and it is stale or missing (after step 2, or a checkout whose committed tree is gone)."""
+ return has_engine_tool(repo) and is_stale(repo) and not Path(repo, *MARKER).is_file()
+
+
+def sync_command(repo: str, python: str) -> list[str]:
+ return [python, str(Path(repo, 'tools', 'engine.py')), 'sync']
diff --git a/frontend-qt/nspb_qt/model.py b/frontend-qt/nspb_qt/model.py
index 54e4eabe..d8b7cfbc 100644
--- a/frontend-qt/nspb_qt/model.py
+++ b/frontend-qt/nspb_qt/model.py
@@ -16,6 +16,7 @@ from pathlib import Path
from PySide6.QtCore import QSettings, Q_ARG, QMetaObject, QObject, Qt, QThread, Signal, Slot
from . import build_failure as bf
+from . import engine
from . import input_staging
from . import inputs
from . import platform_util as pu
@@ -153,7 +154,40 @@ class Model(QObject):
@staticmethod
def is_repo(d: str) -> bool:
- return bool(d) and Path(d, 'dm', 'tools', 'nx_extract', 'Program.cs').is_file()
+ return engine.is_checkout(d) # the committed tree, a synced engine/, or tools/engine.py to make one (#214)
+
+ @property
+ def engine_path(self) -> str:
+ """The folder holding dm/ and scripts/: engine/ when synced, else the committed tree (engine.py's rule)."""
+ return engine.engine_root(self.repo_path)
+
+ def sync_engine(self, then: Callable[[bool], None] | None = None) -> None:
+ """Run `tools/engine.py sync` on a thread when engine/ is stale (after a pull, at the start, before a build that
+ needs it). A no-op for a checkout from before #214 or a current engine/. `then(ok)` runs after it."""
+ if not engine.is_stale(self.repo_path) or getattr(self, '_engine_syncing', False):
+ if then: then(True)
+ return
+ py = self.python_path or ('python' if pu.IS_WIN else '/usr/bin/python3')
+ cmd = engine.sync_command(self.repo_path, py)
+ self._engine_syncing = True
+ self.log('$ ' + ' '.join(cmd) + '\n')
+
+ def work() -> None:
+ ok = False
+ try:
+ p = subprocess.Popen(cmd, cwd=self.repo_path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
+ errors='replace', env=pu.child_env(), **pu.no_window_kwargs())
+ assert p.stdout is not None
+ for line in p.stdout:
+ self.log(line)
+ ok = p.wait() == 0
+ except OSError as e:
+ self.log(f'{e}\n')
+ self._engine_syncing = False
+ self.repoChanged.emit()
+ if then: then(ok)
+
+ threading.Thread(target=work, daemon=True).start()
def _find_repo(self) -> str:
saved = os.environ.get('NSPB_REPO', '')
@@ -183,7 +217,9 @@ class Model(QObject):
def supports_work_dir(self) -> bool:
"""Whether the checkout's build script declares --work-dir (an upstream checkout or an
old ZIP does not; passing it there stops at argparse — issue #5)."""
- f = Path(self.repo_path, 'scripts', 'build_ipa_from_nsp.py')
+ f = Path(self.engine_path, 'scripts', 'build_ipa_from_nsp.py')
+ if not f.is_file() and engine.has_engine_tool(self.repo_path):
+ return True # engine/ is not made yet: the fork's engine always has --work-dir (patch 0007)
try:
text = f.read_text(encoding='utf-8', errors='replace')
except OSError:
@@ -194,6 +230,7 @@ class Model(QObject):
def check_deps(self) -> None:
self.deps_checked = False
+ self.sync_engine() # at the start: a checkout whose engine/ is missing or older than its pin (#214)
threading.Thread(target=self._check_deps_worker, daemon=True).start()
def _check_deps_worker(self) -> None:
@@ -527,6 +564,7 @@ class Model(QObject):
self.repo_busy = False
if ok and self.is_repo(target):
self.repo_path = target
+ self.sync_engine()
elif not ok:
shutil.rmtree(target, ignore_errors=True) # a failed clone leaves a stub
self.repoChanged.emit()
@@ -617,6 +655,7 @@ class Model(QObject):
if stream(['pull', '--ff-only', 'origin', 'main'], 'pull'):
self.repo_behind = 0
self.repo_status = self.t('repo.pulled')
+ self.sync_engine() # the pull may have moved the pin or the patches
else:
self.repo_status = self.t('repo.pullFail')
self.log(self.repo_status + '\n')
@@ -949,6 +988,13 @@ class Model(QObject):
threading.Thread(target=run_all, daemon=True).start()
+ @Slot(bool)
+ def _build_after_sync(self, ok: bool) -> None:
+ if ok and not engine.needs_sync_before_build(self.repo_path):
+ self.start_build()
+ else:
+ self.finished.emit(False, self.t('msg.repoInvalidEmpty'))
+
@Slot(str)
def _emit_log(self, s: str) -> None:
"""Main-thread half of log(): see log() for why the hop exists."""
@@ -1014,6 +1060,10 @@ class Model(QObject):
if not self.repo_ok:
self.finished.emit(False, self.t('msg.repoInvalidEmpty'))
return
+ if engine.needs_sync_before_build(self.repo_path):
+ # engine/ is the only engine and is stale or missing: make it, then build (a failed sync names its fix in the log)
+ self.sync_engine(lambda ok: QMetaObject.invokeMethod(self, '_build_after_sync', Qt.QueuedConnection, Q_ARG(bool, ok)))
+ return
py = self.python_path or ('python' if pu.IS_WIN else '/usr/bin/python3')
base_dir = str(Path(self.base_path).parent)
all_files = [self.base_path, self.update_path] + self.dlc_paths
@@ -1048,8 +1098,8 @@ class Model(QObject):
entry = ['-m', 'nspb_qt.win.driver']
cwd = str(Path(__file__).resolve().parents[1])
else:
- entry = [str(Path(self.repo_path, 'scripts', 'build_ipa_from_nsp.py'))]
- cwd = self.repo_path
+ entry = [str(Path(self.engine_path, 'scripts', 'build_ipa_from_nsp.py'))]
+ cwd = self.engine_path
args = entry + [input_dir, '-k', self.keys_path, '--progress', 'lines']
has_work_dir = self.supports_work_dir()
if has_work_dir:
@@ -1102,7 +1152,7 @@ class Model(QObject):
self.log(f'[shim] {why}\n')
except Exception as e: # a broken shim must be visible, not fatal
self.log(f'[shim] could not prepare the Windows shim: {e}\n')
- env['NSPB_REPO'] = self.repo_path
+ env['NSPB_REPO'] = self.engine_path
def work() -> None:
ok = False
@@ -1252,9 +1302,9 @@ class Model(QObject):
self.stagesChanged.emit()
def work_() -> None:
- args = [self.python_path, str(Path(self.repo_path, 'dm', 'tools', 'romfs_image.py')),
+ args = [self.python_path, str(Path(self.engine_path, 'dm', 'tools', 'romfs_image.py')),
'--reuse', romfs, str(Path(dest, 'romfs.img'))]
- env = pu.child_env({'PYTHONPATH': str(Path(self.repo_path, 'dm', 'tools'))})
+ env = pu.child_env({'PYTHONPATH': str(Path(self.engine_path, 'dm', 'tools'))})
ok = True
try:
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
diff --git a/frontend-qt/nspb_qt/win/build_app.py b/frontend-qt/nspb_qt/win/build_app.py
index 6dbacdd5..552ae40e 100644
--- a/frontend-qt/nspb_qt/win/build_app.py
+++ b/frontend-qt/nspb_qt/win/build_app.py
@@ -45,6 +45,12 @@ else: # pragma: no cover - script mode
HERE = Path(__file__).resolve().parent
+
+def _engine_root() -> str:
+ """The engine's folder when NSPB_REPO is unset: engine/ when synced, else the committed tree (LNPANDY/NX2Apple#214)."""
+ from nspb_qt import engine
+ return engine.engine_root(str(Path(__file__).resolve().parents[3]))
+
#: Frameworks the app links. `Cocoa` is a macOS framework and is dropped; the rest is
#: `dm/host/Makefile`'s link line minus Cocoa, plus what the UIKit side and the launcher use
#: (Xcode would add UIKit through Swift module autolinking).
@@ -443,7 +449,7 @@ def main(argv: list[str] | None = None) -> int:
die('--prepare-xcode generates an Xcode project, which needs a Mac with Xcode and '
'xcodegen; on Windows the IPA is packed directly (drop the option)')
env = os.environ
- repo = Path(env.get('NSPB_REPO') or Path(__file__).resolve().parents[3])
+ repo = Path(env.get('NSPB_REPO') or _engine_root())
game = env.get('GAME') or die('GAME is not set (the game profile ID)')
w = Path(env.get('W') or die('W is not set (the work folder)'))
diff --git a/frontend-qt/nspb_qt/win/driver.py b/frontend-qt/nspb_qt/win/driver.py
index 347f02b6..78ca134e 100644
--- a/frontend-qt/nspb_qt/win/driver.py
+++ b/frontend-qt/nspb_qt/win/driver.py
@@ -56,7 +56,9 @@ REFUSED_OPTIONS = {
def repo_dir() -> Path:
- return Path(__file__).resolve().parents[3]
+ """The engine's folder: the checkout's engine/ when synced, else its committed tree (LNPANDY/NX2Apple#214)."""
+ from .. import engine
+ return Path(engine.engine_root(str(Path(__file__).resolve().parents[3])))
def load_upstream(repo: Path) -> ModuleType:
diff --git a/frontend-qt/tests/test_engine_layout.py b/frontend-qt/tests/test_engine_layout.py
new file mode 100644
index 00000000..2e9be771
--- /dev/null
+++ b/frontend-qt/tests/test_engine_layout.py
@@ -0,0 +1,86 @@
+"""Which engine the Qt front end builds from (LNPANDY/NX2Apple#214): engine/ when synced and current, the committed tree of a
+checkout from before the switch, and engine/ as the only choice (with a sync first) once the committed tree is gone."""
+from __future__ import annotations
+
+import importlib
+import shutil
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / 'frontend-qt'))
+
+engine = importlib.import_module('nspb_qt.engine')
+
+MARKER = Path('dm/tools/nx_extract/Program.cs')
+PIN = 'a' * 40
+
+
+class EngineLayoutTests(unittest.TestCase):
+ def setUp(self):
+ self.r = Path(tempfile.mkdtemp(prefix='qt-engine-'))
+ self.addCleanup(shutil.rmtree, self.r, True)
+
+ def touch(self, rel: str | Path, text: str = 'x') -> None:
+ f = self.r / rel
+ f.parent.mkdir(parents=True, exist_ok=True)
+ f.write_text(text)
+
+ def new_layout(self) -> None:
+ """tools/engine.py, the pin and a one-patch series."""
+ self.touch('tools/engine.py')
+ self.touch('.upstream-ref', PIN + '\n')
+ self.touch('patches/series', '# order\n0001-a.patch\n')
+ self.touch('patches/0001-a.patch', 'patch text\n')
+
+ def synced_engine(self) -> None:
+ """engine/ as sync leaves it: the tree, a branch, and the stamp naming the key and the head."""
+ self.touch(Path('engine') / MARKER)
+ self.touch('engine/.git/HEAD', 'ref: refs/heads/nx2apple\n')
+ self.touch('engine/.git/refs/heads/nx2apple', 'b' * 40 + '\n')
+ self.touch('engine/.git/nx2apple-stamp', f'nx2apple-engine 1\npin {PIN}\nkey {engine.stamp_key(self.r)}\nhead {"b" * 40}\n')
+
+ def test_a_checkout_from_before_the_switch_uses_its_committed_tree(self):
+ self.touch(MARKER)
+ self.assertTrue(engine.is_checkout(str(self.r)))
+ self.assertEqual(engine.engine_root(str(self.r)), str(self.r))
+ self.assertFalse(engine.is_stale(str(self.r)))
+
+ def test_a_current_engine_wins_over_the_committed_tree(self):
+ self.touch(MARKER)
+ self.new_layout()
+ self.synced_engine()
+ self.assertFalse(engine.is_stale(str(self.r)))
+ self.assertEqual(engine.engine_root(str(self.r)), str(self.r / 'engine'))
+
+ def test_a_stale_engine_falls_back_to_the_committed_tree(self):
+ self.touch(MARKER)
+ self.new_layout()
+ self.synced_engine()
+ self.touch('patches/0001-a.patch', 'changed\n')
+ self.assertTrue(engine.is_stale(str(self.r)))
+ self.assertEqual(engine.engine_root(str(self.r)), str(self.r))
+ self.assertFalse(engine.needs_sync_before_build(str(self.r)))
+
+ def test_a_commit_made_in_engine_makes_it_stale(self):
+ self.new_layout()
+ self.synced_engine()
+ self.touch('engine/.git/refs/heads/nx2apple', 'c' * 40 + '\n')
+ self.assertTrue(engine.is_stale(str(self.r)))
+
+ def test_without_a_committed_tree_the_engine_is_synced_before_a_build(self):
+ self.new_layout()
+ self.assertTrue(engine.is_checkout(str(self.r)))
+ self.assertTrue(engine.needs_sync_before_build(str(self.r)))
+ self.assertEqual(engine.engine_root(str(self.r)), str(self.r / 'engine'))
+ self.synced_engine()
+ self.assertFalse(engine.needs_sync_before_build(str(self.r)))
+
+ def test_the_sync_command_runs_the_checkouts_tool(self):
+ self.assertEqual(engine.sync_command('/c', 'py'), ['py', str(Path('/c/tools/engine.py')), 'sync'])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/frontend/Sources/NSPortBuilder/ContentView.swift b/frontend/Sources/NSPortBuilder/ContentView.swift
index 50213833..730d0ae0 100644
--- a/frontend/Sources/NSPortBuilder/ContentView.swift
+++ b/frontend/Sources/NSPortBuilder/ContentView.swift
@@ -291,7 +291,7 @@ final class BuildModel: ObservableObject {
}
nonisolated static func isRepo(_ dir: String) -> Bool {
- FileManager.default.fileExists(atPath: dir + "/dm/tools/nx_extract/Program.cs")
+ EngineLayout.isCheckout(dir) // the committed tree, a synced engine/, or tools/engine.py to make one (#214)
}
/// Every file of the repository the launcher itself runs by path: one list, next to the calls that use it
@@ -304,7 +304,8 @@ final class BuildModel: ObservableObject {
/// The tools of `requiredTools` that `dir` does not have (empty: the build tools are as new as this app needs).
nonisolated static func missingTools(in dir: String) -> [String] {
- requiredTools.filter { !FileManager.default.fileExists(atPath: dir + "/" + $0) }
+ let root = EngineLayout.root(dir) // engine/ once synced, else the committed tree (Engine.swift)
+ return requiredTools.filter { !FileManager.default.fileExists(atPath: root + "/" + $0) }
}
/// Whether the checkout's build script has this fork's `--work-dir` (issue #5): an upstream
@@ -312,7 +313,8 @@ final class BuildModel: ObservableObject {
/// app passes it. Read from the script's own argparse rather than assumed — the option is the
/// fork's, not upstream's.
nonisolated static func repoSupportsWorkDir(_ dir: String) -> Bool {
- let script = dir + "/" + buildScript
+ let script = EngineLayout.root(dir) + "/" + buildScript
+ if !FileManager.default.fileExists(atPath: script), EngineLayout.hasEngineTool(dir) { return true } // engine/ not made yet: the fork's engine has it
guard let text = try? String(contentsOfFile: script, encoding: .utf8) else { return false }
return text.contains("'--work-dir'") || text.contains("\"--work-dir\"")
}
@@ -427,6 +429,7 @@ final class BuildModel: ObservableObject {
if ok && Self.isRepo(dest) {
self.repoPath = dest
self.repoStatus = String(format: self.t("repo.downloadDone"), self.shortPath(dest))
+ self.syncEngine()
} else {
self.repoStatus = self.t("repo.downloadFail")
try? FileManager.default.removeItem(atPath: dest) // a failed clone leaves an empty folder
@@ -487,6 +490,7 @@ final class BuildModel: ObservableObject {
if r.ok {
self.repoBehind = 0; self.repoStatus = self.t("repo.pulled")
if !r.out.isEmpty { self.append(r.out + "\n") }
+ self.syncEngine() // the pull may have moved the pin or the patches
} else {
self.repoStatus = self.pullFailureMessage(r.out)
self.append(r.out + "\n")
@@ -513,6 +517,7 @@ final class BuildModel: ObservableObject {
if pulled.ok {
self.repoBehind = 0; self.autoPulled = behind
self.repoStatus = String(format: self.t("repo.autoPulled"), behind)
+ self.syncEngine()
} else {
self.repoStatus = self.pullFailureMessage(pulled.out)
self.append(pulled.out + "\n")
@@ -607,6 +612,7 @@ final class BuildModel: ObservableObject {
self.repoBehind = 0
self.repoStatus = String(format: self.t(automatic ? "repo.adoptedAuto" : "repo.adopted"), branch)
if automatic { self.autoAdopted = true }
+ self.syncEngine()
} else {
self.repoStatus = self.t(automatic ? "repo.adoptFailAuto" : "repo.adoptFail")
}
@@ -614,6 +620,30 @@ final class BuildModel: ObservableObject {
}
}
+ /// What waits for the running `tools/engine.py sync` (nil: none runs), and whether another one is due when it ends.
+ private var engineSyncWaiting: [(Bool) -> Void]?
+ private var engineSyncAgain = false
+
+ /// `tools/engine.py sync` when engine/ is stale (#214, Engine.swift): at launch, after a clone, a pull or an adoption, and before a
+ /// build that has no other engine. A no-op for a checkout from before #214 or a current engine/. `then(ok)` runs on the main queue.
+ func syncEngine(then: ((Bool) -> Void)? = nil) {
+ if engineSyncWaiting != nil { engineSyncAgain = true; if let then { engineSyncWaiting?.append(then) }; return }
+ guard EngineLayout.isStale(repoPath) else { then?(true); return }
+ engineSyncWaiting = then.map { [$0] } ?? []
+ let repo = repoPath, chosen = pythonPath
+ append("$ python3 tools/engine.py sync\n")
+ DispatchQueue.global().async {
+ let py = chosen.isEmpty ? Self.findPython().path : chosen
+ let ok = EngineLayout.runSync(repo: repo, python: py) { line in DispatchQueue.main.async { self.append(line + "\n") } }
+ DispatchQueue.main.async {
+ let waiting = self.engineSyncWaiting ?? [], again = self.engineSyncAgain && ok
+ self.engineSyncWaiting = nil; self.engineSyncAgain = false
+ if again { self.syncEngine { ok in waiting.forEach { $0(ok) } } } else { waiting.forEach { $0(ok) } }
+ self.objectWillChange.send() // repoOK and the engine's paths may have changed
+ }
+ }
+ }
+
// ---- App self-update (GitHub Releases) ----
// The rules (what is verified, which copies replace themselves, never during a build) are in AppUpdate.swift.
@@ -1154,6 +1184,15 @@ final class BuildModel: ObservableObject {
// empty, and the old fallback was Apple's /usr/bin/python3 (3.9), which the script then
// rejected with "require Python 3.12 or newer" whatever the machine actually had (#53).
let py = pythonPath.isEmpty ? Self.findPython().path : pythonPath
+ // engine/ is the only engine and is stale or missing (#214): make it, then start again
+ if EngineLayout.needsSyncBeforeBuild(repoPath) {
+ syncEngine { ok in
+ self.running = false
+ if ok && !EngineLayout.needsSyncBeforeBuild(self.repoPath) { self.startBuild() }
+ else { self.fail(String(format: self.t("msg.toolsOld"), "engine/ (python3 tools/engine.py sync, see the log)")) }
+ }
+ return
+ }
// The tools this app runs, before a build that takes an hour finds one missing at its last step (#87)
let missing = Self.missingTools(in: repoPath)
if !missing.isEmpty {
@@ -1237,7 +1276,8 @@ final class BuildModel: ObservableObject {
return
}
}
- var args = [repoPath + "/" + Self.buildScript, inputDir, "-k", keysPath,
+ let engine = EngineLayout.root(repoPath) // engine/ once synced, else the committed tree (#214)
+ var args = [engine + "/" + Self.buildScript, inputDir, "-k", keysPath,
"--progress", "json"]
// A compatibility switch the player turned on: written to a small env file the pipeline
// copies into the app bundle (main.m sets one variable per line at boot). Only for an IPA;
@@ -1277,7 +1317,7 @@ final class BuildModel: ObservableObject {
let p = Process()
p.executableURL = URL(fileURLWithPath: py)
p.arguments = args
- p.currentDirectoryURL = URL(fileURLWithPath: repoPath)
+ p.currentDirectoryURL = URL(fileURLWithPath: engine)
// The root of the runtime the apphost looks for: derived from the detected installation,
// not assumed to be ~/.dotnet (Homebrew keeps it in …/opt/dotnet/libexec). Only set when the
// derivation actually answered — a wrong root is worse than none, and the pipeline derives
@@ -1866,9 +1906,10 @@ final class BuildModel: ObservableObject {
let p = Process()
p.executableURL = URL(fileURLWithPath: pythonPath.isEmpty ? "/usr/bin/python3" : pythonPath)
// dest/romfs.img: one file, which main.m's per-game lookup takes before any folder.
- p.arguments = [repoPath + "/" + Self.romfsImageTool, "--reuse", romfs, dest + "/romfs.img"]
+ let engine = EngineLayout.root(repoPath)
+ p.arguments = [engine + "/" + Self.romfsImageTool, "--reuse", romfs, dest + "/romfs.img"]
let romfsPy = p.executableURL!.path
- p.environment = ["PATH": Self.toolPath("/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin", python: romfsPy), "PYTHONPATH": repoPath + "/dm/tools"]
+ p.environment = ["PATH": Self.toolPath("/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin", python: romfsPy), "PYTHONPATH": engine + "/dm/tools"]
.merging(BundledTools.environment(for: romfsPy)) { $1 }
Self.inOwnGroup(p, python: p.executableURL!.path)
let pipe = Pipe(); p.standardOutput = pipe; p.standardError = pipe
@@ -2789,7 +2830,7 @@ final class BuildModel: ObservableObject {
}
let macMetal = firstOutputLine("/usr/bin/xcrun", ["-sdk", "macosx", "metal", "--version"])
- let lock = repo.isEmpty ? nil : try? String(contentsOfFile: repo + "/" + MetalPin.lockPath, encoding: .utf8)
+ let lock = repo.isEmpty ? nil : try? String(contentsOfFile: EngineLayout.root(repo) + "/" + MetalPin.lockPath, encoding: .utf8)
let iosMetal = needsIOS && lock != nil ? firstOutputLine("/usr/bin/xcrun", ["-sdk", "iphoneos", "metal", "--version"]) : nil
out.append(metalDep(runs: macMetal != nil, lock: lock, now: ["macosx": macMetal, "iphoneos": iosMetal], needsIOS: needsIOS))
diff --git a/frontend/Sources/NSPortBuilder/Diagnostics.swift b/frontend/Sources/NSPortBuilder/Diagnostics.swift
index 7794fec9..bd36728f 100644
--- a/frontend/Sources/NSPortBuilder/Diagnostics.swift
+++ b/frontend/Sources/NSPortBuilder/Diagnostics.swift
@@ -298,7 +298,8 @@ extension BuildModel {
let env = ["HOME": NSHomeDirectory(), "PATH": "/usr/bin:/bin", "TMPDIR": NSTemporaryDirectory()]
var dotnetEnv = env
if !dotnet.isEmpty { dotnetEnv["DOTNET_ROOT"] = (dotnet as NSString).deletingLastPathComponent }
- let read = { (file: String) in try? String(contentsOfFile: repo + "/" + file, encoding: .utf8) }
+ let engine = EngineLayout.root(repo) // engine/ once synced, else the committed tree (#214)
+ let read = { (file: String) in try? String(contentsOfFile: engine + "/" + file, encoding: .utf8) }
let pins = read("dm/gfx/tools/dmsh_chain.lock").map(Diagnostics.chainPins) ?? (nil, nil)
let inputs = Diagnostics.Inputs(
appVersion: Self.appVersion,
diff --git a/frontend/Sources/NSPortBuilder/Engine.swift b/frontend/Sources/NSPortBuilder/Engine.swift
new file mode 100644
index 00000000..a252d7c5
--- /dev/null
+++ b/frontend/Sources/NSPortBuilder/Engine.swift
@@ -0,0 +1,111 @@
+import CryptoKit
+import Foundation
+
+/// Where the engine of a checkout is, and whether it needs `tools/engine.py sync` (LNPANDY/NX2Apple#214).
+///
+/// Since #214 the engine is built into `<checkout>/engine/`: seedds/nxnative at `.upstream-ref` plus `patches/`, made by
+/// `tools/engine.py sync`. Until the committed copy goes (step 2), a checkout has both, and one from before the switch has only the
+/// committed copy. The rule, the same as the Qt front end's (frontend-qt/nspb_qt/engine.py):
+///
+/// - `engine/` when it is synced and current,
+/// - else the committed tree (`dm/`, `scripts/` at the checkout's root) when the checkout has one,
+/// - else `engine/` as it is (stale or missing: the build syncs it first).
+///
+/// `stampKey` is tools/engine.py's digest of the pin and the series, compared with the stamp sync wrote into engine/.git, so the
+/// launcher tells a stale engine without starting Python. A release app carries the synced engine as `Resources/engine.git`
+/// (build-mac-app.yml), the seed sync fetches from before it goes to the network.
+enum EngineLayout {
+ /// The file that tells an engine tree (BuildModel.isRepo always looked for it).
+ static let marker = "dm/tools/nx_extract/Program.cs"
+
+ static func hasEngineTool(_ repo: String) -> Bool {
+ let fm = FileManager.default
+ return !repo.isEmpty && fm.fileExists(atPath: repo + "/tools/engine.py") && fm.fileExists(atPath: repo + "/.upstream-ref")
+ }
+
+ static func series(_ repo: String) -> [String]? {
+ guard let text = try? String(contentsOfFile: repo + "/patches/series", encoding: .utf8) else { return nil }
+ return text.split(separator: "\n", omittingEmptySubsequences: false).compactMap { line in
+ let s = (line.split(separator: "#", maxSplits: 1, omittingEmptySubsequences: false).first ?? "").trimmingCharacters(in: .whitespaces)
+ return s.isEmpty ? nil : s
+ }
+ }
+
+ static func hex(_ d: SHA256.Digest) -> String { d.map { String(format: "%02x", $0) }.joined() }
+
+ /// tools/engine.py's stamp_key: sha256 of "pin <sha>\n", then "patch <name>\n<sha256 of the file>\n" for each patch in order.
+ static func stampKey(_ repo: String) -> String? {
+ guard let pin = try? String(contentsOfFile: repo + "/.upstream-ref", encoding: .utf8), let names = series(repo) else { return nil }
+ var h = SHA256()
+ h.update(data: Data(("pin " + pin.trimmingCharacters(in: .whitespacesAndNewlines) + "\n").utf8))
+ for name in names {
+ guard let data = FileManager.default.contents(atPath: repo + "/patches/" + name) else { return nil }
+ h.update(data: Data(("patch " + name + "\n" + hex(SHA256.hash(data: data)) + "\n").utf8))
+ }
+ return hex(h.finalize())
+ }
+
+ /// HEAD of engine/ read from its files (no git process): the detached id, or the id its branch points to.
+ static func head(_ eng: String) -> String {
+ func read(_ p: String) -> String? { (try? String(contentsOfFile: eng + "/.git/" + p, encoding: .utf8))?.trimmingCharacters(in: .whitespacesAndNewlines) }
+ guard let h = read("HEAD") else { return "" }
+ guard h.hasPrefix("ref: ") else { return h }
+ let ref = String(h.dropFirst(5))
+ if let id = read(ref) { return id }
+ for line in (read("packed-refs") ?? "").split(separator: "\n") where line.hasSuffix(" " + ref) {
+ return String(line.split(separator: " ").first ?? "")
+ }
+ return ""
+ }
+
+ /// True when the checkout can make engine/ and engine/ is missing or not built from the current pin and series.
+ static func isStale(_ repo: String) -> Bool {
+ guard hasEngineTool(repo) else { return false }
+ let eng = repo + "/engine"
+ guard let text = try? String(contentsOfFile: eng + "/.git/nx2apple-stamp", encoding: .utf8) else { return true }
+ var stamp: [String: String] = [:]
+ for line in text.split(separator: "\n") {
+ let kv = line.split(separator: " ", maxSplits: 1)
+ if kv.count == 2 { stamp[String(kv[0])] = String(kv[1]) }
+ }
+ return stamp["key"] != stampKey(repo) || stamp["head"] != head(eng)
+ }
+
+ /// The folder holding dm/ and scripts/ for this checkout (see the rule above).
+ static func root(_ repo: String) -> String {
+ guard !repo.isEmpty else { return repo }
+ let fm = FileManager.default
+ if fm.fileExists(atPath: repo + "/engine/" + marker) && !isStale(repo) { return repo + "/engine" }
+ if fm.fileExists(atPath: repo + "/" + marker) { return repo }
+ return hasEngineTool(repo) ? repo + "/engine" : repo
+ }
+
+ /// A checkout the launcher can build from: the committed tree, a synced engine/, or tools/engine.py to make one.
+ static func isCheckout(_ dir: String) -> Bool {
+ let fm = FileManager.default
+ return !dir.isEmpty && (fm.fileExists(atPath: dir + "/" + marker) || fm.fileExists(atPath: dir + "/engine/" + marker) || hasEngineTool(dir))
+ }
+
+ /// engine/ is the only engine and it is stale or missing (after step 2, or a checkout whose committed tree is gone).
+ static func needsSyncBeforeBuild(_ repo: String) -> Bool {
+ hasEngineTool(repo) && isStale(repo) && !FileManager.default.fileExists(atPath: repo + "/" + marker)
+ }
+
+ /// The engine a release app carries (build-mac-app.yml), when this is such an app.
+ static func seed(in resources: URL? = BundledTools.resources) -> String? {
+ guard let dir = resources?.appendingPathComponent("engine.git"), FileManager.default.fileExists(atPath: dir.path + "/HEAD") else { return nil }
+ return dir.path
+ }
+
+ /// `python tools/engine.py [--seed <the app's engine.git>] sync`
+ static func syncArguments(repo: String, python: String, seed: String? = EngineLayout.seed()) -> [String] {
+ [python, repo + "/tools/engine.py"] + (seed.map { ["--seed", $0] } ?? []) + ["sync"]
+ }
+
+ /// Run the sync and stream its lines (blocking: call it off the main thread). git comes from the system's or Homebrew's PATH.
+ static func runSync(repo: String, python: String, line: @escaping (String) -> Void) -> Bool {
+ let env = ["PATH": BuildModel.toolPath("/usr/bin:/bin:/opt/homebrew/bin:/usr/local/bin", python: python), "HOME": NSHomeDirectory(),
+ "LANG": "en_US.UTF-8", "TMPDIR": NSTemporaryDirectory()].merging(BundledTools.environment(for: python)) { $1 }
+ return BuildModel.runCaptured(syncArguments(repo: repo, python: python), env: env, line: line)
+ }
+}
diff --git a/frontend/Sources/NSPortBuilder/LauncherView.swift b/frontend/Sources/NSPortBuilder/LauncherView.swift
index bbef2460..f2b48839 100644
--- a/frontend/Sources/NSPortBuilder/LauncherView.swift
+++ b/frontend/Sources/NSPortBuilder/LauncherView.swift
@@ -169,6 +169,7 @@ struct ContentView: View {
if performStartupChecks { library.discover(); fetchArtwork(); m.refreshDevices(); checkTools() }
if performStartupChecks {
m.checkDeps()
+ m.syncEngine() // a checkout whose engine/ is missing or older than its pin (#214)
if m.repoOK && m.repoIsGit { m.checkRepoUpdates(silent: true) } else { m.adoptAtLaunch() }
m.checkAppUpdate(silent: true)
BuildNotifier.shared.watch(m, library: library) // BuildNotifications.swift
diff --git a/frontend/Sources/NSPortBuilder/Library.swift b/frontend/Sources/NSPortBuilder/Library.swift
index 36e4e826..240807e1 100644
--- a/frontend/Sources/NSPortBuilder/Library.swift
+++ b/frontend/Sources/NSPortBuilder/Library.swift
@@ -142,7 +142,7 @@ struct LaunchSetting: Decodable, Identifiable {
/// render_base in the game's profile ("1600x900 1280x720": docked, then handheld).
static func renderBase(titleID: String, repo: String) -> [String: [Int]]? {
- let path = repo + "/dm/games/" + titleID + "/game.tsv"
+ let path = EngineLayout.root(repo) + "/dm/games/" + titleID + "/game.tsv"
guard !repo.isEmpty, let text = try? String(contentsOfFile: path, encoding: .utf8),
let line = text.split(separator: "\n").first(where: { $0.hasPrefix("render_base\t") }) else { return nil }
let sizes = line.dropFirst("render_base\t".count).split(separator: " ").map { $0.split(separator: "x").compactMap { Int($0) } }
@@ -561,7 +561,7 @@ final class GameLibrary: ObservableObject {
}
static func title(for path: String, repo: String, titleID: String) -> String {
- if let text = try? String(contentsOfFile: repo + "/dm/games/" + titleID + "/game.tsv", encoding: .utf8),
+ if let text = try? String(contentsOfFile: EngineLayout.root(repo) + "/dm/games/" + titleID + "/game.tsv", encoding: .utf8),
let row = text.split(separator: "\n").first(where: { $0.hasPrefix("name\t") }) {
return String(row.dropFirst(5))
}
@@ -581,7 +581,7 @@ final class GameLibrary: ObservableObject {
.filter { $0.isLetter || $0.isNumber }
}
let name = normalized(game.name)
- let images = URL(fileURLWithPath: repo).appendingPathComponent("docs/images")
+ let images = URL(fileURLWithPath: EngineLayout.root(repo)).appendingPathComponent("docs/images")
let files = (try? FileManager.default.contentsOfDirectory(at: images, includingPropertiesForKeys: nil)) ?? []
let matches = files.filter {
["jpg", "jpeg", "png", "webp"].contains($0.pathExtension.lowercased()) && name.hasSuffix(normalized($0.deletingPathExtension().lastPathComponent))
diff --git a/frontend/Tests/NSPortBuilderTests/EngineLayoutTests.swift b/frontend/Tests/NSPortBuilderTests/EngineLayoutTests.swift
new file mode 100644
index 00000000..12bd6551
--- /dev/null
+++ b/frontend/Tests/NSPortBuilderTests/EngineLayoutTests.swift
@@ -0,0 +1,104 @@
+import XCTest
+@testable import NSPortBuilder
+
+/// #214: which engine the launcher builds from. engine/ when synced and current, the committed tree of a checkout from before the
+/// switch, and engine/ as the only choice (with a sync first) once the committed tree is gone. The stamp key matches tools/engine.py's.
+final class EngineLayoutTests: XCTestCase {
+ private var root: URL!
+ private var repo: String { root.path }
+ private let pin = String(repeating: "a", count: 40)
+
+ override func setUpWithError() throws {
+ root = FileManager.default.temporaryDirectory.appendingPathComponent("nx2apple-engine214-" + UUID().uuidString)
+ try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+ }
+
+ override func tearDownWithError() throws { try? FileManager.default.removeItem(at: root) }
+
+ private func touch(_ rel: String, _ text: String = "x") throws {
+ let url = root.appendingPathComponent(rel)
+ try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)
+ try text.write(to: url, atomically: true, encoding: .utf8)
+ }
+
+ private func newLayout() throws {
+ try touch("tools/engine.py")
+ try touch(".upstream-ref", pin + "\n")
+ try touch("patches/series", "# order\n0001-a.patch\n\n0002-b.patch # trailing note\n")
+ try touch("patches/0001-a.patch", "patch a\n")
+ try touch("patches/0002-b.patch", "patch b\n")
+ }
+
+ private func syncedEngine() throws {
+ let head = String(repeating: "b", count: 40)
+ try touch("engine/" + EngineLayout.marker)
+ try touch("engine/.git/HEAD", "ref: refs/heads/nx2apple\n")
+ try touch("engine/.git/refs/heads/nx2apple", head + "\n")
+ try touch("engine/.git/nx2apple-stamp", "nx2apple-engine 1\npin \(pin)\nkey \(EngineLayout.stampKey(repo)!)\nhead \(head)\n")
+ }
+
+ func testACheckoutFromBeforeTheSwitchUsesItsCommittedTree() throws {
+ try touch(EngineLayout.marker)
+ XCTAssertTrue(BuildModel.isRepo(repo))
+ XCTAssertEqual(EngineLayout.root(repo), repo)
+ XCTAssertFalse(EngineLayout.isStale(repo))
+ XCTAssertFalse(EngineLayout.needsSyncBeforeBuild(repo))
+ }
+
+ func testACurrentEngineWinsAndTheBuildScriptComesFromIt() throws {
+ try touch(EngineLayout.marker)
+ try newLayout()
+ try syncedEngine()
+ XCTAssertFalse(EngineLayout.isStale(repo))
+ XCTAssertEqual(EngineLayout.root(repo), repo + "/engine")
+ XCTAssertEqual(BuildModel.missingTools(in: repo), BuildModel.requiredTools) // looked for under engine/, which lacks them here
+ try touch("engine/scripts/build_ipa_from_nsp.py", "p.add_argument('--work-dir')\n")
+ try touch("engine/dm/tools/romfs_image.py")
+ XCTAssertEqual(BuildModel.missingTools(in: repo), [])
+ XCTAssertTrue(BuildModel.repoSupportsWorkDir(repo))
+ }
+
+ func testAStaleEngineFallsBackToTheCommittedTree() throws {
+ try touch(EngineLayout.marker)
+ try newLayout()
+ try syncedEngine()
+ try touch("patches/0002-b.patch", "changed\n")
+ XCTAssertTrue(EngineLayout.isStale(repo))
+ XCTAssertEqual(EngineLayout.root(repo), repo)
+ XCTAssertFalse(EngineLayout.needsSyncBeforeBuild(repo))
+ }
+
+ func testACommitInEngineMakesItStale() throws {
+ try newLayout()
+ try syncedEngine()
+ try touch("engine/.git/refs/heads/nx2apple", String(repeating: "c", count: 40) + "\n")
+ XCTAssertTrue(EngineLayout.isStale(repo))
+ }
+
+ func testWithoutACommittedTreeTheEngineIsSyncedBeforeABuild() throws {
+ try newLayout()
+ XCTAssertTrue(BuildModel.isRepo(repo))
+ XCTAssertTrue(EngineLayout.needsSyncBeforeBuild(repo))
+ XCTAssertTrue(BuildModel.repoSupportsWorkDir(repo)) // the fork's engine always has --work-dir
+ XCTAssertEqual(EngineLayout.root(repo), repo + "/engine")
+ try syncedEngine()
+ XCTAssertFalse(EngineLayout.needsSyncBeforeBuild(repo))
+ }
+
+ func testTheSyncCommandPassesTheAppsSeed() {
+ XCTAssertEqual(EngineLayout.syncArguments(repo: "/c", python: "py", seed: nil), ["py", "/c/tools/engine.py", "sync"])
+ XCTAssertEqual(EngineLayout.syncArguments(repo: "/c", python: "py", seed: "/A.app/Contents/Resources/engine.git"),
+ ["py", "/c/tools/engine.py", "--seed", "/A.app/Contents/Resources/engine.git", "sync"])
+ }
+
+ /// The launcher's key and tools/engine.py's agree, so a sync's stamp reads as current here.
+ func testTheStampKeyMatchesTheTool() throws {
+ try newLayout()
+ let tools = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
+ .deletingLastPathComponent().appendingPathComponent("tools").path
+ let code = "import sys; sys.path.insert(0, sys.argv[1]); import engine, pathlib; print(engine.stamp_key(pathlib.Path(sys.argv[2])))"
+ let r = BuildModel.runQuiet(["/usr/bin/env", "python3", "-c", code, tools, repo], env: ["PATH": "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"])
+ try XCTSkipUnless(r.ok, "python3 did not run: \(r.out)")
+ XCTAssertEqual(r.out.trimmingCharacters(in: .whitespacesAndNewlines), EngineLayout.stampKey(repo))
+ }
+}