Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

store: Manim, Typst, Excalidraw, Strudel and Phaser install and run on a fresh machine

dee8a47a15b0285
diff --git a/store/agents/excalidraw/toolchain/doctor.sh b/store/agents/excalidraw/toolchain/doctor.sh
index 40b708050..39c9a3a0a 100755
--- a/store/agents/excalidraw/toolchain/doctor.sh
+++ b/store/agents/excalidraw/toolchain/doctor.sh
@@ -1,6 +1,9 @@
#!/usr/bin/env bash
set -u; cd "$(dirname "$0")/.."; bad=0
-if [ -f node_modules/@excalidraw/excalidraw/dist/excalidraw.production.min.js ]; then echo "ok excalidraw $(node -p "require('@excalidraw/excalidraw/package.json').version")"; else echo "miss node_modules — run toolchain/setup.sh"; bad=1; fi
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
+if harness_node 18; then echo "ok node $(node --version) (the pane server)"; else bad=1; fi
+if [ -f node_modules/@excalidraw/excalidraw/dist/excalidraw.production.min.js ]; then echo "ok excalidraw $(node -p "require('@excalidraw/excalidraw/package.json').version" 2>/dev/null)"; else echo "miss node_modules — run toolchain/setup.sh"; bad=1; fi
if [ -f viewer.mjs ] && [ -f viewer/index.html ] && [ -f viewer/app.js ]; then echo "ok pane (viewer.mjs, viewer/)"; else echo "miss viewer.mjs or viewer/ — the checkout is incomplete"; bad=1; fi
if command -v python3 >/dev/null 2>&1; then echo "ok $(python3 --version)"; else echo "miss python3"; bad=1; fi
exit $bad
diff --git a/store/agents/excalidraw/toolchain/runtimes.sh b/store/agents/excalidraw/toolchain/runtimes.sh
new file mode 100644
index 000000000..4e21942d0
--- /dev/null
+++ b/store/agents/excalidraw/toolchain/runtimes.sh
@@ -0,0 +1,187 @@
+# shellcheck shell=bash
+# runtimes.sh — the interpreters a store package runs on, found on this machine or fetched, never
+# asked of the person who pressed Get.
+#
+# Sourced (not run) by a package's setup, doctor and viewer scripts. A new Mac has Apple's Python 3.9,
+# no Homebrew and, for most people, no Node on PATH; a package that answers "brew install [email protected]"
+# has failed its install. So:
+#
+# harness_node 18 node >= 18 on PATH, with npm: the machine's own, else the Node Harness
+# itself runs on (~/.harness/runtime/current-node, laid down with the CLI)
+# harness_uv uv on PATH: the machine's own, else a pinned, checksummed release
+# fetched into ~/.harness/runtime
+# harness_venv DIR 3.12 [MIN [BELOW]]
+# DIR is a venv on CPython 3.12, which uv downloads when the machine
+# has none. A venv already there on a Python in [MIN, BELOW) is kept.
+# harness_pip DIR PKG… install into that venv, through uv
+# harness_micromamba micromamba, pinned and checksummed, for native libraries PyPI has
+# no wheel for (pycairo on macOS)
+# harness_conda_env DIR SPEC… DIR is a conda-forge environment with those packages
+#
+# Every function prints `miss …` and returns 1 when it cannot deliver, so `harness_node 18 || exit 1`
+# reads like the rest of a setup script. Nothing here touches the user's shell profile or anything
+# outside ~/.harness/runtime and the package's own directory.
+#
+# ONE copy is written by hand: store/tools/runtimes.sh in the OpenHarness repository. Each package
+# carries an identical copy (a package installs alone, so it cannot reach this one);
+# `node store/tools/sync-runtimes.mjs` rewrites them and the CLI's store spec fails when one drifts.
+
+HARNESS_RUNTIME="${ADAPTER_RUNTIME_DIR:-${HOME:-}/.harness/runtime}"
+# The Pythons uv downloads live with Harness's other runtimes, not in uv's own ~/.local/share/uv:
+# a package's venv points at its interpreter, and `uv python uninstall` there must not break it.
+UV_PYTHON_INSTALL_DIR="${UV_PYTHON_INSTALL_DIR:-$HARNESS_RUNTIME/python}"
+export UV_PYTHON_INSTALL_DIR
+HARNESS_UV_VERSION=0.12.15
+HARNESS_MICROMAMBA_VERSION=2.9.0-0
+
+_harness_platform() {
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64) echo darwin-arm64 ;;
+ Darwin-x86_64) echo darwin-x64 ;;
+ Linux-x86_64) echo linux-x64 ;;
+ Linux-aarch64 | Linux-arm64) echo linux-arm64 ;;
+ *) return 1 ;;
+ esac
+}
+
+_harness_sha256() {
+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1
+ else sha256sum "$1" | cut -d' ' -f1; fi
+}
+
+# _harness_fetch URL DEST SHA256 — DEST exists only if what arrived matches the pinned checksum.
+_harness_fetch() {
+ local part="$2.part.$$"
+ if ! curl -fsSL --retry 3 --connect-timeout 20 --max-time 900 -o "$part" "$1"; then
+ rm -f "$part"; echo "miss could not download $1 — check this machine's internet connection"; return 1
+ fi
+ if [ "$(_harness_sha256 "$part")" != "$3" ]; then
+ rm -f "$part"; echo "miss $1 did not match its pinned checksum; nothing was installed"; return 1
+ fi
+ mv -f "$part" "$2"
+}
+
+_harness_node_at_least() {
+ command -v node >/dev/null 2>&1 || return 1
+ node -e '
+ const [a, b = 0] = process.argv[1].split(".").map(Number)
+ const [x, y] = process.versions.node.split(".").map(Number)
+ process.exit(x > a || (x === a && y >= b) ? 0 : 1)
+ ' "$1" >/dev/null 2>&1
+}
+
+harness_node() {
+ local min="${1:-18}" recorded saved="$PATH" own=0
+ _harness_node_at_least "$min" && own=1
+ # A node without npm beside it (a distro's nodejs package) cannot run a setup's `npm ci`.
+ if [ "$own" = 1 ] && command -v npm >/dev/null 2>&1; then return 0; fi
+ recorded="$(cat "$HARNESS_RUNTIME/current-node" 2>/dev/null || true)"
+ if [ -n "$recorded" ] && [ -x "${recorded%/*}/node" ]; then
+ PATH="${recorded%/*}:$PATH"; export PATH; hash -r 2>/dev/null || true
+ _harness_node_at_least "$min" && return 0
+ PATH="$saved"; export PATH; hash -r 2>/dev/null || true
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min — this machine's newest is $("${recorded%/*}/node" -v), Harness's own; update Harness"
+ return 1
+ fi
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min, and Harness's own Node is not in $HARNESS_RUNTIME — run \`harness start\` once to lay it down"
+ return 1
+}
+
+harness_uv() {
+ local dir asset sum tmp
+ command -v uv >/dev/null 2>&1 && return 0
+ for dir in "${HOME:-}/.local/bin" "${HOME:-}/.cargo/bin"; do
+ [ -n "${HOME:-}" ] || break
+ if [ -x "$dir/uv" ]; then PATH="$dir:$PATH"; export PATH; return 0; fi
+ done
+ dir="$HARNESS_RUNTIME/uv-$HARNESS_UV_VERSION"
+ if [ ! -x "$dir/uv" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=uv-aarch64-apple-darwin sum=dc304b9ed1b24174572290fba60ac3f6fe63c73a671f0439e62a91375841964d ;;
+ darwin-x64) asset=uv-x86_64-apple-darwin sum=e9ca61775532368fe518ab03e7a354c7ecab8ccb3c7d941c775fcc4a362b801b ;;
+ linux-x64) asset=uv-x86_64-unknown-linux-gnu sum=f97935763c04be3e692460a7aaeaaab8fc3b78fcf8b389da820b38ae7423a638 ;;
+ linux-arm64) asset=uv-aarch64-unknown-linux-gnu sum=0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08 ;;
+ *) echo "miss uv — there is no uv release for $(uname -s) $(uname -m); install it from https://docs.astral.sh/uv/"; return 1 ;;
+ esac
+ echo " fetching uv $HARNESS_UV_VERSION into $HARNESS_RUNTIME (it brings Python and the packages)"
+ mkdir -p "$HARNESS_RUNTIME" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ tmp="$(mktemp -d "$HARNESS_RUNTIME/.uv.XXXXXX")" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ if ! _harness_fetch "https://github.com/astral-sh/uv/releases/download/$HARNESS_UV_VERSION/$asset.tar.gz" "$tmp/uv.tar.gz" "$sum"; then
+ rm -rf "$tmp"; return 1
+ fi
+ tar -xzf "$tmp/uv.tar.gz" -C "$tmp" && chmod +x "$tmp/$asset/uv" "$tmp/$asset/uvx" \
+ || { rm -rf "$tmp"; echo "miss the uv archive would not unpack"; return 1; }
+ # Two installs at once both get here; whichever renames first wins and the other's copy goes.
+ # (mv onto a directory that exists would nest the copy inside it, hence the test.)
+ [ -e "$dir" ] || mv "$tmp/$asset" "$dir" 2>/dev/null || true
+ rm -rf "$tmp"
+ [ -x "$dir/uv" ] || { echo "miss uv did not land in $dir"; return 1; }
+ fi
+ PATH="$dir:$PATH"; export PATH
+}
+
+# harness_venv DIR VERSION [MIN [BELOW]] — MIN defaults to VERSION; BELOW is exclusive and optional.
+harness_venv() {
+ local dir="${1:?harness_venv needs a directory}" want="${2:?harness_venv needs a Python version}"
+ local min="${3:-$2}" below="${4:-}"
+ if [ -x "$dir/bin/python" ] && "$dir/bin/python" -c '
+import sys
+at = lambda s: tuple(int(p) for p in s.split("."))
+v = sys.version_info[:2]
+sys.exit(0 if v >= at(sys.argv[1]) and (not sys.argv[2] or v < at(sys.argv[2])) else 1)
+' "$min" "$below" 2>/dev/null; then
+ echo "ok $("$dir/bin/python" --version) in $dir"
+ return 0
+ fi
+ harness_uv || return 1
+ echo " python $want in $dir (uv downloads it when this machine has none)"
+ rm -rf "$dir"
+ # only-managed: the same python-build-standalone interpreter on every machine, so a package tested
+ # once is tested everywhere — not whichever Homebrew or pyenv Python happened to be first on PATH.
+ uv venv --quiet --seed --python "$want" --python-preference only-managed "$dir" \
+ || { echo "miss could not make a Python $want environment in $dir"; return 1; }
+ echo "ok $("$dir/bin/python" --version) in $dir"
+}
+
+harness_pip() {
+ local dir="${1:?harness_pip needs a venv}"
+ shift
+ harness_uv || return 1
+ uv pip install --quiet --python "$dir/bin/python" "$@"
+}
+
+harness_micromamba() {
+ local dir asset sum
+ dir="$HARNESS_RUNTIME/micromamba-$HARNESS_MICROMAMBA_VERSION"
+ if [ ! -x "$dir/micromamba" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=micromamba-osx-arm64 sum=ec2a072f028e1a7cf20f3e2e74d5a8127cf5a5f27636375b5359811565f4e5be ;;
+ darwin-x64) asset=micromamba-osx-64 sum=1e71054bb3ac9a076e21f7ec48acfef536f9b3f1408f371a942784bf5ef83d8a ;;
+ linux-x64) asset=micromamba-linux-64 sum=366cd9cd8be14df1ab8ed50352a82111082a36686b2d389fdb79a92c3fafb3e3 ;;
+ linux-arm64) asset=micromamba-linux-aarch64 sum=9f93b974adcb4d166996af969b6cd371287d1a3e52733704727884d9b74cb7a7 ;;
+ *) echo "miss micromamba — there is no release for $(uname -s) $(uname -m)"; return 1 ;;
+ esac
+ echo " fetching micromamba $HARNESS_MICROMAMBA_VERSION into $HARNESS_RUNTIME (native libraries PyPI has no wheel for)"
+ mkdir -p "$dir" || { echo "miss cannot write $dir"; return 1; }
+ _harness_fetch "https://github.com/mamba-org/micromamba-releases/releases/download/$HARNESS_MICROMAMBA_VERSION/$asset" "$dir/micromamba.$$" "$sum" || return 1
+ chmod +x "$dir/micromamba.$$" && mv -f "$dir/micromamba.$$" "$dir/micromamba" \
+ || { rm -f "$dir/micromamba.$$"; echo "miss micromamba would not install into $dir"; return 1; }
+ fi
+ HARNESS_MICROMAMBA="$dir/micromamba"
+ export HARNESS_MICROMAMBA
+}
+
+# harness_conda_env DIR SPEC… — conda-forge only, packages cached under ~/.harness/runtime/mamba.
+harness_conda_env() {
+ local dir="${1:?harness_conda_env needs a directory}"
+ shift
+ # micromamba takes a relative --prefix as a name under its root prefix and "succeeds" there.
+ case "$dir" in /*) ;; *) dir="$PWD/$dir" ;; esac
+ harness_micromamba || return 1
+ MAMBA_ROOT_PREFIX="$HARNESS_RUNTIME/mamba" "$HARNESS_MICROMAMBA" create --yes --quiet \
+ --prefix "$dir" --override-channels --channel conda-forge "$@" >/dev/null \
+ && [ -d "$dir/conda-meta" ] \
+ || { echo "miss could not make the conda-forge environment in $dir ($*)"; return 1; }
+}
diff --git a/store/agents/excalidraw/toolchain/setup.sh b/store/agents/excalidraw/toolchain/setup.sh
index 9bcaac62a..4898832f2 100755
--- a/store/agents/excalidraw/toolchain/setup.sh
+++ b/store/agents/excalidraw/toolchain/setup.sh
@@ -1,10 +1,13 @@
#!/usr/bin/env bash
# Runs once at install, cwd = the install dir. The pane's dependencies — Excalidraw's own package and
-# the React it renders with — into node_modules, from the lockfile. Nothing global.
+# the React it renders with — into node_modules, from the lockfile. Nothing global. Node is this
+# machine's, or the one Harness itself runs on (npm comes beside it).
set -euo pipefail
cd "$(dirname "$0")/.."
-command -v node >/dev/null 2>&1 || { echo "miss node >= 18 on PATH"; exit 1; }
-command -v npm >/dev/null 2>&1 || { echo "miss npm on PATH"; exit 1; }
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
+harness_node 18 || exit 1
+command -v npm >/dev/null 2>&1 || { echo "miss npm beside $(command -v node)"; exit 1; }
command -v python3 >/dev/null 2>&1 || { echo "miss python3 (the scene helper and the verdict)"; exit 1; }
echo " npm ci (Excalidraw $(node -p "require('./package.json').dependencies['@excalidraw/excalidraw']"))"
npm ci --silent --no-audit --no-fund
diff --git a/store/agents/excalidraw/toolchain/test_scripts.py b/store/agents/excalidraw/toolchain/test_scripts.py
index 596b53928..7f8d38c9d 100644
--- a/store/agents/excalidraw/toolchain/test_scripts.py
+++ b/store/agents/excalidraw/toolchain/test_scripts.py
@@ -1,6 +1,7 @@
"""setup.sh, doctor.sh, init-workspace.sh and viewer.sh, run for real against a scratch install whose
PATH holds only stub commands (and the few coreutils the scripts use), so every ok / miss line and
-exit path is reached without npm, a network or the installed node_modules:
+exit path is reached without npm, a network or the installed node_modules. Node is found the way
+runtimes.sh finds it on a new machine too: not on PATH, but recorded in the runtime dir Harness keeps.
python3 -m unittest toolchain/test_scripts.py
"""
@@ -11,15 +12,20 @@ PACKAGE = Path(__file__).resolve().parent.parent
# A bash line tracer (BASH_ENV sourcing a DEBUG trap that appends to SHCOV_OUT) is passed through when set.
TRACER = {k: os.environ[k] for k in ("BASH_ENV", "SHCOV_OUT") if k in os.environ}
BASH = "/bin/bash"
-COREUTILS = ("dirname", "mkdir")
+COREUTILS = ("dirname", "mkdir", "cat")
BUNDLE = "node_modules/@excalidraw/excalidraw/dist/excalidraw.production.min.js"
-# `node -p "require(...)"`: the versions the scripts print.
-NODE = 'case "$2" in *react/package.json*) echo 18.3.1 ;; *) echo 0.17.6 ;; esac'
+# node: runtimes.sh's version probe (`node -e … 18`, answered by $NODE_OLD and never logged), its
+# version, and `node -p "require(...)"` for the versions the scripts print.
+NODE = """case "$1" in
+ -e) exit "${NODE_OLD:-0}" ;;
+ -v|--version) echo v22.23.2 ;;
+ -p) case "$2" in *react/package.json*) echo 18.3.1 ;; *) echo 0.17.6 ;; esac ;;
+esac"""
class Sandbox:
- """An install dir with the package's scripts, a bin/ of stubs as the whole PATH, and a log of
- every stub call."""
+ """An install dir with the package's scripts, a bin/ of stubs as the whole PATH, an empty runtime
+ dir for runtimes.sh, and a log of every stub call."""
def __init__(self, test: unittest.TestCase):
tmp = tempfile.TemporaryDirectory()
@@ -32,18 +38,32 @@ class Sandbox:
(self.install / "viewer.sh").symlink_to(PACKAGE / "viewer.sh")
self.bin = self.root / "bin"
self.bin.mkdir()
+ self.runtime = self.root / "runtime"
+ self.runtime.mkdir()
self.calls = self.root / "calls.log"
self.calls.touch()
for name in COREUTILS:
real = next(p for p in (Path("/bin") / name, Path("/usr/bin") / name) if p.exists())
(self.bin / name).symlink_to(real)
- def stub(self, name: str, body: str = "") -> Path:
- path = self.bin / name
- path.write_text(f'#!/bin/bash\necho "{name} $*" >> "$CALLS"\n{body}\n')
+ def stub(self, name: str, body: str = "", where: Path | None = None) -> Path:
+ path = (where or self.bin) / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.unlink(missing_ok=True) # never write through a link to a real command
+ # runtimes.sh's `node -e` probe is not a call the scripts make: it stays out of the log.
+ probe = '[ "$1" = -e ] || ' if name == "node" else ""
+ path.write_text(f'#!/bin/bash\n{probe}echo "{name} $*" >> "$CALLS"\n{body}\n')
path.chmod(0o755)
return path
+ def harness_node(self, npm: str | None = None) -> Path:
+ """No node on PATH; Harness's own recorded in the runtime dir, npm beside it when given."""
+ node = self.stub("node", NODE, where=self.root / "harness-node" / "bin")
+ if npm is not None:
+ self.stub("npm", npm, where=node.parent)
+ (self.runtime / "current-node").write_text(f"{node}\n")
+ return node
+
def file(self, rel: str) -> None:
path = self.install / rel
path.parent.mkdir(parents=True, exist_ok=True)
@@ -51,7 +71,7 @@ class Sandbox:
def run(self, script: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
return subprocess.run([BASH, str(self.install / script)], cwd=cwd or self.install,
- env={"PATH": str(self.bin), "CALLS": str(self.calls), **TRACER, **env},
+ env={"PATH": str(self.bin), "CALLS": str(self.calls), "ADAPTER_RUNTIME_DIR": str(self.runtime), **TRACER, **env},
capture_output=True, text=True, timeout=60)
def logged(self) -> list[str]:
@@ -67,13 +87,24 @@ class Doctor(unittest.TestCase):
box.file(rel)
r = box.run("toolchain/doctor.sh")
self.assertEqual(r.returncode, 0, r.stderr)
- self.assertEqual(r.stdout.splitlines(), ["ok excalidraw 0.17.6", "ok pane (viewer.mjs, viewer/)", "ok Python 3.12.1"])
+ self.assertEqual(r.stdout.splitlines(), ["ok node v22.23.2 (the pane server)", "ok excalidraw 0.17.6",
+ "ok pane (viewer.mjs, viewer/)", "ok Python 3.12.1"])
+
+ def test_harness_node_serves_when_path_has_none(self):
+ box = Sandbox(self)
+ box.harness_node()
+ box.stub("python3", 'echo "Python 3.9.6"')
+ for rel in (BUNDLE, "viewer.mjs", "viewer/index.html", "viewer/app.js"):
+ box.file(rel)
+ r = box.run("toolchain/doctor.sh")
+ self.assertEqual((r.returncode, r.stdout.splitlines()[:2]), (0, ["ok node v22.23.2 (the pane server)", "ok excalidraw 0.17.6"]), r.stderr)
def test_every_miss_is_reported_not_just_the_first(self):
box = Sandbox(self)
r = box.run("toolchain/doctor.sh")
self.assertEqual(r.returncode, 1)
- self.assertEqual(r.stdout.splitlines(), ["miss node_modules — run toolchain/setup.sh",
+ self.assertEqual(r.stdout.splitlines(), [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down",
+ "miss node_modules — run toolchain/setup.sh",
"miss viewer.mjs or viewer/ — the checkout is incomplete",
"miss python3"])
@@ -85,11 +116,14 @@ class Doctor(unittest.TestCase):
box.file(rel)
r = box.run("toolchain/doctor.sh")
self.assertEqual(r.returncode, 1)
- self.assertEqual(r.stdout.splitlines()[1], "miss viewer.mjs or viewer/ — the checkout is incomplete")
+ self.assertEqual(r.stdout.splitlines()[2], "miss viewer.mjs or viewer/ — the checkout is incomplete")
+
+
+NPM_CI = f"mkdir -p node_modules/@excalidraw/excalidraw/dist && : > {BUNDLE}"
class Setup(unittest.TestCase):
- def sandbox(self, *missing: str, npm: str = f"mkdir -p node_modules/@excalidraw/excalidraw/dist && : > {BUNDLE}") -> Sandbox:
+ def sandbox(self, *missing: str, npm: str = NPM_CI) -> Sandbox:
box = Sandbox(self)
for name, body in (("node", NODE), ("npm", npm), ("python3", "")):
if name not in missing:
@@ -103,12 +137,32 @@ class Setup(unittest.TestCase):
self.assertEqual(r.stdout.splitlines(), [" npm ci (Excalidraw 0.17.6)", "ok excalidraw 0.17.6 · react 18.3.1"])
self.assertIn("npm ci --silent --no-audit --no-fund", box.logged())
+ def test_a_machine_without_node_uses_harnesss_own_and_the_npm_beside_it(self):
+ box = self.sandbox("node", "npm")
+ node = box.harness_node(npm=NPM_CI)
+ r = box.run("toolchain/setup.sh")
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(r.stdout.splitlines(), [" npm ci (Excalidraw 0.17.6)", "ok excalidraw 0.17.6 · react 18.3.1"])
+ self.assertIn("npm ci --silent --no-audit --no-fund", box.logged())
+ self.assertTrue((box.install / BUNDLE).is_file())
+ self.assertTrue(node.is_file())
+
+ def test_an_old_node_on_path_is_passed_over_for_harnesss(self):
+ box = self.sandbox("node", "npm")
+ box.stub("node", 'exit 1') # a Node 16: the >= 18 probe fails
+ box.harness_node(npm=NPM_CI)
+ r = box.run("toolchain/setup.sh")
+ self.assertEqual((r.returncode, r.stdout.splitlines()[-1]), (0, "ok excalidraw 0.17.6 · react 18.3.1"), r.stderr)
+
def test_each_missing_tool_is_a_miss(self):
- for tool, line in (("node", "miss node >= 18 on PATH"), ("npm", "miss npm on PATH"),
- ("python3", "miss python3 (the scene helper and the verdict)")):
+ for tool, line in (("node", None), ("npm", None), ("python3", "miss python3 (the scene helper and the verdict)")):
with self.subTest(tool=tool):
- r = self.sandbox(tool).run("toolchain/setup.sh")
+ box = self.sandbox(tool)
+ line = line or {"node": f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down",
+ "npm": f"miss npm beside {box.bin}/node"}[tool]
+ r = box.run("toolchain/setup.sh")
self.assertEqual((r.returncode, r.stdout.strip()), (1, line))
+ self.assertNotIn("npm ci --silent --no-audit --no-fund", box.logged())
def test_a_failed_npm_ci_fails_setup(self):
r = self.sandbox(npm="exit 1").run("toolchain/setup.sh")
@@ -146,6 +200,20 @@ class Viewer(unittest.TestCase):
self.assertEqual((r.returncode, r.stdout), (0, "port 4123\n"), r.stderr)
self.assertEqual(box.logged(), [f"node {box.install}/viewer.mjs"])
+ def test_a_login_shell_without_node_runs_harnesss_own(self):
+ box = Sandbox(self)
+ node = box.harness_node()
+ r = box.run("viewer.sh", cwd=box.root, HARNESS_VIEWER_PORT="4123", HARNESS_WORKSPACE=str(box.root))
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(box.logged(), [f"node {box.install}/viewer.mjs"])
+ self.assertTrue(node.is_file())
+
+ def test_no_node_anywhere_never_starts(self):
+ box = Sandbox(self)
+ r = box.run("viewer.sh", cwd=box.root, HARNESS_VIEWER_PORT="4123", HARNESS_WORKSPACE=str(box.root))
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.strip(), f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down")
+
def test_needs_a_port_and_a_workspace(self):
box = Sandbox(self)
box.stub("node")
diff --git a/store/agents/excalidraw/viewer.sh b/store/agents/excalidraw/viewer.sh
index 9a40a19cc..af39f05f3 100755
--- a/store/agents/excalidraw/viewer.sh
+++ b/store/agents/excalidraw/viewer.sh
@@ -1,5 +1,10 @@
#!/usr/bin/env bash
-# The pane. Harness runs this here with HARNESS_VIEWER_PORT and HARNESS_WORKSPACE.
+# The pane. Harness runs this here with HARNESS_VIEWER_PORT and HARNESS_WORKSPACE, through a login
+# shell whose PATH may have no node: runtimes.sh finds this machine's or Harness's own.
set -euo pipefail
: "${HARNESS_VIEWER_PORT:?}"; : "${HARNESS_WORKSPACE:?}"
-exec node "$(cd "$(dirname "$0")" && pwd)/viewer.mjs"
+DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=toolchain/runtimes.sh
+. "$DIR/toolchain/runtimes.sh"
+harness_node 18 || exit 1
+exec node "$DIR/viewer.mjs"
diff --git a/store/agents/manim/README.md b/store/agents/manim/README.md
index 0d924cb36..fcd7521da 100644
--- a/store/agents/manim/README.md
+++ b/store/agents/manim/README.md
@@ -6,8 +6,10 @@ animate in the Video Viewer pane as the scenes render. Runs on Claude Code.
- `harness.json` — engine, template, skill, toolchain, `viewer.use: autonomous/video-viewer`.
- `skills/manim/` — the Manim skill (ours): the library, the commands, the rules of a good scene.
-- `toolchain/setup.sh` makes one venv with the pinned Manim (`MANIM_VERSION`); `doctor.sh` checks
- ffmpeg and LaTeX; `init-workspace.sh` renders the starter; `verdict.py` judges the newest render.
+- `toolchain/setup.sh` makes one environment with the pinned Manim (`MANIM_VERSION`): conda-forge's
+ Python, pycairo and manimpango, Manim from PyPI on top, no Homebrew and no ffmpeg binary (PyAV encodes);
+ `doctor.sh` checks it and LaTeX; `init-workspace.sh` renders the starter; `verdict.py` judges the
+ newest render.
- `toolchain/render.py` is how the agent renders: `manim render` with `--media_dir out
--save_sections`, plus `.harness/render.json` while it runs (scene, animation n of ~m, section,
clips so far, the error and its line on failure) and `.harness/renders/<video>.json` when a scene
diff --git a/store/agents/manim/toolchain/doctor.sh b/store/agents/manim/toolchain/doctor.sh
index 5e1af3c20..38328d91e 100755
--- a/store/agents/manim/toolchain/doctor.sh
+++ b/store/agents/manim/toolchain/doctor.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -u; cd "$(dirname "$0")/.."; bad=0
-if [ -x .venv/bin/manim ]; then echo "ok manim $(cat MANIM_VERSION)"; else echo "miss .venv/bin/manim — run toolchain/setup.sh"; bad=1; fi
-if [ "$(uname -s)" = Darwin ] && ! (command -v pkg-config >/dev/null 2>&1 && pkg-config --exists cairo 2>/dev/null); then echo "miss cairo and pkg-config (brew install cairo pkgconf)"; bad=1; fi
-if command -v ffmpeg >/dev/null 2>&1; then echo "ok ffmpeg"; else echo "miss ffmpeg on PATH (brew install ffmpeg)"; bad=1; fi
-if command -v latex >/dev/null 2>&1; then echo "ok latex (Tex/MathTex available)"; else echo "warn latex not on PATH — Tex/MathTex scenes need it (brew install --cask mactex-no-gui); Text() works without"; fi
+# The version pip recorded, and a pycairo that loads: quick, without importing Manim itself.
+v="$(.venv/bin/python -c 'import cairo, importlib.metadata as m; print(m.version("manim"))' 2>/dev/null)"
+if [ -n "$v" ]; then echo "ok manim $v"; else echo "miss manim with pycairo in .venv — run toolchain/setup.sh"; bad=1; fi
+if command -v latex >/dev/null 2>&1; then echo "ok latex (Tex/MathTex available)"; else echo "warn latex not on PATH — Tex/MathTex scenes need a TeX distribution (MacTeX, TeX Live); Text() works without"; fi
exit $bad
diff --git a/store/agents/manim/toolchain/render.py b/store/agents/manim/toolchain/render.py
index 080655395..1b4a313a0 100644
--- a/store/agents/manim/toolchain/render.py
+++ b/store/agents/manim/toolchain/render.py
@@ -73,10 +73,25 @@ def with_manim_args(argv: list[str]) -> list[str]:
return ["render", *args]
+# What Tex and MathTex run. LaTeX is not part of the toolchain (it is gigabytes), so on most machines a
+# formula scene fails on the first of these; the doctor warns about it.
+TEX_PROGRAMS = {"latex", "pdflatex", "xelatex", "lualatex", "dvisvgm"}
+
+
+def missing_tex(exc: BaseException | None) -> str | None:
+ """The TeX program a Tex/MathTex scene tried to run and this machine does not have."""
+ if isinstance(exc, FileNotFoundError) and os.path.basename(str(exc.filename or "")) in TEX_PROGRAMS:
+ return os.path.basename(str(exc.filename))
+ return None
+
+
def error_from(exc: BaseException | None, tb) -> dict:
if exc is None:
return {"type": "Error", "message": "manim exited with an error"}
out = {"type": type(exc).__name__, "message": str(exc).strip().splitlines()[0] if str(exc).strip() else type(exc).__name__}
+ tex = missing_tex(exc)
+ if tex:
+ out["message"] = f"Tex/MathTex need LaTeX and `{tex}` is not on this machine — write the formula with Text(…) instead"
if isinstance(exc, SyntaxError) and exc.filename:
out.update(file=rel(exc.filename), line=exc.lineno, code=(exc.text or "").strip() or None)
return out
@@ -356,6 +371,12 @@ def install_hooks(status: Status) -> None:
def print_exception(*args, **kwargs):
exc_type, exc, tb = sys.exc_info()
safely(status.fail, exc, tb, isinstance(exc, KeyboardInterrupt))
+ if missing_tex(exc):
+ # One line that says what to do, not a screen of subprocess frames ending in "No such file".
+ error = error_from(exc, tb)
+ where = f"{error['file']}:{error['line']}: " if error.get("file") and error.get("line") else ""
+ print(f"{where}{error['message']}", file=sys.stderr)
+ return None
return orig_print_exception(*args, **kwargs)
console.print_exception = print_exception
diff --git a/store/agents/manim/toolchain/runtimes.sh b/store/agents/manim/toolchain/runtimes.sh
new file mode 100644
index 000000000..4e21942d0
--- /dev/null
+++ b/store/agents/manim/toolchain/runtimes.sh
@@ -0,0 +1,187 @@
+# shellcheck shell=bash
+# runtimes.sh — the interpreters a store package runs on, found on this machine or fetched, never
+# asked of the person who pressed Get.
+#
+# Sourced (not run) by a package's setup, doctor and viewer scripts. A new Mac has Apple's Python 3.9,
+# no Homebrew and, for most people, no Node on PATH; a package that answers "brew install [email protected]"
+# has failed its install. So:
+#
+# harness_node 18 node >= 18 on PATH, with npm: the machine's own, else the Node Harness
+# itself runs on (~/.harness/runtime/current-node, laid down with the CLI)
+# harness_uv uv on PATH: the machine's own, else a pinned, checksummed release
+# fetched into ~/.harness/runtime
+# harness_venv DIR 3.12 [MIN [BELOW]]
+# DIR is a venv on CPython 3.12, which uv downloads when the machine
+# has none. A venv already there on a Python in [MIN, BELOW) is kept.
+# harness_pip DIR PKG… install into that venv, through uv
+# harness_micromamba micromamba, pinned and checksummed, for native libraries PyPI has
+# no wheel for (pycairo on macOS)
+# harness_conda_env DIR SPEC… DIR is a conda-forge environment with those packages
+#
+# Every function prints `miss …` and returns 1 when it cannot deliver, so `harness_node 18 || exit 1`
+# reads like the rest of a setup script. Nothing here touches the user's shell profile or anything
+# outside ~/.harness/runtime and the package's own directory.
+#
+# ONE copy is written by hand: store/tools/runtimes.sh in the OpenHarness repository. Each package
+# carries an identical copy (a package installs alone, so it cannot reach this one);
+# `node store/tools/sync-runtimes.mjs` rewrites them and the CLI's store spec fails when one drifts.
+
+HARNESS_RUNTIME="${ADAPTER_RUNTIME_DIR:-${HOME:-}/.harness/runtime}"
+# The Pythons uv downloads live with Harness's other runtimes, not in uv's own ~/.local/share/uv:
+# a package's venv points at its interpreter, and `uv python uninstall` there must not break it.
+UV_PYTHON_INSTALL_DIR="${UV_PYTHON_INSTALL_DIR:-$HARNESS_RUNTIME/python}"
+export UV_PYTHON_INSTALL_DIR
+HARNESS_UV_VERSION=0.12.15
+HARNESS_MICROMAMBA_VERSION=2.9.0-0
+
+_harness_platform() {
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64) echo darwin-arm64 ;;
+ Darwin-x86_64) echo darwin-x64 ;;
+ Linux-x86_64) echo linux-x64 ;;
+ Linux-aarch64 | Linux-arm64) echo linux-arm64 ;;
+ *) return 1 ;;
+ esac
+}
+
+_harness_sha256() {
+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1
+ else sha256sum "$1" | cut -d' ' -f1; fi
+}
+
+# _harness_fetch URL DEST SHA256 — DEST exists only if what arrived matches the pinned checksum.
+_harness_fetch() {
+ local part="$2.part.$$"
+ if ! curl -fsSL --retry 3 --connect-timeout 20 --max-time 900 -o "$part" "$1"; then
+ rm -f "$part"; echo "miss could not download $1 — check this machine's internet connection"; return 1
+ fi
+ if [ "$(_harness_sha256 "$part")" != "$3" ]; then
+ rm -f "$part"; echo "miss $1 did not match its pinned checksum; nothing was installed"; return 1
+ fi
+ mv -f "$part" "$2"
+}
+
+_harness_node_at_least() {
+ command -v node >/dev/null 2>&1 || return 1
+ node -e '
+ const [a, b = 0] = process.argv[1].split(".").map(Number)
+ const [x, y] = process.versions.node.split(".").map(Number)
+ process.exit(x > a || (x === a && y >= b) ? 0 : 1)
+ ' "$1" >/dev/null 2>&1
+}
+
+harness_node() {
+ local min="${1:-18}" recorded saved="$PATH" own=0
+ _harness_node_at_least "$min" && own=1
+ # A node without npm beside it (a distro's nodejs package) cannot run a setup's `npm ci`.
+ if [ "$own" = 1 ] && command -v npm >/dev/null 2>&1; then return 0; fi
+ recorded="$(cat "$HARNESS_RUNTIME/current-node" 2>/dev/null || true)"
+ if [ -n "$recorded" ] && [ -x "${recorded%/*}/node" ]; then
+ PATH="${recorded%/*}:$PATH"; export PATH; hash -r 2>/dev/null || true
+ _harness_node_at_least "$min" && return 0
+ PATH="$saved"; export PATH; hash -r 2>/dev/null || true
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min — this machine's newest is $("${recorded%/*}/node" -v), Harness's own; update Harness"
+ return 1
+ fi
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min, and Harness's own Node is not in $HARNESS_RUNTIME — run \`harness start\` once to lay it down"
+ return 1
+}
+
+harness_uv() {
+ local dir asset sum tmp
+ command -v uv >/dev/null 2>&1 && return 0
+ for dir in "${HOME:-}/.local/bin" "${HOME:-}/.cargo/bin"; do
+ [ -n "${HOME:-}" ] || break
+ if [ -x "$dir/uv" ]; then PATH="$dir:$PATH"; export PATH; return 0; fi
+ done
+ dir="$HARNESS_RUNTIME/uv-$HARNESS_UV_VERSION"
+ if [ ! -x "$dir/uv" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=uv-aarch64-apple-darwin sum=dc304b9ed1b24174572290fba60ac3f6fe63c73a671f0439e62a91375841964d ;;
+ darwin-x64) asset=uv-x86_64-apple-darwin sum=e9ca61775532368fe518ab03e7a354c7ecab8ccb3c7d941c775fcc4a362b801b ;;
+ linux-x64) asset=uv-x86_64-unknown-linux-gnu sum=f97935763c04be3e692460a7aaeaaab8fc3b78fcf8b389da820b38ae7423a638 ;;
+ linux-arm64) asset=uv-aarch64-unknown-linux-gnu sum=0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08 ;;
+ *) echo "miss uv — there is no uv release for $(uname -s) $(uname -m); install it from https://docs.astral.sh/uv/"; return 1 ;;
+ esac
+ echo " fetching uv $HARNESS_UV_VERSION into $HARNESS_RUNTIME (it brings Python and the packages)"
+ mkdir -p "$HARNESS_RUNTIME" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ tmp="$(mktemp -d "$HARNESS_RUNTIME/.uv.XXXXXX")" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ if ! _harness_fetch "https://github.com/astral-sh/uv/releases/download/$HARNESS_UV_VERSION/$asset.tar.gz" "$tmp/uv.tar.gz" "$sum"; then
+ rm -rf "$tmp"; return 1
+ fi
+ tar -xzf "$tmp/uv.tar.gz" -C "$tmp" && chmod +x "$tmp/$asset/uv" "$tmp/$asset/uvx" \
+ || { rm -rf "$tmp"; echo "miss the uv archive would not unpack"; return 1; }
+ # Two installs at once both get here; whichever renames first wins and the other's copy goes.
+ # (mv onto a directory that exists would nest the copy inside it, hence the test.)
+ [ -e "$dir" ] || mv "$tmp/$asset" "$dir" 2>/dev/null || true
+ rm -rf "$tmp"
+ [ -x "$dir/uv" ] || { echo "miss uv did not land in $dir"; return 1; }
+ fi
+ PATH="$dir:$PATH"; export PATH
+}
+
+# harness_venv DIR VERSION [MIN [BELOW]] — MIN defaults to VERSION; BELOW is exclusive and optional.
+harness_venv() {
+ local dir="${1:?harness_venv needs a directory}" want="${2:?harness_venv needs a Python version}"
+ local min="${3:-$2}" below="${4:-}"
+ if [ -x "$dir/bin/python" ] && "$dir/bin/python" -c '
+import sys
+at = lambda s: tuple(int(p) for p in s.split("."))
+v = sys.version_info[:2]
+sys.exit(0 if v >= at(sys.argv[1]) and (not sys.argv[2] or v < at(sys.argv[2])) else 1)
+' "$min" "$below" 2>/dev/null; then
+ echo "ok $("$dir/bin/python" --version) in $dir"
+ return 0
+ fi
+ harness_uv || return 1
+ echo " python $want in $dir (uv downloads it when this machine has none)"
+ rm -rf "$dir"
+ # only-managed: the same python-build-standalone interpreter on every machine, so a package tested
+ # once is tested everywhere — not whichever Homebrew or pyenv Python happened to be first on PATH.
+ uv venv --quiet --seed --python "$want" --python-preference only-managed "$dir" \
+ || { echo "miss could not make a Python $want environment in $dir"; return 1; }
+ echo "ok $("$dir/bin/python" --version) in $dir"
+}
+
+harness_pip() {
+ local dir="${1:?harness_pip needs a venv}"
+ shift
+ harness_uv || return 1
+ uv pip install --quiet --python "$dir/bin/python" "$@"
+}
+
+harness_micromamba() {
+ local dir asset sum
+ dir="$HARNESS_RUNTIME/micromamba-$HARNESS_MICROMAMBA_VERSION"
+ if [ ! -x "$dir/micromamba" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=micromamba-osx-arm64 sum=ec2a072f028e1a7cf20f3e2e74d5a8127cf5a5f27636375b5359811565f4e5be ;;
+ darwin-x64) asset=micromamba-osx-64 sum=1e71054bb3ac9a076e21f7ec48acfef536f9b3f1408f371a942784bf5ef83d8a ;;
+ linux-x64) asset=micromamba-linux-64 sum=366cd9cd8be14df1ab8ed50352a82111082a36686b2d389fdb79a92c3fafb3e3 ;;
+ linux-arm64) asset=micromamba-linux-aarch64 sum=9f93b974adcb4d166996af969b6cd371287d1a3e52733704727884d9b74cb7a7 ;;
+ *) echo "miss micromamba — there is no release for $(uname -s) $(uname -m)"; return 1 ;;
+ esac
+ echo " fetching micromamba $HARNESS_MICROMAMBA_VERSION into $HARNESS_RUNTIME (native libraries PyPI has no wheel for)"
+ mkdir -p "$dir" || { echo "miss cannot write $dir"; return 1; }
+ _harness_fetch "https://github.com/mamba-org/micromamba-releases/releases/download/$HARNESS_MICROMAMBA_VERSION/$asset" "$dir/micromamba.$$" "$sum" || return 1
+ chmod +x "$dir/micromamba.$$" && mv -f "$dir/micromamba.$$" "$dir/micromamba" \
+ || { rm -f "$dir/micromamba.$$"; echo "miss micromamba would not install into $dir"; return 1; }
+ fi
+ HARNESS_MICROMAMBA="$dir/micromamba"
+ export HARNESS_MICROMAMBA
+}
+
+# harness_conda_env DIR SPEC… — conda-forge only, packages cached under ~/.harness/runtime/mamba.
+harness_conda_env() {
+ local dir="${1:?harness_conda_env needs a directory}"
+ shift
+ # micromamba takes a relative --prefix as a name under its root prefix and "succeeds" there.
+ case "$dir" in /*) ;; *) dir="$PWD/$dir" ;; esac
+ harness_micromamba || return 1
+ MAMBA_ROOT_PREFIX="$HARNESS_RUNTIME/mamba" "$HARNESS_MICROMAMBA" create --yes --quiet \
+ --prefix "$dir" --override-channels --channel conda-forge "$@" >/dev/null \
+ && [ -d "$dir/conda-meta" ] \
+ || { echo "miss could not make the conda-forge environment in $dir ($*)"; return 1; }
+}
diff --git a/store/agents/manim/toolchain/setup.sh b/store/agents/manim/toolchain/setup.sh
index ec8cde6f6..827bcf5cd 100755
--- a/store/agents/manim/toolchain/setup.sh
+++ b/store/agents/manim/toolchain/setup.sh
@@ -1,24 +1,29 @@
#!/usr/bin/env bash
-# Runs once at install, cwd = the install dir. One venv with the pinned Manim Community release.
-# ffmpeg must be on the machine (brew install ffmpeg); LaTeX is optional (only for Tex/MathTex).
+# Runs once at install, cwd = the install dir. One environment in .venv with the pinned Manim Community
+# release. pycairo has no wheel on PyPI for macOS or Linux, and manimpango none for Linux: both build
+# from source against a cairo and a pango this machine may not have. So the environment is conda-forge's
+# Python with its pycairo and manimpango (the libraries inside it), and Manim goes in on top from PyPI,
+# which leaves those two as they are. No ffmpeg binary is needed: PyAV carries its own FFmpeg libraries.
+# LaTeX is optional (only for Tex/MathTex).
set -euo pipefail
cd "$(dirname "$0")/.."
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
VERSION="$(cat MANIM_VERSION)"
-# 3.12 first: it is the interpreter the most wheels exist for; pycairo still builds from source on
-# every version, which is what cairo + pkg-config below are for.
-PY=""; for c in python3.12 python3.11 python3.13 python3; do if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' 2>/dev/null; then PY="$c"; break; fi; done
-[ -n "$PY" ] || { echo "miss python 3.11+ (brew install [email protected])"; exit 1; }
-echo "ok $($PY --version)"
-if [ "$(uname -s)" = Darwin ]; then
- need=""
- command -v pkg-config >/dev/null 2>&1 || need="$need pkgconf"
- pkg-config --exists cairo 2>/dev/null || need="$need cairo"
- if [ -n "$need" ]; then echo "miss pycairo builds from source and needs:$need — run: brew install$need"; exit 1; fi
- echo "ok cairo $(pkg-config --modversion cairo) (manimpango ships its own pango)"
+# An environment that already has this Manim and a pycairo that loads is kept, however it was made
+# (a venv on Homebrew's cairo included): nothing to fetch.
+if .venv/bin/python -c 'import sys, cairo, manim; sys.exit(manim.__version__ != sys.argv[1])' "$VERSION" >/dev/null 2>&1; then
+ echo "ok manim ${VERSION} already in .venv"
+ exit 0
fi
-[ -x .venv/bin/python ] || "$PY" -m venv .venv
-.venv/bin/python -m pip install --quiet --upgrade pip >/dev/null 2>&1 || true
+# Manim needs Python 3.11+. A .venv on an older one, or whose pycairo no longer loads (its
+# cairo uninstalled), is made again.
+if ! .venv/bin/python -c 'import sys, cairo; sys.exit(sys.version_info < (3, 11))' >/dev/null 2>&1; then
+ echo " python 3.12, pycairo and manimpango from conda-forge into .venv (a few minutes the first time)"
+ rm -rf .venv
+ harness_conda_env .venv python=3.12 pycairo manimpango pip || exit 1
+fi
+echo "ok $(.venv/bin/python --version) with pycairo in .venv"
echo " installing manim ${VERSION} (a couple of minutes the first time)"
-.venv/bin/python -m pip install --quiet "manim==${VERSION}"
+harness_pip .venv "manim==${VERSION}" || { echo "miss manim ${VERSION} would not install into .venv"; exit 1; }
echo "ok manim $(.venv/bin/manim --version 2>/dev/null | head -1)"
-command -v ffmpeg >/dev/null 2>&1 && echo "ok ffmpeg $(ffmpeg -version 2>/dev/null | head -1 | cut -d' ' -f3)" || echo "warn ffmpeg not on PATH — renders need it: brew install ffmpeg"
diff --git a/store/agents/manim/toolchain/test_render.py b/store/agents/manim/toolchain/test_render.py
index f1892a5d8..d197fd91f 100644
--- a/store/agents/manim/toolchain/test_render.py
+++ b/store/agents/manim/toolchain/test_render.py
@@ -4,7 +4,7 @@ Python), and one real render, a failing scene and a syntax error on the pinned M
python3 -m unittest toolchain/test_render.py # the real renders skip
"$MANIM_PYTHON" -m unittest toolchain/test_render.py
"""
-import contextlib, importlib.util, io, json, os, runpy, subprocess, sys, tempfile, types, unittest
+import contextlib, importlib.util, io, json, os, runpy, shutil, subprocess, sys, tempfile, types, unittest
from pathlib import Path
from unittest import mock
sys.path.insert(0, str(Path(__file__).parent))
@@ -152,6 +152,23 @@ class Errors(Workspace):
error = render.error_from(exc, exc.__traceback__)
self.assertEqual(error, {"type": "ValueError", "message": "x"})
+ def test_a_tex_program_this_machine_lacks_says_what_to_do(self):
+ scene = self.put("scenes/formula.py", "def construct():\n raise FileNotFoundError(2, 'No such file or directory', 'latex')\n")
+ try:
+ scene["construct"]()
+ except FileNotFoundError as exc:
+ error = render.error_from(exc, exc.__traceback__)
+ self.assertEqual(error, {"type": "FileNotFoundError", "message": "Tex/MathTex need LaTeX and `latex` is not on this machine — write the formula with Text(…) instead",
+ "file": "scenes/formula.py", "line": 2, "code": "raise FileNotFoundError(2, 'No such file or directory', 'latex')", "function": "construct"})
+ self.assertEqual(render.missing_tex(FileNotFoundError(2, "No such file or directory", "/Library/TeX/texbin/dvisvgm")), "dvisvgm")
+
+ def test_any_other_missing_file_is_itself(self):
+ for exc in (FileNotFoundError(2, "No such file or directory", "assets/logo.png"), FileNotFoundError("no filename"), PermissionError(13, "Permission denied", "latex")):
+ with self.subTest(exc=exc):
+ self.assertIsNone(render.missing_tex(exc))
+ self.assertNotIn("LaTeX", render.error_from(exc, None)["message"])
+ self.assertIsNone(render.missing_tex(None))
+
class Labels(unittest.TestCase):
def test_labels_are_short(self):
@@ -434,6 +451,25 @@ class Hooks(unittest.TestCase):
args = self.status.fail.call_args.args
self.assertEqual((args[0], args[2]), (error, cancelled))
+ def test_a_missing_latex_is_one_line_not_a_traceback(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ ws = Path(tmp).resolve()
+ source = ws / "scenes" / "formula.py"
+ source.parent.mkdir()
+ source.write_text("def construct():\n raise FileNotFoundError(2, 'No such file or directory', 'latex')\n")
+ namespace: dict = {}
+ exec(compile(source.read_text(), str(source), "exec"), namespace)
+ for where, construct, line in ((ws, namespace["construct"], "scenes/formula.py:2: "), (ws / "elsewhere", namespace["construct"], "")):
+ with self.subTest(where=line or "no workspace frame"), mock.patch.object(render, "WS", where):
+ err = io.StringIO()
+ try:
+ construct()
+ except FileNotFoundError as exc:
+ with contextlib.redirect_stderr(err):
+ self.assertIsNone(self.console.print_exception(show_locals=False), "Manim's traceback is not printed")
+ self.assertIs(self.status.fail.call_args.args[0], exc)
+ self.assertEqual(err.getvalue(), f"{line}Tex/MathTex need LaTeX and `latex` is not on this machine — write the formula with Text(…) instead\n")
+
def test_a_status_that_raises_never_stops_the_render(self):
self.status.begin.side_effect = self.status.finish.side_effect = self.status.sync.side_effect = RuntimeError("feed broke")
self.assertEqual(self.Scene().render(), "rendered")
@@ -555,6 +591,13 @@ class Broken(Scene):
def construct(self):
self.play(FadeIn(Sqaure()))
'''
+FORMULA = '''from manim import *
+
+
+class Formula(Scene):
+ def construct(self):
+ self.play(Write(MathTex(r"e^{i\\pi} + 1 = 0")))
+'''
SYNTAX = '''from manim import *
class Bad(Scene:
pass
@@ -569,7 +612,7 @@ class RealManim(Workspace):
def setUp(self):
super().setUp()
- for name, source in (("tiny", TINY), ("broken", BROKEN), ("syntax", SYNTAX)):
+ for name, source in (("tiny", TINY), ("broken", BROKEN), ("syntax", SYNTAX), ("formula", FORMULA)):
(self.ws / "scenes").mkdir(exist_ok=True)
(self.ws / "scenes" / f"{name}.py").write_text(source)
self.unhook = self.hooks_restorer()
@@ -641,5 +684,12 @@ class RealManim(Workspace):
self.assertEqual((feed["state"], feed["scene"], feed["output"]), ("failed", "Bad", "out/videos/syntax/480p15/Bad.mp4"))
self.assertEqual({k: feed["error"][k] for k in ("type", "file", "line", "code")}, {"type": "SyntaxError", "file": "scenes/syntax.py", "line": 2, "code": "class Bad(Scene:"})
+ @unittest.skipIf(shutil.which("latex"), "LaTeX is on this machine")
+ def test_a_formula_without_latex_is_one_clear_line(self):
+ code, err = self.render("formula.py", "Formula")
+ self.assertEqual(code, 1)
+ self.assertEqual(err, "scenes/formula.py:6: Tex/MathTex need LaTeX and `latex` is not on this machine — write the formula with Text(…) instead\n")
+ self.assertEqual({k: self.status_file()["error"][k] for k in ("type", "file", "line")}, {"type": "FileNotFoundError", "file": "scenes/formula.py", "line": 6})
+
if __name__ == "__main__": unittest.main()
diff --git a/store/agents/manim/toolchain/test_scripts.py b/store/agents/manim/toolchain/test_scripts.py
index 0b54ec6ec..d24e0e5c2 100644
--- a/store/agents/manim/toolchain/test_scripts.py
+++ b/store/agents/manim/toolchain/test_scripts.py
@@ -1,6 +1,8 @@
"""setup.sh, doctor.sh and init-workspace.sh, run for real against a scratch install whose PATH holds
only stub commands (and the few coreutils the scripts use), so every ok / miss / warn line is reached
-without a network, a Python venv or Manim on the machine:
+without a network, a conda-forge environment or Manim on the machine. uv and micromamba are stubs too
+(micromamba laid where runtimes.sh keeps it): what runtimes.sh does when they are not there yet is
+store/tools/test_runtimes.py's business, apart from the one miss a setup with no network ends on.
python3 -m unittest toolchain/test_scripts.py
"""
@@ -11,18 +13,32 @@ PACKAGE = Path(__file__).resolve().parent.parent
# A bash line tracer (BASH_ENV sourcing a DEBUG trap that appends to SHCOV_OUT) is passed through when set.
TRACER = {k: os.environ[k] for k in ("BASH_ENV", "SHCOV_OUT") if k in os.environ}
BASH = "/bin/bash"
-COREUTILS = ("dirname", "cat", "mkdir", "cp", "head", "cut")
+COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "head", "cut", "chmod", "mv")
VERSION = (PACKAGE / "MANIM_VERSION").read_text().strip()
+MICROMAMBA = "micromamba-2.9.0-0" # runtimes.sh's pinned release, where it keeps it
+
+# The environment's python. What the scripts ask it: its version (VENV_PY); whether this Manim is in
+# it with a pycairo that loads (the keep probe: Manim is there when uv's stub laid bin/manim for that
+# version); whether pycairo loads on a new enough Python (CAIRO_EXIT); the version pip recorded.
+VENV_PYTHON = """here="$(dirname "$0")"
+case "$1" in
+ --version) echo "Python ${VENV_PY:-3.12.14}" ;;
+ -c) [ "${CAIRO_EXIT:-0}" = 0 ] || exit "$CAIRO_EXIT"
+ case "$2" in
+ *manim.__version__*) [ "$("$here/manim" --version 2>/dev/null)" = "Manim Community v$3" ] ;;
+ *importlib.metadata*) [ -x "$here/manim" ] && "$here/manim" --version | cut -dv -f2 ;;
+ esac ;;
+esac"""
class Sandbox:
- """An install dir with the package's scripts, a bin/ of stubs as the whole PATH, and a log of
- every stub call."""
+ """An install dir with the package's scripts, a bin/ of stubs as the whole PATH, a runtime dir
+ for runtimes.sh, and a log of every stub call."""
def __init__(self, test: unittest.TestCase):
tmp = tempfile.TemporaryDirectory()
test.addCleanup(tmp.cleanup)
- self.root = Path(tmp.name)
+ self.root = Path(tmp.name).resolve()
self.install = self.root / "install"
(self.install / "toolchain").mkdir(parents=True)
for script in PACKAGE.glob("toolchain/*.sh"):
@@ -30,6 +46,8 @@ class Sandbox:
shutil.copy(PACKAGE / "MANIM_VERSION", self.install / "MANIM_VERSION")
self.bin = self.root / "bin"
self.bin.mkdir()
+ self.runtime = self.root / "runtime"
+ self.runtime.mkdir()
self.templates = self.root / "templates"
self.calls = self.root / "calls.log"
self.calls.touch()
@@ -40,129 +58,148 @@ class Sandbox:
def stub(self, name: str, body: str = "", where: Path | None = None) -> Path:
path = (where or self.bin) / name
path.parent.mkdir(parents=True, exist_ok=True)
+ path.unlink(missing_ok=True) # never write through a link to a real command
path.write_text(f'#!/bin/bash\necho "{name} $*" >> "$CALLS"\n{body}\n')
path.chmod(0o755)
return path
def run(self, script: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
return subprocess.run([BASH, str(self.install / "toolchain" / script)], cwd=cwd or self.install,
- env={"PATH": str(self.bin), "CALLS": str(self.calls), **TRACER, **env},
+ env={"PATH": str(self.bin), "CALLS": str(self.calls), "ADAPTER_RUNTIME_DIR": str(self.runtime), **TRACER, **env},
capture_output=True, text=True, timeout=60)
def logged(self) -> list[str]:
return self.calls.read_text().splitlines()
# The machine's pieces, as the scripts ask about them.
- def uname(self, system: str) -> None:
- self.stub("uname", f"echo {system}")
-
- def pkg_config(self, cairo: bool = True) -> None:
- self.stub("pkg-config", f'[ "$1" = --modversion ] && {{ echo 1.18.4; exit 0; }}; exit {0 if cairo else 1}')
-
- def ffmpeg(self) -> None:
- self.stub("ffmpeg", 'echo "ffmpeg version 7.1.1 Copyright (c) the FFmpeg developers"')
-
- def venv(self, where: Path) -> None:
- """A .venv whose python runs pip (failing when $PIP_EXIT says so) and whose manim has a version."""
- self.stub("python", '[ "$*" = "-m pip install --quiet --upgrade pip" ] && exit 1\n[ "$2" = pip ] && exit "${PIP_EXIT:-0}"\nexit 0', where=where / ".venv" / "bin")
- self.stub("manim", f'echo "Manim Community v{VERSION}"', where=where / ".venv" / "bin")
+ def venv(self, manim: str | None = VERSION) -> None:
+ """.venv as a conda environment leaves it, with Manim `manim` in it (None: not yet)."""
+ self.stub("python", VENV_PYTHON, where=self.install / ".venv" / "bin")
+ (self.install / ".venv" / "conda-meta").mkdir(exist_ok=True)
+ if manim:
+ self.stub("manim", f'echo "Manim Community v{manim}"', where=self.install / ".venv" / "bin")
+
+ def micromamba(self) -> None:
+ """micromamba already in the runtime dir: `create --prefix DIR` makes DIR an environment with a
+ python, or fails with $MAMBA_EXIT."""
+ python = self.stub("python", VENV_PYTHON, where=self.templates)
+ self.stub("micromamba", f"""[ "${{MAMBA_EXIT:-0}}" = 0 ] || exit "$MAMBA_EXIT"
+while [ $# -gt 0 ]; do [ "$1" = --prefix ] && prefix="$2"; shift; done
+mkdir -p "$prefix/conda-meta" "$prefix/bin" && cp "{python}" "$prefix/bin/python"
+""", where=self.runtime / MICROMAMBA)
+
+ def uv(self) -> None:
+ """uv on PATH: `uv pip install --python PY manim==V` lays PY's bin/manim for V, or fails with $PIP_EXIT."""
+ self.stub("uv", """[ "${PIP_EXIT:-0}" = 0 ] || exit "$PIP_EXIT"
+for arg in "$@"; do case "$arg" in manim==*) v="${arg#manim==}" ;; esac; done
+printf '#!/bin/bash\\necho "Manim Community v%s"\\n' "$v" > .venv/bin/manim && chmod +x .venv/bin/manim""")
class Doctor(unittest.TestCase):
- def test_everything_there_on_a_mac(self):
+ def test_everything_there(self):
box = Sandbox(self)
- box.uname("Darwin"); box.pkg_config(); box.ffmpeg(); box.stub("latex")
- box.venv(box.install)
+ box.venv(); box.stub("latex")
r = box.run("doctor.sh")
self.assertEqual(r.returncode, 0, r.stderr)
- self.assertEqual(r.stdout.splitlines(), [f"ok manim {VERSION}", "ok ffmpeg", "ok latex (Tex/MathTex available)"])
+ self.assertEqual(r.stdout.splitlines(), [f"ok manim {VERSION}", "ok latex (Tex/MathTex available)"])
- def test_nothing_there_on_linux_misses_and_warns(self):
+ def test_no_ffmpeg_and_no_latex_is_still_a_working_install(self):
box = Sandbox(self)
- box.uname("Linux") # cairo is not checked off a Mac
+ box.venv()
r = box.run("doctor.sh")
- self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.returncode, 0, r.stderr)
self.assertEqual(r.stdout.splitlines(), [
- "miss .venv/bin/manim — run toolchain/setup.sh",
- "miss ffmpeg on PATH (brew install ffmpeg)",
- "warn latex not on PATH — Tex/MathTex scenes need it (brew install --cask mactex-no-gui); Text() works without",
+ f"ok manim {VERSION}",
+ "warn latex not on PATH — Tex/MathTex scenes need a TeX distribution (MacTeX, TeX Live); Text() works without",
])
- def test_a_mac_without_pkg_config_or_without_cairo(self):
- for label, prepare in (("no pkg-config", lambda box: None), ("no cairo", lambda box: box.pkg_config(cairo=False))):
+ def test_no_environment_or_one_whose_pycairo_does_not_load_is_a_miss(self):
+ for label, prepare, env in (("no .venv", lambda box: None, {}), ("no manim", lambda box: box.venv(manim=None), {}),
+ ("pycairo broken", lambda box: box.venv(), {"CAIRO_EXIT": "1"})):
with self.subTest(label):
box = Sandbox(self)
- box.uname("Darwin"); box.ffmpeg(); box.venv(box.install); prepare(box)
- r = box.run("doctor.sh")
+ prepare(box)
+ r = box.run("doctor.sh", **env)
self.assertEqual(r.returncode, 1)
- self.assertIn("miss cairo and pkg-config (brew install cairo pkgconf)", r.stdout.splitlines())
-
- def test_a_missing_ffmpeg_alone_fails_the_doctor(self):
- box = Sandbox(self)
- box.uname("Linux"); box.venv(box.install); box.stub("latex")
- r = box.run("doctor.sh")
- self.assertEqual((r.returncode, r.stdout.splitlines()[1]), (1, "miss ffmpeg on PATH (brew install ffmpeg)"))
+ self.assertEqual(r.stdout.splitlines()[0], "miss manim with pycairo in .venv — run toolchain/setup.sh")
class Setup(unittest.TestCase):
- def sandbox(self, system: str = "Linux", pythons: dict[str, bool] | None = None) -> Sandbox:
- """`pythons`: which interpreters exist and whether each is new enough."""
+ MAMBA_CREATE = "micromamba create --yes --quiet --prefix {install}/.venv --override-channels --channel conda-forge python=3.12 pycairo manimpango pip"
+ PIP = f"uv pip install --quiet --python .venv/bin/python manim=={VERSION}"
+
+ def sandbox(self) -> Sandbox:
box = Sandbox(self)
- box.uname(system)
- box.venv(box.templates)
- for name, new_enough in (pythons if pythons is not None else {"python3.12": True}).items():
- box.stub(name, f'case "$1" in\n -c) exit {0 if new_enough else 1} ;;\n --version) echo "Python 3.x ({name})" ;;\n'
- f' -m) mkdir -p .venv && cp -R "{box.templates}/.venv/bin" .venv/bin ;;\nesac')
+ box.micromamba()
+ box.uv()
return box
- def test_installs_manim_into_a_venv(self):
+ def test_a_new_machine_gets_a_conda_forge_python_with_pycairo_then_manim(self):
box = self.sandbox()
- box.ffmpeg()
r = box.run("setup.sh")
self.assertEqual(r.returncode, 0, r.stderr)
self.assertEqual(r.stdout.splitlines(), [
- "ok Python 3.x (python3.12)",
+ " python 3.12, pycairo and manimpango from conda-forge into .venv (a few minutes the first time)",
+ "ok Python 3.12.14 with pycairo in .venv",
f" installing manim {VERSION} (a couple of minutes the first time)",
f"ok manim Manim Community v{VERSION}",
- "ok ffmpeg 7.1.1",
])
- self.assertIn("python3.12 -m venv .venv", box.logged())
- self.assertIn(f"python -m pip install --quiet manim=={VERSION}", box.logged())
+ log = box.logged()
+ self.assertIn(self.MAMBA_CREATE.format(install=box.install), log)
+ self.assertIn(self.PIP, log)
- def test_the_first_new_enough_python_wins_and_an_existing_venv_is_kept(self):
- box = self.sandbox(pythons={"python3.11": False, "python3.13": True, "python3": True})
- box.venv(box.install)
+ def test_an_environment_that_already_has_this_manim_is_kept_without_a_download(self):
+ box = self.sandbox()
+ box.venv()
+ (box.install / ".venv" / "conda-meta").rmdir() # a pip venv on Homebrew's cairo, as older installs made
r = box.run("setup.sh")
+ self.assertEqual((r.returncode, r.stdout.splitlines()), (0, [f"ok manim {VERSION} already in .venv"]), r.stderr)
+ self.assertFalse(any(line.startswith(("micromamba", "uv")) for line in box.logged()))
+
+ def test_a_new_pin_goes_into_the_environment_there(self):
+ box = self.sandbox()
+ box.venv(manim="0.18.1")
+ r = box.run("setup.sh", VENV_PY="3.11.9")
self.assertEqual(r.returncode, 0, r.stderr)
- self.assertEqual(r.stdout.splitlines()[0], "ok Python 3.x (python3.13)")
- self.assertFalse(any("-m venv" in call for call in box.logged()), "the venv was already there")
- self.assertEqual(r.stdout.splitlines()[-1], "warn ffmpeg not on PATH — renders need it: brew install ffmpeg")
+ self.assertEqual(r.stdout.splitlines()[0], "ok Python 3.11.9 with pycairo in .venv")
+ self.assertEqual(r.stdout.splitlines()[-1], f"ok manim Manim Community v{VERSION}")
+ self.assertIn(self.PIP, box.logged())
+ self.assertFalse(any(line.startswith("micromamba") for line in box.logged()))
- def test_no_python_new_enough_is_a_miss(self):
- r = self.sandbox(pythons={"python3": False}).run("setup.sh")
- self.assertEqual((r.returncode, r.stdout.strip()), (1, "miss python 3.11+ (brew install [email protected])"))
+ def test_an_environment_whose_pycairo_does_not_load_is_made_again(self):
+ box = self.sandbox()
+ box.venv()
+ (box.install / ".venv" / "stale").write_text("from the old environment")
+ # Only the first two probes fail (the old .venv); the conda-forge one answers as a working one.
+ box.stub("python", 'exit 1', where=box.install / ".venv" / "bin")
+ r = box.run("setup.sh")
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(r.stdout.splitlines()[0], " python 3.12, pycairo and manimpango from conda-forge into .venv (a few minutes the first time)")
+ self.assertIn(self.MAMBA_CREATE.format(install=box.install), box.logged())
+ self.assertFalse((box.install / ".venv" / "stale").exists())
- def test_a_mac_needs_cairo_to_build_pycairo(self):
- box = self.sandbox("Darwin")
- box.ffmpeg()
+ def test_no_network_for_micromamba_is_a_miss(self):
+ box = Sandbox(self)
+ box.uv()
+ box.stub("uname", 'case "$1" in -s) echo Darwin ;; -m) echo arm64 ;; esac')
+ box.stub("curl", "exit 6")
r = box.run("setup.sh")
self.assertEqual(r.returncode, 1)
- self.assertEqual(r.stdout.splitlines()[-1], "miss pycairo builds from source and needs: pkgconf cairo — run: brew install pkgconf cairo")
- box = self.sandbox("Darwin")
- box.pkg_config(cairo=False)
- self.assertEqual(box.run("setup.sh").stdout.splitlines()[-1], "miss pycairo builds from source and needs: cairo — run: brew install cairo")
-
- def test_a_mac_with_cairo(self):
- box = self.sandbox("Darwin")
- box.pkg_config(); box.ffmpeg()
- r = box.run("setup.sh")
- self.assertEqual(r.returncode, 0, r.stderr)
- self.assertEqual(r.stdout.splitlines()[1], "ok cairo 1.18.4 (manimpango ships its own pango)")
+ self.assertEqual(r.stdout.splitlines()[-1], "miss could not download https://github.com/mamba-org/micromamba-releases/releases/download/"
+ "2.9.0-0/micromamba-osx-arm64 — check this machine's internet connection")
+
+ def test_an_environment_conda_forge_will_not_make_is_a_miss(self):
+ box = self.sandbox()
+ r = box.run("setup.sh", MAMBA_EXIT="1")
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.splitlines()[-1], f"miss could not make the conda-forge environment in {box.install}/.venv (python=3.12 pycairo manimpango pip)")
+ self.assertFalse(any(line.startswith("uv") for line in box.logged()))
def test_a_failed_install_fails_setup(self):
box = self.sandbox()
r = box.run("setup.sh", PIP_EXIT="1")
self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.splitlines()[-1], f"miss manim {VERSION} would not install into .venv")
self.assertFalse(any(line.startswith("ok manim") for line in r.stdout.splitlines()))
diff --git a/store/agents/manim/toolchain/test_verdict.py b/store/agents/manim/toolchain/test_verdict.py
index c68df4d3a..5e8bb35a3 100644
--- a/store/agents/manim/toolchain/test_verdict.py
+++ b/store/agents/manim/toolchain/test_verdict.py
@@ -1,4 +1,5 @@
-import contextlib, io, json, os, runpy, shutil, subprocess, sys, tempfile, time, unittest
+import contextlib, importlib.util, io, json, os, runpy, shutil, subprocess, sys, tempfile, time, types, unittest
+from fractions import Fraction
from pathlib import Path
from unittest import mock
sys.path.insert(0, str(Path(__file__).parent))
@@ -82,24 +83,75 @@ class Probe(unittest.TestCase):
def test_reads_the_first_video_stream(self):
out = json.dumps({"streams": [{"width": 854, "height": 480, "nb_frames": "93", "duration": "6.2"}], "format": {"duration": "6.3"}})
with self.fake(out) as run:
- self.assertEqual(verdict.probe(Path("a.mp4")), {"width": 854, "height": 480, "frames": 93, "duration": 6.2})
+ self.assertEqual(verdict.ffprobe(Path("a.mp4")), {"width": 854, "height": 480, "frames": 93, "duration": 6.2})
self.assertEqual(run.call_args.args[0][0], "ffprobe")
def test_a_container_without_stream_facts_falls_back_to_the_format(self):
with self.fake(json.dumps({"streams": [{"width": 64, "height": 48}], "format": {"duration": "2.0"}})):
- self.assertEqual(verdict.probe(Path("a.webm")), {"width": 64, "height": 48, "frames": None, "duration": 2.0})
+ self.assertEqual(verdict.ffprobe(Path("a.webm")), {"width": 64, "height": 48, "frames": None, "duration": 2.0})
with self.fake(""):
- self.assertEqual(verdict.probe(Path("a.mp4")), {"width": None, "height": None, "frames": None, "duration": None})
+ self.assertEqual(verdict.ffprobe(Path("a.mp4")), {"width": None, "height": None, "frames": None, "duration": None})
def test_no_ffprobe_is_no_facts(self):
with mock.patch.object(verdict.subprocess, "run", side_effect=FileNotFoundError("ffprobe")):
- self.assertEqual(verdict.probe(Path("a.mp4")), {})
+ self.assertEqual(verdict.ffprobe(Path("a.mp4")), {})
@unittest.skipIf(not (shutil.which("ffmpeg") and shutil.which("ffprobe")), "ffmpeg/ffprobe not on PATH")
def test_the_real_ffprobe(self):
with tempfile.TemporaryDirectory() as d:
video = Path(d) / "t.mp4"
subprocess.run(["ffmpeg", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc=size=64x48:rate=10", "-t", "2", "-pix_fmt", "yuv420p", str(video)], check=True)
+ self.assertEqual(verdict.ffprobe(video), {"width": 64, "height": 48, "frames": 20, "duration": 2.0})
+
+
+class PyAV(unittest.TestCase):
+ """probe() through PyAV, which is in Manim's venv on every machine, ffmpeg binary or not."""
+
+ def av(self, stream=None, container_duration=None, opens=None):
+ """A stand-in `av` module whose open() yields one container with `stream` as its video."""
+ container = mock.MagicMock()
+ container.__enter__.return_value = container
+ container.streams.video = [stream]
+ container.duration = container_duration
+ return mock.patch.dict(sys.modules, {"av": types.SimpleNamespace(open=opens or mock.Mock(return_value=container), time_base=1_000_000)})
+
+ @staticmethod
+ def stream(duration, time_base, frames=116):
+ return types.SimpleNamespace(duration=duration, time_base=time_base, frames=frames, codec_context=types.SimpleNamespace(width=854, height=480))
+
+ def test_the_streams_own_duration(self):
+ with self.av(self.stream(118779, Fraction(1, 15360))):
+ self.assertEqual(verdict.probe(Path("Intro.mp4")), {"width": 854, "height": 480, "frames": 116, "duration": 7.733008})
+
+ def test_the_containers_duration_when_the_stream_keeps_none(self):
+ with self.av(self.stream(None, Fraction(1, 100), frames=0), container_duration=7_740_000):
+ self.assertEqual(verdict.probe(Path("Intro.gif")), {"width": 854, "height": 480, "frames": None, "duration": 7.74})
+ with self.av(self.stream(0, None, frames=0)):
+ self.assertEqual(verdict.probe(Path("still.gif"))["duration"], None)
+
+ def test_a_file_pyav_cannot_read_is_no_facts(self):
+ with self.av(opens=mock.Mock(side_effect=ValueError("Invalid data found when processing input"))):
+ self.assertEqual(verdict.probe(Path("scene.py")), {})
+
+ def test_without_pyav_ffprobe_is_asked(self):
+ with mock.patch.dict(sys.modules, {"av": None}), mock.patch.object(verdict, "ffprobe", return_value={"duration": 2.0}) as ffprobe:
+ self.assertEqual(verdict.probe(Path("a.mp4")), {"duration": 2.0})
+ ffprobe.assert_called_once_with(Path("a.mp4"))
+
+ @unittest.skipUnless(importlib.util.find_spec("av"), "PyAV is not importable here; run with the harness venv's python")
+ def test_the_real_pyav(self):
+ import av
+ with tempfile.TemporaryDirectory() as d:
+ video = Path(d) / "t.mp4"
+ with av.open(str(video), "w") as out:
+ stream = out.add_stream("libx264", rate=10)
+ stream.width, stream.height, stream.pix_fmt = 64, 48, "yuv420p"
+ for i in range(20):
+ frame = av.VideoFrame(64, 48, "yuv420p")
+ for plane in frame.planes:
+ plane.update(bytes([i * 10]) * plane.buffer_size)
+ out.mux(stream.encode(frame))
+ out.mux(stream.encode())
self.assertEqual(verdict.probe(video), {"width": 64, "height": 48, "frames": 20, "duration": 2.0})
diff --git a/store/agents/manim/toolchain/verdict.py b/store/agents/manim/toolchain/verdict.py
index 82fa79ddb..49a2267ad 100755
--- a/store/agents/manim/toolchain/verdict.py
+++ b/store/agents/manim/toolchain/verdict.py
@@ -30,6 +30,26 @@ def newest_render(root: Path) -> Path | None:
def probe(path: Path) -> dict:
+ """The first video stream's size, frame count and duration: through PyAV, the FFmpeg binding Manim
+ encodes with (in its venv, which render.py runs this in), else ffprobe. Manim needs no ffmpeg
+ binary, so neither may its verdict. {} when neither can read the file."""
+ try:
+ import av
+ except ImportError:
+ return ffprobe(path)
+ try:
+ with av.open(str(path)) as container:
+ s = container.streams.video[0]
+ if s.duration and s.time_base:
+ duration = round(float(s.duration * s.time_base), 6)
+ else: # a container that keeps no per-stream duration
+ duration = round(container.duration / av.time_base, 6) if container.duration else None
+ return {"width": s.codec_context.width or None, "height": s.codec_context.height or None, "frames": s.frames or None, "duration": duration}
+ except Exception:
+ return {}
+
+
+def ffprobe(path: Path) -> dict:
try:
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,nb_frames,duration:format=duration", "-of", "json", str(path)], capture_output=True, text=True, timeout=60)
d = json.loads(r.stdout or "{}")
diff --git a/store/agents/phaser/toolchain/doctor.sh b/store/agents/phaser/toolchain/doctor.sh
index 5e6654b9a..862f214ae 100755
--- a/store/agents/phaser/toolchain/doctor.sh
+++ b/store/agents/phaser/toolchain/doctor.sh
@@ -1,7 +1,10 @@
#!/usr/bin/env bash
set -u; cd "$(dirname "$0")/.."; bad=0
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
+if harness_node 18; then echo "ok node $(node --version) (vite and the pane)"; else bad=1; fi
if [ -x node_modules/.bin/vite ]; then
- echo "ok phaser $(node -p "require('./node_modules/phaser/package.json').version") · vite $(node -p "require('./node_modules/vite/package.json').version")"
+ echo "ok phaser $(node -p "require('./node_modules/phaser/package.json').version" 2>/dev/null) · vite $(node -p "require('./node_modules/vite/package.json').version" 2>/dev/null)"
else
echo "miss node_modules — run toolchain/setup.sh"; bad=1
fi
diff --git a/store/agents/phaser/toolchain/runtimes.sh b/store/agents/phaser/toolchain/runtimes.sh
new file mode 100644
index 000000000..4e21942d0
--- /dev/null
+++ b/store/agents/phaser/toolchain/runtimes.sh
@@ -0,0 +1,187 @@
+# shellcheck shell=bash
+# runtimes.sh — the interpreters a store package runs on, found on this machine or fetched, never
+# asked of the person who pressed Get.
+#
+# Sourced (not run) by a package's setup, doctor and viewer scripts. A new Mac has Apple's Python 3.9,
+# no Homebrew and, for most people, no Node on PATH; a package that answers "brew install [email protected]"
+# has failed its install. So:
+#
+# harness_node 18 node >= 18 on PATH, with npm: the machine's own, else the Node Harness
+# itself runs on (~/.harness/runtime/current-node, laid down with the CLI)
+# harness_uv uv on PATH: the machine's own, else a pinned, checksummed release
+# fetched into ~/.harness/runtime
+# harness_venv DIR 3.12 [MIN [BELOW]]
+# DIR is a venv on CPython 3.12, which uv downloads when the machine
+# has none. A venv already there on a Python in [MIN, BELOW) is kept.
+# harness_pip DIR PKG… install into that venv, through uv
+# harness_micromamba micromamba, pinned and checksummed, for native libraries PyPI has
+# no wheel for (pycairo on macOS)
+# harness_conda_env DIR SPEC… DIR is a conda-forge environment with those packages
+#
+# Every function prints `miss …` and returns 1 when it cannot deliver, so `harness_node 18 || exit 1`
+# reads like the rest of a setup script. Nothing here touches the user's shell profile or anything
+# outside ~/.harness/runtime and the package's own directory.
+#
+# ONE copy is written by hand: store/tools/runtimes.sh in the OpenHarness repository. Each package
+# carries an identical copy (a package installs alone, so it cannot reach this one);
+# `node store/tools/sync-runtimes.mjs` rewrites them and the CLI's store spec fails when one drifts.
+
+HARNESS_RUNTIME="${ADAPTER_RUNTIME_DIR:-${HOME:-}/.harness/runtime}"
+# The Pythons uv downloads live with Harness's other runtimes, not in uv's own ~/.local/share/uv:
+# a package's venv points at its interpreter, and `uv python uninstall` there must not break it.
+UV_PYTHON_INSTALL_DIR="${UV_PYTHON_INSTALL_DIR:-$HARNESS_RUNTIME/python}"
+export UV_PYTHON_INSTALL_DIR
+HARNESS_UV_VERSION=0.12.15
+HARNESS_MICROMAMBA_VERSION=2.9.0-0
+
+_harness_platform() {
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64) echo darwin-arm64 ;;
+ Darwin-x86_64) echo darwin-x64 ;;
+ Linux-x86_64) echo linux-x64 ;;
+ Linux-aarch64 | Linux-arm64) echo linux-arm64 ;;
+ *) return 1 ;;
+ esac
+}
+
+_harness_sha256() {
+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1
+ else sha256sum "$1" | cut -d' ' -f1; fi
+}
+
+# _harness_fetch URL DEST SHA256 — DEST exists only if what arrived matches the pinned checksum.
+_harness_fetch() {
+ local part="$2.part.$$"
+ if ! curl -fsSL --retry 3 --connect-timeout 20 --max-time 900 -o "$part" "$1"; then
+ rm -f "$part"; echo "miss could not download $1 — check this machine's internet connection"; return 1
+ fi
+ if [ "$(_harness_sha256 "$part")" != "$3" ]; then
+ rm -f "$part"; echo "miss $1 did not match its pinned checksum; nothing was installed"; return 1
+ fi
+ mv -f "$part" "$2"
+}
+
+_harness_node_at_least() {
+ command -v node >/dev/null 2>&1 || return 1
+ node -e '
+ const [a, b = 0] = process.argv[1].split(".").map(Number)
+ const [x, y] = process.versions.node.split(".").map(Number)
+ process.exit(x > a || (x === a && y >= b) ? 0 : 1)
+ ' "$1" >/dev/null 2>&1
+}
+
+harness_node() {
+ local min="${1:-18}" recorded saved="$PATH" own=0
+ _harness_node_at_least "$min" && own=1
+ # A node without npm beside it (a distro's nodejs package) cannot run a setup's `npm ci`.
+ if [ "$own" = 1 ] && command -v npm >/dev/null 2>&1; then return 0; fi
+ recorded="$(cat "$HARNESS_RUNTIME/current-node" 2>/dev/null || true)"
+ if [ -n "$recorded" ] && [ -x "${recorded%/*}/node" ]; then
+ PATH="${recorded%/*}:$PATH"; export PATH; hash -r 2>/dev/null || true
+ _harness_node_at_least "$min" && return 0
+ PATH="$saved"; export PATH; hash -r 2>/dev/null || true
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min — this machine's newest is $("${recorded%/*}/node" -v), Harness's own; update Harness"
+ return 1
+ fi
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min, and Harness's own Node is not in $HARNESS_RUNTIME — run \`harness start\` once to lay it down"
+ return 1
+}
+
+harness_uv() {
+ local dir asset sum tmp
+ command -v uv >/dev/null 2>&1 && return 0
+ for dir in "${HOME:-}/.local/bin" "${HOME:-}/.cargo/bin"; do
+ [ -n "${HOME:-}" ] || break
+ if [ -x "$dir/uv" ]; then PATH="$dir:$PATH"; export PATH; return 0; fi
+ done
+ dir="$HARNESS_RUNTIME/uv-$HARNESS_UV_VERSION"
+ if [ ! -x "$dir/uv" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=uv-aarch64-apple-darwin sum=dc304b9ed1b24174572290fba60ac3f6fe63c73a671f0439e62a91375841964d ;;
+ darwin-x64) asset=uv-x86_64-apple-darwin sum=e9ca61775532368fe518ab03e7a354c7ecab8ccb3c7d941c775fcc4a362b801b ;;
+ linux-x64) asset=uv-x86_64-unknown-linux-gnu sum=f97935763c04be3e692460a7aaeaaab8fc3b78fcf8b389da820b38ae7423a638 ;;
+ linux-arm64) asset=uv-aarch64-unknown-linux-gnu sum=0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08 ;;
+ *) echo "miss uv — there is no uv release for $(uname -s) $(uname -m); install it from https://docs.astral.sh/uv/"; return 1 ;;
+ esac
+ echo " fetching uv $HARNESS_UV_VERSION into $HARNESS_RUNTIME (it brings Python and the packages)"
+ mkdir -p "$HARNESS_RUNTIME" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ tmp="$(mktemp -d "$HARNESS_RUNTIME/.uv.XXXXXX")" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ if ! _harness_fetch "https://github.com/astral-sh/uv/releases/download/$HARNESS_UV_VERSION/$asset.tar.gz" "$tmp/uv.tar.gz" "$sum"; then
+ rm -rf "$tmp"; return 1
+ fi
+ tar -xzf "$tmp/uv.tar.gz" -C "$tmp" && chmod +x "$tmp/$asset/uv" "$tmp/$asset/uvx" \
+ || { rm -rf "$tmp"; echo "miss the uv archive would not unpack"; return 1; }
+ # Two installs at once both get here; whichever renames first wins and the other's copy goes.
+ # (mv onto a directory that exists would nest the copy inside it, hence the test.)
+ [ -e "$dir" ] || mv "$tmp/$asset" "$dir" 2>/dev/null || true
+ rm -rf "$tmp"
+ [ -x "$dir/uv" ] || { echo "miss uv did not land in $dir"; return 1; }
+ fi
+ PATH="$dir:$PATH"; export PATH
+}
+
+# harness_venv DIR VERSION [MIN [BELOW]] — MIN defaults to VERSION; BELOW is exclusive and optional.
+harness_venv() {
+ local dir="${1:?harness_venv needs a directory}" want="${2:?harness_venv needs a Python version}"
+ local min="${3:-$2}" below="${4:-}"
+ if [ -x "$dir/bin/python" ] && "$dir/bin/python" -c '
+import sys
+at = lambda s: tuple(int(p) for p in s.split("."))
+v = sys.version_info[:2]
+sys.exit(0 if v >= at(sys.argv[1]) and (not sys.argv[2] or v < at(sys.argv[2])) else 1)
+' "$min" "$below" 2>/dev/null; then
+ echo "ok $("$dir/bin/python" --version) in $dir"
+ return 0
+ fi
+ harness_uv || return 1
+ echo " python $want in $dir (uv downloads it when this machine has none)"
+ rm -rf "$dir"
+ # only-managed: the same python-build-standalone interpreter on every machine, so a package tested
+ # once is tested everywhere — not whichever Homebrew or pyenv Python happened to be first on PATH.
+ uv venv --quiet --seed --python "$want" --python-preference only-managed "$dir" \
+ || { echo "miss could not make a Python $want environment in $dir"; return 1; }
+ echo "ok $("$dir/bin/python" --version) in $dir"
+}
+
+harness_pip() {
+ local dir="${1:?harness_pip needs a venv}"
+ shift
+ harness_uv || return 1
+ uv pip install --quiet --python "$dir/bin/python" "$@"
+}
+
+harness_micromamba() {
+ local dir asset sum
+ dir="$HARNESS_RUNTIME/micromamba-$HARNESS_MICROMAMBA_VERSION"
+ if [ ! -x "$dir/micromamba" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=micromamba-osx-arm64 sum=ec2a072f028e1a7cf20f3e2e74d5a8127cf5a5f27636375b5359811565f4e5be ;;
+ darwin-x64) asset=micromamba-osx-64 sum=1e71054bb3ac9a076e21f7ec48acfef536f9b3f1408f371a942784bf5ef83d8a ;;
+ linux-x64) asset=micromamba-linux-64 sum=366cd9cd8be14df1ab8ed50352a82111082a36686b2d389fdb79a92c3fafb3e3 ;;
+ linux-arm64) asset=micromamba-linux-aarch64 sum=9f93b974adcb4d166996af969b6cd371287d1a3e52733704727884d9b74cb7a7 ;;
+ *) echo "miss micromamba — there is no release for $(uname -s) $(uname -m)"; return 1 ;;
+ esac
+ echo " fetching micromamba $HARNESS_MICROMAMBA_VERSION into $HARNESS_RUNTIME (native libraries PyPI has no wheel for)"
+ mkdir -p "$dir" || { echo "miss cannot write $dir"; return 1; }
+ _harness_fetch "https://github.com/mamba-org/micromamba-releases/releases/download/$HARNESS_MICROMAMBA_VERSION/$asset" "$dir/micromamba.$$" "$sum" || return 1
+ chmod +x "$dir/micromamba.$$" && mv -f "$dir/micromamba.$$" "$dir/micromamba" \
+ || { rm -f "$dir/micromamba.$$"; echo "miss micromamba would not install into $dir"; return 1; }
+ fi
+ HARNESS_MICROMAMBA="$dir/micromamba"
+ export HARNESS_MICROMAMBA
+}
+
+# harness_conda_env DIR SPEC… — conda-forge only, packages cached under ~/.harness/runtime/mamba.
+harness_conda_env() {
+ local dir="${1:?harness_conda_env needs a directory}"
+ shift
+ # micromamba takes a relative --prefix as a name under its root prefix and "succeeds" there.
+ case "$dir" in /*) ;; *) dir="$PWD/$dir" ;; esac
+ harness_micromamba || return 1
+ MAMBA_ROOT_PREFIX="$HARNESS_RUNTIME/mamba" "$HARNESS_MICROMAMBA" create --yes --quiet \
+ --prefix "$dir" --override-channels --channel conda-forge "$@" >/dev/null \
+ && [ -d "$dir/conda-meta" ] \
+ || { echo "miss could not make the conda-forge environment in $dir ($*)"; return 1; }
+}
diff --git a/store/agents/phaser/toolchain/setup.sh b/store/agents/phaser/toolchain/setup.sh
index db978daa6..74e111904 100755
--- a/store/agents/phaser/toolchain/setup.sh
+++ b/store/agents/phaser/toolchain/setup.sh
@@ -2,12 +2,15 @@
# Runs once at install, cwd = the install dir. One node_modules here — Phaser, Vite, terser — that
# every workspace links to, so a new game is instant and there is one copy of Phaser on the machine.
# Phaser's own agent skills are vendored in skills/ (MIT, see PROVENANCE.md), so nothing is fetched.
+# Node is this machine's, or the one Harness itself runs on (npm comes beside it).
set -euo pipefail
cd "$(dirname "$0")/.."
# shellcheck disable=SC1091
. ./VERSIONS
-command -v node >/dev/null 2>&1 && node -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 18 ? 0 : 1)' || { echo "miss node >= 18 on PATH"; exit 1; }
-command -v npm >/dev/null 2>&1 || { echo "miss npm on PATH"; exit 1; }
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
+harness_node 18 || exit 1
+command -v npm >/dev/null 2>&1 || { echo "miss npm beside $(command -v node)"; exit 1; }
command -v python3 >/dev/null 2>&1 || { echo "miss python3 (the verdict)"; exit 1; }
echo " npm ci (phaser ${PHASER} + vite ${VITE}, about a minute the first time)"
if [ -f package-lock.json ]; then npm ci --silent --no-audit --no-fund; else npm install --silent --no-audit --no-fund; fi
diff --git a/store/agents/phaser/toolchain/test_scripts.py b/store/agents/phaser/toolchain/test_scripts.py
index 27c893273..5418a5f7f 100644
--- a/store/agents/phaser/toolchain/test_scripts.py
+++ b/store/agents/phaser/toolchain/test_scripts.py
@@ -2,7 +2,8 @@
Each test builds a throwaway package root: the real scripts linked in file by file, fixtures beside
them, and a PATH that holds only the tools the case allows (a stub npm, a node that can pretend to
-be old, python3 or not). Nothing is installed and nothing touches the network or this folder.
+be old, python3 or not), and a runtime dir where runtimes.sh looks for the node Harness runs on.
+Nothing is installed and nothing touches the network or this folder.
"""
import json
import os
@@ -15,7 +16,7 @@ from pathlib import Path
PKG = Path(__file__).resolve().parent.parent
NODE = shutil.which("node")
PYTHON = shutil.which("python3")
-BASE_TOOLS = ("bash", "dirname", "find", "wc", "tr", "mkdir", "ln", "chmod")
+BASE_TOOLS = ("bash", "dirname", "find", "wc", "tr", "mkdir", "ln", "chmod", "cat")
VERSIONS = "PHASER=9.9.9\nVITE=7.7.7\nTERSER=5.0.0\nSKILLS_COMMIT=abcdef1234567890\n"
@@ -39,7 +40,8 @@ class Sandbox:
self.pkg = self.root / "pkg"
self.bin = self.root / "bin"
self.ws = self.root / "ws"
- for d in (self.pkg, self.bin, self.ws):
+ self.runtime = self.root / "runtime" # never ~/.harness/runtime: that node would answer every miss
+ for d in (self.pkg, self.bin, self.ws, self.runtime):
d.mkdir()
self.tools(*BASE_TOOLS)
@@ -52,6 +54,7 @@ class Sandbox:
def file(self, rel: str, text: str = "", base: Path | None = None, mode: int = 0o644) -> Path:
path = (base or self.pkg) / rel
path.parent.mkdir(parents=True, exist_ok=True)
+ path.unlink(missing_ok=True) # a tool is a link to the real one: never write through it
path.write_text(text)
path.chmod(mode)
return path
@@ -72,10 +75,23 @@ class Sandbox:
raise unittest.SkipTest("node is not on PATH")
self.stub("node", f'[ "$1" = -e ] && exit 1\nexec "{NODE}" "$@"\n')
- def run(self, rel: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
+ def harness_node(self, npm: str | None = None) -> Path:
+ """The real node where Harness keeps its own, recorded in the runtime dir and not on PATH; a stub
+ npm beside it when given."""
+ if NODE is None:
+ raise unittest.SkipTest("node is not on PATH")
+ node = self.root / "harness-node" / "bin" / "node"
+ node.parent.mkdir(parents=True)
+ node.symlink_to(NODE)
+ if npm is not None:
+ self.file("npm", npm, base=node.parent, mode=0o755)
+ (self.runtime / "current-node").write_text(f"{node}\n")
+ return node
+
+ def run(self, rel: str, *args: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
full = dict((k, v) for k, v in os.environ.items() if not k.startswith(("HARNESS_", "VITE", "NPM_")))
- full.update(PATH=str(self.bin), **env)
- return subprocess.run([str(self.pkg / rel)], cwd=cwd or self.root, env=full,
+ full.update(PATH=str(self.bin), ADAPTER_RUNTIME_DIR=str(self.runtime), **env)
+ return subprocess.run([str(self.pkg / rel), *args], cwd=cwd or self.root, env=full,
capture_output=True, text=True, timeout=60)
@@ -87,6 +103,7 @@ class Doctor(unittest.TestCase):
def test_a_complete_install_is_ready(self):
box = Sandbox(self)
box.link("toolchain/doctor.sh")
+ box.link("toolchain/runtimes.sh")
box.tools("node", "python3")
box.file("node_modules/.bin/vite", "#!/bin/sh\n", mode=0o755)
box.file("node_modules/phaser/package.json", '{"version": "4.2.1"}')
@@ -97,20 +114,35 @@ class Doctor(unittest.TestCase):
r = box.run("toolchain/doctor.sh")
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
out = lines(r)
- self.assertEqual(out[:3], ["ok phaser 4.2.1 · vite 6.4.3",
- "ok pane: the game frame around vite (viewer.mjs, viewer/)",
- "ok skills: 3"])
- self.assertRegex(out[3], r"^ok Python 3\.\d+")
- self.assertEqual(len(out), 4)
+ self.assertRegex(out[0], r"^ok node v\d+\.\d+\.\d+ \(vite and the pane\)$")
+ self.assertEqual(out[1:4], ["ok phaser 4.2.1 · vite 6.4.3",
+ "ok pane: the game frame around vite (viewer.mjs, viewer/)",
+ "ok skills: 3"])
+ self.assertRegex(out[4], r"^ok Python 3\.\d+")
+ self.assertEqual(len(out), 5)
+
+ def test_harness_node_serves_when_path_has_none(self):
+ box = Sandbox(self)
+ box.link("toolchain/doctor.sh")
+ box.link("toolchain/runtimes.sh")
+ box.harness_node()
+ box.file("node_modules/.bin/vite", "#!/bin/sh\n", mode=0o755)
+ box.file("node_modules/phaser/package.json", '{"version": "4.2.1"}')
+ box.file("node_modules/vite/package.json", '{"version": "6.4.3"}')
+ r = box.run("toolchain/doctor.sh")
+ self.assertRegex(lines(r)[0], r"^ok node v\d+\.\d+\.\d+ \(vite and the pane\)$")
+ self.assertEqual(lines(r)[1], "ok phaser 4.2.1 · vite 6.4.3")
def test_an_empty_install_without_python_says_what_is_missing(self):
box = Sandbox(self)
box.link("toolchain/doctor.sh")
+ box.link("toolchain/runtimes.sh")
box.file("viewer.mjs") # the frame's files are not all there
box.file("skills/scenes/SKILL.md") # nor is the harness skill
r = box.run("toolchain/doctor.sh")
self.assertEqual(r.returncode, 1)
- self.assertEqual(lines(r), ["miss node_modules — run toolchain/setup.sh",
+ self.assertEqual(lines(r), [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down",
+ "miss node_modules — run toolchain/setup.sh",
"miss viewer.mjs or viewer/ — the checkout is incomplete",
"miss skills/ — the checkout is incomplete",
"miss python3 (the verdict)"])
@@ -121,6 +153,7 @@ class Setup(unittest.TestCase):
skills: bool = True) -> Sandbox:
box = Sandbox(self)
box.link("toolchain/setup.sh")
+ box.link("toolchain/runtimes.sh")
box.file("VERSIONS", VERSIONS)
if node == "real":
box.tools("node")
@@ -169,12 +202,22 @@ class Setup(unittest.TestCase):
with self.subTest(node=node):
box = self.box(node=node)
r = self.run_setup(box)
- self.assertEqual((r.returncode, lines(r)), (1, ["miss node >= 18 on PATH"]))
+ self.assertEqual((r.returncode, lines(r)), (1, [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down"]))
self.assertEqual(self.npm_calls(), [])
+ def test_a_machine_without_node_or_with_an_old_one_uses_harnesss_and_the_npm_beside_it(self):
+ for node in ("none", "old"):
+ with self.subTest(node=node):
+ box = self.box(node=node, npm=False)
+ box.harness_node(npm=NPM)
+ r = self.run_setup(box)
+ self.assertEqual((r.returncode, lines(r)[-1]), (0, "ok skills: 3 (2 from phaserjs/phaser @ abcdef1, plus harness-phaser)"), r.stdout + r.stderr)
+ self.assertEqual(self.npm_calls()[-1], "ci --silent --no-audit --no-fund")
+
def test_no_npm(self):
- r = self.run_setup(self.box(npm=False))
- self.assertEqual((r.returncode, lines(r)), (1, ["miss npm on PATH"]))
+ box = self.box(npm=False)
+ r = self.run_setup(box)
+ self.assertEqual((r.returncode, lines(r)), (1, [f"miss npm beside {box.bin}/node"]))
def test_no_python(self):
box = self.box(python=False)
@@ -237,10 +280,12 @@ class InitWorkspace(unittest.TestCase):
class ViewerScript(unittest.TestCase):
- def box(self) -> Sandbox:
+ def box(self, node: bool = True) -> Sandbox:
box = Sandbox(self)
box.link("viewer.sh")
- box.stub("node", 'echo "cwd=$PWD"\necho "args=$*"\n')
+ box.link("toolchain/runtimes.sh")
+ if node:
+ box.stub("node", '[ "$1" = -e ] && exit 0\necho "cwd=$PWD"\necho "args=$*"\n')
return box
def test_needs_the_port_and_the_workspace(self):
@@ -259,6 +304,19 @@ class ViewerScript(unittest.TestCase):
self.assertEqual(lines(r), [f"cwd={box.ws}", f"args={box.pkg / 'viewer.mjs'}"])
self.assertEqual(os.readlink(box.ws / "node_modules"), str(box.pkg / "node_modules"))
+ def test_a_login_shell_without_node_runs_harnesss_own(self):
+ box = self.box(node=False)
+ node = box.file("harness-node/bin/node", '#!/bin/bash\n[ "$1" = -e ] && exit 0\necho "harness node $*"\n', base=box.root, mode=0o755)
+ (box.runtime / "current-node").write_text(f"{node}\n")
+ r = box.run("viewer.sh", HARNESS_VIEWER_PORT="4173", HARNESS_WORKSPACE=str(box.ws))
+ self.assertEqual((r.returncode, lines(r)), (0, [f"harness node {box.pkg / 'viewer.mjs'}"]), r.stderr)
+
+ def test_no_node_anywhere_never_starts(self):
+ box = self.box(node=False)
+ r = box.run("viewer.sh", HARNESS_VIEWER_PORT="4173", HARNESS_WORKSPACE=str(box.ws))
+ self.assertEqual((r.returncode, lines(r)), (1, [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down"]))
+ self.assertFalse((box.ws / "node_modules").exists())
+
def test_leaves_an_existing_node_modules_alone(self):
box = self.box()
(box.ws / "node_modules").mkdir()
@@ -267,5 +325,33 @@ class ViewerScript(unittest.TestCase):
self.assertFalse((box.ws / "node_modules").is_symlink())
+class WithNode(unittest.TestCase):
+ """toolchain/with-node.sh, which the verdict runs vite through."""
+
+ def box(self) -> Sandbox:
+ box = Sandbox(self)
+ box.link("toolchain/with-node.sh")
+ box.link("toolchain/runtimes.sh")
+ return box
+
+ def test_node_on_path_runs_the_command(self):
+ box = self.box()
+ box.tools("node")
+ r = box.run("toolchain/with-node.sh", "node", "-e", "console.log('ran')")
+ self.assertEqual((r.returncode, r.stdout), (0, "ran\n"), r.stderr)
+
+ def test_harnesss_node_is_put_on_path_when_there_is_none(self):
+ box = self.box()
+ node = box.harness_node()
+ r = box.run("toolchain/with-node.sh", "bash", "-c", "command -v node")
+ self.assertEqual((r.returncode, r.stdout), (0, f"{node}\n"), r.stderr)
+
+ def test_no_node_is_127_with_the_miss_on_stderr(self):
+ box = self.box()
+ r = box.run("toolchain/with-node.sh", "vite", "build")
+ self.assertEqual((r.returncode, r.stdout), (127, ""))
+ self.assertEqual(r.stderr, f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down\n")
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/store/agents/phaser/toolchain/test_verdict.py b/store/agents/phaser/toolchain/test_verdict.py
index 9e291cf70..b50a24740 100644
--- a/store/agents/phaser/toolchain/test_verdict.py
+++ b/store/agents/phaser/toolchain/test_verdict.py
@@ -234,11 +234,43 @@ class Main(unittest.TestCase):
def test_a_build_that_hangs_is_a_build_finding(self):
self.game()
hung = subprocess.TimeoutExpired(cmd=["vite", "build"], timeout=600)
+ vite = stub_vite(self.bin, "sys.exit(0)")
with mock.patch.object(verdict.subprocess, "run", side_effect=hung):
- code, _, v = self.run_main("vite")
+ code, _, v = self.run_main(vite)
self.assertEqual(code, 1)
self.assertIn("timed out after 600 seconds", v["findings"][0]["message"])
+ def node_env(self, recorded: bool):
+ """The agent's shell as a new Mac has it: no node on PATH, and Harness's own recorded, or not."""
+ runtime = self.bin / "runtime"
+ runtime.mkdir()
+ if recorded:
+ node = self.bin / "harness-node" / "bin" / "node"
+ node.parent.mkdir(parents=True)
+ node.write_text(f'#!/bin/bash\n[ "$1" = -e ] && exit 0\necho "harness node ran $1" > "{self.bin}/node.log"\n')
+ node.chmod(0o755)
+ (runtime / "current-node").write_text(f"{node}\n")
+ return mock.patch.dict(os.environ, {"PATH": "/usr/bin:/bin", "ADAPTER_RUNTIME_DIR": str(runtime)})
+
+ def test_vite_runs_on_harnesss_node_when_the_shell_has_none(self):
+ self.game()
+ vite = self.bin / "vite"
+ vite.write_text("#!/usr/bin/env node\n") # what node_modules/.bin/vite is
+ vite.chmod(0o755)
+ with self.node_env(recorded=True):
+ code, _, v = self.run_main(str(vite))
+ self.assertEqual((self.bin / "node.log").read_text(), f"harness node ran {vite}\n")
+ self.assertEqual((code, v["findings"], v["ready"]), (0, [], True))
+
+ def test_no_node_anywhere_is_a_build_finding_that_says_so(self):
+ self.game()
+ with self.node_env(recorded=False):
+ code, _, v = self.run_main(stub_vite(self.bin, "sys.exit(0)"))
+ self.assertEqual(code, 1)
+ self.assertEqual(v["findings"][0], {"severity": "error", "kind": "build", "message":
+ f"miss node >= 18, and Harness's own Node is not in {self.bin / 'runtime'} — run `harness start` once to lay it down"})
+ self.assertEqual(self.calls(), [])
+
def test_no_build_reads_the_scenes_but_never_runs_vite(self):
self.game()
write(self.ws / "out" / "dist" / "index.html", "<html>") # from an earlier build
diff --git a/store/agents/phaser/toolchain/verdict.py b/store/agents/phaser/toolchain/verdict.py
index 57a331747..3c1d8d479 100644
--- a/store/agents/phaser/toolchain/verdict.py
+++ b/store/agents/phaser/toolchain/verdict.py
@@ -14,10 +14,12 @@ Phases:
whether the game is *fun*; it can tell that nothing would respond to the player.
Ready = it builds and something would respond. The artifact is the built page when there is one;
-the pane is the dev server either way.
+the pane is the dev server either way. Vite runs on node, which the agent's shell may not have on
+PATH: with-node.sh puts this machine's or Harness's own there first.
"""
from __future__ import annotations
+import errno
import json
import os
import re
@@ -28,6 +30,7 @@ from pathlib import Path
WS = Path(os.environ.get("HARNESS_WORKSPACE") or os.getcwd()).resolve()
VITE = os.environ.get("VITE") or str(WS / "node_modules" / ".bin" / "vite")
+WITH_NODE = Path(__file__).resolve().parent / "with-node.sh"
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
SCENE = re.compile(r"extends\s+(?:Phaser\.)?Scene\b")
@@ -171,8 +174,10 @@ def main(argv: list[str]) -> int:
build_ran = bool(written and scenes and not no_build)
if build_ran:
try:
+ if not os.access(VITE, os.X_OK): # said plainly: through the wrapper it would be an exec error on stderr
+ raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), VITE)
run = subprocess.run(
- [VITE, "build", "--outDir", "out/dist", "--minify", "false", "--logLevel", "warn"],
+ [str(WITH_NODE), VITE, "build", "--outDir", "out/dist", "--minify", "false", "--logLevel", "warn"],
capture_output=True, text=True, cwd=WS, timeout=600)
if run.returncode != 0:
build_err = build_message(run.stderr + "\n" + run.stdout)
diff --git a/store/agents/phaser/toolchain/with-node.sh b/store/agents/phaser/toolchain/with-node.sh
new file mode 100755
index 000000000..7ba169de9
--- /dev/null
+++ b/store/agents/phaser/toolchain/with-node.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# with-node.sh CMD [ARG…] — CMD with a node >= 18 on PATH: this machine's, else the one Harness itself
+# runs on. The verdict is run with python3 from the agent's shell, which may have no node at all.
+# No node anywhere is exit 127, as for a command not found, with runtimes.sh's miss line on stderr.
+# shellcheck source=runtimes.sh
+. "${0%/*}/runtimes.sh"
+harness_node 18 >&2 || exit 127
+exec "$@"
diff --git a/store/agents/phaser/viewer.sh b/store/agents/phaser/viewer.sh
index 3019304c9..1ea11b531 100755
--- a/store/agents/phaser/viewer.sh
+++ b/store/agents/phaser/viewer.sh
@@ -1,10 +1,14 @@
#!/usr/bin/env bash
# The pane: Vite's dev server on the workspace, in a game frame (viewer.mjs), on the port Harness
# hands it, loopback only. Vite's HMR is what makes the pane live — the agent saves a scene, the game
-# reloads in place — and the frame around it adds sizes, pause, debug and readable errors.
+# reloads in place — and the frame around it adds sizes, pause, debug and readable errors. Harness runs
+# it through a login shell whose PATH may have no node: runtimes.sh finds this machine's or Harness's own.
set -euo pipefail
: "${HARNESS_VIEWER_PORT:?}"; : "${HARNESS_WORKSPACE:?}"
DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=toolchain/runtimes.sh
+. "$DIR/toolchain/runtimes.sh"
+harness_node 18 || exit 1
cd "$HARNESS_WORKSPACE"
# A workspace the user made by hand, or one whose link was lost: point it at the shared install.
[ -e node_modules ] || ln -s "$DIR/node_modules" node_modules
diff --git a/store/agents/strudel/toolchain/doctor.sh b/store/agents/strudel/toolchain/doctor.sh
index 191354d44..164d08a1d 100755
--- a/store/agents/strudel/toolchain/doctor.sh
+++ b/store/agents/strudel/toolchain/doctor.sh
@@ -1,7 +1,9 @@
#!/usr/bin/env bash
set -u; cd "$(dirname "$0")/.."; bad=0
-if [ -f node_modules/@strudel/repl/dist/index.js ]; then echo "ok strudel $(node -p "require('@strudel/repl/package.json').version") (the pane's REPL)"; else echo "miss node_modules — run toolchain/setup.sh"; bad=1; fi
-if command -v node >/dev/null 2>&1; then echo "ok node $(node --version) (the pane server and the syntax check)"; else echo "miss node >= 18"; bad=1; fi
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
+if harness_node 18; then echo "ok node $(node --version) (the pane server and the syntax check)"; else bad=1; fi
+if [ -f node_modules/@strudel/repl/dist/index.js ]; then echo "ok strudel $(node -p "require('@strudel/repl/package.json').version" 2>/dev/null) (the pane's REPL)"; else echo "miss node_modules — run toolchain/setup.sh"; bad=1; fi
if command -v python3 >/dev/null 2>&1; then echo "ok $(python3 --version) (the verdict)"; else echo "miss python3"; bad=1; fi
echo "note the pane needs a click to start audio, and sample banks (bd, sd, hh) need the internet — synths do not"
exit $bad
diff --git a/store/agents/strudel/toolchain/runtimes.sh b/store/agents/strudel/toolchain/runtimes.sh
new file mode 100644
index 000000000..4e21942d0
--- /dev/null
+++ b/store/agents/strudel/toolchain/runtimes.sh
@@ -0,0 +1,187 @@
+# shellcheck shell=bash
+# runtimes.sh — the interpreters a store package runs on, found on this machine or fetched, never
+# asked of the person who pressed Get.
+#
+# Sourced (not run) by a package's setup, doctor and viewer scripts. A new Mac has Apple's Python 3.9,
+# no Homebrew and, for most people, no Node on PATH; a package that answers "brew install [email protected]"
+# has failed its install. So:
+#
+# harness_node 18 node >= 18 on PATH, with npm: the machine's own, else the Node Harness
+# itself runs on (~/.harness/runtime/current-node, laid down with the CLI)
+# harness_uv uv on PATH: the machine's own, else a pinned, checksummed release
+# fetched into ~/.harness/runtime
+# harness_venv DIR 3.12 [MIN [BELOW]]
+# DIR is a venv on CPython 3.12, which uv downloads when the machine
+# has none. A venv already there on a Python in [MIN, BELOW) is kept.
+# harness_pip DIR PKG… install into that venv, through uv
+# harness_micromamba micromamba, pinned and checksummed, for native libraries PyPI has
+# no wheel for (pycairo on macOS)
+# harness_conda_env DIR SPEC… DIR is a conda-forge environment with those packages
+#
+# Every function prints `miss …` and returns 1 when it cannot deliver, so `harness_node 18 || exit 1`
+# reads like the rest of a setup script. Nothing here touches the user's shell profile or anything
+# outside ~/.harness/runtime and the package's own directory.
+#
+# ONE copy is written by hand: store/tools/runtimes.sh in the OpenHarness repository. Each package
+# carries an identical copy (a package installs alone, so it cannot reach this one);
+# `node store/tools/sync-runtimes.mjs` rewrites them and the CLI's store spec fails when one drifts.
+
+HARNESS_RUNTIME="${ADAPTER_RUNTIME_DIR:-${HOME:-}/.harness/runtime}"
+# The Pythons uv downloads live with Harness's other runtimes, not in uv's own ~/.local/share/uv:
+# a package's venv points at its interpreter, and `uv python uninstall` there must not break it.
+UV_PYTHON_INSTALL_DIR="${UV_PYTHON_INSTALL_DIR:-$HARNESS_RUNTIME/python}"
+export UV_PYTHON_INSTALL_DIR
+HARNESS_UV_VERSION=0.12.15
+HARNESS_MICROMAMBA_VERSION=2.9.0-0
+
+_harness_platform() {
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64) echo darwin-arm64 ;;
+ Darwin-x86_64) echo darwin-x64 ;;
+ Linux-x86_64) echo linux-x64 ;;
+ Linux-aarch64 | Linux-arm64) echo linux-arm64 ;;
+ *) return 1 ;;
+ esac
+}
+
+_harness_sha256() {
+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1
+ else sha256sum "$1" | cut -d' ' -f1; fi
+}
+
+# _harness_fetch URL DEST SHA256 — DEST exists only if what arrived matches the pinned checksum.
+_harness_fetch() {
+ local part="$2.part.$$"
+ if ! curl -fsSL --retry 3 --connect-timeout 20 --max-time 900 -o "$part" "$1"; then
+ rm -f "$part"; echo "miss could not download $1 — check this machine's internet connection"; return 1
+ fi
+ if [ "$(_harness_sha256 "$part")" != "$3" ]; then
+ rm -f "$part"; echo "miss $1 did not match its pinned checksum; nothing was installed"; return 1
+ fi
+ mv -f "$part" "$2"
+}
+
+_harness_node_at_least() {
+ command -v node >/dev/null 2>&1 || return 1
+ node -e '
+ const [a, b = 0] = process.argv[1].split(".").map(Number)
+ const [x, y] = process.versions.node.split(".").map(Number)
+ process.exit(x > a || (x === a && y >= b) ? 0 : 1)
+ ' "$1" >/dev/null 2>&1
+}
+
+harness_node() {
+ local min="${1:-18}" recorded saved="$PATH" own=0
+ _harness_node_at_least "$min" && own=1
+ # A node without npm beside it (a distro's nodejs package) cannot run a setup's `npm ci`.
+ if [ "$own" = 1 ] && command -v npm >/dev/null 2>&1; then return 0; fi
+ recorded="$(cat "$HARNESS_RUNTIME/current-node" 2>/dev/null || true)"
+ if [ -n "$recorded" ] && [ -x "${recorded%/*}/node" ]; then
+ PATH="${recorded%/*}:$PATH"; export PATH; hash -r 2>/dev/null || true
+ _harness_node_at_least "$min" && return 0
+ PATH="$saved"; export PATH; hash -r 2>/dev/null || true
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min — this machine's newest is $("${recorded%/*}/node" -v), Harness's own; update Harness"
+ return 1
+ fi
+ [ "$own" = 1 ] && return 0
+ echo "miss node >= $min, and Harness's own Node is not in $HARNESS_RUNTIME — run \`harness start\` once to lay it down"
+ return 1
+}
+
+harness_uv() {
+ local dir asset sum tmp
+ command -v uv >/dev/null 2>&1 && return 0
+ for dir in "${HOME:-}/.local/bin" "${HOME:-}/.cargo/bin"; do
+ [ -n "${HOME:-}" ] || break
+ if [ -x "$dir/uv" ]; then PATH="$dir:$PATH"; export PATH; return 0; fi
+ done
+ dir="$HARNESS_RUNTIME/uv-$HARNESS_UV_VERSION"
+ if [ ! -x "$dir/uv" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=uv-aarch64-apple-darwin sum=dc304b9ed1b24174572290fba60ac3f6fe63c73a671f0439e62a91375841964d ;;
+ darwin-x64) asset=uv-x86_64-apple-darwin sum=e9ca61775532368fe518ab03e7a354c7ecab8ccb3c7d941c775fcc4a362b801b ;;
+ linux-x64) asset=uv-x86_64-unknown-linux-gnu sum=f97935763c04be3e692460a7aaeaaab8fc3b78fcf8b389da820b38ae7423a638 ;;
+ linux-arm64) asset=uv-aarch64-unknown-linux-gnu sum=0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08 ;;
+ *) echo "miss uv — there is no uv release for $(uname -s) $(uname -m); install it from https://docs.astral.sh/uv/"; return 1 ;;
+ esac
+ echo " fetching uv $HARNESS_UV_VERSION into $HARNESS_RUNTIME (it brings Python and the packages)"
+ mkdir -p "$HARNESS_RUNTIME" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ tmp="$(mktemp -d "$HARNESS_RUNTIME/.uv.XXXXXX")" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ if ! _harness_fetch "https://github.com/astral-sh/uv/releases/download/$HARNESS_UV_VERSION/$asset.tar.gz" "$tmp/uv.tar.gz" "$sum"; then
+ rm -rf "$tmp"; return 1
+ fi
+ tar -xzf "$tmp/uv.tar.gz" -C "$tmp" && chmod +x "$tmp/$asset/uv" "$tmp/$asset/uvx" \
+ || { rm -rf "$tmp"; echo "miss the uv archive would not unpack"; return 1; }
+ # Two installs at once both get here; whichever renames first wins and the other's copy goes.
+ # (mv onto a directory that exists would nest the copy inside it, hence the test.)
+ [ -e "$dir" ] || mv "$tmp/$asset" "$dir" 2>/dev/null || true
+ rm -rf "$tmp"
+ [ -x "$dir/uv" ] || { echo "miss uv did not land in $dir"; return 1; }
+ fi
+ PATH="$dir:$PATH"; export PATH
+}
+
+# harness_venv DIR VERSION [MIN [BELOW]] — MIN defaults to VERSION; BELOW is exclusive and optional.
+harness_venv() {
+ local dir="${1:?harness_venv needs a directory}" want="${2:?harness_venv needs a Python version}"
+ local min="${3:-$2}" below="${4:-}"
+ if [ -x "$dir/bin/python" ] && "$dir/bin/python" -c '
+import sys
+at = lambda s: tuple(int(p) for p in s.split("."))
+v = sys.version_info[:2]
+sys.exit(0 if v >= at(sys.argv[1]) and (not sys.argv[2] or v < at(sys.argv[2])) else 1)
+' "$min" "$below" 2>/dev/null; then
+ echo "ok $("$dir/bin/python" --version) in $dir"
+ return 0
+ fi
+ harness_uv || return 1
+ echo " python $want in $dir (uv downloads it when this machine has none)"
+ rm -rf "$dir"
+ # only-managed: the same python-build-standalone interpreter on every machine, so a package tested
+ # once is tested everywhere — not whichever Homebrew or pyenv Python happened to be first on PATH.
+ uv venv --quiet --seed --python "$want" --python-preference only-managed "$dir" \
+ || { echo "miss could not make a Python $want environment in $dir"; return 1; }
+ echo "ok $("$dir/bin/python" --version) in $dir"
+}
+
+harness_pip() {
+ local dir="${1:?harness_pip needs a venv}"
+ shift
+ harness_uv || return 1
+ uv pip install --quiet --python "$dir/bin/python" "$@"
+}
+
+harness_micromamba() {
+ local dir asset sum
+ dir="$HARNESS_RUNTIME/micromamba-$HARNESS_MICROMAMBA_VERSION"
+ if [ ! -x "$dir/micromamba" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=micromamba-osx-arm64 sum=ec2a072f028e1a7cf20f3e2e74d5a8127cf5a5f27636375b5359811565f4e5be ;;
+ darwin-x64) asset=micromamba-osx-64 sum=1e71054bb3ac9a076e21f7ec48acfef536f9b3f1408f371a942784bf5ef83d8a ;;
+ linux-x64) asset=micromamba-linux-64 sum=366cd9cd8be14df1ab8ed50352a82111082a36686b2d389fdb79a92c3fafb3e3 ;;
+ linux-arm64) asset=micromamba-linux-aarch64 sum=9f93b974adcb4d166996af969b6cd371287d1a3e52733704727884d9b74cb7a7 ;;
+ *) echo "miss micromamba — there is no release for $(uname -s) $(uname -m)"; return 1 ;;
+ esac
+ echo " fetching micromamba $HARNESS_MICROMAMBA_VERSION into $HARNESS_RUNTIME (native libraries PyPI has no wheel for)"
+ mkdir -p "$dir" || { echo "miss cannot write $dir"; return 1; }
+ _harness_fetch "https://github.com/mamba-org/micromamba-releases/releases/download/$HARNESS_MICROMAMBA_VERSION/$asset" "$dir/micromamba.$$" "$sum" || return 1
+ chmod +x "$dir/micromamba.$$" && mv -f "$dir/micromamba.$$" "$dir/micromamba" \
+ || { rm -f "$dir/micromamba.$$"; echo "miss micromamba would not install into $dir"; return 1; }
+ fi
+ HARNESS_MICROMAMBA="$dir/micromamba"
+ export HARNESS_MICROMAMBA
+}
+
+# harness_conda_env DIR SPEC… — conda-forge only, packages cached under ~/.harness/runtime/mamba.
+harness_conda_env() {
+ local dir="${1:?harness_conda_env needs a directory}"
+ shift
+ # micromamba takes a relative --prefix as a name under its root prefix and "succeeds" there.
+ case "$dir" in /*) ;; *) dir="$PWD/$dir" ;; esac
+ harness_micromamba || return 1
+ MAMBA_ROOT_PREFIX="$HARNESS_RUNTIME/mamba" "$HARNESS_MICROMAMBA" create --yes --quiet \
+ --prefix "$dir" --override-channels --channel conda-forge "$@" >/dev/null \
+ && [ -d "$dir/conda-meta" ] \
+ || { echo "miss could not make the conda-forge environment in $dir ($*)"; return 1; }
+}
diff --git a/store/agents/strudel/toolchain/setup.sh b/store/agents/strudel/toolchain/setup.sh
index 7e184f60e..0480ef92e 100755
--- a/store/agents/strudel/toolchain/setup.sh
+++ b/store/agents/strudel/toolchain/setup.sh
@@ -1,11 +1,14 @@
#!/usr/bin/env bash
# Runs once at install, cwd = the install dir. Strudel's REPL — the whole engine — into this
# package's node_modules, from the lockfile, as npm publishes it. Nothing global, nothing vendored
-# into this package, no CDN at run time.
+# into this package, no CDN at run time. Node is this machine's, or the one Harness itself runs on
+# (npm comes beside it).
set -euo pipefail
cd "$(dirname "$0")/.."
-command -v node >/dev/null 2>&1 || { echo "miss node >= 18 on PATH"; exit 1; }
-command -v npm >/dev/null 2>&1 || { echo "miss npm on PATH"; exit 1; }
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
+harness_node 18 || exit 1
+command -v npm >/dev/null 2>&1 || { echo "miss npm beside $(command -v node)"; exit 1; }
command -v python3 >/dev/null 2>&1 || { echo "miss python3 (the verdict)"; exit 1; }
echo " npm ci (@strudel/repl $(node -p "require('./package.json').dependencies['@strudel/repl']"))"
npm ci --silent --no-audit --no-fund
diff --git a/store/agents/strudel/toolchain/test_scripts.py b/store/agents/strudel/toolchain/test_scripts.py
index 23e1ca3dc..66316e1b5 100644
--- a/store/agents/strudel/toolchain/test_scripts.py
+++ b/store/agents/strudel/toolchain/test_scripts.py
@@ -3,8 +3,8 @@
Each test builds a package root in a temp dir with this checkout's scripts symlinked in file by file,
and runs them with PATH set to a bin dir that holds only the tools the test grants: real ones
(bash, node, python3, …) linked in, or stubs written here. So a "miss node" branch is a PATH without
-node, npm is a stub that lays out node_modules or fails, and nothing touches the network or this
-checkout.
+node and an empty runtime dir, Harness's own node is one recorded in that dir, npm is a stub that lays
+out node_modules or fails, and nothing touches the network or this checkout.
"""
from __future__ import annotations
@@ -19,7 +19,7 @@ from pathlib import Path
HERE = Path(__file__).resolve().parent
PKG = HERE.parent
REPL_VERSION = json.loads((PKG / "package.json").read_text())["dependencies"]["@strudel/repl"]
-BASE_TOOLS = ("bash", "dirname", "mkdir")
+BASE_TOOLS = ("bash", "dirname", "mkdir", "cat")
class Sandbox:
@@ -30,6 +30,8 @@ class Sandbox:
self.root = self.dir / "strudel"
self.bin = self.dir / "bin"
self.bin.mkdir()
+ self.runtime = self.dir / "runtime" # runtimes.sh looks here for Harness's node, never in ~
+ self.runtime.mkdir()
(self.root / "toolchain").mkdir(parents=True)
for tool in BASE_TOOLS + tools:
self.grant(tool)
@@ -42,15 +44,28 @@ class Sandbox:
assert found, f"{tool} is needed on this machine to run the test"
(self.bin / tool).symlink_to(found)
- def stub(self, name: str, body: str) -> None:
- path = self.bin / name
+ def stub(self, name: str, body: str, where: Path | None = None) -> Path:
+ path = (where or self.bin) / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.unlink(missing_ok=True) # a granted tool is a link to the real one: never write through it
path.write_text("#!/bin/bash\n" + body)
path.chmod(0o755)
-
- def run(self, rel: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
+ return path
+
+ def harness_node(self) -> Path:
+ """The real node, linked where Harness keeps its own and recorded in the runtime dir — not on PATH."""
+ found = shutil.which("node")
+ assert found, "node is needed on this machine to run the test"
+ node = self.dir / "harness-node" / "bin" / "node"
+ node.parent.mkdir(parents=True)
+ node.symlink_to(found)
+ (self.runtime / "current-node").write_text(f"{node}\n")
+ return node
+
+ def run(self, rel: str, *args: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
clean = {k: v for k, v in os.environ.items() if not k.startswith("HARNESS_")}
- return subprocess.run([str(self.root / rel)], cwd=cwd or self.dir, capture_output=True, text=True,
- env={**clean, "PATH": str(self.bin), **env}, timeout=60)
+ return subprocess.run([str(self.root / rel), *args], cwd=cwd or self.dir, capture_output=True, text=True,
+ env={**clean, "PATH": str(self.bin), "ADAPTER_RUNTIME_DIR": str(self.runtime), **env}, timeout=60)
def lines(done: subprocess.CompletedProcess) -> list[str]:
@@ -61,18 +76,21 @@ class Setup(unittest.TestCase):
def sandbox(self, *tools: str) -> Sandbox:
box = Sandbox(self, tools)
box.link("toolchain/setup.sh")
+ box.link("toolchain/runtimes.sh")
box.link("package.json")
return box
def test_node_is_required(self) -> None:
- done = self.sandbox().run("toolchain/setup.sh")
+ box = self.sandbox()
+ done = box.run("toolchain/setup.sh")
self.assertEqual(done.returncode, 1)
- self.assertEqual(lines(done), ["miss node >= 18 on PATH"])
+ self.assertEqual(lines(done), [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down"])
def test_npm_is_required(self) -> None:
- done = self.sandbox("node").run("toolchain/setup.sh")
+ box = self.sandbox("node")
+ done = box.run("toolchain/setup.sh")
self.assertEqual(done.returncode, 1)
- self.assertEqual(lines(done), ["miss npm on PATH"])
+ self.assertEqual(lines(done), [f"miss npm beside {box.bin}/node"])
def test_python3_is_required(self) -> None:
box = self.sandbox("node")
@@ -81,15 +99,26 @@ class Setup(unittest.TestCase):
self.assertEqual(done.returncode, 1)
self.assertEqual(lines(done), ["miss python3 (the verdict)"])
- def npm(self, box: Sandbox, body: str) -> None:
- box.stub("npm", f'echo "$PWD $*" >> "{box.dir}/npm.calls"\n' + body)
+ def npm(self, box: Sandbox, body: str, where: Path | None = None) -> None:
+ box.stub("npm", f'echo "$PWD $*" >> "{box.dir}/npm.calls"\n' + body, where=where)
- def test_npm_ci_lays_out_the_repl(self) -> None:
- box = self.sandbox("node", "python3")
- self.npm(box, f"""mkdir -p node_modules/@strudel/repl/dist
+ NPM_CI = f"""mkdir -p node_modules/@strudel/repl/dist
echo 'export {{}}' > node_modules/@strudel/repl/dist/index.js
echo '{{"name": "@strudel/repl", "version": "{REPL_VERSION}"}}' > node_modules/@strudel/repl/package.json
-""")
+"""
+
+ def test_a_machine_without_node_uses_harnesss_own_and_the_npm_beside_it(self) -> None:
+ box = self.sandbox("python3")
+ node = box.harness_node()
+ self.npm(box, self.NPM_CI, where=node.parent)
+ done = box.run("toolchain/setup.sh")
+ self.assertEqual(done.returncode, 0, done.stderr)
+ self.assertEqual(lines(done)[-1], f"ok strudel {REPL_VERSION} · AGPL-3.0-or-later, from npm, unmodified")
+ self.assertEqual((box.dir / "npm.calls").read_text(), f"{box.root} ci --silent --no-audit --no-fund\n")
+
+ def test_npm_ci_lays_out_the_repl(self) -> None:
+ box = self.sandbox("node", "python3")
+ self.npm(box, self.NPM_CI)
done = box.run("toolchain/setup.sh")
self.assertEqual(done.returncode, 0, done.stderr)
self.assertEqual(lines(done), [
@@ -118,6 +147,7 @@ class Doctor(unittest.TestCase):
def test_a_ready_machine(self) -> None:
box = Sandbox(self, ("node", "python3"))
box.link("toolchain/doctor.sh")
+ box.link("toolchain/runtimes.sh")
repl = box.root / "node_modules" / "@strudel" / "repl"
(repl / "dist").mkdir(parents=True)
(repl / "dist" / "index.js").write_text("export {}\n")
@@ -125,19 +155,29 @@ class Doctor(unittest.TestCase):
done = box.run("toolchain/doctor.sh")
self.assertEqual(done.returncode, 0, done.stdout + done.stderr)
out = lines(done)
- self.assertEqual(out[0], "ok strudel 9.9.9 (the pane's REPL)")
- self.assertRegex(out[1], r"^ok node v\d+\.\d+\.\d+ \(the pane server and the syntax check\)$")
+ self.assertRegex(out[0], r"^ok node v\d+\.\d+\.\d+ \(the pane server and the syntax check\)$")
+ self.assertEqual(out[1], "ok strudel 9.9.9 (the pane's REPL)")
self.assertRegex(out[2], r"^ok Python 3\.\d+\.\d+ \(the verdict\)$")
self.assertTrue(out[3].startswith("note the pane needs a click to start audio"))
self.assertEqual(len(out), 4)
+ def test_harness_node_serves_when_path_has_none(self) -> None:
+ box = Sandbox(self, ("python3",))
+ box.link("toolchain/doctor.sh")
+ box.link("toolchain/runtimes.sh")
+ box.harness_node()
+ done = box.run("toolchain/doctor.sh")
+ self.assertRegex(lines(done)[0], r"^ok node v\d+\.\d+\.\d+ \(the pane server and the syntax check\)$")
+
def test_an_empty_machine(self) -> None:
box = Sandbox(self)
box.link("toolchain/doctor.sh")
+ box.link("toolchain/runtimes.sh")
done = box.run("toolchain/doctor.sh")
self.assertEqual(done.returncode, 1)
out = lines(done)
- self.assertEqual(out[:3], ["miss node_modules — run toolchain/setup.sh", "miss node >= 18", "miss python3"])
+ self.assertEqual(out[:3], [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down",
+ "miss node_modules — run toolchain/setup.sh", "miss python3"])
self.assertTrue(out[3].startswith("note "))
@@ -176,10 +216,12 @@ class InitWorkspace(unittest.TestCase):
class ViewerLauncher(unittest.TestCase):
- def sandbox(self) -> Sandbox:
+ def sandbox(self, node: bool = True) -> Sandbox:
box = Sandbox(self, ("pwd",))
box.link("viewer.sh")
- box.stub("node", 'echo "node $* in $PWD port=$HARNESS_VIEWER_PORT ws=$HARNESS_WORKSPACE"\n')
+ box.link("toolchain/runtimes.sh")
+ if node:
+ box.stub("node", 'echo "node $* in $PWD port=$HARNESS_VIEWER_PORT ws=$HARNESS_WORKSPACE"\n')
return box
def test_it_needs_a_port_and_a_workspace(self) -> None:
@@ -198,6 +240,46 @@ class ViewerLauncher(unittest.TestCase):
self.assertEqual(done.returncode, 0, done.stderr)
self.assertEqual(done.stdout, f"node {box.root}/viewer.mjs in {box.dir} port=4100 ws=/Users/example/track\n")
+ def test_a_login_shell_without_node_runs_harnesss_own(self) -> None:
+ box = self.sandbox(node=False)
+ node = box.stub("node", 'echo "harness node $*"\n', where=box.dir / "harness-node" / "bin")
+ (box.runtime / "current-node").write_text(f"{node}\n")
+ done = box.run("viewer.sh", HARNESS_VIEWER_PORT="4100", HARNESS_WORKSPACE="/Users/example/track")
+ self.assertEqual((done.returncode, done.stdout), (0, f"harness node {box.root}/viewer.mjs\n"), done.stderr)
+
+ def test_no_node_anywhere_never_starts(self) -> None:
+ box = self.sandbox(node=False)
+ done = box.run("viewer.sh", HARNESS_VIEWER_PORT="4100", HARNESS_WORKSPACE="/Users/example/track")
+ self.assertEqual(done.returncode, 1)
+ self.assertEqual(lines(done), [f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down"])
+
+
+class WithNode(unittest.TestCase):
+ """toolchain/with-node.sh, which the verdict runs `node --check` through."""
+
+ def sandbox(self, *tools: str) -> Sandbox:
+ box = Sandbox(self, tools)
+ box.link("toolchain/with-node.sh")
+ box.link("toolchain/runtimes.sh")
+ return box
+
+ def test_node_on_path_runs_the_command(self) -> None:
+ box = self.sandbox("node")
+ done = box.run("toolchain/with-node.sh", "node", "-e", "console.log('ran')")
+ self.assertEqual((done.returncode, done.stdout), (0, "ran\n"), done.stderr)
+
+ def test_harnesss_node_is_put_on_path_when_there_is_none(self) -> None:
+ box = self.sandbox()
+ node = box.harness_node()
+ done = box.run("toolchain/with-node.sh", "bash", "-c", 'command -v node')
+ self.assertEqual((done.returncode, done.stdout), (0, f"{node}\n"), done.stderr)
+
+ def test_no_node_is_127_with_the_miss_on_stderr(self) -> None:
+ box = self.sandbox()
+ done = box.run("toolchain/with-node.sh", "node", "--check")
+ self.assertEqual((done.returncode, done.stdout), (127, ""))
+ self.assertEqual(done.stderr, f"miss node >= 18, and Harness's own Node is not in {box.runtime} — run `harness start` once to lay it down\n")
+
if __name__ == "__main__":
unittest.main()
diff --git a/store/agents/strudel/toolchain/test_verdict.py b/store/agents/strudel/toolchain/test_verdict.py
index 08b9e1ca6..8fd270b8e 100644
--- a/store/agents/strudel/toolchain/test_verdict.py
+++ b/store/agents/strudel/toolchain/test_verdict.py
@@ -140,6 +140,12 @@ class Judging(unittest.TestCase):
mock.patch.object(verdict.subprocess, "run", side_effect=error):
self.assertEqual(verdict.node_check('s("sbd")'), [])
+ def test_no_node_on_path_or_in_harness_is_the_scan_alone(self) -> None:
+ done = subprocess.CompletedProcess(["with-node.sh"], 127, stdout="", stderr="miss node >= 18, and Harness's own Node is not in …\n")
+ with mock.patch.object(verdict.subprocess, "run", return_value=done) as run:
+ self.assertEqual(verdict.node_check('s("sbd")'), [])
+ self.assertEqual(run.call_args.args[0], [str(HERE / "with-node.sh"), "node", "--check", "--input-type=module"])
+
def test_a_node_failure_without_a_syntax_error_line(self) -> None:
done = subprocess.CompletedProcess(["node"], 1, stdout="", stderr="node: bad option\n")
with mock.patch.object(verdict.subprocess, "run", return_value=done):
diff --git a/store/agents/strudel/toolchain/verdict.py b/store/agents/strudel/toolchain/verdict.py
index 3dd31d85b..d3e9b1ef2 100644
--- a/store/agents/strudel/toolchain/verdict.py
+++ b/store/agents/strudel/toolchain/verdict.py
@@ -12,8 +12,9 @@ human confirms it.
Check is static. Strudel's own transpiler parses a pattern as ECMAScript 2022 with top-level await
allowed (`parse(input, { ecmaVersion: 2022, allowAwaitOutsideFunction: true })` in
@strudel/transpiler), so `node --check --input-type=module` is exactly the parser the REPL will use
-and is the real syntax gate here. Without node we fall back to a bracket/quote scanner. What is not
-checked: whether a name exists in Strudel's eval scope, and whether it sounds good.
+and is the real syntax gate here. node is this machine's or, when the agent's shell has none on PATH,
+the one Harness runs on (with-node.sh finds it). Without either we fall back to a bracket/quote
+scanner. What is not checked: whether a name exists in Strudel's eval scope, and whether it sounds good.
"""
from __future__ import annotations
@@ -27,6 +28,7 @@ from pathlib import Path
WS = Path(os.environ.get("HARNESS_WORKSPACE") or os.getcwd()).resolve()
DEFAULT = "track.strudel"
+WITH_NODE = Path(__file__).resolve().parent / "with-node.sh"
# Sounds superdough registers itself, with no bank to download: registerSynthSounds() (the four
# oscillators plus user/one, sbd, supersaw, pulse, bytebeat, bus and the four noises) and
@@ -123,12 +125,12 @@ def strip_code(text: str) -> tuple[str, list[dict], str]:
def node_check(text: str) -> list[dict]:
- """The REPL's own parser, when node is on PATH. Silent when node is not."""
+ """The REPL's own parser, when there is a node to run it. Silent when there is not (127)."""
try:
- done = subprocess.run(["node", "--check", "--input-type=module"], input=text, capture_output=True, text=True, timeout=20)
+ done = subprocess.run([str(WITH_NODE), "node", "--check", "--input-type=module"], input=text, capture_output=True, text=True, timeout=20)
except (OSError, subprocess.SubprocessError):
return []
- if done.returncode == 0:
+ if done.returncode in (0, 127):
return []
detail = ""
for raw in (done.stderr or "").splitlines():
diff --git a/store/agents/strudel/toolchain/with-node.sh b/store/agents/strudel/toolchain/with-node.sh
new file mode 100755
index 000000000..7ba169de9
--- /dev/null
+++ b/store/agents/strudel/toolchain/with-node.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# with-node.sh CMD [ARG…] — CMD with a node >= 18 on PATH: this machine's, else the one Harness itself
+# runs on. The verdict is run with python3 from the agent's shell, which may have no node at all.
+# No node anywhere is exit 127, as for a command not found, with runtimes.sh's miss line on stderr.
+# shellcheck source=runtimes.sh
+. "${0%/*}/runtimes.sh"
+harness_node 18 >&2 || exit 127
+exec "$@"
diff --git a/store/agents/strudel/viewer.sh b/store/agents/strudel/viewer.sh
index 9a40a19cc..af39f05f3 100755
--- a/store/agents/strudel/viewer.sh
+++ b/store/agents/strudel/viewer.sh
@@ -1,5 +1,10 @@
#!/usr/bin/env bash
-# The pane. Harness runs this here with HARNESS_VIEWER_PORT and HARNESS_WORKSPACE.
+# The pane. Harness runs this here with HARNESS_VIEWER_PORT and HARNESS_WORKSPACE, through a login
+# shell whose PATH may have no node: runtimes.sh finds this machine's or Harness's own.
set -euo pipefail
: "${HARNESS_VIEWER_PORT:?}"; : "${HARNESS_WORKSPACE:?}"
-exec node "$(cd "$(dirname "$0")" && pwd)/viewer.mjs"
+DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=toolchain/runtimes.sh
+. "$DIR/toolchain/runtimes.sh"
+harness_node 18 || exit 1
+exec node "$DIR/viewer.mjs"
diff --git a/store/agents/typst/toolchain/setup.sh b/store/agents/typst/toolchain/setup.sh
index 5a1520e20..a3d7bc9ec 100755
--- a/store/agents/typst/toolchain/setup.sh
+++ b/store/agents/typst/toolchain/setup.sh
@@ -1,21 +1,27 @@
#!/usr/bin/env bash
# Runs once at install, cwd = the install dir. Puts the pinned Typst release binary in bin/ — a
-# single static executable from typst/typst's GitHub release for this machine, nothing installed
-# outside this directory.
+# single static executable from typst/typst's GitHub release for this machine, checked against the
+# release's sha256 before anything is unpacked, nothing installed outside this directory.
set -euo pipefail
cd "$(dirname "$0")/.."
VERSION="$(cat TYPST_VERSION)"
+# GitHub's own digests for the release assets. They move with TYPST_VERSION: a bump pins them anew.
+SUMS_FOR=v0.15.1
case "$(uname -s)-$(uname -m)" in
- Darwin-arm64) ASSET="typst-aarch64-apple-darwin.tar.xz" ;;
- Darwin-x86_64) ASSET="typst-x86_64-apple-darwin.tar.xz" ;;
- Linux-x86_64) ASSET="typst-x86_64-unknown-linux-musl.tar.xz" ;;
- Linux-aarch64) ASSET="typst-aarch64-unknown-linux-musl.tar.xz" ;;
+ Darwin-arm64) ASSET="typst-aarch64-apple-darwin.tar.xz" SUM=48f62ed034aa3a7978309579ac6ca00045e2ef0da73114e8af27cfd8e74dc05a ;;
+ Darwin-x86_64) ASSET="typst-x86_64-apple-darwin.tar.xz" SUM=7f9fdd9584866245de9a79e0add8f9236fae6f40a8a45e2c4771ccc14db4e0fa ;;
+ Linux-x86_64) ASSET="typst-x86_64-unknown-linux-musl.tar.xz" SUM=a6d077d0a95eed5a2eba715b2dae06be954f624ccbf85758a03f389ded33118c ;;
+ Linux-aarch64) ASSET="typst-aarch64-unknown-linux-musl.tar.xz" SUM=5aa8d74a3d906e60ea12a66ac2f37f8eef1b14cbad7182a745e393a10c23dcee ;;
*) echo "miss no Typst release for $(uname -s)-$(uname -m)"; exit 1 ;;
esac
if [ -x bin/typst ] && bin/typst --version 2>/dev/null | grep -q "${VERSION#v}"; then echo "ok typst ${VERSION} already here"; exit 0; fi
+[ "$VERSION" = "$SUMS_FOR" ] || { echo "miss the checksums in toolchain/setup.sh are for typst ${SUMS_FOR}, not ${VERSION}"; exit 1; }
mkdir -p bin tmp
echo " downloading typst ${VERSION} (${ASSET})"
-curl -fsSL -o "tmp/${ASSET}" "https://github.com/typst/typst/releases/download/${VERSION}/${ASSET}"
+URL="https://github.com/typst/typst/releases/download/${VERSION}/${ASSET}"
+curl -fsSL --retry 3 -o "tmp/${ASSET}" "$URL" || { rm -rf tmp; echo "miss could not download $URL — check this machine's internet connection"; exit 1; }
+if command -v shasum >/dev/null 2>&1; then GOT="$(shasum -a 256 "tmp/${ASSET}" | cut -d' ' -f1)"; else GOT="$(sha256sum "tmp/${ASSET}" | cut -d' ' -f1)"; fi
+[ "$GOT" = "$SUM" ] || { rm -rf tmp; echo "miss ${ASSET} did not match its pinned checksum; nothing was installed"; exit 1; }
tar -xJf "tmp/${ASSET}" -C tmp
cp "tmp/${ASSET%.tar.xz}/typst" bin/typst && chmod +x bin/typst && rm -rf tmp
echo "ok $(bin/typst --version)"
diff --git a/store/agents/typst/toolchain/test_scripts.py b/store/agents/typst/toolchain/test_scripts.py
index 6c016cc8c..eebcd9b2a 100644
--- a/store/agents/typst/toolchain/test_scripts.py
+++ b/store/agents/typst/toolchain/test_scripts.py
@@ -4,15 +4,17 @@ without a network or a Typst on the machine:
python3 -m unittest toolchain/test_scripts.py
"""
-import os, shutil, subprocess, tempfile, unittest
+import os, re, shutil, subprocess, tempfile, unittest
from pathlib import Path
PACKAGE = Path(__file__).resolve().parent.parent
# A bash line tracer (BASH_ENV sourcing a DEBUG trap that appends to SHCOV_OUT) is passed through when set.
TRACER = {k: os.environ[k] for k in ("BASH_ENV", "SHCOV_OUT") if k in os.environ}
BASH = "/bin/bash"
-COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "chmod", "grep")
+COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "chmod", "grep", "cut")
VERSION = (PACKAGE / "TYPST_VERSION").read_text().strip()
+# The sha256 setup.sh pins for each release asset, read from the script itself.
+SUMS = dict(re.findall(r'ASSET="([^"]+)" SUM=([0-9a-f]{64})', (PACKAGE / "toolchain" / "setup.sh").read_text()))
class Sandbox:
@@ -39,6 +41,7 @@ class Sandbox:
def stub(self, name: str, body: str = "", where: Path | None = None) -> Path:
path = (where or self.bin) / name
path.parent.mkdir(parents=True, exist_ok=True)
+ path.unlink(missing_ok=True) # never write through a link to a real command
path.write_text(f'#!/bin/bash\necho "{name} $*" >> "$CALLS"\n{body}\n')
path.chmod(0o755)
return path
@@ -78,14 +81,21 @@ class Doctor(unittest.TestCase):
class Setup(unittest.TestCase):
def sandbox(self, system="Darwin", machine="arm64") -> Sandbox:
- box = Sandbox(self)
+ box = self.box = Sandbox(self)
box.stub("uname", f'[ "$1" = -s ] && echo {system} || echo {machine}')
# curl writes the archive it was asked for; tar unpacks a typst that knows its version.
box.stub("curl", 'while [ $# -gt 1 ]; do [ "$1" = -o ] && out="$2"; shift; done; echo archive > "$out"')
box.stub("tar", f'dir="$4/$(basename "$2" .tar.xz)"; mkdir -p "$dir"; printf \'#!/bin/bash\\necho "typst {VERSION[1:]} (fetched)"\\n\' > "$dir/typst"')
(box.bin / "basename").symlink_to("/usr/bin/basename")
+ self.sha("shasum")
return box
+ def sha(self, name: str) -> None:
+ """shasum (`-a 256 FILE`) or sha256sum (`FILE`) answering the pinned sum for the asset, or $BAD_SUM."""
+ box = self.box
+ cases = "".join(f' *{asset}) sum={digest} ;;\n' for asset, digest in SUMS.items())
+ box.stub(name, f'for file in "$@"; do :; done\ncase "$file" in\n{cases}esac\necho "${{BAD_SUM:-$sum}} $file"')
+
def test_downloads_the_release_for_each_machine(self):
for system, machine, asset in (("Darwin", "arm64", "typst-aarch64-apple-darwin.tar.xz"),
("Darwin", "x86_64", "typst-x86_64-apple-darwin.tar.xz"),
@@ -96,10 +106,34 @@ class Setup(unittest.TestCase):
r = box.run("setup.sh")
self.assertEqual(r.returncode, 0, r.stderr)
self.assertEqual(r.stdout.splitlines(), [f" downloading typst {VERSION} ({asset})", f"ok typst {VERSION[1:]} (fetched)"])
- self.assertIn(f"curl -fsSL -o tmp/{asset} https://github.com/typst/typst/releases/download/{VERSION}/{asset}", box.logged())
+ self.assertIn(f"curl -fsSL --retry 3 -o tmp/{asset} https://github.com/typst/typst/releases/download/{VERSION}/{asset}", box.logged())
+ self.assertIn(f"shasum -a 256 tmp/{asset}", box.logged())
self.assertTrue(os.access(box.install / "bin" / "typst", os.X_OK))
self.assertFalse((box.install / "tmp").exists(), "the download is cleaned up")
+ def test_a_machine_without_shasum_checks_with_sha256sum(self):
+ box = self.sandbox("Linux", "x86_64")
+ (box.bin / "shasum").unlink()
+ self.sha("sha256sum")
+ r = box.run("setup.sh")
+ self.assertEqual((r.returncode, r.stdout.splitlines()[-1]), (0, f"ok typst {VERSION[1:]} (fetched)"), r.stderr)
+ self.assertIn("sha256sum tmp/typst-x86_64-unknown-linux-musl.tar.xz", box.logged())
+
+ def test_an_archive_that_does_not_match_its_checksum_is_never_unpacked(self):
+ box = self.sandbox()
+ r = box.run("setup.sh", BAD_SUM="0" * 64)
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.splitlines()[-1], "miss typst-aarch64-apple-darwin.tar.xz did not match its pinned checksum; nothing was installed")
+ self.assertFalse(any(c.startswith("tar") for c in box.logged()))
+ self.assertFalse((box.install / "bin" / "typst").exists() or (box.install / "tmp").exists())
+
+ def test_a_version_the_checksums_are_not_for_is_a_miss(self):
+ box = self.sandbox()
+ (box.install / "TYPST_VERSION").write_text("v9.9.9\n")
+ r = box.run("setup.sh")
+ self.assertEqual((r.returncode, r.stdout.strip()), (1, f"miss the checksums in toolchain/setup.sh are for typst {VERSION}, not v9.9.9"))
+ self.assertFalse(any(c.startswith("curl") for c in box.logged()))
+
def test_an_unknown_machine_is_a_miss(self):
r = self.sandbox("FreeBSD", "riscv64").run("setup.sh")
self.assertEqual(r.returncode, 1)
@@ -123,8 +157,10 @@ class Setup(unittest.TestCase):
box = self.sandbox()
box.stub("curl", "exit 22")
r = box.run("setup.sh")
- self.assertNotEqual(r.returncode, 0)
- self.assertFalse((box.install / "bin" / "typst").exists())
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.splitlines()[-1], f"miss could not download https://github.com/typst/typst/releases/download/{VERSION}/"
+ "typst-aarch64-apple-darwin.tar.xz — check this machine's internet connection")
+ self.assertFalse((box.install / "bin" / "typst").exists() or (box.install / "tmp").exists())
class InitWorkspace(unittest.TestCase):