Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

store: Blender installs on a machine with no Python 3.11, no Homebrew and no ffmpeg

deee24e4e6c6262
diff --git a/store/agents/blender/toolchain/doctor.sh b/store/agents/blender/toolchain/doctor.sh
index 49b5e7ec6..f32f382bb 100755
--- a/store/agents/blender/toolchain/doctor.sh
+++ b/store/agents/blender/toolchain/doctor.sh
@@ -1,4 +1,4 @@
#!/usr/bin/env bash
set -u; cd "$(dirname "$0")/.."
if [ -x .venv/bin/python ] && .venv/bin/python -c 'import bpy' 2>/dev/null; then echo "ok blender $(.venv/bin/python -c 'import bpy; print(bpy.app.version_string)')"; else echo "miss .venv with bpy — run toolchain/setup.sh"; exit 1; fi
-if command -v ffmpeg >/dev/null 2>&1; then echo "ok ffmpeg (turntables)"; else echo "warn ffmpeg not on PATH — turntables stay as frames (brew install ffmpeg)"; fi
+if command -v ffmpeg >/dev/null 2>&1 || .venv/bin/python -c 'import imageio_ffmpeg; imageio_ffmpeg.get_ffmpeg_exe()' 2>/dev/null; then echo "ok ffmpeg (turntables)"; else echo "warn no ffmpeg — turntables stay as frames until toolchain/setup.sh runs again"; fi
diff --git a/store/agents/blender/toolchain/harness_blender.py b/store/agents/blender/toolchain/harness_blender.py
index ad5ce66ef..8531acd17 100644
--- a/store/agents/blender/toolchain/harness_blender.py
+++ b/store/agents/blender/toolchain/harness_blender.py
@@ -258,9 +258,9 @@ def turntable(path: str | Path = "out/turntable.mp4", seconds: float = 4.0, fps:
s.frame_start, s.frame_end, s.render.fps = saved["start"], saved["end"], saved["fps"]
s.frame_set(saved["current"])
bpy.context.preferences.edit.keyframe_new_interpolation_type = saved["interp"]
- ffmpeg = shutil.which("ffmpeg")
+ ffmpeg = _ffmpeg()
if ffmpeg is None:
- print(f"warn no ffmpeg on PATH — turntable frames are in {frames_dir}; brew install ffmpeg to get an mp4")
+ print(f"warn no ffmpeg — turntable frames are in {frames_dir}; run toolchain/setup.sh again to get an mp4")
return out
_activity("Encoding turntable")
subprocess.run([ffmpeg, "-y", "-loglevel", "error", "-framerate", str(fps), "-i", str(frames_dir / "frame_%04d.png"),
@@ -269,6 +269,18 @@ def turntable(path: str | Path = "out/turntable.mp4", seconds: float = 4.0, fps:
return out
+
+def _ffmpeg() -> str | None:
+ """The machine's ffmpeg, else the one imageio-ffmpeg carries in the venv (setup installs it)."""
+ found = shutil.which("ffmpeg")
+ if found:
+ return found
+ try:
+ import imageio_ffmpeg
+ return imageio_ffmpeg.get_ffmpeg_exe()
+ except Exception: # not installed, or its binary is missing for this platform
+ return None
+
# ---------------------------------------------------------------------------------------------------
# Exports
diff --git a/store/agents/blender/toolchain/runtimes.sh b/store/agents/blender/toolchain/runtimes.sh
new file mode 100644
index 000000000..7fe40936c
--- /dev/null
+++ b/store/agents/blender/toolchain/runtimes.sh
@@ -0,0 +1,176 @@
+# shellcheck shell=bash
+# runtimes.sh — the interpreters a store package runs on, found on this machine or fetched, never
+# asked of the person who pressed Get.
+#
+# Sourced (not run) by a package's setup, doctor and viewer scripts. A new Mac has Apple's Python 3.9,
+# no Homebrew and, for most people, no Node on PATH; a package that answers "brew install [email protected]"
+# has failed its install. So:
+#
+# harness_node 18 node >= 18 on PATH: the machine's own, else the Node Harness itself
+# runs on (~/.harness/runtime/current-node, laid down with the CLI)
+# harness_uv uv on PATH: the machine's own, else a pinned, checksummed release
+# fetched into ~/.harness/runtime
+# harness_venv DIR 3.12 [MIN [BELOW]]
+# DIR is a venv on CPython 3.12, which uv downloads when the machine
+# has none. A venv already there on a Python in [MIN, BELOW) is kept.
+# harness_pip DIR PKG… install into that venv, through uv
+# harness_micromamba micromamba, pinned and checksummed, for native libraries PyPI has
+# no wheel for (pycairo on macOS)
+# harness_conda_env DIR SPEC… DIR is a conda-forge environment with those packages
+#
+# Every function prints `miss …` and returns 1 when it cannot deliver, so `harness_node 18 || exit 1`
+# reads like the rest of a setup script. Nothing here touches the user's shell profile or anything
+# outside ~/.harness/runtime and the package's own directory.
+#
+# ONE copy is written by hand: store/tools/runtimes.sh in the OpenHarness repository. Each package
+# carries an identical copy (a package installs alone, so it cannot reach this one);
+# `node store/tools/sync-runtimes.mjs` rewrites them and the CLI's store spec fails when one drifts.
+
+HARNESS_RUNTIME="${ADAPTER_RUNTIME_DIR:-${HOME:-}/.harness/runtime}"
+HARNESS_UV_VERSION=0.12.15
+HARNESS_MICROMAMBA_VERSION=2.9.0-0
+
+_harness_platform() {
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64) echo darwin-arm64 ;;
+ Darwin-x86_64) echo darwin-x64 ;;
+ Linux-x86_64) echo linux-x64 ;;
+ Linux-aarch64 | Linux-arm64) echo linux-arm64 ;;
+ *) return 1 ;;
+ esac
+}
+
+_harness_sha256() {
+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1
+ else sha256sum "$1" | cut -d' ' -f1; fi
+}
+
+# _harness_fetch URL DEST SHA256 — DEST exists only if what arrived matches the pinned checksum.
+_harness_fetch() {
+ local part="$2.part.$$"
+ if ! curl -fsSL --retry 3 --connect-timeout 20 --max-time 900 -o "$part" "$1"; then
+ rm -f "$part"; echo "miss could not download $1 — check this machine's internet connection"; return 1
+ fi
+ if [ "$(_harness_sha256 "$part")" != "$3" ]; then
+ rm -f "$part"; echo "miss $1 did not match its pinned checksum; nothing was installed"; return 1
+ fi
+ mv -f "$part" "$2"
+}
+
+_harness_node_at_least() {
+ command -v node >/dev/null 2>&1 || return 1
+ node -e '
+ const [a, b = 0] = process.argv[1].split(".").map(Number)
+ const [x, y] = process.versions.node.split(".").map(Number)
+ process.exit(x > a || (x === a && y >= b) ? 0 : 1)
+ ' "$1" >/dev/null 2>&1
+}
+
+harness_node() {
+ local min="${1:-18}" recorded dir
+ _harness_node_at_least "$min" && return 0
+ recorded="$(cat "$HARNESS_RUNTIME/current-node" 2>/dev/null || true)"
+ if [ -n "$recorded" ] && [ -x "${recorded%/*}/node" ]; then
+ PATH="${recorded%/*}:$PATH"; export PATH; hash -r 2>/dev/null || true
+ _harness_node_at_least "$min" && return 0
+ echo "miss node >= $min — this machine's newest is $(node -v), Harness's own; update Harness"
+ return 1
+ fi
+ echo "miss node >= $min, and Harness's own Node is not in $HARNESS_RUNTIME — run \`harness start\` once to lay it down"
+ return 1
+}
+
+harness_uv() {
+ local dir asset sum tmp
+ command -v uv >/dev/null 2>&1 && return 0
+ for dir in "${HOME:-}/.local/bin" "${HOME:-}/.cargo/bin"; do
+ [ -n "${HOME:-}" ] || break
+ if [ -x "$dir/uv" ]; then PATH="$dir:$PATH"; export PATH; return 0; fi
+ done
+ dir="$HARNESS_RUNTIME/uv-$HARNESS_UV_VERSION"
+ if [ ! -x "$dir/uv" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=uv-aarch64-apple-darwin sum=dc304b9ed1b24174572290fba60ac3f6fe63c73a671f0439e62a91375841964d ;;
+ darwin-x64) asset=uv-x86_64-apple-darwin sum=e9ca61775532368fe518ab03e7a354c7ecab8ccb3c7d941c775fcc4a362b801b ;;
+ linux-x64) asset=uv-x86_64-unknown-linux-gnu sum=f97935763c04be3e692460a7aaeaaab8fc3b78fcf8b389da820b38ae7423a638 ;;
+ linux-arm64) asset=uv-aarch64-unknown-linux-gnu sum=0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08 ;;
+ *) echo "miss uv — there is no uv release for $(uname -s) $(uname -m); install it from https://docs.astral.sh/uv/"; return 1 ;;
+ esac
+ echo " fetching uv $HARNESS_UV_VERSION into $HARNESS_RUNTIME (it brings Python and the packages)"
+ mkdir -p "$HARNESS_RUNTIME" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ tmp="$(mktemp -d "$HARNESS_RUNTIME/.uv.XXXXXX")" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ if ! _harness_fetch "https://github.com/astral-sh/uv/releases/download/$HARNESS_UV_VERSION/$asset.tar.gz" "$tmp/uv.tar.gz" "$sum"; then
+ rm -rf "$tmp"; return 1
+ fi
+ tar -xzf "$tmp/uv.tar.gz" -C "$tmp" && chmod +x "$tmp/$asset/uv" "$tmp/$asset/uvx" \
+ || { rm -rf "$tmp"; echo "miss the uv archive would not unpack"; return 1; }
+ # Two installs at once both get here; whichever renames first wins and the other's copy goes.
+ mv "$tmp/$asset" "$dir" 2>/dev/null || true
+ rm -rf "$tmp"
+ [ -x "$dir/uv" ] || { echo "miss uv did not land in $dir"; return 1; }
+ fi
+ PATH="$dir:$PATH"; export PATH
+}
+
+# harness_venv DIR VERSION [MIN [BELOW]] — MIN defaults to VERSION; BELOW is exclusive and optional.
+harness_venv() {
+ local dir="${1:?harness_venv needs a directory}" want="${2:?harness_venv needs a Python version}"
+ local min="${3:-$2}" below="${4:-}"
+ if [ -x "$dir/bin/python" ] && "$dir/bin/python" -c '
+import sys
+at = lambda s: tuple(int(p) for p in s.split("."))
+v = sys.version_info[:2]
+sys.exit(0 if v >= at(sys.argv[1]) and (not sys.argv[2] or v < at(sys.argv[2])) else 1)
+' "$min" "$below" 2>/dev/null; then
+ echo "ok $("$dir/bin/python" --version) in $dir"
+ return 0
+ fi
+ harness_uv || return 1
+ echo " python $want in $dir (uv downloads it when this machine has none)"
+ rm -rf "$dir"
+ # only-managed: the same python-build-standalone interpreter on every machine, so a package tested
+ # once is tested everywhere — not whichever Homebrew or pyenv Python happened to be first on PATH.
+ uv venv --quiet --seed --python "$want" --python-preference only-managed "$dir" \
+ || { echo "miss could not make a Python $want environment in $dir"; return 1; }
+ echo "ok $("$dir/bin/python" --version) in $dir"
+}
+
+harness_pip() {
+ local dir="${1:?harness_pip needs a venv}"
+ shift
+ harness_uv || return 1
+ uv pip install --quiet --python "$dir/bin/python" "$@"
+}
+
+harness_micromamba() {
+ local dir asset sum
+ dir="$HARNESS_RUNTIME/micromamba-$HARNESS_MICROMAMBA_VERSION"
+ if [ ! -x "$dir/micromamba" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=micromamba-osx-arm64 sum=ec2a072f028e1a7cf20f3e2e74d5a8127cf5a5f27636375b5359811565f4e5be ;;
+ darwin-x64) asset=micromamba-osx-64 sum=1e71054bb3ac9a076e21f7ec48acfef536f9b3f1408f371a942784bf5ef83d8a ;;
+ linux-x64) asset=micromamba-linux-64 sum=366cd9cd8be14df1ab8ed50352a82111082a36686b2d389fdb79a92c3fafb3e3 ;;
+ linux-arm64) asset=micromamba-linux-aarch64 sum=9f93b974adcb4d166996af969b6cd371287d1a3e52733704727884d9b74cb7a7 ;;
+ *) echo "miss micromamba — there is no release for $(uname -s) $(uname -m)"; return 1 ;;
+ esac
+ echo " fetching micromamba $HARNESS_MICROMAMBA_VERSION into $HARNESS_RUNTIME (native libraries PyPI has no wheel for)"
+ mkdir -p "$dir" || { echo "miss cannot write $dir"; return 1; }
+ _harness_fetch "https://github.com/mamba-org/micromamba-releases/releases/download/$HARNESS_MICROMAMBA_VERSION/$asset" "$dir/micromamba.$$" "$sum" || return 1
+ chmod +x "$dir/micromamba.$$" && mv -f "$dir/micromamba.$$" "$dir/micromamba"
+ fi
+ HARNESS_MICROMAMBA="$dir/micromamba"
+ export HARNESS_MICROMAMBA
+}
+
+# harness_conda_env DIR SPEC… — conda-forge only, packages cached under ~/.harness/runtime/mamba.
+harness_conda_env() {
+ local dir="${1:?harness_conda_env needs a directory}"
+ shift
+ # micromamba takes a relative --prefix as a name under its root prefix and "succeeds" there.
+ case "$dir" in /*) ;; *) dir="$PWD/$dir" ;; esac
+ harness_micromamba || return 1
+ MAMBA_ROOT_PREFIX="$HARNESS_RUNTIME/mamba" "$HARNESS_MICROMAMBA" create --yes --quiet \
+ --prefix "$dir" --override-channels --channel conda-forge "$@" >/dev/null \
+ && [ -d "$dir/conda-meta" ] \
+ || { echo "miss could not make the conda-forge environment in $dir ($*)"; return 1; }
+}
diff --git a/store/agents/blender/toolchain/setup.sh b/store/agents/blender/toolchain/setup.sh
index e1644f216..0e7c421d4 100755
--- a/store/agents/blender/toolchain/setup.sh
+++ b/store/agents/blender/toolchain/setup.sh
@@ -1,16 +1,16 @@
#!/usr/bin/env bash
# Runs once at install, cwd = the install dir. One venv with `bpy` — Blender as a Python module, the
-# whole of Blender minus its window — at the pinned version. The wheel is Python-version specific.
+# whole of Blender minus its window — at the pinned version. The wheel is built for CPython 3.11 alone,
+# so the venv is on 3.11 whatever this machine has: uv downloads that Python when it is not here.
+# imageio-ffmpeg brings an ffmpeg for turntables, since bpy has no movie encoder of its own.
set -euo pipefail
cd "$(dirname "$0")/.."
+# shellcheck source=runtimes.sh
+. toolchain/runtimes.sh
VERSION="$(cat BPY_VERSION)"
-PY=""; for c in python3.11 python3; do if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import sys; raise SystemExit(0 if sys.version_info[:2] == (3, 11) else 1)' 2>/dev/null; then PY="$c"; break; fi; done
-[ -n "$PY" ] || { echo "miss python 3.11 exactly — the bpy ${VERSION} wheel is built for it (brew install [email protected])"; exit 1; }
-echo "ok $($PY --version)"
-[ -x .venv/bin/python ] || "$PY" -m venv .venv
-.venv/bin/python -m pip install --quiet --upgrade pip >/dev/null 2>&1 || true
+harness_venv .venv 3.11 3.11 3.12 || exit 1
echo " installing bpy ${VERSION} (Blender as a module, ~300 MB, a few minutes the first time)"
-.venv/bin/python -m pip install --quiet "bpy==${VERSION}" numpy
+harness_pip .venv "bpy==${VERSION}" numpy imageio-ffmpeg
echo "ok blender $(.venv/bin/python -c 'import bpy; print(bpy.app.version_string)')"
echo " render check (Workbench, headless)"
.venv/bin/python - <<'PY'
diff --git a/store/agents/blender/toolchain/test_harness_blender.py b/store/agents/blender/toolchain/test_harness_blender.py
index c7ad4ad30..1a9381b55 100644
--- a/store/agents/blender/toolchain/test_harness_blender.py
+++ b/store/agents/blender/toolchain/test_harness_blender.py
@@ -5,7 +5,7 @@
Every test starts from fresh() in its own scratch workspace; renders are 32×18 and turntables a few
frames, so the whole file runs in seconds.
"""
-import atexit, contextlib, importlib, io, json, os, shutil, struct, sys, tempfile, unittest
+import atexit, contextlib, importlib, io, json, os, shutil, struct, sys, tempfile, types, unittest
from pathlib import Path
from unittest import mock
@@ -244,8 +244,9 @@ class Renders(Base):
self.assertEqual((s.frame_start, s.frame_end, s.render.fps, s.frame_current), (5, 90, 24, 7))
self.assertEqual(bpy.context.preferences.edit.keyframe_new_interpolation_type, "BEZIER")
- @unittest.skipIf(shutil.which("ffmpeg") is None, "no ffmpeg on PATH")
def test_a_turntable_is_an_mp4_and_the_scene_is_left_as_found(self):
+ if hb._ffmpeg() is None:
+ self.skipTest("no ffmpeg on PATH or in the venv")
cam, matrix = self.scene_as_found()
out = hb.turntable("out/turntable.mp4", seconds=0.3, fps=10, size=SIZE)
self.assertTrue(out.is_file() and out.stat().st_size > 0)
@@ -260,16 +261,34 @@ class Renders(Base):
frames.mkdir(parents=True)
(frames / "frame_0099.png").write_bytes(b"stale")
printed = io.StringIO()
- with mock.patch.object(hb.shutil, "which", return_value=None), contextlib.redirect_stdout(printed):
+ with mock.patch.object(hb, "_ffmpeg", return_value=None), contextlib.redirect_stdout(printed):
out = hb.turntable("out/spin.mp4", seconds=0.2, fps=10, size=SIZE)
self.assertFalse(out.exists())
self.assertEqual(sorted(p.name for p in frames.iterdir()), ["frame_0001.png", "frame_0002.png"])
- self.assertIn("warn no ffmpeg on PATH", printed.getvalue())
+ self.assertIn(f"warn no ffmpeg — turntable frames are in {frames}; run toolchain/setup.sh again", printed.getvalue())
self.assert_scene_put_back(cam, matrix)
+ def test_ffmpeg_is_the_machines_first(self):
+ with mock.patch.object(hb.shutil, "which", return_value="/usr/local/bin/ffmpeg"):
+ self.assertEqual(hb._ffmpeg(), "/usr/local/bin/ffmpeg")
+
+ def test_ffmpeg_falls_back_to_the_one_imageio_ffmpeg_carries(self):
+ carried = types.SimpleNamespace(get_ffmpeg_exe=lambda: "/venv/imageio_ffmpeg/binaries/ffmpeg")
+ with mock.patch.object(hb.shutil, "which", return_value=None), mock.patch.dict(sys.modules, {"imageio_ffmpeg": carried}):
+ self.assertEqual(hb._ffmpeg(), "/venv/imageio_ffmpeg/binaries/ffmpeg")
+
+ def test_no_ffmpeg_anywhere_is_none(self):
+ def missing():
+ raise RuntimeError("no ffmpeg binary for this platform")
+ with mock.patch.object(hb.shutil, "which", return_value=None), \
+ mock.patch.dict(sys.modules, {"imageio_ffmpeg": types.SimpleNamespace(get_ffmpeg_exe=missing)}):
+ self.assertIsNone(hb._ffmpeg())
+ with mock.patch.object(hb.shutil, "which", return_value=None), mock.patch.dict(sys.modules, {"imageio_ffmpeg": None}):
+ self.assertIsNone(hb._ffmpeg(), "not installed: the import itself fails")
+
def test_a_turntable_shorter_than_a_frame_still_renders_one(self):
self.cube()
- with mock.patch.object(hb.shutil, "which", return_value=None), mock.patch.object(hb, "_activity", wraps=hb._activity) as activity, contextlib.redirect_stdout(io.StringIO()):
+ with mock.patch.object(hb, "_ffmpeg", return_value=None), mock.patch.object(hb, "_activity", wraps=hb._activity) as activity, contextlib.redirect_stdout(io.StringIO()):
hb.turntable("out/blink.mp4", seconds=0.04, fps=10, size=SIZE)
self.assertIn(mock.call("Rendering turntable 1/1", progress=1.0, force=False), activity.call_args_list,
"the progress handler ran (0 frames divided by zero in it)")
diff --git a/store/agents/blender/toolchain/test_scripts.py b/store/agents/blender/toolchain/test_scripts.py
index 925402ac9..a44f663ec 100644
--- a/store/agents/blender/toolchain/test_scripts.py
+++ b/store/agents/blender/toolchain/test_scripts.py
@@ -1,6 +1,7 @@
"""setup.sh, doctor.sh and init-workspace.sh, run for real against a scratch install whose PATH holds
only stub commands (and the few coreutils the scripts use), so every ok / warn / miss line is reached
-without a network, a Python 3.11 or a 300 MB bpy wheel:
+without a network, a Python 3.11 or a 300 MB bpy wheel. uv is a stub too: what runtimes.sh does when
+uv is not on PATH at all is store/tools/test_runtimes.py's business.
python3 -m unittest toolchain/test_scripts.py
"""
@@ -11,14 +12,14 @@ PACKAGE = Path(__file__).resolve().parent.parent
# A bash line tracer (BASH_ENV sourcing a DEBUG trap that appends to SHCOV_OUT) is passed through when set.
TRACER = {k: os.environ[k] for k in ("BASH_ENV", "SHCOV_OUT") if k in os.environ}
BASH = "/bin/bash"
-COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "chmod", "grep")
+COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "chmod", "grep", "mktemp", "cut", "shasum")
VERSION = (PACKAGE / "BPY_VERSION").read_text().strip()
-# The venv's python: answers the bpy import (IMPORT_BPY), its version string, pip (PIP_EXIT; the pip
-# self-upgrade always fails, which setup tolerates) and the heredoc render check (RENDER_EXIT).
+# The venv's python: its version (VENV_PY — what runtimes.sh's keep-or-replace probe answers is
+# VENV_PY_OK), the bpy import (IMPORT_BPY), bpy's version string and the heredoc render check (RENDER_EXIT).
VENV_PYTHON = """case "$1" in
- -c) case "$2" in *version_string*) echo "%s" ;; *) exit "${IMPORT_BPY:-0}" ;; esac ;;
- -m) case "$*" in *--upgrade*) exit 1 ;; *) exit "${PIP_EXIT:-0}" ;; esac ;;
+ --version) echo "Python ${VENV_PY:-3.11.9}" ;;
+ -c) case "$2" in *version_string*) echo "%s" ;; *version_info*) exit "${VENV_PY_OK:-0}" ;; *) exit "${IMPORT_BPY:-0}" ;; esac ;;
-) cat >/dev/null; [ "${RENDER_EXIT:-0}" = 0 ] || exit "$RENDER_EXIT"; echo "ok headless rendering works" ;;
esac""" % VERSION
@@ -47,17 +48,17 @@ class Sandbox:
def stub(self, name: str, body: str = "", where: Path | None = None) -> Path:
path = (where or self.bin) / name
path.parent.mkdir(parents=True, exist_ok=True)
+ path.unlink(missing_ok=True) # never write through a link to a real command
path.write_text(f'#!/bin/bash\necho "{name} $*" >> "$CALLS"\n{body}\n')
path.chmod(0o755)
return path
- def interpreter(self, name: str, version: str) -> Path:
- """A python on PATH: the 3.11 probe answers for `version`; `-m venv` makes a venv of stubs."""
+ def uv(self) -> Path:
+ """uv on PATH: `uv venv … DIR` makes a venv of stubs; `uv pip install` answers PIP_EXIT."""
venv_python = self.stub("python", VENV_PYTHON, where=self.root / "templates")
- return self.stub(name, f"""case "$1" in
- -c) [ "{version[:4]}" = 3.11 ] ;;
- --version) echo "Python {version}" ;;
- -m) mkdir -p "$3/bin" && cp "{venv_python}" "$3/bin/python" ;;
+ return self.stub("uv", f"""case "$1" in
+ venv) for last in "$@"; do :; done; mkdir -p "$last/bin" && cp "{venv_python}" "$last/bin/python" ;;
+ pip) exit "${{PIP_EXIT:-0}}" ;;
esac""")
def venv(self) -> Path:
@@ -80,12 +81,19 @@ class Doctor(unittest.TestCase):
r = box.run("doctor.sh")
self.assertEqual((r.returncode, r.stdout.splitlines()), (0, [f"ok blender {VERSION}", "ok ffmpeg (turntables)"]), r.stderr)
- def test_no_ffmpeg_is_only_a_warning(self):
+ def test_the_venvs_own_ffmpeg_serves(self):
box = Sandbox(self)
box.venv()
r = box.run("doctor.sh")
+ self.assertEqual((r.returncode, r.stdout.splitlines()), (0, [f"ok blender {VERSION}", "ok ffmpeg (turntables)"]), r.stderr)
+
+ def test_no_ffmpeg_is_only_a_warning(self):
+ box = Sandbox(self)
+ box.stub("python", 'case "$2" in *imageio_ffmpeg*) exit 1 ;; *version_string*) echo "%s" ;; esac' % VERSION,
+ where=box.install / ".venv" / "bin")
+ r = box.run("doctor.sh")
self.assertEqual(r.returncode, 0)
- self.assertEqual(r.stdout.splitlines()[-1], "warn ffmpeg not on PATH — turntables stay as frames (brew install ffmpeg)")
+ self.assertEqual(r.stdout.splitlines()[-1], "warn no ffmpeg — turntables stay as frames until toolchain/setup.sh runs again")
def test_no_venv_is_a_miss_and_stops(self):
box = Sandbox(self)
@@ -101,62 +109,60 @@ class Doctor(unittest.TestCase):
class Setup(unittest.TestCase):
- def test_installs_bpy_into_a_venv_and_checks_rendering(self):
+ def test_makes_a_3_11_venv_installs_bpy_and_checks_rendering(self):
box = Sandbox(self)
- box.interpreter("python3.11", "3.11.9")
+ box.uv()
r = box.run("setup.sh")
self.assertEqual(r.returncode, 0, r.stderr)
self.assertEqual(r.stdout.splitlines(), [
- "ok Python 3.11.9",
+ " python 3.11 in .venv (uv downloads it when this machine has none)",
+ "ok Python 3.11.9 in .venv",
f" installing bpy {VERSION} (Blender as a module, ~300 MB, a few minutes the first time)",
f"ok blender {VERSION}",
" render check (Workbench, headless)",
"ok headless rendering works",
])
log = box.logged()
- self.assertIn("python3.11 -m venv .venv", log)
- self.assertIn(f"python -m pip install --quiet bpy=={VERSION} numpy", log)
+ self.assertIn("uv venv --quiet --seed --python 3.11 --python-preference only-managed .venv", log)
+ self.assertIn(f"uv pip install --quiet --python .venv/bin/python bpy=={VERSION} numpy imageio-ffmpeg", log)
self.assertEqual(log[-1], "python -")
- def test_python3_serves_when_it_is_3_11(self):
+ def test_an_existing_3_11_venv_is_reused(self):
box = Sandbox(self)
- box.interpreter("python3", "3.11.2")
+ box.uv()
+ box.venv()
r = box.run("setup.sh")
- self.assertEqual((r.returncode, r.stdout.splitlines()[0]), (0, "ok Python 3.11.2"))
- self.assertIn("python3 -m venv .venv", box.logged())
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(r.stdout.splitlines()[0], "ok Python 3.11.9 in .venv")
+ self.assertFalse(any(line.startswith("uv venv") for line in box.logged()))
- def test_a_python3_11_that_is_another_version_is_passed_over(self):
+ def test_a_venv_on_another_python_is_replaced(self):
box = Sandbox(self)
- box.interpreter("python3.11", "3.12.0")
- box.interpreter("python3", "3.11.4")
- r = box.run("setup.sh")
- self.assertEqual((r.returncode, r.stdout.splitlines()[0]), (0, "ok Python 3.11.4"))
+ box.uv()
+ box.venv()
+ r = box.run("setup.sh", VENV_PY_OK="1")
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertIn("uv venv --quiet --seed --python 3.11 --python-preference only-managed .venv", box.logged())
- def test_no_python_3_11_is_a_miss(self):
+ def test_no_uv_and_no_network_is_a_miss(self):
box = Sandbox(self)
- box.interpreter("python3", "3.12.1")
- r = box.run("setup.sh")
+ box.stub("uname", 'case "$1" in -s) echo Darwin ;; -m) echo arm64 ;; esac')
+ box.stub("curl", "exit 6")
+ r = box.run("setup.sh", ADAPTER_RUNTIME_DIR=str(box.root / "runtime"))
self.assertEqual(r.returncode, 1)
- self.assertEqual(r.stdout.strip(), f"miss python 3.11 exactly — the bpy {VERSION} wheel is built for it (brew install [email protected])")
-
- def test_an_existing_venv_is_reused(self):
- box = Sandbox(self)
- box.interpreter("python3.11", "3.11.9")
- box.venv()
- r = box.run("setup.sh")
- self.assertEqual(r.returncode, 0, r.stderr)
- self.assertNotIn("python3.11 -m venv .venv", box.logged())
+ self.assertEqual(r.stdout.splitlines()[-1], "miss could not download https://github.com/astral-sh/uv/releases/download/"
+ "0.12.15/uv-aarch64-apple-darwin.tar.gz — check this machine's internet connection")
def test_a_failed_install_stops_setup(self):
box = Sandbox(self)
- box.interpreter("python3.11", "3.11.9")
+ box.uv()
r = box.run("setup.sh", PIP_EXIT="1")
self.assertNotEqual(r.returncode, 0)
self.assertNotIn(f"ok blender {VERSION}", r.stdout)
def test_a_render_check_that_fails_fails_setup(self):
box = Sandbox(self)
- box.interpreter("python3.11", "3.11.9")
+ box.uv()
r = box.run("setup.sh", RENDER_EXIT="3")
self.assertEqual(r.returncode, 3)
self.assertEqual(r.stdout.splitlines()[-1], " render check (Workbench, headless)")
diff --git a/store/tools/README.md b/store/tools/README.md
index b544c968b..ba7dfe725 100644
--- a/store/tools/README.md
+++ b/store/tools/README.md
@@ -11,3 +11,20 @@ node store/tools/dsh-delete.mjs <agentId> <machineId>
```
`<machineId>` is this machine's id from `harness status` / `~/.harness/cli/data/machines.json`.
+
+## runtimes.sh — what a package runs on
+
+A new machine has Apple's Python 3.9, no Homebrew and usually no Node on PATH, so a package's setup
+never asks the person to install an interpreter: it sources `runtimes.sh` and gets one. `harness_node`
+falls back to the Node Harness itself runs on, `harness_venv` makes a venv on a pinned CPython that
+uv downloads (uv itself is fetched, pinned and checksummed, when absent), and `harness_conda_env`
+covers native libraries PyPI has no wheel for. Everything lands in the package directory or
+`~/.harness/runtime`.
+
+A package installs alone, so each one carries a copy (`toolchain/runtimes.sh` in an agent,
+`runtimes.sh` in a viewer). Edit only `store/tools/runtimes.sh`, then:
+
+```sh
+node store/tools/sync-runtimes.mjs # rewrite every copy
+node store/tools/sync-runtimes.mjs --check # exit 1 on a copy that drifted or a package missing one
+```
diff --git a/store/tools/runtimes.sh b/store/tools/runtimes.sh
new file mode 100644
index 000000000..7fe40936c
--- /dev/null
+++ b/store/tools/runtimes.sh
@@ -0,0 +1,176 @@
+# shellcheck shell=bash
+# runtimes.sh — the interpreters a store package runs on, found on this machine or fetched, never
+# asked of the person who pressed Get.
+#
+# Sourced (not run) by a package's setup, doctor and viewer scripts. A new Mac has Apple's Python 3.9,
+# no Homebrew and, for most people, no Node on PATH; a package that answers "brew install [email protected]"
+# has failed its install. So:
+#
+# harness_node 18 node >= 18 on PATH: the machine's own, else the Node Harness itself
+# runs on (~/.harness/runtime/current-node, laid down with the CLI)
+# harness_uv uv on PATH: the machine's own, else a pinned, checksummed release
+# fetched into ~/.harness/runtime
+# harness_venv DIR 3.12 [MIN [BELOW]]
+# DIR is a venv on CPython 3.12, which uv downloads when the machine
+# has none. A venv already there on a Python in [MIN, BELOW) is kept.
+# harness_pip DIR PKG… install into that venv, through uv
+# harness_micromamba micromamba, pinned and checksummed, for native libraries PyPI has
+# no wheel for (pycairo on macOS)
+# harness_conda_env DIR SPEC… DIR is a conda-forge environment with those packages
+#
+# Every function prints `miss …` and returns 1 when it cannot deliver, so `harness_node 18 || exit 1`
+# reads like the rest of a setup script. Nothing here touches the user's shell profile or anything
+# outside ~/.harness/runtime and the package's own directory.
+#
+# ONE copy is written by hand: store/tools/runtimes.sh in the OpenHarness repository. Each package
+# carries an identical copy (a package installs alone, so it cannot reach this one);
+# `node store/tools/sync-runtimes.mjs` rewrites them and the CLI's store spec fails when one drifts.
+
+HARNESS_RUNTIME="${ADAPTER_RUNTIME_DIR:-${HOME:-}/.harness/runtime}"
+HARNESS_UV_VERSION=0.12.15
+HARNESS_MICROMAMBA_VERSION=2.9.0-0
+
+_harness_platform() {
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64) echo darwin-arm64 ;;
+ Darwin-x86_64) echo darwin-x64 ;;
+ Linux-x86_64) echo linux-x64 ;;
+ Linux-aarch64 | Linux-arm64) echo linux-arm64 ;;
+ *) return 1 ;;
+ esac
+}
+
+_harness_sha256() {
+ if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | cut -d' ' -f1
+ else sha256sum "$1" | cut -d' ' -f1; fi
+}
+
+# _harness_fetch URL DEST SHA256 — DEST exists only if what arrived matches the pinned checksum.
+_harness_fetch() {
+ local part="$2.part.$$"
+ if ! curl -fsSL --retry 3 --connect-timeout 20 --max-time 900 -o "$part" "$1"; then
+ rm -f "$part"; echo "miss could not download $1 — check this machine's internet connection"; return 1
+ fi
+ if [ "$(_harness_sha256 "$part")" != "$3" ]; then
+ rm -f "$part"; echo "miss $1 did not match its pinned checksum; nothing was installed"; return 1
+ fi
+ mv -f "$part" "$2"
+}
+
+_harness_node_at_least() {
+ command -v node >/dev/null 2>&1 || return 1
+ node -e '
+ const [a, b = 0] = process.argv[1].split(".").map(Number)
+ const [x, y] = process.versions.node.split(".").map(Number)
+ process.exit(x > a || (x === a && y >= b) ? 0 : 1)
+ ' "$1" >/dev/null 2>&1
+}
+
+harness_node() {
+ local min="${1:-18}" recorded dir
+ _harness_node_at_least "$min" && return 0
+ recorded="$(cat "$HARNESS_RUNTIME/current-node" 2>/dev/null || true)"
+ if [ -n "$recorded" ] && [ -x "${recorded%/*}/node" ]; then
+ PATH="${recorded%/*}:$PATH"; export PATH; hash -r 2>/dev/null || true
+ _harness_node_at_least "$min" && return 0
+ echo "miss node >= $min — this machine's newest is $(node -v), Harness's own; update Harness"
+ return 1
+ fi
+ echo "miss node >= $min, and Harness's own Node is not in $HARNESS_RUNTIME — run \`harness start\` once to lay it down"
+ return 1
+}
+
+harness_uv() {
+ local dir asset sum tmp
+ command -v uv >/dev/null 2>&1 && return 0
+ for dir in "${HOME:-}/.local/bin" "${HOME:-}/.cargo/bin"; do
+ [ -n "${HOME:-}" ] || break
+ if [ -x "$dir/uv" ]; then PATH="$dir:$PATH"; export PATH; return 0; fi
+ done
+ dir="$HARNESS_RUNTIME/uv-$HARNESS_UV_VERSION"
+ if [ ! -x "$dir/uv" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=uv-aarch64-apple-darwin sum=dc304b9ed1b24174572290fba60ac3f6fe63c73a671f0439e62a91375841964d ;;
+ darwin-x64) asset=uv-x86_64-apple-darwin sum=e9ca61775532368fe518ab03e7a354c7ecab8ccb3c7d941c775fcc4a362b801b ;;
+ linux-x64) asset=uv-x86_64-unknown-linux-gnu sum=f97935763c04be3e692460a7aaeaaab8fc3b78fcf8b389da820b38ae7423a638 ;;
+ linux-arm64) asset=uv-aarch64-unknown-linux-gnu sum=0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08 ;;
+ *) echo "miss uv — there is no uv release for $(uname -s) $(uname -m); install it from https://docs.astral.sh/uv/"; return 1 ;;
+ esac
+ echo " fetching uv $HARNESS_UV_VERSION into $HARNESS_RUNTIME (it brings Python and the packages)"
+ mkdir -p "$HARNESS_RUNTIME" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ tmp="$(mktemp -d "$HARNESS_RUNTIME/.uv.XXXXXX")" || { echo "miss cannot write $HARNESS_RUNTIME"; return 1; }
+ if ! _harness_fetch "https://github.com/astral-sh/uv/releases/download/$HARNESS_UV_VERSION/$asset.tar.gz" "$tmp/uv.tar.gz" "$sum"; then
+ rm -rf "$tmp"; return 1
+ fi
+ tar -xzf "$tmp/uv.tar.gz" -C "$tmp" && chmod +x "$tmp/$asset/uv" "$tmp/$asset/uvx" \
+ || { rm -rf "$tmp"; echo "miss the uv archive would not unpack"; return 1; }
+ # Two installs at once both get here; whichever renames first wins and the other's copy goes.
+ mv "$tmp/$asset" "$dir" 2>/dev/null || true
+ rm -rf "$tmp"
+ [ -x "$dir/uv" ] || { echo "miss uv did not land in $dir"; return 1; }
+ fi
+ PATH="$dir:$PATH"; export PATH
+}
+
+# harness_venv DIR VERSION [MIN [BELOW]] — MIN defaults to VERSION; BELOW is exclusive and optional.
+harness_venv() {
+ local dir="${1:?harness_venv needs a directory}" want="${2:?harness_venv needs a Python version}"
+ local min="${3:-$2}" below="${4:-}"
+ if [ -x "$dir/bin/python" ] && "$dir/bin/python" -c '
+import sys
+at = lambda s: tuple(int(p) for p in s.split("."))
+v = sys.version_info[:2]
+sys.exit(0 if v >= at(sys.argv[1]) and (not sys.argv[2] or v < at(sys.argv[2])) else 1)
+' "$min" "$below" 2>/dev/null; then
+ echo "ok $("$dir/bin/python" --version) in $dir"
+ return 0
+ fi
+ harness_uv || return 1
+ echo " python $want in $dir (uv downloads it when this machine has none)"
+ rm -rf "$dir"
+ # only-managed: the same python-build-standalone interpreter on every machine, so a package tested
+ # once is tested everywhere — not whichever Homebrew or pyenv Python happened to be first on PATH.
+ uv venv --quiet --seed --python "$want" --python-preference only-managed "$dir" \
+ || { echo "miss could not make a Python $want environment in $dir"; return 1; }
+ echo "ok $("$dir/bin/python" --version) in $dir"
+}
+
+harness_pip() {
+ local dir="${1:?harness_pip needs a venv}"
+ shift
+ harness_uv || return 1
+ uv pip install --quiet --python "$dir/bin/python" "$@"
+}
+
+harness_micromamba() {
+ local dir asset sum
+ dir="$HARNESS_RUNTIME/micromamba-$HARNESS_MICROMAMBA_VERSION"
+ if [ ! -x "$dir/micromamba" ]; then
+ case "$(_harness_platform)" in
+ darwin-arm64) asset=micromamba-osx-arm64 sum=ec2a072f028e1a7cf20f3e2e74d5a8127cf5a5f27636375b5359811565f4e5be ;;
+ darwin-x64) asset=micromamba-osx-64 sum=1e71054bb3ac9a076e21f7ec48acfef536f9b3f1408f371a942784bf5ef83d8a ;;
+ linux-x64) asset=micromamba-linux-64 sum=366cd9cd8be14df1ab8ed50352a82111082a36686b2d389fdb79a92c3fafb3e3 ;;
+ linux-arm64) asset=micromamba-linux-aarch64 sum=9f93b974adcb4d166996af969b6cd371287d1a3e52733704727884d9b74cb7a7 ;;
+ *) echo "miss micromamba — there is no release for $(uname -s) $(uname -m)"; return 1 ;;
+ esac
+ echo " fetching micromamba $HARNESS_MICROMAMBA_VERSION into $HARNESS_RUNTIME (native libraries PyPI has no wheel for)"
+ mkdir -p "$dir" || { echo "miss cannot write $dir"; return 1; }
+ _harness_fetch "https://github.com/mamba-org/micromamba-releases/releases/download/$HARNESS_MICROMAMBA_VERSION/$asset" "$dir/micromamba.$$" "$sum" || return 1
+ chmod +x "$dir/micromamba.$$" && mv -f "$dir/micromamba.$$" "$dir/micromamba"
+ fi
+ HARNESS_MICROMAMBA="$dir/micromamba"
+ export HARNESS_MICROMAMBA
+}
+
+# harness_conda_env DIR SPEC… — conda-forge only, packages cached under ~/.harness/runtime/mamba.
+harness_conda_env() {
+ local dir="${1:?harness_conda_env needs a directory}"
+ shift
+ # micromamba takes a relative --prefix as a name under its root prefix and "succeeds" there.
+ case "$dir" in /*) ;; *) dir="$PWD/$dir" ;; esac
+ harness_micromamba || return 1
+ MAMBA_ROOT_PREFIX="$HARNESS_RUNTIME/mamba" "$HARNESS_MICROMAMBA" create --yes --quiet \
+ --prefix "$dir" --override-channels --channel conda-forge "$@" >/dev/null \
+ && [ -d "$dir/conda-meta" ] \
+ || { echo "miss could not make the conda-forge environment in $dir ($*)"; return 1; }
+}
diff --git a/store/tools/sync-runtimes.mjs b/store/tools/sync-runtimes.mjs
new file mode 100644
index 000000000..726a7c156
--- /dev/null
+++ b/store/tools/sync-runtimes.mjs
@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+// Keep every package's copy of runtimes.sh identical to store/tools/runtimes.sh.
+//
+// A package installs alone (a sparse checkout of its own folder), so it cannot source a file outside
+// itself: each one that needs Python, uv or Harness's Node carries a copy — `toolchain/runtimes.sh`
+// in an agent, `runtimes.sh` beside setup.sh in a viewer.
+//
+// node store/tools/sync-runtimes.mjs rewrite every copy from the canonical file
+// node store/tools/sync-runtimes.mjs --check exit 1, naming each copy that differs or is missing
+//
+// "Missing" means a script in the package sources runtimes.sh and the package has no copy of it.
+import { existsSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'
+import { dirname, join, relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const store = dirname(dirname(fileURLToPath(import.meta.url)))
+
+/** Every package folder, with where its copy lives. */
+export function packages(root = store) {
+ const out = []
+ for (const kind of ['agents', 'viewers']) {
+ const base = join(root, kind)
+ if (!existsSync(base)) continue
+ for (const name of readdirSync(base, { withFileTypes: true })) {
+ if (!name.isDirectory()) continue
+ const dir = join(base, name.name)
+ out.push({ dir, copy: join(dir, kind === 'agents' ? 'toolchain' : '', 'runtimes.sh') })
+ }
+ }
+ return out
+}
+
+/** True when a shell script in the package (not its node_modules or venv) sources runtimes.sh. */
+export function sourcesRuntimes(dir) {
+ const walk = (d, depth) => {
+ for (const e of readdirSync(d, { withFileTypes: true })) {
+ if (['node_modules', '.venv', 'upstream', '.git'].includes(e.name)) continue
+ const p = join(d, e.name)
+ if (e.isDirectory()) { if (depth < 2 && walk(p, depth + 1)) return true; continue }
+ if (!e.name.endsWith('.sh') || e.name === 'runtimes.sh') continue
+ if (/^\s*(\.|source)\s+\S*runtimes\.sh/m.test(readFileSync(p, 'utf8'))) return true
+ }
+ return false
+ }
+ return walk(dir, 0)
+}
+
+export function check(root = store) {
+ const canonical = readFileSync(join(root, 'tools', 'runtimes.sh'), 'utf8')
+ const problems = []
+ for (const { dir, copy } of packages(root)) {
+ if (existsSync(copy)) {
+ if (readFileSync(copy, 'utf8') !== canonical) problems.push(`${relative(root, copy)} differs from tools/runtimes.sh`)
+ } else if (sourcesRuntimes(dir)) {
+ problems.push(`${relative(root, dir)} sources runtimes.sh but has no ${relative(dir, copy)}`)
+ }
+ }
+ return problems
+}
+
+export function sync(root = store) {
+ const canonical = readFileSync(join(root, 'tools', 'runtimes.sh'), 'utf8')
+ const written = []
+ for (const { dir, copy } of packages(root)) {
+ if (!existsSync(copy) && !sourcesRuntimes(dir)) continue
+ if (existsSync(copy) && readFileSync(copy, 'utf8') === canonical) continue
+ writeFileSync(copy, canonical)
+ written.push(relative(root, copy))
+ }
+ return written
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+ if (process.argv.includes('--check')) {
+ const problems = check()
+ for (const p of problems) console.error(p)
+ process.exit(problems.length ? 1 : 0)
+ }
+ for (const p of sync()) console.log(`wrote ${p}`)
+}