Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

autonomous-circuit: test the wrapper against a local stand-in; a failed fetch keeps the copy that works

deed996225f474a
diff --git a/store/agents/autonomous-circuit/README.md b/store/agents/autonomous-circuit/README.md
index ac403c1c9..d89f76fb8 100644
--- a/store/agents/autonomous-circuit/README.md
+++ b/store/agents/autonomous-circuit/README.md
@@ -28,4 +28,5 @@ to own the store entry, the wrapper moves into their repository and the registry
```sh
harness dsh check "$PWD" # conformance (warns: the project arrives with setup)
harness dsh install "$PWD" --link # this checkout as the installed agent
+python3 -m unittest discover -s toolchain # the scripts, against a local stand-in repository
```
diff --git a/store/agents/autonomous-circuit/toolchain/fetch-upstream.sh b/store/agents/autonomous-circuit/toolchain/fetch-upstream.sh
index c00070c45..05dd88c28 100755
--- a/store/agents/autonomous-circuit/toolchain/fetch-upstream.sh
+++ b/store/agents/autonomous-circuit/toolchain/fetch-upstream.sh
@@ -10,7 +10,9 @@ if [ -f upstream/.harness-commit ] && [ "$(cat upstream/.harness-commit)" = "${U
fi
command -v git >/dev/null 2>&1 || { echo "miss git on PATH"; exit 1; }
echo " fetching ${UPSTREAM_REPO} @ ${UPSTREAM_COMMIT:0:12} (${UPSTREAM_SPARSE_MODE} sparse: ${UPSTREAM_SPARSE})"
-rm -rf upstream.partial upstream
+# The copy in upstream/ stays until the new one is complete: a fetch that fails (offline, a bad pin)
+# leaves the install that worked.
+rm -rf upstream.partial
git init -q upstream.partial
git -C upstream.partial remote add origin "${UPSTREAM_REPO}"
# Split the patterns on spaces WITHOUT globbing: `/*` is a sparse pattern, not the filesystem root.
@@ -19,5 +21,6 @@ git -C upstream.partial sparse-checkout set --"${UPSTREAM_SPARSE_MODE}" -- "${pa
git -C upstream.partial fetch -q --depth 1 --filter=blob:none origin "${UPSTREAM_COMMIT}"
git -C upstream.partial checkout -q FETCH_HEAD
echo "${UPSTREAM_COMMIT}" > upstream.partial/.harness-commit
+rm -rf upstream
mv upstream.partial upstream
echo "ok ${UPSTREAM_NAME} @ ${UPSTREAM_COMMIT:0:12} fetched ($(du -sh upstream | cut -f1))"
diff --git a/store/agents/autonomous-circuit/toolchain/test_toolchain.py b/store/agents/autonomous-circuit/toolchain/test_toolchain.py
new file mode 100644
index 000000000..014a77587
--- /dev/null
+++ b/store/agents/autonomous-circuit/toolchain/test_toolchain.py
@@ -0,0 +1,304 @@
+"""python3 -m unittest discover -s toolchain — the wrapper's scripts, against a stand-in for the upstream repository.
+
+A bare git repository in a temp dir plays autonomous-ai/autonomous-circuit: two commits of a small tree with
+the folders the sparse patterns keep and the ones they leave behind, and stub `harness/toolchain/*.sh`
+scripts that print what they were handed. Each test lays out a throwaway install dir with the real
+scripts linked in and a VERSIONS that keeps the real sparse patterns but pins the stand-in. Git runs with
+GIT_ALLOW_PROTOCOL=file and no user or system config, so nothing here can reach the network, and the
+stand-in is checked byte for byte afterwards: the wrapper only ever reads upstream.
+"""
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+TOOLCHAIN = Path(__file__).resolve().parent
+PACKAGE = TOOLCHAIN.parent
+NAME = "autonomous-circuit"
+SCRIPTS = ("fetch-upstream.sh", "setup.sh", "doctor.sh", "init-workspace.sh", "viewer.sh")
+PROJECT_SCRIPTS = ("setup.sh", "doctor.sh", "init-workspace.sh", "viewer.sh")
+# A project script that says which one ran, with what, where; exits with $STUB_EXIT.
+STUB = '#!/bin/sh\necho "{name} dsh=$HARNESS_DSH_DIR toolchain=${{CIRCUIT_TOOLCHAIN:-}} pwd=$(pwd -P)"\nexit "${{STUB_EXIT:-0}}"\n'
+
+
+def versions_value(key: str) -> str:
+ match = re.search(rf'^{key}="([^"]*)"$', (PACKAGE / "VERSIONS").read_text(), re.M)
+ assert match, key
+ return match.group(1)
+
+
+def snapshot(root: Path) -> dict:
+ return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(root.rglob("*")) if p.is_file()}
+
+
+class Upstream:
+ """The stand-in repository, built once per test class."""
+
+ def __init__(self, tmp: Path):
+ self.tmp = tmp
+ (tmp / "gitconfig").write_text("")
+ self.env = {k: v for k, v in os.environ.items() if not k.startswith(("GIT_", "HARNESS_", "CIRCUIT_"))}
+ self.env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(tmp / "gitconfig"), GIT_ALLOW_PROTOCOL="file",
+ GIT_AUTHOR_NAME="Example", GIT_AUTHOR_EMAIL="[email protected]",
+ GIT_COMMITTER_NAME="Example", GIT_COMMITTER_EMAIL="[email protected]")
+ work = tmp / "work"
+ files = {
+ "README.md": "Autonomous Circuit\n",
+ "harness/AGENTS.md": "# Circuit\n",
+ "harness/template/product.json": "{}\n",
+ "skills/pcb/SKILL.md": "# pcb\n",
+ "toolchain/README.md": "tools\n",
+ "products/big/board.json": "{\"big\": true}\n",
+ "examples/demo/board.json": "{\"demo\": true}\n",
+ }
+ for rel, text in files.items():
+ (work / rel).parent.mkdir(parents=True, exist_ok=True)
+ (work / rel).write_text(text)
+ for name in PROJECT_SCRIPTS:
+ path = work / "harness" / "toolchain" / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(STUB.format(name=name[:-3]))
+ path.chmod(0o755)
+ self.git("init", "-q", "-b", "main", str(work))
+ self.git("-C", str(work), "add", "-A")
+ self.git("-C", str(work), "commit", "-q", "-m", "first")
+ self.first = self.git("-C", str(work), "rev-parse", "HEAD").strip()
+ (work / "harness" / "NEW.md").write_text("second\n")
+ self.git("-C", str(work), "add", "-A")
+ self.git("-C", str(work), "commit", "-q", "-m", "second")
+ self.second = self.git("-C", str(work), "rev-parse", "HEAD").strip()
+ self.bare = tmp / f"{NAME}.git"
+ self.git("clone", "-q", "--bare", str(work), str(self.bare))
+ # What GitHub allows: partial clones, and fetching a commit by its id.
+ self.git("-C", str(self.bare), "config", "uploadpack.allowFilter", "true")
+ self.git("-C", str(self.bare), "config", "uploadpack.allowAnySHA1InWant", "true")
+ self.url = self.bare.as_uri()
+
+ def git(self, *args: str) -> str:
+ return subprocess.run(["git", *args], env=self.env, check=True, capture_output=True, text=True).stdout
+
+
+class Wrapper(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.class_tmp = Path(tempfile.mkdtemp())
+ cls.upstream = Upstream(cls.class_tmp)
+ cls.pristine = snapshot(cls.upstream.bare)
+
+ @classmethod
+ def tearDownClass(cls):
+ shutil.rmtree(cls.class_tmp)
+
+ def setUp(self):
+ self.tmp = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, self.tmp)
+ self.pkg = self.tmp / NAME
+ (self.pkg / "toolchain").mkdir(parents=True)
+ for name in SCRIPTS:
+ (self.pkg / "toolchain" / name).symlink_to(TOOLCHAIN / name)
+ self.pin(self.upstream.first)
+
+ def tearDown(self):
+ self.assertEqual(snapshot(self.upstream.bare), self.pristine, "the upstream repository was written to")
+
+ def pin(self, commit: str):
+ text = (PACKAGE / "VERSIONS").read_text()
+ text = re.sub(r'^UPSTREAM_REPO=.*$', f'UPSTREAM_REPO="{self.upstream.url}"', text, flags=re.M)
+ text = re.sub(r'^UPSTREAM_COMMIT=.*$', f'UPSTREAM_COMMIT="{commit}"', text, flags=re.M)
+ (self.pkg / "VERSIONS").write_text(text)
+
+ def run_script(self, name: str, cwd: Path | None = None, path: str | None = None, **env: str):
+ full = dict(self.upstream.env, **env)
+ if path is not None:
+ full["PATH"] = path
+ return subprocess.run([str(self.pkg / "toolchain" / name)], cwd=cwd or self.pkg, env=full,
+ capture_output=True, text=True, timeout=120)
+
+ def fake_fetched(self, commit: str, scripts=PROJECT_SCRIPTS):
+ """An upstream/ as a fetch leaves it, without git: the commit file and the project's scripts."""
+ (self.pkg / "upstream" / "harness" / "toolchain").mkdir(parents=True)
+ (self.pkg / "upstream" / ".harness-commit").write_text(commit + "\n")
+ for name in scripts:
+ path = self.pkg / "upstream" / "harness" / "toolchain" / name
+ path.write_text(STUB.format(name=name[:-3]))
+ path.chmod(0o755)
+
+ def committed(self) -> str:
+ return (self.pkg / "upstream" / ".harness-commit").read_text().strip()
+
+ def real(self, path: Path) -> str:
+ return os.path.realpath(path)
+
+ # VERSIONS
+
+ def test_versions_pins_a_full_commit_of_the_projects_public_repository(self):
+ self.assertEqual(versions_value("UPSTREAM_NAME"), NAME)
+ self.assertEqual(versions_value("UPSTREAM_REPO"), f"https://github.com/autonomous-ai/{NAME}.git")
+ self.assertRegex(versions_value("UPSTREAM_COMMIT"), r"^[0-9a-f]{40}$")
+
+ # fetch-upstream.sh
+
+ def test_fetch_checks_out_the_pinned_commit_sparse_shallow_and_blobless(self):
+ run = self.run_script("fetch-upstream.sh")
+ self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
+ first = self.upstream.first
+ lines = run.stdout.splitlines()
+ self.assertEqual(lines[0], f" fetching {self.upstream.url} @ {first[:12]} "
+ f"({versions_value('UPSTREAM_SPARSE_MODE')} sparse: {versions_value('UPSTREAM_SPARSE')})")
+ self.assertRegex(lines[1], rf"^ok {NAME} @ {first[:12]} fetched \(\S+\)$")
+ self.assertEqual(len(lines), 2)
+ up = self.pkg / "upstream"
+ self.assertEqual(self.committed(), first)
+ self.assertEqual(self.upstream.git("-C", str(up), "rev-parse", "HEAD").strip(), first)
+ self.assertTrue((up / "harness" / "AGENTS.md").is_file())
+ self.assertTrue((up / "skills" / "pcb" / "SKILL.md").is_file())
+ self.assertTrue(os.access(up / "harness" / "toolchain" / "setup.sh", os.X_OK))
+ # The product library and the examples stay behind; so does the second commit.
+ self.assertFalse((up / "products").exists())
+ self.assertFalse((up / "examples").exists())
+ self.assertFalse((up / "harness" / "NEW.md").exists())
+ self.assertTrue((up / ".git" / "shallow").is_file())
+ self.assertEqual(self.upstream.git("-C", str(up), "config", "remote.origin.partialclonefilter").strip(), "blob:none")
+ self.assertFalse((self.pkg / "upstream.partial").exists())
+
+ def test_fetch_is_a_no_op_when_the_pin_is_already_there(self):
+ self.assertEqual(self.run_script("fetch-upstream.sh").returncode, 0)
+ (self.pkg / "upstream" / "kept").write_text("")
+ run = self.run_script("fetch-upstream.sh")
+ self.assertEqual(run.returncode, 0)
+ self.assertEqual(run.stdout, f"ok {NAME} @ {self.upstream.first[:12]} already fetched\n")
+ self.assertTrue((self.pkg / "upstream" / "kept").exists())
+
+ def test_a_new_pin_replaces_the_copy(self):
+ self.assertEqual(self.run_script("fetch-upstream.sh").returncode, 0)
+ (self.pkg / "upstream" / "stale").write_text("")
+ self.pin(self.upstream.second)
+ run = self.run_script("fetch-upstream.sh")
+ self.assertEqual(run.returncode, 0, run.stderr)
+ self.assertIn(f"ok {NAME} @ {self.upstream.second[:12]} fetched", run.stdout)
+ self.assertEqual(self.committed(), self.upstream.second)
+ self.assertTrue((self.pkg / "upstream" / "harness" / "NEW.md").is_file())
+ self.assertFalse((self.pkg / "upstream" / "stale").exists())
+
+ def test_a_failed_fetch_keeps_the_copy_that_works(self):
+ self.assertEqual(self.run_script("fetch-upstream.sh").returncode, 0)
+ self.pin("0123456789abcdef0123456789abcdef01234567") # not in the repository: a bad bump, or offline
+ run = self.run_script("fetch-upstream.sh")
+ self.assertNotEqual(run.returncode, 0)
+ self.assertNotIn("ok ", run.stdout)
+ self.assertEqual(self.committed(), self.upstream.first)
+ self.assertTrue((self.pkg / "upstream" / "harness" / "toolchain" / "doctor.sh").is_file())
+ # and the next attempt starts clean
+ self.pin(self.upstream.second)
+ self.assertEqual(self.run_script("fetch-upstream.sh").returncode, 0)
+ self.assertEqual(self.committed(), self.upstream.second)
+ self.assertFalse((self.pkg / "upstream.partial").exists())
+
+ def test_fetch_without_git_says_so(self):
+ bin_ = self.tmp / "bin"
+ bin_.mkdir()
+ for tool in ("bash", "cat"):
+ (bin_ / tool).symlink_to(shutil.which(tool))
+ run = self.run_script("fetch-upstream.sh", path=str(bin_))
+ self.assertEqual(run.returncode, 1)
+ self.assertEqual(run.stdout, "miss git on PATH\n")
+ self.assertFalse((self.pkg / "upstream").exists())
+
+ # setup.sh
+
+ def test_setup_fetches_then_runs_the_projects_own_setup_in_its_checkout(self):
+ run = self.run_script("setup.sh", cwd=self.tmp)
+ self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
+ lines = run.stdout.splitlines()
+ self.assertEqual(len(lines), 3, lines)
+ self.assertIn(f"ok {NAME} @ {self.upstream.first[:12]} fetched", lines[1])
+ self.assertTrue(lines[2].startswith("setup "), lines)
+ self.assertEqual(self.real(Path(self.dsh_dir(lines[2]))), self.real(self.pkg / "upstream"))
+ self.assertTrue(lines[2].endswith(f" pwd={self.real(self.pkg)}"), lines)
+
+ def dsh_dir(self, line: str) -> str:
+ return re.search(r"dsh=(\S+)", line).group(1)
+
+ def test_setup_fails_with_the_projects_setup(self):
+ run = self.run_script("setup.sh", STUB_EXIT="3")
+ self.assertEqual(run.returncode, 3)
+
+ def test_setup_stops_when_the_fetch_fails(self):
+ self.pin("0123456789abcdef0123456789abcdef01234567")
+ run = self.run_script("setup.sh")
+ self.assertNotEqual(run.returncode, 0)
+ self.assertNotIn("setup dsh=", run.stdout)
+
+ def test_setup_says_when_the_project_has_no_setup(self):
+ self.fake_fetched(versions_value_for(self.pkg), scripts=("doctor.sh",))
+ run = self.run_script("setup.sh")
+ self.assertEqual(run.returncode, 1)
+ self.assertEqual(run.stdout.splitlines(), [
+ f"ok {NAME} @ {self.upstream.first[:12]} already fetched",
+ "miss upstream/harness/toolchain/setup.sh",
+ ])
+
+ # doctor.sh
+
+ def test_doctor_before_setup(self):
+ run = self.run_script("doctor.sh", cwd=self.tmp)
+ self.assertEqual(run.returncode, 1)
+ self.assertEqual(run.stdout, f"miss {NAME} is not fetched — run toolchain/setup.sh\n")
+
+ def test_doctor_at_the_pin_hands_over_to_the_projects_doctor(self):
+ self.fake_fetched(self.upstream.first)
+ run = self.run_script("doctor.sh", cwd=self.tmp)
+ self.assertEqual(run.returncode, 0, run.stderr)
+ ok, project = run.stdout.splitlines()
+ self.assertEqual(ok, f"ok {NAME} @ {self.upstream.first[:12]}")
+ self.assertTrue(project.startswith("doctor "), project)
+ self.assertEqual(self.real(Path(self.dsh_dir(project))), self.real(self.pkg / "upstream"))
+ self.assertEqual(self.real(Path(re.search(r"toolchain=(\S+)", project).group(1))), self.real(self.pkg / "upstream" / "toolchain"))
+ self.assertTrue(project.endswith(f"pwd={self.real(self.pkg)}"), project)
+ self.assertEqual(self.run_script("doctor.sh", STUB_EXIT="4").returncode, 4)
+
+ def test_doctor_keeps_a_toolchain_the_environment_names(self):
+ self.fake_fetched(self.upstream.first)
+ run = self.run_script("doctor.sh", CIRCUIT_TOOLCHAIN="/Users/example/circuit/toolchain")
+ self.assertIn("toolchain=/Users/example/circuit/toolchain ", run.stdout)
+
+ def test_doctor_warns_when_the_copy_is_behind_the_pin(self):
+ self.fake_fetched(self.upstream.first)
+ self.pin(self.upstream.second)
+ run = self.run_script("doctor.sh")
+ self.assertEqual(run.returncode, 0)
+ lines = run.stdout.splitlines()
+ self.assertEqual(lines[0], f"warn {NAME} @ {self.upstream.first[:12]}, VERSIONS pins {self.upstream.second[:12]} — run toolchain/setup.sh")
+ self.assertTrue(lines[1].startswith("doctor "), lines)
+
+ def test_doctor_fails_when_the_project_has_no_doctor(self):
+ self.fake_fetched(self.upstream.first, scripts=())
+ run = self.run_script("doctor.sh")
+ self.assertNotEqual(run.returncode, 0)
+ self.assertEqual(run.stdout, f"ok {NAME} @ {self.upstream.first[:12]}\n")
+
+ # init-workspace.sh, viewer.sh
+
+ def test_init_and_viewer_run_the_projects_own_in_the_workspace(self):
+ self.fake_fetched(self.upstream.first)
+ ws = self.tmp / "workspace"
+ ws.mkdir()
+ for script, word in (("init-workspace.sh", "init-workspace"), ("viewer.sh", "viewer")):
+ run = self.run_script(script, cwd=ws)
+ self.assertEqual(run.returncode, 0, run.stderr)
+ self.assertTrue(run.stdout.startswith(f"{word} "), run.stdout)
+ self.assertEqual(self.real(Path(self.dsh_dir(run.stdout))), self.real(self.pkg / "upstream"))
+ self.assertTrue(run.stdout.rstrip().endswith(f"pwd={self.real(ws)}"), run.stdout)
+ self.assertEqual(self.run_script(script, cwd=ws, STUB_EXIT="5").returncode, 5)
+
+
+def versions_value_for(pkg: Path) -> str:
+ return re.search(r'^UPSTREAM_COMMIT="([^"]*)"$', (pkg / "VERSIONS").read_text(), re.M).group(1)
+
+
+if __name__ == "__main__":
+ unittest.main()