The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
typst: cover the verdict and the shell scripts, fix three verdict bugs
dee505c9d9bdd58
diff --git a/store/agents/typst/toolchain/test_scripts.py b/store/agents/typst/toolchain/test_scripts.pynew file mode 100644index 000000000..52e81777a--- /dev/null+++ b/store/agents/typst/toolchain/test_scripts.py@@ -0,0 +1,148 @@+"""setup.sh, doctor.sh and init-workspace.sh, run for real against a scratch install whose PATH holds+only stub commands (and the few coreutils the scripts use), so every ok / miss line is reached+without a network or a Typst on the machine:++ python3 -m unittest toolchain/test_scripts.py+"""+import os, shutil, subprocess, tempfile, unittest+from pathlib import Path++PACKAGE = Path(__file__).resolve().parent.parent+BASH = "/bin/bash"+COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "chmod", "grep")+VERSION = (PACKAGE / "TYPST_VERSION").read_text().strip()+++class Sandbox:+ """An install dir with the package's scripts, a bin/ of stubs as the whole PATH, and a log of+ every stub call."""++ def __init__(self, test: unittest.TestCase):+ tmp = tempfile.TemporaryDirectory()+ test.addCleanup(tmp.cleanup)+ self.root = Path(tmp.name)+ self.install = self.root / "install"+ (self.install / "toolchain").mkdir(parents=True)+ for script in PACKAGE.glob("toolchain/*.sh"):+ shutil.copy(script, self.install / "toolchain" / script.name)+ shutil.copy(PACKAGE / "TYPST_VERSION", self.install / "TYPST_VERSION")+ self.bin = self.root / "bin"+ self.bin.mkdir()+ self.calls = self.root / "calls.log"+ self.calls.touch()+ for name in COREUTILS:+ real = next(p for p in (Path("/bin") / name, Path("/usr/bin") / name) if p.exists())+ (self.bin / name).symlink_to(real)++ def stub(self, name: str, body: str = "", where: Path | None = None) -> Path:+ path = (where or self.bin) / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(f'#!/bin/bash\necho "{name} $*" >> "$CALLS"\n{body}\n')+ path.chmod(0o755)+ return path++ def run(self, script: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:+ return subprocess.run([BASH, str(self.install / "toolchain" / script)], cwd=cwd or self.install,+ env={"PATH": str(self.bin), "CALLS": str(self.calls), **env},+ capture_output=True, text=True, timeout=60)++ def logged(self) -> list[str]:+ return self.calls.read_text().splitlines()+++class Doctor(unittest.TestCase):+ def test_ready(self):+ box = Sandbox(self)+ box.stub("typst", f'echo "typst {VERSION[1:]} (abc)"', where=box.install / "bin")+ box.stub("python3", 'echo "Python 3.12.1"')+ r = box.run("doctor.sh")+ self.assertEqual(r.returncode, 0, r.stderr)+ self.assertEqual(r.stdout.splitlines(), [f"ok typst {VERSION[1:]} (abc)", "ok Python 3.12.1 for the verdict"])++ def test_no_typst_is_a_miss_and_stops(self):+ box = Sandbox(self)+ box.stub("python3", 'echo "Python 3.12.1"')+ r = box.run("doctor.sh")+ self.assertEqual(r.returncode, 1)+ self.assertEqual(r.stdout.splitlines(), ["miss bin/typst — run toolchain/setup.sh"])++ def test_no_python_is_a_miss(self):+ box = Sandbox(self)+ box.stub("typst", 'echo "typst 0.15.1"', where=box.install / "bin")+ r = box.run("doctor.sh")+ self.assertEqual(r.returncode, 1)+ self.assertEqual(r.stdout.splitlines()[-1], "miss python3 for the verdict")+++class Setup(unittest.TestCase):+ def sandbox(self, system="Darwin", machine="arm64") -> Sandbox:+ box = Sandbox(self)+ box.stub("uname", f'[ "$1" = -s ] && echo {system} || echo {machine}')+ # curl writes the archive it was asked for; tar unpacks a typst that knows its version.+ box.stub("curl", 'while [ $# -gt 1 ]; do [ "$1" = -o ] && out="$2"; shift; done; echo archive > "$out"')+ box.stub("tar", f'dir="$4/$(basename "$2" .tar.xz)"; mkdir -p "$dir"; printf \'#!/bin/bash\\necho "typst {VERSION[1:]} (fetched)"\\n\' > "$dir/typst"')+ (box.bin / "basename").symlink_to("/usr/bin/basename")+ return box++ def test_downloads_the_release_for_each_machine(self):+ for system, machine, asset in (("Darwin", "arm64", "typst-aarch64-apple-darwin.tar.xz"),+ ("Darwin", "x86_64", "typst-x86_64-apple-darwin.tar.xz"),+ ("Linux", "x86_64", "typst-x86_64-unknown-linux-musl.tar.xz"),+ ("Linux", "aarch64", "typst-aarch64-unknown-linux-musl.tar.xz")):+ with self.subTest(machine=f"{system}-{machine}"):+ box = self.sandbox(system, machine)+ r = box.run("setup.sh")+ self.assertEqual(r.returncode, 0, r.stderr)+ self.assertEqual(r.stdout.splitlines(), [f" downloading typst {VERSION} ({asset})", f"ok typst {VERSION[1:]} (fetched)"])+ self.assertIn(f"curl -fsSL -o tmp/{asset} https://github.com/typst/typst/releases/download/{VERSION}/{asset}", box.logged())+ self.assertTrue(os.access(box.install / "bin" / "typst", os.X_OK))+ self.assertFalse((box.install / "tmp").exists(), "the download is cleaned up")++ def test_an_unknown_machine_is_a_miss(self):+ r = self.sandbox("FreeBSD", "riscv64").run("setup.sh")+ self.assertEqual(r.returncode, 1)+ self.assertEqual(r.stdout.strip(), "miss no Typst release for FreeBSD-riscv64")++ def test_the_pinned_version_already_here_is_kept(self):+ box = self.sandbox()+ box.stub("typst", f'echo "typst {VERSION[1:]} (old install)"', where=box.install / "bin")+ r = box.run("setup.sh")+ self.assertEqual((r.returncode, r.stdout.strip()), (0, f"ok typst {VERSION} already here"))+ self.assertFalse(any(c.startswith("curl") for c in box.logged()))++ def test_another_version_is_replaced(self):+ box = self.sandbox()+ box.stub("typst", 'echo "typst 0.1.0"', where=box.install / "bin")+ r = box.run("setup.sh")+ self.assertEqual(r.returncode, 0, r.stderr)+ self.assertEqual(r.stdout.splitlines()[-1], f"ok typst {VERSION[1:]} (fetched)")++ def test_a_failed_download_fails_setup(self):+ box = self.sandbox()+ box.stub("curl", "exit 22")+ r = box.run("setup.sh")+ self.assertNotEqual(r.returncode, 0)+ self.assertFalse((box.install / "bin" / "typst").exists())+++class InitWorkspace(unittest.TestCase):+ def test_compiles_the_template_and_seeds_the_verdict(self):+ box = Sandbox(self)+ ws = box.root / "ws"+ ws.mkdir()+ box.stub("typst", "exit 1", where=box.install / "bin") # a failing compile does not stop init+ box.stub("python3", "exit 1")+ r = box.run("init-workspace.sh", cwd=ws, HARNESS_DSH_DIR=str(box.install))+ self.assertEqual(r.returncode, 0, r.stderr)+ self.assertTrue((ws / ".harness").is_dir() and (ws / "out").is_dir())+ self.assertEqual(box.logged(), ["typst compile main.typ out/main.pdf", f"python3 {box.install}/toolchain/verdict.py"])++ def test_needs_the_install_dir(self):+ box = Sandbox(self)+ r = box.run("init-workspace.sh", cwd=box.root)+ self.assertNotEqual(r.returncode, 0)+ self.assertIn("HARNESS_DSH_DIR", r.stderr)+++if __name__ == "__main__":+ unittest.main()diff --git a/store/agents/typst/toolchain/test_verdict.py b/store/agents/typst/toolchain/test_verdict.pyindex 7888a4f67..a32c5ae27 100644--- a/store/agents/typst/toolchain/test_verdict.py+++ b/store/agents/typst/toolchain/test_verdict.py@@ -1,9 +1,14 @@-import sys, unittest+import contextlib, io, json, os, runpy, sys, unittestfrom pathlib import Path+from unittest import mocksys.path.insert(0, str(Path(__file__).parent))import tempfile+import verdictfrom verdict import judge, page_count+HERE = Path(__file__).resolve().parent++class Judge(unittest.TestCase):def test_compiles_clean(self):v = judge("= Hi", "", 0, "out/main.pdf", 2)@@ -14,10 +19,24 @@ class Judge(unittest.TestCase):v = judge("#foo", diag, 1, None, None)self.assertFalse(v["ready"]); self.assertEqual(v["phases"][1]["state"], "failed")self.assertEqual(v["findings"][0], {"severity": "error", "kind": "typst", "message": "unknown variable: foo", "ref": "main.typ:3:5"})+ self.assertEqual(v["summary"], "1 error")def test_warning_keeps_review_open(self):diag = "warning: unused import\n ┌─ main.typ:1:1\n"v = judge("x", diag, 0, "out/main.pdf", 1)self.assertTrue(v["ready"]); self.assertEqual(v["phases"][2]["state"], "active")+ self.assertEqual(v["summary"], "main.pdf · 1 page · compiles")+ def test_errors_are_counted_and_a_diagnostic_without_a_place_has_no_ref(self):+ v = judge("x", "error: one\nerror: two\n", 1, "out/main.pdf", 3)+ self.assertEqual(v["summary"], "main.pdf · 3 pages · 2 errors")+ self.assertNotIn("ref", v["findings"][0])+ def test_a_failed_compile_without_diagnostics(self):+ v = judge("x", "", 101, None, None)+ self.assertFalse(v["ready"]); self.assertEqual(v["summary"], "not compiled")+ def test_an_empty_source_is_not_a_document_even_if_it_compiles(self):+ v = judge(" \n", "", 0, "out/main.pdf", 1)+ self.assertFalse(v["ready"])+ self.assertEqual([p["state"] for p in v["phases"]], ["active", "pending", "pending"])+class PageCount(unittest.TestCase):def test_page_labels_are_not_pages(self):@@ -25,5 +44,116 @@ class PageCount(unittest.TestCase):with tempfile.NamedTemporaryFile(suffix=".pdf") as f:f.write(body); f.flush()self.assertEqual(page_count(Path(f.name)), 2)+ def test_no_pages_or_no_file_is_none(self):+ with tempfile.TemporaryDirectory() as d:+ (Path(d) / "a.pdf").write_bytes(b"%PDF-1.7\n%%EOF")+ self.assertIsNone(page_count(Path(d) / "a.pdf"))+ self.assertIsNone(page_count(Path(d) / "missing.pdf"))+++# A stand-in for `typst compile --root WS src out`: a source containing "#foo" fails the way Typst+# does; anything else writes a two-page PDF, and "#warn" adds a warning.+FAKE_TYPST = r"""#!/bin/sh+src="$4"; out="$5"+if grep -q '#foo' "$src"; then printf 'error: unknown variable: foo\n \342\224\214\342\224\200 main.typ:1:1\n' >&2; exit 1; fi+grep -q '#warn' "$src" && printf 'warning: unknown font family: x\n \342\224\214\342\224\200 main.typ:2:3\n' >&2+printf '%%PDF-1.7\n<</Type /Page>>\n<</Type /Page>>\n' > "$out"+"""+++class Main(unittest.TestCase):+ """verdict.py on a workspace: what it compiles, what it writes, what it prints and returns."""++ def setUp(self):+ tmp = tempfile.TemporaryDirectory()+ self.addCleanup(tmp.cleanup)+ self.ws = Path(tmp.name) / "ws"+ self.ws.mkdir()+ self.typst = Path(tmp.name) / "typst"+ self.typst.write_text(FAKE_TYPST)+ self.typst.chmod(0o755)+ for name, value in (("WS", self.ws), ("TYPST", str(self.typst))):+ patcher = mock.patch.object(verdict, name, value)+ patcher.start()+ self.addCleanup(patcher.stop)++ def main(self, *argv):+ out = io.StringIO()+ with contextlib.redirect_stdout(out):+ code = verdict.main(["verdict.py", *argv])+ return code, out.getvalue(), json.loads((self.ws / ".harness" / "verdict.json").read_text())++ def test_a_document_that_compiles(self):+ (self.ws / "main.typ").write_text("= Hello\n#warn\n")+ code, printed, v = self.main()+ self.assertEqual(code, 0)+ self.assertEqual(v["artifact"], "out/main.pdf")+ self.assertEqual(v["summary"], "main.pdf · 2 pages · compiles")+ self.assertEqual(printed.splitlines(), ["ready · main.pdf · 2 pages · compiles", " warning unknown font family: x (main.typ:2:3)"])++ def test_a_named_source_relative_or_absolute(self):+ (self.ws / "paper.typ").write_text("= Paper\n")+ self.assertEqual(self.main("paper.typ")[2]["artifact"], "out/paper.pdf")+ self.assertEqual(self.main(str(self.ws / "paper.typ"))[2]["artifact"], "out/paper.pdf")++ def test_an_error_keeps_the_last_good_pdf_as_the_artifact(self):+ (self.ws / "main.typ").write_text("= Hello\n")+ self.main()+ (self.ws / "main.typ").write_text("#foo\n")+ code, printed, v = self.main()+ self.assertEqual(code, 1)+ self.assertFalse(v["ready"])+ self.assertEqual(v["artifact"], "out/main.pdf")+ self.assertEqual(v["summary"], "main.pdf · 2 pages · 1 error")+ self.assertIn(" error unknown variable: foo (main.typ:1:1)", printed)++ def test_no_source_names_the_missing_file(self):+ code, printed, v = self.main("paper.typ")+ self.assertEqual(code, 1)+ self.assertIsNone(v["artifact"])+ self.assertEqual(v["findings"], [{"severity": "error", "kind": "typst", "message": "no paper.typ in the workspace"}])++ def test_no_typst_binary_is_a_finding_not_a_traceback(self):+ (self.ws / "main.typ").write_text("= Hello\n")+ with mock.patch.object(verdict, "TYPST", str(self.ws / "bin" / "typst")):+ code, _, v = self.main()+ self.assertEqual(code, 1)+ self.assertEqual(v["phases"][1]["state"], "failed")+ self.assertTrue(v["findings"][0]["message"].startswith("typst could not run"), v["findings"])++ def test_run_as_a_script(self):+ (self.ws / "main.typ").write_text("= Hello\n")+ env = {"HARNESS_WORKSPACE": str(self.ws), "TYPST": str(self.typst)}+ with mock.patch.dict(os.environ, env), mock.patch.object(sys, "argv", ["verdict.py"]), contextlib.redirect_stdout(io.StringIO()):+ with self.assertRaises(SystemExit) as exit_:+ runpy.run_path(str(HERE / "verdict.py"), run_name="__main__")+ self.assertEqual(exit_.exception.code, 0)+ self.assertTrue((self.ws / "out" / "main.pdf").exists())+++def real_typst() -> str | None:+ """$TYPST as the harness sets it, else the install's own bin/typst."""+ for candidate in (os.environ.get("TYPST"), HERE.parent / "bin" / "typst"):+ if candidate and os.access(candidate, os.X_OK):+ return str(candidate)+ return None++[email protected](real_typst() is None, "no typst binary: set TYPST or run toolchain/setup.sh")+class RealTypst(unittest.TestCase):+ """The diagnostics regex against what the pinned Typst actually prints."""++ def test_diagnostics_from_the_real_compiler(self):+ with tempfile.TemporaryDirectory() as d, mock.patch.object(verdict, "WS", Path(d)), mock.patch.object(verdict, "TYPST", real_typst()), contextlib.redirect_stdout(io.StringIO()):+ (Path(d) / "main.typ").write_text('= Hi\n#text(font: "NoSuchFontAnywhere")[x]\n')+ self.assertEqual(verdict.main(["verdict.py"]), 0)+ v = json.loads((Path(d) / ".harness" / "verdict.json").read_text())+ self.assertEqual(v["summary"], "main.pdf · 1 page · compiles")+ self.assertEqual([(f["severity"], f["ref"]) for f in v["findings"]], [("warning", "main.typ:2:12")])+ (Path(d) / "main.typ").write_text("#foo\n")+ self.assertEqual(verdict.main(["verdict.py"]), 1)+ v = json.loads((Path(d) / ".harness" / "verdict.json").read_text())+ self.assertEqual(v["findings"], [{"severity": "error", "kind": "typst", "message": "unknown variable: foo", "ref": "main.typ:1:1"}])+if __name__ == "__main__": unittest.main()diff --git a/store/agents/typst/toolchain/verdict.py b/store/agents/typst/toolchain/verdict.pyindex 9d7292ec0..f0db017c2 100755--- a/store/agents/typst/toolchain/verdict.py+++ b/store/agents/typst/toolchain/verdict.py@@ -22,7 +22,8 @@ def judge(source: str, diagnostics: str, code: int, pdf: str | None, pages: intfindings.append({"severity": "error" if sev == "error" else "warning", "kind": "typst", "message": msg.strip(), **({"ref": f"{file}:{line}:{col}"} if file else {})})errors = [f for f in findings if f["severity"] == "error"]written = len(source.strip()) > 0- compiled = code == 0 and not errors and pdf is not None+ # An empty file compiles to a blank page: that is not a document yet, so it is not ready either.+ compiled = written and code == 0 and not errors and pdf is not Nonephases = [{"id": "write", "name": "Write", "state": "done" if written else "active"},{"id": "compile", "name": "Compile", "state": ("done" if compiled else "failed") if written else "pending"},@@ -50,11 +51,15 @@ def main(argv: list[str]) -> int:out = WS / "out" / (src.stem + ".pdf")out.parent.mkdir(parents=True, exist_ok=True)source = src.read_text() if src.exists() else ""- code, diag = 1, "error: no main.typ in the workspace"+ code, diag = 1, f"error: no {os.path.relpath(src, WS)} in the workspace"if src.exists():- r = subprocess.run([TYPST, "compile", "--root", str(WS), str(src), str(out)], capture_output=True, text=True, cwd=WS)- code, diag = r.returncode, r.stderr- pdf = os.path.relpath(out, WS) if out.exists() and code == 0 else (os.path.relpath(out, WS) if out.exists() else None)+ try:+ r = subprocess.run([TYPST, "compile", "--root", str(WS), str(src), str(out)], capture_output=True, text=True, cwd=WS)+ code, diag = r.returncode, r.stderr+ except OSError as error: # no typst binary (setup not run): still a verdict, not a traceback+ code, diag = 1, f"error: typst could not run ({error})"+ # The last good PDF stays the artifact while the source does not compile: the pane keeps showing it.+ pdf = os.path.relpath(out, WS) if out.exists() else Noneverdict = judge(source, diag, code, pdf, page_count(out) if out.exists() else None)(WS / ".harness").mkdir(exist_ok=True)(WS / ".harness" / "verdict.json").write_text(json.dumps(verdict, indent=2) + "\n")