Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

typst: cover the verdict and the shell scripts, fix three verdict bugs

dee505c9d9bdd58
diff --git a/store/agents/typst/toolchain/test_scripts.py b/store/agents/typst/toolchain/test_scripts.py
new file mode 100644
index 000000000..52e81777a
--- /dev/null
+++ b/store/agents/typst/toolchain/test_scripts.py
@@ -0,0 +1,148 @@
+"""setup.sh, doctor.sh and init-workspace.sh, run for real against a scratch install whose PATH holds
+only stub commands (and the few coreutils the scripts use), so every ok / miss line is reached
+without a network or a Typst on the machine:
+
+ python3 -m unittest toolchain/test_scripts.py
+"""
+import os, shutil, subprocess, tempfile, unittest
+from pathlib import Path
+
+PACKAGE = Path(__file__).resolve().parent.parent
+BASH = "/bin/bash"
+COREUTILS = ("dirname", "cat", "mkdir", "cp", "rm", "chmod", "grep")
+VERSION = (PACKAGE / "TYPST_VERSION").read_text().strip()
+
+
+class Sandbox:
+ """An install dir with the package's scripts, a bin/ of stubs as the whole PATH, and a log of
+ every stub call."""
+
+ def __init__(self, test: unittest.TestCase):
+ tmp = tempfile.TemporaryDirectory()
+ test.addCleanup(tmp.cleanup)
+ self.root = Path(tmp.name)
+ self.install = self.root / "install"
+ (self.install / "toolchain").mkdir(parents=True)
+ for script in PACKAGE.glob("toolchain/*.sh"):
+ shutil.copy(script, self.install / "toolchain" / script.name)
+ shutil.copy(PACKAGE / "TYPST_VERSION", self.install / "TYPST_VERSION")
+ self.bin = self.root / "bin"
+ self.bin.mkdir()
+ self.calls = self.root / "calls.log"
+ self.calls.touch()
+ for name in COREUTILS:
+ real = next(p for p in (Path("/bin") / name, Path("/usr/bin") / name) if p.exists())
+ (self.bin / name).symlink_to(real)
+
+ def stub(self, name: str, body: str = "", where: Path | None = None) -> Path:
+ path = (where or self.bin) / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(f'#!/bin/bash\necho "{name} $*" >> "$CALLS"\n{body}\n')
+ path.chmod(0o755)
+ return path
+
+ def run(self, script: str, cwd: Path | None = None, **env: str) -> subprocess.CompletedProcess:
+ return subprocess.run([BASH, str(self.install / "toolchain" / script)], cwd=cwd or self.install,
+ env={"PATH": str(self.bin), "CALLS": str(self.calls), **env},
+ capture_output=True, text=True, timeout=60)
+
+ def logged(self) -> list[str]:
+ return self.calls.read_text().splitlines()
+
+
+class Doctor(unittest.TestCase):
+ def test_ready(self):
+ box = Sandbox(self)
+ box.stub("typst", f'echo "typst {VERSION[1:]} (abc)"', where=box.install / "bin")
+ box.stub("python3", 'echo "Python 3.12.1"')
+ r = box.run("doctor.sh")
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(r.stdout.splitlines(), [f"ok typst {VERSION[1:]} (abc)", "ok Python 3.12.1 for the verdict"])
+
+ def test_no_typst_is_a_miss_and_stops(self):
+ box = Sandbox(self)
+ box.stub("python3", 'echo "Python 3.12.1"')
+ r = box.run("doctor.sh")
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.splitlines(), ["miss bin/typst — run toolchain/setup.sh"])
+
+ def test_no_python_is_a_miss(self):
+ box = Sandbox(self)
+ box.stub("typst", 'echo "typst 0.15.1"', where=box.install / "bin")
+ r = box.run("doctor.sh")
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.splitlines()[-1], "miss python3 for the verdict")
+
+
+class Setup(unittest.TestCase):
+ def sandbox(self, system="Darwin", machine="arm64") -> Sandbox:
+ box = Sandbox(self)
+ box.stub("uname", f'[ "$1" = -s ] && echo {system} || echo {machine}')
+ # curl writes the archive it was asked for; tar unpacks a typst that knows its version.
+ box.stub("curl", 'while [ $# -gt 1 ]; do [ "$1" = -o ] && out="$2"; shift; done; echo archive > "$out"')
+ box.stub("tar", f'dir="$4/$(basename "$2" .tar.xz)"; mkdir -p "$dir"; printf \'#!/bin/bash\\necho "typst {VERSION[1:]} (fetched)"\\n\' > "$dir/typst"')
+ (box.bin / "basename").symlink_to("/usr/bin/basename")
+ return box
+
+ def test_downloads_the_release_for_each_machine(self):
+ for system, machine, asset in (("Darwin", "arm64", "typst-aarch64-apple-darwin.tar.xz"),
+ ("Darwin", "x86_64", "typst-x86_64-apple-darwin.tar.xz"),
+ ("Linux", "x86_64", "typst-x86_64-unknown-linux-musl.tar.xz"),
+ ("Linux", "aarch64", "typst-aarch64-unknown-linux-musl.tar.xz")):
+ with self.subTest(machine=f"{system}-{machine}"):
+ box = self.sandbox(system, machine)
+ r = box.run("setup.sh")
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(r.stdout.splitlines(), [f" downloading typst {VERSION} ({asset})", f"ok typst {VERSION[1:]} (fetched)"])
+ self.assertIn(f"curl -fsSL -o tmp/{asset} https://github.com/typst/typst/releases/download/{VERSION}/{asset}", box.logged())
+ self.assertTrue(os.access(box.install / "bin" / "typst", os.X_OK))
+ self.assertFalse((box.install / "tmp").exists(), "the download is cleaned up")
+
+ def test_an_unknown_machine_is_a_miss(self):
+ r = self.sandbox("FreeBSD", "riscv64").run("setup.sh")
+ self.assertEqual(r.returncode, 1)
+ self.assertEqual(r.stdout.strip(), "miss no Typst release for FreeBSD-riscv64")
+
+ def test_the_pinned_version_already_here_is_kept(self):
+ box = self.sandbox()
+ box.stub("typst", f'echo "typst {VERSION[1:]} (old install)"', where=box.install / "bin")
+ r = box.run("setup.sh")
+ self.assertEqual((r.returncode, r.stdout.strip()), (0, f"ok typst {VERSION} already here"))
+ self.assertFalse(any(c.startswith("curl") for c in box.logged()))
+
+ def test_another_version_is_replaced(self):
+ box = self.sandbox()
+ box.stub("typst", 'echo "typst 0.1.0"', where=box.install / "bin")
+ r = box.run("setup.sh")
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertEqual(r.stdout.splitlines()[-1], f"ok typst {VERSION[1:]} (fetched)")
+
+ def test_a_failed_download_fails_setup(self):
+ box = self.sandbox()
+ box.stub("curl", "exit 22")
+ r = box.run("setup.sh")
+ self.assertNotEqual(r.returncode, 0)
+ self.assertFalse((box.install / "bin" / "typst").exists())
+
+
+class InitWorkspace(unittest.TestCase):
+ def test_compiles_the_template_and_seeds_the_verdict(self):
+ box = Sandbox(self)
+ ws = box.root / "ws"
+ ws.mkdir()
+ box.stub("typst", "exit 1", where=box.install / "bin") # a failing compile does not stop init
+ box.stub("python3", "exit 1")
+ r = box.run("init-workspace.sh", cwd=ws, HARNESS_DSH_DIR=str(box.install))
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertTrue((ws / ".harness").is_dir() and (ws / "out").is_dir())
+ self.assertEqual(box.logged(), ["typst compile main.typ out/main.pdf", f"python3 {box.install}/toolchain/verdict.py"])
+
+ def test_needs_the_install_dir(self):
+ box = Sandbox(self)
+ r = box.run("init-workspace.sh", cwd=box.root)
+ self.assertNotEqual(r.returncode, 0)
+ self.assertIn("HARNESS_DSH_DIR", r.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/store/agents/typst/toolchain/test_verdict.py b/store/agents/typst/toolchain/test_verdict.py
index 7888a4f67..a32c5ae27 100644
--- a/store/agents/typst/toolchain/test_verdict.py
+++ b/store/agents/typst/toolchain/test_verdict.py
@@ -1,9 +1,14 @@
-import sys, unittest
+import contextlib, io, json, os, runpy, sys, unittest
from pathlib import Path
+from unittest import mock
sys.path.insert(0, str(Path(__file__).parent))
import tempfile
+import verdict
from verdict import judge, page_count
+HERE = Path(__file__).resolve().parent
+
+
class Judge(unittest.TestCase):
def test_compiles_clean(self):
v = judge("= Hi", "", 0, "out/main.pdf", 2)
@@ -14,10 +19,24 @@ class Judge(unittest.TestCase):
v = judge("#foo", diag, 1, None, None)
self.assertFalse(v["ready"]); self.assertEqual(v["phases"][1]["state"], "failed")
self.assertEqual(v["findings"][0], {"severity": "error", "kind": "typst", "message": "unknown variable: foo", "ref": "main.typ:3:5"})
+ self.assertEqual(v["summary"], "1 error")
def test_warning_keeps_review_open(self):
diag = "warning: unused import\n ┌─ main.typ:1:1\n"
v = judge("x", diag, 0, "out/main.pdf", 1)
self.assertTrue(v["ready"]); self.assertEqual(v["phases"][2]["state"], "active")
+ self.assertEqual(v["summary"], "main.pdf · 1 page · compiles")
+ def test_errors_are_counted_and_a_diagnostic_without_a_place_has_no_ref(self):
+ v = judge("x", "error: one\nerror: two\n", 1, "out/main.pdf", 3)
+ self.assertEqual(v["summary"], "main.pdf · 3 pages · 2 errors")
+ self.assertNotIn("ref", v["findings"][0])
+ def test_a_failed_compile_without_diagnostics(self):
+ v = judge("x", "", 101, None, None)
+ self.assertFalse(v["ready"]); self.assertEqual(v["summary"], "not compiled")
+ def test_an_empty_source_is_not_a_document_even_if_it_compiles(self):
+ v = judge(" \n", "", 0, "out/main.pdf", 1)
+ self.assertFalse(v["ready"])
+ self.assertEqual([p["state"] for p in v["phases"]], ["active", "pending", "pending"])
+
class PageCount(unittest.TestCase):
def test_page_labels_are_not_pages(self):
@@ -25,5 +44,116 @@ class PageCount(unittest.TestCase):
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
f.write(body); f.flush()
self.assertEqual(page_count(Path(f.name)), 2)
+ def test_no_pages_or_no_file_is_none(self):
+ with tempfile.TemporaryDirectory() as d:
+ (Path(d) / "a.pdf").write_bytes(b"%PDF-1.7\n%%EOF")
+ self.assertIsNone(page_count(Path(d) / "a.pdf"))
+ self.assertIsNone(page_count(Path(d) / "missing.pdf"))
+
+
+# A stand-in for `typst compile --root WS src out`: a source containing "#foo" fails the way Typst
+# does; anything else writes a two-page PDF, and "#warn" adds a warning.
+FAKE_TYPST = r"""#!/bin/sh
+src="$4"; out="$5"
+if grep -q '#foo' "$src"; then printf 'error: unknown variable: foo\n \342\224\214\342\224\200 main.typ:1:1\n' >&2; exit 1; fi
+grep -q '#warn' "$src" && printf 'warning: unknown font family: x\n \342\224\214\342\224\200 main.typ:2:3\n' >&2
+printf '%%PDF-1.7\n<</Type /Page>>\n<</Type /Page>>\n' > "$out"
+"""
+
+
+class Main(unittest.TestCase):
+ """verdict.py on a workspace: what it compiles, what it writes, what it prints and returns."""
+
+ def setUp(self):
+ tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(tmp.cleanup)
+ self.ws = Path(tmp.name) / "ws"
+ self.ws.mkdir()
+ self.typst = Path(tmp.name) / "typst"
+ self.typst.write_text(FAKE_TYPST)
+ self.typst.chmod(0o755)
+ for name, value in (("WS", self.ws), ("TYPST", str(self.typst))):
+ patcher = mock.patch.object(verdict, name, value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
+ def main(self, *argv):
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ code = verdict.main(["verdict.py", *argv])
+ return code, out.getvalue(), json.loads((self.ws / ".harness" / "verdict.json").read_text())
+
+ def test_a_document_that_compiles(self):
+ (self.ws / "main.typ").write_text("= Hello\n#warn\n")
+ code, printed, v = self.main()
+ self.assertEqual(code, 0)
+ self.assertEqual(v["artifact"], "out/main.pdf")
+ self.assertEqual(v["summary"], "main.pdf · 2 pages · compiles")
+ self.assertEqual(printed.splitlines(), ["ready · main.pdf · 2 pages · compiles", " warning unknown font family: x (main.typ:2:3)"])
+
+ def test_a_named_source_relative_or_absolute(self):
+ (self.ws / "paper.typ").write_text("= Paper\n")
+ self.assertEqual(self.main("paper.typ")[2]["artifact"], "out/paper.pdf")
+ self.assertEqual(self.main(str(self.ws / "paper.typ"))[2]["artifact"], "out/paper.pdf")
+
+ def test_an_error_keeps_the_last_good_pdf_as_the_artifact(self):
+ (self.ws / "main.typ").write_text("= Hello\n")
+ self.main()
+ (self.ws / "main.typ").write_text("#foo\n")
+ code, printed, v = self.main()
+ self.assertEqual(code, 1)
+ self.assertFalse(v["ready"])
+ self.assertEqual(v["artifact"], "out/main.pdf")
+ self.assertEqual(v["summary"], "main.pdf · 2 pages · 1 error")
+ self.assertIn(" error unknown variable: foo (main.typ:1:1)", printed)
+
+ def test_no_source_names_the_missing_file(self):
+ code, printed, v = self.main("paper.typ")
+ self.assertEqual(code, 1)
+ self.assertIsNone(v["artifact"])
+ self.assertEqual(v["findings"], [{"severity": "error", "kind": "typst", "message": "no paper.typ in the workspace"}])
+
+ def test_no_typst_binary_is_a_finding_not_a_traceback(self):
+ (self.ws / "main.typ").write_text("= Hello\n")
+ with mock.patch.object(verdict, "TYPST", str(self.ws / "bin" / "typst")):
+ code, _, v = self.main()
+ self.assertEqual(code, 1)
+ self.assertEqual(v["phases"][1]["state"], "failed")
+ self.assertTrue(v["findings"][0]["message"].startswith("typst could not run"), v["findings"])
+
+ def test_run_as_a_script(self):
+ (self.ws / "main.typ").write_text("= Hello\n")
+ env = {"HARNESS_WORKSPACE": str(self.ws), "TYPST": str(self.typst)}
+ with mock.patch.dict(os.environ, env), mock.patch.object(sys, "argv", ["verdict.py"]), contextlib.redirect_stdout(io.StringIO()):
+ with self.assertRaises(SystemExit) as exit_:
+ runpy.run_path(str(HERE / "verdict.py"), run_name="__main__")
+ self.assertEqual(exit_.exception.code, 0)
+ self.assertTrue((self.ws / "out" / "main.pdf").exists())
+
+
+def real_typst() -> str | None:
+ """$TYPST as the harness sets it, else the install's own bin/typst."""
+ for candidate in (os.environ.get("TYPST"), HERE.parent / "bin" / "typst"):
+ if candidate and os.access(candidate, os.X_OK):
+ return str(candidate)
+ return None
+
+
[email protected](real_typst() is None, "no typst binary: set TYPST or run toolchain/setup.sh")
+class RealTypst(unittest.TestCase):
+ """The diagnostics regex against what the pinned Typst actually prints."""
+
+ def test_diagnostics_from_the_real_compiler(self):
+ with tempfile.TemporaryDirectory() as d, mock.patch.object(verdict, "WS", Path(d)), mock.patch.object(verdict, "TYPST", real_typst()), contextlib.redirect_stdout(io.StringIO()):
+ (Path(d) / "main.typ").write_text('= Hi\n#text(font: "NoSuchFontAnywhere")[x]\n')
+ self.assertEqual(verdict.main(["verdict.py"]), 0)
+ v = json.loads((Path(d) / ".harness" / "verdict.json").read_text())
+ self.assertEqual(v["summary"], "main.pdf · 1 page · compiles")
+ self.assertEqual([(f["severity"], f["ref"]) for f in v["findings"]], [("warning", "main.typ:2:12")])
+ (Path(d) / "main.typ").write_text("#foo\n")
+ self.assertEqual(verdict.main(["verdict.py"]), 1)
+ v = json.loads((Path(d) / ".harness" / "verdict.json").read_text())
+ self.assertEqual(v["findings"], [{"severity": "error", "kind": "typst", "message": "unknown variable: foo", "ref": "main.typ:1:1"}])
+
if __name__ == "__main__": unittest.main()
diff --git a/store/agents/typst/toolchain/verdict.py b/store/agents/typst/toolchain/verdict.py
index 9d7292ec0..f0db017c2 100755
--- a/store/agents/typst/toolchain/verdict.py
+++ b/store/agents/typst/toolchain/verdict.py
@@ -22,7 +22,8 @@ def judge(source: str, diagnostics: str, code: int, pdf: str | None, pages: int
findings.append({"severity": "error" if sev == "error" else "warning", "kind": "typst", "message": msg.strip(), **({"ref": f"{file}:{line}:{col}"} if file else {})})
errors = [f for f in findings if f["severity"] == "error"]
written = len(source.strip()) > 0
- compiled = code == 0 and not errors and pdf is not None
+ # An empty file compiles to a blank page: that is not a document yet, so it is not ready either.
+ compiled = written and code == 0 and not errors and pdf is not None
phases = [
{"id": "write", "name": "Write", "state": "done" if written else "active"},
{"id": "compile", "name": "Compile", "state": ("done" if compiled else "failed") if written else "pending"},
@@ -50,11 +51,15 @@ def main(argv: list[str]) -> int:
out = WS / "out" / (src.stem + ".pdf")
out.parent.mkdir(parents=True, exist_ok=True)
source = src.read_text() if src.exists() else ""
- code, diag = 1, "error: no main.typ in the workspace"
+ code, diag = 1, f"error: no {os.path.relpath(src, WS)} in the workspace"
if src.exists():
- r = subprocess.run([TYPST, "compile", "--root", str(WS), str(src), str(out)], capture_output=True, text=True, cwd=WS)
- code, diag = r.returncode, r.stderr
- pdf = os.path.relpath(out, WS) if out.exists() and code == 0 else (os.path.relpath(out, WS) if out.exists() else None)
+ try:
+ r = subprocess.run([TYPST, "compile", "--root", str(WS), str(src), str(out)], capture_output=True, text=True, cwd=WS)
+ code, diag = r.returncode, r.stderr
+ except OSError as error: # no typst binary (setup not run): still a verdict, not a traceback
+ code, diag = 1, f"error: typst could not run ({error})"
+ # The last good PDF stays the artifact while the source does not compile: the pane keeps showing it.
+ pdf = os.path.relpath(out, WS) if out.exists() else None
verdict = judge(source, diag, code, pdf, page_count(out) if out.exists() else None)
(WS / ".harness").mkdir(exist_ok=True)
(WS / ".harness" / "verdict.json").write_text(json.dumps(verdict, indent=2) + "\n")