The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
yosys: verdict.py to full coverage; "fail" in a passing check no longer fails the sim
deecbdc64ecb4a1
diff --git a/store/agents/yosys/toolchain/test_verdict.py b/store/agents/yosys/toolchain/test_verdict.pyindex 1b6e12d41..e96953961 100644--- a/store/agents/yosys/toolchain/test_verdict.py+++ b/store/agents/yosys/toolchain/test_verdict.py@@ -1,12 +1,22 @@"""The judge, without a toolchain: every fixture below is real output from iverilog, yosys ornextpnr, trimmed. `python3 -m unittest discover -s toolchain`."""+import io+import json+import os+import runpyimport sysimport tempfileimport unittest+from contextlib import redirect_stdoutfrom pathlib import Path+from unittest import mocksys.path.insert(0, str(Path(__file__).parent))-from verdict import assemble, parse_pnr_report, parse_sim_log, parse_yosys_log, read_step, to_verdict # noqa: E402+import verdict # noqa: E402+from verdict import (assemble, load_json, parse_pnr_log, parse_pnr_report, parse_sim_log, # noqa: E402+ parse_yosys_log, read_step, to_verdict)++SCRIPT = Path(__file__).parent / "verdict.py"DONE = {"state": "done", "exit": 0}FAILED = {"state": "failed", "exit": 1}@@ -45,13 +55,16 @@ End of script."""-def build(steps=None, sim=None, synth=None, pnr=None, bitstream=None, rtl=("rtl/blink.v",)):+BITSTREAM = {"path": "out/blink.bin", "bytes": 104090, "flash": "iceprog out/blink.bin"}+++def build(steps=None, sim=None, synth=None, pnr=None, bitstream=BITSTREAM, rtl=("rtl/blink.v",)):return assemble("blink", steps if steps is not None else dict(ALL_DONE), list(rtl),sim if sim is not None else {"checks": ["PASS blink"], "failures": [], "asserted": True, "diagnostics": []},synth if synth is not None else {"cells": 77, "byType": {"SB_LUT4": 32}, "diagnostics": []},pnr if pnr is not None else {**parse_pnr_report(PNR), "diagnostics": []},- bitstream if bitstream is not None else {"path": "out/blink.bin", "bytes": 104090, "flash": "iceprog out/blink.bin"},+ bitstream,{"signals": [{"name": "clk"}], "end": 985000}, "--up5k", "sg48", "out/blink.svg",)@@ -75,6 +88,16 @@ class SimLog(unittest.TestCase):def test_a_dump_with_no_assertion_is_not_a_test(self):self.assertFalse(parse_sim_log("VCD info: dumpfile out/sim.vcd opened\n")["asserted"])+ def test_a_passing_check_that_mentions_failure_is_not_a_failure(self):+ s = parse_sim_log(" ok no failure after reset\n ok parity flagged on a failed frame\n"+ "PASS uart: 2 checks\n")+ self.assertEqual(s["failures"], [])+ self.assertTrue(s["asserted"])++ def test_the_word_FAIL_anywhere_in_a_check_line_still_fails_it(self):+ s = parse_sim_log(" ok 3 frames, FAIL on the fourth\n")+ self.assertEqual(s["failures"], ["ok 3 frames, FAIL on the fourth"])+class YosysLog(unittest.TestCase):def test_stat_gives_cells_and_types(self):@@ -90,6 +113,11 @@ class YosysLog(unittest.TestCase):self.assertEqual(y["diagnostics"][0]["kind"], "latch")self.assertEqual(y["diagnostics"][0]["severity"], "warning")+ def test_a_stat_block_without_counts(self):+ y = parse_yosys_log("=== empty ===\n\n Number of ports: 0\n")+ self.assertEqual((y["cells"], y["wires"], y["byType"]), (0, 0, {}))+ self.assertEqual(parse_yosys_log("no statistics here")["cells"], 0)+def test_error_is_an_error(self):y = parse_yosys_log("ERROR: Multiple conflicting drivers for bad.\\y")self.assertEqual(y["diagnostics"][0]["severity"], "error")@@ -112,6 +140,14 @@ class PnrReport(unittest.TestCase):self.assertEqual(util[0]["name"], "Logic cells")+class PnrLog(unittest.TestCase):+ def test_errors_and_warnings_with_or_without_the_info_prefix(self):+ d = parse_pnr_log("Info: placing\nERROR: IO 'tx' is unconstrained\nInfo: Warning: timing is tight\n")+ self.assertEqual(d, [+ {"severity": "error", "kind": "nextpnr", "message": "IO 'tx' is unconstrained"},+ {"severity": "warning", "kind": "nextpnr", "message": "timing is tight"}])++class Assemble(unittest.TestCase):def test_a_clean_run_is_ready(self):r = build()@@ -191,6 +227,43 @@ class Assemble(unittest.TestCase):r = build(steps=steps, synth=synth, bitstream=None)self.assertEqual(len([f for f in r["findings"] if "check -assert" in f["message"]]), 1)+ def test_verilog_diagnostics_are_findings_on_their_line(self):+ sim = parse_sim_log("rtl/blink.v:19: error: Unknown module type: countr\n")+ steps = {**ALL_DONE, "sim": {"state": "failed", "exit": 1, "log": "out/logs/sim.log", "tail": "1 error(s)"}}+ r = build(steps=steps, sim=sim, bitstream=None)+ self.assertEqual(r["findings"][0], {"severity": "error", "kind": "verilog",+ "message": "Unknown module type: countr", "ref": "rtl/blink.v:19"})+ # the parsed error explains the red phase; the log's last line is not added on top+ self.assertEqual(len(r["findings"]), 1)+ self.assertEqual(r["phases"][1]["state"], "failed")+ self.assertTrue(r["summary"].endswith("1 error"))++ def test_a_simulation_that_did_not_finish_says_so(self):+ steps = {**ALL_DONE, "sim": {"state": "failed", "exit": 1, "log": "out/logs/sim.log"}}+ r = build(steps=steps, sim={"checks": [], "failures": [], "asserted": False, "diagnostics": []},+ bitstream=None)+ self.assertEqual(r["findings"][0]["kind"], "simulation")+ self.assertIn("simulation did not finish", r["findings"][0]["message"])+ self.assertIsNone(r["simulation"]["vcd"])++ def test_nextpnr_errors_point_at_the_constraints(self):+ steps = {**ALL_DONE, "pnr": dict(FAILED), "pack": {"state": "skipped"}}+ pnr = {"utilization": [], "clocks": [], "diagnostics": parse_pnr_log("ERROR: IO 'tx' is unconstrained\n")}+ r = build(steps=steps, pnr=pnr, bitstream=None)+ self.assertEqual(r["findings"], [{"severity": "error", "kind": "nextpnr", "message": "IO 'tx' is unconstrained",+ "ref": "constraints/blink.pcf"}])+ self.assertEqual(r["summary"], "blink · sim passes · 77 cells · 1 error")++ def test_a_running_step_makes_its_phase_active(self):+ steps = {"sim": dict(DONE), "waves": dict(DONE), "synth": {"state": "running", "log": "out/logs/synth.log"}}+ r = build(steps=steps, pnr={"utilization": [], "clocks": [], "diagnostics": []}, bitstream=None)+ self.assertEqual([p["state"] for p in r["phases"]], ["done", "done", "active", "pending", "pending"])++ def test_several_errors_are_counted_in_the_summary(self):+ steps = {**ALL_DONE, "synth": dict(FAILED), "pnr": dict(FAILED)}+ r = build(steps=steps, bitstream=None)+ self.assertTrue(r["summary"].endswith("2 errors"))+def test_a_latch_finding_explains_itself(self):synth = {"cells": 4, "byType": {}, "diagnostics": [{"severity": "warning", "kind": "latch", "message": r"Latch inferred for signal `\bad.\y'"}]}@@ -219,6 +292,16 @@ class Verdict(unittest.TestCase):self.assertLessEqual(len(to_verdict(build(sim=sim), "a")["summary"]), 200)+class LoadJson(unittest.TestCase):+ def test_missing_or_broken_json_is_none(self):+ with tempfile.TemporaryDirectory() as d:+ self.assertIsNone(load_json(Path(d) / "nope.json"))+ (Path(d) / "bad.json").write_text("{not json")+ self.assertIsNone(load_json(Path(d) / "bad.json"))+ (Path(d) / "ok.json").write_text('{"a": 1}')+ self.assertEqual(load_json(Path(d) / "ok.json"), {"a": 1})++class Steps(unittest.TestCase):"""What flow.sh leaves beside each log: <step>.start, <step>.time, <step>.exit."""@@ -252,5 +335,101 @@ class Steps(unittest.TestCase):self.assertFalse(r["ready"])+def workspace(root: Path, *, top="blink", pack=True, pnr_ok=True):+ """A workspace as flow.sh leaves it after a whole run."""+ logs = root / "out" / "logs"+ logs.mkdir(parents=True)+ (root / "rtl").mkdir()+ (root / "rtl" / f"{top}.v").write_text("module blink; endmodule\n")+ (root / "out" / ".top").write_text(top + "\n")+ (logs / "run.json").write_text(json.dumps({"top": top, "pid": 1, "startedAt": 1000, "finishedAt": 9000}))+ for k, sid in enumerate(("sim", "waves", "synth", "schematic", "svg", "pnr", "pack")):+ if sid == "pack" and not pack:+ continue+ failed = sid == "pnr" and not pnr_ok+ (logs / f"{sid}.start").write_text(f"{1000 + k}\n")+ (logs / f"{sid}.time").write_text(f"{1000 + k} {1500 + k}\n")+ (logs / f"{sid}.exit").write_text("1\n" if failed else "0\n")+ (logs / f"{sid}.log").write_text({"sim": " ok one\nPASS blink\n", "synth": STAT}.get(sid, "")+ + ("ERROR: IO 'tx' is unconstrained\n" if failed else ""))+ (root / "out" / f"{top}_pnr.json").write_text(json.dumps(PNR))+ (root / "out" / f"{top}_routed.json").write_text("{}")+ (root / "out" / f"{top}.svg").write_text("<svg/>")+ (root / "out" / f"{top}.bin").write_bytes(b"\0" * 32)+ (root / "out" / "waves.json").write_text(json.dumps({"signals": [{"name": "clk"}, {"name": "led"}]}))+++class Main(unittest.TestCase):+ """The whole read: out/ as flow.sh leaves it -> out/<top>.report.json and .harness/verdict.json."""++ def run_main(self, root: Path, argv):+ out = io.StringIO()+ with mock.patch.object(verdict, "WS", root), redirect_stdout(out):+ code = verdict.main(argv)+ return code, out.getvalue()++ def test_a_finished_run_is_ready_and_writes_both_files(self):+ with tempfile.TemporaryDirectory() as d:+ root = Path(d)+ workspace(root)+ code, out = self.run_main(root, ["verdict.py", "blink"])+ self.assertEqual(code, 0)+ self.assertTrue(out.startswith("ready · blink · sim passes · 77 cells"), out)+ report = json.loads((root / "out" / "blink.report.json").read_text())+ self.assertEqual(report["bitstream"], {"path": "out/blink.bin", "bytes": 32, "flash": "iceprog out/blink.bin"})+ self.assertEqual((report["pnr"]["report"], report["pnr"]["routed"]), ("out/blink_pnr.json", "out/blink_routed.json"))+ self.assertEqual(report["synthesis"]["schematic"], "out/blink.svg")+ self.assertEqual(report["simulation"]["signals"], 2)+ self.assertEqual(report["steps"][0]["seconds"], 0.5)+ self.assertEqual(report["board"]["device"], "iCE40 UP5K")+ v = json.loads((root / ".harness" / "verdict.json").read_text())+ self.assertEqual((v["ready"], v["artifact"]), (True, "out/blink.report.json"))++ def test_the_top_comes_from_out_dot_top_then_the_testbench_then_a_default(self):+ with tempfile.TemporaryDirectory() as d:+ root = Path(d)+ workspace(root)+ self.assertEqual(self.run_main(root, ["verdict.py"])[0], 0)+ (root / "out" / ".top").unlink()+ (root / "tb").mkdir()+ (root / "tb" / "uart_tb.v").write_text("")+ code, out = self.run_main(root, ["verdict.py"])+ self.assertTrue((root / "out" / "uart.report.json").exists())+ self.assertTrue(out.startswith("ready · uart · sim passes"), out)+ self.assertNotIn("bitstream ready", out) # there is no out/uart.bin+ with tempfile.TemporaryDirectory() as d:+ code, out = self.run_main(Path(d), ["verdict.py"])+ self.assertEqual(code, 1)+ self.assertIn("not ready · top", out)+ self.assertIn(" error no rtl/*.v — nothing to build", out)+ v = json.loads((Path(d) / ".harness" / "verdict.json").read_text())+ self.assertEqual([p["state"] for p in v["phases"]], ["active", "pending", "pending", "pending", "pending"])++ def test_a_failed_place_and_route_reads_its_log_and_ships_no_bitstream(self):+ with tempfile.TemporaryDirectory() as d:+ root = Path(d)+ workspace(root, pnr_ok=False, pack=False)+ (root / "out" / "blink_routed.json").unlink()+ (root / "out" / "blink_pnr.json").unlink()+ code, out = self.run_main(root, ["verdict.py", "blink"])+ self.assertEqual(code, 1)+ self.assertIn(" error IO 'tx' is unconstrained (constraints/blink.pcf)", out)+ report = json.loads((root / "out" / "blink.report.json").read_text())+ self.assertIsNone(report["bitstream"])+ self.assertIsNone(report["pnr"]["routed"])+ self.assertEqual(report["steps"][-1]["state"], "skipped")++ def test_runs_as_a_script_on_the_harness_workspace(self):+ with tempfile.TemporaryDirectory() as d:+ workspace(Path(d))+ with mock.patch.dict(os.environ, {"HARNESS_WORKSPACE": d, "YOSYS_DEVICE": "--hx8k"}), \+ mock.patch.object(sys, "argv", [str(SCRIPT), "blink"]), redirect_stdout(io.StringIO()), \+ self.assertRaises(SystemExit) as exit_:+ runpy.run_path(str(SCRIPT), run_name="__main__")+ self.assertEqual(exit_.exception.code, 0)+ report = json.loads((Path(d) / "out" / "blink.report.json").read_text())+ self.assertEqual(report["board"]["device"], "iCE40 HX8K")++if __name__ == "__main__":unittest.main()diff --git a/store/agents/yosys/toolchain/verdict.py b/store/agents/yosys/toolchain/verdict.pyindex d5c3bff44..46c8908b8 100755--- a/store/agents/yosys/toolchain/verdict.py+++ b/store/agents/yosys/toolchain/verdict.py@@ -80,7 +80,8 @@ def parse_sim_log(text: str) -> dict:diagnostics.append({"severity": "warning" if kind.lower() == "warning" else "error","message": message.strip(), "ref": f"{file}:{line}"})checks = [l.strip() for l in text.splitlines() if re.match(r"\s*(ok|PASS|FAIL)\b", l.strip())]- failures = [l for l in checks if l.upper().startswith("FAIL") or " FAIL" in l.upper()]+ # The word FAIL, as the testbenches print it: not "fail" inside a passing check's description.+ failures = [l for l in checks if re.search(r"\bFAIL\b", l)]# "FAIL blink: 2 checks failed" is the testbench's own tally, not a third failing check.tallies = [l for l in failures if re.search(r"\bchecks?\s+failed", l, re.I)]# A testbench that never says PASS or FAIL has not been asked a question.