Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

strudel: cover the verdict to 100%, fix three false reports

dee4f7d8836e778
diff --git a/store/agents/strudel/toolchain/test_verdict.py b/store/agents/strudel/toolchain/test_verdict.py
index f803e2472..08b9e1ca6 100644
--- a/store/agents/strudel/toolchain/test_verdict.py
+++ b/store/agents/strudel/toolchain/test_verdict.py
@@ -1,13 +1,17 @@
"""python3 -m unittest toolchain/test_verdict.py — what the verdict says about a track."""
from __future__ import annotations
+import contextlib
+import io
import json
import os
+import runpy
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
+from unittest import mock
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
@@ -96,6 +100,52 @@ class Judging(unittest.TestCase):
self.assertTrue(result["ready"])
self.assertIn("tempo", kinds(result, "info"))
+ def test_an_unclosed_block_comment_is_an_error(self) -> None:
+ result = verdict.judge('s("sbd*4")\n/* the bass\n', "track.strudel")
+ self.assertEqual(kinds(result, "error"), ["syntax"])
+ self.assertIn("line 2: a /* comment is never closed", result["findings"][0]["message"])
+
+ def test_a_block_comment_keeps_the_line_count(self) -> None:
+ result = verdict.judge('/* one\ntwo */\ns("sbd"))\n', "track.strudel")
+ self.assertIn("line 3: a stray )", [f["message"] for f in result["findings"]])
+
+ def test_a_string_cut_off_by_the_end_of_the_file(self) -> None:
+ result = verdict.judge('s("sbd*4', "track.strudel")
+ self.assertIn('line 1: a " string is never closed', [f["message"] for f in result["findings"]])
+
+ def test_an_escaped_quote_does_not_close_the_string(self) -> None:
+ result = verdict.judge('setcps(1)\ns("sbd \\" sbd")\n', "track.strudel")
+ self.assertTrue(result["ready"], result["findings"])
+
+ def test_a_template_literal_spans_lines(self) -> None:
+ result = verdict.judge('setcps(1)\ns(`sbd\n sbd`)\n)\n', "track.strudel")
+ self.assertIn("line 4: a stray )", [f["message"] for f in result["findings"]])
+
+ def test_a_line_continuation_inside_a_string_keeps_the_line_count(self) -> None:
+ # A backslash-newline continues a '…' or "…" string onto the next line. Before the fix the
+ # escape skipped the newline without counting it, so every later finding was a line early.
+ result = verdict.judge('s("sbd \\\nsbd")\n)\n', "track.strudel")
+ self.assertIn("line 3: a stray )", [f["message"] for f in result["findings"]])
+
+ def test_a_commented_out_sample_is_not_a_network_dependency(self) -> None:
+ # Before the fix the sound scan read the raw text, comments included, so a voice the agent
+ # had commented out still said "this track needs the internet".
+ result = verdict.judge('setcps(1)\n// s("bd*4")\n/* s("hh*8") */\ns("sbd*4")\n', "track.strudel")
+ self.assertEqual(kinds(result, "warning"), [])
+ self.assertIn("plays offline", result["summary"])
+
+ def test_without_node_the_scan_is_the_only_syntax_check(self) -> None:
+ for error in (OSError("node"), subprocess.TimeoutExpired("node", 20)):
+ with self.subTest(error=type(error).__name__), \
+ mock.patch.object(verdict.subprocess, "run", side_effect=error):
+ self.assertEqual(verdict.node_check('s("sbd")'), [])
+
+ def test_a_node_failure_without_a_syntax_error_line(self) -> None:
+ done = subprocess.CompletedProcess(["node"], 1, stdout="", stderr="node: bad option\n")
+ with mock.patch.object(verdict.subprocess, "run", return_value=done):
+ self.assertEqual(verdict.node_check('s("sbd")')[0]["message"],
+ "node could not parse the pattern: syntax error")
+
def test_the_template_is_ready_and_offline(self) -> None:
result = verdict.judge(TEMPLATE.read_text(), "track.strudel")
self.assertTrue(result["ready"], result["findings"])
@@ -137,5 +187,55 @@ class Writing(unittest.TestCase):
self.assertEqual(written["artifact"], "b-side.strudel")
+class Main(unittest.TestCase):
+ """main() in this process, so what it does is measured, not only what it leaves on disk."""
+
+ def setUp(self) -> None:
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.ws = Path(self.tmp.name).resolve()
+
+ def main(self, *args: str) -> tuple[int, str, dict]:
+ out = io.StringIO()
+ with mock.patch.object(verdict, "WS", self.ws), contextlib.redirect_stdout(out):
+ code = verdict.main(["verdict.py", *args])
+ return code, out.getvalue(), json.loads((self.ws / ".harness" / "verdict.json").read_text())
+
+ def test_the_default_track_prints_the_summary_and_each_finding(self) -> None:
+ (self.ws / "track.strudel").write_text('s("sbd*4")\n')
+ code, printed, written = self.main()
+ self.assertEqual(code, 0)
+ self.assertEqual(written["artifact"], "track.strudel")
+ self.assertEqual(printed.splitlines(), [
+ "ready · track.strudel · 1 line · plays offline",
+ " info no setcps/setcpm: Strudel runs at its default 0.5 cycles per second",
+ ])
+
+ def test_an_absolute_path_and_an_unreadable_one(self) -> None:
+ (self.ws / "b.strudel").write_text(GOOD)
+ self.assertEqual(self.main(str(self.ws / "b.strudel"))[2]["artifact"], "b.strudel")
+ code, printed, written = self.main("missing.strudel")
+ self.assertEqual(code, 1)
+ self.assertEqual(written["summary"], "missing.strudel · unreadable")
+ self.assertEqual([p["state"] for p in written["phases"]], ["active", "pending", "pending"])
+ self.assertIn(" error missing.strudel: ", printed)
+
+ def test_bytes_that_are_not_utf8_are_judged_not_a_crash(self) -> None:
+ # Before the fix read_text() raised UnicodeDecodeError: no verdict, the last one left standing.
+ (self.ws / "track.strudel").write_bytes(b'setcps(1)\n// caf\xe9 set\ns("sbd*4")\n')
+ code, _, written = self.main()
+ self.assertEqual(code, 0)
+ self.assertTrue(written["ready"])
+
+ def test_run_as_a_script(self) -> None:
+ out = io.StringIO()
+ with mock.patch.dict(os.environ, {"HARNESS_WORKSPACE": str(self.ws)}), \
+ mock.patch.object(sys, "argv", ["verdict.py"]), contextlib.redirect_stdout(out), \
+ self.assertRaises(SystemExit) as exit_:
+ runpy.run_path(str(HERE / "verdict.py"), run_name="__main__")
+ self.assertEqual(exit_.exception.code, 1)
+ self.assertIn("not ready · track.strudel · unreadable", out.getvalue())
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/store/agents/strudel/toolchain/verdict.py b/store/agents/strudel/toolchain/verdict.py
index 24ca0c9be..3dd31d85b 100644
--- a/store/agents/strudel/toolchain/verdict.py
+++ b/store/agents/strudel/toolchain/verdict.py
@@ -49,11 +49,13 @@ NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
PAIRS = {")": "(", "]": "[", "}": "{"}
-def strip_code(text: str) -> tuple[str, list[dict]]:
+def strip_code(text: str) -> tuple[str, list[dict], str]:
"""Walk the source once: report unbalanced brackets and unterminated strings, and return the
- code with comments and string bodies blanked out so name scans do not trip over them."""
+ code with comments and string bodies blanked out so name scans do not trip over them, and the
+ source with only the comments taken out, so the sound scan does not read a commented-out voice."""
findings: list[dict] = []
out: list[str] = []
+ kept: list[str] = []
stack: list[tuple[str, int]] = []
i, line, n = 0, 1, len(text)
while i < n:
@@ -61,6 +63,7 @@ def strip_code(text: str) -> tuple[str, list[dict]]:
if c == "\n":
line += 1
out.append(c)
+ kept.append(c)
i += 1
elif c == "/" and i + 1 < n and text[i + 1] == "/":
while i < n and text[i] != "\n":
@@ -73,12 +76,15 @@ def strip_code(text: str) -> tuple[str, list[dict]]:
else:
line += text.count("\n", i, end)
i = end + 2
+ kept.append(" ")
elif c in "\"'`":
- quote, start_line, i = c, line, i + 1
+ quote, start_line, start, i = c, line, i, i + 1
closed = False
while i < n:
ch = text[i]
if ch == "\\":
+ if text[i + 1:i + 2] == "\n": # a line continuation is still a line
+ line += 1
i += 2
continue
if ch == quote:
@@ -93,9 +99,11 @@ def strip_code(text: str) -> tuple[str, list[dict]]:
if not closed:
findings.append({"severity": "error", "kind": "syntax", "message": f"line {start_line}: a {quote} string is never closed"})
out.append(" ")
+ kept.append(text[start:i])
elif c in "([{":
stack.append((c, line))
out.append(c)
+ kept.append(c)
i += 1
elif c in ")]}":
if not stack or stack[-1][0] != PAIRS[c]:
@@ -103,13 +111,15 @@ def strip_code(text: str) -> tuple[str, list[dict]]:
else:
stack.pop()
out.append(c)
+ kept.append(c)
i += 1
else:
out.append(c)
+ kept.append(c)
i += 1
for opener, at in stack:
findings.append({"severity": "error", "kind": "syntax", "message": f"line {at}: {opener} is never closed"})
- return "".join(out), findings
+ return "".join(out), findings, "".join(kept)
def node_check(text: str) -> list[dict]:
@@ -131,7 +141,7 @@ def node_check(text: str) -> list[dict]:
def judge(text: str, rel: str) -> dict:
findings: list[dict] = []
- code, scan = strip_code(text)
+ code, scan, uncommented = strip_code(text)
findings += scan
bare = "".join(ch for ch in code if not ch.isspace())
written = bool(bare)
@@ -152,7 +162,7 @@ def judge(text: str, rel: str) -> dict:
findings.append({"severity": "info", "kind": "tempo", "message": "no setcps/setcpm: Strudel runs at its default 0.5 cycles per second"})
online: list[str] = []
- for _, literal in SOUND_LITERAL.findall(text):
+ for _, literal in SOUND_LITERAL.findall(uncommented):
for token in NAME.findall(literal):
if token not in OFFLINE_SOUNDS and token not in online:
online.append(token)
@@ -161,7 +171,6 @@ def judge(text: str, rel: str) -> dict:
"message": f"{', '.join(online[:8])} {'is' if len(online) == 1 else 'are'} sample-bank sound(s): the pane downloads them at start, so this track needs the internet"})
errors = [f for f in findings if f["severity"] == "error"]
- warnings = [f for f in findings if f["severity"] == "warning"]
ok = written and not errors
lines = len(text.splitlines())
@@ -174,8 +183,6 @@ def judge(text: str, rel: str) -> dict:
bits.append(f"{len(errors)} error{'s' if len(errors) != 1 else ''}")
elif online:
bits.append("plays — samples need the internet")
- elif warnings:
- bits.append(f"plays · {len(warnings)} warning{'s' if len(warnings) != 1 else ''}")
else:
bits.append("plays offline")
@@ -195,7 +202,8 @@ def main(argv: list[str]) -> int:
target = target if target.is_absolute() else WS / target
rel = os.path.relpath(target, WS)
try:
- text = target.read_text()
+ # As the pane reads it (served as UTF-8, decoded with replacement), not a crash.
+ text = target.read_text(encoding="utf-8", errors="replace")
except OSError as error:
verdict = {"spec": 1, "ready": False, "summary": f"{Path(rel).name} · unreadable",
"findings": [{"severity": "error", "kind": "file", "message": f"{rel}: {error}"}],