Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

13.4 KB
Raw Blame
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import unittest
from unittest import mock

SCRIPT = Path(__file__).resolve().parents[1] / "validate-change.py"
spec = importlib.util.spec_from_file_location("validate_change", SCRIPT)
runner = importlib.util.module_from_spec(spec)
spec.loader.exec_module(runner)


@unittest.skipUnless(os.name == "posix", "process groups require POSIX")
class ValidationTests(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self.root = Path(self.tmp.name)
        (self.root / ".harness").mkdir()
        (self.root / ".gitignore").write_text(".harness/\n")
        (self.root / "source.txt").write_text("original\n")
        self.git("init", "-q")
        self.git("add", ".")
        self.git("-c", "user.name=Fixture", "-c", "[email protected]", "commit", "-qm", "fixture")

    def git(self, *args):
        return subprocess.check_output(["git", *args], cwd=self.root, stderr=subprocess.STDOUT)

    def check(self, name, code, timeout=3):
        return {"name": name, "argv": [sys.executable, "-c", code], "timeout_seconds": timeout}

    def run_plan(self, checks, minimum=0, reuse=None, env=None):
        plan = self.root / ".harness" / "plan.json"
        plan.write_text(json.dumps({"reason": "Validation runner integration fixtures", "minimum_free_gib": minimum, "checks": checks}))
        command = [sys.executable, str(SCRIPT), str(plan)]
        if reuse:
            command.extend(["--reuse", str(reuse)])
        result = subprocess.run(command, cwd=self.root, capture_output=True, text=True, timeout=15, env=env)
        paths = sorted((self.root / ".harness" / "validation").glob("*/receipt.json"))
        self.assertTrue(paths, result.stderr + result.stdout)
        self.last_receipt = paths[-1]
        return result, json.loads(paths[-1].read_text())

    def counted_check(self, name="counted"):
        check = self.check(name, "from pathlib import Path; p=Path('.harness/count'); p.write_text(str(int(p.read_text())+1) if p.exists() else '1'); print('checked')")
        check["reuse"] = {"inputs": ["source.txt"], "toolchain": [[sys.executable, "--version"]]}
        return check

    def test_independent_checks_really_overlap_and_capture_logs(self):
        checks = []
        for name, other in [("one", "two"), ("two", "one")]:
            checks.append(self.check(name, f"from pathlib import Path; import time\nPath('.harness/{name}').touch()\nwhile not Path('.harness/{other}').exists(): time.sleep(.02)\nprint('{name} finished')"))
        result, receipt = self.run_plan(checks)
        self.assertEqual(result.returncode, 0, result.stderr + result.stdout)
        self.assertEqual(receipt["status"], "passed")
        self.assertEqual(len(receipt["checks"]), 2)
        for check in receipt["checks"]:
            self.assertIn("finished", Path(check["log"]).read_text())
            self.assertGreater(check["duration_seconds"], 0)
        self.assertEqual(receipt["source"], receipt["source_after"])

    def test_failure_does_not_hide_other_results(self):
        result, receipt = self.run_plan([self.check("bad", "import sys; sys.exit(7)"), self.check("good", "print('ok')")])
        self.assertEqual(result.returncode, 1)
        checks = {c["name"]: c for c in receipt["checks"]}
        self.assertEqual(checks["bad"]["exit_code"], 7)
        self.assertEqual(checks["good"]["status"], "passed")

    def test_timeout_kills_owned_children_even_if_parent_exits_on_term(self):
        child = "import signal,time; from pathlib import Path; signal.signal(signal.SIGTERM, signal.SIG_IGN)\nwhile True:\n with Path('.harness/beats').open('a') as f: f.write('x')\n time.sleep(.02)"
        parent = f"import subprocess,sys,time; subprocess.Popen([sys.executable, '-c', {child!r}]); time.sleep(30)"
        result, receipt = self.run_plan([self.check("stalled", parent, .5)])
        self.assertEqual(result.returncode, 1)
        self.assertEqual(receipt["checks"][0]["status"], "timeout")
        beats = self.root / ".harness" / "beats"
        size = beats.stat().st_size
        self.assertGreater(size, 0)
        time.sleep(.15)
        self.assertEqual(beats.stat().st_size, size, "a child fixture survived cleanup")

    def test_low_disk_blocks_before_starting_commands(self):
        result, receipt = self.run_plan([self.check("never", "from pathlib import Path; Path('.harness/ran').touch()")], minimum=10**9)
        self.assertEqual(result.returncode, 1)
        self.assertEqual(receipt["status"], "blocked")
        self.assertEqual(receipt["checks"][0]["status"], "not_run")
        self.assertFalse((self.root / ".harness" / "ran").exists())

    def test_source_edit_invalidates_successful_checks(self):
        result, receipt = self.run_plan([self.check("edit", "from pathlib import Path; Path('source.txt').write_text('changed')")])
        self.assertEqual(result.returncode, 1)
        self.assertEqual(receipt["checks"][0]["status"], "passed")
        self.assertEqual(receipt["status"], "source_changed")

    def test_untracked_file_content_is_part_of_fingerprint(self):
        new = self.root / "new.txt"
        new.write_text("one")
        before = runner.source_state(self.root)
        new.write_text("two")
        after = runner.source_state(self.root)
        self.assertEqual(before["commit"], after["commit"])
        self.assertNotEqual(before["working_changes_sha256"], after["working_changes_sha256"])
        self.assertTrue(after["dirty"])

    def test_missing_executable_is_blocked_not_passed(self):
        result, receipt = self.run_plan([{"name": "missing", "argv": ["/no/such/validation-tool"], "timeout_seconds": 1}])
        self.assertEqual(result.returncode, 1)
        self.assertEqual(receipt["checks"][0]["status"], "blocked")

    def test_empty_plan_and_unsafe_log_names_are_rejected(self):
        for checks in [[], [self.check("../escape", "pass")]]:
            plan = self.root / ".harness" / "invalid.json"
            plan.write_text(json.dumps({"reason": "fixture", "checks": checks}))
            with self.assertRaises(ValueError):
                runner.read_plan(plan, self.root)

    def test_passed_evidence_survives_unrelated_commit_and_keeps_original_log(self):
        check = self.counted_check()
        result, original = self.run_plan([check])
        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
        previous = self.last_receipt
        (self.root / "notes.md").write_text("unrelated documentation\n")
        self.git("add", "notes.md")
        self.git("-c", "user.name=Fixture", "-c", "[email protected]", "commit", "-qm", "docs only")
        result, receipt = self.run_plan([check], reuse=previous)
        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
        self.assertEqual(receipt["checks"][0]["status"], "reused")
        self.assertEqual(receipt["checks"][0]["log"], original["checks"][0]["log"])
        self.assertNotEqual(receipt["source"]["commit"], original["source"]["commit"])
        self.assertEqual((self.root / ".harness/count").read_text(), "1")
        # Reuse chains retain the original execution duration, not a fictitious run.
        _, chained = self.run_plan([check], reuse=self.last_receipt)
        self.assertEqual(chained["checks"][0]["original_duration_seconds"], original["checks"][0]["duration_seconds"])

    def test_source_toolchain_environment_and_command_changes_each_rerun(self):
        check = self.counted_check()
        version = self.root / ".harness/tool-version"
        version.write_text("one")
        check["reuse"]["toolchain"] = [[sys.executable, "-c", "from pathlib import Path; print(Path('.harness/tool-version').read_text())"]]
        env = dict(os.environ, VALIDATION_FIXTURE_ENV="one")
        self.run_plan([check], env=env)
        changes = [
            lambda: (self.root / "source.txt").write_text("new source"),
            lambda: version.write_text("new toolchain"),
            lambda: env.update(VALIDATION_FIXTURE_ENV="two"),
            lambda: check["argv"].append("new argument"),
        ]
        for number, change in enumerate(changes, 2):
            previous = self.last_receipt
            change()
            result, receipt = self.run_plan([check], reuse=previous, env=env)
            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
            self.assertEqual(receipt["checks"][0]["status"], "passed")
            self.assertEqual((self.root / ".harness/count").read_text(), str(number))

    def test_new_and_deleted_files_in_a_declared_directory_invalidate_evidence(self):
        folder = self.root / "component"
        folder.mkdir()
        (folder / "original.txt").write_text("original")
        self.git("add", "component")
        check = self.counted_check()
        check["reuse"]["inputs"] = ["component"]
        self.run_plan([check])
        previous = self.last_receipt
        (folder / "new.txt").write_text("untracked source")
        _, receipt = self.run_plan([check], reuse=previous)
        self.assertEqual(receipt["checks"][0]["status"], "passed")
        previous = self.last_receipt
        (folder / "original.txt").unlink()
        _, receipt = self.run_plan([check], reuse=previous)
        self.assertEqual(receipt["checks"][0]["status"], "passed")
        self.assertEqual((self.root / ".harness/count").read_text(), "3")

    def test_failed_checks_rerun_without_repeating_independent_passes(self):
        good = self.counted_check()
        bad = self.check("failed", "raise SystemExit(7)")
        bad["reuse"] = good["reuse"]
        self.run_plan([good, bad])
        result, receipt = self.run_plan([good, bad], reuse=self.last_receipt)
        self.assertEqual(result.returncode, 1)
        checks = {item["name"]: item for item in receipt["checks"]}
        self.assertEqual(checks["counted"]["status"], "reused")
        self.assertEqual(checks["failed"]["status"], "failed")
        self.assertEqual((self.root / ".harness/count").read_text(), "1")

    def test_missing_or_tampered_logs_and_changing_source_are_not_reusable(self):
        check = self.counted_check()
        _, receipt = self.run_plan([check])
        previous = self.last_receipt
        Path(receipt["checks"][0]["log"]).write_text("different evidence")
        _, receipt = self.run_plan([check], reuse=previous)
        self.assertEqual(receipt["checks"][0]["status"], "passed")
        previous = self.last_receipt
        Path(receipt["checks"][0]["log"]).unlink()
        _, receipt = self.run_plan([check], reuse=previous)
        self.assertEqual(receipt["checks"][0]["status"], "passed")
        # This edit is outside the declared inputs, so only the unstable source
        # receipt (not a different per-check key) prevents reuse on the next run.
        check["argv"][2] += "; Path('unrelated.txt').write_text('changed during check')"
        _, receipt = self.run_plan([check])
        self.assertEqual(receipt["status"], "source_changed")
        _, receipt = self.run_plan([check], reuse=self.last_receipt)
        self.assertEqual(receipt["checks"][0]["status"], "passed")
        self.assertEqual((self.root / ".harness/count").read_text(), "5")

    def test_toolchain_failure_saves_a_blocked_receipt_before_tests_start(self):
        check = self.counted_check()
        check["reuse"]["toolchain"] = [[sys.executable, "-c", "raise SystemExit(4)"]]
        result, receipt = self.run_plan([check])
        self.assertEqual(result.returncode, 1)
        self.assertEqual(receipt["status"], "blocked")
        self.assertEqual(receipt["checks"][0]["status"], "not_run")
        self.assertFalse((self.root / ".harness/count").exists())

    def test_finished_empty_group_is_not_signaled(self):
        process = mock.Mock(pid=12345)
        process.poll.return_value = 0
        with mock.patch.object(runner.subprocess, "check_output", return_value=" 999\n"), mock.patch.object(runner.os, "killpg", side_effect=PermissionError("macOS denied")) as kill:
            self.assertIsNone(runner.stop_group(process))
            kill.assert_not_called()

    def test_finished_parent_does_not_leave_its_fixture_child_running(self):
        child = "import signal,time; from pathlib import Path; signal.signal(signal.SIGTERM, signal.SIG_IGN)\nwhile True:\n with Path('.harness/beats').open('a') as f: f.write('x')\n time.sleep(.02)"
        parent = f"import subprocess,sys,time; subprocess.Popen([sys.executable, '-c', {child!r}]); time.sleep(.15)"
        result, receipt = self.run_plan([self.check("exited", parent)])
        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
        size = (self.root / ".harness/beats").stat().st_size
        self.assertGreater(size, 0)
        time.sleep(.15)
        self.assertEqual((self.root / ".harness/beats").stat().st_size, size)

    def test_cleanup_errors_preserve_results_and_cannot_report_a_pass(self):
        output = self.root / ".harness"
        with mock.patch.object(runner, "stop_group", return_value="permission denied"):
            checks = runner.run_checks({"checks": [self.check("complete", "print('all tests finished')")]}, self.root, output, 1)
        self.assertEqual(checks[0]["status"], "cleanup_failed")
        self.assertEqual(checks[0]["exit_code"], 0)
        self.assertEqual(checks[0]["cleanup_error"], "permission denied")
        self.assertIn("all tests finished", Path(checks[0]["log"]).read_text())


if __name__ == "__main__":
    unittest.main()