The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Collect CI evidence as soon as the watched run passes (#664)
Deebd8af2a69cc8
diff --git a/AGENTS.md b/AGENTS.mdindex 9896acfbc..8e6847ae4 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -22,8 +22,9 @@ validation and shipping, use [docs/validation-and-release.md](docs/validation-anvalidation cycle. Verify published versions and checksums, then report completion.Desktop's `--wait` follows the exact tag/SHA through the workflow's six-artifactverification; reuse that receipt instead of repeating the downloads manually.-- Prepare the PR and review while checks run. Collect completed CI evidence with- `scripts/record-ci-validation.py RUN_ID --scope SCOPE --pr PR_NUMBER`; retain routine+- Prepare the PR and review while checks run. Once CI has a run ID, start+ `scripts/record-ci-validation.py RUN_ID --scope SCOPE --pr PR_NUMBER --wait` to+ collect its evidence as soon as that attempt passes; retain routineresults in its ignored receipt and the PR body instead of another documentationcommit. Resolve any source differences explicitly before reusing that evidence.- For an authorized Desktop release, start `make release-desktop ARGS="--prepare"`diff --git a/docs/validation-and-release.md b/docs/validation-and-release.mdindex 0c11359df..dc4add13e 100644--- a/docs/validation-and-release.md+++ b/docs/validation-and-release.md@@ -269,6 +269,15 @@ collect its record with one read-only command:python3 scripts/record-ci-validation.py RUN_ID --scope cli --pr PR_NUMBER```+For a run still in progress, add `--wait` and start the collector alongside review.+It observes that run every ten seconds and collects its evidence as soon as the+attempt passes. The default wait budget is 900s (`--wait-timeout`); collection gets+its own 90s budget (`--timeout`). Both durations are recorded separately. A failed+or cancelled run, changed source/attempt, or expired observation budget stops the+command. A timeout or lookup failure does not establish that CI stopped: inspect+and follow the same run ID. The collector never dispatches, cancels or retries CI,+and waiting does not bypass the required job, artifact, source or PR checks below.+Use the required CI scope (`cli`, `tui`, `backend`, `desktop`, `process`, or `full`). The commandwrites a short `validation.md` and machine-readable `receipt.json` under ignored`.harness/validation/`; use the paragraph/table in the PR's verification section.diff --git a/scripts/record-ci-validation.py b/scripts/record-ci-validation.pyindex ca1c69d06..ab40290e1 100644--- a/scripts/record-ci-validation.py+++ b/scripts/record-ci-validation.py@@ -73,16 +73,45 @@ class Client:page += 1-def check_run(run, repository, run_id):+def check_run_identity(run, repository, run_id):if (run.get("id") != run_id or run.get("repository", {}).get("full_name", "").lower() != repository.lower()or run.get("path") != ".github/workflows/ci.yml" or not re.fullmatch(r"[0-9a-f]{40}", run.get("head_sha", ""))):raise ValueError("run repository, workflow or source does not match this CI request")- if run.get("status") != "completed" or run.get("conclusion") != "success":- raise ValueError(f"CI has not passed: {run.get('status')}/{run.get('conclusion')}; follow the same run")if type(run.get("run_attempt")) is not int or run["run_attempt"] < 1:raise ValueError("run attempt is missing")+def check_run(run, repository, run_id):+ check_run_identity(run, repository, run_id)+ if run.get("status") != "completed" or run.get("conclusion") != "success":+ raise ValueError(f"CI has not passed: {run.get('status')}/{run.get('conclusion')}; follow the same run")+++def wait_for_run(client, run_id):+ """Observe one immutable run attempt; never dispatch, cancel, or rerun it."""+ identity, previous_state, announced_at = None, None, 0+ while True:+ if time.monotonic() >= client.deadline:+ raise TimeoutError(f"CI wait deadline exceeded; follow the same run {run_id}")+ run = client.api(f"actions/runs/{run_id}")+ check_run_identity(run, client.repository, run_id)+ current = (run["head_sha"], run["run_attempt"])+ if identity is not None and current != identity:+ raise ValueError("CI run source/attempt changed while waiting")+ identity = current+ state = (run.get("status"), run.get("conclusion"))+ if state[0] not in {"queued", "requested", "waiting", "pending", "in_progress", "completed"}:+ raise ValueError(f"unexpected CI run status: {state[0]}")+ now = time.monotonic()+ if state != previous_state or now - announced_at >= 30:+ print(f"CI {run_id}, attempt {run['run_attempt']}: {state[0]}/{state[1] or 'pending'}", flush=True)+ previous_state, announced_at = state, now+ if state[0] == "completed":+ check_run(run, client.repository, run_id)+ return run+ time.sleep(min(10, max(0, client.deadline - now)))++def check_jobs(run, jobs, scope):names = [job["name"] for job in jobs]if len(names) != len(set(names)) or len({job["id"] for job in jobs}) != len(jobs):@@ -184,9 +213,11 @@ def source_comparison(root, tested, target):"same_tree": tested_tree == target_tree, "changed_files": list(filter(None, paths)), "working_tree_dirty": dirty, "checkout_head": checkout}-def collect(client, root, run_id, scope, target, output, pr_number=None):+def collect(client, root, run_id, scope, target, output, pr_number=None, expected_run=None):run = client.api(f"actions/runs/{run_id}")check_run(run, client.repository, run_id)+ if expected_run is not None and any(run[key] != expected_run[key] for key in ["id", "head_sha", "run_attempt"]):+ raise ValueError("CI run changed between waiting and evidence collection")with ThreadPoolExecutor(max_workers=3) as pool:job_future = pool.submit(client.pages, f"actions/runs/{run_id}/jobs?filter=latest", "jobs")artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "desktop", "full"} else None@@ -294,22 +325,38 @@ def main(argv=None):parser.add_argument("--target", default="HEAD", help="committed source to compare with the tested tree")parser.add_argument("--pr", type=int, help="also verify that this PR still names the target commit")parser.add_argument("--output", type=Path, help="new evidence directory; defaults to ignored .harness/validation/")- parser.add_argument("--timeout", type=float, default=90)+ parser.add_argument("--timeout", type=float, default=90, help="evidence collection budget in seconds (default: 90)")+ parser.add_argument("--wait", action="store_true", help="follow this run attempt, then collect its completed evidence")+ parser.add_argument("--wait-timeout", type=float, default=900, help="separate CI observation budget in seconds (default: 900)")args = parser.parse_args(argv)- if args.run < 1 or (args.pr is not None and args.pr < 1) or not 0 < args.timeout < float("inf") or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", args.repo):+ if args.run < 1 or (args.pr is not None and args.pr < 1) or any(not 0 < value < float("inf") for value in [args.timeout, args.wait_timeout]) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", args.repo):parser.error("supply a valid repository, positive IDs and finite positive timeout")root = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()).resolve()output = args.output or root / ".harness" / "validation" / (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + f"-ci-{args.run}")output.mkdir(parents=True, exist_ok=False)- started, started_at = time.monotonic(), utc_now()+ waiting, observed, collection_started = None, None, Nonetry:- record = collect(Client(args.repo, started + args.timeout), root, args.run, args.scope, args.target, output, args.pr)+ if args.wait:+ started = time.monotonic()+ waiting = {"started_at": utc_now()}+ try:+ observed = wait_for_run(Client(args.repo, started + args.wait_timeout), args.run)+ waiting.update(head_sha=observed["head_sha"], run_attempt=observed["run_attempt"])+ finally:+ waiting.update(finished_at=utc_now(), duration_seconds=round(time.monotonic() - started, 3))+ collection_started, started_at = time.monotonic(), utc_now()+ record = collect(Client(args.repo, collection_started + args.timeout), root, args.run, args.scope,+ args.target, output, args.pr, expected_run=observed)except (ValueError, KeyError, TypeError, OSError, RuntimeError, subprocess.SubprocessError, zipfile.BadZipFile) as error:record = {"schema": 1, "status": "collection_failed", "run_id": args.run, "error": str(error)}- record["collection"] = {"started_at": started_at, "finished_at": utc_now(), "duration_seconds": round(time.monotonic() - started, 3)}+ if waiting is not None:+ record["waiting"] = waiting+ if collection_started is not None:+ record["collection"] = {"started_at": started_at, "finished_at": utc_now(), "duration_seconds": round(time.monotonic() - collection_started, 3)}(output / "receipt.json").write_text(json.dumps(record, indent=2) + "\n")(output / "validation.md").write_text(markdown(record))- print(f"{record['status']} ({record['collection']['duration_seconds']:g}s): {output / 'validation.md'}")+ elapsed = ", ".join(f"{record[phase]['duration_seconds']:g}s {phase}" for phase in ["waiting", "collection"] if phase in record)+ print(f"{record['status']} ({elapsed}): {output / 'validation.md'}")if "error" in record:print(record["error"], file=sys.stderr)return {"passed": 0, "source_review_required": 3}.get(record["status"], 1)diff --git a/scripts/tests/test_record_ci_validation.py b/scripts/tests/test_record_ci_validation.pyindex 755c176a0..079aec51f 100644--- a/scripts/tests/test_record_ci_validation.py+++ b/scripts/tests/test_record_ci_validation.py@@ -7,6 +7,7 @@ from pathlib import Pathimport subprocessimport tempfileimport unittest+from unittest import mockimport zipfileSCRIPT = Path(__file__).resolve().parents[1] / "record-ci-validation.py"@@ -55,7 +56,7 @@ class EvidenceTests(unittest.TestCase):archive.writestr(name, json.dumps(summary))return stream.getvalue()- def collect(self, scope="cli", target="HEAD", with_pr=False, pr_after=None):+ def collect(self, scope="cli", target="HEAD", with_pr=False, pr_after=None, expected_run=None):fixture = selfclass FakeClient:@@ -79,7 +80,7 @@ class EvidenceTests(unittest.TestCase):fixture.command_args = argsreturn fixture.archives[args[-1]] if hasattr(fixture, "archives") else fixture.archive- return recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None)+ return recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None, expected_run)def test_complete_record_checks_identity_digest_coverage_and_pr(self):result = self.collect(with_pr=True)@@ -248,6 +249,12 @@ class EvidenceTests(unittest.TestCase):with self.assertRaisesRegex(ValueError, "PR head/base changed"):self.collect(with_pr=True, pr_after=after)+ def test_waited_attempt_cannot_be_replaced_before_collection(self):+ self.assertEqual(self.collect(expected_run=self.run)["status"], "passed")+ for change in [dict(run_attempt=2), dict(head_sha="b" * 40), dict(id=124)]:+ with self.subTest(change=change), self.assertRaisesRegex(ValueError, "between waiting"):+ self.collect(expected_run=dict(self.run, **change))+def test_stable_pr_pointing_elsewhere_needs_source_review(self):self.pr["head"]["sha"] = "b" * 40result = self.collect(with_pr=True)@@ -270,5 +277,103 @@ class EvidenceTests(unittest.TestCase):client.pages("jobs", "jobs")+class WaitTests(unittest.TestCase):+ def setUp(self):+ self.now = 0+ self.run = dict(id=123, repository=dict(full_name="owner/repo"), path=".github/workflows/ci.yml",+ head_sha="a" * 40, run_attempt=1, status="completed", conclusion="success")+ self.client = recorder.Client("owner/repo", 120)+ self.patch(recorder.time, "monotonic", side_effect=lambda: self.now)+ self.sleep = self.patch(recorder.time, "sleep", side_effect=self.advance)+ self.patch(recorder, "print", create=True)++ def patch(self, obj, name, **kwargs):+ patcher = mock.patch.object(obj, name, **kwargs)+ value = patcher.start()+ self.addCleanup(patcher.stop)+ return value++ def advance(self, seconds):+ self.now += seconds++ def observe(self, runs):+ self.client.api = mock.Mock(side_effect=runs)+ return recorder.wait_for_run(self.client, 123)++ def test_live_states_and_completed_fast_path_follow_one_run(self):+ for states in [[], ["queued", "pending", "waiting", "requested", "in_progress"]]:+ with self.subTest(states=states):+ self.now = 0+ self.sleep.reset_mock()+ result = self.observe([dict(self.run, status=s, conclusion=None) for s in states] + [self.run])+ self.assertEqual(result, self.run)+ self.assertEqual(self.sleep.call_count, len(states))+ self.assertTrue(all(call == mock.call("actions/runs/123") for call in self.client.api.call_args_list))++ def test_failed_or_cancelled_runs_stop_without_following_a_later_rerun(self):+ for conclusion in ["failure", "cancelled", "timed_out", "action_required", "skipped"]:+ with self.subTest(conclusion=conclusion), self.assertRaisesRegex(ValueError, "CI has not passed"):+ self.observe([dict(self.run, conclusion=conclusion), dict(self.run, run_attempt=2)])+ self.assertEqual(self.client.api.call_count, 1)+ self.sleep.assert_not_called()++ def test_identity_changes_and_unknown_states_fail_closed(self):+ for change in [dict(id=124), dict(repository=dict(full_name="other/repo")), dict(path="other.yml"),+ dict(head_sha="b" * 40), dict(run_attempt=2), dict(status="unknown")]:+ self.now = 0+ with self.subTest(change=change), self.assertRaises(ValueError):+ self.observe([dict(self.run, status="queued", conclusion=None), dict(self.run, **change)])++ def test_wait_deadline_does_not_restart_or_cancel_the_job(self):+ self.client.deadline = 12+ pending = dict(self.run, status="in_progress", conclusion=None)+ with self.assertRaisesRegex(TimeoutError, "follow the same run 123"):+ self.observe([pending, pending])+ self.assertEqual(self.now, 12)+ self.assertEqual(self.sleep.call_args_list, [mock.call(10), mock.call(2)])+ self.assertEqual(self.client.api.call_count, 2)++ def test_cli_keeps_waiting_and_collection_budgets_and_receipts_separate(self):+ with tempfile.TemporaryDirectory() as folder:+ root, output = Path(folder), Path(folder) / "evidence"+ self.patch(recorder.subprocess, "check_output", return_value=str(root))+ self.patch(recorder, "markdown", return_value="verified\n")++ def wait(client, run_id):+ self.assertEqual((client.deadline, run_id), (120, 123))+ self.advance(40)+ return self.run++ def collect(client, *args, expected_run=None):+ self.assertEqual(client.deadline, 130) # A fresh 90s after waiting 40s.+ self.assertEqual(expected_run, self.run)+ self.advance(5)+ return dict(status="passed")++ self.patch(recorder, "wait_for_run", side_effect=wait)+ self.patch(recorder, "collect", side_effect=collect)+ result = recorder.main(["123", "--repo", "owner/repo", "--scope", "process", "--wait",+ "--wait-timeout", "120", "--output", str(output)])+ receipt = json.loads((output / "receipt.json").read_text())+ self.assertEqual(result, 0)+ self.assertEqual(receipt["waiting"]["duration_seconds"], 40)+ self.assertEqual(receipt["waiting"]["head_sha"], self.run["head_sha"])+ self.assertEqual(receipt["collection"]["duration_seconds"], 5)++ def test_cli_preserves_wait_failure_without_attempting_collection(self):+ with tempfile.TemporaryDirectory() as folder:+ output = Path(folder) / "evidence"+ self.patch(recorder.subprocess, "check_output", return_value=folder)+ self.patch(recorder, "wait_for_run", side_effect=TimeoutError("CI wait deadline exceeded"))+ collect = self.patch(recorder, "collect")+ result = recorder.main(["123", "--scope", "process", "--wait", "--output", str(output)])+ receipt = json.loads((output / "receipt.json").read_text())+ self.assertEqual(result, 1)+ self.assertIn("waiting", receipt)+ self.assertNotIn("collection", receipt)+ self.assertIn("CI wait deadline", receipt["error"])+ collect.assert_not_called()++if __name__ == "__main__":unittest.main()