Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Collect CI evidence as soon as the watched run passes (#664)

Deebd8af2a69cc8
diff --git a/AGENTS.md b/AGENTS.md
index 9896acfbc..8e6847ae4 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -22,8 +22,9 @@ validation and shipping, use [docs/validation-and-release.md](docs/validation-an
validation cycle. Verify published versions and checksums, then report completion.
Desktop's `--wait` follows the exact tag/SHA through the workflow's six-artifact
verification; reuse that receipt instead of repeating the downloads manually.
-- Prepare the PR and review while checks run. Collect completed CI evidence with
- `scripts/record-ci-validation.py RUN_ID --scope SCOPE --pr PR_NUMBER`; retain routine
+- Prepare the PR and review while checks run. Once CI has a run ID, start
+ `scripts/record-ci-validation.py RUN_ID --scope SCOPE --pr PR_NUMBER --wait` to
+ collect its evidence as soon as that attempt passes; retain routine
results in its ignored receipt and the PR body instead of another documentation
commit. Resolve any source differences explicitly before reusing that evidence.
- For an authorized Desktop release, start `make release-desktop ARGS="--prepare"`
diff --git a/docs/validation-and-release.md b/docs/validation-and-release.md
index 0c11359df..dc4add13e 100644
--- a/docs/validation-and-release.md
+++ b/docs/validation-and-release.md
@@ -269,6 +269,15 @@ collect its record with one read-only command:
python3 scripts/record-ci-validation.py RUN_ID --scope cli --pr PR_NUMBER
```
+For a run still in progress, add `--wait` and start the collector alongside review.
+It observes that run every ten seconds and collects its evidence as soon as the
+attempt passes. The default wait budget is 900s (`--wait-timeout`); collection gets
+its own 90s budget (`--timeout`). Both durations are recorded separately. A failed
+or cancelled run, changed source/attempt, or expired observation budget stops the
+command. A timeout or lookup failure does not establish that CI stopped: inspect
+and follow the same run ID. The collector never dispatches, cancels or retries CI,
+and waiting does not bypass the required job, artifact, source or PR checks below.
+
Use the required CI scope (`cli`, `tui`, `backend`, `desktop`, `process`, or `full`). The command
writes a short `validation.md` and machine-readable `receipt.json` under ignored
`.harness/validation/`; use the paragraph/table in the PR's verification section.
diff --git a/scripts/record-ci-validation.py b/scripts/record-ci-validation.py
index ca1c69d06..ab40290e1 100644
--- a/scripts/record-ci-validation.py
+++ b/scripts/record-ci-validation.py
@@ -73,16 +73,45 @@ class Client:
page += 1
-def check_run(run, repository, run_id):
+def check_run_identity(run, repository, run_id):
if (run.get("id") != run_id or run.get("repository", {}).get("full_name", "").lower() != repository.lower()
or run.get("path") != ".github/workflows/ci.yml" or not re.fullmatch(r"[0-9a-f]{40}", run.get("head_sha", ""))):
raise ValueError("run repository, workflow or source does not match this CI request")
- if run.get("status") != "completed" or run.get("conclusion") != "success":
- raise ValueError(f"CI has not passed: {run.get('status')}/{run.get('conclusion')}; follow the same run")
if type(run.get("run_attempt")) is not int or run["run_attempt"] < 1:
raise ValueError("run attempt is missing")
+def check_run(run, repository, run_id):
+ check_run_identity(run, repository, run_id)
+ if run.get("status") != "completed" or run.get("conclusion") != "success":
+ raise ValueError(f"CI has not passed: {run.get('status')}/{run.get('conclusion')}; follow the same run")
+
+
+def wait_for_run(client, run_id):
+ """Observe one immutable run attempt; never dispatch, cancel, or rerun it."""
+ identity, previous_state, announced_at = None, None, 0
+ while True:
+ if time.monotonic() >= client.deadline:
+ raise TimeoutError(f"CI wait deadline exceeded; follow the same run {run_id}")
+ run = client.api(f"actions/runs/{run_id}")
+ check_run_identity(run, client.repository, run_id)
+ current = (run["head_sha"], run["run_attempt"])
+ if identity is not None and current != identity:
+ raise ValueError("CI run source/attempt changed while waiting")
+ identity = current
+ state = (run.get("status"), run.get("conclusion"))
+ if state[0] not in {"queued", "requested", "waiting", "pending", "in_progress", "completed"}:
+ raise ValueError(f"unexpected CI run status: {state[0]}")
+ now = time.monotonic()
+ if state != previous_state or now - announced_at >= 30:
+ print(f"CI {run_id}, attempt {run['run_attempt']}: {state[0]}/{state[1] or 'pending'}", flush=True)
+ previous_state, announced_at = state, now
+ if state[0] == "completed":
+ check_run(run, client.repository, run_id)
+ return run
+ time.sleep(min(10, max(0, client.deadline - now)))
+
+
def check_jobs(run, jobs, scope):
names = [job["name"] for job in jobs]
if len(names) != len(set(names)) or len({job["id"] for job in jobs}) != len(jobs):
@@ -184,9 +213,11 @@ def source_comparison(root, tested, target):
"same_tree": tested_tree == target_tree, "changed_files": list(filter(None, paths)), "working_tree_dirty": dirty, "checkout_head": checkout}
-def collect(client, root, run_id, scope, target, output, pr_number=None):
+def collect(client, root, run_id, scope, target, output, pr_number=None, expected_run=None):
run = client.api(f"actions/runs/{run_id}")
check_run(run, client.repository, run_id)
+ if expected_run is not None and any(run[key] != expected_run[key] for key in ["id", "head_sha", "run_attempt"]):
+ raise ValueError("CI run changed between waiting and evidence collection")
with ThreadPoolExecutor(max_workers=3) as pool:
job_future = pool.submit(client.pages, f"actions/runs/{run_id}/jobs?filter=latest", "jobs")
artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "desktop", "full"} else None
@@ -294,22 +325,38 @@ def main(argv=None):
parser.add_argument("--target", default="HEAD", help="committed source to compare with the tested tree")
parser.add_argument("--pr", type=int, help="also verify that this PR still names the target commit")
parser.add_argument("--output", type=Path, help="new evidence directory; defaults to ignored .harness/validation/")
- parser.add_argument("--timeout", type=float, default=90)
+ parser.add_argument("--timeout", type=float, default=90, help="evidence collection budget in seconds (default: 90)")
+ parser.add_argument("--wait", action="store_true", help="follow this run attempt, then collect its completed evidence")
+ parser.add_argument("--wait-timeout", type=float, default=900, help="separate CI observation budget in seconds (default: 900)")
args = parser.parse_args(argv)
- if args.run < 1 or (args.pr is not None and args.pr < 1) or not 0 < args.timeout < float("inf") or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", args.repo):
+ if args.run < 1 or (args.pr is not None and args.pr < 1) or any(not 0 < value < float("inf") for value in [args.timeout, args.wait_timeout]) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", args.repo):
parser.error("supply a valid repository, positive IDs and finite positive timeout")
root = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()).resolve()
output = args.output or root / ".harness" / "validation" / (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + f"-ci-{args.run}")
output.mkdir(parents=True, exist_ok=False)
- started, started_at = time.monotonic(), utc_now()
+ waiting, observed, collection_started = None, None, None
try:
- record = collect(Client(args.repo, started + args.timeout), root, args.run, args.scope, args.target, output, args.pr)
+ if args.wait:
+ started = time.monotonic()
+ waiting = {"started_at": utc_now()}
+ try:
+ observed = wait_for_run(Client(args.repo, started + args.wait_timeout), args.run)
+ waiting.update(head_sha=observed["head_sha"], run_attempt=observed["run_attempt"])
+ finally:
+ waiting.update(finished_at=utc_now(), duration_seconds=round(time.monotonic() - started, 3))
+ collection_started, started_at = time.monotonic(), utc_now()
+ record = collect(Client(args.repo, collection_started + args.timeout), root, args.run, args.scope,
+ args.target, output, args.pr, expected_run=observed)
except (ValueError, KeyError, TypeError, OSError, RuntimeError, subprocess.SubprocessError, zipfile.BadZipFile) as error:
record = {"schema": 1, "status": "collection_failed", "run_id": args.run, "error": str(error)}
- record["collection"] = {"started_at": started_at, "finished_at": utc_now(), "duration_seconds": round(time.monotonic() - started, 3)}
+ if waiting is not None:
+ record["waiting"] = waiting
+ if collection_started is not None:
+ record["collection"] = {"started_at": started_at, "finished_at": utc_now(), "duration_seconds": round(time.monotonic() - collection_started, 3)}
(output / "receipt.json").write_text(json.dumps(record, indent=2) + "\n")
(output / "validation.md").write_text(markdown(record))
- print(f"{record['status']} ({record['collection']['duration_seconds']:g}s): {output / 'validation.md'}")
+ elapsed = ", ".join(f"{record[phase]['duration_seconds']:g}s {phase}" for phase in ["waiting", "collection"] if phase in record)
+ print(f"{record['status']} ({elapsed}): {output / 'validation.md'}")
if "error" in record:
print(record["error"], file=sys.stderr)
return {"passed": 0, "source_review_required": 3}.get(record["status"], 1)
diff --git a/scripts/tests/test_record_ci_validation.py b/scripts/tests/test_record_ci_validation.py
index 755c176a0..079aec51f 100644
--- a/scripts/tests/test_record_ci_validation.py
+++ b/scripts/tests/test_record_ci_validation.py
@@ -7,6 +7,7 @@ from pathlib import Path
import subprocess
import tempfile
import unittest
+from unittest import mock
import zipfile
SCRIPT = Path(__file__).resolve().parents[1] / "record-ci-validation.py"
@@ -55,7 +56,7 @@ class EvidenceTests(unittest.TestCase):
archive.writestr(name, json.dumps(summary))
return stream.getvalue()
- def collect(self, scope="cli", target="HEAD", with_pr=False, pr_after=None):
+ def collect(self, scope="cli", target="HEAD", with_pr=False, pr_after=None, expected_run=None):
fixture = self
class FakeClient:
@@ -79,7 +80,7 @@ class EvidenceTests(unittest.TestCase):
fixture.command_args = args
return fixture.archives[args[-1]] if hasattr(fixture, "archives") else fixture.archive
- return recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None)
+ return recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None, expected_run)
def test_complete_record_checks_identity_digest_coverage_and_pr(self):
result = self.collect(with_pr=True)
@@ -248,6 +249,12 @@ class EvidenceTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "PR head/base changed"):
self.collect(with_pr=True, pr_after=after)
+ def test_waited_attempt_cannot_be_replaced_before_collection(self):
+ self.assertEqual(self.collect(expected_run=self.run)["status"], "passed")
+ for change in [dict(run_attempt=2), dict(head_sha="b" * 40), dict(id=124)]:
+ with self.subTest(change=change), self.assertRaisesRegex(ValueError, "between waiting"):
+ self.collect(expected_run=dict(self.run, **change))
+
def test_stable_pr_pointing_elsewhere_needs_source_review(self):
self.pr["head"]["sha"] = "b" * 40
result = self.collect(with_pr=True)
@@ -270,5 +277,103 @@ class EvidenceTests(unittest.TestCase):
client.pages("jobs", "jobs")
+class WaitTests(unittest.TestCase):
+ def setUp(self):
+ self.now = 0
+ self.run = dict(id=123, repository=dict(full_name="owner/repo"), path=".github/workflows/ci.yml",
+ head_sha="a" * 40, run_attempt=1, status="completed", conclusion="success")
+ self.client = recorder.Client("owner/repo", 120)
+ self.patch(recorder.time, "monotonic", side_effect=lambda: self.now)
+ self.sleep = self.patch(recorder.time, "sleep", side_effect=self.advance)
+ self.patch(recorder, "print", create=True)
+
+ def patch(self, obj, name, **kwargs):
+ patcher = mock.patch.object(obj, name, **kwargs)
+ value = patcher.start()
+ self.addCleanup(patcher.stop)
+ return value
+
+ def advance(self, seconds):
+ self.now += seconds
+
+ def observe(self, runs):
+ self.client.api = mock.Mock(side_effect=runs)
+ return recorder.wait_for_run(self.client, 123)
+
+ def test_live_states_and_completed_fast_path_follow_one_run(self):
+ for states in [[], ["queued", "pending", "waiting", "requested", "in_progress"]]:
+ with self.subTest(states=states):
+ self.now = 0
+ self.sleep.reset_mock()
+ result = self.observe([dict(self.run, status=s, conclusion=None) for s in states] + [self.run])
+ self.assertEqual(result, self.run)
+ self.assertEqual(self.sleep.call_count, len(states))
+ self.assertTrue(all(call == mock.call("actions/runs/123") for call in self.client.api.call_args_list))
+
+ def test_failed_or_cancelled_runs_stop_without_following_a_later_rerun(self):
+ for conclusion in ["failure", "cancelled", "timed_out", "action_required", "skipped"]:
+ with self.subTest(conclusion=conclusion), self.assertRaisesRegex(ValueError, "CI has not passed"):
+ self.observe([dict(self.run, conclusion=conclusion), dict(self.run, run_attempt=2)])
+ self.assertEqual(self.client.api.call_count, 1)
+ self.sleep.assert_not_called()
+
+ def test_identity_changes_and_unknown_states_fail_closed(self):
+ for change in [dict(id=124), dict(repository=dict(full_name="other/repo")), dict(path="other.yml"),
+ dict(head_sha="b" * 40), dict(run_attempt=2), dict(status="unknown")]:
+ self.now = 0
+ with self.subTest(change=change), self.assertRaises(ValueError):
+ self.observe([dict(self.run, status="queued", conclusion=None), dict(self.run, **change)])
+
+ def test_wait_deadline_does_not_restart_or_cancel_the_job(self):
+ self.client.deadline = 12
+ pending = dict(self.run, status="in_progress", conclusion=None)
+ with self.assertRaisesRegex(TimeoutError, "follow the same run 123"):
+ self.observe([pending, pending])
+ self.assertEqual(self.now, 12)
+ self.assertEqual(self.sleep.call_args_list, [mock.call(10), mock.call(2)])
+ self.assertEqual(self.client.api.call_count, 2)
+
+ def test_cli_keeps_waiting_and_collection_budgets_and_receipts_separate(self):
+ with tempfile.TemporaryDirectory() as folder:
+ root, output = Path(folder), Path(folder) / "evidence"
+ self.patch(recorder.subprocess, "check_output", return_value=str(root))
+ self.patch(recorder, "markdown", return_value="verified\n")
+
+ def wait(client, run_id):
+ self.assertEqual((client.deadline, run_id), (120, 123))
+ self.advance(40)
+ return self.run
+
+ def collect(client, *args, expected_run=None):
+ self.assertEqual(client.deadline, 130) # A fresh 90s after waiting 40s.
+ self.assertEqual(expected_run, self.run)
+ self.advance(5)
+ return dict(status="passed")
+
+ self.patch(recorder, "wait_for_run", side_effect=wait)
+ self.patch(recorder, "collect", side_effect=collect)
+ result = recorder.main(["123", "--repo", "owner/repo", "--scope", "process", "--wait",
+ "--wait-timeout", "120", "--output", str(output)])
+ receipt = json.loads((output / "receipt.json").read_text())
+ self.assertEqual(result, 0)
+ self.assertEqual(receipt["waiting"]["duration_seconds"], 40)
+ self.assertEqual(receipt["waiting"]["head_sha"], self.run["head_sha"])
+ self.assertEqual(receipt["collection"]["duration_seconds"], 5)
+
+ def test_cli_preserves_wait_failure_without_attempting_collection(self):
+ with tempfile.TemporaryDirectory() as folder:
+ output = Path(folder) / "evidence"
+ self.patch(recorder.subprocess, "check_output", return_value=folder)
+ self.patch(recorder, "wait_for_run", side_effect=TimeoutError("CI wait deadline exceeded"))
+ collect = self.patch(recorder, "collect")
+ result = recorder.main(["123", "--scope", "process", "--wait", "--output", str(output)])
+ receipt = json.loads((output / "receipt.json").read_text())
+ self.assertEqual(result, 1)
+ self.assertIn("waiting", receipt)
+ self.assertNotIn("collection", receipt)
+ self.assertIn("CI wait deadline", receipt["error"])
+ collect.assert_not_called()
+
+
if __name__ == "__main__":
unittest.main()