Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Collect source-checked CI evidence for a fast PR handoff (#636)

Dee27af2fd88619
diff --git a/AGENTS.md b/AGENTS.md
index 52963af04..870ba5237 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -22,3 +22,7 @@ validation and shipping, use [docs/validation-and-release.md](docs/validation-an
validation cycle. Verify published versions and checksums, then report completion.
Desktop's `--wait` follows the exact tag/SHA through the workflow's six-artifact
verification; reuse that receipt instead of repeating the downloads manually.
+- Prepare the PR and review while checks run. Collect completed CI evidence with
+ `scripts/record-ci-validation.py RUN_ID --scope SCOPE --pr PR_NUMBER`; retain routine
+ results in its ignored receipt and the PR body instead of another documentation
+ commit. Resolve any source differences explicitly before reusing that evidence.
diff --git a/docs/validation-and-release.md b/docs/validation-and-release.md
index ce0cc1ca4..13058f670 100644
--- a/docs/validation-and-release.md
+++ b/docs/validation-and-release.md
@@ -157,6 +157,37 @@ before merge and then again on the merged commit. Check the merged source identi
and rerun only validation invalidated by the merge. Native checks absent from CI
still need their own evidence.
+Prepare the PR description and review the diff while CI is running. Once it passes,
+collect its record with one read-only command:
+
+```sh
+python3 scripts/record-ci-validation.py RUN_ID --scope cli --pr PR_NUMBER
+```
+
+Use the required CI scope (`cli`, `tui`, `backend`, `process`, or `full`). The command
+writes a short `validation.md` and machine-readable `receipt.json` under ignored
+`.harness/validation/`; use the paragraph/table in the PR's verification section.
+It checks the exact run/repository/workflow, required jobs, completed steps, source
+trees, and an optional PR's head/base stability. For CLI it also downloads the
+coverage summary by immutable artifact ID and verifies its archive checksum and
+file/case totals. Network calls are bounded; the default collection budget is 90s.
+
+Exit 0 means the requested CI scope passed and its committed source tree matches
+the selected target (`HEAD` by default). Exit 3 still saves the record but flags
+different source, a dirty working tree or a PR head mismatch for review. It lists
+changed paths without deciding that they are harmless. Review their impact and
+reuse only applicable evidence; a different SHA with the same tree is accepted.
+`--target COMMIT` supports historical audits and labels them as covering that
+commit rather than the current working copy. Missing objects need a fetch before
+comparison; failed/pending runs, missing/skipped required jobs, corrupt/expired
+artifacts, and changing run/PR state cannot become a successful record.
+
+Keep routine validation evidence in these receipts, CI artifacts and the PR body.
+Do not add a documentation commit or recreate all raw logs just to record another
+ordinary check. Commit a performance report when the comparison itself is useful
+repository documentation. The collector records CI time separately; request,
+implementation, review/merge and publication timestamps still need their own record.
+
For deterministic local checks, the runner can do that comparison and reuse the
original logs. Add an explicit `reuse` contract to each eligible check:
diff --git a/scripts/record-ci-validation.py b/scripts/record-ci-validation.py
new file mode 100644
index 000000000..d7f21eee9
--- /dev/null
+++ b/scripts/record-ci-validation.py
@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+"""Collect a completed CI run into JSON and a PR-ready validation paragraph.
+
+Read-only GitHub access; this does not run checks, edit PRs, approve or merge code.
+Exit 3 means the CI passed but source differences still require review.
+"""
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import time
+import zipfile
+
+CLI_JOBS = {"cli-contracts", "typecheck-test"} | {f"cli-tests ({i})" for i in range(1, 5)} | {
+ f"serial-native ({system}, {node})" for system in ["ubuntu-latest", "macos-latest"] for node in ["20.19.0", "22.23.2"]
+}
+TUI_JOBS = {"tui-test (ubuntu-latest, x86_64-unknown-linux-musl)", "tui-test (ubuntu-24.04-arm, aarch64-unknown-linux-musl)"}
+SCOPES = {"cli": CLI_JOBS, "tui": TUI_JOBS, "backend": {"backend-desk"}, "process": set()}
+SCOPES["full"] = CLI_JOBS | TUI_JOBS | SCOPES["backend"]
+
+
+def utc_now():
+ return datetime.now(timezone.utc).isoformat()
+
+
+def seconds(start, end):
+ value = (datetime.fromisoformat(end.replace("Z", "+00:00")) - datetime.fromisoformat(start.replace("Z", "+00:00"))).total_seconds()
+ if value < 0:
+ raise ValueError("invalid CI timestamps")
+ return round(value, 3)
+
+
+class Client:
+ def __init__(self, repository, deadline):
+ self.repository, self.deadline = repository, deadline
+
+ def command(self, *args, binary=False):
+ remaining = self.deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("evidence collection deadline exceeded")
+ env = dict(os.environ, GH_PROMPT_DISABLED="1")
+ env.pop("GH_DEBUG", None)
+ result = subprocess.run(["gh", *args], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, timeout=min(30, remaining), env=env)
+ if result.returncode:
+ raise RuntimeError(f"GitHub lookup failed ({result.returncode}): {' '.join(args[:2])}")
+ return result.stdout if binary else json.loads(result.stdout)
+
+ def api(self, path):
+ return self.command("api", f"repos/{self.repository}/{path}")
+
+ def pages(self, path, key):
+ values, page = [], 1
+ separator = "&" if "?" in path else "?"
+ while True:
+ data = self.api(f"{path}{separator}per_page=100&page={page}")
+ values.extend(data[key])
+ if len(values) >= data["total_count"]:
+ if len(values) != data["total_count"]:
+ raise ValueError(f"inconsistent pagination for {key}")
+ return values
+ if not data[key]:
+ raise ValueError(f"incomplete pagination for {key}")
+ page += 1
+
+
+def check_run(run, repository, run_id):
+ if (run.get("id") != run_id or run.get("repository", {}).get("full_name", "").lower() != repository.lower()
+ or run.get("path") != ".github/workflows/ci.yml" or not re.fullmatch(r"[0-9a-f]{40}", run.get("head_sha", ""))):
+ raise ValueError("run repository, workflow or source does not match this CI request")
+ if run.get("status") != "completed" or run.get("conclusion") != "success":
+ raise ValueError(f"CI has not passed: {run.get('status')}/{run.get('conclusion')}; follow the same run")
+ if type(run.get("run_attempt")) is not int or run["run_attempt"] < 1:
+ raise ValueError("run attempt is missing")
+
+
+def check_jobs(run, jobs, scope):
+ names = [job["name"] for job in jobs]
+ if len(names) != len(set(names)) or len({job["id"] for job in jobs}) != len(jobs):
+ raise ValueError("duplicate CI jobs")
+ required = SCOPES[scope] | {"process-checks"}
+ missing = required - set(names)
+ if missing:
+ raise ValueError(f"requested {scope} scope is missing jobs: {', '.join(sorted(missing))}")
+ for job in jobs:
+ if (job.get("run_id") != run["id"] or job.get("head_sha") != run["head_sha"]
+ or not 1 <= job.get("run_attempt", 0) <= run["run_attempt"]):
+ raise ValueError(f"job belongs to a different run/source/attempt: {job['name']}")
+ if job.get("status") != "completed" or job.get("conclusion") not in {"success", "skipped"}:
+ raise ValueError(f"job did not pass: {job['name']}")
+ if job["name"] in required and job["conclusion"] != "success":
+ raise ValueError(f"required job was skipped: {job['name']}")
+ if any(step.get("conclusion") not in {"success", "skipped"} for step in job.get("steps", [])):
+ raise ValueError(f"job contains a failed or unfinished step: {job['name']}")
+
+
+def read_cli_summary(artifact, archive, run):
+ provenance = artifact.get("workflow_run", {})
+ if (artifact.get("name") != "cli-test-summary" or artifact.get("expired") is not False
+ or provenance.get("id") != run["id"] or provenance.get("head_sha") != run["head_sha"]):
+ raise ValueError("CLI summary artifact is expired or belongs to another source/run")
+ digest = "sha256:" + hashlib.sha256(archive).hexdigest()
+ if artifact.get("digest") != digest:
+ raise ValueError("CLI summary archive digest mismatch")
+ with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
+ if bundle.namelist() != ["cli-test-summary.json"] or bundle.infolist()[0].file_size > 8 * 1024 ** 2:
+ raise ValueError("unexpected CLI summary archive contents")
+ summary = json.loads(bundle.read("cli-test-summary.json"))
+ if not isinstance(summary, dict):
+ raise ValueError("invalid CLI summary object")
+ for key in ["files", "passed", "skipped", "todo"]:
+ if type(summary.get(key)) is not int or summary[key] < 0:
+ raise ValueError("invalid CLI summary counts")
+ files = summary.get("verified_files", [])
+ shards = summary.get("shards", [])
+ if (summary.get("schema") != 1 or summary.get("status") != "passed" or not summary["files"]
+ or len(files) != summary["files"] or len(set(files)) != len(files)
+ or len(shards) != 4 or {shard["shard"] for shard in shards} != {1, 2, 3, 4}
+ or any(sum(shard[key] for shard in shards) != summary[key] for key in ["files", "passed", "skipped", "todo"])):
+ raise ValueError("CLI summary coverage is incomplete or inconsistent")
+ return summary
+
+
+def source_comparison(root, tested, target):
+ def git(*args):
+ return subprocess.check_output(["git", *args], cwd=root, stderr=subprocess.PIPE, timeout=10).decode().strip()
+ target_sha = git("rev-parse", "--verify", "--end-of-options", f"{target}^{{commit}}")
+ tested_tree, target_tree = (git("rev-parse", f"{sha}^{{tree}}") for sha in [tested, target_sha])
+ paths = git("diff", "--name-only", "--no-renames", "-z", tested, target_sha, "--").split("\0")
+ checkout = git("rev-parse", "HEAD")
+ dirty = bool(git("status", "--porcelain", "--untracked-files=normal")) if target_sha == checkout else None
+ return {"tested_sha": tested, "target_sha": target_sha, "tested_tree": tested_tree, "target_tree": target_tree,
+ "same_tree": tested_tree == target_tree, "changed_files": list(filter(None, paths)), "working_tree_dirty": dirty, "checkout_head": checkout}
+
+
+def collect(client, root, run_id, scope, target, output, pr_number=None):
+ run = client.api(f"actions/runs/{run_id}")
+ check_run(run, client.repository, run_id)
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ job_future = pool.submit(client.pages, f"actions/runs/{run_id}/jobs?filter=latest", "jobs")
+ artifact_future = pool.submit(client.pages, f"actions/runs/{run_id}/artifacts", "artifacts") if scope in {"cli", "full"} else None
+ pr_future = pool.submit(client.api, f"pulls/{pr_number}") if pr_number else None
+ jobs = job_future.result()
+ artifacts = artifact_future.result() if artifact_future else []
+ pr = pr_future.result() if pr_future else None
+ check_jobs(run, jobs, scope)
+ source = source_comparison(root, run["head_sha"], target)
+ record = {"schema": 1, "repository": client.repository, "scope": scope, "status": "passed",
+ "run": {key: run[key] for key in ["id", "run_attempt", "head_sha", "html_url", "created_at", "updated_at", "status", "conclusion"]},
+ "workflow_seconds": seconds(run["created_at"], run["updated_at"]), "source": source,
+ "jobs": [{"id": job["id"], "name": job["name"], "attempt": job["run_attempt"], "conclusion": job["conclusion"],
+ "started_at": job["started_at"], "completed_at": job["completed_at"],
+ "seconds": seconds(job["started_at"], job["completed_at"]) if job["conclusion"] == "success" else None,
+ "steps": [{"name": step["name"], "conclusion": step["conclusion"]} for step in job.get("steps", [])]} for job in jobs]}
+ if scope in {"cli", "full"}:
+ matches = [artifact for artifact in artifacts if artifact["name"] == "cli-test-summary"]
+ if len(matches) != 1:
+ raise ValueError("expected one nonexpired CLI coverage summary artifact")
+ artifact = matches[0]
+ if artifact.get("size_in_bytes", 16 * 1024 ** 2 + 1) > 16 * 1024 ** 2:
+ raise ValueError("CLI coverage artifact is unexpectedly large")
+ archive = client.command("api", f"repos/{client.repository}/actions/artifacts/{artifact['id']}/zip", binary=True)
+ record["cli_summary"] = read_cli_summary(artifact, archive, run)
+ record["artifact"] = {key: artifact[key] for key in ["id", "name", "digest", "size_in_bytes"]}
+ (output / "cli-test-summary.zip").write_bytes(archive)
+ if pr:
+ record["pr"] = {"number": pr_number, "url": pr["html_url"], "head": pr["head"]["sha"], "base": pr["base"]["sha"]}
+ # Detect reruns and moving PR heads after the independent downloads. Never
+ # accidentally pair a new attempt/head with the old artifact or job verdicts.
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ run_future = pool.submit(client.api, f"actions/runs/{run_id}")
+ pr_future = pool.submit(client.api, f"pulls/{pr_number}") if pr else None
+ after = run_future.result()
+ pr_after = pr_future.result() if pr_future else None
+ check_run(after, client.repository, run_id)
+ if any(after[key] != run[key] for key in ["run_attempt", "head_sha", "updated_at"]):
+ raise ValueError("CI run changed during collection; inspect the same run again")
+ if pr and any(pr_after[key]["sha"] != pr[key]["sha"] for key in ["head", "base"]):
+ raise ValueError("PR head/base changed during collection; review the new state")
+ if source != source_comparison(root, run["head_sha"], target):
+ raise ValueError("local source changed during collection")
+ if not source["same_tree"] or source["working_tree_dirty"] or (pr and pr["head"]["sha"] != source["target_sha"]):
+ record["status"] = "source_review_required"
+ return record
+
+
+def markdown(record):
+ if record["status"] == "collection_failed":
+ return f"CI evidence collection incomplete: {record['error']}\n"
+ run, source = record["run"], record["source"]
+ elapsed = int(record["workflow_seconds"])
+ lines = [f"Required scope `{record['scope']}` verified: [CI run {run['id']}]({run['html_url']}) passed, attempt {run['run_attempt']}.",
+ f"Elapsed **{elapsed // 60}m{elapsed % 60:02}s**, {run['created_at']} → {run['updated_at']} (CI time only).",
+ f"Tested commit `{source['tested_sha']}`; target `{source['target_sha']}`."]
+ if "cli_summary" in record:
+ c = record["cli_summary"]
+ lines.append(f"CLI default suite: **{c['files']} files verified exactly once; {c['passed']:,} passed, {c['skipped']} skipped, {c['todo']} todo**. Summary artifact digest verified.")
+ lines += ["", "| Job | Result | Time |", "| --- | --- | ---: |"]
+ for job in record["jobs"]:
+ name = job["name"].replace("|", "\\|").replace("\n", " ")
+ elapsed = f"{job['seconds']:g}s" if job["seconds"] is not None else "—"
+ lines.append(f"| {name} | {job['conclusion']} | {elapsed} |")
+ lines.append("")
+ if record["status"] == "source_review_required":
+ lines.append("**Source review required before reusing this result.**")
+ if not source["same_tree"]:
+ lines.append("Changed paths since the tested commit: " + ", ".join(f"`{path.replace('`', '')}`" for path in source["changed_files"]) + ".")
+ if source["working_tree_dirty"]:
+ lines.append("The current working tree also contains changes outside this committed CI result.")
+ if "pr" in record and record["pr"]["head"] != source["target_sha"]:
+ lines.append(f"PR head `{record['pr']['head']}` differs from the selected target.")
+ else:
+ lines.append("The tested and target source trees match. Scope selection and code/merge review remain explicit.")
+ if source["working_tree_dirty"] is None:
+ lines.append("The selected target is a historical commit; this record does not cover the current working copy.")
+ return "\n".join(lines) + "\n"
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("run", type=int)
+ parser.add_argument("--repo", default="autonomous-ai/openharness")
+ parser.add_argument("--scope", choices=SCOPES, required=True)
+ parser.add_argument("--target", default="HEAD", help="committed source to compare with the tested tree")
+ parser.add_argument("--pr", type=int, help="also verify that this PR still names the target commit")
+ parser.add_argument("--output", type=Path, help="new evidence directory; defaults to ignored .harness/validation/")
+ parser.add_argument("--timeout", type=float, default=90)
+ args = parser.parse_args(argv)
+ if args.run < 1 or (args.pr is not None and args.pr < 1) or not 0 < args.timeout < float("inf") or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", args.repo):
+ parser.error("supply a valid repository, positive IDs and finite positive timeout")
+ root = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()).resolve()
+ output = args.output or root / ".harness" / "validation" / (datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + f"-ci-{args.run}")
+ output.mkdir(parents=True, exist_ok=False)
+ started, started_at = time.monotonic(), utc_now()
+ try:
+ record = collect(Client(args.repo, started + args.timeout), root, args.run, args.scope, args.target, output, args.pr)
+ except (ValueError, KeyError, TypeError, OSError, RuntimeError, subprocess.SubprocessError, zipfile.BadZipFile) as error:
+ record = {"schema": 1, "status": "collection_failed", "run_id": args.run, "error": str(error)}
+ record["collection"] = {"started_at": started_at, "finished_at": utc_now(), "duration_seconds": round(time.monotonic() - started, 3)}
+ (output / "receipt.json").write_text(json.dumps(record, indent=2) + "\n")
+ (output / "validation.md").write_text(markdown(record))
+ print(f"{record['status']} ({record['collection']['duration_seconds']:g}s): {output / 'validation.md'}")
+ if "error" in record:
+ print(record["error"], file=sys.stderr)
+ return {"passed": 0, "source_review_required": 3}.get(record["status"], 1)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/test_record_ci_validation.py b/scripts/tests/test_record_ci_validation.py
new file mode 100644
index 000000000..43c2f590c
--- /dev/null
+++ b/scripts/tests/test_record_ci_validation.py
@@ -0,0 +1,205 @@
+import copy
+import hashlib
+import importlib.util
+import io
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+import zipfile
+
+SCRIPT = Path(__file__).resolve().parents[1] / "record-ci-validation.py"
+spec = importlib.util.spec_from_file_location("record_ci", SCRIPT)
+recorder = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(recorder)
+
+
+class EvidenceTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.root = Path(self.tmp.name) / "source"
+ self.root.mkdir()
+ self.output = Path(self.tmp.name) / "evidence"
+ self.output.mkdir()
+ (self.root / "source.txt").write_text("tested\n")
+ self.git("init", "-q")
+ self.git("add", ".")
+ self.git("-c", "user.name=Fixture", "-c", "[email protected]", "commit", "-qm", "tested")
+ self.sha = self.git("rev-parse", "HEAD")
+ self.run = {"id": 123, "repository": {"full_name": "owner/repo"}, "path": ".github/workflows/ci.yml",
+ "head_sha": self.sha, "run_attempt": 1, "status": "completed", "conclusion": "success",
+ "html_url": "https://github.com/owner/repo/actions/runs/123",
+ "created_at": "2026-10-03T00:00:00Z", "updated_at": "2026-10-03T00:03:00Z"}
+ self.jobs = [{"id": i, "name": name, "run_id": 123, "run_attempt": 1, "head_sha": self.sha,
+ "status": "completed", "conclusion": "success", "started_at": "2026-10-03T00:00:10Z",
+ "completed_at": "2026-10-03T00:02:59Z", "steps": []}
+ for i, name in enumerate(sorted(recorder.CLI_JOBS | {"process-checks"}), 1)]
+ self.summary = {"schema": 1, "status": "passed", "files": 4, "passed": 4, "skipped": 0, "todo": 0,
+ "verified_files": [f"src/file-{i}.spec.ts" for i in range(4)],
+ "shards": [{"shard": i, "files": 1, "passed": 1, "skipped": 0, "todo": 0} for i in range(1, 5)]}
+ self.archive = self.zip_summary(self.summary)
+ self.artifact = {"id": 987, "name": "cli-test-summary", "expired": False, "size_in_bytes": len(self.archive),
+ "digest": "sha256:" + hashlib.sha256(self.archive).hexdigest(),
+ "workflow_run": {"id": 123, "head_sha": self.sha}}
+ self.pr = {"html_url": "https://github.com/owner/repo/pull/5", "head": {"sha": self.sha}, "base": {"sha": self.sha}}
+ self.after = None
+
+ def git(self, *args):
+ return subprocess.check_output(["git", *args], cwd=self.root, stderr=subprocess.PIPE, text=True).strip()
+
+ def zip_summary(self, summary, name="cli-test-summary.json"):
+ stream = io.BytesIO()
+ with zipfile.ZipFile(stream, "w") as archive:
+ archive.writestr(name, json.dumps(summary))
+ return stream.getvalue()
+
+ def collect(self, scope="cli", target="HEAD", with_pr=False, pr_after=None):
+ fixture = self
+
+ class FakeClient:
+ repository = "owner/repo"
+ run_reads = 0
+ pr_reads = 0
+
+ def api(self, path):
+ if path == "actions/runs/123":
+ self.run_reads += 1
+ return copy.deepcopy(fixture.after if self.run_reads > 1 and fixture.after else fixture.run)
+ if path == "pulls/5":
+ self.pr_reads += 1
+ return copy.deepcopy(pr_after if self.pr_reads > 1 and pr_after else fixture.pr)
+ raise AssertionError(path)
+
+ def pages(self, path, key):
+ return copy.deepcopy(fixture.jobs if key == "jobs" else [fixture.artifact])
+
+ def command(self, *args, binary=False):
+ fixture.command_args = args
+ return fixture.archive
+
+ return recorder.collect(FakeClient(), self.root, 123, scope, target, self.output, 5 if with_pr else None)
+
+ def test_complete_record_checks_identity_digest_coverage_and_pr(self):
+ result = self.collect(with_pr=True)
+ self.assertEqual(result["status"], "passed")
+ self.assertEqual(result["workflow_seconds"], 180)
+ self.assertEqual(result["cli_summary"]["passed"], 4)
+ self.assertEqual(result["pr"]["head"], self.sha)
+ self.assertEqual(self.command_args[-1], "repos/owner/repo/actions/artifacts/987/zip")
+ self.assertIn("4 files verified exactly once", recorder.markdown(result))
+ self.assertEqual((self.output / "cli-test-summary.zip").read_bytes(), self.archive)
+
+ def test_process_scope_needs_no_cli_artifact(self):
+ self.jobs = [job for job in self.jobs if job["name"] == "process-checks"]
+ result = self.collect(scope="process")
+ self.assertEqual(result["status"], "passed")
+ self.assertNotIn("cli_summary", result)
+ self.assertFalse((self.output / "cli-test-summary.zip").exists())
+
+ def test_partial_scope_cannot_be_called_full_ci(self):
+ with self.assertRaisesRegex(ValueError, "missing jobs"):
+ self.collect(scope="full")
+
+ def test_missing_skipped_failed_or_wrong_source_job_is_rejected(self):
+ original = copy.deepcopy(self.jobs)
+ for change in [lambda: self.jobs.pop(), lambda: self.jobs[0].update(conclusion="skipped"),
+ lambda: self.jobs[0].update(conclusion="failure"), lambda: self.jobs[0].update(head_sha="b" * 40),
+ lambda: self.jobs.append(self.jobs[0])]:
+ self.jobs = copy.deepcopy(original)
+ change()
+ with self.assertRaises(ValueError):
+ self.collect()
+
+ def test_pending_failed_or_wrong_workflow_run_is_not_success(self):
+ original = dict(self.run)
+ for change in [dict(status="in_progress"), dict(conclusion="failure"), dict(path=".github/workflows/other.yml"),
+ dict(repository={"full_name": "another/repo"}), dict(id=456)]:
+ self.run = dict(original, **change)
+ with self.subTest(change=change), self.assertRaises(ValueError):
+ self.collect()
+
+ def test_continued_step_failure_cannot_hide_behind_a_green_job(self):
+ self.jobs[0]["steps"] = [{"name": "tests", "conclusion": "failure"}]
+ with self.assertRaisesRegex(ValueError, "failed or unfinished step"):
+ self.collect()
+
+ def test_corrupt_expired_or_mismatched_artifact_is_rejected(self):
+ original = copy.deepcopy(self.artifact)
+ for change in [dict(expired=True), dict(digest="sha256:" + "0" * 64), dict(workflow_run={"id": 456, "head_sha": self.sha})]:
+ self.artifact = dict(original, **change)
+ with self.subTest(change=change), self.assertRaises(ValueError):
+ self.collect()
+
+ def test_archive_paths_are_never_extracted(self):
+ self.archive = self.zip_summary(self.summary, "../escaped.json")
+ self.artifact.update(digest="sha256:" + hashlib.sha256(self.archive).hexdigest())
+ with self.assertRaisesRegex(ValueError, "contents"):
+ self.collect()
+ self.assertFalse((self.output.parent / "escaped.json").exists())
+
+ def test_duplicate_or_inconsistent_coverage_is_rejected(self):
+ for bad in [dict(self.summary, files=5), dict(self.summary, passed=True),
+ dict(self.summary, verified_files=["same"] * 4), dict(self.summary, shards=self.summary["shards"][:3])]:
+ archive = self.zip_summary(bad)
+ artifact = dict(self.artifact, digest="sha256:" + hashlib.sha256(archive).hexdigest())
+ with self.assertRaises(ValueError):
+ recorder.read_cli_summary(artifact, archive, self.run)
+
+ def test_source_changes_and_dirty_working_copy_require_review(self):
+ (self.root / "source.txt").write_text("different\n")
+ result = self.collect()
+ self.assertEqual(result["status"], "source_review_required")
+ self.assertTrue(result["source"]["working_tree_dirty"])
+ self.git("add", ".")
+ self.git("-c", "user.name=Fixture", "-c", "[email protected]", "commit", "-qm", "changed")
+ result = self.collect()
+ self.assertEqual(result["status"], "source_review_required")
+ self.assertEqual(result["source"]["changed_files"], ["source.txt"])
+ historical = self.collect(target=self.sha)
+ self.assertEqual(historical["status"], "passed")
+ self.assertIn("historical commit", recorder.markdown(historical))
+
+ def test_same_tree_with_different_commit_keeps_valid_evidence(self):
+ self.git("-c", "user.name=Fixture", "-c", "[email protected]", "commit", "--allow-empty", "-qm", "same tree")
+ result = self.collect()
+ self.assertEqual(result["status"], "passed")
+ self.assertNotEqual(result["source"]["tested_sha"], result["source"]["target_sha"])
+ self.assertTrue(result["source"]["same_tree"])
+
+ def test_rerun_or_pr_movement_during_collection_is_rejected(self):
+ self.after = dict(self.run, run_attempt=2)
+ with self.assertRaisesRegex(ValueError, "run changed"):
+ self.collect()
+ self.after = None
+ for part in ["head", "base"]:
+ after = copy.deepcopy(self.pr)
+ after[part]["sha"] = "b" * 40
+ with self.assertRaisesRegex(ValueError, "PR head/base changed"):
+ self.collect(with_pr=True, pr_after=after)
+
+ def test_stable_pr_pointing_elsewhere_needs_source_review(self):
+ self.pr["head"]["sha"] = "b" * 40
+ result = self.collect(with_pr=True)
+ self.assertEqual(result["status"], "source_review_required")
+ self.assertIn("PR head", recorder.markdown(result))
+
+ def test_pagination_keeps_all_jobs_and_rejects_truncation(self):
+ client = recorder.Client("owner/repo", 0)
+ calls = []
+
+ def api(path):
+ calls.append(path)
+ return {"total_count": 101, "jobs": list(range(100)) if path.endswith("&page=1") else [100]}
+
+ client.api = api
+ self.assertEqual(client.pages("jobs?filter=latest", "jobs"), list(range(101)))
+ self.assertEqual(len(calls), 2)
+ client.api = lambda _: {"total_count": 1, "jobs": []}
+ with self.assertRaisesRegex(ValueError, "incomplete pagination"):
+ client.pages("jobs", "jobs")
+
+
+if __name__ == "__main__":
+ unittest.main()