The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import copy
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import subprocess
import tempfile
import types
import unittest
from unittest import mock
import zipfile
spec = importlib.util.spec_from_file_location("merge_pr", Path(__file__).resolve().parents[1] / "merge-validated-pr.py")
merger = importlib.util.module_from_spec(spec)
spec.loader.exec_module(merger)
class MergeTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name) / "source"
self.root.mkdir()
self.git("init", "-q")
self.git("config", "user.name", "Fixture")
self.git("config", "user.email", "[email protected]")
(self.root / "source.txt").write_text("base\n")
self.git("add", ".")
self.git("commit", "-qm", "base")
self.base = self.git("rev-parse", "HEAD")
(self.root / "source.txt").write_text("reviewed\n")
self.git("commit", "-qam", "reviewed")
self.head = self.git("rev-parse", "HEAD")
self.tree = self.git("rev-parse", "HEAD^{tree}")
self.output = Path(self.tmp.name) / "result"
self.client = FakeClient(self)
self.patch(merger.ci, "print", create=True)
self.patch(merger.time, "sleep")
def git(self, *args):
return subprocess.check_output(["git", *args], cwd=self.root, stderr=subprocess.PIPE, text=True).strip()
def patch(self, obj, name, **kwargs):
patcher = mock.patch.object(obj, name, **kwargs)
value = patcher.start()
self.addCleanup(patcher.stop)
return value
def finish(self, merge=True, automatic=False):
self.client.required_gate = automatic
result = merger.finish(self.client, self.root, 5, None if automatic else 123,
None if automatic else "process", self.head, self.base,
self.output, merge=merge)
self.assertEqual(json.loads((self.output / "receipt.json").read_text()), result)
return result
def test_automatic_pr_gate_merges_without_queue_or_legacy_job_inventory(self):
self.patch(merger.ci, "collect", side_effect=AssertionError("legacy collector must not run"))
result = self.finish(automatic=True)
self.assertEqual(result["status"], "merged")
self.assertEqual(result["run_id"], 123)
self.assertEqual(result["pr_evidence"]["source_sha"], self.head)
self.assertTrue(result["merge"]["same_tested_tree"])
self.assertEqual(len(self.client.writes), 1)
self.assertIn("sha=" + self.head, self.client.writes[0])
self.assertTrue((self.output / "ci/ci-required.zip").is_file())
self.assertIn("automatic PR CI", (self.output / "merge.md").read_text())
def test_automatic_preview_verifies_the_gate_without_merging(self):
self.assertEqual(self.finish(merge=False, automatic=True)["status"], "ready")
self.assertEqual(self.client.writes, [])
def test_failed_automatic_ci_or_missing_gate_cannot_merge(self):
self.client.run["conclusion"] = "failure"
self.assertEqual(self.finish(automatic=True)["status"], "not_merged")
self.output = self.output.with_name("missing-gate")
self.client.run["conclusion"] = "success"
self.client.gate_jobs = []
result = self.finish(automatic=True)
self.assertIn("required CI gate did not pass", result["error"])
self.assertEqual(self.client.writes, [])
def test_manual_ci_cannot_replace_automatic_pr_ci(self):
self.client.run["event"] = "workflow_dispatch"
result = self.finish(automatic=True)
self.assertEqual(result["status"], "not_merged")
self.assertIn("automatic CI has not run", result["error"])
self.assertEqual(self.client.writes, [])
def test_automatic_gate_must_cover_the_local_reviewed_tree(self):
self.client.tested_tree = "e" * 40
result = self.finish(automatic=True)
self.assertEqual(result["status"], "not_merged")
self.assertIn("does not cover the reviewed source", result["error"])
self.assertEqual(self.client.writes, [])
def test_superseded_automatic_run_or_attempt_cannot_authorize_merge(self):
for change in [dict(id=124), dict(run_attempt=2)]:
self.client = FakeClient(self)
self.client.superseding_run = dict(self.client.run, **change)
self.output = self.output.with_name("superseded-" + next(iter(change)))
with self.subTest(change=change):
result = self.finish(automatic=True)
self.assertEqual(result["status"], "not_merged")
self.assertIn("CI changed during verification", result["error"])
self.assertEqual(self.client.writes, [])
def reused_receipt(self):
record = dict(origin_sha="e" * 40, origin_tree="f" * 40, run_id=99, run_attempt=1,
run_url="https://github.com/owner/repo/actions/runs/99", reason="passed in run 99")
self.client.receipt_patch = dict(suites=["mobile"], reused={"mobile": record},
reuse=dict(status="applied", passed_head="e" * 40, main_parent=self.base))
return record
def main_planner(self, verify):
calls = []
def verify_reuse(root, receipt, base, evidence):
calls.append((root, receipt["reused"], base, evidence))
return verify(receipt)
main = types.SimpleNamespace(GitHubEvidence=lambda client: ("main's reader", client), verify_reuse=verify_reuse)
loader = self.patch(merger.queue.planner, "main_planner", return_value=main)
return loader, calls
def test_reused_results_are_rederived_by_mains_planner_before_merging(self):
record = self.reused_receipt()
loader, calls = self.main_planner(lambda receipt: receipt["reused"])
result = self.finish(automatic=True)
self.assertEqual(result["status"], "merged")
loader.assert_called_once_with(self.root, self.base)
self.assertEqual(calls, [(self.root, {"mobile": record}, self.base, ("main's reader", self.client))])
self.assertEqual(result["pr_evidence"]["reused"], {"mobile": record})
self.assertIn("reused mobile from https://github.com/owner/repo/actions/runs/99", (self.output / "merge.md").read_text())
def test_reuse_that_mains_planner_refuses_or_cannot_check_blocks_the_merge(self):
self.reused_receipt()
def refuse(receipt):
raise ValueError("reused mobile refused: main changed its input backend/src/types/productUsage.ts")
self.main_planner(refuse)
result = self.finish(automatic=True)
self.assertEqual(result["status"], "not_merged")
self.assertIn("productUsage", result["error"])
self.assertEqual(self.client.writes, [])
self.output = self.output.with_name("no-verifier")
self.patch(merger.queue.planner, "main_planner", side_effect=ValueError("main's planner cannot verify reused results (X)"))
result = self.finish(automatic=True)
self.assertIn("cannot verify reused results", result["error"])
self.assertEqual(self.client.writes, [])
def test_a_receipt_that_skips_a_suite_it_did_not_reuse_blocks_the_merge(self):
self.client.receipt_patch = dict(suites=["mobile"])
result = self.finish(automatic=True)
self.assertIn("does not run every suite", result["error"])
self.assertEqual(self.client.writes, [])
def test_default_command_uses_direct_merge_unless_github_requires_queue(self):
direct = self.patch(merger, "finish", return_value={"status": "ready"})
queued = self.patch(merger.queue, "finish", return_value={"status": "ready_for_queue"})
self.patch(merger, "git", return_value=str(self.root))
self.patch(merger.ci, "Client", return_value=self.client)
self.patch(merger.sys, "argv", new=["merge-validated-pr.py", "5", "--reviewed-head", self.head,
"--reviewed-base", self.base])
self.patch(merger, "print", create=True)
self.assertEqual(merger.main(), 0)
self.assertEqual(direct.call_args.args[2:5], (5, None, None))
queued.assert_not_called()
direct.reset_mock()
self.client.rules = [{"type": "merge_queue"}]
self.assertEqual(merger.main(), 0)
queued.assert_called_once()
direct.assert_not_called()
def test_preview_verifies_full_evidence_without_mutating_github(self):
result = self.finish(merge=False)
self.assertEqual(result["status"], "ready")
self.assertFalse(result["merge_authorized"])
self.assertEqual(self.client.writes, [])
self.assertTrue((self.output / "ci/validation.md").is_file())
self.assertEqual(result["reviewed_source"]["tree"], self.tree)
def commit(self, sha, message, name="Ada", email="[email protected]", login="ada"):
return dict(sha=sha, author=dict(login=login) if login else None,
commit=dict(message=message, author=dict(name=name, email=email)))
def test_squash_message_names_harness_once_whichever_address_a_commit_has(self):
pr = dict(number=5, title="Fix it", user=dict(login="ken"))
old = "348238807+harness-for-autonomous[bot]@users.noreply.github.com"
commits = [self.commit("1", "First\n\nWhy.", "harness-for-autonomous[bot]", old, "harness-for-autonomous[bot]"),
self.commit("2", f"Second\n\nCo-authored-by: Harness <{old}>", "Harness by Autonomous", "[email protected]", None),
self.commit("3", "Third\n\nCo-authored-by: harness by autonomous <[email protected]>\nHarness-Agent: Atlas")]
title, message = merger.squash_message(pr, commits)
self.assertEqual(title, "Fix it (#5)")
self.assertEqual(message.count("Co-authored-by: Harness by Autonomous <[email protected]>"), 1)
self.assertNotIn("[bot]", message)
self.assertIn("* First\n\nWhy.", message)
self.assertIn("Harness-Agent: Atlas", message)
self.assertEqual(message.count("Co-authored-by: Ada <[email protected]>"), 1)
self.assertIn("Co-authored-by: Harness by Autonomous <[email protected]>", message.splitlines()[-2:])
def test_squash_message_keeps_other_people_once_and_leaves_out_the_pull_requests_author(self):
pr = dict(number=5, title="Fix it", user=dict(login="ken"))
commits = [self.commit("1", "One\n\nCo-authored-by: Grace <[email protected]>", "Ken", "[email protected]", "ken"),
self.commit("2", "Two\n\nCo-authored-by: Grace <[email protected]>", "Ken", "[email protected]", "ken"),
self.commit("3", "Three", "dependabot[bot]", "49699333+dependabot[bot]@users.noreply.github.com", "dependabot[bot]")]
_, message = merger.squash_message(pr, commits)
self.assertEqual(message.lower().count("co-authored-by: grace <[email protected]>"), 1)
self.assertNotIn("[email protected]", message)
self.assertIn("Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>", message)
def test_a_single_commit_keeps_its_own_title_and_body_as_github_does(self):
pr = dict(number=5, title="Other title", user=dict(login="ada"))
title, message = merger.squash_message(pr, [self.commit("1", "Only\n\nBody.")])
self.assertEqual(title, "Only (#5)")
self.assertEqual(message, "Body.")
def test_a_commit_of_only_co_author_lines_never_makes_an_empty_title(self):
pr = dict(number=5, title="Fix it", user=dict(login="ada"))
only = "Co-authored-by: Grace <[email protected]>"
title, message = merger.squash_message(pr, [self.commit("1", only)])
self.assertEqual(title, "Fix it (#5)")
self.assertEqual(message, only)
title, message = merger.squash_message(pr, [self.commit("1", only), self.commit("2", "Real")])
self.assertEqual(title, "Fix it (#5)")
self.assertIn("* Real", message)
self.assertNotIn("* \n", message + "\n")
def test_a_merge_sends_the_composed_message(self):
self.client.commits = [self.commit("1", "Only\n\nBody.", "Harness by Autonomous", "[email protected]", None)]
result = self.finish()
self.assertEqual(result["status"], "merged")
write = self.client.writes[0]
self.assertIn("commit_title=Only (#5)", write)
field = next(part for part in write if part.startswith("commit_message=@"))
self.assertIn("Co-authored-by: Harness by Autonomous <[email protected]>", Path(field.split("@", 1)[1]).read_text())
def test_commits_that_cannot_be_read_never_stop_a_merge(self):
self.client.commits = RuntimeError("no commits")
result = self.finish()
self.assertEqual(result["status"], "merged")
self.assertEqual(self.client.writes, [("api", "--method", "PUT", "repos/owner/repo/pulls/5/merge",
"-f", f"sha={self.head}", "-f", "merge_method=squash")])
self.assertIn("squash_message_error", result)
def test_merge_sends_exact_head_once_and_verifies_actual_tree(self):
result = self.finish()
self.assertEqual(result["status"], "merged")
self.assertTrue(result["merge"]["same_tested_tree"])
self.assertEqual(result["merge"]["commit"], self.client.merged_sha)
self.assertEqual(len(self.client.writes), 1)
self.assertEqual(self.client.writes[0][:8], ("api", "--method", "PUT", "repos/owner/repo/pulls/5/merge",
"-f", f"sha={self.head}", "-f", "merge_method=squash"))
self.assertIn("merge_request", result["phases"])
self.assertIn("verification", result["phases"])
def test_scoped_ci_reuse_verifies_the_reviewed_target_without_claiming_equal_tested_trees(self):
collect = merger.ci.collect
def reused(*args, **kwargs):
evidence = collect(*args, **kwargs)
evidence["source"].update(tested_sha=self.base, tested_tree=self.git("rev-parse", f"{self.base}^{{tree}}"),
same_tree=False, changed_files=["docs/guide.md"])
evidence["scope_reuse"] = dict(same_inputs=True, tested_sha256="a" * 64)
return evidence
self.patch(merger.ci, "collect", side_effect=reused)
result = self.finish()
self.assertEqual(result["status"], "merged")
self.assertTrue(result["merge"]["same_reviewed_tree"])
self.assertFalse(result["merge"]["same_tested_tree"])
self.assertIn("CI evidence reused", (self.output / "merge.md").read_text())
def test_wrong_pr_draft_target_or_head_stops_before_ci_or_mutation(self):
original = copy.deepcopy(self.client.pr)
for change in [dict(number=6), dict(state="closed"), dict(draft=True), dict(merged=True),
dict(head=dict(sha="b" * 40)),
dict(base=dict(ref="other", repo=dict(full_name="owner/repo"))),
dict(base=dict(ref="main", repo=dict(full_name="other/repo")))]:
self.client.pr = dict(original, **change)
with self.subTest(change=change), self.assertRaises(ValueError):
merger.preflight(self.client, self.root, 5, self.head, self.base)
self.assertEqual(self.client.writes, [])
self.assertEqual(self.client.run_reads, 0)
def test_live_main_movement_is_detected_even_when_pr_base_snapshot_is_stale(self):
self.client.actual_base = self.head
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertIn("main moved", result["error"])
self.assertEqual(self.client.writes, [])
def test_untracked_work_or_different_local_head_cannot_merge(self):
(self.root / "unreviewed.txt").write_text("change\n")
with self.assertRaisesRegex(ValueError, "local checkout"):
merger.preflight(self.client, self.root, 5, self.head, self.base)
(self.root / "unreviewed.txt").unlink()
self.git("commit", "--allow-empty", "-qm", "later")
with self.assertRaisesRegex(ValueError, "local checkout"):
merger.preflight(self.client, self.root, 5, self.head, self.base)
def test_reviewed_head_must_include_reviewed_main(self):
unrelated = self.git("commit-tree", self.tree, "-m", "unrelated")
self.base = self.client.actual_base = unrelated
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertEqual(self.client.writes, [])
def test_failed_ci_and_missing_required_jobs_block_merge(self):
self.client.run["conclusion"] = "failure"
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertEqual(self.client.writes, [])
self.output = self.output.with_name("missing-job")
self.client.run["conclusion"] = "success"
self.client.jobs = []
result = self.finish()
self.assertIn("missing jobs", result["error"])
self.assertEqual(self.client.writes, [])
def test_ci_for_a_different_tree_is_retained_but_cannot_merge(self):
self.client.run["head_sha"] = self.base
self.client.jobs[0]["head_sha"] = self.base
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertEqual(self.client.writes, [])
evidence = json.loads((self.output / "ci/receipt.json").read_text())
self.assertEqual(evidence["status"], "source_review_required")
def test_movement_after_ci_verification_is_rechecked_before_write(self):
original = merger.preflight
def moved(*args, **kwargs):
if kwargs.get("ready"):
self.client.actual_base = self.head
return original(*args, **kwargs)
self.patch(merger, "preflight", side_effect=moved)
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertIn("main moved", result["error"])
self.assertEqual(self.client.writes, [])
def test_unknown_or_blocked_github_mergeability_never_bypasses_rules(self):
for state in ["unknown", "blocked", "unstable", "dirty"]:
self.output = Path(self.tmp.name) / state
self.client.pr["mergeable_state"] = state
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertEqual(self.client.writes, [])
def test_lost_merge_response_is_resolved_by_reading_the_same_pr(self):
self.client.behavior = "timeout_after_merge"
result = self.finish()
self.assertEqual(result["status"], "merged")
self.assertIn("merge_request_error", result)
self.assertEqual(len(self.client.writes), 1)
def test_unconfirmed_request_is_retained_and_never_submitted_twice(self):
self.client.behavior = "timeout_before_merge"
result = self.finish()
self.assertEqual(result["status"], "merge_not_confirmed")
self.assertIn("inspect this same PR", result["error"])
self.assertIn("merge_requested_at", result)
self.assertEqual(len(self.client.writes), 1)
def test_different_merged_tree_requires_review_instead_of_false_success(self):
self.client.merged_tree = self.git("rev-parse", f"{self.base}^{{tree}}")
result = self.finish()
self.assertEqual(result["status"], "merged_source_review_required")
self.assertFalse(result["merge"]["same_tested_tree"])
self.assertEqual(len(self.client.writes), 1)
def test_changed_head_or_target_during_write_cannot_claim_tested_merge(self):
for behavior in ["retarget_during_merge", "head_change_during_merge", "wrong_response_sha"]:
with self.subTest(behavior=behavior):
self.output = Path(self.tmp.name) / behavior
self.client = FakeClient(self)
self.client.behavior = behavior
result = self.finish()
self.assertEqual(result["status"], "merge_not_confirmed")
self.assertEqual(len(self.client.writes), 1)
def test_bad_artifact_stops_before_merge_and_preserves_failure_receipt(self):
self.patch(merger.ci, "collect", side_effect=merger.ci.zipfile.BadZipFile("bad artifact"))
result = self.finish()
self.assertEqual(result["status"], "not_merged")
self.assertEqual(result["error"], "bad artifact")
self.assertEqual(self.client.writes, [])
class FakeClient:
repository = "owner/repo"
def __init__(self, fixture):
self.fixture, self.actual_base = fixture, fixture.base
self.deadline, self.run_reads = 0, 0
self.writes, self.behavior = [], "success"
self.merged_sha, self.merged_tree = "c" * 40, fixture.tree
self.tested_tree = fixture.tree
self.required_gate, self.run_list_reads, self.superseding_run = False, 0, None
self.rules, self.receipt_patch = [], {}
self.commits = [dict(sha=fixture.head, author=None, commit=dict(message="reviewed", author=dict(name="Fixture", email="[email protected]")))]
self.pr = dict(number=5, title="Fixture", user=dict(login="fixture"), state="open", draft=False, merged=False, mergeable=True, mergeable_state="clean",
html_url="https://github.com/owner/repo/pull/5", head=dict(sha=fixture.head),
base=dict(sha=fixture.base, ref="main", repo=dict(full_name=self.repository)))
self.run = dict(id=123, repository=dict(full_name=self.repository), path=".github/workflows/ci.yml",
event="pull_request",
head_sha=fixture.head, run_attempt=1, status="completed", conclusion="success",
html_url="https://github.com/owner/repo/actions/runs/123",
created_at="2026-10-03T00:00:00Z", updated_at="2026-10-03T00:01:00Z")
self.jobs = [dict(id=1, name="process-checks", run_id=123, run_attempt=1, head_sha=fixture.head,
status="completed", conclusion="success", steps=[],
started_at="2026-10-03T00:00:01Z", completed_at="2026-10-03T00:01:00Z")]
self.gate_jobs = [dict(self.jobs[0], name="ci/required")]
def api(self, path):
if path.startswith("pulls/5/commits"):
if isinstance(self.commits, Exception):
raise self.commits
return copy.deepcopy(self.commits) if path.endswith("page=1") else []
if path == "rules/branches/main":
return copy.deepcopy(self.rules)
if path == "pulls/5":
return copy.deepcopy(self.pr)
if path == "git/ref/heads/main":
return dict(ref="refs/heads/main", object=dict(type="commit", sha=self.actual_base))
if path == "actions/runs/123":
self.run_reads += 1
return copy.deepcopy(self.run)
if path == f"git/commits/{self.merged_sha}":
return dict(sha=self.merged_sha, tree=dict(sha=self.merged_tree), parents=[dict(sha=self.actual_base)])
if path == f"git/commits/{self.fixture.head}":
return dict(sha=self.fixture.head, tree=dict(sha=self.tested_tree))
raise AssertionError(path)
def pages(self, path, key):
if key == "workflow_runs":
self.run_list_reads += 1
return [copy.deepcopy(self.superseding_run if self.run_list_reads > 1 and self.superseding_run else self.run)]
if self.required_gate:
if key == "jobs":
return copy.deepcopy(self.gate_jobs)
if key == "artifacts":
receipt = dict(schema=1, kind="ci-required", status="passed", source_sha=self.fixture.head,
source_tree=self.tested_tree, event="pull_request", run_id=123, run_attempt=1,
suites=[], required_jobs=["process-checks"],
results={"plan": "success", "process-checks": "success"})
receipt.update(self.receipt_patch)
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as bundle:
bundle.writestr("ci-required.json", json.dumps(receipt))
self.archive = buf.getvalue()
return [dict(id=20, name="ci-required", expired=False, size_in_bytes=len(self.archive),
workflow_run=dict(id=123, head_sha=self.fixture.head),
digest="sha256:" + hashlib.sha256(self.archive).hexdigest())]
assert key in {"jobs", "artifacts"}, key
return copy.deepcopy(self.jobs) if key == "jobs" else []
def command(self, *args, binary=False):
if binary:
assert args == ("api", "repos/owner/repo/actions/artifacts/20/zip")
return self.archive
self.writes.append(args)
before = json.loads((self.fixture.output / "receipt.json").read_text())
assert before["status"] == "merge_requested" and before["reviewed_head"] == self.fixture.head
if self.behavior == "timeout_before_merge":
raise subprocess.TimeoutExpired("gh", 30)
self.pr.update(merged=True, state="closed", merge_commit_sha=self.merged_sha,
merged_at="2026-10-03T00:01:10Z")
if self.behavior == "timeout_after_merge":
raise subprocess.TimeoutExpired("gh", 30)
if self.behavior == "retarget_during_merge":
self.pr["base"]["ref"] = "other"
if self.behavior == "head_change_during_merge":
self.pr["head"]["sha"] = "b" * 40
return dict(sha="f" * 40 if self.behavior == "wrong_response_sha" else self.merged_sha, merged=True, message="merged")
if __name__ == "__main__":
unittest.main()