graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Gate prompt evolution on pinned formal evidence
Rach Pradhanfc58fe1a3233
diff --git a/examples/dgm_formal_gate.py b/examples/dgm_formal_gate.pynew file mode 100644index 00000000..0804529b--- /dev/null+++ b/examples/dgm_formal_gate.py@@ -0,0 +1,392 @@+"""Opt-in formal baseline gate for the prompt-genome DGM example.++The checked model describes shared harness lifecycle, never the candidate+prompt. An external pin file is a trust input, not a sandbox boundary.+"""+from __future__ import annotations++import argparse+import hashlib+import json+import os+from pathlib import Path+import re+import shutil+import subprocess+import sys+import secrets++REPO = Path(__file__).resolve().parents[1]+SCHEMA = "codegraff.dgm.formal-pin.v1"+RECEIPT_SCHEMA = "codegraff.dgm.formal-receipt.v1"+HEX = re.compile(r"[0-9a-f]{64}\Z")+SOURCE_FILES = (+ "src/agent_async_tools.zig", "src/agent_request.zig", "src/agent_steps.zig",+ "src/agent_tools.zig", "src/agent_stream.zig", "src/agent.zig",+ "src/jev_effort_state.zig", "src/jev_tool.zig", "src/effort_route.zig",+ "src/commands_effort.zig", "src/acp_config.zig", "src/jev_model_scope.zig",+ "src/acp_permission.zig", "src/acp_inbox.zig", "src/acp_live_turn.zig",+ "apps/native/lib/acp-transport.ts", "apps/native/lib/acp-client.ts",+ "apps/native/app/api/acp/route.ts", "src/http2_pool.zig",+ "src/http2_buffered.zig", "src/agent_stream_h2.zig", "src/http_client.zig",+ "examples/dgm_loop.py", "examples/dgm_formal_gate.py", "sdk/py/harness_sdk.py",+ "scripts/check-formal.py", "examples/replay_judge.py",+ "examples/dgm_eval_set.jsonl",+)+++class GateError(RuntimeError):+ pass+++def digest(path):+ h = hashlib.sha256()+ with Path(path).open("rb") as stream:+ for part in iter(lambda: stream.read(1024 * 1024), b""):+ h.update(part)+ return h.hexdigest()+++def canonical(value):+ return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()+++def private_write(path, data):+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)+ with os.fdopen(fd, "wb") as stream:+ stream.write(data)+++def external(path, repo=REPO):+ given = Path(path).expanduser()+ if not given.is_absolute():+ raise GateError("formal pin path must be absolute")+ resolved = given.resolve()+ if resolved == repo or repo in resolved.parents:+ raise GateError("formal pin must be outside the candidate source tree")+ return resolved+++def route(value):+ if not isinstance(value, str) or not value or any(ord(ch) < 33 or ord(ch) > 126 for ch in value):+ raise GateError("model selector must be a nonempty printable token")+ return value+++def create_pin(destination, binary, java, jar, main_model, replay_model, *, repo=REPO):+ """Explicitly copy the verifier bundle and write an external hash pin."""+ repo = Path(repo).resolve()+ destination = external(destination, repo)+ binary, java, jar = (Path(p).expanduser().resolve(strict=True) for p in (binary, java, jar))+ if destination.exists():+ raise GateError("pin destination already exists")+ for path in (binary, java, jar):+ if not path.is_file():+ raise GateError("a pinned executable or archive is missing")+ bundle_files = [Path("scripts/check-formal.py"), Path("examples/replay_judge.py"),+ Path("examples/dgm_eval_set.jsonl")]+ bundle_files += [p.relative_to(repo) for p in sorted((repo / "formal").glob("*.tla"))]+ bundle_files += [p.relative_to(repo) for p in sorted((repo / "formal").glob("*.cfg"))]+ if not any(p.name == "AsyncTools.tla" for p in bundle_files) or not any(+ p.name == "EffortSelection.tla" for p in bundle_files):+ raise GateError("required formal models are missing")+ destination.mkdir(mode=0o700, parents=True)+ bundle = destination / "bundle"+ bundle.mkdir(mode=0o700)+ for rel in bundle_files:+ target = bundle / rel+ target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)+ shutil.copyfile(repo / rel, target)+ target.chmod(0o600)+ manifest = {+ "schema": SCHEMA, "source_root": str(repo), "bundle_root": str(bundle),+ "source_files": {str(rel): digest(repo / rel) for rel in+ sorted(set(map(Path, SOURCE_FILES)) | set(bundle_files))},+ "bundle_files": {str(rel): digest(bundle / rel) for rel in bundle_files},+ "binary": {"path": str(binary), "sha256": digest(binary)},+ "java": {"path": str(java), "sha256": digest(java)},+ "jar": {"path": str(jar), "sha256": digest(jar)},+ "heldout_hash": digest(bundle / "examples/dgm_eval_set.jsonl"),+ "main_model": route(main_model), "replay_model": route(replay_model),+ }+ pin = destination / "pin.json"+ private_write(pin, canonical(manifest))+ (destination / "receipts").mkdir(mode=0o700)+ return pin+++class FormalGate:+ def __init__(self, pin_path, *, repo=REPO, checker=None):+ self.repo = Path(repo).resolve()+ self.pin_path = external(pin_path, self.repo)+ try:+ raw = self.pin_path.read_bytes()+ self.pin = json.loads(raw)+ except (OSError, ValueError) as error:+ raise GateError("formal pin is missing or malformed") from error+ if self.pin.get("schema") != SCHEMA or self.pin.get("source_root") != str(self.repo):+ raise GateError("formal pin does not match this source tree")+ self.pin_sha256 = hashlib.sha256(raw).hexdigest()+ self.bundle = external(self.pin["bundle_root"], self.repo)+ self.receipts = self.pin_path.parent / "receipts"+ self.checker = checker or self._run_checker+ self.checked_identity = None+ self.check_output_sha256 = None+ self.check_log_path = None+ self.key_path = os.environ.get("GRAFF_SCORE_KEY_FILE")+ if not self.key_path:+ raise GateError("opt-in formal gate requires a usable score-signing key")+ try:+ self.key_sha256 = digest(self.key_path)+ except OSError as error:+ raise GateError("opt-in formal gate requires a usable score-signing key") from error+ self.verify_identity()++ def verify_identity(self):+ try:+ if digest(self.pin_path) != self.pin_sha256:+ raise GateError("formal pin changed")+ if (not Path(self.key_path).is_file() or not Path(self.key_path).read_bytes().strip()+ or digest(self.key_path) != self.key_sha256):+ raise GateError("opt-in formal gate requires a usable score-signing key")+ route(self.pin["main_model"])+ route(self.pin["replay_model"])+ source = self.pin["source_files"]+ bundle = self.pin["bundle_files"]+ if not set(SOURCE_FILES) <= set(source) or not bundle or not self.receipts.is_dir():+ raise GateError("formal pin inventory is incomplete")+ formal_source = {p.relative_to(self.repo).as_posix() for suffix in ("*.tla", "*.cfg")+ for p in (self.repo / "formal").glob(suffix)}+ if formal_source != {name for name in source if name.startswith("formal/")}:+ raise GateError("formal source file set changed")+ for relative, expected in source.items():+ if not HEX.fullmatch(expected) or digest(self.repo / relative) != expected:+ raise GateError("pinned source changed: " + relative)+ for relative, expected in bundle.items():+ if not HEX.fullmatch(expected) or digest(self.bundle / relative) != expected:+ raise GateError("pinned formal bundle changed: " + relative)+ for name in ("binary", "java", "jar"):+ entry = self.pin[name]+ if not HEX.fullmatch(entry["sha256"]) or digest(entry["path"]) != entry["sha256"]:+ raise GateError("pinned " + name + " changed")+ if self.pin["heldout_hash"] != bundle["examples/dgm_eval_set.jsonl"]:+ raise GateError("held-out set pin mismatch")+ identity = hashlib.sha256(canonical(self.pin)).hexdigest()+ if self.checked_identity is not None and identity != self.checked_identity:+ raise GateError("formal baseline identity drifted")+ return identity+ except (KeyError, OSError, TypeError, ValueError) as error:+ if isinstance(error, GateError):+ raise+ raise GateError("formal baseline pin could not be verified") from error++ def _run_checker(self):+ env = {key: os.environ[key] for key in ("PATH", "LANG", "LC_ALL") if key in os.environ}+ env.update(JAVA=self.pin["java"]["path"], TLA2TOOLS=self.pin["jar"]["path"],+ PYTHONDONTWRITEBYTECODE="1")+ try:+ result = subprocess.run([sys.executable, str(self.bundle / "scripts/check-formal.py")],+ cwd=self.bundle, env=env, capture_output=True, timeout=900)+ evidence = result.stdout + result.stderr+ except subprocess.TimeoutExpired as error:+ evidence = (error.stdout or b"") + (error.stderr or b"")+ log = self.receipts / ("check-" + secrets.token_hex(8) + ".log")+ private_write(log, evidence[:65536] + b"\n[checker timed out; output bounded]\n")+ raise GateError("formal model check timed out; see private checker log") from error+ log = self.receipts / ("check-" + secrets.token_hex(8) + ".log")+ private_write(log, evidence[:65536] + (b"\n[output truncated]\n" if len(evidence) > 65536 else b""))+ self.check_log_path = log+ if result.returncode != 0:+ raise GateError("formal model check failed; see private checker log")+ return hashlib.sha256(evidence).hexdigest()++ def ensure_checked(self):+ identity = self.verify_identity()+ if self.checked_identity is None:+ try:+ output_sha = self.checker()+ except (OSError, subprocess.TimeoutExpired) as error:+ raise GateError("formal model check unavailable") from error+ if not isinstance(output_sha, str) or not HEX.fullmatch(output_sha):+ raise GateError("formal checker returned no valid receipt")+ if self.verify_identity() != identity:+ raise GateError("formal baseline changed during model check")+ self.checked_identity = identity+ self.check_output_sha256 = output_sha+ return identity++ @property+ def binary(self):+ return self.pin["binary"]["path"]++ @property+ def main_model(self):+ return self.pin["main_model"]++ def replay_env(self):+ self.ensure_checked()+ return {"GRAFF_HARNESS_BIN": self.binary,+ "GRAFF_REPLAY_JUDGE": str(self.bundle / "examples/replay_judge.py"),+ "GRAFF_EVAL_SET_FILE": str(self.bundle / "examples/dgm_eval_set.jsonl"),+ "GRAFF_EVAL_SET_HASH": self.pin["heldout_hash"],+ "GRAFF_EVAL_MODEL": self.pin["replay_model"]}++ def prepare_candidate(self, prompt):+ self.ensure_checked()+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()+ payload = {"schema": RECEIPT_SCHEMA, "phase": "before_evaluation",+ "candidate_prompt_sha256": prompt_sha,+ "formal_identity_sha256": self.checked_identity,+ "checker_output_sha256": self.check_output_sha256,+ "checker_log_file": self.check_log_path.name if self.check_log_path else None,+ "checker_log_sha256": digest(self.check_log_path) if self.check_log_path else None,+ "binary_sha256": self.pin["binary"]["sha256"]}+ path = self.receipts / ("candidate-" + prompt_sha + "-" + secrets.token_hex(8) + ".json")+ data = canonical(payload)+ private_write(path, data)+ return {"path": path, "sha256": hashlib.sha256(data).hexdigest(),+ "prompt_sha256": prompt_sha}++ def finish_candidate(self, prepared, prompt, report, heldout_hash):+ self.ensure_checked() # cheap rehash; never silently rerun TLC on drift+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()+ if prompt_sha != prepared["prompt_sha256"] or digest(prepared["path"]) != prepared["sha256"]:+ raise GateError("candidate prompt or preflight receipt changed")+ if heldout_hash and heldout_hash != self.pin["heldout_hash"]:+ raise GateError("held-out replay used a different evaluation set")+ report_sha = hashlib.sha256(report.encode()).hexdigest()+ if len(prompt.encode()) > 1048576 or len(report.encode()) > 1048576:+ raise GateError("candidate evidence exceeds private receipt bound")+ payload = {"schema": RECEIPT_SCHEMA, "phase": "before_score",+ "preflight_sha256": prepared["sha256"], "candidate_prompt_sha256": prompt_sha,+ "preflight_file": prepared["path"].name,+ "candidate_prompt": prompt, "report": report,+ "report_sha256": report_sha, "heldout_eval_set_hash": heldout_hash,+ "formal_identity_sha256": self.checked_identity,+ "checker_output_sha256": self.check_output_sha256,+ "checker_log_file": self.check_log_path.name if self.check_log_path else None,+ "checker_log_sha256": digest(self.check_log_path) if self.check_log_path else None,+ "binary_sha256": self.pin["binary"]["sha256"],+ "main_model": self.main_model, "replay_model": self.pin["replay_model"]}+ data = canonical(payload)+ if len(data) > 2 * 1048576 + 4096:+ raise GateError("serialized private receipt exceeds bound")+ artifact = self.artifact_digest(data, report_sha, prompt_sha)+ path = self.receipts / ("score-" + artifact + ".json")+ private_write(path, data)+ return artifact++ @staticmethod+ def artifact_digest(data, report_sha, prompt_sha):+ return hashlib.sha256(b"dgm-formal-artifact-v1\0" + hashlib.sha256(data).digest() ++ bytes.fromhex(report_sha) + bytes.fromhex(prompt_sha)).hexdigest()++ def verify_score_receipt(self, row, prompt=None):+ """Validate the private evidence named by a signed formal score row."""+ self.ensure_checked()+ artifact = row.get("artifact_sha", "")+ if row.get("judge_id") != "replay-v1+formal-v1" or not HEX.fullmatch(artifact):+ raise GateError("formal score has no valid artifact reference")+ path = self.receipts / ("score-" + artifact + ".json")+ try:+ with path.open("rb") as stream:+ data = stream.read(2 * 1048576 + 4097)+ if len(data) > 2 * 1048576 + 4096:+ raise GateError("formal score receipt exceeds bound")+ receipt = json.loads(data)+ if data != canonical(receipt):+ raise GateError("formal score receipt is not canonical")+ candidate = receipt["candidate_prompt"]+ report = receipt["report"]+ prompt_sha = hashlib.sha256(candidate.encode()).hexdigest()+ report_sha = hashlib.sha256(report.encode()).hexdigest()+ if (receipt["schema"] != RECEIPT_SCHEMA or receipt["phase"] != "before_score"+ or receipt["candidate_prompt_sha256"] != prompt_sha+ or row.get("prompt_sha") != prompt_sha[:16]+ or receipt["report_sha256"] != report_sha+ or self.artifact_digest(data, report_sha, prompt_sha) != artifact+ or receipt["formal_identity_sha256"] != self.checked_identity+ or not HEX.fullmatch(receipt["checker_output_sha256"])+ or receipt["binary_sha256"] != self.pin["binary"]["sha256"]+ or receipt["main_model"] != self.main_model+ or receipt["replay_model"] != self.pin["replay_model"]+ or receipt["heldout_eval_set_hash"] != row.get("eval_set_hash", "")+ or (row.get("eval_set_hash") and row["eval_set_hash"] != self.pin["heldout_hash"])+ or (prompt is not None and candidate != prompt)):+ raise GateError("formal score receipt does not match its score row")+ preflight_name = receipt["preflight_file"]+ if not re.fullmatch(r"candidate-[0-9a-f]{64}-[0-9a-f]{16}\.json", preflight_name):+ raise GateError("formal preflight name mismatch")+ preflight = self.receipts / preflight_name+ if (preflight.parent != self.receipts+ or digest(preflight) != receipt["preflight_sha256"]):+ raise GateError("formal preflight receipt mismatch")+ before = json.loads(preflight.read_bytes())+ if (before.get("candidate_prompt_sha256") != prompt_sha or+ before.get("formal_identity_sha256") != self.checked_identity or+ before.get("checker_output_sha256") != receipt["checker_output_sha256"] or+ before.get("checker_log_file") != receipt["checker_log_file"] or+ before.get("checker_log_sha256") != receipt["checker_log_sha256"]):+ raise GateError("formal preflight identity mismatch")+ log_name, log_sha = receipt["checker_log_file"], receipt["checker_log_sha256"]+ if log_name is not None or log_sha is not None:+ if (not isinstance(log_name, str) or+ not re.fullmatch(r"check-[0-9a-f]{16}\.log", log_name) or+ not isinstance(log_sha, str) or not HEX.fullmatch(log_sha) or+ digest(self.receipts / log_name) != log_sha):+ raise GateError("archived checker log changed")+ return receipt+ except (OSError, ValueError, KeyError, TypeError) as error:+ raise GateError("formal score receipt unavailable or malformed") from error+++def main():+ parser = argparse.ArgumentParser(description="Pin or verify the optional DGM formal gate")+ action = parser.add_mutually_exclusive_group(required=True)+ action.add_argument("--pin", help="new external directory; must not exist")+ action.add_argument("--check", help="existing external pin.json to verify")+ action.add_argument("--verify-row", help="JSON score row whose formal receipt must verify")+ parser.add_argument("--pin-file", help="external pin.json for --verify-row")+ parser.add_argument("--candidate-sha256", help="full prompt SHA-256 for --check")+ parser.add_argument("--binary")+ parser.add_argument("--java")+ parser.add_argument("--jar")+ parser.add_argument("--main-model")+ parser.add_argument("--replay-model")+ args = parser.parse_args()+ if args.pin:+ if not all((args.binary, args.java, args.jar, args.main_model, args.replay_model)):+ parser.error("--pin requires --binary, --java, --jar, and both model selectors")+ print(create_pin(args.pin, args.binary, args.java, args.jar,+ args.main_model, args.replay_model))+ elif args.check:+ if not args.candidate_sha256 or not HEX.fullmatch(args.candidate_sha256):+ parser.error("--check requires --candidate-sha256")+ gate = FormalGate(args.check)+ gate.ensure_checked()+ print(json.dumps({"schema": "codegraff.dgm.formal-check.v1", "ok": True,+ "candidate_prompt_sha256": args.candidate_sha256,+ "formal_identity_sha256": gate.checked_identity,+ "checker_output_sha256": gate.check_output_sha256,+ "binary_sha256": gate.pin["binary"]["sha256"]}, sort_keys=True))+ else:+ if not args.pin_file:+ parser.error("--verify-row requires --pin-file")+ sys.path.insert(0, str(REPO / "sdk/py"))+ from harness_sdk import verify_score+ gate = FormalGate(args.pin_file)+ with Path(args.verify_row).open("rb") as stream:+ raw = stream.read(1048577)+ if len(raw) > 1048576:+ raise GateError("score row exceeds verifier bound")+ row = json.loads(raw)+ if not verify_score(Path(gate.key_path).read_bytes().strip(), row):+ raise GateError("score row signature invalid")+ gate.verify_score_receipt(row)+ print(json.dumps({"schema": "codegraff.dgm.formal-score-check.v1", "ok": True,+ "artifact_sha": row["artifact_sha"],+ "formal_identity_sha256": gate.checked_identity}, sort_keys=True))+++if __name__ == "__main__":+ main()diff --git a/examples/dgm_loop.py b/examples/dgm_loop.pyindex 9a354fba..947c227b 100644--- a/examples/dgm_loop.py+++ b/examples/dgm_loop.py@@ -24,6 +24,10 @@ printed GRAFF_EVAL_SET_FILE / GRAFF_EVAL_SET_HASH / GRAFF_REPLAY_JUDGE.Usage:python3 examples/dgm_loop.py "summarize src/main.zig's Telemetry struct" 3++Optional shared-harness formal gate (prompt correctness still comes from the+held-out replay judge): create an external pin with dgm_formal_gate.py --pin,+then set GRAFF_DGM_FORMAL_PIN and GRAFF_SCORE_KEY_FILE before this command."""import collectionsimport hashlib@@ -36,9 +40,11 @@ import syssys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "sdk", "py"))from harness_sdk import Harness, prompt_fingerprint, verify_score # noqa: E402+from dgm_formal_gate import FormalGate # noqa: E402ARCHIVE_DIR = os.path.join(".graff", "trajectories")LEGACY_ARCHIVE = "harness.trajectory.jsonl"+_replay_env = Nonedef archive_paths():@@ -75,7 +81,7 @@ SEED = ("You are a focused agent. Do exactly what the task asks, verify your ""result before reporting, and report it plainly with evidence.")-def load_archive():+def load_archive(gate=None):"""genomes: sha → prompt text; scores: sha → [values]; children: sha → n.When GRAFF_SCORE_KEY_FILE is set, score rows must carry a valid HMAC or@@ -95,6 +101,12 @@ def load_archive():if key is not None and not verify_score(key, o):rejected += 1continue+ if gate is not None and o.get("judge_id") == "replay-v1+formal-v1":+ try:+ gate.verify_score_receipt(o, genomes.get(o.get("prompt_sha")))+ except Exception:+ rejected += 1+ continuescores[o["prompt_sha"]].append(o["score"])p = o.get("parent_sha")if p and (p, o["prompt_sha"]) not in seen_edges:@@ -127,7 +139,7 @@ def replay_eval(child_prompt):the editable tree; falls back to the in-tree copy with a warning).Returns the judge's JSON verdict, or None on ANY failure — the callermust treat None as fail-closed, not as a free pass."""- judge_path = os.environ.get("GRAFF_REPLAY_JUDGE")+ judge_path = (_replay_env or os.environ).get("GRAFF_REPLAY_JUDGE")if not judge_path:judge_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"replay_judge.py")@@ -136,7 +148,7 @@ def replay_eval(child_prompt):try:r = subprocess.run([sys.executable, judge_path],input=child_prompt, text=True,- stdout=subprocess.PIPE, timeout=1800)+ stdout=subprocess.PIPE, timeout=1800, env=_replay_env)if r.returncode != 0:return Nonereturn json.loads(r.stdout)@@ -184,38 +196,57 @@ def judge(h, task, report, child):f"efficiency {efficiency:.3f}, ${res.get('total_cost', 0):.4f}")return 0.9 + 0.1 * efficiency, res["eval_set_hash"]def main():+ global _replay_env+ _replay_env = Noneif len(sys.argv) < 2:sys.exit(__doc__)task = sys.argv[1]iters = int(sys.argv[2]) if len(sys.argv) > 2 else 3- h = Harness(yolo=True)- for gen in range(iters):- genomes, scores, children = load_archive()- parent_sha = select_parent(genomes, scores, children)- parent = genomes.get(parent_sha, SEED)- # Modify: the meta step — the harness's own model mutates the genome.- child = h.ask(- "Rewrite this agent system prompt to perform better on tasks "- f"like: {task}\nChange ONE meaningful behavior. Keep it under "- f"120 words. Reply with ONLY the new prompt.\n---\n{parent}")- # Spawn the child as a prompt variant; its run (incl. tool sequence)- # lands in the archive automatically.- report = h.ask(- f'Call the subagent tool exactly once: description "dgm gen {gen}", '- f'prompt {json.dumps(task)}, system_prompt {json.dumps(child)}. '- "Then reply with the subagent report verbatim.")- s, eval_hash = judge(h, task, report, child)- # Evaluate write-back with the lineage edge + signed provenance:- # who judged (judge_id), what artifact (report sha), on which pinned- # eval set — all folded into the record's HMAC (Step 0).- h.score(child, s, notes=f"gen {gen}", parent=parent_sha or SEED,- judge_id="replay-v1",- artifact_sha=hashlib.sha256(report.encode()).hexdigest()[:16],- eval_set_hash=eval_hash)- print(f"gen {gen}: parent={parent_sha or 'seed'} "- f"child={prompt_fingerprint(child)} score={s:.2f}"- f"{' [PASSING]' if s >= 0.9 else ' [failing]'}")- h.close()+ gate = FormalGate(os.environ["GRAFF_DGM_FORMAL_PIN"]) if "GRAFF_DGM_FORMAL_PIN" in os.environ else None+ child_env = None+ if gate is not None:+ gate.ensure_checked() # before the first model call+ child_env = dict(os.environ)+ child_env.update(gate.replay_env())+ child_env.pop("GRAFF_DGM_FORMAL_PIN", None)+ _replay_env = child_env+ h = Harness(yolo=True, binary=gate.binary if gate else None, env=child_env,+ model=gate.main_model if gate else None)+ try:+ for gen in range(iters):+ if gate:+ gate.ensure_checked() # stop drift before a new mutation call+ genomes, scores, children = load_archive(gate)+ parent_sha = select_parent(genomes, scores, children)+ parent = genomes.get(parent_sha, SEED)+ # The genome is a prompt, not a change to harness source or model.+ child = h.ask(+ "Rewrite this agent system prompt to perform better on tasks "+ f"like: {task}\nChange ONE meaningful behavior. Keep it under "+ f"120 words. Reply with ONLY the new prompt.\n---\n{parent}")+ prepared = gate.prepare_candidate(child) if gate else None+ # The optional gate runs before the costly child and replay eval.+ report = h.ask(+ f'Call the subagent tool exactly once: description "dgm gen {gen}", '+ f'prompt {json.dumps(task)}, system_prompt {json.dumps(child)}. '+ "Then reply with the subagent report verbatim.")+ s, eval_hash = judge(h, task, report, child)+ if gate:+ if s >= 0.9 and not eval_hash:+ raise RuntimeError("promotable score has no pinned replay receipt")+ artifact_sha = gate.finish_candidate(prepared, child, report, eval_hash)+ else:+ artifact_sha = hashlib.sha256(report.encode()).hexdigest()[:16]+ # The signed eval_set_hash stays the original held-out digest;+ # the opt-in signed artifact digest binds formal-check evidence.+ h.score(child, s, notes=f"gen {gen}", parent=parent_sha or SEED,+ judge_id="replay-v1+formal-v1" if gate else "replay-v1",+ artifact_sha=artifact_sha, eval_set_hash=eval_hash)+ print(f"gen {gen}: parent={parent_sha or 'seed'} "+ f"child={prompt_fingerprint(child)} score={s:.2f}"+ f"{' [PASSING]' if s >= 0.9 else ' [failing]'}")+ finally:+ h.close()print(f"archive: {ARCHIVE_DIR} — rerun to keep evolving; /trajectory to inspect")diff --git a/examples/test_dgm_formal_gate.py b/examples/test_dgm_formal_gate.pynew file mode 100644index 00000000..00fe1b46--- /dev/null+++ b/examples/test_dgm_formal_gate.py@@ -0,0 +1,234 @@+"""Offline controls for the optional Python DGM formal baseline gate."""+from __future__ import annotations++import hashlib+import json+import os+from pathlib import Path+import sys+import tempfile+import unittest+from unittest.mock import patch++sys.path.insert(0, str(Path(__file__).resolve().parent))+import dgm_formal_gate as formal+import dgm_loop as loop+from harness_sdk import score_signature+++class FormalGateTests(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.root = Path(self.temp.name)+ for name in ("graff", "java", "tla.jar"):+ (self.root / name).write_bytes(name.encode())+ self.key = self.root / "score.key"+ self.key.write_bytes(b"private-test-key")+ self.env = patch.dict(os.environ, {"GRAFF_SCORE_KEY_FILE": str(self.key)})+ self.env.start()+ self.addCleanup(self.env.stop)+ self.pin = formal.create_pin(self.root / "pin", self.root / "graff",+ self.root / "java", self.root / "tla.jar",+ "codex/primary", "codex/replay")+ self.calls = 0++ def checker(self):+ self.calls += 1+ return "a" * 64++ def test_receipt_resolves_from_artifact_and_tlc_runs_once(self):+ gate = formal.FormalGate(self.pin, checker=self.checker)+ def recorded_check():+ output = self.checker()+ gate.check_log_path = gate.receipts / "check-0000000000000000.log"+ formal.private_write(gate.check_log_path, b"finite model checks passed\n")+ return output+ gate.checker = recorded_check+ prompt = "One changed behavior"+ for report in ("first child report", "second child report"):+ before = gate.prepare_candidate(prompt)+ artifact = gate.finish_candidate(before, prompt, report, gate.pin["heldout_hash"])+ row = {"judge_id": "replay-v1+formal-v1", "artifact_sha": artifact,+ "prompt_sha": hashlib.sha256(prompt.encode()).hexdigest()[:16],+ "eval_set_hash": gate.pin["heldout_hash"]}+ receipt = gate.verify_score_receipt(row, prompt)+ self.assertEqual(receipt["report"], report)+ self.assertEqual((gate.receipts / ("score-" + artifact + ".json")).stat().st_mode & 0o777, 0o600)+ self.assertEqual(self.calls, 1)+ self.assertEqual(gate.replay_env()["GRAFF_EVAL_MODEL"], "codex/replay")+ self.assertEqual(gate.main_model, "codex/primary")+ # A later TLC invocation may print different runtime statistics.+ later = formal.FormalGate(self.pin, checker=lambda: "b" * 64)+ later.ensure_checked()+ self.assertEqual(later.verify_score_receipt(row, prompt)["report"], "second child report")+ gate.check_log_path.write_bytes(b"changed model-check log\n")+ with self.assertRaises(formal.GateError):+ later.verify_score_receipt(row, prompt)++ def test_tamper_and_missing_receipt_fail_closed(self):+ gate = formal.FormalGate(self.pin, checker=self.checker)+ prompt = "A prompt"+ before = gate.prepare_candidate(prompt)+ artifact = gate.finish_candidate(before, prompt, "report", gate.pin["heldout_hash"])+ row = {"judge_id": "replay-v1+formal-v1", "artifact_sha": artifact,+ "prompt_sha": hashlib.sha256(prompt.encode()).hexdigest()[:16],+ "eval_set_hash": gate.pin["heldout_hash"]}+ with self.assertRaises(formal.GateError):+ gate.verify_score_receipt(dict(row, eval_set_hash="b" * 64))+ with self.assertRaises(formal.GateError):+ gate.verify_score_receipt(dict(row, artifact_sha="0" * 64))+ path = gate.receipts / ("score-" + artifact + ".json")+ path.write_bytes(path.read_bytes().replace(b"report", b"forged"))+ with self.assertRaises(formal.GateError):+ gate.verify_score_receipt(row)++ def test_key_and_source_identity_drift_fail_closed(self):+ gate = formal.FormalGate(self.pin, checker=self.checker)+ gate.ensure_checked()+ self.key.write_bytes(b"rotated-key")+ with self.assertRaises(formal.GateError):+ gate.ensure_checked()+ self.key.write_bytes(b"private-test-key")+ (self.pin.parent / "bundle" / "formal" / "AsyncTools.tla").write_text("tampered")+ with self.assertRaises(formal.GateError):+ gate.ensure_checked()+ self.assertEqual(self.calls, 1)++ def test_external_pin_and_model_validation(self):+ with self.assertRaises(formal.GateError):+ formal.create_pin(formal.REPO / "not-external", self.root / "graff",+ self.root / "java", self.root / "tla.jar", "a", "b")+ with self.assertRaises(formal.GateError):+ formal.route(" ")++ def test_signed_formal_archive_row_requires_matching_receipt(self):+ gate = formal.FormalGate(self.pin, checker=self.checker)+ prompt = "archived candidate"+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()[:16]+ before = gate.prepare_candidate(prompt)+ artifact = gate.finish_candidate(before, prompt, "private report", gate.pin["heldout_hash"])+ row = {"kind": "score", "prompt_sha": prompt_sha, "parent_sha": "",+ "score": 0.9, "score_run_id": "run", "judge_id": "replay-v1+formal-v1",+ "artifact_sha": artifact, "eval_set_hash": gate.pin["heldout_hash"]}+ row["sig"] = score_signature(self.key.read_bytes(), prompt_sha, "", 0.9,+ "run", row["judge_id"], artifact, row["eval_set_hash"])+ archive = self.root / "archive.jsonl"+ archive.write_text(json.dumps({"kind": "prompt", "prompt_sha": prompt_sha,+ "text": prompt}) + "\n" + json.dumps(row) + "\n")+ with patch.object(loop, "archive_paths", return_value=[str(archive)]):+ genomes, scores, _ = loop.load_archive(gate)+ self.assertEqual(scores[prompt_sha], [0.9])+ (gate.receipts / ("score-" + artifact + ".json")).unlink()+ _, scores, _ = loop.load_archive(gate)+ self.assertNotIn(prompt_sha, scores)+++class LoopGateTests(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.root = Path(self.temp.name)+ for name in ("graff", "java", "tla.jar"):+ (self.root / name).write_bytes(name.encode())+ self.key = self.root / "score.key"+ self.key.write_bytes(b"private-test-key")+ self.env = patch.dict(os.environ, {"GRAFF_SCORE_KEY_FILE": str(self.key)})+ self.env.start()+ self.addCleanup(self.env.stop)+ self.pin = formal.create_pin(self.root / "pin", self.root / "graff",+ self.root / "java", self.root / "tla.jar",+ "codex/primary", "codex/replay")++ class FakeHarness:+ instances = []+ def __init__(self, **kwargs):+ self.kwargs = kwargs+ self.calls = []+ self.scores = []+ self.closed = False+ self.instances.append(self)+ def ask(self, prompt):+ self.calls.append(prompt)+ return "candidate prompt" if len(self.calls) == 1 else "child report"+ def score(self, *args, **kwargs):+ self.scores.append((args, kwargs))+ def close(self):+ self.closed = True++ def test_gate_failure_precedes_paid_model_launch(self):+ self.FakeHarness.instances.clear()+ class RejectGate:+ def __init__(self, path):+ pass+ def ensure_checked(self):+ raise formal.GateError("offline checker rejected")+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": "/unused"}), \+ patch.object(loop, "FormalGate", RejectGate), \+ patch.object(loop, "Harness", self.FakeHarness), \+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):+ with self.assertRaises(formal.GateError):+ loop.main()+ self.assertEqual(self.FakeHarness.instances, [])++ def test_default_flow_unchanged_without_gate(self):+ self.FakeHarness.instances.clear()+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": ""}), \+ patch.object(loop, "Harness", self.FakeHarness), \+ patch.object(loop, "load_archive", return_value=({}, {}, {})), \+ patch.object(loop, "judge", return_value=(0.8, "heldout")), \+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):+ del os.environ["GRAFF_DGM_FORMAL_PIN"]+ loop.main()+ harness = self.FakeHarness.instances[0]+ self.assertEqual(len(harness.calls), 2)+ self.assertEqual(len(harness.scores), 1)+ self.assertEqual(harness.scores[0][1]["judge_id"], "replay-v1")+ self.assertTrue(harness.closed)++ def test_opt_in_loop_records_evidence_and_pinned_routes(self):+ self.FakeHarness.instances.clear()+ gate = formal.FormalGate(self.pin, checker=lambda: "a" * 64)+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": str(self.pin)}), \+ patch.object(loop, "FormalGate", lambda _: gate), \+ patch.object(loop, "Harness", self.FakeHarness), \+ patch.object(loop, "load_archive", return_value=({}, {}, {})), \+ patch.object(loop, "judge", return_value=(0.9, gate.pin["heldout_hash"])), \+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):+ loop.main()+ harness = self.FakeHarness.instances[0]+ self.assertEqual(harness.kwargs["binary"], str((self.root / "graff").resolve()))+ self.assertEqual(harness.kwargs["model"], "codex/primary")+ self.assertEqual(harness.kwargs["env"]["GRAFF_EVAL_MODEL"], "codex/replay")+ self.assertEqual(len(harness.scores), 1)+ score = harness.scores[0][1]+ self.assertEqual(score["judge_id"], "replay-v1+formal-v1")+ self.assertEqual(score["eval_set_hash"], gate.pin["heldout_hash"])+ self.assertTrue((gate.receipts / ("score-" + score["artifact_sha"] + ".json")).is_file())+ self.assertTrue(harness.closed)++ def test_drift_during_candidate_execution_prevents_score(self):+ self.FakeHarness.instances.clear()+ gate = formal.FormalGate(self.pin, checker=lambda: "a" * 64)+ target = self.pin.parent / "bundle" / "formal" / "AsyncTools.tla"+ class MutatingHarness(self.FakeHarness):+ def ask(self, prompt):+ result = super().ask(prompt)+ if len(self.calls) == 2:+ target.write_text("changed during child report")+ return result+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": str(self.pin)}), \+ patch.object(loop, "FormalGate", lambda _: gate), \+ patch.object(loop, "Harness", MutatingHarness), \+ patch.object(loop, "load_archive", return_value=({}, {}, {})), \+ patch.object(loop, "judge", return_value=(0.9, gate.pin["heldout_hash"])), \+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):+ with self.assertRaises(formal.GateError):+ loop.main()+ harness = self.FakeHarness.instances[0]+ self.assertEqual(harness.scores, [])+ self.assertTrue(harness.closed)+++if __name__ == "__main__":+ unittest.main()diff --git a/formal/dgm-integration.md b/formal/dgm-integration.mdindex d4cbd717..3fef1ede 100644--- a/formal/dgm-integration.md+++ b/formal/dgm-integration.md@@ -46,6 +46,46 @@ identity. Preserve the original replay suite hash and bind the formal receiptinto score provenance. Retain formal failure separately from replay failure,unknown measurement and an ordinary low efficiency score.+## Python example opt-in++Create a new verifier directory outside the candidate checkout using+`examples/dgm_formal_gate.py --pin`, supplying the harness executable, Java,+TLC archive, and explicit mutation and replay model selectors. Set+`GRAFF_DGM_FORMAL_PIN` to the resulting pin file and `GRAFF_SCORE_KEY_FILE`+to the existing signing key before running `examples/dgm_loop.py`. The helper+checks the pinned formal bundle once per unchanged identity. Normal mode remains+unchanged when no formal pin is configured.++Mutation and replay roles may intentionally use different model selectors;+record both rather than silently conflating them. Selectors and binary hashes+are not proof of the provider route actually served during inference. Historical+scores remain possible parents, but do not become formally checked ancestry.+Only new evaluations admitted through formal mode carry its evidence.++```sh+python3 examples/dgm_formal_gate.py --pin "$PIN_DIR" \+ --binary "$HARNESS_BINARY" --java "$JAVA" --jar "$TLA2TOOLS" \+ --main-model "$MAIN_MODEL" --replay-model "$REPLAY_MODEL"+GRAFF_DGM_FORMAL_PIN="$PIN_DIR/pin.json" \+ GRAFF_SCORE_KEY_FILE="$SCORE_KEY_FILE" \+ python3 examples/dgm_loop.py "Your evaluation task" 3+```++The pin directory must be new and outside the source checkout. Set the model+selectors explicitly for the mutation and replay roles. Receipts and bounded+checker logs stay private. To audit an archived signed score, set the same+signing-key environment variable and run:++```sh+python3 examples/dgm_formal_gate.py --verify-row "$SCORE_ROW_JSON" \+ --pin-file "$PIN_DIR/pin.json"+```++Formal score rows resolve to `score-<artifact digest>.json` receipts. Validation+checks the signature, prompt/report hashes, original held-out hash, preflight+receipt, pinned identity and archived checker evidence. It also runs the current+pinned model checks; their raw output need not match an earlier successful log.+## Other native promotion pathsThe repository also has native learning and persona promotion paths. Integrating