Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Gate prompt evolution on pinned formal evidence

Rach Pradhanfc58fe1a3233
diff --git a/examples/dgm_formal_gate.py b/examples/dgm_formal_gate.py
new file mode 100644
index 00000000..0804529b
--- /dev/null
+++ b/examples/dgm_formal_gate.py
@@ -0,0 +1,392 @@
+"""Opt-in formal baseline gate for the prompt-genome DGM example.
+
+The checked model describes shared harness lifecycle, never the candidate
+prompt. An external pin file is a trust input, not a sandbox boundary.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import secrets
+
+REPO = Path(__file__).resolve().parents[1]
+SCHEMA = "codegraff.dgm.formal-pin.v1"
+RECEIPT_SCHEMA = "codegraff.dgm.formal-receipt.v1"
+HEX = re.compile(r"[0-9a-f]{64}\Z")
+SOURCE_FILES = (
+ "src/agent_async_tools.zig", "src/agent_request.zig", "src/agent_steps.zig",
+ "src/agent_tools.zig", "src/agent_stream.zig", "src/agent.zig",
+ "src/jev_effort_state.zig", "src/jev_tool.zig", "src/effort_route.zig",
+ "src/commands_effort.zig", "src/acp_config.zig", "src/jev_model_scope.zig",
+ "src/acp_permission.zig", "src/acp_inbox.zig", "src/acp_live_turn.zig",
+ "apps/native/lib/acp-transport.ts", "apps/native/lib/acp-client.ts",
+ "apps/native/app/api/acp/route.ts", "src/http2_pool.zig",
+ "src/http2_buffered.zig", "src/agent_stream_h2.zig", "src/http_client.zig",
+ "examples/dgm_loop.py", "examples/dgm_formal_gate.py", "sdk/py/harness_sdk.py",
+ "scripts/check-formal.py", "examples/replay_judge.py",
+ "examples/dgm_eval_set.jsonl",
+)
+
+
+class GateError(RuntimeError):
+ pass
+
+
+def digest(path):
+ h = hashlib.sha256()
+ with Path(path).open("rb") as stream:
+ for part in iter(lambda: stream.read(1024 * 1024), b""):
+ h.update(part)
+ return h.hexdigest()
+
+
+def canonical(value):
+ return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()
+
+
+def private_write(path, data):
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ with os.fdopen(fd, "wb") as stream:
+ stream.write(data)
+
+
+def external(path, repo=REPO):
+ given = Path(path).expanduser()
+ if not given.is_absolute():
+ raise GateError("formal pin path must be absolute")
+ resolved = given.resolve()
+ if resolved == repo or repo in resolved.parents:
+ raise GateError("formal pin must be outside the candidate source tree")
+ return resolved
+
+
+def route(value):
+ if not isinstance(value, str) or not value or any(ord(ch) < 33 or ord(ch) > 126 for ch in value):
+ raise GateError("model selector must be a nonempty printable token")
+ return value
+
+
+def create_pin(destination, binary, java, jar, main_model, replay_model, *, repo=REPO):
+ """Explicitly copy the verifier bundle and write an external hash pin."""
+ repo = Path(repo).resolve()
+ destination = external(destination, repo)
+ binary, java, jar = (Path(p).expanduser().resolve(strict=True) for p in (binary, java, jar))
+ if destination.exists():
+ raise GateError("pin destination already exists")
+ for path in (binary, java, jar):
+ if not path.is_file():
+ raise GateError("a pinned executable or archive is missing")
+ bundle_files = [Path("scripts/check-formal.py"), Path("examples/replay_judge.py"),
+ Path("examples/dgm_eval_set.jsonl")]
+ bundle_files += [p.relative_to(repo) for p in sorted((repo / "formal").glob("*.tla"))]
+ bundle_files += [p.relative_to(repo) for p in sorted((repo / "formal").glob("*.cfg"))]
+ if not any(p.name == "AsyncTools.tla" for p in bundle_files) or not any(
+ p.name == "EffortSelection.tla" for p in bundle_files):
+ raise GateError("required formal models are missing")
+ destination.mkdir(mode=0o700, parents=True)
+ bundle = destination / "bundle"
+ bundle.mkdir(mode=0o700)
+ for rel in bundle_files:
+ target = bundle / rel
+ target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+ shutil.copyfile(repo / rel, target)
+ target.chmod(0o600)
+ manifest = {
+ "schema": SCHEMA, "source_root": str(repo), "bundle_root": str(bundle),
+ "source_files": {str(rel): digest(repo / rel) for rel in
+ sorted(set(map(Path, SOURCE_FILES)) | set(bundle_files))},
+ "bundle_files": {str(rel): digest(bundle / rel) for rel in bundle_files},
+ "binary": {"path": str(binary), "sha256": digest(binary)},
+ "java": {"path": str(java), "sha256": digest(java)},
+ "jar": {"path": str(jar), "sha256": digest(jar)},
+ "heldout_hash": digest(bundle / "examples/dgm_eval_set.jsonl"),
+ "main_model": route(main_model), "replay_model": route(replay_model),
+ }
+ pin = destination / "pin.json"
+ private_write(pin, canonical(manifest))
+ (destination / "receipts").mkdir(mode=0o700)
+ return pin
+
+
+class FormalGate:
+ def __init__(self, pin_path, *, repo=REPO, checker=None):
+ self.repo = Path(repo).resolve()
+ self.pin_path = external(pin_path, self.repo)
+ try:
+ raw = self.pin_path.read_bytes()
+ self.pin = json.loads(raw)
+ except (OSError, ValueError) as error:
+ raise GateError("formal pin is missing or malformed") from error
+ if self.pin.get("schema") != SCHEMA or self.pin.get("source_root") != str(self.repo):
+ raise GateError("formal pin does not match this source tree")
+ self.pin_sha256 = hashlib.sha256(raw).hexdigest()
+ self.bundle = external(self.pin["bundle_root"], self.repo)
+ self.receipts = self.pin_path.parent / "receipts"
+ self.checker = checker or self._run_checker
+ self.checked_identity = None
+ self.check_output_sha256 = None
+ self.check_log_path = None
+ self.key_path = os.environ.get("GRAFF_SCORE_KEY_FILE")
+ if not self.key_path:
+ raise GateError("opt-in formal gate requires a usable score-signing key")
+ try:
+ self.key_sha256 = digest(self.key_path)
+ except OSError as error:
+ raise GateError("opt-in formal gate requires a usable score-signing key") from error
+ self.verify_identity()
+
+ def verify_identity(self):
+ try:
+ if digest(self.pin_path) != self.pin_sha256:
+ raise GateError("formal pin changed")
+ if (not Path(self.key_path).is_file() or not Path(self.key_path).read_bytes().strip()
+ or digest(self.key_path) != self.key_sha256):
+ raise GateError("opt-in formal gate requires a usable score-signing key")
+ route(self.pin["main_model"])
+ route(self.pin["replay_model"])
+ source = self.pin["source_files"]
+ bundle = self.pin["bundle_files"]
+ if not set(SOURCE_FILES) <= set(source) or not bundle or not self.receipts.is_dir():
+ raise GateError("formal pin inventory is incomplete")
+ formal_source = {p.relative_to(self.repo).as_posix() for suffix in ("*.tla", "*.cfg")
+ for p in (self.repo / "formal").glob(suffix)}
+ if formal_source != {name for name in source if name.startswith("formal/")}:
+ raise GateError("formal source file set changed")
+ for relative, expected in source.items():
+ if not HEX.fullmatch(expected) or digest(self.repo / relative) != expected:
+ raise GateError("pinned source changed: " + relative)
+ for relative, expected in bundle.items():
+ if not HEX.fullmatch(expected) or digest(self.bundle / relative) != expected:
+ raise GateError("pinned formal bundle changed: " + relative)
+ for name in ("binary", "java", "jar"):
+ entry = self.pin[name]
+ if not HEX.fullmatch(entry["sha256"]) or digest(entry["path"]) != entry["sha256"]:
+ raise GateError("pinned " + name + " changed")
+ if self.pin["heldout_hash"] != bundle["examples/dgm_eval_set.jsonl"]:
+ raise GateError("held-out set pin mismatch")
+ identity = hashlib.sha256(canonical(self.pin)).hexdigest()
+ if self.checked_identity is not None and identity != self.checked_identity:
+ raise GateError("formal baseline identity drifted")
+ return identity
+ except (KeyError, OSError, TypeError, ValueError) as error:
+ if isinstance(error, GateError):
+ raise
+ raise GateError("formal baseline pin could not be verified") from error
+
+ def _run_checker(self):
+ env = {key: os.environ[key] for key in ("PATH", "LANG", "LC_ALL") if key in os.environ}
+ env.update(JAVA=self.pin["java"]["path"], TLA2TOOLS=self.pin["jar"]["path"],
+ PYTHONDONTWRITEBYTECODE="1")
+ try:
+ result = subprocess.run([sys.executable, str(self.bundle / "scripts/check-formal.py")],
+ cwd=self.bundle, env=env, capture_output=True, timeout=900)
+ evidence = result.stdout + result.stderr
+ except subprocess.TimeoutExpired as error:
+ evidence = (error.stdout or b"") + (error.stderr or b"")
+ log = self.receipts / ("check-" + secrets.token_hex(8) + ".log")
+ private_write(log, evidence[:65536] + b"\n[checker timed out; output bounded]\n")
+ raise GateError("formal model check timed out; see private checker log") from error
+ log = self.receipts / ("check-" + secrets.token_hex(8) + ".log")
+ private_write(log, evidence[:65536] + (b"\n[output truncated]\n" if len(evidence) > 65536 else b""))
+ self.check_log_path = log
+ if result.returncode != 0:
+ raise GateError("formal model check failed; see private checker log")
+ return hashlib.sha256(evidence).hexdigest()
+
+ def ensure_checked(self):
+ identity = self.verify_identity()
+ if self.checked_identity is None:
+ try:
+ output_sha = self.checker()
+ except (OSError, subprocess.TimeoutExpired) as error:
+ raise GateError("formal model check unavailable") from error
+ if not isinstance(output_sha, str) or not HEX.fullmatch(output_sha):
+ raise GateError("formal checker returned no valid receipt")
+ if self.verify_identity() != identity:
+ raise GateError("formal baseline changed during model check")
+ self.checked_identity = identity
+ self.check_output_sha256 = output_sha
+ return identity
+
+ @property
+ def binary(self):
+ return self.pin["binary"]["path"]
+
+ @property
+ def main_model(self):
+ return self.pin["main_model"]
+
+ def replay_env(self):
+ self.ensure_checked()
+ return {"GRAFF_HARNESS_BIN": self.binary,
+ "GRAFF_REPLAY_JUDGE": str(self.bundle / "examples/replay_judge.py"),
+ "GRAFF_EVAL_SET_FILE": str(self.bundle / "examples/dgm_eval_set.jsonl"),
+ "GRAFF_EVAL_SET_HASH": self.pin["heldout_hash"],
+ "GRAFF_EVAL_MODEL": self.pin["replay_model"]}
+
+ def prepare_candidate(self, prompt):
+ self.ensure_checked()
+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()
+ payload = {"schema": RECEIPT_SCHEMA, "phase": "before_evaluation",
+ "candidate_prompt_sha256": prompt_sha,
+ "formal_identity_sha256": self.checked_identity,
+ "checker_output_sha256": self.check_output_sha256,
+ "checker_log_file": self.check_log_path.name if self.check_log_path else None,
+ "checker_log_sha256": digest(self.check_log_path) if self.check_log_path else None,
+ "binary_sha256": self.pin["binary"]["sha256"]}
+ path = self.receipts / ("candidate-" + prompt_sha + "-" + secrets.token_hex(8) + ".json")
+ data = canonical(payload)
+ private_write(path, data)
+ return {"path": path, "sha256": hashlib.sha256(data).hexdigest(),
+ "prompt_sha256": prompt_sha}
+
+ def finish_candidate(self, prepared, prompt, report, heldout_hash):
+ self.ensure_checked() # cheap rehash; never silently rerun TLC on drift
+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()
+ if prompt_sha != prepared["prompt_sha256"] or digest(prepared["path"]) != prepared["sha256"]:
+ raise GateError("candidate prompt or preflight receipt changed")
+ if heldout_hash and heldout_hash != self.pin["heldout_hash"]:
+ raise GateError("held-out replay used a different evaluation set")
+ report_sha = hashlib.sha256(report.encode()).hexdigest()
+ if len(prompt.encode()) > 1048576 or len(report.encode()) > 1048576:
+ raise GateError("candidate evidence exceeds private receipt bound")
+ payload = {"schema": RECEIPT_SCHEMA, "phase": "before_score",
+ "preflight_sha256": prepared["sha256"], "candidate_prompt_sha256": prompt_sha,
+ "preflight_file": prepared["path"].name,
+ "candidate_prompt": prompt, "report": report,
+ "report_sha256": report_sha, "heldout_eval_set_hash": heldout_hash,
+ "formal_identity_sha256": self.checked_identity,
+ "checker_output_sha256": self.check_output_sha256,
+ "checker_log_file": self.check_log_path.name if self.check_log_path else None,
+ "checker_log_sha256": digest(self.check_log_path) if self.check_log_path else None,
+ "binary_sha256": self.pin["binary"]["sha256"],
+ "main_model": self.main_model, "replay_model": self.pin["replay_model"]}
+ data = canonical(payload)
+ if len(data) > 2 * 1048576 + 4096:
+ raise GateError("serialized private receipt exceeds bound")
+ artifact = self.artifact_digest(data, report_sha, prompt_sha)
+ path = self.receipts / ("score-" + artifact + ".json")
+ private_write(path, data)
+ return artifact
+
+ @staticmethod
+ def artifact_digest(data, report_sha, prompt_sha):
+ return hashlib.sha256(b"dgm-formal-artifact-v1\0" + hashlib.sha256(data).digest() +
+ bytes.fromhex(report_sha) + bytes.fromhex(prompt_sha)).hexdigest()
+
+ def verify_score_receipt(self, row, prompt=None):
+ """Validate the private evidence named by a signed formal score row."""
+ self.ensure_checked()
+ artifact = row.get("artifact_sha", "")
+ if row.get("judge_id") != "replay-v1+formal-v1" or not HEX.fullmatch(artifact):
+ raise GateError("formal score has no valid artifact reference")
+ path = self.receipts / ("score-" + artifact + ".json")
+ try:
+ with path.open("rb") as stream:
+ data = stream.read(2 * 1048576 + 4097)
+ if len(data) > 2 * 1048576 + 4096:
+ raise GateError("formal score receipt exceeds bound")
+ receipt = json.loads(data)
+ if data != canonical(receipt):
+ raise GateError("formal score receipt is not canonical")
+ candidate = receipt["candidate_prompt"]
+ report = receipt["report"]
+ prompt_sha = hashlib.sha256(candidate.encode()).hexdigest()
+ report_sha = hashlib.sha256(report.encode()).hexdigest()
+ if (receipt["schema"] != RECEIPT_SCHEMA or receipt["phase"] != "before_score"
+ or receipt["candidate_prompt_sha256"] != prompt_sha
+ or row.get("prompt_sha") != prompt_sha[:16]
+ or receipt["report_sha256"] != report_sha
+ or self.artifact_digest(data, report_sha, prompt_sha) != artifact
+ or receipt["formal_identity_sha256"] != self.checked_identity
+ or not HEX.fullmatch(receipt["checker_output_sha256"])
+ or receipt["binary_sha256"] != self.pin["binary"]["sha256"]
+ or receipt["main_model"] != self.main_model
+ or receipt["replay_model"] != self.pin["replay_model"]
+ or receipt["heldout_eval_set_hash"] != row.get("eval_set_hash", "")
+ or (row.get("eval_set_hash") and row["eval_set_hash"] != self.pin["heldout_hash"])
+ or (prompt is not None and candidate != prompt)):
+ raise GateError("formal score receipt does not match its score row")
+ preflight_name = receipt["preflight_file"]
+ if not re.fullmatch(r"candidate-[0-9a-f]{64}-[0-9a-f]{16}\.json", preflight_name):
+ raise GateError("formal preflight name mismatch")
+ preflight = self.receipts / preflight_name
+ if (preflight.parent != self.receipts
+ or digest(preflight) != receipt["preflight_sha256"]):
+ raise GateError("formal preflight receipt mismatch")
+ before = json.loads(preflight.read_bytes())
+ if (before.get("candidate_prompt_sha256") != prompt_sha or
+ before.get("formal_identity_sha256") != self.checked_identity or
+ before.get("checker_output_sha256") != receipt["checker_output_sha256"] or
+ before.get("checker_log_file") != receipt["checker_log_file"] or
+ before.get("checker_log_sha256") != receipt["checker_log_sha256"]):
+ raise GateError("formal preflight identity mismatch")
+ log_name, log_sha = receipt["checker_log_file"], receipt["checker_log_sha256"]
+ if log_name is not None or log_sha is not None:
+ if (not isinstance(log_name, str) or
+ not re.fullmatch(r"check-[0-9a-f]{16}\.log", log_name) or
+ not isinstance(log_sha, str) or not HEX.fullmatch(log_sha) or
+ digest(self.receipts / log_name) != log_sha):
+ raise GateError("archived checker log changed")
+ return receipt
+ except (OSError, ValueError, KeyError, TypeError) as error:
+ raise GateError("formal score receipt unavailable or malformed") from error
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Pin or verify the optional DGM formal gate")
+ action = parser.add_mutually_exclusive_group(required=True)
+ action.add_argument("--pin", help="new external directory; must not exist")
+ action.add_argument("--check", help="existing external pin.json to verify")
+ action.add_argument("--verify-row", help="JSON score row whose formal receipt must verify")
+ parser.add_argument("--pin-file", help="external pin.json for --verify-row")
+ parser.add_argument("--candidate-sha256", help="full prompt SHA-256 for --check")
+ parser.add_argument("--binary")
+ parser.add_argument("--java")
+ parser.add_argument("--jar")
+ parser.add_argument("--main-model")
+ parser.add_argument("--replay-model")
+ args = parser.parse_args()
+ if args.pin:
+ if not all((args.binary, args.java, args.jar, args.main_model, args.replay_model)):
+ parser.error("--pin requires --binary, --java, --jar, and both model selectors")
+ print(create_pin(args.pin, args.binary, args.java, args.jar,
+ args.main_model, args.replay_model))
+ elif args.check:
+ if not args.candidate_sha256 or not HEX.fullmatch(args.candidate_sha256):
+ parser.error("--check requires --candidate-sha256")
+ gate = FormalGate(args.check)
+ gate.ensure_checked()
+ print(json.dumps({"schema": "codegraff.dgm.formal-check.v1", "ok": True,
+ "candidate_prompt_sha256": args.candidate_sha256,
+ "formal_identity_sha256": gate.checked_identity,
+ "checker_output_sha256": gate.check_output_sha256,
+ "binary_sha256": gate.pin["binary"]["sha256"]}, sort_keys=True))
+ else:
+ if not args.pin_file:
+ parser.error("--verify-row requires --pin-file")
+ sys.path.insert(0, str(REPO / "sdk/py"))
+ from harness_sdk import verify_score
+ gate = FormalGate(args.pin_file)
+ with Path(args.verify_row).open("rb") as stream:
+ raw = stream.read(1048577)
+ if len(raw) > 1048576:
+ raise GateError("score row exceeds verifier bound")
+ row = json.loads(raw)
+ if not verify_score(Path(gate.key_path).read_bytes().strip(), row):
+ raise GateError("score row signature invalid")
+ gate.verify_score_receipt(row)
+ print(json.dumps({"schema": "codegraff.dgm.formal-score-check.v1", "ok": True,
+ "artifact_sha": row["artifact_sha"],
+ "formal_identity_sha256": gate.checked_identity}, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/dgm_loop.py b/examples/dgm_loop.py
index 9a354fba..947c227b 100644
--- a/examples/dgm_loop.py
+++ b/examples/dgm_loop.py
@@ -24,6 +24,10 @@ printed GRAFF_EVAL_SET_FILE / GRAFF_EVAL_SET_HASH / GRAFF_REPLAY_JUDGE.
Usage:
python3 examples/dgm_loop.py "summarize src/main.zig's Telemetry struct" 3
+
+Optional shared-harness formal gate (prompt correctness still comes from the
+held-out replay judge): create an external pin with dgm_formal_gate.py --pin,
+then set GRAFF_DGM_FORMAL_PIN and GRAFF_SCORE_KEY_FILE before this command.
"""
import collections
import hashlib
@@ -36,9 +40,11 @@ import sys
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "sdk", "py"))
from harness_sdk import Harness, prompt_fingerprint, verify_score # noqa: E402
+from dgm_formal_gate import FormalGate # noqa: E402
ARCHIVE_DIR = os.path.join(".graff", "trajectories")
LEGACY_ARCHIVE = "harness.trajectory.jsonl"
+_replay_env = None
def archive_paths():
@@ -75,7 +81,7 @@ SEED = ("You are a focused agent. Do exactly what the task asks, verify your "
"result before reporting, and report it plainly with evidence.")
-def load_archive():
+def load_archive(gate=None):
"""genomes: sha → prompt text; scores: sha → [values]; children: sha → n.
When GRAFF_SCORE_KEY_FILE is set, score rows must carry a valid HMAC or
@@ -95,6 +101,12 @@ def load_archive():
if key is not None and not verify_score(key, o):
rejected += 1
continue
+ if gate is not None and o.get("judge_id") == "replay-v1+formal-v1":
+ try:
+ gate.verify_score_receipt(o, genomes.get(o.get("prompt_sha")))
+ except Exception:
+ rejected += 1
+ continue
scores[o["prompt_sha"]].append(o["score"])
p = o.get("parent_sha")
if p and (p, o["prompt_sha"]) not in seen_edges:
@@ -127,7 +139,7 @@ def replay_eval(child_prompt):
the editable tree; falls back to the in-tree copy with a warning).
Returns the judge's JSON verdict, or None on ANY failure — the caller
must treat None as fail-closed, not as a free pass."""
- judge_path = os.environ.get("GRAFF_REPLAY_JUDGE")
+ judge_path = (_replay_env or os.environ).get("GRAFF_REPLAY_JUDGE")
if not judge_path:
judge_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"replay_judge.py")
@@ -136,7 +148,7 @@ def replay_eval(child_prompt):
try:
r = subprocess.run([sys.executable, judge_path],
input=child_prompt, text=True,
- stdout=subprocess.PIPE, timeout=1800)
+ stdout=subprocess.PIPE, timeout=1800, env=_replay_env)
if r.returncode != 0:
return None
return json.loads(r.stdout)
@@ -184,38 +196,57 @@ def judge(h, task, report, child):
f"efficiency {efficiency:.3f}, ${res.get('total_cost', 0):.4f}")
return 0.9 + 0.1 * efficiency, res["eval_set_hash"]
def main():
+ global _replay_env
+ _replay_env = None
if len(sys.argv) < 2:
sys.exit(__doc__)
task = sys.argv[1]
iters = int(sys.argv[2]) if len(sys.argv) > 2 else 3
- h = Harness(yolo=True)
- for gen in range(iters):
- genomes, scores, children = load_archive()
- parent_sha = select_parent(genomes, scores, children)
- parent = genomes.get(parent_sha, SEED)
- # Modify: the meta step — the harness's own model mutates the genome.
- child = h.ask(
- "Rewrite this agent system prompt to perform better on tasks "
- f"like: {task}\nChange ONE meaningful behavior. Keep it under "
- f"120 words. Reply with ONLY the new prompt.\n---\n{parent}")
- # Spawn the child as a prompt variant; its run (incl. tool sequence)
- # lands in the archive automatically.
- report = h.ask(
- f'Call the subagent tool exactly once: description "dgm gen {gen}", '
- f'prompt {json.dumps(task)}, system_prompt {json.dumps(child)}. '
- "Then reply with the subagent report verbatim.")
- s, eval_hash = judge(h, task, report, child)
- # Evaluate write-back with the lineage edge + signed provenance:
- # who judged (judge_id), what artifact (report sha), on which pinned
- # eval set — all folded into the record's HMAC (Step 0).
- h.score(child, s, notes=f"gen {gen}", parent=parent_sha or SEED,
- judge_id="replay-v1",
- artifact_sha=hashlib.sha256(report.encode()).hexdigest()[:16],
- eval_set_hash=eval_hash)
- print(f"gen {gen}: parent={parent_sha or 'seed'} "
- f"child={prompt_fingerprint(child)} score={s:.2f}"
- f"{' [PASSING]' if s >= 0.9 else ' [failing]'}")
- h.close()
+ gate = FormalGate(os.environ["GRAFF_DGM_FORMAL_PIN"]) if "GRAFF_DGM_FORMAL_PIN" in os.environ else None
+ child_env = None
+ if gate is not None:
+ gate.ensure_checked() # before the first model call
+ child_env = dict(os.environ)
+ child_env.update(gate.replay_env())
+ child_env.pop("GRAFF_DGM_FORMAL_PIN", None)
+ _replay_env = child_env
+ h = Harness(yolo=True, binary=gate.binary if gate else None, env=child_env,
+ model=gate.main_model if gate else None)
+ try:
+ for gen in range(iters):
+ if gate:
+ gate.ensure_checked() # stop drift before a new mutation call
+ genomes, scores, children = load_archive(gate)
+ parent_sha = select_parent(genomes, scores, children)
+ parent = genomes.get(parent_sha, SEED)
+ # The genome is a prompt, not a change to harness source or model.
+ child = h.ask(
+ "Rewrite this agent system prompt to perform better on tasks "
+ f"like: {task}\nChange ONE meaningful behavior. Keep it under "
+ f"120 words. Reply with ONLY the new prompt.\n---\n{parent}")
+ prepared = gate.prepare_candidate(child) if gate else None
+ # The optional gate runs before the costly child and replay eval.
+ report = h.ask(
+ f'Call the subagent tool exactly once: description "dgm gen {gen}", '
+ f'prompt {json.dumps(task)}, system_prompt {json.dumps(child)}. '
+ "Then reply with the subagent report verbatim.")
+ s, eval_hash = judge(h, task, report, child)
+ if gate:
+ if s >= 0.9 and not eval_hash:
+ raise RuntimeError("promotable score has no pinned replay receipt")
+ artifact_sha = gate.finish_candidate(prepared, child, report, eval_hash)
+ else:
+ artifact_sha = hashlib.sha256(report.encode()).hexdigest()[:16]
+ # The signed eval_set_hash stays the original held-out digest;
+ # the opt-in signed artifact digest binds formal-check evidence.
+ h.score(child, s, notes=f"gen {gen}", parent=parent_sha or SEED,
+ judge_id="replay-v1+formal-v1" if gate else "replay-v1",
+ artifact_sha=artifact_sha, eval_set_hash=eval_hash)
+ print(f"gen {gen}: parent={parent_sha or 'seed'} "
+ f"child={prompt_fingerprint(child)} score={s:.2f}"
+ f"{' [PASSING]' if s >= 0.9 else ' [failing]'}")
+ finally:
+ h.close()
print(f"archive: {ARCHIVE_DIR} — rerun to keep evolving; /trajectory to inspect")
diff --git a/examples/test_dgm_formal_gate.py b/examples/test_dgm_formal_gate.py
new file mode 100644
index 00000000..00fe1b46
--- /dev/null
+++ b/examples/test_dgm_formal_gate.py
@@ -0,0 +1,234 @@
+"""Offline controls for the optional Python DGM formal baseline gate."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import dgm_formal_gate as formal
+import dgm_loop as loop
+from harness_sdk import score_signature
+
+
+class FormalGateTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ for name in ("graff", "java", "tla.jar"):
+ (self.root / name).write_bytes(name.encode())
+ self.key = self.root / "score.key"
+ self.key.write_bytes(b"private-test-key")
+ self.env = patch.dict(os.environ, {"GRAFF_SCORE_KEY_FILE": str(self.key)})
+ self.env.start()
+ self.addCleanup(self.env.stop)
+ self.pin = formal.create_pin(self.root / "pin", self.root / "graff",
+ self.root / "java", self.root / "tla.jar",
+ "codex/primary", "codex/replay")
+ self.calls = 0
+
+ def checker(self):
+ self.calls += 1
+ return "a" * 64
+
+ def test_receipt_resolves_from_artifact_and_tlc_runs_once(self):
+ gate = formal.FormalGate(self.pin, checker=self.checker)
+ def recorded_check():
+ output = self.checker()
+ gate.check_log_path = gate.receipts / "check-0000000000000000.log"
+ formal.private_write(gate.check_log_path, b"finite model checks passed\n")
+ return output
+ gate.checker = recorded_check
+ prompt = "One changed behavior"
+ for report in ("first child report", "second child report"):
+ before = gate.prepare_candidate(prompt)
+ artifact = gate.finish_candidate(before, prompt, report, gate.pin["heldout_hash"])
+ row = {"judge_id": "replay-v1+formal-v1", "artifact_sha": artifact,
+ "prompt_sha": hashlib.sha256(prompt.encode()).hexdigest()[:16],
+ "eval_set_hash": gate.pin["heldout_hash"]}
+ receipt = gate.verify_score_receipt(row, prompt)
+ self.assertEqual(receipt["report"], report)
+ self.assertEqual((gate.receipts / ("score-" + artifact + ".json")).stat().st_mode & 0o777, 0o600)
+ self.assertEqual(self.calls, 1)
+ self.assertEqual(gate.replay_env()["GRAFF_EVAL_MODEL"], "codex/replay")
+ self.assertEqual(gate.main_model, "codex/primary")
+ # A later TLC invocation may print different runtime statistics.
+ later = formal.FormalGate(self.pin, checker=lambda: "b" * 64)
+ later.ensure_checked()
+ self.assertEqual(later.verify_score_receipt(row, prompt)["report"], "second child report")
+ gate.check_log_path.write_bytes(b"changed model-check log\n")
+ with self.assertRaises(formal.GateError):
+ later.verify_score_receipt(row, prompt)
+
+ def test_tamper_and_missing_receipt_fail_closed(self):
+ gate = formal.FormalGate(self.pin, checker=self.checker)
+ prompt = "A prompt"
+ before = gate.prepare_candidate(prompt)
+ artifact = gate.finish_candidate(before, prompt, "report", gate.pin["heldout_hash"])
+ row = {"judge_id": "replay-v1+formal-v1", "artifact_sha": artifact,
+ "prompt_sha": hashlib.sha256(prompt.encode()).hexdigest()[:16],
+ "eval_set_hash": gate.pin["heldout_hash"]}
+ with self.assertRaises(formal.GateError):
+ gate.verify_score_receipt(dict(row, eval_set_hash="b" * 64))
+ with self.assertRaises(formal.GateError):
+ gate.verify_score_receipt(dict(row, artifact_sha="0" * 64))
+ path = gate.receipts / ("score-" + artifact + ".json")
+ path.write_bytes(path.read_bytes().replace(b"report", b"forged"))
+ with self.assertRaises(formal.GateError):
+ gate.verify_score_receipt(row)
+
+ def test_key_and_source_identity_drift_fail_closed(self):
+ gate = formal.FormalGate(self.pin, checker=self.checker)
+ gate.ensure_checked()
+ self.key.write_bytes(b"rotated-key")
+ with self.assertRaises(formal.GateError):
+ gate.ensure_checked()
+ self.key.write_bytes(b"private-test-key")
+ (self.pin.parent / "bundle" / "formal" / "AsyncTools.tla").write_text("tampered")
+ with self.assertRaises(formal.GateError):
+ gate.ensure_checked()
+ self.assertEqual(self.calls, 1)
+
+ def test_external_pin_and_model_validation(self):
+ with self.assertRaises(formal.GateError):
+ formal.create_pin(formal.REPO / "not-external", self.root / "graff",
+ self.root / "java", self.root / "tla.jar", "a", "b")
+ with self.assertRaises(formal.GateError):
+ formal.route(" ")
+
+ def test_signed_formal_archive_row_requires_matching_receipt(self):
+ gate = formal.FormalGate(self.pin, checker=self.checker)
+ prompt = "archived candidate"
+ prompt_sha = hashlib.sha256(prompt.encode()).hexdigest()[:16]
+ before = gate.prepare_candidate(prompt)
+ artifact = gate.finish_candidate(before, prompt, "private report", gate.pin["heldout_hash"])
+ row = {"kind": "score", "prompt_sha": prompt_sha, "parent_sha": "",
+ "score": 0.9, "score_run_id": "run", "judge_id": "replay-v1+formal-v1",
+ "artifact_sha": artifact, "eval_set_hash": gate.pin["heldout_hash"]}
+ row["sig"] = score_signature(self.key.read_bytes(), prompt_sha, "", 0.9,
+ "run", row["judge_id"], artifact, row["eval_set_hash"])
+ archive = self.root / "archive.jsonl"
+ archive.write_text(json.dumps({"kind": "prompt", "prompt_sha": prompt_sha,
+ "text": prompt}) + "\n" + json.dumps(row) + "\n")
+ with patch.object(loop, "archive_paths", return_value=[str(archive)]):
+ genomes, scores, _ = loop.load_archive(gate)
+ self.assertEqual(scores[prompt_sha], [0.9])
+ (gate.receipts / ("score-" + artifact + ".json")).unlink()
+ _, scores, _ = loop.load_archive(gate)
+ self.assertNotIn(prompt_sha, scores)
+
+
+class LoopGateTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ for name in ("graff", "java", "tla.jar"):
+ (self.root / name).write_bytes(name.encode())
+ self.key = self.root / "score.key"
+ self.key.write_bytes(b"private-test-key")
+ self.env = patch.dict(os.environ, {"GRAFF_SCORE_KEY_FILE": str(self.key)})
+ self.env.start()
+ self.addCleanup(self.env.stop)
+ self.pin = formal.create_pin(self.root / "pin", self.root / "graff",
+ self.root / "java", self.root / "tla.jar",
+ "codex/primary", "codex/replay")
+
+ class FakeHarness:
+ instances = []
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+ self.calls = []
+ self.scores = []
+ self.closed = False
+ self.instances.append(self)
+ def ask(self, prompt):
+ self.calls.append(prompt)
+ return "candidate prompt" if len(self.calls) == 1 else "child report"
+ def score(self, *args, **kwargs):
+ self.scores.append((args, kwargs))
+ def close(self):
+ self.closed = True
+
+ def test_gate_failure_precedes_paid_model_launch(self):
+ self.FakeHarness.instances.clear()
+ class RejectGate:
+ def __init__(self, path):
+ pass
+ def ensure_checked(self):
+ raise formal.GateError("offline checker rejected")
+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": "/unused"}), \
+ patch.object(loop, "FormalGate", RejectGate), \
+ patch.object(loop, "Harness", self.FakeHarness), \
+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):
+ with self.assertRaises(formal.GateError):
+ loop.main()
+ self.assertEqual(self.FakeHarness.instances, [])
+
+ def test_default_flow_unchanged_without_gate(self):
+ self.FakeHarness.instances.clear()
+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": ""}), \
+ patch.object(loop, "Harness", self.FakeHarness), \
+ patch.object(loop, "load_archive", return_value=({}, {}, {})), \
+ patch.object(loop, "judge", return_value=(0.8, "heldout")), \
+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):
+ del os.environ["GRAFF_DGM_FORMAL_PIN"]
+ loop.main()
+ harness = self.FakeHarness.instances[0]
+ self.assertEqual(len(harness.calls), 2)
+ self.assertEqual(len(harness.scores), 1)
+ self.assertEqual(harness.scores[0][1]["judge_id"], "replay-v1")
+ self.assertTrue(harness.closed)
+
+ def test_opt_in_loop_records_evidence_and_pinned_routes(self):
+ self.FakeHarness.instances.clear()
+ gate = formal.FormalGate(self.pin, checker=lambda: "a" * 64)
+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": str(self.pin)}), \
+ patch.object(loop, "FormalGate", lambda _: gate), \
+ patch.object(loop, "Harness", self.FakeHarness), \
+ patch.object(loop, "load_archive", return_value=({}, {}, {})), \
+ patch.object(loop, "judge", return_value=(0.9, gate.pin["heldout_hash"])), \
+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):
+ loop.main()
+ harness = self.FakeHarness.instances[0]
+ self.assertEqual(harness.kwargs["binary"], str((self.root / "graff").resolve()))
+ self.assertEqual(harness.kwargs["model"], "codex/primary")
+ self.assertEqual(harness.kwargs["env"]["GRAFF_EVAL_MODEL"], "codex/replay")
+ self.assertEqual(len(harness.scores), 1)
+ score = harness.scores[0][1]
+ self.assertEqual(score["judge_id"], "replay-v1+formal-v1")
+ self.assertEqual(score["eval_set_hash"], gate.pin["heldout_hash"])
+ self.assertTrue((gate.receipts / ("score-" + score["artifact_sha"] + ".json")).is_file())
+ self.assertTrue(harness.closed)
+
+ def test_drift_during_candidate_execution_prevents_score(self):
+ self.FakeHarness.instances.clear()
+ gate = formal.FormalGate(self.pin, checker=lambda: "a" * 64)
+ target = self.pin.parent / "bundle" / "formal" / "AsyncTools.tla"
+ class MutatingHarness(self.FakeHarness):
+ def ask(self, prompt):
+ result = super().ask(prompt)
+ if len(self.calls) == 2:
+ target.write_text("changed during child report")
+ return result
+ with patch.dict(os.environ, {"GRAFF_DGM_FORMAL_PIN": str(self.pin)}), \
+ patch.object(loop, "FormalGate", lambda _: gate), \
+ patch.object(loop, "Harness", MutatingHarness), \
+ patch.object(loop, "load_archive", return_value=({}, {}, {})), \
+ patch.object(loop, "judge", return_value=(0.9, gate.pin["heldout_hash"])), \
+ patch.object(sys, "argv", ["dgm_loop.py", "task", "1"]):
+ with self.assertRaises(formal.GateError):
+ loop.main()
+ harness = self.FakeHarness.instances[0]
+ self.assertEqual(harness.scores, [])
+ self.assertTrue(harness.closed)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/formal/dgm-integration.md b/formal/dgm-integration.md
index d4cbd717..3fef1ede 100644
--- a/formal/dgm-integration.md
+++ b/formal/dgm-integration.md
@@ -46,6 +46,46 @@ identity. Preserve the original replay suite hash and bind the formal receipt
into score provenance. Retain formal failure separately from replay failure,
unknown measurement and an ordinary low efficiency score.
+## Python example opt-in
+
+Create a new verifier directory outside the candidate checkout using
+`examples/dgm_formal_gate.py --pin`, supplying the harness executable, Java,
+TLC archive, and explicit mutation and replay model selectors. Set
+`GRAFF_DGM_FORMAL_PIN` to the resulting pin file and `GRAFF_SCORE_KEY_FILE`
+to the existing signing key before running `examples/dgm_loop.py`. The helper
+checks the pinned formal bundle once per unchanged identity. Normal mode remains
+unchanged when no formal pin is configured.
+
+Mutation and replay roles may intentionally use different model selectors;
+record both rather than silently conflating them. Selectors and binary hashes
+are not proof of the provider route actually served during inference. Historical
+scores remain possible parents, but do not become formally checked ancestry.
+Only new evaluations admitted through formal mode carry its evidence.
+
+```sh
+python3 examples/dgm_formal_gate.py --pin "$PIN_DIR" \
+ --binary "$HARNESS_BINARY" --java "$JAVA" --jar "$TLA2TOOLS" \
+ --main-model "$MAIN_MODEL" --replay-model "$REPLAY_MODEL"
+GRAFF_DGM_FORMAL_PIN="$PIN_DIR/pin.json" \
+ GRAFF_SCORE_KEY_FILE="$SCORE_KEY_FILE" \
+ python3 examples/dgm_loop.py "Your evaluation task" 3
+```
+
+The pin directory must be new and outside the source checkout. Set the model
+selectors explicitly for the mutation and replay roles. Receipts and bounded
+checker logs stay private. To audit an archived signed score, set the same
+signing-key environment variable and run:
+
+```sh
+python3 examples/dgm_formal_gate.py --verify-row "$SCORE_ROW_JSON" \
+ --pin-file "$PIN_DIR/pin.json"
+```
+
+Formal score rows resolve to `score-<artifact digest>.json` receipts. Validation
+checks the signature, prompt/report hashes, original held-out hash, preflight
+receipt, pinned identity and archived checker evidence. It also runs the current
+pinned model checks; their raw output need not match an earlier successful log.
+
## Other native promotion paths
The repository also has native learning and persona promotion paths. Integrating