Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

feat(evals): add offline trajectory replay and evaluation tools (#189, #190, #191)

virgiliojr949c207f870ade
diff --git a/.gitignore b/.gitignore
index de5c726..c81c624 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,6 @@ docs/BACKERS.md
# Social-card render cache (mkdocs-material[imaging]).
.cache/
+
+# Local evaluation work products
+.eval-work/
diff --git a/docs/research/progressive-disclosure-evals.md b/docs/research/progressive-disclosure-evals.md
index f024e2f..f6d253b 100644
--- a/docs/research/progressive-disclosure-evals.md
+++ b/docs/research/progressive-disclosure-evals.md
@@ -207,10 +207,10 @@ Status values:
| ID | Status | Task | Depends on | Production change? |
|---|---|---|---|---|
-| PD-00 | READY | Freeze current baseline and evaluation contract | none | no |
-| PD-01 | BLOCKED | Implement deterministic manifest + hashing + budget schema | PD-00 | no |
-| PD-02 | BLOCKED | Implement fixture/replay scorer with trajectory metrics | PD-01 | no |
-| PD-03 | BLOCKED | Implement paper-flat baseline pack builder | PD-01 | no |
+| PD-00 | DONE | Freeze current baseline and evaluation contract | none | no |
+| PD-01 | DONE | Implement deterministic manifest + hashing + budget schema | PD-00 | no |
+| PD-02 | DONE | Implement fixture/replay scorer with trajectory metrics | PD-01 | no |
+| PD-03 | DONE | Implement paper-flat baseline pack builder | PD-01 | no |
| PD-04 | BLOCKED | Add dry-run/live runner contract and first harness adapter | PD-02, PD-03 | no |
| PD-05 | BLOCKED | Representation experiment: raw payload vs structured B2S payload | PD-04 | no |
| PD-06 | BLOCKED | Automatic vs explicit activation experiment | PD-04 | no |
@@ -548,10 +548,10 @@ Agents update this table when a task changes status. Keep entries short; link to
| Task | Status | PR / commit | Evidence | Notes |
|---|---|---|---|---|
-| PD-00 | READY | — | — | First implementation task |
-| PD-01 | BLOCKED | — | — | waits for PD-00 |
-| PD-02 | BLOCKED | — | — | waits for PD-01 |
-| PD-03 | BLOCKED | — | — | waits for PD-01 |
+| PD-00 | DONE | — | — | Baseline documented |
+| PD-01 | DONE | — | `tests/evals/test_manifest.py` | Deterministic manifest, source SHA-256, budget/secret validation |
+| PD-02 | DONE | — | `tests/evals/test_score.py`, `test_replay.py` | Offline trajectory scorer and replay runner |
+| PD-03 | DONE | — | `tests/evals/test_paper_flat.py` | Synthetic paper-flat baseline builder |
| PD-04 | BLOCKED | — | — | waits for PD-02/03 |
| PD-05 | BLOCKED | — | — | primary representation experiment |
| PD-06 | BLOCKED | — | — | host capability dependent |
diff --git a/evals/fixtures/pd00-synthetic-book.txt b/evals/fixtures/pd00-synthetic-book.txt
new file mode 100644
index 0000000..c4b79f4
--- /dev/null
+++ b/evals/fixtures/pd00-synthetic-book.txt
@@ -0,0 +1,22 @@
+Synthetic Progressive Disclosure Book
+
+Sumário
+Capítulo 1 — Orientação
+Capítulo 2 — Recuperação
+Capítulo 3 — Verificação
+
+Capítulo 1 — Orientação
+A orientação apresenta o objetivo da avaliação. O leitor identifica o livro,
+o capítulo e a pergunta antes de carregar detalhes. Esta seção usa texto
+sintético, curto e determinístico para exercitar a descoberta de capítulos.
+
+Capítulo 2 — Recuperação
+A recuperação localiza a evidência necessária depois da orientação. O agente
+consulta o sumário, abre o capítulo relevante e evita carregar capítulos sem
+relação com a pergunta. Esta seção permanece sintética e não reproduz obra
+publicada.
+
+Capítulo 3 — Verificação
+A verificação confirma que a resposta usa a evidência do capítulo alvo. O
+relatório separa contagens medidas da estimativa do loop de descoberta. Esta
+seção é o alvo determinístico para a linha de base PD-00.
diff --git a/evals/fixtures/pd03-book.txt b/evals/fixtures/pd03-book.txt
new file mode 100644
index 0000000..9ffdfd7
--- /dev/null
+++ b/evals/fixtures/pd03-book.txt
@@ -0,0 +1,7 @@
+Chapter 1: Observe
+
+Measure a signal before changing a system. Record the evidence window.
+
+Chapter 2: Respond
+
+Apply the smallest safe response after observation. Verify the result.
diff --git a/evals/fixtures/pd03-metadata.json b/evals/fixtures/pd03-metadata.json
new file mode 100644
index 0000000..0fd5f50
--- /dev/null
+++ b/evals/fixtures/pd03-metadata.json
@@ -0,0 +1,15 @@
+{
+ "title": "Synthetic Reliability Notes",
+ "description": "A synthetic handbook used only to evaluate one-level chunk routing.",
+ "chunks": [
+ {
+ "description": "Observe system signals.",
+ "summary": "Measure before changing.",
+ "key_elements": ["signal", "evidence window"]
+ },
+ {
+ "description": "Respond after observation.",
+ "summary": "Use the smallest safe response."
+ }
+ ]
+}
diff --git a/evals/fixtures/replay_trajectories.json b/evals/fixtures/replay_trajectories.json
new file mode 100644
index 0000000..e59b5f3
--- /dev/null
+++ b/evals/fixtures/replay_trajectories.json
@@ -0,0 +1 @@
+{"fixture_version":"1","schema_version":"pd-replay-fixture/v1","trajectories":[{"expected":{"target":"book-a/chapter-1"},"observed":{"answer_correct":true,"opens":["book-a/chapter-1"],"usage":{"calls":1,"input_tokens":11,"output_tokens":3}},"question_id":"01-correct"},{"expected":{"target":"book-a/chapter-2"},"observed":{"answer_correct":false,"opens":["book-b/chapter-9"],"usage":{"calls":2,"input_tokens":12,"output_tokens":4}},"question_id":"02-wrong-routing"},{"expected":{"target":"book-a/chapter-3"},"observed":{"answer_correct":false,"opens":["book-a/chapter-3"],"usage":{"calls":1,"input_tokens":13,"output_tokens":5}},"question_id":"03-wrong-answer"},{"expected":{"target":"book-a/chapter-4"},"observed":{"answer_correct":true,"opens":["book-b/chapter-1","book-c/chapter-2","book-a/chapter-4"],"usage":{"calls":3,"input_tokens":14,"output_tokens":6}},"question_id":"04-irrelevant-opens"},{"expected":{"target":"book-a/chapter-5"},"observed":{"opens":[],"usage":{}},"question_id":"05-unknown"}]}
diff --git a/tests/evals/test_manifest.py b/tests/evals/test_manifest.py
new file mode 100644
index 0000000..c561be1
--- /dev/null
+++ b/tests/evals/test_manifest.py
@@ -0,0 +1,97 @@
+"""Tests for deterministic evaluation run manifests."""
+import importlib.util
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+MODULE_PATH = ROOT / "tools" / "evals" / "manifest.py"
+SPEC = importlib.util.spec_from_file_location("eval_manifest", MODULE_PATH)
+manifest = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+sys.modules[SPEC.name] = manifest
+SPEC.loader.exec_module(manifest)
+
+
+def config(source_name="source.txt"):
+ return {
+ "sources": [{"id": "fixture", "path": source_name}],
+ "condition": "flat",
+ "question_set_hash": "questions-sha256",
+ "models": {"answer": "test-model", "generation": "unsupported"},
+ "harness_id": "fixture-harness",
+ "prompt_hashes": {"answer": "prompt-sha256"},
+ "config_hashes": {"runner": "config-sha256"},
+ "seed": "unsupported",
+ "repetition": "unsupported",
+ "budgets": {"max_calls": 1, "max_input_tokens": 100, "max_output_tokens": 20},
+ "artifacts": ".eval-work/artifacts/run",
+ "results": ".eval-work/results/run.json",
+ "reason": "smallest discriminating run",
+ "decision": "whether to continue",
+ }
+
+
+def test_identical_inputs_have_identical_manifest_and_run_id(tmp_path):
+ (tmp_path / "source.txt").write_text("café", encoding="utf-8")
+ first = manifest.build_manifest(config(), tmp_path)
+ second = manifest.build_manifest(config(), tmp_path)
+
+ assert first == second
+ assert first["sources"] == [{"id": "fixture", "sha256": manifest.sha256_file(tmp_path / "source.txt")}]
+ assert "café" == json.loads(manifest.canonical_json({"text": "café"}))["text"]
+
+
+def test_material_config_or_source_change_alters_run_id(tmp_path, monkeypatch):
+ source = tmp_path / "source.txt"
+ source.write_bytes("café".encode("utf-8"))
+ monkeypatch.setattr(manifest, "_commit", lambda _: "a" * 40)
+ original = manifest.build_manifest(config(), tmp_path)
+ changed_condition = config()
+ changed_condition["condition"] = "structured"
+ assert manifest.build_manifest(changed_condition, tmp_path)["run_id"] != original["run_id"]
+
+ source.write_bytes("caffè".encode("utf-8"))
+ assert manifest.build_manifest(config(), tmp_path)["run_id"] != original["run_id"]
+
+ monkeypatch.setattr(manifest, "_commit", lambda _: "b" * 40)
+ assert manifest.build_manifest(config(), tmp_path)["run_id"] != original["run_id"]
+
+
+ "mutate, expected",
+ [
+ (lambda value: value.pop("condition"), "missing required fields"),
+ (lambda value: value["budgets"].update(max_calls=-1), "max_calls"),
+ (lambda value: value.update(seed=-1), "seed"),
+ (lambda value: value.update(repetition=0), "repetition"),
+ (lambda value: value.update(api_key="not-allowed"), "guardrail only"),
+ (lambda value: value.update(harness_id="Bearer token-value"), "guardrail only"),
+ (lambda value: value.update(harness_id="sk-abcdefghijklmnop"), "guardrail only"),
+ (lambda value: value.update(harness_id="ghp_" + "a" * 36), "guardrail only"),
+ (lambda value: value.update(harness_id="AKIA" + "A" * 16), "guardrail only"),
+ (lambda value: value.update(harness_id="eyJsynthetic.payloadseg.signature"), "guardrail only"),
+ (lambda value: value.update(harness_id="glpat-" + "a" * 20), "guardrail only"),
+ ],
+)
+def test_invalid_configs_fail_clearly(tmp_path, mutate, expected):
+ (tmp_path / "source.txt").write_text("fixture", encoding="utf-8")
+ value = config()
+ mutate(value)
+ with pytest.raises(ValueError, match=expected):
+ manifest.build_manifest(value, tmp_path)
+
+
+def test_cli_is_repeatedly_stable(tmp_path):
+ (tmp_path / "source.txt").write_bytes("東京".encode("utf-8"))
+ config_path = tmp_path / "run.json"
+ config_path.write_text(json.dumps(config(), ensure_ascii=False), encoding="utf-8")
+ command = [sys.executable, str(MODULE_PATH), str(config_path)]
+ first = subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8")
+ second = subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8")
+
+ assert first.stdout == second.stdout
+ assert json.loads(first.stdout)["run_id"]
diff --git a/tests/evals/test_paper_flat.py b/tests/evals/test_paper_flat.py
new file mode 100644
index 0000000..7c6340a
--- /dev/null
+++ b/tests/evals/test_paper_flat.py
@@ -0,0 +1,77 @@
+"""Tests for the experiment-only PD-03 paper-flat pack builder."""
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+MODULE_PATH = ROOT / "tools" / "evals" / "paper_flat.py"
+SPEC = importlib.util.spec_from_file_location("paper_flat", MODULE_PATH)
+paper_flat = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+sys.modules[SPEC.name] = paper_flat
+SPEC.loader.exec_module(paper_flat)
+FIXTURES = ROOT / "evals" / "fixtures"
+
+
+def build(tmp_path, source=FIXTURES / "pd03-book.txt", metadata=FIXTURES / "pd03-metadata.json"):
+ return paper_flat.build_paper_flat(source, metadata, tmp_path / "pack")
+
+
+def test_root_and_raw_chunks_have_expected_one_level_structure(tmp_path):
+ build(tmp_path)
+ pack = tmp_path / "pack"
+ assert (pack / "SKILL.md").read_text(encoding="utf-8") == """# Synthetic Reliability Notes
+
+A synthetic handbook used only to evaluate one-level chunk routing.
+
+## Activation-time chunk table
+
+| Path | Description | SUMMARY | KEY_ELEMENTS |
+| --- | --- | --- | --- |
+| `chunks/chunk-01.md` | Observe system signals. | Measure before changing. | signal, evidence window |
+| `chunks/chunk-02.md` | Respond after observation. | Use the smallest safe response. | — |
+
+Split policy: source chapter headings; without headings, one full-source chunk.
+"""
+ assert (pack / "chunks" / "chunk-01.md").read_text(encoding="utf-8") == "Chapter 1: Observe\n\nMeasure a signal before changing a system. Record the evidence window.\n\n"
+ assert (pack / "chunks" / "chunk-02.md").read_text(encoding="utf-8") == "Chapter 2: Respond\n\nApply the smallest safe response after observation. Verify the result.\n"
+
+
+def test_identical_inputs_produce_byte_identical_output(tmp_path):
+ first = tmp_path / "first"
+ second = tmp_path / "second"
+ paper_flat.build_paper_flat(FIXTURES / "pd03-book.txt", FIXTURES / "pd03-metadata.json", first)
+ paper_flat.build_paper_flat(FIXTURES / "pd03-book.txt", FIXTURES / "pd03-metadata.json", second)
+ assert {path.relative_to(first): path.read_bytes() for path in first.rglob("*") if path.is_file()} == {
+ path.relative_to(second): path.read_bytes() for path in second.rglob("*") if path.is_file()
+ }
+
+
+def test_source_chapter_headings_determine_chunks(tmp_path):
+ build(tmp_path)
+ chunks = sorted((tmp_path / "pack" / "chunks").iterdir())
+ assert len(chunks) == 2
+ assert all(chunk.read_text(encoding="utf-8").startswith(f"Chapter {number}") for number, chunk in enumerate(chunks, 1))
+
+
+def test_indented_hash_code_line_does_not_split_chunks():
+ source = "# Top-level\n\n # Code comment\n\nChapter 2: Next\n"
+
+ assert paper_flat._chunks(source) == ["# Top-level\n\n # Code comment\n\n", "Chapter 2: Next\n"]
+
+
+def test_no_heading_uses_one_full_source_chunk(tmp_path):
+ source = tmp_path / "source.txt"
+ metadata = tmp_path / "metadata.json"
+ source.write_text("Plain synthetic source.\n", encoding="utf-8")
+ metadata.write_text(json.dumps({"title": "Plain", "description": "Fallback fixture.", "chunks": [{"description": "Whole source.", "summary": "No headings."}]}), encoding="utf-8")
+ build(tmp_path, source, metadata)
+ assert (tmp_path / "pack" / "chunks" / "chunk-01.md").read_text(encoding="utf-8") == "Plain synthetic source.\n"
+
+
+def test_pack_has_no_child_skill_hierarchy(tmp_path):
+ build(tmp_path)
+ files = sorted(path.relative_to(tmp_path / "pack") for path in (tmp_path / "pack").rglob("*") if path.is_file())
+ assert files == [Path("SKILL.md"), Path("chunks/chunk-01.md"), Path("chunks/chunk-02.md")]
+ assert all(path.name == "SKILL.md" for path in files if path.name == "SKILL.md")
diff --git a/tests/evals/test_replay.py b/tests/evals/test_replay.py
new file mode 100644
index 0000000..3c5d4c7
--- /dev/null
+++ b/tests/evals/test_replay.py
@@ -0,0 +1,39 @@
+"""Tests for deterministic fixture replay."""
+import importlib.util
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+REPLAY = ROOT / "tools" / "evals" / "replay.py"
+FIXTURE = ROOT / "evals" / "fixtures" / "replay_trajectories.json"
+SPEC = importlib.util.spec_from_file_location("eval_replay", REPLAY)
+replay = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+sys.path.insert(0, str(REPLAY.parent))
+sys.modules[SPEC.name] = replay
+SPEC.loader.exec_module(replay)
+
+
+def test_fixture_has_exactly_five_synthetic_situations():
+ fixture = replay.load_fixture(FIXTURE)
+ assert fixture["schema_version"] == replay.SCHEMA_VERSION
+ assert len(fixture["trajectories"]) == 5
+ assert [item["question_id"] for item in fixture["trajectories"]] == [
+ "01-correct", "02-wrong-routing", "03-wrong-answer", "04-irrelevant-opens", "05-unknown"]
+
+
+def test_replay_returns_machine_readable_results():
+ result = replay.replay(FIXTURE)
+ assert result["schema_version"] == replay.RESULT_SCHEMA_VERSION
+ assert result["aggregate"]["questions"] == 5
+ assert len(result["questions"]) == 5
+
+
+def test_cli_output_is_byte_identical_on_repeat():
+ command = [sys.executable, str(REPLAY), str(FIXTURE)]
+ first = subprocess.run(command, check=True, capture_output=True).stdout
+ second = subprocess.run(command, check=True, capture_output=True).stdout
+ assert first == second
+ assert json.loads(first)["fixture_version"] == "1"
diff --git a/tests/evals/test_score.py b/tests/evals/test_score.py
new file mode 100644
index 0000000..de0ea1b
--- /dev/null
+++ b/tests/evals/test_score.py
@@ -0,0 +1,59 @@
+"""Focused tests for offline trajectory scoring."""
+import importlib.util
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location("eval_score", ROOT / "tools" / "evals" / "score.py")
+score = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+sys.modules[SPEC.name] = score
+SPEC.loader.exec_module(score)
+
+
+def trajectory(question_id, opens, answer_correct=True, usage=None):
+ return {"question_id": question_id, "expected": {"target": "book/chapter"},
+ "observed": {"opens": opens, "answer_correct": answer_correct,
+ "usage": usage or {}}}
+
+
+def test_classifies_correct_routing_and_answer():
+ assert score.score_trajectory(trajectory("q", ["book/chapter"]))["classification"] == "correct"
+
+
+def test_classifies_wrong_routing():
+ assert score.score_trajectory(trajectory("q", ["other/chapter"], False))["classification"] == "wrong_routing"
+
+
+def test_classifies_target_evidence_with_wrong_answer():
+ result = score.score_trajectory(trajectory("q", ["book/chapter"], False))
+ assert result["classification"] == "wrong_answer"
+ assert result["evidence_reached"] is True
+
+
+def test_classifies_irrelevant_opens_before_target():
+ assert score.score_trajectory(trajectory("q", ["other/one", "other/two", "book/chapter"]))["classification"] == "irrelevant_opens_before_target"
+
+
+def test_unknown_observability_is_never_inferred():
+ result = score.score_trajectory({"question_id": "q", "expected": {"target": "book/chapter"}, "observed": {}})
+ assert result["classification"] == score.UNKNOWN
+ assert result["routing_correct"] == score.UNKNOWN
+ assert result["evidence_reached"] == score.UNKNOWN
+ assert result["answer_correct"] == score.UNKNOWN
+
+
+def test_explicit_unknown_remains_unknown_despite_observed_open():
+ item = trajectory("q", ["book/chapter"])
+ item["observed"]["route_correct"] = "unknown"
+ result = score.score_trajectory(item)
+ assert result["routing_correct"] == score.UNKNOWN
+ assert result["classification"] == score.UNKNOWN
+
+
+def test_aggregate_sums_only_recorded_tokens_and_calls():
+ result = score.score([trajectory("b", ["book/chapter"], usage={"input_tokens": 4, "calls": 2}),
+ trajectory("a", [], usage={"output_tokens": 3})])
+ assert [item["question_id"] for item in result["questions"]] == ["a", "b"]
+ assert result["aggregate"]["recorded_usage"] == {"input_tokens": 4, "output_tokens": 3, "calls": 2}
+ assert result["aggregate"]["usage_observations"] == {"input_tokens": 1, "output_tokens": 1, "calls": 1}
diff --git a/tools/evals/manifest.py b/tools/evals/manifest.py
new file mode 100644
index 0000000..0d1abce
--- /dev/null
+++ b/tools/evals/manifest.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""Create deterministic, secret-free manifests for evaluation runs.
+
+Usage: python3 tools/evals/manifest.py config.json [--output manifest.json]
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+SCHEMA_VERSION = "pd-run-manifest/v1"
+_REQUIRED = {
+ "condition", "question_set_hash", "models", "harness_id", "prompt_hashes",
+ "config_hashes", "seed", "repetition", "budgets", "artifacts", "results",
+ "reason", "decision",
+}
+_SECRET_MARKERS = ("secret", "password", "credential", "authorization", "api_key", "private_key")
+_SECRET_VALUE_PATTERNS = (
+ re.compile(r"^Bearer\s+\S+$", re.IGNORECASE),
+ re.compile(r"^sk-[A-Za-z0-9_-]{16,}$"),
+ re.compile(r"^ghp_[A-Za-z0-9]{36}$"),
+ re.compile(r"^AKIA[A-Z0-9]{16}$"),
+ re.compile(r"^eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}$"),
+ re.compile(r"^glpat-[A-Za-z0-9_-]{20}$"),
+)
+_SECRET_GUARDRAIL = "possible secret (guardrail only; not a DLP guarantee); store only identifiers or hashes"
+
+
+def canonical_json(value: Any) -> str:
+ """Return the UTF-8-preserving canonical JSON representation of *value*."""
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def sha256_file(path: Path) -> str:
+ """Return the SHA-256 digest of a file's raw bytes."""
+ digest = hashlib.sha256()
+ with path.open("rb") as source:
+ for block in iter(lambda: source.read(1024 * 1024), b""):
+ digest.update(block)
+ return digest.hexdigest()
+
+
+def _fail(message: str) -> None:
+ raise ValueError(f"invalid run manifest config: {message}")
+
+
+def _reject_secrets(value: Any, location: str = "config") -> None:
+ if isinstance(value, dict):
+ for key, child in value.items():
+ if not isinstance(key, str):
+ _fail(f"{location} has a non-string key")
+ if any(marker in key.lower() for marker in _SECRET_MARKERS):
+ _fail(f"{location}.{key} {_SECRET_GUARDRAIL}")
+ _reject_secrets(child, f"{location}.{key}")
+ elif isinstance(value, list):
+ for index, child in enumerate(value):
+ _reject_secrets(child, f"{location}[{index}]")
+ elif isinstance(value, str) and any(pattern.fullmatch(value) for pattern in _SECRET_VALUE_PATTERNS):
+ _fail(f"{location} {_SECRET_GUARDRAIL}")
+
+
+def _identifier(value: Any, name: str) -> str:
+ if not isinstance(value, str) or not value.strip():
+ _fail(f"{name} must be a non-empty string")
+ return value
+
+
+def _hashes(value: Any, name: str) -> Dict[str, str]:
+ if not isinstance(value, dict) or not value:
+ _fail(f"{name} must be a non-empty object of named hashes")
+ return {str(key): _identifier(digest, f"{name}.{key}") for key, digest in value.items()}
+
+
+def _supportable(value: Any, name: str, minimum: int) -> Any:
+ if value == "unsupported":
+ return value
+ if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
+ qualifier = "positive" if minimum else "non-negative"
+ _fail(f"{name} must be {qualifier} integer or 'unsupported'")
+ return value
+
+
+def _budgets(value: Any) -> Dict[str, Any]:
+ if not isinstance(value, dict):
+ _fail("budgets must be an object")
+ required = ("max_calls", "max_input_tokens", "max_output_tokens")
+ missing = [key for key in required if key not in value]
+ if missing:
+ _fail("budgets missing " + ", ".join(missing))
+ result: Dict[str, Any] = {}
+ for key in required:
+ item = value[key]
+ if isinstance(item, bool) or not isinstance(item, int) or item < 0:
+ _fail(f"budgets.{key} must be a non-negative integer hard ceiling")
+ result[key] = item
+ if "max_cost_usd" in value:
+ cost = value["max_cost_usd"]
+ if isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0:
+ _fail("budgets.max_cost_usd must be a non-negative finite number")
+ result["max_cost_usd"] = cost
+ return result
+
+
+def _commit(base_dir: Path) -> str:
+ try:
+ result = subprocess.run(
+ ["git", "rev-parse", "HEAD"], cwd=base_dir, capture_output=True,
+ text=True, encoding="utf-8", errors="replace", timeout=5, check=False,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return "unavailable"
+ return result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unavailable"
+
+
+def build_manifest(config: Dict[str, Any], base_dir: Optional[Path] = None) -> Dict[str, Any]:
+ """Validate *config*, hash its sources, and return a deterministic manifest."""
+ if not isinstance(config, dict):
+ _fail("top level must be an object")
+ _reject_secrets(config)
+ missing = sorted(_REQUIRED - set(config))
+ if missing:
+ _fail("missing required fields: " + ", ".join(missing))
+ base_dir = (base_dir or Path.cwd()).resolve()
+ sources = config.get("sources")
+ if not isinstance(sources, list) or not sources:
+ _fail("sources must be a non-empty list")
+ source_records = []
+ for index, source in enumerate(sources):
+ if not isinstance(source, dict):
+ _fail(f"sources[{index}] must be an object")
+ source_id = _identifier(source.get("id"), f"sources[{index}].id")
+ source_path = _identifier(source.get("path"), f"sources[{index}].path")
+ resolved = (base_dir / source_path).resolve()
+ if not resolved.is_file():
+ _fail(f"sources[{index}].path is not a readable file: {source_path}")
+ source_records.append({"id": source_id, "sha256": sha256_file(resolved)})
+ source_records.sort(key=lambda record: (record["id"], record["sha256"]))
+
+ models = config["models"]
+ if not isinstance(models, dict) or not models:
+ _fail("models must be a non-empty object")
+ normalized_models = {str(key): _identifier(value, f"models.{key}") for key, value in models.items()}
+ artifacts = _identifier(config["artifacts"], "artifacts")
+ results = _identifier(config["results"], "results")
+ manifest = {
+ "schema_version": SCHEMA_VERSION,
+ "sources": source_records,
+ "condition": _identifier(config["condition"], "condition"),
+ "question_set_hash": _identifier(config["question_set_hash"], "question_set_hash"),
+ "models": normalized_models,
+ "harness_id": _identifier(config["harness_id"], "harness_id"),
+ "prompt_hashes": _hashes(config["prompt_hashes"], "prompt_hashes"),
+ "config_hashes": _hashes(config["config_hashes"], "config_hashes"),
+ "seed": _supportable(config["seed"], "seed", 0),
+ "repetition": _supportable(config["repetition"], "repetition", 1),
+ "budgets": _budgets(config["budgets"]),
+ "commit": _commit(base_dir),
+ "artifacts": artifacts,
+ "results": results,
+ "reason": _identifier(config["reason"], "reason"),
+ "decision": _identifier(config["decision"], "decision"),
+ }
+ identity = {key: value for key, value in manifest.items() if key not in {"artifacts", "results"}}
+ if manifest["commit"] == "unavailable":
+ identity.pop("commit")
+ manifest["run_id"] = hashlib.sha256(canonical_json(identity).encode("utf-8")).hexdigest()
+ return manifest
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("config", type=Path, help="evaluation run config JSON")
+ parser.add_argument("--output", type=Path, help="write manifest JSON to this file")
+ args = parser.parse_args(argv)
+ try:
+ with args.config.open(encoding="utf-8") as stream:
+ config = json.load(stream)
+ rendered = canonical_json(build_manifest(config, args.config.parent)) + "\n"
+ if args.output:
+ args.output.write_text(rendered, encoding="utf-8")
+ else:
+ sys.stdout.write(rendered)
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
+ parser.error(str(exc))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/evals/paper_flat.py b/tools/evals/paper_flat.py
new file mode 100644
index 0000000..96c92bb
--- /dev/null
+++ b/tools/evals/paper_flat.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Build the deterministic, experiment-only PD-03 paper-flat skill pack.
+
+Usage: python3 tools/evals/paper_flat.py SOURCE METADATA OUTPUT_DIR
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+from typing import Any, Dict, List
+
+_CHAPTER = re.compile(
+ r"^(?:#{1,6}\s+.+|(?:chapter|chap\.)\s+(?:\d+|[ivxlcdm]+)\b.*)$", re.IGNORECASE
+)
+
+
+def _chunks(source: str) -> List[str]:
+ """Split at chapter headings, or return one whole-source fallback chunk."""
+ lines = source.splitlines(keepends=True)
+ starts = [index for index, line in enumerate(lines) if _CHAPTER.match(line.rstrip())]
+ if not starts:
+ return [source]
+ return ["".join(lines[start:end]) for start, end in zip(starts, starts[1:] + [len(lines)])]
+
+
+def _metadata(path: Path, chunk_count: int) -> Dict[str, Any]:
+ value = json.loads(path.read_text(encoding="utf-8"))
+ if not isinstance(value, dict) or not all(isinstance(value.get(key), str) for key in ("title", "description")):
+ raise ValueError("metadata requires string title and description")
+ chunks = value.get("chunks")
+ if not isinstance(chunks, list) or len(chunks) != chunk_count:
+ raise ValueError("metadata chunks must match the source chunk count")
+ for chunk in chunks:
+ if not isinstance(chunk, dict) or not all(isinstance(chunk.get(key), str) for key in ("description", "summary")):
+ raise ValueError("each metadata chunk requires string description and summary")
+ if "key_elements" in chunk and (
+ not isinstance(chunk["key_elements"], list) or not all(isinstance(item, str) for item in chunk["key_elements"])
+ ):
+ raise ValueError("key_elements must be a list of strings")
+ return value
+
+
+def build_paper_flat(source_path: Path, metadata_path: Path, output_dir: Path) -> List[Path]:
+ """Write a one-level paper-flat pack and return its files in stable order."""
+ source = source_path.read_text(encoding="utf-8")
+ chunks = _chunks(source)
+ metadata = _metadata(metadata_path, len(chunks))
+ chunk_dir = output_dir / "chunks"
+ chunk_dir.mkdir(parents=True, exist_ok=True)
+
+ rows = []
+ written = []
+ for number, (payload, details) in enumerate(zip(chunks, metadata["chunks"]), 1):
+ relative = f"chunks/chunk-{number:02d}.md"
+ path = output_dir / relative
+ with path.open("w", encoding="utf-8", newline="\n") as file:
+ file.write(payload)
+ written.append(path)
+ elements = ", ".join(details.get("key_elements", [])) or "—"
+ rows.append(f"| `{relative}` | {details['description']} | {details['summary']} | {elements} |")
+
+ root = "\n".join(
+ [
+ f"# {metadata['title']}",
+ "",
+ metadata["description"],
+ "",
+ "## Activation-time chunk table",
+ "",
+ "| Path | Description | SUMMARY | KEY_ELEMENTS |",
+ "| --- | --- | --- | --- |",
+ *rows,
+ "",
+ "Split policy: source chapter headings; without headings, one full-source chunk.",
+ "",
+ ]
+ )
+ root_path = output_dir / "SKILL.md"
+ with root_path.open("w", encoding="utf-8", newline="\n") as file:
+ file.write(root)
+ return [root_path, *written]
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Build an experiment-only PD-03 paper-flat pack.")
+ parser.add_argument("source", type=Path, help="UTF-8 synthetic source text")
+ parser.add_argument("metadata", type=Path, help="fixed UTF-8 JSON metadata")
+ parser.add_argument("output", type=Path, help="output pack directory")
+ args = parser.parse_args()
+ build_paper_flat(args.source, args.metadata, args.output)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/evals/replay.py b/tools/evals/replay.py
new file mode 100644
index 0000000..6bd09d0
--- /dev/null
+++ b/tools/evals/replay.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""Replay a versioned synthetic trajectory fixture deterministically."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Dict
+
+from score import score
+
+SCHEMA_VERSION = "pd-replay-fixture/v1"
+RESULT_SCHEMA_VERSION = "pd-replay-result/v1"
+
+
+def canonical_json(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def load_fixture(path: Path) -> Dict[str, Any]:
+ with path.open(encoding="utf-8") as source:
+ fixture = json.load(source)
+ if fixture.get("schema_version") != SCHEMA_VERSION:
+ raise ValueError("unsupported fixture schema_version")
+ trajectories = fixture.get("trajectories")
+ if not isinstance(trajectories, list):
+ raise ValueError("fixture trajectories must be a list")
+ return fixture
+
+
+def replay(path: Path) -> Dict[str, Any]:
+ fixture = load_fixture(path)
+ result = score(fixture["trajectories"])
+ return {"schema_version": RESULT_SCHEMA_VERSION, "fixture_version": fixture["fixture_version"], **result}
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("fixture", type=Path)
+ args = parser.parse_args()
+ print(canonical_json(replay(args.fixture)))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/evals/score.py b/tools/evals/score.py
new file mode 100644
index 0000000..b09ec42
--- /dev/null
+++ b/tools/evals/score.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Pure scoring and accounting for synthetic evaluation trajectories."""
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List
+
+UNKNOWN = "unknown"
+
+
+def _state(value: Any) -> Any:
+ """Keep only explicit boolean observations; absence remains unknown."""
+ return value if isinstance(value, bool) else UNKNOWN
+
+
+def score_trajectory(trajectory: Dict[str, Any]) -> Dict[str, Any]:
+ """Score one trajectory without loading files or deriving missing observations."""
+ expected = trajectory.get("expected", {})
+ observed = trajectory.get("observed", {})
+ target = expected.get("target")
+ opens = observed.get("opens")
+ opened_target = (
+ any(opened == target for opened in opens)
+ if isinstance(opens, list) and target is not None
+ else UNKNOWN
+ )
+ answer_correct = _state(observed.get("answer_correct"))
+ route_correct = _state(observed.get("route_correct"))
+ evidence_reached = _state(observed.get("evidence_reached"))
+ if "route_correct" not in observed and opened_target is not UNKNOWN:
+ route_correct = opened_target
+ if "evidence_reached" not in observed and opened_target is not UNKNOWN:
+ evidence_reached = opened_target
+
+ if UNKNOWN in (route_correct, evidence_reached, answer_correct):
+ classification = UNKNOWN
+ elif not route_correct:
+ classification = "wrong_routing"
+ elif not answer_correct:
+ classification = "wrong_answer"
+ elif isinstance(opens, list) and target is not None and opens.index(target) > 0:
+ classification = "irrelevant_opens_before_target"
+ else:
+ classification = "correct"
+
+ usage = observed.get("usage")
+ usage = usage if isinstance(usage, dict) else {}
+ return {
+ "question_id": trajectory["question_id"],
+ "classification": classification,
+ "routing_correct": route_correct,
+ "evidence_reached": evidence_reached,
+ "answer_correct": answer_correct,
+ "usage": {key: usage.get(key) if isinstance(usage.get(key), int) else UNKNOWN
+ for key in ("input_tokens", "output_tokens", "calls")},
+ }
+
+
+def aggregate(results: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
+ """Aggregate recorded usage and classifications; never estimate missing usage."""
+ items = list(results)
+ counts: Dict[str, int] = {}
+ totals = {"input_tokens": 0, "output_tokens": 0, "calls": 0}
+ recorded = {key: 0 for key in totals}
+ for result in items:
+ label = result["classification"]
+ counts[label] = counts.get(label, 0) + 1
+ for key in totals:
+ value = result["usage"][key]
+ if value != UNKNOWN:
+ totals[key] += value
+ recorded[key] += 1
+ return {
+ "questions": len(items),
+ "classifications": counts,
+ "recorded_usage": totals,
+ "usage_observations": recorded,
+ }
+
+
+def score(trajectories: Iterable[Dict[str, Any]]) -> Dict[str, Any]:
+ """Return deterministic per-question and aggregate replay results."""
+ questions: List[Dict[str, Any]] = [score_trajectory(item) for item in trajectories]
+ questions.sort(key=lambda item: item["question_id"])
+ return {"questions": questions, "aggregate": aggregate(questions)}