Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
feat(evals): add offline trajectory replay and evaluation tools (#189, #190, #191)
virgiliojr949c207f870ade
diff --git a/.gitignore b/.gitignoreindex de5c726..c81c624 100644--- a/.gitignore+++ b/.gitignore@@ -25,3 +25,6 @@ docs/BACKERS.md# Social-card render cache (mkdocs-material[imaging])..cache/++# Local evaluation work products+.eval-work/diff --git a/docs/research/progressive-disclosure-evals.md b/docs/research/progressive-disclosure-evals.mdindex f024e2f..f6d253b 100644--- a/docs/research/progressive-disclosure-evals.md+++ b/docs/research/progressive-disclosure-evals.md@@ -207,10 +207,10 @@ Status values:| ID | Status | Task | Depends on | Production change? ||---|---|---|---|---|-| PD-00 | READY | Freeze current baseline and evaluation contract | none | no |-| PD-01 | BLOCKED | Implement deterministic manifest + hashing + budget schema | PD-00 | no |-| PD-02 | BLOCKED | Implement fixture/replay scorer with trajectory metrics | PD-01 | no |-| PD-03 | BLOCKED | Implement paper-flat baseline pack builder | PD-01 | no |+| PD-00 | DONE | Freeze current baseline and evaluation contract | none | no |+| PD-01 | DONE | Implement deterministic manifest + hashing + budget schema | PD-00 | no |+| PD-02 | DONE | Implement fixture/replay scorer with trajectory metrics | PD-01 | no |+| PD-03 | DONE | Implement paper-flat baseline pack builder | PD-01 | no || PD-04 | BLOCKED | Add dry-run/live runner contract and first harness adapter | PD-02, PD-03 | no || PD-05 | BLOCKED | Representation experiment: raw payload vs structured B2S payload | PD-04 | no || PD-06 | BLOCKED | Automatic vs explicit activation experiment | PD-04 | no |@@ -548,10 +548,10 @@ Agents update this table when a task changes status. Keep entries short; link to| Task | Status | PR / commit | Evidence | Notes ||---|---|---|---|---|-| PD-00 | READY | — | — | First implementation task |-| PD-01 | BLOCKED | — | — | waits for PD-00 |-| PD-02 | BLOCKED | — | — | waits for PD-01 |-| PD-03 | BLOCKED | — | — | waits for PD-01 |+| PD-00 | DONE | — | — | Baseline documented |+| PD-01 | DONE | — | `tests/evals/test_manifest.py` | Deterministic manifest, source SHA-256, budget/secret validation |+| PD-02 | DONE | — | `tests/evals/test_score.py`, `test_replay.py` | Offline trajectory scorer and replay runner |+| PD-03 | DONE | — | `tests/evals/test_paper_flat.py` | Synthetic paper-flat baseline builder || PD-04 | BLOCKED | — | — | waits for PD-02/03 || PD-05 | BLOCKED | — | — | primary representation experiment || PD-06 | BLOCKED | — | — | host capability dependent |diff --git a/evals/fixtures/pd00-synthetic-book.txt b/evals/fixtures/pd00-synthetic-book.txtnew file mode 100644index 0000000..c4b79f4--- /dev/null+++ b/evals/fixtures/pd00-synthetic-book.txt@@ -0,0 +1,22 @@+Synthetic Progressive Disclosure Book++Sumário+Capítulo 1 — Orientação+Capítulo 2 — Recuperação+Capítulo 3 — Verificação++Capítulo 1 — Orientação+A orientação apresenta o objetivo da avaliação. O leitor identifica o livro,+o capítulo e a pergunta antes de carregar detalhes. Esta seção usa texto+sintético, curto e determinístico para exercitar a descoberta de capítulos.++Capítulo 2 — Recuperação+A recuperação localiza a evidência necessária depois da orientação. O agente+consulta o sumário, abre o capítulo relevante e evita carregar capítulos sem+relação com a pergunta. Esta seção permanece sintética e não reproduz obra+publicada.++Capítulo 3 — Verificação+A verificação confirma que a resposta usa a evidência do capítulo alvo. O+relatório separa contagens medidas da estimativa do loop de descoberta. Esta+seção é o alvo determinístico para a linha de base PD-00.diff --git a/evals/fixtures/pd03-book.txt b/evals/fixtures/pd03-book.txtnew file mode 100644index 0000000..9ffdfd7--- /dev/null+++ b/evals/fixtures/pd03-book.txt@@ -0,0 +1,7 @@+Chapter 1: Observe++Measure a signal before changing a system. Record the evidence window.++Chapter 2: Respond++Apply the smallest safe response after observation. Verify the result.diff --git a/evals/fixtures/pd03-metadata.json b/evals/fixtures/pd03-metadata.jsonnew file mode 100644index 0000000..0fd5f50--- /dev/null+++ b/evals/fixtures/pd03-metadata.json@@ -0,0 +1,15 @@+{+ "title": "Synthetic Reliability Notes",+ "description": "A synthetic handbook used only to evaluate one-level chunk routing.",+ "chunks": [+ {+ "description": "Observe system signals.",+ "summary": "Measure before changing.",+ "key_elements": ["signal", "evidence window"]+ },+ {+ "description": "Respond after observation.",+ "summary": "Use the smallest safe response."+ }+ ]+}diff --git a/evals/fixtures/replay_trajectories.json b/evals/fixtures/replay_trajectories.jsonnew file mode 100644index 0000000..e59b5f3--- /dev/null+++ b/evals/fixtures/replay_trajectories.json@@ -0,0 +1 @@+{"fixture_version":"1","schema_version":"pd-replay-fixture/v1","trajectories":[{"expected":{"target":"book-a/chapter-1"},"observed":{"answer_correct":true,"opens":["book-a/chapter-1"],"usage":{"calls":1,"input_tokens":11,"output_tokens":3}},"question_id":"01-correct"},{"expected":{"target":"book-a/chapter-2"},"observed":{"answer_correct":false,"opens":["book-b/chapter-9"],"usage":{"calls":2,"input_tokens":12,"output_tokens":4}},"question_id":"02-wrong-routing"},{"expected":{"target":"book-a/chapter-3"},"observed":{"answer_correct":false,"opens":["book-a/chapter-3"],"usage":{"calls":1,"input_tokens":13,"output_tokens":5}},"question_id":"03-wrong-answer"},{"expected":{"target":"book-a/chapter-4"},"observed":{"answer_correct":true,"opens":["book-b/chapter-1","book-c/chapter-2","book-a/chapter-4"],"usage":{"calls":3,"input_tokens":14,"output_tokens":6}},"question_id":"04-irrelevant-opens"},{"expected":{"target":"book-a/chapter-5"},"observed":{"opens":[],"usage":{}},"question_id":"05-unknown"}]}diff --git a/tests/evals/test_manifest.py b/tests/evals/test_manifest.pynew file mode 100644index 0000000..c561be1--- /dev/null+++ b/tests/evals/test_manifest.py@@ -0,0 +1,97 @@+"""Tests for deterministic evaluation run manifests."""+import importlib.util+import json+import subprocess+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[2]+MODULE_PATH = ROOT / "tools" / "evals" / "manifest.py"+SPEC = importlib.util.spec_from_file_location("eval_manifest", MODULE_PATH)+manifest = importlib.util.module_from_spec(SPEC)+assert SPEC.loader is not None+sys.modules[SPEC.name] = manifest+SPEC.loader.exec_module(manifest)+++def config(source_name="source.txt"):+ return {+ "sources": [{"id": "fixture", "path": source_name}],+ "condition": "flat",+ "question_set_hash": "questions-sha256",+ "models": {"answer": "test-model", "generation": "unsupported"},+ "harness_id": "fixture-harness",+ "prompt_hashes": {"answer": "prompt-sha256"},+ "config_hashes": {"runner": "config-sha256"},+ "seed": "unsupported",+ "repetition": "unsupported",+ "budgets": {"max_calls": 1, "max_input_tokens": 100, "max_output_tokens": 20},+ "artifacts": ".eval-work/artifacts/run",+ "results": ".eval-work/results/run.json",+ "reason": "smallest discriminating run",+ "decision": "whether to continue",+ }+++def test_identical_inputs_have_identical_manifest_and_run_id(tmp_path):+ (tmp_path / "source.txt").write_text("café", encoding="utf-8")+ first = manifest.build_manifest(config(), tmp_path)+ second = manifest.build_manifest(config(), tmp_path)++ assert first == second+ assert first["sources"] == [{"id": "fixture", "sha256": manifest.sha256_file(tmp_path / "source.txt")}]+ assert "café" == json.loads(manifest.canonical_json({"text": "café"}))["text"]+++def test_material_config_or_source_change_alters_run_id(tmp_path, monkeypatch):+ source = tmp_path / "source.txt"+ source.write_bytes("café".encode("utf-8"))+ monkeypatch.setattr(manifest, "_commit", lambda _: "a" * 40)+ original = manifest.build_manifest(config(), tmp_path)+ changed_condition = config()+ changed_condition["condition"] = "structured"+ assert manifest.build_manifest(changed_condition, tmp_path)["run_id"] != original["run_id"]++ source.write_bytes("caffè".encode("utf-8"))+ assert manifest.build_manifest(config(), tmp_path)["run_id"] != original["run_id"]++ monkeypatch.setattr(manifest, "_commit", lambda _: "b" * 40)+ assert manifest.build_manifest(config(), tmp_path)["run_id"] != original["run_id"]+++ "mutate, expected",+ [+ (lambda value: value.pop("condition"), "missing required fields"),+ (lambda value: value["budgets"].update(max_calls=-1), "max_calls"),+ (lambda value: value.update(seed=-1), "seed"),+ (lambda value: value.update(repetition=0), "repetition"),+ (lambda value: value.update(api_key="not-allowed"), "guardrail only"),+ (lambda value: value.update(harness_id="Bearer token-value"), "guardrail only"),+ (lambda value: value.update(harness_id="sk-abcdefghijklmnop"), "guardrail only"),+ (lambda value: value.update(harness_id="ghp_" + "a" * 36), "guardrail only"),+ (lambda value: value.update(harness_id="AKIA" + "A" * 16), "guardrail only"),+ (lambda value: value.update(harness_id="eyJsynthetic.payloadseg.signature"), "guardrail only"),+ (lambda value: value.update(harness_id="glpat-" + "a" * 20), "guardrail only"),+ ],+)+def test_invalid_configs_fail_clearly(tmp_path, mutate, expected):+ (tmp_path / "source.txt").write_text("fixture", encoding="utf-8")+ value = config()+ mutate(value)+ with pytest.raises(ValueError, match=expected):+ manifest.build_manifest(value, tmp_path)+++def test_cli_is_repeatedly_stable(tmp_path):+ (tmp_path / "source.txt").write_bytes("東京".encode("utf-8"))+ config_path = tmp_path / "run.json"+ config_path.write_text(json.dumps(config(), ensure_ascii=False), encoding="utf-8")+ command = [sys.executable, str(MODULE_PATH), str(config_path)]+ first = subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8")+ second = subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8")++ assert first.stdout == second.stdout+ assert json.loads(first.stdout)["run_id"]diff --git a/tests/evals/test_paper_flat.py b/tests/evals/test_paper_flat.pynew file mode 100644index 0000000..7c6340a--- /dev/null+++ b/tests/evals/test_paper_flat.py@@ -0,0 +1,77 @@+"""Tests for the experiment-only PD-03 paper-flat pack builder."""+import importlib.util+import json+import sys+from pathlib import Path++ROOT = Path(__file__).resolve().parents[2]+MODULE_PATH = ROOT / "tools" / "evals" / "paper_flat.py"+SPEC = importlib.util.spec_from_file_location("paper_flat", MODULE_PATH)+paper_flat = importlib.util.module_from_spec(SPEC)+assert SPEC.loader is not None+sys.modules[SPEC.name] = paper_flat+SPEC.loader.exec_module(paper_flat)+FIXTURES = ROOT / "evals" / "fixtures"+++def build(tmp_path, source=FIXTURES / "pd03-book.txt", metadata=FIXTURES / "pd03-metadata.json"):+ return paper_flat.build_paper_flat(source, metadata, tmp_path / "pack")+++def test_root_and_raw_chunks_have_expected_one_level_structure(tmp_path):+ build(tmp_path)+ pack = tmp_path / "pack"+ assert (pack / "SKILL.md").read_text(encoding="utf-8") == """# Synthetic Reliability Notes++A synthetic handbook used only to evaluate one-level chunk routing.++## Activation-time chunk table++| Path | Description | SUMMARY | KEY_ELEMENTS |+| --- | --- | --- | --- |+| `chunks/chunk-01.md` | Observe system signals. | Measure before changing. | signal, evidence window |+| `chunks/chunk-02.md` | Respond after observation. | Use the smallest safe response. | — |++Split policy: source chapter headings; without headings, one full-source chunk.+"""+ assert (pack / "chunks" / "chunk-01.md").read_text(encoding="utf-8") == "Chapter 1: Observe\n\nMeasure a signal before changing a system. Record the evidence window.\n\n"+ assert (pack / "chunks" / "chunk-02.md").read_text(encoding="utf-8") == "Chapter 2: Respond\n\nApply the smallest safe response after observation. Verify the result.\n"+++def test_identical_inputs_produce_byte_identical_output(tmp_path):+ first = tmp_path / "first"+ second = tmp_path / "second"+ paper_flat.build_paper_flat(FIXTURES / "pd03-book.txt", FIXTURES / "pd03-metadata.json", first)+ paper_flat.build_paper_flat(FIXTURES / "pd03-book.txt", FIXTURES / "pd03-metadata.json", second)+ assert {path.relative_to(first): path.read_bytes() for path in first.rglob("*") if path.is_file()} == {+ path.relative_to(second): path.read_bytes() for path in second.rglob("*") if path.is_file()+ }+++def test_source_chapter_headings_determine_chunks(tmp_path):+ build(tmp_path)+ chunks = sorted((tmp_path / "pack" / "chunks").iterdir())+ assert len(chunks) == 2+ assert all(chunk.read_text(encoding="utf-8").startswith(f"Chapter {number}") for number, chunk in enumerate(chunks, 1))+++def test_indented_hash_code_line_does_not_split_chunks():+ source = "# Top-level\n\n # Code comment\n\nChapter 2: Next\n"++ assert paper_flat._chunks(source) == ["# Top-level\n\n # Code comment\n\n", "Chapter 2: Next\n"]+++def test_no_heading_uses_one_full_source_chunk(tmp_path):+ source = tmp_path / "source.txt"+ metadata = tmp_path / "metadata.json"+ source.write_text("Plain synthetic source.\n", encoding="utf-8")+ metadata.write_text(json.dumps({"title": "Plain", "description": "Fallback fixture.", "chunks": [{"description": "Whole source.", "summary": "No headings."}]}), encoding="utf-8")+ build(tmp_path, source, metadata)+ assert (tmp_path / "pack" / "chunks" / "chunk-01.md").read_text(encoding="utf-8") == "Plain synthetic source.\n"+++def test_pack_has_no_child_skill_hierarchy(tmp_path):+ build(tmp_path)+ files = sorted(path.relative_to(tmp_path / "pack") for path in (tmp_path / "pack").rglob("*") if path.is_file())+ assert files == [Path("SKILL.md"), Path("chunks/chunk-01.md"), Path("chunks/chunk-02.md")]+ assert all(path.name == "SKILL.md" for path in files if path.name == "SKILL.md")diff --git a/tests/evals/test_replay.py b/tests/evals/test_replay.pynew file mode 100644index 0000000..3c5d4c7--- /dev/null+++ b/tests/evals/test_replay.py@@ -0,0 +1,39 @@+"""Tests for deterministic fixture replay."""+import importlib.util+import json+import subprocess+import sys+from pathlib import Path++ROOT = Path(__file__).resolve().parents[2]+REPLAY = ROOT / "tools" / "evals" / "replay.py"+FIXTURE = ROOT / "evals" / "fixtures" / "replay_trajectories.json"+SPEC = importlib.util.spec_from_file_location("eval_replay", REPLAY)+replay = importlib.util.module_from_spec(SPEC)+assert SPEC.loader is not None+sys.path.insert(0, str(REPLAY.parent))+sys.modules[SPEC.name] = replay+SPEC.loader.exec_module(replay)+++def test_fixture_has_exactly_five_synthetic_situations():+ fixture = replay.load_fixture(FIXTURE)+ assert fixture["schema_version"] == replay.SCHEMA_VERSION+ assert len(fixture["trajectories"]) == 5+ assert [item["question_id"] for item in fixture["trajectories"]] == [+ "01-correct", "02-wrong-routing", "03-wrong-answer", "04-irrelevant-opens", "05-unknown"]+++def test_replay_returns_machine_readable_results():+ result = replay.replay(FIXTURE)+ assert result["schema_version"] == replay.RESULT_SCHEMA_VERSION+ assert result["aggregate"]["questions"] == 5+ assert len(result["questions"]) == 5+++def test_cli_output_is_byte_identical_on_repeat():+ command = [sys.executable, str(REPLAY), str(FIXTURE)]+ first = subprocess.run(command, check=True, capture_output=True).stdout+ second = subprocess.run(command, check=True, capture_output=True).stdout+ assert first == second+ assert json.loads(first)["fixture_version"] == "1"diff --git a/tests/evals/test_score.py b/tests/evals/test_score.pynew file mode 100644index 0000000..de0ea1b--- /dev/null+++ b/tests/evals/test_score.py@@ -0,0 +1,59 @@+"""Focused tests for offline trajectory scoring."""+import importlib.util+import sys+from pathlib import Path++ROOT = Path(__file__).resolve().parents[2]+SPEC = importlib.util.spec_from_file_location("eval_score", ROOT / "tools" / "evals" / "score.py")+score = importlib.util.module_from_spec(SPEC)+assert SPEC.loader is not None+sys.modules[SPEC.name] = score+SPEC.loader.exec_module(score)+++def trajectory(question_id, opens, answer_correct=True, usage=None):+ return {"question_id": question_id, "expected": {"target": "book/chapter"},+ "observed": {"opens": opens, "answer_correct": answer_correct,+ "usage": usage or {}}}+++def test_classifies_correct_routing_and_answer():+ assert score.score_trajectory(trajectory("q", ["book/chapter"]))["classification"] == "correct"+++def test_classifies_wrong_routing():+ assert score.score_trajectory(trajectory("q", ["other/chapter"], False))["classification"] == "wrong_routing"+++def test_classifies_target_evidence_with_wrong_answer():+ result = score.score_trajectory(trajectory("q", ["book/chapter"], False))+ assert result["classification"] == "wrong_answer"+ assert result["evidence_reached"] is True+++def test_classifies_irrelevant_opens_before_target():+ assert score.score_trajectory(trajectory("q", ["other/one", "other/two", "book/chapter"]))["classification"] == "irrelevant_opens_before_target"+++def test_unknown_observability_is_never_inferred():+ result = score.score_trajectory({"question_id": "q", "expected": {"target": "book/chapter"}, "observed": {}})+ assert result["classification"] == score.UNKNOWN+ assert result["routing_correct"] == score.UNKNOWN+ assert result["evidence_reached"] == score.UNKNOWN+ assert result["answer_correct"] == score.UNKNOWN+++def test_explicit_unknown_remains_unknown_despite_observed_open():+ item = trajectory("q", ["book/chapter"])+ item["observed"]["route_correct"] = "unknown"+ result = score.score_trajectory(item)+ assert result["routing_correct"] == score.UNKNOWN+ assert result["classification"] == score.UNKNOWN+++def test_aggregate_sums_only_recorded_tokens_and_calls():+ result = score.score([trajectory("b", ["book/chapter"], usage={"input_tokens": 4, "calls": 2}),+ trajectory("a", [], usage={"output_tokens": 3})])+ assert [item["question_id"] for item in result["questions"]] == ["a", "b"]+ assert result["aggregate"]["recorded_usage"] == {"input_tokens": 4, "output_tokens": 3, "calls": 2}+ assert result["aggregate"]["usage_observations"] == {"input_tokens": 1, "output_tokens": 1, "calls": 1}diff --git a/tools/evals/manifest.py b/tools/evals/manifest.pynew file mode 100644index 0000000..0d1abce--- /dev/null+++ b/tools/evals/manifest.py@@ -0,0 +1,196 @@+#!/usr/bin/env python3+"""Create deterministic, secret-free manifests for evaluation runs.++Usage: python3 tools/evals/manifest.py config.json [--output manifest.json]+"""+from __future__ import annotations++import argparse+import hashlib+import json+import math+import re+import subprocess+import sys+from pathlib import Path+from typing import Any, Dict, List, Optional++SCHEMA_VERSION = "pd-run-manifest/v1"+_REQUIRED = {+ "condition", "question_set_hash", "models", "harness_id", "prompt_hashes",+ "config_hashes", "seed", "repetition", "budgets", "artifacts", "results",+ "reason", "decision",+}+_SECRET_MARKERS = ("secret", "password", "credential", "authorization", "api_key", "private_key")+_SECRET_VALUE_PATTERNS = (+ re.compile(r"^Bearer\s+\S+$", re.IGNORECASE),+ re.compile(r"^sk-[A-Za-z0-9_-]{16,}$"),+ re.compile(r"^ghp_[A-Za-z0-9]{36}$"),+ re.compile(r"^AKIA[A-Z0-9]{16}$"),+ re.compile(r"^eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}$"),+ re.compile(r"^glpat-[A-Za-z0-9_-]{20}$"),+)+_SECRET_GUARDRAIL = "possible secret (guardrail only; not a DLP guarantee); store only identifiers or hashes"+++def canonical_json(value: Any) -> str:+ """Return the UTF-8-preserving canonical JSON representation of *value*."""+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)+++def sha256_file(path: Path) -> str:+ """Return the SHA-256 digest of a file's raw bytes."""+ digest = hashlib.sha256()+ with path.open("rb") as source:+ for block in iter(lambda: source.read(1024 * 1024), b""):+ digest.update(block)+ return digest.hexdigest()+++def _fail(message: str) -> None:+ raise ValueError(f"invalid run manifest config: {message}")+++def _reject_secrets(value: Any, location: str = "config") -> None:+ if isinstance(value, dict):+ for key, child in value.items():+ if not isinstance(key, str):+ _fail(f"{location} has a non-string key")+ if any(marker in key.lower() for marker in _SECRET_MARKERS):+ _fail(f"{location}.{key} {_SECRET_GUARDRAIL}")+ _reject_secrets(child, f"{location}.{key}")+ elif isinstance(value, list):+ for index, child in enumerate(value):+ _reject_secrets(child, f"{location}[{index}]")+ elif isinstance(value, str) and any(pattern.fullmatch(value) for pattern in _SECRET_VALUE_PATTERNS):+ _fail(f"{location} {_SECRET_GUARDRAIL}")+++def _identifier(value: Any, name: str) -> str:+ if not isinstance(value, str) or not value.strip():+ _fail(f"{name} must be a non-empty string")+ return value+++def _hashes(value: Any, name: str) -> Dict[str, str]:+ if not isinstance(value, dict) or not value:+ _fail(f"{name} must be a non-empty object of named hashes")+ return {str(key): _identifier(digest, f"{name}.{key}") for key, digest in value.items()}+++def _supportable(value: Any, name: str, minimum: int) -> Any:+ if value == "unsupported":+ return value+ if isinstance(value, bool) or not isinstance(value, int) or value < minimum:+ qualifier = "positive" if minimum else "non-negative"+ _fail(f"{name} must be {qualifier} integer or 'unsupported'")+ return value+++def _budgets(value: Any) -> Dict[str, Any]:+ if not isinstance(value, dict):+ _fail("budgets must be an object")+ required = ("max_calls", "max_input_tokens", "max_output_tokens")+ missing = [key for key in required if key not in value]+ if missing:+ _fail("budgets missing " + ", ".join(missing))+ result: Dict[str, Any] = {}+ for key in required:+ item = value[key]+ if isinstance(item, bool) or not isinstance(item, int) or item < 0:+ _fail(f"budgets.{key} must be a non-negative integer hard ceiling")+ result[key] = item+ if "max_cost_usd" in value:+ cost = value["max_cost_usd"]+ if isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0:+ _fail("budgets.max_cost_usd must be a non-negative finite number")+ result["max_cost_usd"] = cost+ return result+++def _commit(base_dir: Path) -> str:+ try:+ result = subprocess.run(+ ["git", "rev-parse", "HEAD"], cwd=base_dir, capture_output=True,+ text=True, encoding="utf-8", errors="replace", timeout=5, check=False,+ )+ except (OSError, subprocess.SubprocessError):+ return "unavailable"+ return result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else "unavailable"+++def build_manifest(config: Dict[str, Any], base_dir: Optional[Path] = None) -> Dict[str, Any]:+ """Validate *config*, hash its sources, and return a deterministic manifest."""+ if not isinstance(config, dict):+ _fail("top level must be an object")+ _reject_secrets(config)+ missing = sorted(_REQUIRED - set(config))+ if missing:+ _fail("missing required fields: " + ", ".join(missing))+ base_dir = (base_dir or Path.cwd()).resolve()+ sources = config.get("sources")+ if not isinstance(sources, list) or not sources:+ _fail("sources must be a non-empty list")+ source_records = []+ for index, source in enumerate(sources):+ if not isinstance(source, dict):+ _fail(f"sources[{index}] must be an object")+ source_id = _identifier(source.get("id"), f"sources[{index}].id")+ source_path = _identifier(source.get("path"), f"sources[{index}].path")+ resolved = (base_dir / source_path).resolve()+ if not resolved.is_file():+ _fail(f"sources[{index}].path is not a readable file: {source_path}")+ source_records.append({"id": source_id, "sha256": sha256_file(resolved)})+ source_records.sort(key=lambda record: (record["id"], record["sha256"]))++ models = config["models"]+ if not isinstance(models, dict) or not models:+ _fail("models must be a non-empty object")+ normalized_models = {str(key): _identifier(value, f"models.{key}") for key, value in models.items()}+ artifacts = _identifier(config["artifacts"], "artifacts")+ results = _identifier(config["results"], "results")+ manifest = {+ "schema_version": SCHEMA_VERSION,+ "sources": source_records,+ "condition": _identifier(config["condition"], "condition"),+ "question_set_hash": _identifier(config["question_set_hash"], "question_set_hash"),+ "models": normalized_models,+ "harness_id": _identifier(config["harness_id"], "harness_id"),+ "prompt_hashes": _hashes(config["prompt_hashes"], "prompt_hashes"),+ "config_hashes": _hashes(config["config_hashes"], "config_hashes"),+ "seed": _supportable(config["seed"], "seed", 0),+ "repetition": _supportable(config["repetition"], "repetition", 1),+ "budgets": _budgets(config["budgets"]),+ "commit": _commit(base_dir),+ "artifacts": artifacts,+ "results": results,+ "reason": _identifier(config["reason"], "reason"),+ "decision": _identifier(config["decision"], "decision"),+ }+ identity = {key: value for key, value in manifest.items() if key not in {"artifacts", "results"}}+ if manifest["commit"] == "unavailable":+ identity.pop("commit")+ manifest["run_id"] = hashlib.sha256(canonical_json(identity).encode("utf-8")).hexdigest()+ return manifest+++def main(argv: Optional[List[str]] = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("config", type=Path, help="evaluation run config JSON")+ parser.add_argument("--output", type=Path, help="write manifest JSON to this file")+ args = parser.parse_args(argv)+ try:+ with args.config.open(encoding="utf-8") as stream:+ config = json.load(stream)+ rendered = canonical_json(build_manifest(config, args.config.parent)) + "\n"+ if args.output:+ args.output.write_text(rendered, encoding="utf-8")+ else:+ sys.stdout.write(rendered)+ except (OSError, json.JSONDecodeError, ValueError) as exc:+ parser.error(str(exc))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tools/evals/paper_flat.py b/tools/evals/paper_flat.pynew file mode 100644index 0000000..96c92bb--- /dev/null+++ b/tools/evals/paper_flat.py@@ -0,0 +1,97 @@+#!/usr/bin/env python3+"""Build the deterministic, experiment-only PD-03 paper-flat skill pack.++Usage: python3 tools/evals/paper_flat.py SOURCE METADATA OUTPUT_DIR+"""+from __future__ import annotations++import argparse+import json+import re+from pathlib import Path+from typing import Any, Dict, List++_CHAPTER = re.compile(+ r"^(?:#{1,6}\s+.+|(?:chapter|chap\.)\s+(?:\d+|[ivxlcdm]+)\b.*)$", re.IGNORECASE+)+++def _chunks(source: str) -> List[str]:+ """Split at chapter headings, or return one whole-source fallback chunk."""+ lines = source.splitlines(keepends=True)+ starts = [index for index, line in enumerate(lines) if _CHAPTER.match(line.rstrip())]+ if not starts:+ return [source]+ return ["".join(lines[start:end]) for start, end in zip(starts, starts[1:] + [len(lines)])]+++def _metadata(path: Path, chunk_count: int) -> Dict[str, Any]:+ value = json.loads(path.read_text(encoding="utf-8"))+ if not isinstance(value, dict) or not all(isinstance(value.get(key), str) for key in ("title", "description")):+ raise ValueError("metadata requires string title and description")+ chunks = value.get("chunks")+ if not isinstance(chunks, list) or len(chunks) != chunk_count:+ raise ValueError("metadata chunks must match the source chunk count")+ for chunk in chunks:+ if not isinstance(chunk, dict) or not all(isinstance(chunk.get(key), str) for key in ("description", "summary")):+ raise ValueError("each metadata chunk requires string description and summary")+ if "key_elements" in chunk and (+ not isinstance(chunk["key_elements"], list) or not all(isinstance(item, str) for item in chunk["key_elements"])+ ):+ raise ValueError("key_elements must be a list of strings")+ return value+++def build_paper_flat(source_path: Path, metadata_path: Path, output_dir: Path) -> List[Path]:+ """Write a one-level paper-flat pack and return its files in stable order."""+ source = source_path.read_text(encoding="utf-8")+ chunks = _chunks(source)+ metadata = _metadata(metadata_path, len(chunks))+ chunk_dir = output_dir / "chunks"+ chunk_dir.mkdir(parents=True, exist_ok=True)++ rows = []+ written = []+ for number, (payload, details) in enumerate(zip(chunks, metadata["chunks"]), 1):+ relative = f"chunks/chunk-{number:02d}.md"+ path = output_dir / relative+ with path.open("w", encoding="utf-8", newline="\n") as file:+ file.write(payload)+ written.append(path)+ elements = ", ".join(details.get("key_elements", [])) or "—"+ rows.append(f"| `{relative}` | {details['description']} | {details['summary']} | {elements} |")++ root = "\n".join(+ [+ f"# {metadata['title']}",+ "",+ metadata["description"],+ "",+ "## Activation-time chunk table",+ "",+ "| Path | Description | SUMMARY | KEY_ELEMENTS |",+ "| --- | --- | --- | --- |",+ *rows,+ "",+ "Split policy: source chapter headings; without headings, one full-source chunk.",+ "",+ ]+ )+ root_path = output_dir / "SKILL.md"+ with root_path.open("w", encoding="utf-8", newline="\n") as file:+ file.write(root)+ return [root_path, *written]+++def main() -> int:+ parser = argparse.ArgumentParser(description="Build an experiment-only PD-03 paper-flat pack.")+ parser.add_argument("source", type=Path, help="UTF-8 synthetic source text")+ parser.add_argument("metadata", type=Path, help="fixed UTF-8 JSON metadata")+ parser.add_argument("output", type=Path, help="output pack directory")+ args = parser.parse_args()+ build_paper_flat(args.source, args.metadata, args.output)+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tools/evals/replay.py b/tools/evals/replay.pynew file mode 100644index 0000000..6bd09d0--- /dev/null+++ b/tools/evals/replay.py@@ -0,0 +1,45 @@+#!/usr/bin/env python3+"""Replay a versioned synthetic trajectory fixture deterministically."""+from __future__ import annotations++import argparse+import json+from pathlib import Path+from typing import Any, Dict++from score import score++SCHEMA_VERSION = "pd-replay-fixture/v1"+RESULT_SCHEMA_VERSION = "pd-replay-result/v1"+++def canonical_json(value: Any) -> str:+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)+++def load_fixture(path: Path) -> Dict[str, Any]:+ with path.open(encoding="utf-8") as source:+ fixture = json.load(source)+ if fixture.get("schema_version") != SCHEMA_VERSION:+ raise ValueError("unsupported fixture schema_version")+ trajectories = fixture.get("trajectories")+ if not isinstance(trajectories, list):+ raise ValueError("fixture trajectories must be a list")+ return fixture+++def replay(path: Path) -> Dict[str, Any]:+ fixture = load_fixture(path)+ result = score(fixture["trajectories"])+ return {"schema_version": RESULT_SCHEMA_VERSION, "fixture_version": fixture["fixture_version"], **result}+++def main() -> None:+ parser = argparse.ArgumentParser()+ parser.add_argument("fixture", type=Path)+ args = parser.parse_args()+ print(canonical_json(replay(args.fixture)))+++if __name__ == "__main__":+ main()diff --git a/tools/evals/score.py b/tools/evals/score.pynew file mode 100644index 0000000..b09ec42--- /dev/null+++ b/tools/evals/score.py@@ -0,0 +1,84 @@+#!/usr/bin/env python3+"""Pure scoring and accounting for synthetic evaluation trajectories."""+from __future__ import annotations++from typing import Any, Dict, Iterable, List++UNKNOWN = "unknown"+++def _state(value: Any) -> Any:+ """Keep only explicit boolean observations; absence remains unknown."""+ return value if isinstance(value, bool) else UNKNOWN+++def score_trajectory(trajectory: Dict[str, Any]) -> Dict[str, Any]:+ """Score one trajectory without loading files or deriving missing observations."""+ expected = trajectory.get("expected", {})+ observed = trajectory.get("observed", {})+ target = expected.get("target")+ opens = observed.get("opens")+ opened_target = (+ any(opened == target for opened in opens)+ if isinstance(opens, list) and target is not None+ else UNKNOWN+ )+ answer_correct = _state(observed.get("answer_correct"))+ route_correct = _state(observed.get("route_correct"))+ evidence_reached = _state(observed.get("evidence_reached"))+ if "route_correct" not in observed and opened_target is not UNKNOWN:+ route_correct = opened_target+ if "evidence_reached" not in observed and opened_target is not UNKNOWN:+ evidence_reached = opened_target++ if UNKNOWN in (route_correct, evidence_reached, answer_correct):+ classification = UNKNOWN+ elif not route_correct:+ classification = "wrong_routing"+ elif not answer_correct:+ classification = "wrong_answer"+ elif isinstance(opens, list) and target is not None and opens.index(target) > 0:+ classification = "irrelevant_opens_before_target"+ else:+ classification = "correct"++ usage = observed.get("usage")+ usage = usage if isinstance(usage, dict) else {}+ return {+ "question_id": trajectory["question_id"],+ "classification": classification,+ "routing_correct": route_correct,+ "evidence_reached": evidence_reached,+ "answer_correct": answer_correct,+ "usage": {key: usage.get(key) if isinstance(usage.get(key), int) else UNKNOWN+ for key in ("input_tokens", "output_tokens", "calls")},+ }+++def aggregate(results: Iterable[Dict[str, Any]]) -> Dict[str, Any]:+ """Aggregate recorded usage and classifications; never estimate missing usage."""+ items = list(results)+ counts: Dict[str, int] = {}+ totals = {"input_tokens": 0, "output_tokens": 0, "calls": 0}+ recorded = {key: 0 for key in totals}+ for result in items:+ label = result["classification"]+ counts[label] = counts.get(label, 0) + 1+ for key in totals:+ value = result["usage"][key]+ if value != UNKNOWN:+ totals[key] += value+ recorded[key] += 1+ return {+ "questions": len(items),+ "classifications": counts,+ "recorded_usage": totals,+ "usage_observations": recorded,+ }+++def score(trajectories: Iterable[Dict[str, Any]]) -> Dict[str, Any]:+ """Return deterministic per-question and aggregate replay results."""+ questions: List[Dict[str, Any]] = [score_trajectory(item) for item in trajectories]+ questions.sort(key=lambda item: item["question_id"])+ return {"questions": questions, "aggregate": aggregate(questions)}