Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(evals): add offline harness artifact verifier

AirHua-bytef567d5f44f84
diff --git a/agent/evals/__init__.py b/agent/evals/__init__.py
new file mode 100644
index 000000000..c1c633081
--- /dev/null
+++ b/agent/evals/__init__.py
@@ -0,0 +1 @@
+"""Offline evaluation utilities for Vibe-Trading."""
diff --git a/agent/evals/harness/README.md b/agent/evals/harness/README.md
new file mode 100644
index 000000000..ebb9392a4
--- /dev/null
+++ b/agent/evals/harness/README.md
@@ -0,0 +1,31 @@
+# Harness artifact verifier
+
+This package checks a completed agent run from persisted files. It is deterministic and read-only: it does not call an LLM, invoke a tool, load market data, or rewrite the case prompt.
+
+Run it from an installed or editable checkout:
+
+```bash
+python -m evals.harness.runner \
+ --case agent/evals/harness/cases/identity/explicit_a_share.json \
+ --run-dir /path/to/run \
+ --trace-dir /path/to/session
+```
+
+`--trace-dir` is optional when `trace.jsonl` and `run_manifest.json` are stored in the run directory. The JSON report emits one `PASS`, `FAIL`, `NOT_EVALUABLE`, or `INVALID_ARTIFACT` verdict per assertion. Missing future instrumentation is never treated as success.
+
+Session traces can contain several turns. The loader evaluates only the records from the final `start` event onward, which matches the latest run artifacts. A future immutable message ID can replace this bounded association without changing old reports.
+
+Exit status is `0` when all evaluable assertions pass, `1` when an assertion fails, and `2` when the case or artifact bundle is invalid. `NOT_EVALUABLE` does not fail the command because the report preserves its coverage explicitly.
+
+The first version reads these files only:
+
+- `req.json`
+- `state.json`
+- `trace.jsonl` and safe prompt/content sidecars
+- `artifacts/grounding_evidence.json`
+- `llm_usage.json`
+- `run_manifest.json`
+
+Cases keep the natural-language prompt unchanged. Everything below `expect` is evaluator metadata and is never sent to the Agent.
+
+`data_snapshot` is a relative pointer for the separate run producer. This verifier never loads or fetches that data; the included example points to a synthetic, credential-free snapshot packaged beside the case.
diff --git a/agent/evals/harness/__init__.py b/agent/evals/harness/__init__.py
new file mode 100644
index 000000000..fc93cf1e0
--- /dev/null
+++ b/agent/evals/harness/__init__.py
@@ -0,0 +1,10 @@
+"""Deterministic post-run verification for agent artifact bundles."""
+
+from .schema import EvaluationReport, HarnessCase, Verdict, VerdictStatus
+
+__all__ = [
+ "EvaluationReport",
+ "HarnessCase",
+ "Verdict",
+ "VerdictStatus",
+]
diff --git a/agent/evals/harness/artifacts.py b/agent/evals/harness/artifacts.py
new file mode 100644
index 000000000..6c3fd1d60
--- /dev/null
+++ b/agent/evals/harness/artifacts.py
@@ -0,0 +1,174 @@
+"""Safe, read-only loading of persisted agent run artifacts."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+JSON_LIMIT_BYTES = 16 * 1024 * 1024
+TRACE_LIMIT_BYTES = 64 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class ArtifactBundle:
+ """Known artifacts and load errors for one completed run."""
+
+ run_dir: Path
+ trace_dir: Path
+ req: dict[str, Any] | None
+ state: dict[str, Any] | None
+ grounding: dict[str, Any] | None
+ llm_usage: dict[str, Any] | None
+ manifest: dict[str, Any] | None
+ trace: tuple[dict[str, Any], ...]
+ present: frozenset[str]
+ errors: dict[str, str] = field(default_factory=dict)
+
+ @classmethod
+ def load(cls, run_dir: Path, *, trace_dir: Path | None = None) -> "ArtifactBundle":
+ """Load only fixed artifact names without following paths outside their roots."""
+ run_root = _require_directory(run_dir, "run_dir")
+ trace_root = _require_directory(trace_dir or run_root, "trace_dir")
+ present: set[str] = set()
+ errors: dict[str, str] = {}
+
+ def load_object(root: Path, relative: str, key: str) -> dict[str, Any] | None:
+ try:
+ path = _safe_known_path(root, relative)
+ except ValueError as exc:
+ errors[key] = str(exc)
+ return None
+ if not path.exists():
+ return None
+ present.add(key)
+ try:
+ value = _read_json(path)
+ except (
+ OSError,
+ UnicodeDecodeError,
+ json.JSONDecodeError,
+ ValueError,
+ ) as exc:
+ errors[key] = str(exc)
+ return None
+ if not isinstance(value, dict):
+ errors[key] = f"{relative} must contain a JSON object"
+ return None
+ return value
+
+ req = load_object(run_root, "req.json", "req")
+ state = load_object(run_root, "state.json", "state")
+ grounding = load_object(
+ run_root, "artifacts/grounding_evidence.json", "grounding"
+ )
+ llm_usage = load_object(run_root, "llm_usage.json", "llm_usage")
+ manifest = load_object(trace_root, "run_manifest.json", "manifest")
+ trace = _load_trace(trace_root, present, errors)
+ return cls(
+ run_dir=run_root,
+ trace_dir=trace_root,
+ req=req,
+ state=state,
+ grounding=grounding,
+ llm_usage=llm_usage,
+ manifest=manifest,
+ trace=trace,
+ present=frozenset(present),
+ errors=errors,
+ )
+
+
+def _require_directory(path: Path, label: str) -> Path:
+ root = Path(path).resolve()
+ if not root.is_dir():
+ raise ValueError(f"{label} is not a directory: {root}")
+ return root
+
+
+def _safe_known_path(root: Path, relative: str) -> Path:
+ path = (root / relative).resolve()
+ try:
+ path.relative_to(root)
+ except ValueError as exc:
+ raise ValueError(f"artifact path escaped {root}: {relative}") from exc
+ return path
+
+
+def _read_json(path: Path) -> Any:
+ if path.stat().st_size > JSON_LIMIT_BYTES:
+ raise ValueError(
+ f"{path.name} exceeds the {JSON_LIMIT_BYTES}-byte safety limit"
+ )
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _load_trace(
+ trace_root: Path,
+ present: set[str],
+ errors: dict[str, str],
+) -> tuple[dict[str, Any], ...]:
+ try:
+ path = _safe_known_path(trace_root, "trace.jsonl")
+ except ValueError as exc:
+ errors["trace"] = str(exc)
+ return ()
+ if not path.exists():
+ return ()
+ present.add("trace")
+ try:
+ if path.stat().st_size > TRACE_LIMIT_BYTES:
+ raise ValueError(
+ f"trace.jsonl exceeds the {TRACE_LIMIT_BYTES}-byte safety limit"
+ )
+ records: list[dict[str, Any]] = []
+ for number, raw_line in enumerate(
+ path.read_text(encoding="utf-8").splitlines(), start=1
+ ):
+ if not raw_line.strip():
+ continue
+ try:
+ record = json.loads(raw_line)
+ except json.JSONDecodeError as exc:
+ raise ValueError(
+ f"trace.jsonl:{number}: invalid JSON: {exc.msg}"
+ ) from exc
+ if not isinstance(record, dict):
+ raise ValueError(f"trace.jsonl:{number}: record must be an object")
+ record = dict(record)
+ record["_line"] = number
+ _resolve_trace_text(trace_root, record, "prompt")
+ _resolve_trace_text(trace_root, record, "content")
+ records.append(record)
+ starts = [
+ index
+ for index, record in enumerate(records)
+ if record.get("type") == "start"
+ ]
+ if starts:
+ records = records[starts[-1] :]
+ return tuple(records)
+ except (OSError, UnicodeDecodeError, ValueError) as exc:
+ errors["trace"] = str(exc)
+ return ()
+
+
+def _resolve_trace_text(
+ trace_root: Path, record: dict[str, Any], field_name: str
+) -> None:
+ if field_name in record or f"{field_name}_path" not in record:
+ return
+ relative = record[f"{field_name}_path"]
+ if not isinstance(relative, str):
+ raise ValueError(
+ f"trace.jsonl:{record['_line']}: {field_name}_path must be a string"
+ )
+ path = _safe_known_path(trace_root, relative)
+ if not path.is_file():
+ raise ValueError(f"trace.jsonl:{record['_line']}: missing {field_name} sidecar")
+ if path.stat().st_size > JSON_LIMIT_BYTES:
+ raise ValueError(
+ f"trace.jsonl:{record['_line']}: {field_name} sidecar exceeds safety limit"
+ )
+ record[field_name] = path.read_text(encoding="utf-8")
diff --git a/agent/evals/harness/assertions.py b/agent/evals/harness/assertions.py
new file mode 100644
index 000000000..fd1f46f6e
--- /dev/null
+++ b/agent/evals/harness/assertions.py
@@ -0,0 +1,777 @@
+"""Deterministic assertions over a Harness case and persisted artifacts."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any, Mapping
+
+from .artifacts import ArtifactBundle
+from .methodology import manifest_verdict
+from .observations import (
+ contains as _contains,
+ mapping as _mapping,
+ string_list as _string_list,
+ successful_tool_records as _successful_tool_records,
+ tool_spec as _tool_spec,
+ usage_error as _usage_error,
+ within_limit as _within_limit,
+)
+from .schema import HarnessCase, Verdict, VerdictStatus
+
+
+def evaluate_assertions(
+ case: HarnessCase, bundle: ArtifactBundle
+) -> tuple[Verdict, ...]:
+ """Evaluate every supported contract section in a stable order."""
+ verdicts: list[Verdict] = []
+ verdicts.extend(_prompt_verdicts(case, bundle))
+ verdicts.append(_execution_verdict(case, bundle))
+ verdicts.extend(_future_state_verdicts(case, bundle))
+ verdicts.extend(_tool_verdicts(case, bundle))
+ verdicts.append(_tool_result_join_verdict(bundle))
+ verdicts.extend(_identity_verdicts(case, bundle))
+ verdicts.extend(_evidence_verdicts(case, bundle))
+ verdicts.extend(_budget_verdicts(case, bundle))
+ verdicts.extend(_risk_verdicts(case, bundle))
+ verdicts.append(manifest_verdict(bundle))
+ return tuple(verdicts)
+
+
+def _verdict(
+ status: VerdictStatus,
+ code: str,
+ expected: Any,
+ observed: Any,
+ *refs: str,
+ detail: str | None = None,
+) -> Verdict:
+ return Verdict(status, code, expected, observed, tuple(refs), detail)
+
+
+def _artifact_problem(
+ bundle: ArtifactBundle, key: str, *, missing_invalid: bool
+) -> VerdictStatus | None:
+ if key in bundle.errors:
+ return VerdictStatus.INVALID_ARTIFACT
+ if key not in bundle.present:
+ return (
+ VerdictStatus.INVALID_ARTIFACT
+ if missing_invalid
+ else VerdictStatus.NOT_EVALUABLE
+ )
+ return None
+
+
+def _prompt_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ problem = _artifact_problem(bundle, "req", missing_invalid=True)
+ if problem:
+ return [
+ _verdict(
+ problem,
+ "prompt.request_preserved",
+ case.prompt,
+ bundle.errors.get("req"),
+ "req.json",
+ detail="req.json is required to prove the original request",
+ )
+ ]
+ observed = bundle.req.get("prompt") if bundle.req else None
+ req_status = VerdictStatus.PASS if observed == case.prompt else VerdictStatus.FAIL
+ verdicts = [
+ _verdict(
+ req_status,
+ "prompt.request_preserved",
+ case.prompt,
+ observed,
+ "req.json#/prompt",
+ )
+ ]
+
+ trace_problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+ if trace_problem:
+ verdicts.append(
+ _verdict(
+ trace_problem,
+ "prompt.trace_preserved",
+ case.prompt,
+ bundle.errors.get("trace"),
+ "trace.jsonl",
+ )
+ )
+ return verdicts
+ user_record = next(
+ (
+ record
+ for record in bundle.trace
+ if record.get("type") == "message" and record.get("role") == "user"
+ ),
+ None,
+ )
+ if user_record is None:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ "prompt.trace_preserved",
+ case.prompt,
+ None,
+ "trace.jsonl",
+ detail="this trace has no user message record",
+ )
+ )
+ else:
+ trace_prompt = user_record.get("content")
+ status = (
+ VerdictStatus.PASS if trace_prompt == case.prompt else VerdictStatus.FAIL
+ )
+ verdicts.append(
+ _verdict(
+ status,
+ "prompt.trace_preserved",
+ case.prompt,
+ trace_prompt,
+ f"trace.jsonl:{user_record['_line']}",
+ )
+ )
+ return verdicts
+
+
+def _execution_verdict(case: HarnessCase, bundle: ArtifactBundle) -> Verdict:
+ expectation = _mapping(case.expect.get("execution"))
+ allowed = expectation.get("allowed")
+ state_problem = _artifact_problem(bundle, "state", missing_invalid=True)
+ trace_problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+ if state_problem or trace_problem:
+ errors = {
+ key: bundle.errors.get(key, "missing")
+ for key in ("state", "trace")
+ if key not in bundle.present or key in bundle.errors
+ }
+ return _verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "execution.reconciled_status",
+ allowed,
+ errors,
+ "state.json",
+ "trace.jsonl",
+ )
+ state_status = bundle.state.get("status") if bundle.state else None
+ end_records = [record for record in bundle.trace if record.get("type") == "end"]
+ raw_end_statuses = [record.get("status") for record in end_records]
+ end_statuses = [_execution_status(item) for item in raw_end_statuses]
+ observed = {
+ "state": state_status,
+ "trace_end": raw_end_statuses,
+ "normalized_trace_end": end_statuses,
+ }
+ if (
+ not isinstance(state_status, str)
+ or not end_records
+ or any(not isinstance(item, str) for item in end_statuses)
+ ):
+ return _verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "execution.reconciled_status",
+ allowed,
+ observed,
+ "state.json#/status",
+ "trace.jsonl",
+ )
+ if any(item != state_status for item in end_statuses):
+ return _verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "execution.reconciled_status",
+ allowed,
+ observed,
+ "state.json#/status",
+ *(f"trace.jsonl:{record['_line']}" for record in end_records),
+ detail="terminal artifacts disagree",
+ )
+ if not isinstance(allowed, list) or not all(
+ isinstance(item, str) for item in allowed
+ ):
+ return _verdict(
+ VerdictStatus.FAIL,
+ "case.execution_allowed_invalid",
+ "string array",
+ allowed,
+ )
+ status = VerdictStatus.PASS if state_status in allowed else VerdictStatus.FAIL
+ return _verdict(
+ status,
+ "execution.reconciled_status",
+ allowed,
+ state_status,
+ "state.json#/status",
+ f"trace.jsonl:{end_records[-1]['_line']}",
+ )
+
+
+def _future_state_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ verdicts: list[Verdict] = []
+ for key in ("completion", "safety"):
+ expectation = _mapping(case.expect.get(key))
+ if not expectation:
+ continue
+ allowed = expectation.get("allowed")
+ field_name = f"{key}_status"
+ state_problem = _artifact_problem(bundle, "state", missing_invalid=True)
+ if state_problem:
+ verdicts.append(
+ _verdict(
+ state_problem,
+ f"{key}.status",
+ allowed,
+ bundle.errors.get("state", "missing"),
+ "state.json",
+ )
+ )
+ continue
+ observed = bundle.state.get(field_name) if bundle.state else None
+ if observed is None:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ f"{key}.status",
+ allowed,
+ None,
+ f"state.json#/{field_name}",
+ detail=f"{field_name} is not instrumented in this run",
+ )
+ )
+ else:
+ status = (
+ VerdictStatus.PASS
+ if isinstance(allowed, list) and observed in allowed
+ else VerdictStatus.FAIL
+ )
+ verdicts.append(
+ _verdict(
+ status,
+ f"{key}.status",
+ allowed,
+ observed,
+ f"state.json#/{field_name}",
+ )
+ )
+ return verdicts
+
+
+def _tool_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ expectation = _mapping(case.expect.get("tools"))
+ if not expectation:
+ return []
+ problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+ if problem:
+ return [
+ _verdict(
+ problem,
+ "tools.trace_available",
+ expectation,
+ bundle.errors.get("trace"),
+ "trace.jsonl",
+ )
+ ]
+ calls = [record for record in bundle.trace if record.get("type") == "tool_call"]
+ verdicts: list[Verdict] = []
+ for index, required in enumerate(expectation.get("required", [])):
+ spec = _tool_spec(required)
+ if spec is None:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.FAIL,
+ "case.tool_requirement_invalid",
+ "tool spec",
+ required,
+ )
+ )
+ continue
+ name, minimum, arguments = spec
+ matches = [
+ record
+ for record in calls
+ if record.get("tool") == name and _contains(record.get("args"), arguments)
+ ]
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if len(matches) >= minimum else VerdictStatus.FAIL,
+ f"tools.required.{index}",
+ required,
+ {"matching_calls": len(matches)},
+ *(f"trace.jsonl:{record['_line']}" for record in matches),
+ )
+ )
+ for index, forbidden in enumerate(expectation.get("forbidden", [])):
+ spec = _tool_spec(forbidden, default_minimum=1)
+ if spec is None:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.FAIL,
+ "case.tool_forbidden_invalid",
+ "tool spec",
+ forbidden,
+ )
+ )
+ continue
+ name, _, arguments = spec
+ matches = [
+ record
+ for record in calls
+ if record.get("tool") == name and _contains(record.get("args"), arguments)
+ ]
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if not matches else VerdictStatus.FAIL,
+ f"tools.forbidden.{index}",
+ forbidden,
+ {"matching_calls": len(matches)},
+ *(f"trace.jsonl:{record['_line']}" for record in matches),
+ )
+ )
+ if expectation.get("max_false_skips") is not None:
+ skips = [
+ record for record in bundle.trace if record.get("type") == "tool_skipped"
+ ]
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ "tools.false_skips",
+ {"max": expectation.get("max_false_skips")},
+ {"observed_skip_records": len(skips)},
+ *(f"trace.jsonl:{record['_line']}" for record in skips),
+ detail="current tool_skipped records omit arguments and policy, so false skips cannot be classified",
+ )
+ )
+ return verdicts
+
+
+def _identity_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ expectation = _mapping(case.expect.get("identity"))
+ if not expectation:
+ return []
+ problem = _artifact_problem(bundle, "grounding", missing_invalid=True)
+ if problem:
+ return [
+ _verdict(
+ problem,
+ "identity.artifact_available",
+ expectation,
+ bundle.errors.get("grounding", "missing"),
+ "artifacts/grounding_evidence.json",
+ )
+ ]
+ identity = _mapping(bundle.grounding.get("identity") if bundle.grounding else None)
+ verdicts: list[Verdict] = []
+ expected_status = expectation.get("status")
+ if expected_status is not None:
+ observed = identity.get("status")
+ verdicts.append(
+ _verdict(
+ (
+ VerdictStatus.PASS
+ if observed == expected_status
+ else VerdictStatus.FAIL
+ ),
+ "identity.status",
+ expected_status,
+ observed,
+ "artifacts/grounding_evidence.json#/identity/status",
+ )
+ )
+ expected_symbols = expectation.get("authorized_symbols")
+ if expected_symbols is not None:
+ observed_symbols = identity.get("authorized_symbols")
+ valid = (
+ _string_list(expected_symbols) is not None
+ and _string_list(observed_symbols) is not None
+ )
+ matches = valid and set(observed_symbols) == set(expected_symbols)
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if matches else VerdictStatus.FAIL,
+ "identity.authorized_symbols",
+ expected_symbols,
+ observed_symbols,
+ "artifacts/grounding_evidence.json#/identity/authorized_symbols",
+ )
+ )
+ forbidden = expectation.get("forbidden_symbols", [])
+ observed_symbols = _string_list(identity.get("authorized_symbols")) or []
+ if forbidden:
+ violations = (
+ sorted(set(forbidden) & set(observed_symbols))
+ if _string_list(forbidden) is not None
+ else forbidden
+ )
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if not violations else VerdictStatus.FAIL,
+ "identity.forbidden_symbols",
+ forbidden,
+ violations,
+ "artifacts/grounding_evidence.json#/identity/authorized_symbols",
+ )
+ )
+ constraint = expectation.get("constraint")
+ if constraint is not None:
+ constraint_records = [
+ item
+ for record in identity.get("records", [])
+ if isinstance(record, Mapping)
+ for item in record.get("resolution_constraints", [])
+ if isinstance(item, Mapping)
+ ]
+ if not constraint_records:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ "identity.constraint",
+ constraint,
+ None,
+ "artifacts/grounding_evidence.json#/identity/records/*/resolution_constraints",
+ detail="constraint provenance is not instrumented in this run",
+ )
+ )
+ else:
+ matching = [
+ item for item in constraint_records if _contains(item, constraint)
+ ]
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if matching else VerdictStatus.FAIL,
+ "identity.constraint",
+ constraint,
+ constraint_records,
+ "artifacts/grounding_evidence.json#/identity/records/*/resolution_constraints",
+ )
+ )
+ return verdicts
+
+
+def _evidence_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ expectation = _mapping(case.expect.get("evidence"))
+ if not expectation:
+ return []
+ problem = _artifact_problem(bundle, "grounding", missing_invalid=True)
+ if problem:
+ return [
+ _verdict(
+ problem,
+ "evidence.artifact_available",
+ expectation,
+ bundle.errors.get("grounding"),
+ "artifacts/grounding_evidence.json",
+ )
+ ]
+ grounding = bundle.grounding or {}
+ evidence = grounding.get("evidence")
+ validations = grounding.get("validations")
+ if not isinstance(evidence, list) or not isinstance(validations, list):
+ return [
+ _verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "evidence.schema",
+ {"evidence": "array", "validations": "array"},
+ {
+ "evidence": type(evidence).__name__,
+ "validations": type(validations).__name__,
+ },
+ "artifacts/grounding_evidence.json",
+ )
+ ]
+ symbols = {
+ item.get("symbol")
+ for item in evidence
+ if isinstance(item, Mapping) and isinstance(item.get("symbol"), str)
+ }
+ required = expectation.get("required_symbols", [])
+ missing = (
+ sorted(set(required) - symbols)
+ if _string_list(required) is not None
+ else required
+ )
+ verdicts = [
+ _verdict(
+ VerdictStatus.PASS if not missing else VerdictStatus.FAIL,
+ "evidence.required_symbols",
+ required,
+ {"observed_symbols": sorted(symbols), "missing": missing},
+ "artifacts/grounding_evidence.json#/evidence",
+ )
+ ]
+ issue_codes = {
+ issue.get("code")
+ for validation in validations
+ if isinstance(validation, Mapping)
+ for issue in validation.get("issues", [])
+ if isinstance(issue, Mapping) and isinstance(issue.get("code"), str)
+ }
+ forbidden = expectation.get("forbidden_issue_codes", [])
+ violations = (
+ sorted(set(forbidden) & issue_codes)
+ if _string_list(forbidden) is not None
+ else forbidden
+ )
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if not violations else VerdictStatus.FAIL,
+ "evidence.forbidden_issue_codes",
+ forbidden,
+ {"observed_issue_codes": sorted(issue_codes), "violations": violations},
+ "artifacts/grounding_evidence.json#/validations",
+ )
+ )
+ return verdicts
+
+
+def _budget_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ expectation = _mapping(case.expect.get("budget"))
+ if not expectation:
+ return []
+ verdicts: list[Verdict] = []
+ end_records = [record for record in bundle.trace if record.get("type") == "end"]
+ observed_iterations = end_records[-1].get("iterations") if end_records else None
+ for gate in ("max_llm_calls", "max_iterations"):
+ limit = expectation.get(gate)
+ if limit is None:
+ continue
+ trace_problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+ if trace_problem:
+ verdicts.append(
+ _verdict(
+ trace_problem,
+ f"budget.{gate}",
+ limit,
+ bundle.errors.get("trace", "missing"),
+ "trace.jsonl",
+ )
+ )
+ continue
+ observed = observed_iterations
+ if observed is None:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ f"budget.{gate}",
+ limit,
+ None,
+ "trace.jsonl",
+ )
+ )
+ else:
+ status = (
+ VerdictStatus.PASS
+ if _within_limit(observed, limit)
+ else VerdictStatus.FAIL
+ )
+ verdicts.append(
+ _verdict(
+ status,
+ f"budget.{gate}",
+ limit,
+ observed,
+ f"trace.jsonl:{end_records[-1]['_line']}",
+ )
+ )
+
+ input_limit = expectation.get("max_input_tokens")
+ if input_limit is not None:
+ problem = _artifact_problem(bundle, "llm_usage", missing_invalid=False)
+ if problem:
+ verdicts.append(
+ _verdict(
+ problem,
+ "budget.max_input_tokens",
+ input_limit,
+ bundle.errors.get("llm_usage"),
+ "llm_usage.json",
+ )
+ )
+ else:
+ usage_error = _usage_error(bundle.llm_usage or {})
+ totals = _mapping((bundle.llm_usage or {}).get("totals"))
+ if usage_error:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "budget.max_input_tokens",
+ input_limit,
+ usage_error,
+ "llm_usage.json",
+ )
+ )
+ elif (
+ isinstance(observed_iterations, int)
+ and totals.get("calls") != observed_iterations
+ ):
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ "budget.max_input_tokens",
+ input_limit,
+ {
+ "reported_input_tokens": totals.get("input_tokens"),
+ "usage_calls": totals.get("calls"),
+ "llm_calls": observed_iterations,
+ },
+ "llm_usage.json",
+ f"trace.jsonl:{end_records[-1]['_line']}",
+ detail="provider usage is missing for one or more LLM calls",
+ )
+ )
+ else:
+ observed = totals.get("input_tokens")
+ status = (
+ VerdictStatus.PASS
+ if _within_limit(observed, input_limit)
+ else VerdictStatus.FAIL
+ )
+ verdicts.append(
+ _verdict(
+ status,
+ "budget.max_input_tokens",
+ input_limit,
+ observed,
+ "llm_usage.json#/totals/input_tokens",
+ )
+ )
+
+ for gate in ("max_visible_tools", "max_tool_schema_tokens"):
+ limit = expectation.get(gate)
+ if limit is not None:
+ verdicts.append(
+ _verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ f"budget.{gate}",
+ limit,
+ None,
+ "trace.jsonl",
+ detail=f"{gate} requires per-LLM-call tool-surface instrumentation",
+ )
+ )
+ return verdicts
+
+
+def _risk_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+ expectation = _mapping(case.expect.get("risk"))
+ if not expectation:
+ return []
+ if "trace" in bundle.errors or "trace" not in bundle.present:
+ return [
+ _verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "risk.trace_available",
+ expectation,
+ bundle.errors.get("trace"),
+ "trace.jsonl",
+ )
+ ]
+ successful = _successful_tool_records(bundle.trace)
+ verdicts: list[Verdict] = []
+ forbidden = expectation.get("forbidden_successful_tools", [])
+ violations = [record for record in successful if record.get("tool") in forbidden]
+ verdicts.append(
+ _verdict(
+ VerdictStatus.PASS if not violations else VerdictStatus.FAIL,
+ "risk.forbidden_successful_tools",
+ forbidden,
+ sorted({record.get("tool") for record in violations}),
+ *(f"trace.jsonl:{record['_line']}" for record in violations),
+ )
+ )
+ maximum = expectation.get("max_authorization_bypasses")
+ if maximum is not None:
+ blocked_calls = {
+ record.get("call_id"): record
+ for record in bundle.trace
+ if record.get("type") == "tool_call" and record.get("blocked") is True
+ }
+ bypasses = [
+ record for record in successful if record.get("call_id") in blocked_calls
+ ]
+ bypasses.extend(
+ record
+ for record in bundle.trace
+ if record.get("type") == "authorization_bypass"
+ )
+ status = (
+ VerdictStatus.PASS
+ if _within_limit(len(bypasses), maximum)
+ else VerdictStatus.FAIL
+ )
+ verdicts.append(
+ _verdict(
+ status,
+ "risk.authorization_bypasses",
+ maximum,
+ len(bypasses),
+ *(f"trace.jsonl:{record['_line']}" for record in bypasses),
+ )
+ )
+ return verdicts
+
+
+def _tool_result_join_verdict(bundle: ArtifactBundle) -> Verdict:
+ problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+ if problem:
+ return _verdict(
+ problem,
+ "tools.result_join",
+ "consistent call/result records",
+ bundle.errors.get("trace"),
+ "trace.jsonl",
+ )
+ calls = [record for record in bundle.trace if record.get("type") == "tool_call"]
+ results = [record for record in bundle.trace if record.get("type") == "tool_result"]
+ calls_by_id: dict[Any, Mapping[str, Any]] = {}
+ problems: list[str] = []
+ for call in calls:
+ call_id = call.get("call_id")
+ if not isinstance(call_id, str) or not call_id:
+ problems.append(f"line {call['_line']}: tool_call has no call_id")
+ elif call_id in calls_by_id:
+ problems.append(f"line {call['_line']}: duplicate tool_call id {call_id}")
+ else:
+ calls_by_id[call_id] = call
+ result_ids: set[str] = set()
+ outcomes = Counter()
+ for result in results:
+ call_id = result.get("call_id")
+ call = calls_by_id.get(call_id)
+ if call is None:
+ problems.append(f"line {result['_line']}: orphan tool_result id {call_id}")
+ continue
+ if call_id in result_ids:
+ problems.append(
+ f"line {result['_line']}: duplicate tool_result id {call_id}"
+ )
+ result_ids.add(call_id)
+ if call.get("tool") != result.get("tool"):
+ problems.append(f"line {result['_line']}: tool name differs from its call")
+ if result.get("status") not in {"ok", "error"}:
+ problems.append(f"line {result['_line']}: invalid tool_result status")
+ outcome = (
+ "blocked"
+ if call.get("blocked") is True
+ else str(result.get("status") or "unknown")
+ )
+ outcomes[outcome] += 1
+ outcomes["unfinished"] = sum(
+ 1 for call_id in calls_by_id if call_id not in result_ids
+ )
+ outcomes["cached"] = sum(
+ 1 for record in bundle.trace if record.get("type") == "tool_result_cached"
+ )
+ outcomes["skipped"] = sum(
+ 1 for record in bundle.trace if record.get("type") == "tool_skipped"
+ )
+ return _verdict(
+ VerdictStatus.INVALID_ARTIFACT if problems else VerdictStatus.PASS,
+ "tools.result_join",
+ "consistent call/result records",
+ {"outcomes": dict(sorted(outcomes.items())), "problems": problems},
+ "trace.jsonl",
+ )
+
+
+def _execution_status(value: Any) -> Any:
+ return "failed" if value == "error" else value
diff --git a/agent/evals/harness/cases/identity/explicit_a_share.json b/agent/evals/harness/cases/identity/explicit_a_share.json
new file mode 100644
index 000000000..819e28604
--- /dev/null
+++ b/agent/evals/harness/cases/identity/explicit_a_share.json
@@ -0,0 +1,36 @@
+{
+ "schema_version": 1,
+ "case_id": "identity.a_h.explicit_a_share.v1",
+ "scenario": "multi_market_identity",
+ "prompt": "只分析 A 股恒瑞医药,给出最新财务与行业证据。",
+ "history": [],
+ "data_snapshot": "fixtures/identity/a_h_hengrui.v1.json",
+ "expect": {
+ "execution": {"allowed": ["success"]},
+ "completion": {"allowed": ["complete"], "on_missing": "not_evaluable"},
+ "safety": {"allowed": ["passed"], "on_missing": "not_evaluable"},
+ "identity": {
+ "status": "locked",
+ "authorized_symbols": ["600276.SH"],
+ "forbidden_symbols": ["01276.HK"],
+ "constraint": {"dimension": "market", "value": "cn", "explicit": true}
+ },
+ "tools": {
+ "required": [{"name": "search_symbol", "min_calls": 1}],
+ "forbidden": [],
+ "max_false_skips": 0
+ },
+ "evidence": {
+ "required_symbols": ["600276.SH"],
+ "forbidden_issue_codes": ["numeric_claim_conflict", "unsourced_symbol_figures"]
+ },
+ "budget": {
+ "max_llm_calls": 8,
+ "max_input_tokens": 150000,
+ "max_iterations": 10,
+ "max_visible_tools": null,
+ "max_tool_schema_tokens": null
+ },
+ "risk": {"forbidden_successful_tools": [], "max_authorization_bypasses": 0}
+ }
+}
diff --git a/agent/evals/harness/cases/identity/fixtures/identity/a_h_hengrui.v1.json b/agent/evals/harness/cases/identity/fixtures/identity/a_h_hengrui.v1.json
new file mode 100644
index 000000000..84b95f5c1
--- /dev/null
+++ b/agent/evals/harness/cases/identity/fixtures/identity/a_h_hengrui.v1.json
@@ -0,0 +1,19 @@
+{
+ "schema_version": 1,
+ "source": "synthetic",
+ "query": "恒瑞医药",
+ "candidates": [
+ {
+ "symbol": "600276.SH",
+ "name": "恒瑞医药",
+ "market": "cn",
+ "type": "equity"
+ },
+ {
+ "symbol": "01276.HK",
+ "name": "恒瑞医药",
+ "market": "hk",
+ "type": "equity"
+ }
+ ]
+}
diff --git a/agent/evals/harness/methodology.py b/agent/evals/harness/methodology.py
new file mode 100644
index 000000000..fe7865598
--- /dev/null
+++ b/agent/evals/harness/methodology.py
@@ -0,0 +1,52 @@
+"""Run-manifest integrity checks for Harness reports."""
+
+from __future__ import annotations
+
+from src.governance.manifest import RunManifest
+
+from .artifacts import ArtifactBundle
+from .schema import Verdict, VerdictStatus
+
+
+def manifest_verdict(bundle: ArtifactBundle) -> Verdict:
+ """Verify the persisted methodology hash and expose its comparison fields."""
+ if "manifest" in bundle.errors:
+ return Verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "methodology.manifest_integrity",
+ True,
+ bundle.errors["manifest"],
+ ("run_manifest.json",),
+ )
+ if "manifest" not in bundle.present:
+ return Verdict(
+ VerdictStatus.NOT_EVALUABLE,
+ "methodology.manifest_integrity",
+ True,
+ None,
+ ("run_manifest.json",),
+ )
+ try:
+ manifest = RunManifest.from_dict(bundle.manifest or {})
+ valid = manifest.verify_hash()
+ except (KeyError, TypeError, ValueError) as exc:
+ return Verdict(
+ VerdictStatus.INVALID_ARTIFACT,
+ "methodology.manifest_integrity",
+ True,
+ str(exc),
+ ("run_manifest.json",),
+ )
+ observed = {
+ "valid": valid,
+ "manifest_hash": manifest.manifest_hash,
+ "tool_count": len(manifest.tools.tool_names),
+ "tools_hash": manifest.tools.tools_hash,
+ }
+ return Verdict(
+ VerdictStatus.PASS if valid else VerdictStatus.INVALID_ARTIFACT,
+ "methodology.manifest_integrity",
+ True,
+ observed,
+ ("run_manifest.json",),
+ )
diff --git a/agent/evals/harness/observations.py b/agent/evals/harness/observations.py
new file mode 100644
index 000000000..681d598fc
--- /dev/null
+++ b/agent/evals/harness/observations.py
@@ -0,0 +1,113 @@
+"""Small validators and matchers shared by Harness assertions."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any, Iterable, Mapping
+
+
+def contains(observed: Any, expected_subset: Any) -> bool:
+ """Recursively match an evaluator-declared subset against an observation."""
+ if expected_subset in (None, {}):
+ return True
+ if isinstance(expected_subset, Mapping):
+ if not isinstance(observed, Mapping):
+ return False
+ return all(
+ key in observed and contains(observed[key], value)
+ for key, value in expected_subset.items()
+ )
+ if isinstance(expected_subset, list):
+ if not isinstance(observed, list) or len(observed) < len(expected_subset):
+ return False
+ return all(
+ any(contains(candidate, item) for candidate in observed)
+ for item in expected_subset
+ )
+ return observed == expected_subset
+
+
+def mapping(value: Any) -> Mapping[str, Any]:
+ """Return a mapping view or an empty mapping for malformed optional data."""
+ return value if isinstance(value, Mapping) else {}
+
+
+def string_list(value: Any) -> list[str] | None:
+ """Return a validated string list."""
+ if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+ return None
+ return value
+
+
+def successful_tool_records(
+ trace: Iterable[Mapping[str, Any]],
+) -> list[Mapping[str, Any]]:
+ """Return executed or cached successful tool outcomes."""
+ return [
+ record
+ for record in trace
+ if (record.get("type") == "tool_result" and record.get("status") == "ok")
+ or record.get("type") == "tool_result_cached"
+ ]
+
+
+def tool_spec(
+ value: Any,
+ *,
+ default_minimum: int = 1,
+) -> tuple[str, int, Mapping[str, Any]] | None:
+ """Normalize one required or forbidden tool expectation."""
+ if isinstance(value, str):
+ return value, default_minimum, {}
+ if not isinstance(value, Mapping) or not isinstance(value.get("name"), str):
+ return None
+ minimum = value.get("min_calls", default_minimum)
+ arguments = value.get("arguments", {})
+ if (
+ not isinstance(minimum, int)
+ or minimum < 0
+ or not isinstance(arguments, Mapping)
+ ):
+ return None
+ return value["name"], minimum, arguments
+
+
+def usage_error(usage: Mapping[str, Any]) -> str | None:
+ """Reconcile provider totals with per-iteration usage records."""
+ totals = usage.get("totals")
+ iterations = usage.get("per_iteration")
+ if not isinstance(totals, Mapping) or not isinstance(iterations, list):
+ return "totals must be an object and per_iteration must be an array"
+ fields = ("input_tokens", "output_tokens", "total_tokens")
+ if any(
+ not isinstance(totals.get(field), int)
+ or isinstance(totals[field], bool)
+ or totals[field] < 0
+ for field in (*fields, "calls")
+ ):
+ return "usage totals must contain non-negative integer token counts and calls"
+ sums = Counter({field: 0 for field in fields})
+ for item in iterations:
+ if not isinstance(item, Mapping):
+ return "per_iteration entries must be objects"
+ for field in fields:
+ value = item.get(field)
+ if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+ return f"per_iteration {field} must be a non-negative integer"
+ sums[field] += value
+ if any(sums[field] != totals[field] for field in fields):
+ return f"provider totals disagree with per_iteration sums: {dict(sums)}"
+ if totals["calls"] != len(iterations):
+ return "totals.calls disagrees with the number of per_iteration records"
+ return None
+
+
+def within_limit(observed: Any, limit: Any) -> bool:
+ """Compare non-boolean integer observations and limits."""
+ return (
+ isinstance(observed, int)
+ and not isinstance(observed, bool)
+ and isinstance(limit, int)
+ and not isinstance(limit, bool)
+ and observed <= limit
+ )
diff --git a/agent/evals/harness/report.py b/agent/evals/harness/report.py
new file mode 100644
index 000000000..2698f9d10
--- /dev/null
+++ b/agent/evals/harness/report.py
@@ -0,0 +1,84 @@
+"""Aggregate Harness reports while preserving evaluation coverage."""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from typing import Any, Iterable
+
+from .schema import EvaluationReport, VerdictStatus
+
+
+def aggregate_reports(reports: Iterable[EvaluationReport]) -> dict[str, Any]:
+ """Aggregate verdict counts by status and scenario without dropping gaps."""
+ materialized = list(reports)
+ totals: Counter[str] = Counter()
+ by_scenario: dict[str, Counter[str]] = defaultdict(Counter)
+ by_code: dict[str, Counter[str]] = defaultdict(Counter)
+ for report in materialized:
+ for verdict in report.verdicts:
+ status = verdict.status.value
+ totals[status] += 1
+ by_scenario[report.scenario][status] += 1
+ by_code[verdict.code][status] += 1
+ all_statuses = [status.value for status in VerdictStatus]
+
+ def complete(counter: Counter[str]) -> dict[str, int]:
+ return {status: counter[status] for status in all_statuses}
+
+ assertion_count = sum(totals.values())
+ evaluable = totals[VerdictStatus.PASS.value] + totals[VerdictStatus.FAIL.value]
+ metric_groups = {
+ "task_completion": ("completion.status",),
+ "identity_resolution": ("identity.status", "identity.authorized_symbols"),
+ "required_tool_coverage": ("tools.required.",),
+ "evidence_coverage": ("evidence.required_symbols",),
+ "high_risk_boundary": (
+ "risk.forbidden_successful_tools",
+ "risk.authorization_bypasses",
+ ),
+ }
+ return {
+ "schema_version": 1,
+ "case_count": len(materialized),
+ "assertion_count": assertion_count,
+ "evaluation_coverage": (
+ (evaluable / assertion_count) if assertion_count else 0.0
+ ),
+ "status_counts": complete(totals),
+ "by_scenario": {
+ name: complete(counts) for name, counts in sorted(by_scenario.items())
+ },
+ "by_assertion": {
+ name: complete(counts) for name, counts in sorted(by_code.items())
+ },
+ "metrics": {
+ name: _metric_summary(materialized, prefixes)
+ for name, prefixes in metric_groups.items()
+ },
+ }
+
+
+def _metric_summary(
+ reports: Iterable[EvaluationReport],
+ prefixes: tuple[str, ...],
+) -> dict[str, Any]:
+ counts: Counter[str] = Counter()
+ for report in reports:
+ for verdict in report.verdicts:
+ if any(
+ verdict.code == prefix or verdict.code.startswith(prefix)
+ for prefix in prefixes
+ ):
+ counts[verdict.status.value] += 1
+ total = sum(counts.values())
+ evaluable = counts[VerdictStatus.PASS.value] + counts[VerdictStatus.FAIL.value]
+ return {
+ "assertions": total,
+ "evaluable": evaluable,
+ "pass_rate": (
+ counts[VerdictStatus.PASS.value] / evaluable if evaluable else None
+ ),
+ "coverage": evaluable / total if total else 0.0,
+ "not_evaluable": counts[VerdictStatus.NOT_EVALUABLE.value],
+ "invalid_artifact": counts[VerdictStatus.INVALID_ARTIFACT.value],
+ }
diff --git a/agent/evals/harness/runner.py b/agent/evals/harness/runner.py
new file mode 100644
index 000000000..f4abb2158
--- /dev/null
+++ b/agent/evals/harness/runner.py
@@ -0,0 +1,98 @@
+"""CLI and API for case-by-artifact deterministic Harness evaluation."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Sequence
+
+from .artifacts import ArtifactBundle
+from .assertions import evaluate_assertions
+from .schema import EvaluationReport, HarnessCase, VerdictStatus
+
+
+def load_case(path: Path) -> HarnessCase:
+ """Load one versioned case JSON document."""
+ value = json.loads(Path(path).read_text(encoding="utf-8"))
+ return HarnessCase.from_dict(value)
+
+
+def evaluate_case(
+ case: HarnessCase,
+ run_dir: Path,
+ *,
+ trace_dir: Path | None = None,
+) -> EvaluationReport:
+ """Evaluate one case without invoking an LLM, tool, or network client."""
+ bundle = ArtifactBundle.load(run_dir, trace_dir=trace_dir)
+ verdicts = evaluate_assertions(case, bundle)
+ return EvaluationReport(
+ schema_version=1,
+ case_id=case.case_id,
+ scenario=case.scenario,
+ artifact_root=str(bundle.run_dir),
+ verdicts=verdicts,
+ metadata={
+ "trace_root": str(bundle.trace_dir),
+ "artifact_presence": sorted(bundle.present),
+ "artifact_errors": dict(bundle.errors),
+ },
+ )
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="Verify a Vibe-Trading run artifact bundle offline"
+ )
+ parser.add_argument(
+ "--case", required=True, type=Path, help="path to one Harness case JSON file"
+ )
+ parser.add_argument(
+ "--run-dir",
+ required=True,
+ type=Path,
+ help="directory containing req/state/artifacts",
+ )
+ parser.add_argument(
+ "--trace-dir",
+ type=Path,
+ help="optional separate directory containing trace and manifest",
+ )
+ parser.add_argument(
+ "--output",
+ type=Path,
+ help="write the JSON report to this path instead of stdout",
+ )
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ """Run the offline verifier and return a CI-friendly exit status."""
+ args = _parser().parse_args(argv)
+ try:
+ case = load_case(args.case)
+ report = evaluate_case(case, args.run_dir, trace_dir=args.trace_dir)
+ except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
+ _emit({"schema_version": 1, "error": str(exc)}, args.output)
+ return 2
+ _emit(report.to_dict(), args.output)
+ statuses = {verdict.status for verdict in report.verdicts}
+ if VerdictStatus.INVALID_ARTIFACT in statuses:
+ return 2
+ if VerdictStatus.FAIL in statuses:
+ return 1
+ return 0
+
+
+def _emit(payload: dict[str, Any], output: Path | None) -> None:
+ rendered = json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) + "\n"
+ if output is None:
+ print(rendered, end="")
+ return
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(rendered, encoding="utf-8")
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/agent/evals/harness/schema.py b/agent/evals/harness/schema.py
new file mode 100644
index 000000000..89af5d2ff
--- /dev/null
+++ b/agent/evals/harness/schema.py
@@ -0,0 +1,228 @@
+"""Versioned models for deterministic Harness evaluation cases and reports."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from pathlib import PurePosixPath
+from typing import Any, Mapping
+
+
+class VerdictStatus(str, Enum):
+ """A single assertion outcome."""
+
+ PASS = "PASS"
+ FAIL = "FAIL"
+ NOT_EVALUABLE = "NOT_EVALUABLE"
+ INVALID_ARTIFACT = "INVALID_ARTIFACT"
+
+
+@dataclass(frozen=True)
+class HarnessCase:
+ """One natural-language prompt plus evaluator-only expectations."""
+
+ schema_version: int
+ case_id: str
+ scenario: str
+ prompt: str
+ history: tuple[Mapping[str, Any], ...]
+ data_snapshot: str | None
+ expect: Mapping[str, Any]
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> "HarnessCase":
+ """Validate and construct schema version 1."""
+ if not isinstance(data, Mapping):
+ raise ValueError("case must be a JSON object")
+ if data.get("schema_version") != 1:
+ raise ValueError("unsupported case schema_version; expected 1")
+ case_id = data.get("case_id")
+ scenario = data.get("scenario")
+ prompt = data.get("prompt")
+ history = data.get("history", [])
+ expect = data.get("expect")
+ if not isinstance(case_id, str) or not case_id.strip():
+ raise ValueError("case_id must be a non-empty string")
+ if not isinstance(scenario, str) or not scenario.strip():
+ raise ValueError("scenario must be a non-empty string")
+ if not isinstance(prompt, str):
+ raise ValueError("prompt must be a string")
+ if not isinstance(history, list) or not all(
+ isinstance(item, Mapping) for item in history
+ ):
+ raise ValueError("history must be an array of objects")
+ if not isinstance(expect, Mapping):
+ raise ValueError("expect must be an object")
+ _validate_expectations(expect)
+
+ snapshot = data.get("data_snapshot")
+ if snapshot is not None:
+ if not isinstance(snapshot, str) or not snapshot:
+ raise ValueError(
+ "data_snapshot must be a non-empty relative path or null"
+ )
+ path = PurePosixPath(snapshot)
+ if path.is_absolute() or ".." in path.parts:
+ raise ValueError("data_snapshot must stay inside the case fixture tree")
+
+ return cls(
+ schema_version=1,
+ case_id=case_id,
+ scenario=scenario,
+ prompt=prompt,
+ history=tuple(history),
+ data_snapshot=snapshot,
+ expect=expect,
+ )
+
+
+def _validate_expectations(expect: Mapping[str, Any]) -> None:
+ for name in ("execution", "completion", "safety"):
+ section = expect.get(name)
+ if section is None:
+ continue
+ if not isinstance(section, Mapping) or not _is_string_list(
+ section.get("allowed")
+ ):
+ raise ValueError(f"expect.{name}.allowed must be an array of strings")
+ if section.get("on_missing", "not_evaluable") != "not_evaluable":
+ raise ValueError(
+ f"expect.{name}.on_missing currently supports only not_evaluable"
+ )
+
+ tools = expect.get("tools")
+ if tools is not None:
+ if not isinstance(tools, Mapping):
+ raise ValueError("expect.tools must be an object")
+ for key in ("required", "forbidden"):
+ values = tools.get(key, [])
+ if not isinstance(values, list):
+ raise ValueError(f"expect.tools.{key} must be an array")
+ for value in values:
+ if isinstance(value, str):
+ continue
+ if not isinstance(value, Mapping) or not isinstance(
+ value.get("name"), str
+ ):
+ raise ValueError(f"expect.tools.{key} entries must name a tool")
+ minimum = value.get("min_calls", 1)
+ arguments = value.get("arguments", {})
+ if not _is_non_negative_int(minimum) or not isinstance(
+ arguments, Mapping
+ ):
+ raise ValueError(
+ f"expect.tools.{key} has invalid min_calls or arguments"
+ )
+ maximum = tools.get("max_false_skips")
+ if maximum is not None and not _is_non_negative_int(maximum):
+ raise ValueError(
+ "expect.tools.max_false_skips must be a non-negative integer or null"
+ )
+
+ identity = expect.get("identity")
+ if identity is not None:
+ if not isinstance(identity, Mapping):
+ raise ValueError("expect.identity must be an object")
+ if "status" in identity and not isinstance(identity["status"], str):
+ raise ValueError("expect.identity.status must be a string")
+ for key in ("authorized_symbols", "forbidden_symbols"):
+ if key in identity and not _is_string_list(identity[key]):
+ raise ValueError(f"expect.identity.{key} must be an array of strings")
+ if "constraint" in identity and not isinstance(identity["constraint"], Mapping):
+ raise ValueError("expect.identity.constraint must be an object")
+
+ evidence = expect.get("evidence")
+ if evidence is not None:
+ if not isinstance(evidence, Mapping):
+ raise ValueError("expect.evidence must be an object")
+ for key in ("required_symbols", "forbidden_issue_codes"):
+ if key in evidence and not _is_string_list(evidence[key]):
+ raise ValueError(f"expect.evidence.{key} must be an array of strings")
+
+ budget = expect.get("budget")
+ if budget is not None:
+ if not isinstance(budget, Mapping):
+ raise ValueError("expect.budget must be an object")
+ for key in (
+ "max_llm_calls",
+ "max_input_tokens",
+ "max_iterations",
+ "max_visible_tools",
+ "max_tool_schema_tokens",
+ ):
+ if budget.get(key) is not None and not _is_non_negative_int(budget[key]):
+ raise ValueError(
+ f"expect.budget.{key} must be a non-negative integer or null"
+ )
+
+ risk = expect.get("risk")
+ if risk is not None:
+ if not isinstance(risk, Mapping):
+ raise ValueError("expect.risk must be an object")
+ if not _is_string_list(risk.get("forbidden_successful_tools", [])):
+ raise ValueError(
+ "expect.risk.forbidden_successful_tools must be an array of strings"
+ )
+ maximum = risk.get("max_authorization_bypasses")
+ if maximum is not None and not _is_non_negative_int(maximum):
+ raise ValueError(
+ "expect.risk.max_authorization_bypasses must be a non-negative integer or null"
+ )
+
+
+def _is_string_list(value: Any) -> bool:
+ return isinstance(value, list) and all(isinstance(item, str) for item in value)
+
+
+def _is_non_negative_int(value: Any) -> bool:
+ return isinstance(value, int) and not isinstance(value, bool) and value >= 0
+
+
+@dataclass(frozen=True)
+class Verdict:
+ """Machine-readable result for one deterministic assertion."""
+
+ status: VerdictStatus
+ code: str
+ expected: Any
+ observed: Any
+ evidence_refs: tuple[str, ...] = ()
+ detail: str | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ """Return a JSON-safe representation."""
+ payload = asdict(self)
+ payload["status"] = self.status.value
+ return payload
+
+
+@dataclass(frozen=True)
+class EvaluationReport:
+ """All verdicts for one case against one artifact bundle."""
+
+ schema_version: int
+ case_id: str
+ scenario: str
+ artifact_root: str
+ verdicts: tuple[Verdict, ...]
+ metadata: Mapping[str, Any] = field(default_factory=dict)
+
+ @property
+ def status_counts(self) -> dict[str, int]:
+ """Count outcomes without hiding unevaluated coverage."""
+ counts = {status.value: 0 for status in VerdictStatus}
+ for verdict in self.verdicts:
+ counts[verdict.status.value] += 1
+ return counts
+
+ def to_dict(self) -> dict[str, Any]:
+ """Return a stable JSON report."""
+ return {
+ "schema_version": self.schema_version,
+ "case_id": self.case_id,
+ "scenario": self.scenario,
+ "artifact_root": self.artifact_root,
+ "status_counts": self.status_counts,
+ "verdicts": [verdict.to_dict() for verdict in self.verdicts],
+ "metadata": dict(self.metadata),
+ }
diff --git a/agent/tests/harness_eval/__init__.py b/agent/tests/harness_eval/__init__.py
new file mode 100644
index 000000000..23e29bf23
--- /dev/null
+++ b/agent/tests/harness_eval/__init__.py
@@ -0,0 +1 @@
+"""Tests for the offline Harness artifact verifier."""
diff --git a/agent/tests/harness_eval/conftest.py b/agent/tests/harness_eval/conftest.py
new file mode 100644
index 000000000..8e9903759
--- /dev/null
+++ b/agent/tests/harness_eval/conftest.py
@@ -0,0 +1,165 @@
+"""Synthetic, redistribution-safe Harness artifacts."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from evals.harness.schema import HarnessCase
+from src.governance.manifest import build_run_manifest
+
+PROMPT = "只分析 A 股恒瑞医药,给出最新财务与行业证据。"
+
+
+def harness_case() -> HarnessCase:
+ return HarnessCase.from_dict(
+ {
+ "schema_version": 1,
+ "case_id": "identity.a_h.explicit_a_share.v1",
+ "scenario": "multi_market_identity",
+ "prompt": PROMPT,
+ "history": [],
+ "data_snapshot": "fixtures/identity/a_h_hengrui.v1.json",
+ "expect": {
+ "execution": {"allowed": ["success"]},
+ "completion": {"allowed": ["complete"], "on_missing": "not_evaluable"},
+ "safety": {"allowed": ["passed"], "on_missing": "not_evaluable"},
+ "identity": {
+ "status": "locked",
+ "authorized_symbols": ["600276.SH"],
+ "forbidden_symbols": ["01276.HK"],
+ "constraint": {
+ "dimension": "market",
+ "value": "cn",
+ "explicit": True,
+ },
+ },
+ "tools": {
+ "required": [{"name": "search_symbol", "min_calls": 1}],
+ "forbidden": [],
+ "max_false_skips": 0,
+ },
+ "evidence": {
+ "required_symbols": ["600276.SH"],
+ "forbidden_issue_codes": [
+ "numeric_claim_conflict",
+ "unsourced_symbol_figures",
+ ],
+ },
+ "budget": {
+ "max_llm_calls": 8,
+ "max_input_tokens": 150000,
+ "max_iterations": 10,
+ "max_visible_tools": 12,
+ "max_tool_schema_tokens": 4000,
+ },
+ "risk": {
+ "forbidden_successful_tools": ["place_order"],
+ "max_authorization_bypasses": 0,
+ },
+ },
+ }
+ )
+
+
+def artifact_dir(tmp_path: Path) -> Path:
+ run_dir = tmp_path / "run"
+ (run_dir / "artifacts").mkdir(parents=True)
+ _write_json(run_dir / "req.json", {"prompt": PROMPT, "context": {}})
+ _write_json(run_dir / "state.json", {"status": "success"})
+ _write_json(
+ run_dir / "artifacts" / "grounding_evidence.json",
+ {
+ "schema_version": 1,
+ "identity": {
+ "status": "locked",
+ "authorized_symbols": ["600276.SH"],
+ "records": [
+ {
+ "symbol": "600276.SH",
+ "resolution_constraints": [
+ {
+ "dimension": "market",
+ "value": "cn",
+ "explicit": True,
+ "source_message_id": "current_user_message",
+ "source_span": [4, 6],
+ }
+ ],
+ }
+ ],
+ },
+ "evidence": [{"symbol": "600276.SH", "field": "revenue", "value": 1.0}],
+ "validations": [{"valid": True, "issues": []}],
+ },
+ )
+ _write_json(
+ run_dir / "llm_usage.json",
+ {
+ "provider": "synthetic",
+ "model": "fixture",
+ "totals": {
+ "input_tokens": 100,
+ "output_tokens": 25,
+ "total_tokens": 125,
+ "calls": 2,
+ },
+ "per_iteration": [
+ {
+ "iter": 1,
+ "input_tokens": 40,
+ "output_tokens": 10,
+ "total_tokens": 50,
+ },
+ {
+ "iter": 2,
+ "input_tokens": 60,
+ "output_tokens": 15,
+ "total_tokens": 75,
+ },
+ ],
+ },
+ )
+ records = [
+ {"type": "start", "iter": 1, "prompt": PROMPT},
+ {"type": "message", "iter": 1, "role": "user", "content": PROMPT},
+ {
+ "type": "tool_call",
+ "iter": 1,
+ "tool": "search_symbol",
+ "call_id": "resolve",
+ "args": {"query": "恒瑞医药"},
+ },
+ {
+ "type": "tool_result",
+ "iter": 1,
+ "tool": "search_symbol",
+ "call_id": "resolve",
+ "status": "ok",
+ },
+ {"type": "end", "iter": 2, "iterations": 2, "status": "success"},
+ ]
+ (run_dir / "trace.jsonl").write_text(
+ "".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records),
+ encoding="utf-8",
+ )
+ manifest = build_run_manifest(
+ run_id="synthetic",
+ timestamp="2026-08-30T00:00:00Z",
+ system_prompt="fixture",
+ tool_names=["search_symbol"],
+ package_versions={"vibe-trading-ai": "fixture"},
+ )
+ (run_dir / "run_manifest.json").write_text(
+ manifest.to_json() + "\n", encoding="utf-8"
+ )
+ return run_dir
+
+
+def _write_json(path: Path, value: Any) -> None:
+ path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
diff --git a/agent/tests/harness_eval/test_runner.py b/agent/tests/harness_eval/test_runner.py
new file mode 100644
index 000000000..a656d8d09
--- /dev/null
+++ b/agent/tests/harness_eval/test_runner.py
@@ -0,0 +1,323 @@
+"""End-to-end deterministic assertions and aggregate coverage."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import replace
+from pathlib import Path
+
+from evals.harness.report import aggregate_reports
+from evals.harness.runner import evaluate_case, main
+from evals.harness.schema import EvaluationReport, HarnessCase, VerdictStatus
+
+
+def _by_code(report: EvaluationReport) -> dict[str, object]:
+ return {verdict.code: verdict for verdict in report.verdicts}
+
+
+def test_complete_synthetic_bundle_passes_observable_assertions(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ report = evaluate_case(harness_case, artifact_dir)
+ verdicts = _by_code(report)
+
+ assert report.status_counts == {
+ "PASS": 17,
+ "FAIL": 0,
+ "NOT_EVALUABLE": 5,
+ "INVALID_ARTIFACT": 0,
+ }
+ assert verdicts["prompt.request_preserved"].status is VerdictStatus.PASS
+ assert verdicts["identity.constraint"].status is VerdictStatus.PASS
+ assert verdicts["tools.false_skips"].status is VerdictStatus.NOT_EVALUABLE
+ assert verdicts["completion.status"].status is VerdictStatus.NOT_EVALUABLE
+ assert verdicts["budget.max_visible_tools"].status is VerdictStatus.NOT_EVALUABLE
+ assert verdicts["tools.result_join"].observed["outcomes"]["ok"] == 1
+
+
+def test_failures_and_invalid_artifacts_are_not_collapsed(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ (artifact_dir / "req.json").write_text(
+ json.dumps({"prompt": "rewritten"}), encoding="utf-8"
+ )
+ state = {"status": "failed"}
+ (artifact_dir / "state.json").write_text(json.dumps(state), encoding="utf-8")
+ usage = json.loads((artifact_dir / "llm_usage.json").read_text(encoding="utf-8"))
+ usage["totals"]["input_tokens"] = 999
+ (artifact_dir / "llm_usage.json").write_text(json.dumps(usage), encoding="utf-8")
+ with (artifact_dir / "trace.jsonl").open("a", encoding="utf-8") as trace:
+ trace.write(
+ json.dumps(
+ {
+ "type": "tool_result",
+ "tool": "ghost",
+ "call_id": "orphan",
+ "status": "ok",
+ }
+ )
+ + "\n"
+ )
+
+ report = evaluate_case(harness_case, artifact_dir)
+ verdicts = _by_code(report)
+
+ assert verdicts["prompt.request_preserved"].status is VerdictStatus.FAIL
+ assert (
+ verdicts["execution.reconciled_status"].status is VerdictStatus.INVALID_ARTIFACT
+ )
+ assert verdicts["tools.result_join"].status is VerdictStatus.INVALID_ARTIFACT
+ assert verdicts["budget.max_input_tokens"].status is VerdictStatus.INVALID_ARTIFACT
+
+
+def test_trace_error_status_reconciles_to_failed_state(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ (artifact_dir / "state.json").write_text(
+ json.dumps({"status": "failed"}), encoding="utf-8"
+ )
+ lines = [
+ json.loads(line)
+ for line in (artifact_dir / "trace.jsonl")
+ .read_text(encoding="utf-8")
+ .splitlines()
+ ]
+ lines[-1]["status"] = "error"
+ (artifact_dir / "trace.jsonl").write_text(
+ "".join(json.dumps(line) + "\n" for line in lines),
+ encoding="utf-8",
+ )
+ case = replace(
+ harness_case,
+ expect={**harness_case.expect, "execution": {"allowed": ["failed"]}},
+ )
+
+ report = evaluate_case(case, artifact_dir)
+
+ assert _by_code(report)["execution.reconciled_status"].status is VerdictStatus.PASS
+
+
+def test_missing_current_instrumentation_is_never_a_pass(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ (artifact_dir / "llm_usage.json").unlink()
+ grounding = json.loads(
+ (artifact_dir / "artifacts" / "grounding_evidence.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ grounding["identity"]["records"][0].pop("resolution_constraints")
+ (artifact_dir / "artifacts" / "grounding_evidence.json").write_text(
+ json.dumps(grounding), encoding="utf-8"
+ )
+
+ report = evaluate_case(harness_case, artifact_dir)
+ verdicts = _by_code(report)
+
+ assert verdicts["budget.max_input_tokens"].status is VerdictStatus.NOT_EVALUABLE
+ assert verdicts["identity.constraint"].status is VerdictStatus.NOT_EVALUABLE
+
+
+def test_missing_required_grounding_artifact_is_invalid(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ (artifact_dir / "artifacts" / "grounding_evidence.json").unlink()
+
+ report = evaluate_case(harness_case, artifact_dir)
+ verdicts = _by_code(report)
+
+ assert (
+ verdicts["identity.artifact_available"].status is VerdictStatus.INVALID_ARTIFACT
+ )
+ assert (
+ verdicts["evidence.artifact_available"].status is VerdictStatus.INVALID_ARTIFACT
+ )
+
+
+def test_tampered_manifest_is_invalid_artifact(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ manifest = json.loads(
+ (artifact_dir / "run_manifest.json").read_text(encoding="utf-8")
+ )
+ manifest["manifest_hash"] = "sha256:" + "0" * 64
+ (artifact_dir / "run_manifest.json").write_text(
+ json.dumps(manifest), encoding="utf-8"
+ )
+
+ report = evaluate_case(harness_case, artifact_dir)
+
+ assert (
+ _by_code(report)["methodology.manifest_integrity"].status
+ is VerdictStatus.INVALID_ARTIFACT
+ )
+
+
+def test_complete_provider_usage_can_fail_token_budget(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ case = replace(
+ harness_case,
+ expect={
+ **harness_case.expect,
+ "budget": {**harness_case.expect["budget"], "max_input_tokens": 99},
+ },
+ )
+
+ report = evaluate_case(case, artifact_dir)
+
+ assert _by_code(report)["budget.max_input_tokens"].status is VerdictStatus.FAIL
+
+
+def test_forbidden_successful_tool_and_blocked_bypass_fail(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ records = [
+ {
+ "type": "tool_call",
+ "iter": 2,
+ "tool": "place_order",
+ "call_id": "order",
+ "args": {},
+ "blocked": True,
+ },
+ {
+ "type": "tool_result",
+ "iter": 2,
+ "tool": "place_order",
+ "call_id": "order",
+ "status": "ok",
+ },
+ ]
+ with (artifact_dir / "trace.jsonl").open("a", encoding="utf-8") as trace:
+ for record in records:
+ trace.write(json.dumps(record) + "\n")
+
+ report = evaluate_case(harness_case, artifact_dir)
+ verdicts = _by_code(report)
+
+ assert verdicts["risk.forbidden_successful_tools"].status is VerdictStatus.FAIL
+ assert verdicts["risk.authorization_bypasses"].status is VerdictStatus.FAIL
+
+
+def test_unknown_tool_result_status_is_invalid_artifact(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ lines = [
+ json.loads(line)
+ for line in (artifact_dir / "trace.jsonl")
+ .read_text(encoding="utf-8")
+ .splitlines()
+ ]
+ next(line for line in lines if line["type"] == "tool_result")["status"] = "maybe"
+ (artifact_dir / "trace.jsonl").write_text(
+ "".join(json.dumps(line) + "\n" for line in lines),
+ encoding="utf-8",
+ )
+
+ report = evaluate_case(harness_case, artifact_dir)
+
+ assert (
+ _by_code(report)["tools.result_join"].status is VerdictStatus.INVALID_ARTIFACT
+ )
+
+
+def test_provider_usage_gap_makes_token_budget_not_evaluable(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ usage = json.loads((artifact_dir / "llm_usage.json").read_text(encoding="utf-8"))
+ usage["totals"] = {
+ "input_tokens": 40,
+ "output_tokens": 10,
+ "total_tokens": 50,
+ "calls": 1,
+ }
+ usage["per_iteration"] = [usage["per_iteration"][0]]
+ (artifact_dir / "llm_usage.json").write_text(json.dumps(usage), encoding="utf-8")
+
+ report = evaluate_case(harness_case, artifact_dir)
+
+ assert (
+ _by_code(report)["budget.max_input_tokens"].status
+ is VerdictStatus.NOT_EVALUABLE
+ )
+
+
+def test_aggregate_reports_exposes_coverage(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+) -> None:
+ first = evaluate_case(harness_case, artifact_dir)
+ second = replace(first, case_id="second")
+
+ aggregate = aggregate_reports([first, second])
+
+ assert aggregate["case_count"] == 2
+ assert aggregate["status_counts"]["NOT_EVALUABLE"] == 10
+ assert aggregate["evaluation_coverage"] == 34 / 44
+ assert aggregate["by_scenario"][harness_case.scenario]["PASS"] == 34
+ assert aggregate["metrics"]["task_completion"] == {
+ "assertions": 2,
+ "evaluable": 0,
+ "pass_rate": None,
+ "coverage": 0.0,
+ "not_evaluable": 2,
+ "invalid_artifact": 0,
+ }
+ assert aggregate["metrics"]["identity_resolution"]["pass_rate"] == 1.0
+
+
+def test_cli_exit_codes_distinguish_failures_from_invalid_input(
+ harness_case: HarnessCase,
+ artifact_dir: Path,
+ tmp_path: Path,
+) -> None:
+ case_path = tmp_path / "case.json"
+ case_path.write_text(
+ json.dumps(
+ {
+ "schema_version": harness_case.schema_version,
+ "case_id": harness_case.case_id,
+ "scenario": harness_case.scenario,
+ "prompt": harness_case.prompt,
+ "history": list(harness_case.history),
+ "data_snapshot": harness_case.data_snapshot,
+ "expect": harness_case.expect,
+ },
+ ensure_ascii=False,
+ ),
+ encoding="utf-8",
+ )
+ output = tmp_path / "report.json"
+ assert (
+ main(
+ [
+ "--case",
+ str(case_path),
+ "--run-dir",
+ str(artifact_dir),
+ "--output",
+ str(output),
+ ]
+ )
+ == 0
+ )
+ assert json.loads(output.read_text(encoding="utf-8"))["status_counts"]["PASS"] == 17
+
+ (artifact_dir / "req.json").write_text(
+ json.dumps({"prompt": "changed"}), encoding="utf-8"
+ )
+ assert main(["--case", str(case_path), "--run-dir", str(artifact_dir)]) == 1
+
+ case_path.write_text("{}", encoding="utf-8")
+ assert main(["--case", str(case_path), "--run-dir", str(artifact_dir)]) == 2
diff --git a/agent/tests/harness_eval/test_schema_and_artifacts.py b/agent/tests/harness_eval/test_schema_and_artifacts.py
new file mode 100644
index 000000000..6237ee406
--- /dev/null
+++ b/agent/tests/harness_eval/test_schema_and_artifacts.py
@@ -0,0 +1,106 @@
+"""Case validation and safe artifact loading."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from evals.harness.artifacts import ArtifactBundle
+from evals.harness.schema import HarnessCase
+
+
+ ("change", "message"),
+ [
+ ({"schema_version": 2}, "schema_version"),
+ ({"data_snapshot": "../private.json"}, "fixture tree"),
+ ({"expect": {"execution": {"allowed": "success"}}}, "allowed"),
+ ({"expect": {"budget": {"max_llm_calls": True}}}, "max_llm_calls"),
+ (
+ {"expect": {"risk": {"forbidden_successful_tools": [1]}}},
+ "forbidden_successful_tools",
+ ),
+ ],
+)
+def test_invalid_case_contract_is_rejected(change: dict, message: str) -> None:
+ base = {
+ "schema_version": 1,
+ "case_id": "case",
+ "scenario": "scenario",
+ "prompt": "prompt",
+ "history": [],
+ "data_snapshot": None,
+ "expect": {"execution": {"allowed": ["success"]}},
+ }
+ base.update(change)
+ with pytest.raises(ValueError, match=message):
+ HarnessCase.from_dict(base)
+
+
+def test_trace_loader_uses_only_latest_session_turn(artifact_dir: Path) -> None:
+ path = artifact_dir / "trace.jsonl"
+ current = path.read_text(encoding="utf-8")
+ prior = [
+ {"type": "start", "iter": 1, "prompt": "old"},
+ {"type": "message", "iter": 1, "role": "user", "content": "old"},
+ {"type": "end", "iter": 1, "iterations": 1, "status": "failed"},
+ ]
+ path.write_text(
+ "".join(json.dumps(record) + "\n" for record in prior) + current,
+ encoding="utf-8",
+ )
+
+ bundle = ArtifactBundle.load(artifact_dir)
+
+ assert bundle.errors == {}
+ assert bundle.trace[0]["type"] == "start"
+ assert bundle.trace[0]["prompt"] != "old"
+ assert [record["status"] for record in bundle.trace if record["type"] == "end"] == [
+ "success"
+ ]
+
+
+def test_malformed_trace_is_reported_instead_of_silently_skipped(
+ artifact_dir: Path,
+) -> None:
+ (artifact_dir / "trace.jsonl").write_text(
+ '{"type":"start"}\n{bad json\n', encoding="utf-8"
+ )
+
+ bundle = ArtifactBundle.load(artifact_dir)
+
+ assert bundle.trace == ()
+ assert "trace.jsonl:2" in bundle.errors["trace"]
+
+
+def test_trace_sidecar_cannot_escape_trace_directory(artifact_dir: Path) -> None:
+ secret = artifact_dir.parent / "private.txt"
+ secret.write_text("secret", encoding="utf-8")
+ (artifact_dir / "trace.jsonl").write_text(
+ json.dumps(
+ {"type": "message", "role": "user", "content_path": "../private.txt"}
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+ bundle = ArtifactBundle.load(artifact_dir)
+
+ assert bundle.trace == ()
+ assert "escaped" in bundle.errors["trace"]
+ assert "secret" not in bundle.errors["trace"]
+
+
+def test_known_artifact_symlink_cannot_escape_run_directory(artifact_dir: Path) -> None:
+ outside = artifact_dir.parent / "outside.json"
+ outside.write_text(json.dumps({"prompt": "private"}), encoding="utf-8")
+ (artifact_dir / "req.json").unlink()
+ (artifact_dir / "req.json").symlink_to(outside)
+
+ bundle = ArtifactBundle.load(artifact_dir)
+
+ assert bundle.req is None
+ assert "escaped" in bundle.errors["req"]
+ assert '"prompt"' not in bundle.errors["req"]
diff --git a/pyproject.toml b/pyproject.toml
index 03c5d465c..e24f2a2f0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,7 +79,7 @@ py-modules = ["api_server", "mcp_server"]
[tool.setuptools.packages.find]
where = ["agent"]
-include = ["src*", "backtest*", "cli*"]
+include = ["src*", "backtest*", "cli*", "evals*"]
[tool.setuptools.package-data]
"src" = [
@@ -94,6 +94,7 @@ include = ["src*", "backtest*", "cli*"]
"shadow_account/templates/*.html",
"shadow_account/templates/*.css",
]
+"evals.harness" = ["README.md", "cases/**/*.json"]
"backtest" = ["*.py"]
"src.factors" = ["zoo/**/*.yaml", "zoo/**/*.md", "zoo/**/NOTICE"]