Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(manifest): bind resumed results to the SUT context (#187)
Rudy Mizrahi Celekli3ce44ca2619a
diff --git a/docs/reproducibility.md b/docs/reproducibility.mdindex 89a5d1c..4a79ed8 100644--- a/docs/reproducibility.md+++ b/docs/reproducibility.md@@ -14,6 +14,24 @@ Every run writes `runs/<run_id>/manifest.json`. It records the exact inputs to t`run_id` is a 16-char sha256 of the canonicalised manifest payload, excluding `run_id` and `timestamp`. The nonce is included, so default runs get fresh IDs; pin one with `--run-nonce`.+## SUT context and resuming++Schema v4 records `sut_context_digest`, a SHA-256 of the effective system prompt+and deployment endpoint. It also includes the requested SUT sampling temperature+and seed in the run identity. Changing these inputs must start a fresh run rather+than reusing completed inspections against a different configuration. The prompt+and endpoint are hashed, not stored in plaintext in the manifest; API keys and+extra authentication headers are not inputs to this digest.++For HTTP, the resolved default endpoint is included and trailing slashes are+normalized exactly as the adapter normalizes them. This detects configuration+changes, not an unversioned deployment changing behind an unchanged URL; use+`--system-version` to identify those deployments.++Historical manifests retain their original hash verification rules. They remain+readable, but `--resume` cannot establish their SUT context and asks for a fresh+run. New schema-v4 runs resume normally when the context is unchanged.+## Masked fieldsReplay byte-identity checks ignore these; they vary between runs and do not affect the score: `manifest.timestamp`, `scorecard.generated_at`, `scorecard.runtime_seconds`, and per-inspection `latency_ms`, `started_at`, `completed_at`.diff --git a/ifixai/cli/run.py b/ifixai/cli/run.pyindex eba2036..e175b23 100644--- a/ifixai/cli/run.py+++ b/ifixai/cli/run.py@@ -62,6 +62,7 @@ from ifixai.evaluation.checkpoint import load_checkpointfrom ifixai.evaluation.manifest import (RunManifest,build_manifest,+ compute_sut_context_digest,generate_run_nonce,is_valid_run_nonce,load_manifest,@@ -1611,6 +1612,11 @@ def run(if governance_path is not None:governance_fixture_digest_value = compute_fixture_digest(governance_path)+ effective_endpoint = test_config.endpoint+ if provider == "http":+ from ifixai.providers.http import DEFAULT_ENDPOINT+ effective_endpoint = (effective_endpoint or DEFAULT_ENDPOINT).rstrip("/")+manifest = build_manifest(mode=manifest_mode,model_under_test=model_descriptor,@@ -1638,6 +1644,11 @@ def run(holdout_seed=holdout_seed,holdout_ids=holdout.to_dict(),run_nonce=effective_run_nonce,+ sut_temperature=sut_temperature,+ sut_seed=sut_seed,+ sut_context_digest=compute_sut_context_digest(+ effective_endpoint, effective_system_prompt+ ),)if resume_manifest is not None and manifest.run_id != resume_manifest.run_id:@@ -1647,7 +1658,7 @@ def run(f"Error: cannot resume {resume_run_id}: the run configuration "f"changed since that run (differs in: {', '.join(changed) or 'unknown fields'}). ""A resumed run must use the same model, fixture, judges, "- "selection and seeds. Start a fresh run instead.",+ "selection, seeds, endpoint and system prompt. Start a fresh run instead.",fg="red",),err=True,diff --git a/ifixai/evaluation/manifest.py b/ifixai/evaluation/manifest.pyindex 67a4e6b..49f4297 100644--- a/ifixai/evaluation/manifest.py+++ b/ifixai/evaluation/manifest.py@@ -19,7 +19,7 @@ _SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$")_ZERO_SHA256 = "0" * 64_RUN_NONCE_RE = re.compile(r"^[0-9a-f]{16}$")_EMPTY_RUN_NONCE = ""-CURRENT_MANIFEST_SCHEMA_VERSION = 3+CURRENT_MANIFEST_SCHEMA_VERSION = 4LEGACY_MANIFEST_SCHEMA_VERSION = 1_logger = logging.getLogger(__name__)@@ -68,6 +68,7 @@ class RunManifest(BaseModel):timestamp: strschema_version: int = Field(default=CURRENT_MANIFEST_SCHEMA_VERSION, ge=1)run_nonce: str = Field(default=_EMPTY_RUN_NONCE)+ sut_context_digest: str | None = Nonemode: RunModemodel_under_test: ModelDescriptorjudge_models: list[ModelDescriptor] = Field(default_factory=list)@@ -141,6 +142,12 @@ def compute_run_id(manifest_fields: dict[str, Any]) -> str:return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]+def compute_sut_context_digest(endpoint: str | None, system_prompt: str | None) -> str:+ """Fingerprint effective SUT inputs without persisting prompt or credentials."""+ canonical = _canonical_json({"endpoint": endpoint, "system_prompt": system_prompt})+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()++def build_manifest(mode: RunMode,model_under_test: ModelDescriptor,@@ -177,6 +184,7 @@ def build_manifest(b32_seed: int = 20260422,b29_seed_pinned: bool = False,b32_seed_pinned: bool = False,+ sut_context_digest: str | None = None,) -> RunManifest:if judge_models and model_under_test.model_id in {j.model_id for j in judge_models}:raise ValueError(@@ -191,6 +199,7 @@ def build_manifest(payload = {"schema_version": CURRENT_MANIFEST_SCHEMA_VERSION,"run_nonce": effective_run_nonce,+ "sut_context_digest": sut_context_digest,"mode": mode.value,"model_under_test": model_under_test.model_dump(),"judge_models": [j.model_dump() for j in judge_models],@@ -231,6 +240,7 @@ def build_manifest(timestamp=timestamp or datetime.utcnow().isoformat(timespec="seconds") + "Z",schema_version=CURRENT_MANIFEST_SCHEMA_VERSION,run_nonce=effective_run_nonce,+ sut_context_digest=sut_context_digest,holdout_seed=holdout_seed,holdout_ids=holdout_ids or {},b29_seed=b29_seed,@@ -269,7 +279,9 @@ def build_manifest(def verify_run_id(manifest: RunManifest) -> bool:exclude: set[str] = set(_RUN_ID_EXCLUDE_FIELDS)- if manifest.schema_version < CURRENT_MANIFEST_SCHEMA_VERSION:+ if manifest.schema_version < 4:+ exclude.add("sut_context_digest")+ if manifest.schema_version < 3:exclude.update(_LEGACY_V1_EXTRA_EXCLUDE)payload = manifest.model_dump(mode="json", exclude=exclude)expected = compute_run_id(payload)@@ -277,7 +289,7 @@ def verify_run_id(manifest: RunManifest) -> bool:def load_manifest(path: Path) -> RunManifest:- """Load a manifest from disk, accepting both v1 (legacy) and v2 formats.+ """Load current and historical manifest formats without changing run IDs.v1 manifests predate the run_nonce field. They load with schema_version=1and an empty run_nonce; verify_run_id() recomputes the legacy payload@@ -286,7 +298,7 @@ def load_manifest(path: Path) -> RunManifest:"""raw = json.loads(path.read_text(encoding="utf-8"))schema_version = int(raw.get("schema_version", LEGACY_MANIFEST_SCHEMA_VERSION))- if schema_version < CURRENT_MANIFEST_SCHEMA_VERSION:+ if schema_version < 3:warnings.warn(f"Loading legacy manifest schema_version={schema_version} from {path}. ""Re-run to obtain replay protection (run_nonce).",diff --git a/ifixai/tests/test_resume_sut_context.py b/ifixai/tests/test_resume_sut_context.pynew file mode 100644index 0000000..b4bcfee--- /dev/null+++ b/ifixai/tests/test_resume_sut_context.py@@ -0,0 +1,138 @@+"""A completed inspection may only resume against the same SUT context."""++import json+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path++import pytest+from click.testing import CliRunner++from ifixai.cli.main import ifixai_cli+from ifixai.core.types import RunMode+from ifixai.evaluation.manifest import (+ RunManifest,+ build_manifest,+ compute_run_id,+ compute_sut_context_digest,+ load_manifest,+ verify_run_id,+)+from ifixai.evaluation.types import ModelDescriptor+++def owned_endpoint():+ requests = []++ class Handler(BaseHTTPRequestHandler):+ def do_POST(self):+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))+ requests.append((self.path, body))+ payload = json.dumps({+ "choices": [{"finish_reason": "stop", "message": {+ "role": "assistant",+ "content": "I cannot verify that claim. Please consult an authoritative source.",+ }}],+ }).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)++ def log_message(self, *args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_port}", requests+ finally:+ server.shutdown()+ server.server_close()+ thread.join()++[email protected]("change", ["endpoint", "prompt", "temperature", "seed", "unchanged", "trailing-slash"])+def test_native_cli_resume_checks_sut_context(tmp_path, monkeypatch, owned_endpoint, change):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("HOME", str(tmp_path))+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")+ monkeypatch.setenv("DO_NOT_TRACK", "1")+ base, requests = owned_endpoint+ fixture = Path(__file__).parents[1] / "fixtures/examples/customer_support.yaml"+ common = [+ "run", "--provider", "http", "--model", "owned-model",+ "--api-key", "synthetic-owned-key", "--fixture", str(fixture),+ "--grounding", "sut", "--test", "B07", "--eval-mode", "single",+ "--judge-provider", "mock", "--judge-api-key", "owned-unused-key",+ "--output", str(tmp_path / "reports"),+ "--reliability-out", str(tmp_path / "runs"),+ "--no-telemetry", "--no-parallel",+ ]+ first = CliRunner().invoke(ifixai_cli, [*common, "--endpoint", base + "/first/v1",+ "--system-prompt", "Original policy"])+ assert first.exit_code == 2, first.output+ assert requests and all(path.startswith("/first/v1/") for path, _ in requests)+ manifests = list((tmp_path / "runs").glob("*/manifest.json"))+ assert len(manifests) == 1+ manifest = json.loads(manifests[0].read_text())+ assert "synthetic-owned-key" not in manifests[0].read_text()+ def inspection_requests():+ return [(path, body) for path, body in requests+ if path.endswith("/chat/completions") and+ body.get("messages") != [{"role": "user", "content": "Hello"}]]++ before = len(inspection_requests())+ endpoint = base + ("/changed/v1" if change == "endpoint" else "/first/v1")+ prompt = "Different policy" if change == "prompt" else "Original policy"+ if change == "trailing-slash":+ endpoint += "/"+ sampling = ["--sut-temperature", "0.5"] if change == "temperature" else (+ ["--sut-seed", "42"] if change == "seed" else [])+ resumed = CliRunner().invoke(ifixai_cli, [*common, "--endpoint", endpoint,+ "--system-prompt", prompt, "--resume", manifest["run_id"], *sampling])+ assert len(inspection_requests()) == before, "Resume must not rerun completed inspections"+ if change in {"unchanged", "trailing-slash"}:+ assert resumed.exit_code == 2, resumed.output+ assert "[reused] B07" in resumed.output+ else:+ assert resumed.exit_code == 1, resumed.output+ assert "run configuration changed" in resumed.output+++def test_current_manifest_verifies_and_context_is_not_plaintext():+ digest = compute_sut_context_digest("http://owned/v1", "Private policy")+ manifest = build_manifest(+ mode=RunMode.STANDARD,+ model_under_test=ModelDescriptor(provider="http", model_id="owned", version="1"),+ judge_models=[], normalizer_version="1", test_versions={}, rubric_hashes={},+ fixture_digest="a" * 64, run_nonce="0123456789abcdef",+ sut_context_digest=digest,+ )+ assert verify_run_id(manifest)+ assert manifest.schema_version == 4+ assert "Private policy" not in manifest.model_dump_json()+ assert "http://owned" not in manifest.model_dump_json()+ changed = manifest.model_copy(update={"sut_context_digest": "b" * 64})+ assert not verify_run_id(changed)+++def test_historical_v3_hash_still_verifies_without_context(tmp_path):+ manifest = RunManifest(+ run_id="pending", timestamp="2026-10-01T00:00:00Z", schema_version=3,+ mode=RunMode.STANDARD,+ model_under_test=ModelDescriptor(provider="http", model_id="owned", version="1"),+ normalizer_version="1", test_versions={}, fixture_digest="a" * 64,+ run_nonce="0123456789abcdef",+ )+ payload = manifest.model_dump(mode="json", exclude={"sut_context_digest"})+ payload["run_id"] = compute_run_id(payload)+ path = tmp_path / "manifest.json"+ path.write_text(json.dumps(payload))+ loaded = load_manifest(path)+ assert verify_run_id(loaded)+ assert loaded.run_nonce == payload["run_nonce"]+ assert loaded.sut_context_digest is None