Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(replay): reach an operator with aisoc replay, an API job and a console page (#907)

Beenu Arora032ddacf1e00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0a7e963b8..ea9af2a95 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -331,6 +331,13 @@ jobs:
# non-importable copy is how the gap silently returns.
- name: Verify vendored llm_contract_rules is in sync
run: python scripts/sync_vendored_llm_contract.py --check
+ # Same reasoning again, for the replay scorer. The API orchestrates
+ # replay evaluation and grades the result, and `packages/aisoc-benchmark`
+ # is not on the image's build context. A drifted mirror would mean two
+ # definitions of "hallucination rate" in the tree, both calling
+ # themselves the same published number.
+ - name: Verify vendored aisoc_benchmark is in sync
+ run: python scripts/sync_vendored_benchmark.py --check
# Fabricated security data must never render as a tenant's real state.
# Around three dozen console components served a `MOCK_*` / `DEMO_*`
# array of plausible alerts, connectors and coverage numbers whenever the
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 9ab40b6e0..e3344a040 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -72,6 +72,7 @@ jobs:
migrations: ${{ steps.areas.outputs.migrations }}
backup: ${{ steps.areas.outputs.backup }}
upgrade: ${{ steps.areas.outputs.upgrade }}
+ replay: ${{ steps.areas.outputs.replay }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
@@ -90,6 +91,11 @@ jobs:
# encryption works.
backup_paths='scripts/backup.sh scripts/restore.sh scripts/backup_crypt.py .github/workflows/integration.yml'
upgrade_paths='services/api/migrations/ services/api/app/scripts/run_migrations.py .github/workflows/integration.yml'
+ # Replay evaluation spans four services, the CLI and the scorer, and
+ # the only thing that proves the whole chain is the end-to-end run.
+ # Every tree it crosses is listed, because a change to any one of
+ # them can break a link none of the unit suites can see.
+ replay_paths='services/api/app/services/replay_evaluation/ services/api/app/api/v1/endpoints/evaluations.py services/api/app/_vendor/aisoc_benchmark/ services/agents/app/replay/ services/agents/app/api/replay_router.py services/actions/app/api/replay_history_router.py services/actions/app/services/alert_history.py services/connectors/app/api/router.py packages/aisoc-benchmark/ packages/aisoc-cli/ tests/e2e/test_replay_cli_end_to_end.py .github/workflows/integration.yml'
base_ref=""
if [ -n "${{ github.base_ref }}" ]; then
@@ -100,7 +106,7 @@ jobs:
fi
emit_all_true() {
- for area in spine migrations backup upgrade; do
+ for area in spine migrations backup upgrade replay; do
echo "${area}=true" >> "$GITHUB_OUTPUT"
done
}
@@ -139,6 +145,7 @@ jobs:
decide migrations "${migration_paths}"
decide backup "${backup_paths}"
decide upgrade "${upgrade_paths}"
+ decide replay "${replay_paths}"
# ── The spine, end to end ──────────────────────────────────────────────────
spine:
@@ -412,6 +419,96 @@ jobs:
if: always()
run: docker compose down -v --remove-orphans
+ # ── Replay evaluation, end to end ──────────────────────────────────────────
+ # Phase 1's acceptance bar: the CLI, run against a mocked Splunk ES holding
+ # 200 recorded closed notables, produces a report that reproduces byte for
+ # byte on a second run with the deterministic model path.
+ #
+ # It cannot be a unit test in any service's suite. The chain crosses
+ # `services/actions` (the SIEM credentials and the closed-finding readers),
+ # `services/connectors` (the production `normalize()`), `services/agents`
+ # (triage) and `services/api` (the orchestration, the vault and the tenant
+ # session), and all four package their code as a top-level `app`, so one
+ # process can hold exactly one of them. The harness starts each from the
+ # working tree rather than from a published image, so this proves the code
+ # in the diff.
+ #
+ # The end-to-end run is also what found the defect that made this necessary:
+ # `POST /connectors/{id}/normalize` shipped with no caller and its client
+ # built a URL missing the prefix the connectors service mounts under, so
+ # every request would have 404'd. Three green unit suites could not see it.
+ #
+ # AISOC_REPLAY_E2E_REQUIRED turns an unreachable database into a failure
+ # rather than a skip, for the same reason as the jobs below: a gate that
+ # quietly declines to run looks identical to one that passed.
+ replay-e2e:
+ name: Replay evaluation (CLI to API to report, four services)
+ needs: changes
+ if: needs.changes.outputs.replay == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 25
+ services:
+ postgres:
+ image: postgres:16
+ env:
+ POSTGRES_USER: aisoc
+ POSTGRES_PASSWORD: aisoc_ci_secret
+ POSTGRES_DB: aisoc
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready -U aisoc"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 10
+ env:
+ # The two roles the default deployment ships. The services connect as
+ # aisoc_app, which is DML-only and which the RLS policies apply to;
+ # aisoc is the owner and runs DDL and nothing else. Wiring this job as a
+ # single superuser would measure a role nothing runs as, which is what
+ # `check_runtime_db_role.py` refuses.
+ AISOC_E2E_DATABASE_URL: postgresql+asyncpg://aisoc_app:aisoc_app_ci_secret@localhost:5432/aisoc
+ AISOC_E2E_DATABASE_MIGRATION_URL: postgresql+asyncpg://aisoc:aisoc_ci_secret@localhost:5432/aisoc
+ AISOC_APP_DB_PASSWORD: aisoc_app_ci_secret
+ AISOC_REPLAY_E2E_REQUIRED: '1'
+ ENVIRONMENT: development
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: '3.11'
+ cache: pip
+
+ # The union of what the four services import on this path. Same version
+ # bounds as ci.yml's per-service jobs, because a chain that only works
+ # on a different resolution is not the chain that ships.
+ - name: Install the four services' shared dependencies
+ run: >-
+ pip install --quiet
+ pytest pytest-asyncio uvicorn
+ "fastapi>=0.117,<0.142" "pydantic[email]" pydantic-settings "sqlalchemy[asyncio]" asyncpg
+ structlog "PyJWT>=2.8,<3" "cryptography>=46,<51" "passlib[bcrypt]" tenacity pyyaml
+ prometheus-client "opentelemetry-sdk" "opentelemetry-api"
+ "opentelemetry-exporter-otlp-proto-grpc"
+ "opentelemetry-instrumentation-fastapi"
+ "opentelemetry-instrumentation-httpx"
+ "strawberry-graphql[fastapi]"
+ neo4j redis celery httpx aiofiles email-validator
+ "sqlglot>=30,<31" aiosqlite "croniter>=2.0,<7.0"
+ aiokafka apscheduler qdrant-client bcrypt
+ langgraph langchain-core langchain-openai
+ click rich jsonschema
+
+ # The CLI is driven as an installed console script would be, off its own
+ # source tree, so the command under test is the one the package ships.
+ - name: Replay evaluation, CLI to API to report
+ run: python -m pytest tests/e2e/test_replay_cli_end_to_end.py -v --tb=short
+
+ - name: Service logs on failure
+ if: failure()
+ run: for log in .e2e-*.log; do echo "::group::${log}"; tail -n 200 "${log}"; echo "::endgroup::"; done
+
# ── Migration correctness ──────────────────────────────────────────────────
migrations:
name: Migrations (fresh apply, idempotent re-run, alembic round-trip)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f5f0fbbf..ec565aa40 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,77 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+- **Replay evaluation reaches an operator: `aisoc replay`, an async API job,
+ an "Evaluate on your history" console page, and JSON, Markdown and PDF
+ export** (gap-closure Phase 1.4 and 1.5). Phases 1.1 through 1.3 built
+ readers that could list a customer's closed findings, a runner that could
+ replay them through the production triage path writing nothing, and a
+ scorer that could grade the result. None of the three had a caller. This is
+ the surface that drives them, and building it found a defect that three
+ green unit suites could not see.
+
+ **The shape was forced, not chosen.** No process can hold two of these
+ services: `services/actions` owns the SIEM credential path and the readers,
+ `services/agents` owns triage, `services/connectors` owns `normalize()`,
+ and all three package their code as a top-level `app`. So the API
+ orchestrates, which is what it already does for `/cases/{id}/investigate`
+ and for live actions, and two internal routes were added for it to drive:
+ `POST /replay/history` on actions and `POST /replay/run` on agents. Scoring
+ is the one link with no round trip, through a byte-identical mirror of
+ `packages/aisoc-benchmark` under `services/api/app/_vendor/`, because the
+ API image is built with `./services/api` as its context and nothing under
+ `packages/` exists at runtime.
+
+ **The defect the end-to-end run found.** `POST /connectors/{id}/normalize`
+ shipped in Phase 1.2 with no caller, and the client that builds its URL
+ omitted the `/api/v1` prefix the connectors service mounts its router
+ under. Every normalize request would have returned 404, so no replay could
+ have run on any deployment. The unit test covering that function asserted
+ the wrong URL and passed. It now reads the mount out of the connectors
+ service's own source with `ast` and compares in both directions.
+
+ **What the console does with a thin corpus.** No figure is rendered without
+ the count behind it: recall beside its malicious-case count, the headline
+ beside its answered-decision count, and the history read beside how many of
+ those findings carried an analyst label at all. Below the floor of malicious
+ cases the headline card reads "withheld" and prints the sentence explaining
+ why, rather than a dash or a zero. Zero would say the agent got every answer
+ wrong, which is a different fact with a different remedy.
+
+ **Reproducibility, stated precisely.** A report reproduces byte for byte
+ between two runs over the same pinned window, apart from the two wall-clock
+ latency figures, which measure the host. `strip_latency` lives beside the
+ renderer that emits that line so the two cannot drift, and is exposed as
+ `--exclude-latency` on the CLI and `exclude_latency=true` on the export
+ route. The export serves the artefact stored when the run completed rather
+ than re-rendering it, because a report re-rendered by a newer renderer is a
+ different artefact from the one the operator read.
+
+ **Measured.** Phase 1's "Done when" now holds end to end:
+ `tests/e2e/test_replay_cli_end_to_end.py` runs the CLI twice against a
+ mocked Splunk ES holding 200 recorded closed notables, through four
+ services started from the working tree, and the two reports are identical
+ as bytes. 200 findings read, 200 labelled, 60 replayed and graded, 40
+ malicious, headline printed rather than withheld. A second assertion
+ fetches both reports without the exclusion and fails if more than the one
+ latency line differs, so the comparison cannot pass on a stripped artefact
+ that hid something else. Suites: `services/actions` 767 to 780,
+ `services/agents` 1280 to 1306, `services/api` 2852 to 2877,
+ `packages/aisoc-benchmark` 57 to 62, `packages/aisoc-cli` 40 to 54.
+
+ **Migration 065**, not 064: `064_sandbox_upload_policy.sql` landed from
+ another phase while this one was in flight. Two tenant-scoped tables with
+ row-level-security policies and `aisoc_app` grants, verified applied
+ against a live `postgres:16`.
+
+ **Gates:** `integration.yml :: replay-e2e` (four services, a real database,
+ `AISOC_REPLAY_E2E_REQUIRED=1` so an unreachable database is a failure
+ rather than a skip), `ci.yml` cli job (`test_replay_command.py`), `ci.yml`
+ api job (`test_replay_evaluation.py`), `ci.yml` agents job
+ (`test_replay_route.py`), `ci.yml` actions job
+ (`test_replay_history_route.py`), and
+ `scripts/sync_vendored_benchmark.py --check`. Claim-to-gate matrix gains
+ three rows, all GATED.
- **Autonomy promoted by a measured track record, not by a settings toggle**
(gap-closure Phase 2.3). Phase 2.2 made agreement measurable. This is what
the measurement is for.
diff --git a/GAP_CLOSURE_PROGRESS.md b/GAP_CLOSURE_PROGRESS.md
index 2f79123a2..dd10b3a83 100644
--- a/GAP_CLOSURE_PROGRESS.md
+++ b/GAP_CLOSURE_PROGRESS.md
@@ -240,6 +240,10 @@ must not be recorded as done until the fourth exists. Note also that a true
end-to-end run spans three services, so its home is an integration test with
containers rather than any service's unit suite.
+Deviations D9 onward were recorded after the phase sections below and sit
+at the end of this file. The numbering is one sequence; only the placement
+differs, because two phases were in flight at once.
+
---
## Phase 1: Replay evaluation on a customer's own history
@@ -247,15 +251,38 @@ containers rather than any service's unit suite.
- [x] **1.1 History readers.** Shipped in [#903](https://github.com/beenuar/AiSOC/pull/903). Five readers on the clients in `services/actions`, which already own the credential path and already hold the writeback going the other way: `SplunkClient.list_closed_notables`, `SentinelClient.list_closed_incidents`, `ElasticClient.list_closed_signals`, `QRadarClient.list_closed_offenses`, `DefenderClient.list_resolved_alerts`. One taxonomy module (`app/services/alert_history.py`) rather than five that could disagree. 30 tests drive each reader's real HTTP path against vendor-shaped payloads; the `services/actions` suite goes 737 to 767. Claim-to-gate row added, matrix 147 rows to 148, GATED 139 to 140. Two vendor decisions recorded in `apps/docs/docs/evaluation/replay.md`: Elastic ships no disposition field so an untagged deployment yields no labels, and QRadar "Non-Issue" is `benign` not `benign_true_positive` because it makes no claim about whether the rule was right.
- [x] **1.2 Replay runner.** Shipped in [#904](https://github.com/beenuar/AiSOC/pull/904). `services/agents/app/replay/` holds the split, the shadow sinks and the runner; it holds no triage. Persistence is injected through `app/workers/triage_persistence.py`, whose default is `LiveTriageWriter` doing exactly what the worker did inline, so the measured path is the production one rather than a copy. `CostTracker` gained a `persist` flag so a replay measures spend without billing it. Normalisation reaches the real connector through a new `POST /connectors/{id}/normalize`, because both services package their code as top-level `app` and one process can hold one of them. Verdict, confidence, evidence, tool calls, model id, tokens, measured cost and latency are all recorded per decision. See D6 and D7 below for the two places the plan and the tree disagreed.
- [x] **1.3 Scoring.** Shipped in [#904](https://github.com/beenuar/AiSOC/pull/904). `packages/aisoc-benchmark/aisoc_benchmark/replay.py` reuses the existing `_INDICATOR_PATTERNS` for hallucination so there is one definition, and adds per-class precision and recall with malicious recall first, a confusion matrix, abstention rate, reliability bins with an expected calibration error, per-rule and per-source breakdowns, and seeded bootstrap intervals. Below 30 malicious cases the headline accuracy is withheld with the count and the reason. A rate with no denominator reads "not measured".
-- [ ] **1.4 Surfaces.** CLI `aisoc replay`, async API job with tenant-scoped tables, the "Evaluate on your history" console page, and JSON, Markdown and PDF export. Markdown and JSON rendering already ship with 1.3 (`format_replay_report`, `ReplayScore.as_dict`). The remaining work is scoped in D8 below, because the shape it has to take is not the obvious one and rediscovering that would cost a session.
-- [~] **1.5 Gates and docs.** Recorded vendor payload tests per reader shipped with 1.1. The leakage test shipped in [#904](https://github.com/beenuar/AiSOC/pull/904) (`services/agents/tests/test_replay_leakage.py`), covering all three stores a test-window decision can travel back through, each with a sensitivity half that runs the unprotected configuration and asserts it leaks. Three claim-to-gate rows added, matrix 148 rows to 151, GATED 140 to 143. `apps/docs/docs/evaluation/replay.md` covers the method, the limits and the privacy position. What remains is the documentation of the 1.4 surfaces once they exist.
+- [x] **1.4 Surfaces.** Shipped in [#907](https://github.com/beenuar/AiSOC/pull/907). Built to the shape D8 records: the API orchestrates, driving two new internal routes (`POST /replay/history` on actions, `POST /replay/run` on agents) and scoring in process through a byte-identical mirror of `packages/aisoc-benchmark` under `services/api/app/_vendor/`, since the API image's build context excludes `packages/`. `aisoc replay` drives the API and renders nothing of its own; progress goes to stderr so `aisoc replay ... > report.md` is the report and nothing else. The console page shows every rate beside the count it was computed over, and prints the withheld-headline sentence in place of a number rather than a dash or a zero. Export reuses `format_replay_report` and `ReplayScore.as_dict` from 1.3 and serves the stored artefact rather than re-rendering it; the PDF is that same Markdown through WeasyPrint, and answers 503 naming the native libraries when they are absent rather than serving an empty file. Migration **065**, not the 064 D8 names: `064_sandbox_upload_policy.sql` landed from Phase 11 while this was in flight, which is exactly why D8 says to check the directory. See D15 for the defect the end-to-end run found.
+- [x] **1.5 Gates and docs.** Recorded vendor payload tests per reader shipped with 1.1. The leakage test shipped in [#904](https://github.com/beenuar/AiSOC/pull/904) (`services/agents/tests/test_replay_leakage.py`), covering all three stores a test-window decision can travel back through, each with a sensitivity half that runs the unprotected configuration and asserts it leaks. Three claim-to-gate rows added, matrix 148 rows to 151, GATED 140 to 143. `apps/docs/docs/evaluation/replay.md` covers the method, the limits and the privacy position, and now the three surfaces as well: a "Running one" section for the console, the CLI and the API, and a "Reproducibility, stated precisely" section that names what is excluded and why. Its "what exists today" note no longer hedges, because nothing on the page is unbuilt. Three more claim-to-gate rows added for 1.4, all GATED. The absolute tally moves with whatever else lands, so recount it with `scripts/check_claim_gate_matrix.py` rather than reading a number off this line.
**Done when:** the CLI, run against a mocked Splunk ES holding 200 recorded
closed notables, produces a report that reproduces byte for byte on a second run
with the deterministic model path.
-**Not yet met.** Three of its four links are proven and the fourth is not built;
-the decomposition and what each link rests on are in D8 above.
+**Met.** `tests/e2e/test_replay_cli_end_to_end.py` is the fourth link, and it
+runs exactly that: a mock Splunk ES serving 200 closed notables over the two
+REST endpoints `SplunkClient.run_search` actually calls, four services started
+from the working tree with uvicorn, a real administrator created by the
+deployment's own bootstrap script and signed in through the real login route
+(the dev-mode bypass would have skipped the `connectors:write` and
+`reports:read` checks these routes declare), and the CLI invoked twice. The two
+reports are identical as bytes. Measured on this worktree: 200 findings read,
+200 carrying an analyst label, 60 replayed and graded, 40 malicious, headline
+printed rather than withheld.
+
+Two things stop that from being a vacuous pass. A second assertion fetches both
+reports **without** the latency exclusion and fails if more than that one line
+differs, so a stripped artefact cannot hide a field that quietly stopped
+reproducing. And further assertions require the report to carry the 200 findings
+read, the 60-finding graded window and a printed headline, because a withheld
+headline reproduces just as reliably while printing far fewer numbers, so a run
+that was accidentally thin would weaken the proof without failing it.
+
+Wall-clock latency is the only excluded field, and the exclusion lives beside
+the renderer that emits the line rather than in the CLI, so the producer and the
+remover cannot drift. The claim is therefore "byte for byte apart from the two
+latency figures, over a pinned window, on the deterministic model path", and
+each of those qualifiers is load-bearing: an unpinned window is a different
+window on a second run, and a hosted model may legitimately differ.
## Phase 2: Live shadow mode and evidence-gated autonomy
@@ -602,3 +629,35 @@ the fix was proven by re-staling `ROADMAP.md` and watching the test fail.
**Backgrounding a long run kills it here.** The first baseline attempt was
started with `nohup ... &` and was dead within a minute, having written 41
progress dots. Long suites run in the foreground.
+
+### D15. The route Phase 1.2 added could never have been reached, and only the end-to-end run could tell
+
+D8 predicted that three proven links are not the same claim as one end-to-end
+run. This is what the fourth link found the first time it ran.
+
+`POST /connectors/{id}/normalize` shipped in 1.2 as the resolution to D6: both
+services package their code as top-level `app`, so the agents service reaches
+the production connector mapping over HTTP rather than copying it. The client
+built its URL as `{CONNECTORS_SERVICE_URL}/connectors/{id}/normalize`. The
+connectors service mounts that router with `prefix="/api/v1"`, and the API's
+own caller in `endpoints/connectors.py` appends the same segment for exactly
+that reason. So every normalize request the agents service could have made
+would have returned 404, and no replay could have run on any deployment.
+
+Three suites were green over it. The unit test covering `fetch_normalized`
+asserted the URL the code produced rather than the URL the service serves,
+which is the one-directional shape this repository keeps finding: it compared
+the producer against a copy of itself and printed OK. Nothing else called the
+function, so nothing else could notice.
+
+**Resolution:** the prefix is a named constant with the reason recorded beside
+it, and the test now parses `services/connectors/app/main.py` and
+`app/api/router.py` with `ast`, derives what that service actually serves, and
+asserts the requested URL is in that set. It was proven against the pre-fix
+value: with the prefix removed it fails naming both URLs. This service cannot
+import that one, so reading the other tree's source is the only way to compare
+in the direction that drifts.
+
+The durable lesson is D8's, sharpened: an end-to-end test is not a slower
+version of the unit tests underneath it. It is the only thing that exercises
+the joins, and the joins are where a caller-less mechanism hides.
diff --git a/ROADMAP.md b/ROADMAP.md
index 0ef0b5b5b..015a43bee 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -44,7 +44,7 @@ A proof-first, security-first program to make every README claim gate-backed, cl
- **Phase D (breadth):** D1 eight connectors (QRadar/Exabeam/Securonix/Devo/Netskope/Windows-Sysmon/Zeek-Suricata/syslog-CEF) · D2 AI/LLM-usage audit connector + `llm-*` detections + hot/cold lake tiering · D3 live-vendor mock-server smoke.
- **Phase E (prove it):** E1 CI-gated benchmark scoreboard tied to a deterministic live-agent MITRE-accuracy run.
-The claim-to-gate matrix stands at **164 rows — 156 GATED / 8 PARTIAL / 0 NO GATE** — **every product claim is backed by a failing test** and the ratchet (`MAX_NO_GATE=0`) forbids any regression. The 8 remaining PARTIAL rows are honest, named deferrals; none was relabelled without building the gate it names. Count the table rows with `python3 scripts/check_claim_gate_matrix.py` rather than trusting a figure quoted in prose — this line has gone stale before. The six lettered deferrals are scoped in [`docs/audit/DEFERRED_SUBPHASES.md`](docs/audit/DEFERRED_SUBPHASES.md).
+The claim-to-gate matrix stands at **167 rows — 159 GATED / 8 PARTIAL / 0 NO GATE** — **every product claim is backed by a failing test** and the ratchet (`MAX_NO_GATE=0`) forbids any regression. The 8 remaining PARTIAL rows are honest, named deferrals; none was relabelled without building the gate it names. Count the table rows with `python3 scripts/check_claim_gate_matrix.py` rather than trusting a figure quoted in prose — this line has gone stale before. The six lettered deferrals are scoped in [`docs/audit/DEFERRED_SUBPHASES.md`](docs/audit/DEFERRED_SUBPHASES.md).
## v4.0 — Shipped
diff --git a/apps/docs/docs/compliance/evidence-pack.md b/apps/docs/docs/compliance/evidence-pack.md
index 12b4f5b2e..6e73b0623 100644
--- a/apps/docs/docs/compliance/evidence-pack.md
+++ b/apps/docs/docs/compliance/evidence-pack.md
@@ -33,7 +33,7 @@ and says so.
| Prompt-injection resistance | 32-payload adversarial corpus with a recall floor and a false-positive ceiling | `services/agents/tests/adversarial/` |
| Claim-to-gate traceability | Every product claim mapped to the CI job that fails when it stops being true | `docs/audit/CLAIM_TO_GATE_MATRIX.md` |
-The last row is the one worth reading first. It is the honest index: 156 rows
+The last row is the one worth reading first. It is the honest index: 159 rows
`GATED`, 8 `PARTIAL` with the specific gap named, and a ratchet that refuses
to let a row sit at `NO GATE`.
diff --git a/apps/docs/docs/evaluation/replay.md b/apps/docs/docs/evaluation/replay.md
index 68f0fda8c..f77755898 100644
--- a/apps/docs/docs/evaluation/replay.md
+++ b/apps/docs/docs/evaluation/replay.md
@@ -14,10 +14,10 @@ Replay evaluation answers the question that actually matters: **how does AiSOC
triage compare with what your analysts already decided, on your data?**
:::note What exists today
-This page documents the **history readers**, the **replay runner** and the
-**scoring report**. The `aisoc replay` command and the console page land next
-and this page grows with them. Nothing here describes a capability that is not
-in the tree.
+Everything on this page is in the tree and reachable: the history readers, the
+replay runner, the scoring report, the `aisoc replay` command, the API job and
+the **Evaluate on your history** console page. Nothing here describes a
+capability that does not exist.
:::
## The one rule worth reading first
@@ -222,6 +222,115 @@ fused confidence score, the deterministic narrative and entity resolution.
Every report states this in its method section. Replay measures triage on a
single normalized finding, not the whole pipeline.
+## Running one
+
+There are three ways in, and all three drive the same job. The API
+orchestrates it, because no single process can hold the three services
+involved: `services/actions` owns the SIEM credentials and the readers,
+`services/agents` owns triage, and `services/connectors` owns `normalize()`.
+All three package their code as a top-level `app`, so a process that imported
+two of them would get one of them.
+
+### The console
+
+**Evaluate on your history**, in the left-hand navigation. Pick a connected
+source and a window, and the page shows the report when the run finishes.
+
+Two things about how it presents a result are deliberate.
+
+**No number appears without the sample size behind it.** Recall on malicious
+renders with the count of malicious cases it was computed over, the headline
+accuracy with the count of answered decisions, and the history read with how
+many of those findings carried an analyst label at all. A precision of 1.00
+over two predictions is not a precision of 1.00 over two hundred, and a panel
+that prints only the ratio has thrown that away.
+
+**When the window is too thin for a headline, the page prints the reason
+rather than a number.** Below the floor of malicious cases the headline card
+reads "withheld" and the sentence explaining why is rendered underneath: how
+many malicious cases the window held, how many are required, and why a figure
+computed over fewer would describe the queue rather than the agent. It is not
+a dash, and it is not a zero. Zero would say the agent got every answer wrong,
+which is a different fact with a different remedy.
+
+The method block sits beside the report, not behind a tab: the split point,
+the frozen-context counts, the bootstrap seed and resample count, how many
+writes shadow mode intercepted, and the list of enrichments a replayed finding
+does not carry.
+
+### The CLI
+
+```bash
+aisoc replay --connector-id <uuid> --output report.md
+```
+
+The tenant comes from the credential (`--api-key`, or `AISOC_API_KEY`). There
+is no tenant flag, because a flag would be a value you chose that nothing
+checked.
+
+Useful options:
+
+| Option | What it does |
+|---|---|
+| `--since` / `--until` | Pin the window. Omitting them dates it from now, so a second run covers a different window and produces a different report. |
+| `--format markdown\|json\|pdf` | Which export to write. All three come from the artefact stored when the run completed. |
+| `--output PATH` | Where to write it. Without it, markdown and JSON go to stdout and every progress line goes to stderr, so `aisoc replay ... > report.md` produces the report and nothing else. |
+| `--exclude-latency` | Replace the two wall-clock latency figures with a note. See below. |
+| `--no-wait` / `--collect <id>` | Queue a run and collect it later. |
+| `--seed` / `--resamples` | Change the bootstrap. Both travel into the report either way. |
+
+### The API
+
+```
+POST /api/v1/evaluations/replay -> 202 with an id
+GET /api/v1/evaluations/replay/{id} -> poll to `completed` or `failed`
+GET /api/v1/evaluations/replay/{id}/export?format=markdown|json|pdf
+GET /api/v1/evaluations/replay/{id}/decisions
+```
+
+Starting a run needs `connectors:write`, the same bar as testing a connector,
+because it does the same kind of thing: an outbound call to your SIEM with
+stored credentials. Reading a report needs `reports:read`.
+
+Results live in two tenant-scoped tables with row-level-security policies. The
+report is stored as the renderer produced it and served back unchanged, rather
+than re-rendered on each request: a report re-rendered later by a newer
+renderer is a different artefact from the one you read. The per-decision rows
+are kept separately, and carry the evidence triage was given, so a number you
+disagree with can be re-derived rather than re-argued.
+
+## Reproducibility, stated precisely
+
+**A report reproduces byte for byte between two runs over the same pinned
+window, apart from the two wall-clock latency figures.**
+
+Everything else is a property of the input and the code: the bootstrap is
+seeded and records its seed, the split is a total order over `(closed_at,
+finding_id)`, and the renderer emits no timestamp and iterates nothing
+unsorted. Mean and p95 latency measure the machine the replay ran on and will
+differ between two runs on one host, let alone two.
+
+`--exclude-latency` on the CLI, and `exclude_latency=true` on the export
+route, replace those two figures with a note. That is a filter over the stored
+artefact, not a second rendering, and the function that removes the line lives
+beside the one that emits it so the two cannot drift.
+
+Two caveats worth stating plainly:
+
+- **Pin the window.** An omitted `--since` or `--until` dates the window from
+ now, which is a different window on a second run and therefore a different
+ report for a reason that has nothing to do with determinism.
+- **Reproducibility is not accuracy.** A deterministic model path reproduces
+ because it is deterministic. Pointed at a hosted model, two runs may differ,
+ and that is a property of the model rather than of this pipeline.
+
+`tests/e2e/test_replay_cli_end_to_end.py` is what holds this claim up. It runs
+the CLI twice against a mocked Splunk ES holding 200 recorded closed notables,
+through the API, the actions service, the connectors service and the agents
+service, and compares the bytes. It also fetches both reports *without* the
+exclusion and asserts that latency is the only line that differs, so the
+comparison cannot pass on a stripped artefact that hid something else.
+
## Privacy: where your data goes
Reading your history happens inside your deployment, against credentials you
@@ -256,6 +365,11 @@ through the same triage path production uses.
escalation is the only stage of this path that calls tools. The field is
recorded rather than omitted so a future change that gives triage a tool
shows up as a number moving off zero.
+- **PDF export needs a native stack.** WeasyPrint's Pango, Cairo and GLib
+ libraries are installed in the API container image and are routinely absent
+ on a local checkout. When they are, the PDF route answers 503 naming them,
+ rather than serving an empty file; JSON and Markdown carry the same report
+ and need nothing extra.
- **Hallucination counting errs high.** Indicators are extracted from the
agent's own reasoning with the same pattern set the synthetic-corpus grader
checks against, so a phrase shaped like a domain is checked and, if absent
diff --git a/apps/web/src/app/(app)/evaluate/page.tsx b/apps/web/src/app/(app)/evaluate/page.tsx
new file mode 100644
index 000000000..a4e18a0c9
--- /dev/null
+++ b/apps/web/src/app/(app)/evaluate/page.tsx
@@ -0,0 +1,17 @@
+import { ReplayEvaluationView } from '@/components/evaluations/ReplayEvaluationView';
+
+export const metadata = {
+ title: 'Evaluate on your history',
+ description:
+ "Measure AiSOC triage against your own analysts' past decisions, on your own closed findings.",
+};
+
+// Every read here is tenant-scoped and credential-backed, and a run in flight
+// changes between requests. Nothing about it is cacheable.
+export const dynamic = 'force-dynamic';
+export const revalidate = 0;
+export const fetchCache = 'force-no-store';
+
+export default function EvaluatePage() {
+ return <ReplayEvaluationView />;
+}
diff --git a/apps/web/src/components/evaluations/ReplayEvaluationView.tsx b/apps/web/src/components/evaluations/ReplayEvaluationView.tsx
new file mode 100644
index 000000000..ac7186567
--- /dev/null
+++ b/apps/web/src/components/evaluations/ReplayEvaluationView.tsx
@@ -0,0 +1,534 @@
+'use client';
+
+/**
+ * "Evaluate on your history": triage measured against this tenant's own analysts.
+ *
+ * Gap-closure Phase 1.4.
+ *
+ * The rule that shapes this whole view is that **no number appears without the
+ * sample size behind it**. A precision of 1.00 across two predictions is not a
+ * precision of 1.00 across two hundred, and a page that prints only the ratio
+ * has thrown the distinction away. So the headline is rendered inside a block
+ * that carries the counts, not above one.
+ *
+ * The thin-corpus case is the one this view exists to get right. Below the
+ * platform's floor of malicious cases the API returns `headline_accuracy: null`
+ * with a `headline_withheld_reason`, and this renders the reason in place of
+ * the number rather than a dash, a zero or an empty card. Zero would say the
+ * agent got every answer wrong, which is a different fact with a different
+ * remedy; a dash says nothing at all. The reason says which it is and why.
+ *
+ * Nothing here is mocked. A failed run shows its own error, an empty list shows
+ * an empty state, and a run still going shows its status. There is no
+ * fabricated sample report behind any of those.
+ */
+
+import { useCallback, useMemo, useState } from 'react';
+import useSWR from 'swr';
+import { clsx } from 'clsx';
+import {
+ connectorsApi,
+ evaluationsApi,
+ type Connector,
+ type ReplayCapabilities,
+ type ReplayEvaluationDetail,
+ type ReplayEvaluationSummary,
+ type ReplayExportFormat,
+} from '@/lib/api';
+import { EmptyState } from '@/components/ui/EmptyState';
+import { ErrorState } from '@/components/ui/ErrorState';
+
+/** Poll while a run is moving. Stops as soon as it reaches a terminal state. */
+const POLL_MS = 3000;
+
+function pct(value: number | null | undefined): string {
+ // "not measured" rather than "0%". A rate with no denominator and a rate
+ // that genuinely is zero are different facts, and only one of them is a
+ // statement about the agent.
+ return value === null || value === undefined ? 'not measured' : `${(value * 100).toFixed(1)}%`;
+}
+
+function when(value: string | null): string {
+ if (!value) return '';
+ const parsed = new Date(value);
+ return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString();
+}
+
+function windowLabel(run: ReplayEvaluationSummary): string {
+ const start = new Date(run.window_start);
+ const end = new Date(run.window_end);
+ if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime())) {
+ return `${run.window_start} to ${run.window_end}`;
+ }
+ return `${start.toLocaleDateString()} to ${end.toLocaleDateString()}`;
+}
+
+const STATUS_STYLES: Record<string, string> = {
+ queued: 'bg-slate-500/15 text-slate-300 border-slate-500/30',
+ running: 'bg-sky-500/15 text-sky-300 border-sky-500/30',
+ completed: 'bg-emerald-500/15 text-emerald-300 border-emerald-500/30',
+ failed: 'bg-rose-500/15 text-rose-300 border-rose-500/30',
+};
+
+function StatusPill({ status }: { status: string }) {
+ return (
+ <span
+ className={clsx(
+ 'inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium',
+ STATUS_STYLES[status] ?? STATUS_STYLES.queued,
+ )}
+ >
+ {status}
+ </span>
+ );
+}
+
+/**
+ * One measurement and the count it was computed over, always together.
+ */
+function Figure({
+ label,
+ value,
+ over,
+}: {
+ label: string;
+ value: string;
+ over: string;
+}) {
+ return (
+ <div className="rounded-lg border border-slate-800 bg-slate-900/40 p-4">
+ <div className="text-xs uppercase tracking-wide text-slate-400">{label}</div>
+ <div className="mt-1 text-2xl font-semibold text-slate-100">{value}</div>
+ <div className="mt-1 text-xs text-slate-400">{over}</div>
+ </div>
+ );
+}
+
+// ───────────────────────────────────────────────────────────────────────────
+// Starting a run
+// ───────────────────────────────────────────────────────────────────────────
+
+function StartPanel({
+ capabilities,
+ connectors,
+ onStarted,
+}: {
+ capabilities: ReplayCapabilities | undefined;
+ connectors: Connector[];
+ onStarted: (id: string) => void;
+}) {
+ const replayable = useMemo(() => {
+ // `Connector.type` is the wire field the list route returns; the
+ // capabilities route speaks `connector_type`. They are the same value
+ // under two names, and this is the only place the two meet.
+ const supported = new Set((capabilities?.connectors ?? []).map((c) => c.connector_type));
+ return connectors.filter((c) => supported.has(c.type));
+ }, [capabilities, connectors]);
+
+ // The chosen connector is derived rather than synchronised into state by an
+ // effect. The list arrives asynchronously, and seeding a default from an
+ // effect would render one frame with nothing selected and then re-render.
+ const [chosen, setChosen] = useState('');
+ const connectorId = chosen || replayable[0]?.id || '';
+ const [days, setDays] = useState(capabilities?.default_window_days ?? 90);
+ const [submitting, setSubmitting] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+
+ const start = useCallback(async () => {
+ if (!connectorId) return;
+ setSubmitting(true);
+ setError(null);
+ try {
+ const until = new Date();
+ const since = new Date(until.getTime() - days * 24 * 60 * 60 * 1000);
+ const run = await evaluationsApi.start({
+ connector_id: connectorId,
+ since: since.toISOString(),
+ until: until.toISOString(),
+ });
+ onStarted(run.id);
+ } catch (err) {
+ setError(err instanceof Error ? err.message : 'Could not start the evaluation.');
+ } finally {
+ setSubmitting(false);
+ }
+ }, [connectorId, days, onStarted]);
+
+ if (replayable.length === 0) {
+ return (
+ <EmptyState
+ title="No connected source can be replayed yet"
+ description={
+ 'Replay reads the findings your analysts already closed. It needs a connected ' +
+ (capabilities?.connectors ?? []).map((c) => c.label).join(', ') +
+ ' instance. Connect one, then come back.'
+ }
+ action={
+ <a
+ href="/connectors"
+ className="rounded-md bg-sky-600 px-4 py-2 text-sm font-medium text-white"
+ >
+ Connect a source
+ </a>
+ }
+ />
+ );
+ }
+
+ return (
+ <div className="rounded-xl border border-slate-800 bg-slate-900/40 p-5">
+ <h2 className="text-sm font-semibold text-slate-200">Run an evaluation</h2>
+ <p className="mt-1 max-w-3xl text-sm text-slate-400">
+ AiSOC reads the findings your analysts closed in this window, replays them through the
+ same triage path production runs, and grades the result against the labels they chose.
+ Nothing is written back to your SIEM and no alert, memory or action is created.
+ </p>
+
+ <div className="mt-4 flex flex-wrap items-end gap-4">
+ <label className="flex flex-col gap-1 text-xs text-slate-400">
+ Source
+ <select
+ className="min-w-[16rem] rounded-md border border-slate-700 bg-slate-950 px-3 py-2 text-sm text-slate-100"
+ value={connectorId}
+ onChange={(event) => setChosen(event.target.value)}
+ >
+ {replayable.map((connector) => (
+ <option key={connector.id} value={connector.id}>
+ {connector.name}
+ </option>
+ ))}
+ </select>
+ </label>
+
+ <label className="flex flex-col gap-1 text-xs text-slate-400">
+ Window
+ <select
+ className="rounded-md border border-slate-700 bg-slate-950 px-3 py-2 text-sm text-slate-100"
+ value={days}
+ onChange={(event) => setDays(Number(event.target.value))}
+ >
+ {[30, 60, 90, 180, 365].map((option) => (
+ <option key={option} value={option}>
+ Last {option} days
+ </option>
+ ))}
+ </select>
+ </label>
+
+ <button
+ type="button"
+ onClick={start}
+ disabled={submitting || !connectorId}
+ className="rounded-md bg-sky-600 px-4 py-2 text-sm font-medium text-white disabled:opacity-50"
+ >
+ {submitting ? 'Starting...' : 'Evaluate on my history'}
+ </button>
+ </div>
+
+ {capabilities ? (
+ <p className="mt-3 text-xs text-slate-500">
+ The earlier {Math.round(capabilities.default_train_fraction * 100)}% of the window fixes
+ what the agent is allowed to know; only the later period is graded. A headline accuracy
+ is printed only when that period holds at least {capabilities.min_malicious_for_headline}{' '}
+ confirmed-malicious cases.
+ </p>
+ ) : null}
+
+ {error ? <p className="mt-3 text-sm text-rose-400">{error}</p> : null}
+ </div>
+ );
+}
+
+// ───────────────────────────────────────────────────────────────────────────
+// The report
+// ───────────────────────────────────────────────────────────────────────────
+
+function Headline({ run }: { run: ReplayEvaluationDetail }) {
+ const withheld = run.headline_accuracy === null;
+ return (
+ <div className="rounded-xl border border-slate-800 bg-slate-900/40 p-5">
+ <div className="grid gap-4 sm:grid-cols-3">
+ <Figure
+ label="Recall on malicious"
+ value={pct(run.malicious_recall)}
+ over={`over ${run.malicious_support} confirmed-malicious case${run.malicious_support === 1 ? '' : 's'}`}
+ />
+ <Figure
+ label="Headline accuracy"
+ value={withheld ? 'withheld' : pct(run.headline_accuracy)}
+ over={withheld ? 'see the reason below' : `over ${run.graded} answered decisions`}
+ />
+ <Figure
+ label="History read"
+ value={String(run.findings_read)}
+ over={`${run.findings_labelled} carried an analyst label, ${run.decisions_recorded} replayed`}
+ />
+ </div>
+
+ {withheld && run.headline_withheld_reason ? (
+ <p className="mt-4 rounded-lg border border-amber-500/30 bg-amber-500/10 p-3 text-sm text-amber-200">
+ {run.headline_withheld_reason}
+ </p>
+ ) : null}
+ </div>
+ );
+}
+
+/**
+ * How the number was produced, next to the number itself.
+ *
+ * The method block is not a footnote here. A replayed finding carries none of
+ * the fusion-stage enrichment a live alert has, and the report says which
+ * enrichments are missing; an operator deciding whether to believe a figure
+ * needs that on the same screen.
+ */
+function MethodPanel({ run }: { run: ReplayEvaluationDetail }) {
+ const method = (run.method ?? {}) as Record<string, any>;
+ const split = (method.split ?? {}) as Record<string, any>;
+ const frozen = (method.frozen_context ?? {}) as Record<string, any>;
+ const limits: string[] = Array.isArray(method.envelope_limits) ? method.envelope_limits : [];
+ const writes = (method.shadow_writes_attempted ?? {}) as Record<string, number>;
+ const attempted = Object.values(writes).reduce((total, count) => total + (count || 0), 0);
+
+ return (
+ <div className="rounded-xl border border-slate-800 bg-slate-900/40 p-5">
+ <h3 className="text-sm font-semibold text-slate-200">Method and sample sizes</h3>
+
+ <dl className="mt-3 grid gap-x-8 gap-y-2 text-sm sm:grid-cols-2">
+ <div className="flex justify-between gap-4 border-b border-slate-800 py-1">
+ <dt className="text-slate-400">Source</dt>
+ <dd className="text-slate-200">{run.vendor}</dd>
+ </div>
+ <div className="flex justify-between gap-4 border-b border-slate-800 py-1">
+ <dt className="text-slate-400">Window</dt>
+ <dd className="text-slate-200">{windowLabel(run)}</dd>
+ </div>
+ <div className="flex justify-between gap-4 border-b border-slate-800 py-1">
+ <dt className="text-slate-400">Split</dt>
+ <dd className="text-slate-200">
+ {split.train_findings ?? 0} train / {split.test_findings ?? 0} graded, cut at{' '}
+ {when(split.split_at ?? null)}
+ </dd>
+ </div>
+ <div className="flex justify-between gap-4 border-b border-slate-800 py-1">
+ <dt className="text-slate-400">Confidence intervals</dt>
+ <dd className="text-slate-200">
+ bootstrap, {run.bootstrap_resamples} resamples, seed {run.bootstrap_seed}
+ </dd>
+ </div>
+ <div className="flex justify-between gap-4 border-b border-slate-800 py-1">
+ <dt className="text-slate-400">Frozen context</dt>
+ <dd className="text-slate-200">
+ {frozen.statements_frozen ?? 0} statements, {frozen.priors_frozen ?? 0} priors
+ {frozen.statements_dropped_after_split
+ ? `, ${frozen.statements_dropped_after_split} dropped as later than the split`
+ : ''}
+ </dd>
+ </div>
+ <div className="flex justify-between gap-4 border-b border-slate-800 py-1">
+ <dt className="text-slate-400">Writes suppressed</dt>
+ <dd className="text-slate-200">
+ {attempted} attempted, 0 performed
+ </dd>
+ </div>
+ </dl>
+
+ {limits.length > 0 ? (
+ <div className="mt-4">
+ <h4 className="text-xs font-semibold uppercase tracking-wide text-slate-400">
+ What a replayed finding does not carry
+ </h4>
+ <ul className="mt-2 list-disc space-y-1 pl-5 text-sm text-slate-400">
+ {limits.map((limit) => (
+ <li key={limit}>{limit}</li>
+ ))}
+ </ul>
+ </div>
+ ) : null}
+ </div>
+ );
+}
+
+function ExportBar({ id }: { id: string }) {
+ const formats: Array<{ format: ReplayExportFormat; label: string }> = [
+ { format: 'markdown', label: 'Markdown' },
+ { format: 'json', label: 'JSON' },
+ { format: 'pdf', label: 'PDF' },
+ ];
+ const [excludeLatency, setExcludeLatency] = useState(false);
+
+ return (
+ <div className="flex flex-wrap items-center gap-3">
+ <span className="text-xs uppercase tracking-wide text-slate-400">Export</span>
+ {formats.map(({ format, label }) => (
+ <a
+ key={format}
+ href={evaluationsApi.exportUrl(id, format, excludeLatency)}
+ className="rounded-md border border-slate-700 px-3 py-1.5 text-sm text-slate-200 hover:border-slate-500"
+ >
+ {label}
+ </a>
+ ))}
+ <label className="flex items-center gap-2 text-xs text-slate-400">
+ <input
+ type="checkbox"
+ checked={excludeLatency}
+ onChange={(event) => setExcludeLatency(event.target.checked)}
+ />
+ Exclude wall-clock latency, so two runs over one window compare cleanly
+ </label>
+ </div>
+ );
+}
+
+function ReportPanel({ run }: { run: ReplayEvaluationDetail }) {
+ if (run.status === 'failed') {
+ return (
+ <ErrorState
+ title="This evaluation did not complete"
+ description={
+ run.error ??
+ 'No reason was recorded. Nothing was measured, so no report was produced.'
+ }
+ />
+ );
+ }
+ if (run.status !== 'completed') {
+ return (
+ <div className="rounded-xl border border-slate-800 bg-slate-900/40 p-5 text-sm text-slate-400">
+ This evaluation is {run.status}. Replaying a window of findings through triage takes a
+ few minutes; the report appears here when it finishes.
+ </div>
+ );
+ }
+
+ return (
+ <div className="space-y-4">
+ <Headline run={run} />
+ <MethodPanel run={run} />
+ <ExportBar id={run.id} />
+ <div className="rounded-xl border border-slate-800 bg-slate-900/40 p-5">
+ <h3 className="text-sm font-semibold text-slate-200">Full report</h3>
+ {/*
+ Rendered as preformatted text rather than through a Markdown
+ renderer. The report embeds connector-supplied values and model
+ output (rule ids, vendor names, hallucinated indicator examples),
+ and this is the one presentation that cannot turn any of it into
+ markup. The same bytes are what the Markdown export downloads.
+ */}
+ <pre className="mt-3 max-h-[36rem] overflow-auto whitespace-pre-wrap break-words rounded-lg bg-slate-950 p-4 font-mono text-xs leading-relaxed text-slate-300">
+ {run.report_markdown}
+ </pre>
+ </div>
+ </div>
+ );
+}
+
+// ───────────────────────────────────────────────────────────────────────────
+// The page
+// ───────────────────────────────────────────────────────────────────────────
+
+export function ReplayEvaluationView() {
+ const [selected, setSelected] = useState<string | null>(null);
+
+ const { data: capabilities } = useSWR('replay-capabilities', () =>
+ evaluationsApi.capabilities(),
+ );
+ const { data: connectorList } = useSWR('replay-connectors', () => connectorsApi.list());
+ // `connectorsApi.list()` returns the envelope, not the array. Unwrapped once
+ // here rather than in the panel, so the panel takes a plain list.
+ const connectors: Connector[] = Array.isArray(connectorList)
+ ? connectorList
+ : (connectorList?.connectors ?? []);
+ const {
+ data: runs,
+ error: runsError,
+ isLoading,
+ mutate: refreshRuns,
+ } = useSWR('replay-evaluations', () => evaluationsApi.list(), {
+ refreshInterval: POLL_MS,
+ });
+
+ const activeId = selected ?? runs?.[0]?.id ?? null;
+ const activeSummary = runs?.find((run) => run.id === activeId);
+
+ const { data: detail } = useSWR(
+ activeId ? ['replay-evaluation', activeId] : null,
+ () => evaluationsApi.get(activeId as string),
+ {
+ // Stop polling the moment the run stops moving. A terminal run will
+ // never change again, and a page left open should not keep asking.
+ refreshInterval: activeSummary?.is_terminal ? 0 : POLL_MS,
+ },
+ );
+
+ const onStarted = useCallback(
+ (id: string) => {
+ setSelected(id);
+ void refreshRuns();
+ },
+ [refreshRuns],
+ );
+
+ return (
+ <div className="space-y-6 p-6">
+ <header>
+ <h1 className="text-xl font-semibold text-slate-100">Evaluate on your history</h1>
+ <p className="mt-1 max-w-3xl text-sm text-slate-400">
+ Measure AiSOC triage against your own analysts&apos; past decisions, on your own data,
+ before trusting it. Every rate below travels with the number of cases it was computed
+ over.
+ </p>
+ </header>
+
+ <StartPanel capabilities={capabilities} connectors={connectors} onStarted={onStarted} />
+
+ {runsError ? (
+ <ErrorState title="Could not load past evaluations" error={runsError} />
+ ) : null}
+
+ {!runsError && !isLoading && (runs?.length ?? 0) === 0 ? (
+ <EmptyState
+ title="No evaluation has been run yet"
+ description="Start one above. It reads only findings your analysts already closed, and writes nothing back."
+ />
+ ) : null}
+
+ {(runs?.length ?? 0) > 0 ? (
+ <div className="grid gap-6 lg:grid-cols-[18rem_1fr]">
+ <aside className="space-y-2">
+ <h2 className="text-xs font-semibold uppercase tracking-wide text-slate-400">
+ Evaluations
+ </h2>
+ {runs?.map((run) => (
+ <button
+ key={run.id}
+ type="button"
+ onClick={() => setSelected(run.id)}
+ className={clsx(
+ 'w-full rounded-lg border p-3 text-left',
+ run.id === activeId
+ ? 'border-sky-500/50 bg-sky-500/10'
+ : 'border-slate-800 bg-slate-900/40 hover:border-slate-700',
+ )}
+ >
+ <div className="flex items-center justify-between gap-2">
+ <span className="text-sm text-slate-200">{run.vendor}</span>
+ <StatusPill status={run.status} />
+ </div>
+ <div className="mt-1 text-xs text-slate-400">{windowLabel(run)}</div>
+ <div className="mt-1 text-xs text-slate-500">
+ {run.status === 'completed'
+ ? `${run.graded} graded, ${run.malicious_support} malicious`
+ : when(run.created_at)}
+ </div>
+ </button>
+ ))}
+ </aside>
+
+ <section>{detail ? <ReportPanel run={detail} /> : null}</section>
+ </div>
+ ) : null}
+ </div>
+ );
+}
diff --git a/apps/web/src/components/layout/Sidebar.tsx b/apps/web/src/components/layout/Sidebar.tsx
index 9ed180dde..fc1a15d26 100644
--- a/apps/web/src/components/layout/Sidebar.tsx
+++ b/apps/web/src/components/layout/Sidebar.tsx
@@ -193,6 +193,11 @@ const navSections: NavSection[] = [
href: '/federated-search',
icon: <SearchIcon />,
},
+ {
+ label: 'Evaluate on History',
+ href: '/evaluate',
+ icon: <ChartBarIcon />,
+ },
{
label: 'Explore',
href: '/explore',
diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts
index 214a10e77..c975bf8fc 100644
--- a/apps/web/src/lib/api.ts
+++ b/apps/web/src/lib/api.ts
@@ -6090,3 +6090,120 @@ export const apiKeysApi = {
revoke: (id: string) =>
request<void>(`/api/v1/api-keys/${id}`, { method: 'DELETE' }),
};
+
+// ─── Replay evaluation (gap-closure Phase 1.4) ───────────────────────────────
+//
+// Wraps `services/api/app/api/v1/endpoints/evaluations.py`: measure triage
+// against a tenant's own analysts, on their own closed findings.
+//
+// Every field the console needs to show sample sizes beside the headline is
+// on the summary, not buried in `score`. That is deliberate on both sides: a
+// headline accuracy rendered without the count behind it is the single most
+// misleading thing this surface could print, so the two travel together
+// through the wire shape rather than by a convention the UI has to remember.
+//
+// `headline_accuracy` is `null` when the window held too few malicious cases
+// for a headline to mean anything, and `headline_withheld_reason` carries the
+// sentence explaining that. Null is not zero: a zero in an accuracy column
+// says the agent got every answer wrong, which is a different fact with a
+// different remedy.
+
+export type ReplayEvaluationStatus = 'queued' | 'running' | 'completed' | 'failed';
+
+export interface ReplayEvaluationSummary {
+ id: string;
+ status: ReplayEvaluationStatus;
+ error: string | null;
+ connector_id: string;
+ vendor: string;
+ window_start: string;
+ window_end: string;
+ train_fraction: number;
+ bootstrap_seed: number;
+ bootstrap_resamples: number;
+ findings_read: number;
+ findings_labelled: number;
+ decisions_recorded: number;
+ graded: number;
+ malicious_support: number;
+ headline_accuracy: number | null;
+ headline_withheld_reason: string | null;
+ malicious_recall: number | null;
+ created_at: string;
+ started_at: string | null;
+ completed_at: string | null;
+ is_terminal: boolean;
+}
+
+export interface ReplayEvaluationDetail extends ReplayEvaluationSummary {
+ /** The full `ReplayScore.as_dict()`. */
+ score: Record<string, unknown> | null;
+ /** Split point, frozen-context provenance, attempted writes, envelope gaps. */
+ method: Record<string, unknown> | null;
+ /** The artefact stored when the run completed, not a re-render. */
+ report_markdown: string | null;
+}
+
+export interface ReplayableConnectorInfo {
+ connector_type: string;
+ vendor: string;
+ label: string;
+}
+
+export interface ReplayCapabilities {
+ connectors: ReplayableConnectorInfo[];
+ default_window_days: number;
+ default_train_fraction: number;
+ default_bootstrap_seed: number;
+ default_bootstrap_resamples: number;
+ /** Below this many malicious cases no headline accuracy is printed. */
+ min_malicious_for_headline: number;
+ max_findings: number;
+}
+
+export interface StartReplayInput {
+ connector_id: string;
+ since?: string;
+ until?: string;
+ train_fraction?: number;
+ limit?: number;
+}
+
+export type ReplayExportFormat = 'markdown' | 'json' | 'pdf';
+
+export const evaluationsApi = {
+ capabilities: () =>
+ request<ReplayCapabilities>('/api/v1/evaluations/replay/capabilities'),
+
+ list: (limit = 50) =>
+ request<ReplayEvaluationSummary[]>('/api/v1/evaluations/replay', {
+ params: { limit },
+ }),
+
+ get: (id: string) =>
+ request<ReplayEvaluationDetail>(`/api/v1/evaluations/replay/${id}`),
+
+ decisions: (id: string, limit = 2000) =>
+ request<Array<Record<string, unknown>>>(
+ `/api/v1/evaluations/replay/${id}/decisions`,
+ { params: { limit } },
+ ),
+
+ start: (data: StartReplayInput) =>
+ request<ReplayEvaluationSummary>('/api/v1/evaluations/replay', {
+ method: 'POST',
+ body: JSON.stringify(data),
+ }),
+
+ /**
+ * Where a download points. Built rather than fetched because the browser
+ * does the download, and the route already sets `Content-Disposition`.
+ *
+ * `exclude_latency` is offered on every format: the two wall-clock figures
+ * are the only part of the report that does not reproduce between runs, so
+ * an operator diffing two exports wants them gone.
+ */
+ exportUrl: (id: string, format: ReplayExportFormat, excludeLatency = false): string =>
+ `${API_BASE}/api/v1/evaluations/replay/${id}/export` +
+ `?format=${format}&exclude_latency=${excludeLatency ? 'true' : 'false'}`,
+};
diff --git a/docs/audit/CLAIM_TO_GATE_MATRIX.md b/docs/audit/CLAIM_TO_GATE_MATRIX.md
index 49f08ab50..c6d299d87 100644
--- a/docs/audit/CLAIM_TO_GATE_MATRIX.md
+++ b/docs/audit/CLAIM_TO_GATE_MATRIX.md
@@ -172,10 +172,13 @@ Statuses: `GATED` (a CI job fails when the claim stops being true) · `PARTIAL`
| Every autonomy transition is in the hash-chained audit log with its evidence | `plans/aisoc_gap_closure_plan.plan.md` (2.3); `apps/docs/docs/operations/shadow-mode.md` | `integration.yml` (`TestTheTransitionsAreInTheHashChainedLog` - a promotion and the demotion that follows it are replayed through `audit_hash.verify_chain`, the same pure function an external auditor would run over a CSV export, and the second entry is asserted to chain onto the first rather than starting a new one. A `DELETE` against the trail is asserted to raise, because a grant whose record can be quietly removed has no audit trail) | GATED | The chain and the snapshot's presence are gated. The snapshot's *sufficiency* is a judgement: it carries the counts, the thresholds by value, absolute window bounds, the first and last decision id, the models and a digest of the rules, which is what a later reader needs, but no test can prove a question nobody has asked yet is answerable |
| A track record cannot unlock a high-blast containment | `apps/docs/docs/operations/shadow-mode.md` (what a grant can and cannot unlock); `services/actions/app/live_actions/dispatcher.py` | `ci.yml` actions job (`test_earned_autonomy_wiring.py` - an earned grant raises the tier ceiling to L3 and no further, so MINIMAL, LOW and MEDIUM blast are reachable and HIGH and CRITICAL are not, whatever the evidence says; the capability contract still applies on top, so a verb declared `analyst` or `mandatory_human` stays gated; and a grant is asserted never to lower a bar, only to widen the branch that was already heading for the approval queue) | GATED | The ceiling is gated. The reasoning behind it is a judgement and is stated where the code is: agreement on triage verdicts is evidence about the agent's judgement, not evidence that a high-blast containment was the right call, and one number must not be allowed to unlock both |
| A replay report prints "not measured" where there was no denominator, and no headline accuracy on a corpus too thin to support one | `apps/docs/docs/evaluation/replay.md`; `packages/aisoc-benchmark/aisoc_benchmark/replay.py` | `ci.yml` benchmark job (`test_replay_metrics.py` — a class the agent never predicted reports `None` precision and renders as "not measured", because `0.0` in a recall column says the agent missed every case and "never asked" is a different fact; below 30 malicious cases the headline is withheld with the count and the reason, and the rendered report is asserted to contain no percentage in that section. Three populations are asserted apart: unlabelled findings excluded from accuracy and counted, triage refusals scored as neither answer nor abstention, and abstentions counted against malicious recall. A verdict outside the taxonomy gets its own confusion-matrix column rather than being folded into one the agent did not say. Bootstrap intervals are seeded and the seed and resample count travel in the report, so two runs over one corpus render byte-identically) | GATED | Hallucination extraction reuses the existing `_INDICATOR_PATTERNS`, which over-counts rather than under-counts: a phrase shaped like a domain is checked and, if absent from the evidence, counted. The published rate is a ceiling and the per-decision list travels with it so a disputed count can be re-derived |
+| An operator can measure triage against their own analysts from the CLI, the API or the console, and the report reproduces byte for byte | `apps/docs/docs/evaluation/replay.md` (Running one; Reproducibility, stated precisely); `packages/aisoc-cli` `aisoc replay` | `integration.yml :: replay-e2e` (`tests/e2e/test_replay_cli_end_to_end.py` — the CLI is run **twice** against a mocked Splunk ES holding 200 recorded closed notables, through the API, `services/actions`, `services/connectors` and `services/agents`, each started from the working tree rather than from a published image, and the two reports are compared as bytes. The comparison cannot pass vacuously: a second assertion fetches both reports *without* the latency exclusion and fails if more than that one line differs, so a stripped artefact cannot hide a field that stopped reproducing; further assertions require the report to carry the 200 findings read, the 60-finding graded window and a printed headline, because a withheld headline reproduces just as reliably while printing far fewer numbers. `AISOC_REPLAY_E2E_REQUIRED=1` makes an unreachable database a failure rather than a skip) + `ci.yml` cli job (`test_replay_command.py` — the CLI renders nothing of its own and writes the bytes the API returned, progress goes to stderr so `aisoc replay ... > report.md` is the report and nothing else, an omitted window is not filled in from the local clock, and a failed evaluation exits non-zero with the server's reason and writes no file) + `ci.yml` benchmark job (`test_replay_metrics.py` — two corpora differing only in latency render differently and identically once excluded; the stripper removes exactly the line the renderer emits, in both directions) | GATED | Reproducibility is a property of the deterministic model path and of a **pinned** window. An omitted `--since` / `--until` dates the window from now, which is a different window on a second run; pointed at a hosted model two runs may legitimately differ. No hosted provider has been exercised end to end here |
+| A replay evaluation's results are tenant-scoped, and its tenant comes from the credential rather than the request | `apps/docs/docs/evaluation/replay.md` (The API); `services/api/migrations/065_replay_evaluations.sql` | `isolation.yml :: route-scope` (`check_route_auth.py` — the six new `/evaluations/replay*` routes carry an auth dependency or the gate fails; `check_route_tenant_scope.py` — no route, query parameter or request-body field on this surface names a tenant, and the internal routes it drives resolve theirs from the credential: `services/agents` through the vendored `tenant_scope` where a service token with no `X-AiSOC-Tenant-ID` resolves to an **empty** scope that refuses rather than widens, and `services/actions` by taking no tenant at all, since the credentials it is handed are the scope) + `isolation.yml` (`check_rls_policy_shape.py` — both new tables carry `FORCE` and the `OR current_tenant_id() IS NULL` arm, verified applied against a live `postgres:16`) + `ci.yml` api job (`test_replay_evaluation.py` — a connector in another tenant resolves to no row and the run fails rather than reading it; an evaluation id in another tenant is a 404 rather than a 403, since telling the two apart would confirm the id exists) + `ci.yml` agents job (`test_replay_route.py` — a service token with no tenant header is refused with no downstream call made, and the tenant the credential named is the one forwarded to the connectors service) | GATED | The per-decision rows carry the evidence triage was given, so a tenant's own alert content is at rest in `aisoc_replay_decisions`. That is the point (a disputed score has to be re-derivable) and it is stated on the docs page rather than left to be discovered |
+| The API scores a replay with the grader the benchmark package ships, not a second copy of it | `services/api/app/_vendor/aisoc_benchmark/VENDORED.md` | `ci.yml :: vendored-sync` (`scripts/sync_vendored_benchmark.py --check` — every module of `packages/aisoc-benchmark/aisoc_benchmark/` must be byte-identical in `services/api/app/_vendor/aisoc_benchmark/`, checked in **both** directions so a module present only in the mirror fails too, and refusing to report a clean mirror when the source directory yields no modules) | GATED | A mirror rather than an import, because the `aisoc-core-api` image is built with `./services/api` as its context and nothing under `packages/` exists at runtime. Same constraint that produced `_vendor/narrative.py` and `_vendor/nl_query/`. A second grader would eventually disagree with the first about what "hallucination rate" means while both kept calling themselves the same number |
## Summary
-- GATED: 156
+- GATED: 159
- PARTIAL: 8
- NO GATE: 0 (**every claim is backed by a failing test.** The last NO GATE — the weekly benchmark scoreboard running live against `main` — closed in Phase E1: `scripts/check_scoreboard.py` ties the published scoreboard to a deterministic per-PR live-agent MITRE-accuracy run, while the funded weekly `wet-eval.yml` appends the LLM-tier rows. The remaining PARTIAL rows are honest, named deferrals — each states the specific gap and what closes it — not unproven claims.)
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index e47df3956..82a97198f 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -12225,6 +12225,233 @@ paths:
$ref: '#/components/schemas/HTTPValidationError'
security:
- HTTPBearer: []
+ /api/v1/evaluations/replay/capabilities:
+ get:
+ tags:
+ - evaluations
+ summary: Which connectors can be replayed on this deployment
+ operationId: capabilities_api_v1_evaluations_replay_capabilities_get
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/CapabilitiesResponse'
+ security:
+ - HTTPBearer: []
+ /api/v1/evaluations/replay:
+ post:
+ tags:
+ - evaluations
+ summary: Start a replay evaluation against this tenant's own history
+ description: 'Queue a run and return its id immediately.
+
+
+ 202 rather than 200: replaying a few hundred findings through a model is
+
+ not a request-response operation, and a route that blocked on it would
+
+ time out at whichever proxy is in front of it while the work continued.'
+ operationId: start_replay_api_v1_evaluations_replay_post
+ security:
+ - HTTPBearer: []
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/StartReplayRequest'
+ responses:
+ '202':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/EvaluationSummary'
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
+ get:
+ tags:
+ - evaluations
+ summary: This tenant's replay evaluations
+ operationId: list_replays_api_v1_evaluations_replay_get
+ security:
+ - HTTPBearer: []
+ parameters:
+ - name: limit
+ in: query
+ required: false
+ schema:
+ type: integer
+ maximum: 200
+ minimum: 1
+ default: 50
+ title: Limit
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluationSummary'
+ title: Response List Replays Api V1 Evaluations Replay Get
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
+ /api/v1/evaluations/replay/{evaluation_id}:
+ get:
+ tags:
+ - evaluations
+ summary: One replay evaluation
+ operationId: get_replay_api_v1_evaluations_replay__evaluation_id__get
+ security:
+ - HTTPBearer: []
+ parameters:
+ - name: evaluation_id
+ in: path
+ required: true
+ schema:
+ type: string
+ format: uuid
+ title: Evaluation Id
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/EvaluationDetail'
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
+ /api/v1/evaluations/replay/{evaluation_id}/decisions:
+ get:
+ tags:
+ - evaluations
+ summary: The decisions behind a report
+ description: 'Every replayed finding, so a disputed number can be re-derived.
+
+
+ Read separately from the report because these rows carry the evidence
+
+ handed to triage and are measured in megabytes, and the page that shows
+
+ the headline should not pay for them.'
+ operationId: get_replay_decisions_api_v1_evaluations_replay__evaluation_id__decisions_get
+ security:
+ - HTTPBearer: []
+ parameters:
+ - name: evaluation_id
+ in: path
+ required: true
+ schema:
+ type: string
+ format: uuid
+ title: Evaluation Id
+ - name: limit
+ in: query
+ required: false
+ schema:
+ type: integer
+ maximum: 2000
+ minimum: 1
+ default: 2000
+ title: Limit
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ type: array
+ items:
+ type: object
+ additionalProperties: true
+ title: Response Get Replay Decisions Api V1 Evaluations Replay Evaluation
+ Id Decisions Get
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
+ /api/v1/evaluations/replay/{evaluation_id}/export:
+ get:
+ tags:
+ - evaluations
+ summary: Export a report as JSON, Markdown or PDF
+ description: 'Serve the stored report, not a fresh render of it.
+
+
+ All three formats come from the artefact written when the run completed.
+
+ Re-rendering later would produce a document that no longer matches what
+
+ the operator read, and this phase''s acceptance bar is a report that
+
+ reproduces byte for byte.'
+ operationId: export_replay_api_v1_evaluations_replay__evaluation_id__export_get
+ security:
+ - HTTPBearer: []
+ parameters:
+ - name: evaluation_id
+ in: path
+ required: true
+ schema:
+ type: string
+ format: uuid
+ title: Evaluation Id
+ - name: format
+ in: query
+ required: false
+ schema:
+ enum:
+ - json
+ - markdown
+ - pdf
+ type: string
+ default: markdown
+ title: Format
+ - name: exclude_latency
+ in: query
+ required: false
+ schema:
+ type: boolean
+ description: Replace the two wall-clock latency figures with a note. Everything
+ else in the report is a property of the input and the code, so two runs
+ over one pinned window are then byte-identical; latency measures the host
+ and never will be.
+ default: false
+ title: Exclude Latency
+ description: Replace the two wall-clock latency figures with a note. Everything
+ else in the report is a property of the input and the code, so two runs
+ over one pinned window are then byte-identical; latency measures the host
+ and never will be.
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema: {}
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
/api/v1/kb/ingest:
post:
tags:
@@ -17746,6 +17973,50 @@ components:
among analyst-dispositioned alerts whose ai_score landed in that bucket
— i.e., the empirical hit-rate the model claimed.'
+ CapabilitiesResponse:
+ properties:
+ connectors:
+ items:
+ $ref: '#/components/schemas/ReplayableConnectorInfo'
+ type: array
+ title: Connectors
+ default_window_days:
+ type: integer
+ title: Default Window Days
+ default_train_fraction:
+ type: number
+ title: Default Train Fraction
+ default_bootstrap_seed:
+ type: integer
+ title: Default Bootstrap Seed
+ default_bootstrap_resamples:
+ type: integer
+ title: Default Bootstrap Resamples
+ min_malicious_for_headline:
+ type: integer
+ title: Min Malicious For Headline
+ max_findings:
+ type: integer
+ title: Max Findings
+ type: object
+ required:
+ - connectors
+ - default_window_days
+ - default_train_fraction
+ - default_bootstrap_seed
+ - default_bootstrap_resamples
+ - min_malicious_for_headline
+ - max_findings
+ title: CapabilitiesResponse
+ description: 'What this deployment can replay, so the console offers nothing
+ it cannot.
+
+
+ ``min_malicious_for_headline`` travels to the client so the page can say
+
+ why a headline is withheld before a run is started, rather than only
+
+ after.'
CaseMetrics:
properties:
open:
@@ -20261,6 +20532,235 @@ components:
asserts it fires on every positive fixture and stays silent on every
negative one. Approval now requires this verdict to pass.'
+ EvaluationDetail:
+ properties:
+ id:
+ type: string
+ format: uuid
+ title: Id
+ status:
+ type: string
+ title: Status
+ error:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Error
+ connector_id:
+ type: string
+ title: Connector Id
+ vendor:
+ type: string
+ title: Vendor
+ window_start:
+ type: string
+ format: date-time
+ title: Window Start
+ window_end:
+ type: string
+ format: date-time
+ title: Window End
+ train_fraction:
+ type: number
+ title: Train Fraction
+ bootstrap_seed:
+ type: integer
+ title: Bootstrap Seed
+ bootstrap_resamples:
+ type: integer
+ title: Bootstrap Resamples
+ findings_read:
+ type: integer
+ title: Findings Read
+ findings_labelled:
+ type: integer
+ title: Findings Labelled
+ decisions_recorded:
+ type: integer
+ title: Decisions Recorded
+ graded:
+ type: integer
+ title: Graded
+ malicious_support:
+ type: integer
+ title: Malicious Support
+ headline_accuracy:
+ anyOf:
+ - type: number
+ - type: 'null'
+ title: Headline Accuracy
+ headline_withheld_reason:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Headline Withheld Reason
+ malicious_recall:
+ anyOf:
+ - type: number
+ - type: 'null'
+ title: Malicious Recall
+ created_at:
+ type: string
+ format: date-time
+ title: Created At
+ started_at:
+ anyOf:
+ - type: string
+ format: date-time
+ - type: 'null'
+ title: Started At
+ completed_at:
+ anyOf:
+ - type: string
+ format: date-time
+ - type: 'null'
+ title: Completed At
+ is_terminal:
+ type: boolean
+ title: Is Terminal
+ score:
+ anyOf:
+ - additionalProperties: true
+ type: object
+ - type: 'null'
+ title: Score
+ method:
+ anyOf:
+ - additionalProperties: true
+ type: object
+ - type: 'null'
+ title: Method
+ report_markdown:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Report Markdown
+ type: object
+ required:
+ - id
+ - status
+ - connector_id
+ - vendor
+ - window_start
+ - window_end
+ - train_fraction
+ - bootstrap_seed
+ - bootstrap_resamples
+ - findings_read
+ - findings_labelled
+ - decisions_recorded
+ - graded
+ - malicious_support
+ - created_at
+ - is_terminal
+ title: EvaluationDetail
+ description: A completed run with its report and the provenance behind it.
+ EvaluationSummary:
+ properties:
+ id:
+ type: string
+ format: uuid
+ title: Id
+ status:
+ type: string
+ title: Status
+ error:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Error
+ connector_id:
+ type: string
+ title: Connector Id
+ vendor:
+ type: string
+ title: Vendor
+ window_start:
+ type: string
+ format: date-time
+ title: Window Start
+ window_end:
+ type: string
+ format: date-time
+ title: Window End
+ train_fraction:
+ type: number
+ title: Train Fraction
+ bootstrap_seed:
+ type: integer
+ title: Bootstrap Seed
+ bootstrap_resamples:
+ type: integer
+ title: Bootstrap Resamples
+ findings_read:
+ type: integer
+ title: Findings Read
+ findings_labelled:
+ type: integer
+ title: Findings Labelled
+ decisions_recorded:
+ type: integer
+ title: Decisions Recorded
+ graded:
+ type: integer
+ title: Graded
+ malicious_support:
+ type: integer
+ title: Malicious Support
+ headline_accuracy:
+ anyOf:
+ - type: number
+ - type: 'null'
+ title: Headline Accuracy
+ headline_withheld_reason:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Headline Withheld Reason
+ malicious_recall:
+ anyOf:
+ - type: number
+ - type: 'null'
+ title: Malicious Recall
+ created_at:
+ type: string
+ format: date-time
+ title: Created At
+ started_at:
+ anyOf:
+ - type: string
+ format: date-time
+ - type: 'null'
+ title: Started At
+ completed_at:
+ anyOf:
+ - type: string
+ format: date-time
+ - type: 'null'
+ title: Completed At
+ is_terminal:
+ type: boolean
+ title: Is Terminal
+ type: object
+ required:
+ - id
+ - status
+ - connector_id
+ - vendor
+ - window_start
+ - window_end
+ - train_fraction
+ - bootstrap_seed
+ - bootstrap_resamples
+ - findings_read
+ - findings_labelled
+ - decisions_recorded
+ - graded
+ - malicious_support
+ - created_at
+ - is_terminal
+ title: EvaluationSummary
+ description: One run as the list and the poll return it.
EventListResponse:
properties:
items:
@@ -26386,6 +26886,23 @@ components:
required:
- yaml
title: ReplaceRulesRequest
+ ReplayableConnectorInfo:
+ properties:
+ connector_type:
+ type: string
+ title: Connector Type
+ vendor:
+ type: string
+ title: Vendor
+ label:
+ type: string
+ title: Label
+ type: object
+ required:
+ - connector_type
+ - vendor
+ - label
+ title: ReplayableConnectorInfo
ReportDefinitionBody:
properties:
name:
@@ -28304,6 +28821,51 @@ components:
- disposition
title: SourceWritebackRequest
description: What the caller believes about the alert.
+ StartReplayRequest:
+ properties:
+ connector_id:
+ type: string
+ format: uuid
+ title: Connector Id
+ since:
+ anyOf:
+ - type: string
+ format: date-time
+ - type: 'null'
+ title: Since
+ until:
+ anyOf:
+ - type: string
+ format: date-time
+ - type: 'null'
+ title: Until
+ train_fraction:
+ type: number
+ exclusiveMaximum: 1.0
+ exclusiveMinimum: 0.0
+ title: Train Fraction
+ default: 0.7
+ limit:
+ type: integer
+ maximum: 2000.0
+ minimum: 1.0
+ title: Limit
+ default: 1000
+ bootstrap_seed:
+ type: integer
+ title: Bootstrap Seed
+ default: 20260926
+ bootstrap_resamples:
+ type: integer
+ maximum: 20000.0
+ minimum: 0.0
+ title: Bootstrap Resamples
+ default: 1000
+ type: object
+ required:
+ - connector_id
+ title: StartReplayRequest
+ description: Which connector, which window, and how the report should be computed.
StepDispatchRequest:
properties:
capability:
diff --git a/packages/aisoc-benchmark/aisoc_benchmark/__init__.py b/packages/aisoc-benchmark/aisoc_benchmark/__init__.py
index c9abe6343..a8f0ae43f 100644
--- a/packages/aisoc-benchmark/aisoc_benchmark/__init__.py
+++ b/packages/aisoc-benchmark/aisoc_benchmark/__init__.py
@@ -14,6 +14,8 @@ from .metrics import (
score_incident,
)
from .replay import (
+ LATENCY_FIELDS,
+ LATENCY_LINE_PREFIX,
MALICIOUS,
MIN_MALICIOUS_FOR_HEADLINE,
CalibrationBin,
@@ -22,10 +24,13 @@ from .replay import (
SegmentScore,
format_replay_report,
score_replay,
+ strip_latency,
)
from .runner import format_report, load_corpus, run_benchmark
__all__ = [
+ "LATENCY_FIELDS",
+ "LATENCY_LINE_PREFIX",
"MALICIOUS",
"MIN_MALICIOUS_FOR_HEADLINE",
"AgentVerdict",
@@ -46,5 +51,6 @@ __all__ = [
"score_incident",
"format_replay_report",
"score_replay",
+ "strip_latency",
]
__version__ = "0.1.0"
diff --git a/packages/aisoc-benchmark/aisoc_benchmark/replay.py b/packages/aisoc-benchmark/aisoc_benchmark/replay.py
index cf8a1a412..2bd97c9f1 100644
--- a/packages/aisoc-benchmark/aisoc_benchmark/replay.py
+++ b/packages/aisoc-benchmark/aisoc_benchmark/replay.py
@@ -63,6 +63,8 @@ __all__ = [
"BOOTSTRAP_SEED",
"CALIBRATION_BINS",
"GRADED_DISPOSITIONS",
+ "LATENCY_FIELDS",
+ "LATENCY_LINE_PREFIX",
"MALICIOUS",
"MIN_MALICIOUS_FOR_HEADLINE",
"UNLABELED",
@@ -72,8 +74,18 @@ __all__ = [
"SegmentScore",
"format_replay_report",
"score_replay",
+ "strip_latency",
]
+#: The one line of the rendered report that measures the host rather than the
+#: agent. Declared here, used by :func:`format_replay_report` to emit it and
+#: by :func:`strip_latency` to remove it, so the producer and the remover
+#: cannot drift into disagreeing about which line that is.
+LATENCY_LINE_PREFIX = "- Mean latency:"
+
+#: The matching fields in :meth:`ReplayScore.as_dict`, for the JSON export.
+LATENCY_FIELDS: tuple[str, ...] = ("mean_latency_ms", "p95_latency_ms")
+
#: The canonical disposition that means "this was a real threat". Written once
#: because "malicious recall" has to mean the same thing in the metric, the
#: renderer and the promotion gate Phase 2 builds on top of it.
@@ -557,7 +569,7 @@ def format_replay_report(score: ReplayScore, *, method: dict[str, Any] | None =
f"- Tokens: {score.total_tokens}",
f"- Measured spend: {measured} (list-price estimate for unbilled calls: {estimated}; "
f"calls with no price at all: {score.unpriced_calls})",
- f"- Mean latency: {'not measured' if score.mean_latency_ms is None else f'{score.mean_latency_ms:.0f} ms'}, "
+ f"{LATENCY_LINE_PREFIX} {'not measured' if score.mean_latency_ms is None else f'{score.mean_latency_ms:.0f} ms'}, "
f"p95 {'not measured' if score.p95_latency_ms is None else f'{score.p95_latency_ms:.0f} ms'}",
f"- Models: {', '.join(score.models) if score.models else 'none, no model call was placed'}",
"",
@@ -573,6 +585,31 @@ def format_replay_report(score: ReplayScore, *, method: dict[str, Any] | None =
return "\n".join(lines)
+def strip_latency(report: str) -> str:
+ """Return the report without the two wall-clock latency figures.
+
+ Everything else in this report is a property of the input and the code,
+ so two runs over one pinned window reproduce it byte for byte. Mean and
+ p95 latency are not: they measure the machine the replay ran on, and they
+ will differ between two runs on the same host, let alone two hosts.
+
+ So the reproducibility claim is stated precisely rather than loosely. The
+ report reproduces byte for byte **apart from these two figures**, and this
+ function is what produces the artefact that claim is about. It removes the
+ line :data:`LATENCY_LINE_PREFIX` names rather than matching on the word
+ "latency" anywhere, because a future section that discusses latency in
+ prose must not be silently deleted from an operator's export.
+
+ The line is replaced rather than dropped, so the surrounding structure and
+ the line count are unchanged and a diff of two reports points at content
+ rather than at an offset.
+ """
+ return "\n".join(
+ f"{LATENCY_LINE_PREFIX} excluded from this export (wall clock, not reproducible)" if line.startswith(LATENCY_LINE_PREFIX) else line
+ for line in report.split("\n")
+ )
+
+
def _calibration(answered: list[dict[str, Any]]) -> tuple[list[CalibrationBin], float | None]:
"""Reliability bins and expected calibration error.
diff --git a/packages/aisoc-benchmark/tests/test_replay_metrics.py b/packages/aisoc-benchmark/tests/test_replay_metrics.py
index 083ae668a..a1a5d59cf 100644
--- a/packages/aisoc-benchmark/tests/test_replay_metrics.py
+++ b/packages/aisoc-benchmark/tests/test_replay_metrics.py
@@ -14,9 +14,12 @@ from typing import Any
import pytest # type: ignore[import-not-found]
from aisoc_benchmark.metrics import extract_checkable_indicators
from aisoc_benchmark.replay import (
+ LATENCY_FIELDS,
+ LATENCY_LINE_PREFIX,
MIN_MALICIOUS_FOR_HEADLINE,
format_replay_report,
score_replay,
+ strip_latency,
)
MALICIOUS = "true_positive"
@@ -303,3 +306,77 @@ def test_a_history_with_no_labels_at_all_says_so() -> None:
assert score.headline_accuracy is None
assert "nothing to grade against" in (score.headline_withheld_reason or "")
assert score.unlabeled == 50
+
+
+# --------------------------------------------------------------------------
+# The reproducibility claim, stated precisely
+# --------------------------------------------------------------------------
+#
+# Gap-closure Phase 1.4. Everything in this report is a property of the input
+# and the code apart from the two wall-clock latency figures, and the claim is
+# worth nothing if it is stated loosely. `strip_latency` is what produces the
+# artefact the claim is about, so it is checked in both directions: it must
+# remove the line the renderer emits, and it must remove nothing else.
+
+
+def test_two_runs_differing_only_in_latency_are_identical_once_it_is_excluded() -> None:
+ """The exact shape of what changes between two runs on one history.
+
+ Latency is the only field that moves, and it moves every time. Everything
+ downstream of that fact depends on this holding.
+ """
+ fast = _corpus(malicious=40, benign=60, hits=31)
+ slow = [{**row, "latency_ms": row["latency_ms"] + 137} for row in fast]
+
+ assert format_replay_report(score_replay(fast)) != format_replay_report(score_replay(slow))
+ assert strip_latency(format_replay_report(score_replay(fast))) == strip_latency(format_replay_report(score_replay(slow)))
+
+
+def test_stripping_latency_changes_exactly_one_line() -> None:
+ report = format_replay_report(score_replay(_corpus(malicious=40, benign=60, hits=31)))
+
+ stripped = strip_latency(report)
+
+ original_lines = report.split("\n")
+ stripped_lines = stripped.split("\n")
+ # The line is replaced rather than dropped, so a diff of two reports
+ # points at content rather than at an offset.
+ assert len(original_lines) == len(stripped_lines)
+ differing = [(a, b) for a, b in zip(original_lines, stripped_lines, strict=True) if a != b]
+ assert len(differing) == 1
+ assert differing[0][0].startswith(LATENCY_LINE_PREFIX)
+
+
+def test_the_renderer_emits_the_line_the_stripper_looks_for() -> None:
+ """Both directions, because a prefix that drifted would silently strip nothing.
+
+ The producer and the remover share one constant. This asserts the
+ constant is actually what lands in the rendered report, so a renderer
+ change that stopped using it fails here rather than leaving
+ `--exclude-latency` a no-op that still claims to have excluded something.
+ """
+ report = format_replay_report(score_replay(_corpus(malicious=40, benign=60, hits=31)))
+
+ assert sum(1 for line in report.split("\n") if line.startswith(LATENCY_LINE_PREFIX)) == 1
+
+
+def test_prose_mentioning_latency_elsewhere_is_not_deleted() -> None:
+ """Anchored on the line prefix, not on the word.
+
+ A future section discussing latency must not be silently removed from an
+ operator's export.
+ """
+ report = "## Cost and latency\n\n- Tokens: 0\n" + LATENCY_LINE_PREFIX + " 4 ms, p95 9 ms\n\nLatency is reported, not scored.\n"
+
+ stripped = strip_latency(report)
+
+ assert "## Cost and latency" in stripped
+ assert "Latency is reported, not scored." in stripped
+ assert "4 ms" not in stripped
+
+
+def test_the_latency_fields_named_for_the_json_export_exist_on_the_score() -> None:
+ score = score_replay(_corpus(malicious=40, benign=60, hits=31)).as_dict()
+
+ for field in LATENCY_FIELDS:
+ assert field in score
diff --git a/packages/aisoc-cli/src/aisoc_cli/main.py b/packages/aisoc-cli/src/aisoc_cli/main.py
index 8fad67b7c..30ed459a9 100644
--- a/packages/aisoc-cli/src/aisoc_cli/main.py
+++ b/packages/aisoc-cli/src/aisoc_cli/main.py
@@ -13,6 +13,7 @@ Commands:
aisoc mcp serve [--transport] Launch the MCP server for IDE assistants
aisoc mcp install --host <h> Wire AiSOC into Claude / Cursor / Continue
aisoc submit <file> POST an alert/event JSON payload to the AiSOC API
+ aisoc replay --connector-id <id> Measure triage against your own closed findings
"""
from __future__ import annotations
@@ -23,6 +24,7 @@ import os
import shutil
import subprocess
import sys
+import time
from importlib import resources
from pathlib import Path
from string import Template
@@ -42,6 +44,15 @@ from rich.table import Table
console = Console()
+#: Progress and diagnostics go here, never to stdout.
+#:
+#: ``aisoc replay`` writes the report itself to stdout when no ``--output`` is
+#: given, so ``aisoc replay ... > report.md`` has to produce the report and
+#: nothing else. A progress line on stdout would land in the middle of the
+#: file, and this phase's acceptance bar is a report that reproduces byte for
+#: byte.
+err_console = Console(stderr=True)
+
PLUGIN_TYPES = ("enricher", "connector", "responder", "detection", "widget")
DEFAULT_AUTHOR = "Your Name <[email protected]>"
@@ -978,5 +989,384 @@ def submit(
)
+# ── replay command ───────────────────────────────────────────────────────────
+#
+# `aisoc replay` measures AiSOC triage against a tenant's own analysts, on
+# their own closed findings, before anyone is asked to trust it.
+#
+# The CLI drives the API and does no evaluation of its own. That is forced
+# rather than chosen: the history readers live in `services/actions`, triage in
+# `services/agents` and `normalize()` in `services/connectors`, and all three
+# package their code as top-level `app`, so one Python process can hold exactly
+# one of them. The API is the service that already orchestrates across all
+# three, holds the vault and owns the tenant session.
+#
+# POST {api_url}/api/v1/evaluations/replay -> 202 with an id
+# GET {api_url}/api/v1/evaluations/replay/{id} -> poll to a terminal state
+# GET .../export?format=markdown|json|pdf -> the stored artefact
+#
+# Determinism, which is this phase's acceptance bar
+# -------------------------------------------------
+# Two runs over one history must produce the same bytes. Three things here
+# could break that and each is handled rather than hoped about:
+#
+# * the window. `--since` / `--until` are passed through exactly as given,
+# and when they are omitted the API dates the window from now — which is a
+# different window on a second run, and therefore a different report. The
+# help text says so rather than leaving it to be discovered.
+# * the artefact. Nothing here renders anything. The file written is the
+# bytes the API returned, which are the bytes stored when the run
+# completed.
+# * wall-clock latency. `--exclude-latency` asks the API for the report with
+# its two latency figures replaced by a note. Everything else is a
+# property of the input and the code.
+
+_DEFAULT_POLL_SECONDS = 3.0
+_DEFAULT_TIMEOUT_SECONDS = 3600.0
+
+#: Statuses the API will never move a run out of.
+_TERMINAL = frozenset({"completed", "failed"})
+
+
+def _replay_headers(api_key: str | None) -> dict[str, str]:
+ headers = {"Accept": "application/json"}
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ return headers
+
+
+def _api_error(response: httpx.Response) -> str:
+ """The server's own reason, or the status when it did not give one."""
+ try:
+ body = response.json()
+ except ValueError:
+ return f"HTTP {response.status_code}: {response.text[:300]}"
+ detail = body.get("detail") if isinstance(body, dict) else None
+ return f"HTTP {response.status_code}: {detail or response.text[:300]}"
+
+
+def _poll_until_terminal(
+ client: httpx.Client,
+ url: str,
+ headers: dict[str, str],
+ *,
+ poll_seconds: float,
+ timeout_seconds: float,
+) -> dict[str, Any]:
+ """Poll one evaluation until it stops moving, or give up and say so.
+
+ A timeout here is reported as a timeout, never as a failed evaluation:
+ the run is still going server-side and the id is printed so it can be
+ collected later.
+ """
+ deadline = time.monotonic() + timeout_seconds
+ last_status = ""
+ while True:
+ response = client.get(url, headers=headers)
+ if response.status_code >= 400:
+ raise click.ClickException(f"Could not read the evaluation: {_api_error(response)}")
+ body: dict[str, Any] = response.json()
+ status = str(body.get("status") or "")
+ if status != last_status:
+ err_console.print(f"[dim]status:[/dim] {status}")
+ last_status = status
+ if status in _TERMINAL:
+ return body
+ if time.monotonic() >= deadline:
+ raise click.ClickException(
+ f"Timed out after {timeout_seconds:.0f}s with the run still '{status}'. "
+ f"It is still running on the server; collect it with "
+ f"`aisoc replay --collect {body.get('id')}`."
+ )
+ time.sleep(poll_seconds)
+
+
+def _write_report(
+ client: httpx.Client,
+ base_url: str,
+ evaluation_id: str,
+ headers: dict[str, str],
+ *,
+ export_format: str,
+ exclude_latency: bool,
+ output: Path | None,
+) -> None:
+ """Fetch the stored artefact and write it out, unmodified.
+
+ Nothing is rendered, reformatted or annotated on the way through. The
+ bytes written are the bytes the API returned.
+ """
+ response = client.get(
+ f"{base_url}/api/v1/evaluations/replay/{evaluation_id}/export",
+ headers=headers,
+ params={"format": export_format, "exclude_latency": str(exclude_latency).lower()},
+ )
+ if response.status_code >= 400:
+ raise click.ClickException(f"Export failed: {_api_error(response)}")
+
+ if output is not None:
+ output.write_bytes(response.content)
+ err_console.print(f"[green]report written[/green] {output} ({len(response.content)} bytes)")
+ return
+
+ if export_format == "pdf":
+ raise click.ClickException("PDF is binary; pass --output to write it to a file.")
+ # stdout, so `aisoc replay ... > report.md` produces the report and
+ # nothing else. Every progress line above went to stderr.
+ sys.stdout.write(response.text)
+
+
+def _print_summary(body: dict[str, Any]) -> None:
+ """The sample sizes beside the headline, on stderr.
+
+ Both are shown together on purpose. A headline accuracy with no sample
+ size behind it is the single most misleading thing this surface could
+ print, and a withheld headline has to say why it was withheld rather
+ than showing a blank.
+ """
+ headline = body.get("headline_accuracy")
+ withheld = body.get("headline_withheld_reason")
+ if headline is None:
+ headline_line = "[yellow]withheld[/yellow]" + (f"\n {withheld}" if withheld else "")
+ else:
+ headline_line = (
+ f"{float(headline) * 100:.1f}% over {body.get('graded', 0)} answered decisions"
+ )
+
+ recall = body.get("malicious_recall")
+ recall_line = "not measured" if recall is None else f"{float(recall) * 100:.1f}%"
+
+ err_console.print(
+ Panel(
+ f"[bold]evaluation:[/bold] {body.get('id')}\n"
+ f"[bold]source:[/bold] {body.get('vendor')} ({body.get('connector_id')})\n"
+ f"[bold]window:[/bold] {body.get('window_start')} to {body.get('window_end')}\n"
+ f"[bold]findings read:[/bold] {body.get('findings_read', 0)} "
+ f"({body.get('findings_labelled', 0)} carried an analyst label)\n"
+ f"[bold]replayed:[/bold] {body.get('decisions_recorded', 0)}, "
+ f"answered {body.get('graded', 0)}, malicious cases {body.get('malicious_support', 0)}\n"
+ f"[bold]malicious recall:[/bold] {recall_line}\n"
+ f"[bold]headline accuracy:[/bold] {headline_line}",
+ title="[bold green]replay evaluation[/bold green]",
+ )
+ )
+
+
+ "--api-url",
+ envvar="AISOC_API_URL",
+ default=_DEFAULT_API_URL,
+ show_default=True,
+ help="Base URL for the AiSOC API.",
+)
+ "--api-key",
+ envvar="AISOC_API_KEY",
+ default=None,
+ help="AiSOC bearer token. The tenant is derived from it; there is no tenant flag.",
+)
+ "--connector-id",
+ default=None,
+ help="UUID of the saved connector whose SIEM history to replay.",
+)
+ "--collect",
+ "collect_id",
+ default=None,
+ help="Skip starting a run and collect an existing evaluation by id.",
+)
+ "--since",
+ default=None,
+ help=(
+ "Window start, ISO-8601. Omitting it dates the window from now, which means a second "
+ "run covers a different window and produces a different report. Pin both ends to compare runs."
+ ),
+)
[email protected]("--until", default=None, help="Window end, ISO-8601. See --since.")
+ "--train-fraction",
+ type=float,
+ default=0.7,
+ show_default=True,
+ help="Time split. The earlier fraction fixes the frozen context; only the later period is graded.",
+)
+ "--limit",
+ type=int,
+ default=1000,
+ show_default=True,
+ help="Maximum findings to read from the window.",
+)
+ "--seed",
+ type=int,
+ default=None,
+ help="Bootstrap seed. Defaults to the platform's, and travels into the report either way.",
+)
+ "--resamples",
+ type=int,
+ default=None,
+ help="Bootstrap resample count. Defaults to the platform's, and travels into the report.",
+)
+ "--format",
+ "export_format",
+ type=click.Choice(["markdown", "json", "pdf"]),
+ default="markdown",
+ show_default=True,
+ help="Export format. All three come from the artefact stored when the run completed.",
+)
+ "--output",
+ "-o",
+ type=click.Path(dir_okay=False, writable=True, path_type=Path),
+ default=None,
+ help="Write the report here. Without it, markdown and JSON go to stdout.",
+)
+ "--exclude-latency",
+ is_flag=True,
+ default=False,
+ help=(
+ "Replace the two wall-clock latency figures with a note, so two runs over one pinned "
+ "window are byte-identical. Latency measures the host and never reproduces."
+ ),
+)
+ "--wait/--no-wait",
+ default=True,
+ show_default=True,
+ help="Poll to completion. With --no-wait the id is printed and the run continues server-side.",
+)
+ "--poll-interval",
+ type=float,
+ default=_DEFAULT_POLL_SECONDS,
+ show_default=True,
+ help="Seconds between polls.",
+)
+ "--timeout",
+ type=float,
+ default=_DEFAULT_TIMEOUT_SECONDS,
+ show_default=True,
+ help="Give up waiting after this many seconds. The run continues server-side.",
+)
+def replay(
+ api_url: str,
+ api_key: str | None,
+ connector_id: str | None,
+ collect_id: str | None,
+ since: str | None,
+ until: str | None,
+ train_fraction: float,
+ limit: int,
+ seed: int | None,
+ resamples: int | None,
+ export_format: str,
+ output: Path | None,
+ exclude_latency: bool,
+ wait: bool,
+ poll_interval: float,
+ timeout: float,
+) -> None:
+ """Measure AiSOC triage against your own analysts' past decisions.
+
+ Reads the findings your analysts already closed in a window, replays them
+ through the same triage path production runs (writing nothing, reading a
+ frozen world), and grades the result against the labels they chose.
+
+ The report leads with recall on malicious, prints every rate beside the
+ count it was computed over, and withholds a headline accuracy when the
+ window holds too few malicious cases for one to mean anything.
+
+ The tenant comes from the credential. There is no tenant flag, because a
+ flag would be a value you chose that nothing checked.
+ """
+ if not collect_id and not connector_id:
+ raise click.ClickException(
+ "Pass --connector-id to start a run, or --collect <id> to fetch one."
+ )
+
+ base_url = api_url.rstrip("/")
+ headers = _replay_headers(api_key)
+
+ with httpx.Client(timeout=60.0) as client:
+ if collect_id:
+ evaluation_id = collect_id
+ err_console.print(f"[dim]collecting[/dim] {evaluation_id}")
+ else:
+ payload: dict[str, Any] = {
+ "connector_id": connector_id,
+ "train_fraction": train_fraction,
+ "limit": limit,
+ }
+ # Only sent when given. An omitted bound lets the API date the
+ # window, and sending a locally-computed "now" instead would put
+ # the CLI's clock into a report the API is responsible for.
+ if since:
+ payload["since"] = since
+ if until:
+ payload["until"] = until
+ if seed is not None:
+ payload["bootstrap_seed"] = seed
+ if resamples is not None:
+ payload["bootstrap_resamples"] = resamples
+
+ try:
+ response = client.post(
+ f"{base_url}/api/v1/evaluations/replay",
+ headers=headers,
+ json=payload,
+ )
+ except httpx.HTTPError as exc:
+ raise click.ClickException(
+ f"AiSOC API unreachable at {base_url}: {exc}\nIs the stack running? Try `aisoc serve`."
+ ) from exc
+ if response.status_code >= 400:
+ raise click.ClickException(
+ f"Could not start the evaluation: {_api_error(response)}"
+ )
+ started = response.json()
+ evaluation_id = str(started.get("id"))
+ err_console.print(f"[green]queued[/green] {evaluation_id}")
+
+ if not wait:
+ err_console.print(
+ f"[dim]not waiting. Collect it with:[/dim] aisoc replay --collect {evaluation_id}"
+ )
+ return
+
+ body = _poll_until_terminal(
+ client,
+ f"{base_url}/api/v1/evaluations/replay/{evaluation_id}",
+ headers,
+ poll_seconds=poll_interval,
+ timeout_seconds=timeout,
+ )
+
+ if body.get("status") == "failed":
+ # The server's reason, verbatim. A replay that could not read a
+ # window has measured nothing, and saying so is the useful answer.
+ raise click.ClickException(
+ f"The evaluation failed: {body.get('error') or 'no reason recorded'}"
+ )
+
+ _print_summary(body)
+ _write_report(
+ client,
+ base_url,
+ evaluation_id,
+ headers,
+ export_format=export_format,
+ exclude_latency=exclude_latency,
+ output=output,
+ )
+
+
if __name__ == "__main__":
cli()
diff --git a/packages/aisoc-cli/tests/test_replay_command.py b/packages/aisoc-cli/tests/test_replay_command.py
new file mode 100644
index 000000000..c285a34ab
--- /dev/null
+++ b/packages/aisoc-cli/tests/test_replay_command.py
@@ -0,0 +1,346 @@
+"""CLI surface tests for ``aisoc replay``.
+
+Gap-closure Phase 1.4 gate.
+
+The contract these pin:
+
+* ``aisoc replay`` starts a run, polls it to a terminal state, and writes the
+ artefact the API stored. It renders nothing of its own.
+* Progress goes to stderr and the report goes to stdout, so
+ ``aisoc replay ... > report.md`` produces the report and nothing else.
+* A failed evaluation exits non-zero with the server's own reason. A replay
+ that could not read a window has measured nothing, and saying so is the
+ useful answer.
+* The tenant is never a flag. It comes from the credential.
+* ``--exclude-latency`` reaches the API, which is where the exclusion is
+ implemented, next to the renderer that emits the line.
+
+``httpx.MockTransport`` stands in for the API. Nothing here makes a network
+call.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+from click.testing import CliRunner, Result
+
+from aisoc_cli.main import cli
+
+EVALUATION_ID = "3f2a1b4c-5d6e-4f70-8192-a3b4c5d6e7f8"
+
+REPORT = """# Replay evaluation
+
+## What was graded
+
+- Decisions replayed: 60
+- Carried an analyst label: 60
+
+## Recall on malicious
+
+- Malicious cases in the test window: 31
+- Recall: 83.9% (95% CI 71.0% to 93.5%)
+"""
+
+
+def _summary(status: str, **overrides: Any) -> dict[str, Any]:
+ body: dict[str, Any] = {
+ "id": EVALUATION_ID,
+ "status": status,
+ "error": None,
+ "connector_id": "c0ffee00-0000-0000-0000-000000000001",
+ "vendor": "splunk",
+ "window_start": "2026-03-01T00:00:00+00:00",
+ "window_end": "2026-06-01T00:00:00+00:00",
+ "train_fraction": 0.7,
+ "bootstrap_seed": 20260926,
+ "bootstrap_resamples": 1000,
+ "findings_read": 200,
+ "findings_labelled": 188,
+ "decisions_recorded": 60,
+ "graded": 58,
+ "malicious_support": 31,
+ "headline_accuracy": 0.8448,
+ "headline_withheld_reason": None,
+ "malicious_recall": 0.8387,
+ "created_at": "2026-06-01T09:00:00+00:00",
+ "started_at": "2026-06-01T09:00:01+00:00",
+ "completed_at": "2026-06-01T09:04:12+00:00",
+ "is_terminal": status in {"completed", "failed"},
+ }
+ body.update(overrides)
+ return body
+
+
+class _Api:
+ """A scripted API. Records every request so the contract can be asserted."""
+
+ def __init__(
+ self, *, statuses: list[str] | None = None, summary_overrides: dict[str, Any] | None = None
+ ) -> None:
+ self.requests: list[httpx.Request] = []
+ self._statuses = list(statuses or ["completed"])
+ self._overrides = summary_overrides or {}
+
+ def __call__(self, request: httpx.Request) -> httpx.Response:
+ self.requests.append(request)
+ path = request.url.path
+ if request.method == "POST" and path.endswith("/evaluations/replay"):
+ return httpx.Response(202, json=_summary("queued"))
+ if path.endswith("/export"):
+ return httpx.Response(
+ 200, text=REPORT, headers={"content-type": "text/markdown; charset=utf-8"}
+ )
+ status = self._statuses.pop(0) if len(self._statuses) > 1 else self._statuses[0]
+ return httpx.Response(200, json=_summary(status, **self._overrides))
+
+
+def _install(
+ monkeypatch: pytest.MonkeyPatch,
+ handler: Any,
+ *,
+ patch_sleep: bool = True,
+) -> None:
+ """Point the CLI's httpx client at a mock transport, and stop it sleeping.
+
+ Patched by dotted path rather than by reaching through ``cli_main`` for
+ its ``httpx`` and ``time`` attributes. Same effect, and it does not assert
+ that the module re-exports its own imports, which it does not.
+ """
+ real_client = httpx.Client
+ monkeypatch.setattr(
+ "aisoc_cli.main.httpx.Client",
+ lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),
+ )
+ if patch_sleep:
+ monkeypatch.setattr("aisoc_cli.main.time.sleep", lambda _seconds: None)
+
+
+def runner() -> CliRunner:
+ # stderr kept separate so the "report on stdout, progress on stderr"
+ # property can be asserted rather than assumed.
+ return CliRunner()
+
+
+def api(monkeypatch: pytest.MonkeyPatch) -> _Api:
+ scripted = _Api()
+ _install(monkeypatch, scripted)
+ return scripted
+
+
+def _invoke(runner: CliRunner, *args: str) -> Result:
+ return runner.invoke(cli, ["replay", *args], catch_exceptions=False)
+
+
+def test_replay_is_registered_and_discoverable(runner: CliRunner) -> None:
+ result = runner.invoke(cli, ["--help"])
+
+ assert result.exit_code == 0
+ assert "replay" in result.output
+
+
+def test_a_run_is_started_polled_and_its_report_written(
+ runner: CliRunner, api: _Api, tmp_path: Path
+) -> None:
+ output = tmp_path / "report.md"
+
+ result = _invoke(
+ runner,
+ "--connector-id",
+ "c0ffee00-0000-0000-0000-000000000001",
+ "--since",
+ "2026-03-01T00:00:00Z",
+ "--until",
+ "2026-06-01T00:00:00Z",
+ "--output",
+ str(output),
+ )
+
+ assert result.exit_code == 0
+ # The bytes written are the bytes the API returned. Nothing is rendered,
+ # reformatted or annotated on the way through.
+ assert output.read_text() == REPORT
+
+ posted = [r for r in api.requests if r.method == "POST"]
+ assert len(posted) == 1
+ body = json.loads(posted[0].content)
+ assert body["connector_id"] == "c0ffee00-0000-0000-0000-000000000001"
+ assert body["since"] == "2026-03-01T00:00:00Z"
+ assert body["until"] == "2026-06-01T00:00:00Z"
+
+
+def test_an_omitted_window_is_not_filled_in_locally(
+ runner: CliRunner, api: _Api, tmp_path: Path
+) -> None:
+ """The CLI must not put its own clock into a report the API owns.
+
+ Sending a locally-computed "now" would make the window depend on the
+ machine the CLI ran on rather than on the deployment.
+ """
+ _invoke(
+ runner,
+ "--connector-id",
+ "c0ffee00-0000-0000-0000-000000000001",
+ "--output",
+ str(tmp_path / "r.md"),
+ )
+
+ body = json.loads([r for r in api.requests if r.method == "POST"][0].content)
+ assert "since" not in body
+ assert "until" not in body
+
+
+def test_the_report_goes_to_stdout_and_progress_to_stderr(runner: CliRunner, api: _Api) -> None:
+ """`aisoc replay ... > report.md` has to produce the report and nothing else."""
+ result = _invoke(runner, "--connector-id", "c0ffee00-0000-0000-0000-000000000001")
+
+ assert result.exit_code == 0
+ assert result.stdout == REPORT
+ assert "queued" in result.stderr
+ assert "replay evaluation" in result.stderr
+
+
+def test_exclude_latency_reaches_the_api(runner: CliRunner, api: _Api, tmp_path: Path) -> None:
+ """The exclusion is implemented beside the renderer, not in the CLI."""
+ _invoke(
+ runner,
+ "--connector-id",
+ "c0ffee00-0000-0000-0000-000000000001",
+ "--exclude-latency",
+ "--output",
+ str(tmp_path / "r.md"),
+ )
+
+ export = [r for r in api.requests if r.url.path.endswith("/export")][0]
+ assert export.url.params["exclude_latency"] == "true"
+ assert export.url.params["format"] == "markdown"
+
+
+def test_the_seed_and_resamples_are_passed_through_when_given(
+ runner: CliRunner, api: _Api, tmp_path: Path
+) -> None:
+ _invoke(
+ runner,
+ "--connector-id",
+ "c0ffee00-0000-0000-0000-000000000001",
+ "--seed",
+ "7",
+ "--resamples",
+ "200",
+ "--output",
+ str(tmp_path / "r.md"),
+ )
+
+ body = json.loads([r for r in api.requests if r.method == "POST"][0].content)
+ assert body["bootstrap_seed"] == 7
+ assert body["bootstrap_resamples"] == 200
+
+
+def test_a_failed_evaluation_exits_non_zero_with_the_servers_reason(
+ runner: CliRunner, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ scripted = _Api(
+ statuses=["failed"],
+ summary_overrides={"error": "the actions service is unreachable at http://actions:8085"},
+ )
+ _install(monkeypatch, scripted)
+
+ result = runner.invoke(
+ cli, ["replay", "--connector-id", "c0ffee00-0000-0000-0000-000000000001"]
+ )
+
+ assert result.exit_code != 0
+ assert "unreachable" in result.stderr
+ # Nothing is written when there is nothing measured.
+ assert result.stdout == ""
+
+
+def test_polling_continues_until_a_terminal_status(
+ runner: CliRunner, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ scripted = _Api(statuses=["queued", "running", "running", "completed"])
+ _install(monkeypatch, scripted)
+
+ result = runner.invoke(
+ cli, ["replay", "--connector-id", "c0ffee00-0000-0000-0000-000000000001"]
+ )
+
+ assert result.exit_code == 0
+ polls = [
+ r for r in scripted.requests if r.method == "GET" and not r.url.path.endswith("/export")
+ ]
+ assert len(polls) == 4
+
+
+def test_no_wait_prints_the_id_and_stops(runner: CliRunner, api: _Api) -> None:
+ result = _invoke(runner, "--connector-id", "c0ffee00-0000-0000-0000-000000000001", "--no-wait")
+
+ assert result.exit_code == 0
+ assert EVALUATION_ID in result.stderr
+ assert not [r for r in api.requests if r.url.path.endswith("/export")]
+
+
+def test_collect_fetches_an_existing_run_without_starting_one(runner: CliRunner, api: _Api) -> None:
+ result = _invoke(runner, "--collect", EVALUATION_ID)
+
+ assert result.exit_code == 0
+ assert not [r for r in api.requests if r.method == "POST"]
+ assert result.stdout == REPORT
+
+
+def test_neither_connector_nor_collect_is_refused_with_both_options(runner: CliRunner) -> None:
+ result = runner.invoke(cli, ["replay"])
+
+ assert result.exit_code != 0
+ assert "--connector-id" in result.stderr
+ assert "--collect" in result.stderr
+
+
+def test_there_is_no_tenant_flag(runner: CliRunner) -> None:
+ """A tenant flag would be a value the operator chose that nothing checked."""
+ result = runner.invoke(cli, ["replay", "--help"])
+
+ assert result.exit_code == 0
+ assert "--tenant" not in result.output
+ # Click re-wraps help text, so the sentence is matched with whitespace
+ # collapsed rather than against whatever the terminal width produced.
+ collapsed = " ".join(result.output.split())
+ assert "There is no tenant flag" in collapsed
+
+
+def test_pdf_to_stdout_is_refused_rather_than_written_as_mojibake(
+ runner: CliRunner, api: _Api
+) -> None:
+ result = runner.invoke(cli, ["replay", "--collect", EVALUATION_ID, "--format", "pdf"])
+
+ assert result.exit_code != 0
+ assert "--output" in result.stderr
+
+
+def test_an_unreachable_api_says_so_and_names_the_url(
+ runner: CliRunner, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def refuse(request: httpx.Request) -> httpx.Response:
+ raise httpx.ConnectError("connection refused", request=request)
+
+ _install(monkeypatch, refuse, patch_sleep=False)
+
+ result = runner.invoke(
+ cli,
+ [
+ "replay",
+ "--connector-id",
+ "c0ffee00-0000-0000-0000-000000000001",
+ "--api-url",
+ "http://127.0.0.1:8000",
+ ],
+ )
+
+ assert result.exit_code != 0
+ assert "http://127.0.0.1:8000" in result.stderr
diff --git a/scripts/mypy_baseline.json b/scripts/mypy_baseline.json
index f70342544..abc7fa1d5 100644
--- a/scripts/mypy_baseline.json
+++ b/scripts/mypy_baseline.json
@@ -84,7 +84,7 @@
"src/aisoc_cli/main.py": {
"import-not-found": 7,
"import-untyped": 2,
- "untyped-decorator": 48
+ "untyped-decorator": 65
},
"tests/test_cli.py": {
"import-not-found": 2,
@@ -104,6 +104,10 @@
"import-not-found": 1,
"untyped-decorator": 1
},
+ "tests/test_replay_command.py": {
+ "import-not-found": 3,
+ "untyped-decorator": 2
+ },
"tests/test_submit_command.py": {
"attr-defined": 1,
"import-not-found": 3,
diff --git a/scripts/sync_vendored_benchmark.py b/scripts/sync_vendored_benchmark.py
new file mode 100644
index 000000000..2d1329c3b
--- /dev/null
+++ b/scripts/sync_vendored_benchmark.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Keep ``services/api/app/_vendor/aisoc_benchmark/`` in lockstep with its source.
+
+Gap-closure Phase 1.4.
+
+The replay evaluation job is orchestrated by ``services/api``: that service
+holds the vault and the tenant session, and it is already the one that proxies
+to ``services/agents`` and ``services/actions``. Scoring is the one link in
+that chain with no round trip, because ``packages/aisoc-benchmark`` is a
+distribution rather than a service.
+
+It still cannot be imported directly. The ``aisoc-core-api`` image is built
+with ``services/api`` as its build context, so nothing under ``packages/`` is
+present at runtime. Same constraint that already produced
+``_vendor/narrative.py``, ``_vendor/llm_contract_rules.py`` and
+``_vendor/nl_query/``, and the same answer: a mirror plus a gate.
+
+The alternative was a second scorer inside the API, and the module being
+mirrored opens by explaining why there is exactly one grader. A copy that
+drifted would publish two definitions of "hallucination rate" while both
+called themselves the same number.
+
+Run modes
+---------
+* ``python scripts/sync_vendored_benchmark.py`` - copy source to vendored.
+* ``python scripts/sync_vendored_benchmark.py --check`` - fail (exit 1) if the
+ mirror is missing, stale, or has grown a file the source does not have. CI
+ uses this mode.
+
+Both directions are compared. A file present in the mirror and absent from the
+source fails too, because the failure this repository keeps finding is the
+one-directional check that prints OK while drift accumulates in the direction
+things actually move.
+
+Dependency-free so it runs on a bare interpreter.
+"""
+
+from __future__ import annotations
+
+import argparse
+import filecmp
+import shutil
+import sys
+from pathlib import Path
+
+# `scripts/` is on sys.path when this file is run as a program, but not when a
+# test loads it by path with importlib. gate_toolkit sits beside it either way.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from gate_toolkit import repo_root, self_test_if_requested # noqa: E402
+
+self_test_if_requested(__file__)
+
+SOURCE_REL = Path("packages/aisoc-benchmark/aisoc_benchmark")
+VENDORED_REL = Path("services/api/app/_vendor/aisoc_benchmark")
+
+#: Files that live only in the mirror and have no source counterpart.
+#: ``VENDORED.md`` documents the mirror; it is not part of the package.
+MIRROR_ONLY = frozenset({"VENDORED.md"})
+
+
+def _python_files(directory: Path) -> dict[str, Path]:
+ return {p.name: p for p in sorted(directory.glob("*.py"))}
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--check", action="store_true", help="verify without writing")
+ args = parser.parse_args()
+
+ root = repo_root()
+ source = root / SOURCE_REL
+ vendored = root / VENDORED_REL
+
+ if not source.is_dir():
+ print(f"FAIL: source package missing at {SOURCE_REL}")
+ return 2
+
+ source_files = _python_files(source)
+ if not source_files:
+ # A gate that passes over an empty tree has verified nothing.
+ print(f"FAIL: no Python modules found under {SOURCE_REL}; refusing to report a clean mirror")
+ return 2
+
+ if not args.check:
+ vendored.mkdir(parents=True, exist_ok=True)
+ for name, path in source_files.items():
+ shutil.copy2(path, vendored / name)
+ for stale in _python_files(vendored):
+ if stale not in source_files:
+ (vendored / stale).unlink()
+ print(f"OK: mirrored {len(source_files)} modules into {VENDORED_REL}")
+ return 0
+
+ if not vendored.is_dir():
+ print(f"FAIL: vendored mirror missing at {VENDORED_REL}. Run: python {Path(__file__).name}")
+ return 1
+
+ problems: list[str] = []
+ vendored_files = _python_files(vendored)
+
+ for name, path in source_files.items():
+ mirror = vendored_files.get(name)
+ if mirror is None:
+ problems.append(f"missing in mirror: {name}")
+ continue
+ if not filecmp.cmp(path, mirror, shallow=False):
+ problems.append(f"differs from source: {name}")
+
+ for name in vendored_files:
+ if name not in source_files and name not in MIRROR_ONLY:
+ problems.append(f"present in mirror but not in source: {name}")
+
+ if problems:
+ print(f"FAIL: {VENDORED_REL} has drifted from {SOURCE_REL}")
+ for problem in problems:
+ print(f" - {problem}")
+ print(f"\nRun: python scripts/{Path(__file__).name}")
+ return 1
+
+ print(f"OK: {len(source_files)} modules mirrored byte-identically from {SOURCE_REL}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/services/actions/app/api/replay_history_router.py b/services/actions/app/api/replay_history_router.py
new file mode 100644
index 000000000..a62d8f877
--- /dev/null
+++ b/services/actions/app/api/replay_history_router.py
@@ -0,0 +1,247 @@
+"""Expose the closed-finding readers so replay evaluation can reach them.
+
+Gap-closure Phase 1.4.
+
+Phase 1.1 put five readers on the five SIEM clients in this service, because
+this service already owns the credential path. Nothing called them. This is
+the route that does.
+
+Why the route is here and not in ``services/api``
+-------------------------------------------------
+The API orchestrates the replay job, but it cannot read a customer's SIEM
+itself without a second copy of every client. ``services/actions`` holds the
+clients, the retry behaviour, the pagination loops and the vendor quirks that
+Phase 1.1 recorded one at a time. A reader in the API would be a sixth place
+for "what does a closed Splunk notable look like" to be written down.
+
+So the API decrypts ``auth_config`` from the vault and forwards it, exactly
+the trust model ``POST /connectors/{id}/normalize`` already uses from Phase
+1.2. Nothing is persisted here and no credential is stored: the client is
+built, used for one window, and dropped.
+
+Why the client factory is imported rather than rewritten
+--------------------------------------------------------
+``app.executors.siem`` already builds all five clients from a flat credential
+mapping, and this repository has twice found a second factory reading keys the
+first one does not. ``_splunk_client`` and friends are reused as-is, so a
+credential key that works for writeback works for the read in the same
+deployment. The one client ``siem.py`` builds inline rather than through a
+factory is Defender, and that arm is built here from the same
+``DEFENDER_CLIENT_PARAM_KEYS`` it declares.
+
+What a vendor with no usable credentials gets
+---------------------------------------------
+A 422 naming the keys that would have built a client, not an empty list. An
+empty list reads as "this customer closed nothing in that window", which is a
+statement about their SOC rather than about their configuration, and it would
+travel into a report as a sample size of zero.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Literal
+
+import structlog
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel, Field
+
+from app.clients.defender_client import DefenderClient
+from app.executors.siem import (
+ DEFENDER_CLIENT_PARAM_KEYS,
+ ELASTIC_CLIENT_PARAM_KEYS,
+ QRADAR_CLIENT_PARAM_KEYS,
+ SENTINEL_CLIENT_PARAM_KEYS,
+ SPLUNK_CLIENT_PARAM_KEYS,
+ _elastic_client,
+ _qradar_client,
+ _sentinel_client,
+ _splunk_client,
+)
+from app.security.authz import require_service_auth
+from app.services.alert_history import (
+ ClosedFinding,
+ parse_defender_alert,
+ parse_elastic_signal,
+ parse_qradar_offense,
+ parse_sentinel_incident,
+ parse_splunk_notable,
+)
+
+logger = structlog.get_logger(__name__)
+
+router = APIRouter(prefix="/replay", tags=["replay"])
+
+#: The five vendors Phase 1.1 wrote readers for. A literal rather than a free
+#: string so an unsupported vendor is a 422 from the schema with the supported
+#: set in the message, rather than a 500 from a dictionary lookup.
+Vendor = Literal["splunk", "sentinel", "elastic", "qradar", "defender"]
+
+#: Hard ceiling on one window's read. Replay grades a window, and a caller who
+#: asks for more than this is asking for a different job than the one the
+#: report describes. Refused rather than truncated: a truncated list reads as
+#: the whole window and would be reported as one.
+MAX_FINDINGS = 5000
+
+#: Which credential keys each vendor's client factory reads, so a refusal can
+#: name them. Sourced from ``app.executors.siem`` rather than retyped, which
+#: is the defect this repository found in the dry-run credential strip: a list
+#: of key names that had drifted from what the factory actually read.
+_CREDENTIAL_KEYS: dict[str, tuple[str, ...]] = {
+ "splunk": SPLUNK_CLIENT_PARAM_KEYS,
+ "sentinel": SENTINEL_CLIENT_PARAM_KEYS,
+ "elastic": ELASTIC_CLIENT_PARAM_KEYS,
+ "qradar": QRADAR_CLIENT_PARAM_KEYS,
+ "defender": DEFENDER_CLIENT_PARAM_KEYS,
+}
+
+
+class HistoryRequest(BaseModel):
+ """One vendor, one window, one set of credentials the API already decrypted.
+
+ There is no tenant field. This service never resolves a tenant: the
+ credentials it is handed *are* the scope, because they reach exactly one
+ customer's SIEM. A tenant field here would be a value the caller chose
+ that nothing checked.
+ """
+
+ vendor: Vendor
+ #: Decrypted by ``services/api`` from the connector instance's vault
+ #: record before the call. Same envelope as the connector test-connection
+ #: and normalize proxies.
+ credentials: dict[str, Any] = Field(default_factory=dict)
+ since: datetime
+ until: datetime
+ limit: int = Field(default=1000, ge=1, le=MAX_FINDINGS)
+ #: Splunk only: the saved search or inline SPL the deployment stores its
+ #: notables under. Deployments rename it, so a hardcoded search returns
+ #: nothing on a site that did.
+ search_override: str | None = None
+ #: Elastic only: the signals index pattern.
+ index: str | None = None
+
+
+class HistoryResponse(BaseModel):
+ vendor: str
+ #: Rows the vendor returned. Reported separately from ``labelled`` so a
+ #: customer whose analysts close without a disposition sees that, rather
+ #: than a small graded sample with no explanation.
+ count: int
+ labelled: int
+ unlabeled: int
+ findings: list[dict[str, Any]]
+
+
+def _unusable(vendor: str) -> HTTPException:
+ keys = ", ".join(_CREDENTIAL_KEYS.get(vendor, ()))
+ return HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=(
+ f"no usable {vendor} credentials in this request; the client factory reads {keys}. "
+ f"Returning an empty history instead would be reported as a window in which "
+ f"nobody closed anything."
+ ),
+ )
+
+
+async def _read_splunk(body: HistoryRequest) -> list[ClosedFinding]:
+ client = _splunk_client(body.credentials)
+ if client is None:
+ raise _unusable("splunk")
+ rows = await client.list_closed_notables(
+ body.since,
+ body.until,
+ limit=body.limit,
+ search_override=body.search_override,
+ )
+ return [parse_splunk_notable(row) for row in rows]
+
+
+async def _read_sentinel(body: HistoryRequest) -> list[ClosedFinding]:
+ client = _sentinel_client(body.credentials)
+ if client is None:
+ raise _unusable("sentinel")
+ rows = await client.list_closed_incidents(body.since, body.until, limit=body.limit)
+ return [parse_sentinel_incident(row) for row in rows]
+
+
+async def _read_elastic(body: HistoryRequest) -> list[ClosedFinding]:
+ client = _elastic_client(body.credentials)
+ if client is None:
+ raise _unusable("elastic")
+ kwargs: dict[str, Any] = {"limit": body.limit}
+ if body.index:
+ kwargs["index"] = body.index
+ rows = await client.list_closed_signals(body.since, body.until, **kwargs)
+ return [parse_elastic_signal(row) for row in rows]
+
+
+async def _read_qradar(body: HistoryRequest) -> list[ClosedFinding]:
+ client = _qradar_client(body.credentials)
+ if client is None:
+ raise _unusable("qradar")
+ rows = await client.list_closed_offenses(body.since, body.until, limit=body.limit)
+ return [parse_qradar_offense(row) for row in rows]
+
+
+async def _read_defender(body: HistoryRequest) -> list[ClosedFinding]:
+ creds = body.credentials
+ required = (creds.get("mde_tenant_id"), creds.get("mde_client_id"), creds.get("mde_client_secret"))
+ if not all(required):
+ raise _unusable("defender")
+ client = DefenderClient(str(required[0]), str(required[1]), str(required[2]))
+ rows = await client.list_resolved_alerts(body.since, body.until, limit=body.limit)
+ return [parse_defender_alert(row) for row in rows]
+
+
+_READERS = {
+ "splunk": _read_splunk,
+ "sentinel": _read_sentinel,
+ "elastic": _read_elastic,
+ "qradar": _read_qradar,
+ "defender": _read_defender,
+}
+
+
[email protected]("/history", response_model=HistoryResponse, summary="Closed findings for one window")
+async def read_history(
+ body: HistoryRequest,
+ _auth: Annotated[None, Depends(require_service_auth)] = None,
+) -> HistoryResponse:
+ """Read the findings a customer's analysts closed, with the labels they chose.
+
+ A vendor error surfaces as a 502 naming the vendor rather than an empty
+ list, for the same reason a missing credential does: replay must not grade
+ a window it could not read and report the result as a measurement.
+ """
+ if body.until <= body.since:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=f"until ({body.until.isoformat()}) must be after since ({body.since.isoformat()})",
+ )
+
+ try:
+ findings = await _READERS[body.vendor](body)
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001 - every vendor failure is a 502, never an empty window
+ logger.exception("replay.history.read_failed", vendor=body.vendor)
+ raise HTTPException(
+ status_code=status.HTTP_502_BAD_GATEWAY,
+ detail=f"reading {body.vendor} history failed: {type(exc).__name__}",
+ ) from exc
+
+ labelled = sum(1 for f in findings if f.is_labelled)
+ logger.info(
+ "replay.history.read",
+ vendor=body.vendor,
+ count=len(findings),
+ labelled=labelled,
+ )
+ return HistoryResponse(
+ vendor=body.vendor,
+ count=len(findings),
+ labelled=labelled,
+ unlabeled=len(findings) - labelled,
+ findings=[f.as_dict() for f in findings],
+ )
diff --git a/services/actions/app/main.py b/services/actions/app/main.py
index f2a542095..28a29c8a7 100644
--- a/services/actions/app/main.py
+++ b/services/actions/app/main.py
@@ -23,6 +23,7 @@ from fastapi import FastAPI
from app._health import install_health_routes
from app.api.live_actions_router import router as live_actions_router
+from app.api.replay_history_router import router as replay_history_router
from app.api.router import router as legacy_router
from app.live_actions import register_builtin_executors
@@ -47,6 +48,9 @@ app.state.mark_not_ready = _mark_not_ready
app.include_router(legacy_router, prefix="/api/v1")
app.include_router(live_actions_router, prefix="/api/v1")
+# Gap-closure Phase 1.4: the read half of the SIEM relationship. Phase 1.1 put
+# five closed-finding readers on the clients here and nothing called them.
+app.include_router(replay_history_router, prefix="/api/v1")
@app.on_event("startup")
diff --git a/services/actions/app/services/alert_history.py b/services/actions/app/services/alert_history.py
index 62f8c91d8..40e765899 100644
--- a/services/actions/app/services/alert_history.py
+++ b/services/actions/app/services/alert_history.py
@@ -126,6 +126,35 @@ class ClosedFinding:
"""Whether this row may contribute to accuracy."""
return self.disposition != UNLABELED
+ def as_dict(self) -> dict[str, Any]:
+ """Serialise for the boundary into ``services/agents``.
+
+ The key names are exactly the field names, because the replay side
+ rebuilds this row with ``HistoricalFinding.from_mapping`` and
+ ``scripts/check_replay_contract_parity.py`` compares the two field
+ sets in both directions. A hand-written key that drifted from a field
+ name would land a value in no column at all, and the receiving side
+ would default it rather than fail.
+
+ ``closed_at`` goes out as ISO-8601 rather than an epoch, because the
+ receiving parser refuses an unparseable close time instead of
+ defaulting it, and a naive epoch would have to guess a timezone to be
+ read back.
+ """
+ return {
+ "vendor": self.vendor,
+ "finding_id": self.finding_id,
+ "title": self.title,
+ "disposition": self.disposition,
+ "vendor_disposition": self.vendor_disposition,
+ "closed_at": self.closed_at.isoformat(),
+ "closed_by": self.closed_by,
+ "reason": self.reason,
+ "rule_id": self.rule_id,
+ "severity": self.severity,
+ "raw": dict(self.raw),
+ }
+
def _coerce_time(value: Any) -> datetime:
"""Best-effort vendor timestamp to an aware UTC datetime.
diff --git a/services/actions/tests/test_replay_history_route.py b/services/actions/tests/test_replay_history_route.py
new file mode 100644
index 000000000..c44153c0e
--- /dev/null
+++ b/services/actions/tests/test_replay_history_route.py
@@ -0,0 +1,236 @@
+"""The history route: every vendor reachable, every failure named.
+
+Gap-closure Phase 1.4 gate.
+
+Phase 1.1 proved the five readers parse vendor-shaped payloads. This proves
+the route in front of them reaches each one, refuses rather than returning an
+empty window, and translates the credential mapping the API forwards into a
+client that the reader can use.
+
+Every vendor payload here is **synthetic**, hand-built to the shape the
+vendor's API documents. None came from a customer.
+
+The assertions that matter are the refusals. An evaluation that turns "your
+credentials do not build a client" into an empty list has told an operator
+their analysts closed nothing, which is a statement about their SOC rather
+than about their configuration.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import httpx
+import pytest
+import respx
+from app.api.replay_history_router import _CREDENTIAL_KEYS, MAX_FINDINGS
+from app.main import app
+from app.services.alert_history import UNLABELED
+from fastapi.testclient import TestClient
+
+SINCE = datetime(2026, 9, 1, tzinfo=UTC)
+UNTIL = datetime(2026, 9, 26, tzinfo=UTC)
+
+_SPLUNK_NOTABLES = {
+ "results": [
+ {
+ "event_id": "ES-001",
+ "rule_name": "Beaconing to a rare destination",
+ "rule_id": "rule-beacon",
+ "disposition": "disposition:1",
+ "review_time": "1790294400",
+ "reviewer": "a.analyst",
+ "comment": "Confirmed beacon, host isolated.",
+ "urgency": "high",
+ },
+ {
+ "event_id": "ES-002",
+ "rule_name": "Bulk export by finance",
+ "rule_id": "rule-export",
+ "disposition": "disposition:6",
+ "review_time": "1790298000",
+ "reviewer": "b.analyst",
+ },
+ ]
+}
+
+
+def client(monkeypatch: pytest.MonkeyPatch) -> TestClient:
+ # The route is behind `require_service_auth`, which fails closed unless a
+ # token is configured or dev mode is on. Dev mode here rather than a token
+ # so the tests exercise the handler rather than the shared auth dependency,
+ # which has its own coverage.
+ monkeypatch.setenv("AISOC_DEV_MODE", "1")
+ monkeypatch.setenv("AISOC_ACTIONS_SERVICE_TOKEN", "")
+ from app.core.config import get_settings
+
+ get_settings.cache_clear()
+ yield TestClient(app)
+ get_settings.cache_clear()
+
+
+def _body(vendor: str, credentials: dict, **extra) -> dict:
+ return {
+ "vendor": vendor,
+ "credentials": credentials,
+ "since": SINCE.isoformat(),
+ "until": UNTIL.isoformat(),
+ **extra,
+ }
+
+
+def test_splunk_history_reaches_the_real_reader_and_reports_both_counts(client: TestClient) -> None:
+ respx.post(url__regex=r"https://splunk:8089/services/search/jobs$").mock(return_value=httpx.Response(201, json={"sid": "sid-1"}))
+ respx.get(url__regex=r".*/results.*").mock(return_value=httpx.Response(200, json=_SPLUNK_NOTABLES))
+
+ response = client.post(
+ "/api/v1/replay/history",
+ json=_body("splunk", {"splunk_url": "https://splunk:8089", "splunk_token": "t"}),
+ )
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body["vendor"] == "splunk"
+ assert body["count"] == 2
+ # The undetermined row is present and excluded, and both numbers are
+ # reported. A customer whose analysts close without a disposition has to
+ # see that rather than a small graded sample with no explanation.
+ assert body["labelled"] == 1
+ assert body["unlabeled"] == 1
+ assert [f["disposition"] for f in body["findings"]] == ["true_positive", UNLABELED]
+
+
+def test_the_serialised_finding_carries_every_field_the_replay_side_rebuilds(client: TestClient) -> None:
+ """The boundary is by name, so the names have to survive the trip."""
+ respx.post(url__regex=r".*/services/search/jobs$").mock(return_value=httpx.Response(201, json={"sid": "s"}))
+ respx.get(url__regex=r".*/results.*").mock(return_value=httpx.Response(200, json=_SPLUNK_NOTABLES))
+
+ response = client.post(
+ "/api/v1/replay/history",
+ json=_body("splunk", {"splunk_url": "https://splunk:8089", "splunk_token": "t"}),
+ )
+
+ finding = response.json()["findings"][0]
+ assert set(finding) == {
+ "vendor",
+ "finding_id",
+ "title",
+ "disposition",
+ "vendor_disposition",
+ "closed_at",
+ "closed_by",
+ "reason",
+ "rule_id",
+ "severity",
+ "raw",
+ }
+ # ISO-8601, not an epoch. The receiving parser refuses an unparseable
+ # close time rather than defaulting it, and a naive epoch would have to
+ # guess a timezone to be read back.
+ assert finding["closed_at"] == "2026-09-25T00:00:00+00:00"
+ # The vendor row travels untouched so the replay runner can hand it to the
+ # same connector normalize() production uses.
+ assert finding["raw"]["event_id"] == "ES-001"
+
+
+ "vendor",
+ ["splunk", "sentinel", "elastic", "qradar", "defender"],
+)
+def test_a_vendor_with_no_usable_credentials_is_refused_by_name(client: TestClient, vendor: str) -> None:
+ response = client.post("/api/v1/replay/history", json=_body(vendor, {}))
+
+ assert response.status_code == 422
+ detail = response.json()["detail"]
+ assert vendor in detail
+ # The refusal names the keys that would have built a client, sourced from
+ # the factory rather than retyped, so a key the factory stops reading
+ # cannot linger in the message.
+ for key in _CREDENTIAL_KEYS[vendor]:
+ assert key in detail
+
+
+def test_an_unsupported_vendor_is_refused_by_the_schema(client: TestClient) -> None:
+ response = client.post("/api/v1/replay/history", json=_body("crowdstrike", {"anything": "here"}))
+
+ assert response.status_code == 422
+
+
+def test_an_inverted_window_is_refused_rather_than_read_backwards(client: TestClient) -> None:
+ response = client.post(
+ "/api/v1/replay/history",
+ json={
+ "vendor": "splunk",
+ "credentials": {"splunk_url": "https://splunk:8089", "splunk_token": "t"},
+ "since": UNTIL.isoformat(),
+ "until": SINCE.isoformat(),
+ },
+ )
+
+ assert response.status_code == 422
+ assert "must be after" in response.json()["detail"]
+
+
+def test_a_vendor_error_is_a_502_naming_the_vendor_not_an_empty_window(client: TestClient) -> None:
+ """The distinction this whole surface rests on.
+
+ An empty list reads as "nobody closed anything in that window", which is a
+ measurement. A read that failed is not a measurement, and the two must not
+ produce the same response body.
+ """
+ respx.post(url__regex=r".*/services/search/jobs$").mock(return_value=httpx.Response(503, text="maintenance"))
+
+ response = client.post(
+ "/api/v1/replay/history",
+ json=_body("splunk", {"splunk_url": "https://splunk:8089", "splunk_token": "t"}),
+ )
+
+ assert response.status_code == 502
+ assert "splunk" in response.json()["detail"]
+
+
+def test_the_splunk_saved_search_the_wizard_collected_is_honoured(client: TestClient) -> None:
+ """Deployments rename the saved search, and a hardcoded one returns zero rows."""
+ created = respx.post(url__regex=r".*/services/search/jobs$").mock(return_value=httpx.Response(201, json={"sid": "s"}))
+ respx.get(url__regex=r".*/results.*").mock(return_value=httpx.Response(200, json={"results": []}))
+
+ client.post(
+ "/api/v1/replay/history",
+ json=_body(
+ "splunk",
+ {"splunk_url": "https://splunk:8089", "splunk_token": "t"},
+ search_override="`my_review_lookup`",
+ ),
+ )
+
+ assert "my_review_lookup" in created.calls[0].request.content.decode()
+
+
+def test_a_limit_above_the_ceiling_is_refused_rather_than_truncated(client: TestClient) -> None:
+ response = client.post(
+ "/api/v1/replay/history",
+ json=_body(
+ "splunk",
+ {"splunk_url": "https://splunk:8089", "splunk_token": "t"},
+ limit=MAX_FINDINGS + 1,
+ ),
+ )
+
+ assert response.status_code == 422
+
+
+def test_the_route_takes_no_tenant_from_the_caller() -> None:
+ """There is no tenant field, and that is the property rather than an omission.
+
+ This service never resolves a tenant. The credentials it is handed are the
+ scope, because they reach exactly one customer's SIEM. A tenant field here
+ would be a value the caller chose that nothing checked.
+ """
+ from app.api.replay_history_router import HistoryRequest
+
+ assert not any("tenant" in name for name in HistoryRequest.model_fields)
diff --git a/services/agents/app/api/replay_router.py b/services/agents/app/api/replay_router.py
new file mode 100644
index 000000000..c4e17caa4
--- /dev/null
+++ b/services/agents/app/api/replay_router.py
@@ -0,0 +1,187 @@
+"""Expose the replay runner so the API can orchestrate an evaluation.
+
+Gap-closure Phase 1.4.
+
+Phase 1.2 built ``app.replay.ReplayRunner``: it drives the production triage
+path over a frozen historical window and writes nothing. Nothing called it.
+This is the route that does, and it is the second of the two internal routes
+the orchestration in ``services/api`` needs.
+
+The tenant comes from the credential
+------------------------------------
+There is no ``tenant_id`` field on the request. The caller is either a console
+session, whose token carries a verified tenant claim, or the API acting as a
+trusted service, which declares the tenant it is acting for on
+``X-AiSOC-Tenant-ID``. Both resolve through the vendored
+``app.security.tenant_scope``, and a service token with no tenant header
+resolves to an empty scope that refuses rather than widens.
+
+That matters more here than on a read route. The tenant travels into the
+envelope handed to triage, so a caller-supplied tenant would decide which
+customer's business-context rules and institutional memory the graded verdict
+was produced under.
+
+This route writes nothing either
+--------------------------------
+The runner constructs the production worker with a
+:class:`~app.replay.shadow.ShadowTriageWriter`, and the response carries the
+attempted-write counters so the caller can assert on a number rather than on
+an absence. Those counters travel into the report's method section.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+import structlog
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel, Field
+
+from app.replay import (
+ DEFAULT_TRAIN_FRACTION,
+ ContextSnapshot,
+ HistoricalFinding,
+ NormalizerUnavailable,
+ ReplayRunner,
+ capture_context,
+ split_by_time,
+)
+from app.replay.connector_normalizer import fetch_normalized
+from app.security.tenant_scope import (
+ TenantPrincipal,
+ require_console_or_service_auth,
+ scoped_tenant_or_403,
+)
+
+logger = structlog.get_logger()
+
+router = APIRouter(prefix="/replay", tags=["replay"])
+
+ScopedPrincipal = Annotated[TenantPrincipal, Depends(require_console_or_service_auth)]
+
+#: Matches ``MAX_NORMALIZE_ROWS`` on the connectors service. A window larger
+#: than one normalize batch would have to be split, and a split batch is a
+#: second ordering to get wrong.
+MAX_FINDINGS = 2000
+
+
+class FrozenContext(BaseModel):
+ """Organisation memory and outcome priors as the caller captured them.
+
+ Optional. When absent the snapshot is empty, which is the honest default:
+ an empty frozen context grades the agent with no institutional memory at
+ all, which understates production rather than overstating it.
+
+ Priors are deliberately not derived from the train window's labels here or
+ anywhere. A human-authored prior suppresses a matching repeat without
+ re-triage, so seeding priors from ground truth would score the ground
+ truth.
+ """
+
+ statements: list[dict[str, Any]] = Field(default_factory=list)
+ priors: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+
+class ReplayRequest(BaseModel):
+ """One window of closed findings, and the connector whose mapping to use."""
+
+ #: The connector id whose ``normalize()`` the connectors service should
+ #: run. Not a vendor name: two deployments of the same vendor can be
+ #: reached by different connectors.
+ connector_id: str = Field(..., min_length=1)
+ #: ``ClosedFinding.as_dict()`` rows from the actions service.
+ findings: list[dict[str, Any]] = Field(..., min_length=1)
+ train_fraction: float = Field(default=DEFAULT_TRAIN_FRACTION, gt=0.0, lt=1.0)
+ context: FrozenContext | None = None
+
+
+class ReplayResponse(BaseModel):
+ decisions: list[dict[str, Any]]
+ #: Split point, frozen-context provenance, attempted writes and the
+ #: enrichment gaps a replayed envelope does not carry. Travels into the
+ #: report so a reader can decide whether to believe the numbers.
+ method: dict[str, Any]
+
+
[email protected]("/run", response_model=ReplayResponse, summary="Replay closed findings through production triage")
+async def run_replay(body: ReplayRequest, principal: ScopedPrincipal) -> ReplayResponse:
+ """Split, freeze, and replay the test window, writing nothing.
+
+ Every failure here is a refusal with a reason rather than a short result.
+ A replay that could not normalize its input, or whose history has no
+ usable close times, has nothing to measure, and returning a partial run
+ would publish a number over whichever rows happened to survive.
+ """
+ tenant_id = scoped_tenant_or_403(principal)
+
+ if len(body.findings) > MAX_FINDINGS:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=(
+ f"{len(body.findings)} findings exceeds the {MAX_FINDINGS}-finding limit for one replay; "
+ f"narrow the window rather than grading a truncated one"
+ ),
+ )
+
+ try:
+ findings = [HistoricalFinding.from_mapping(row) for row in body.findings]
+ except ValueError as exc:
+ # An unparseable close time is refused rather than defaulted. A
+ # finding with an invented close time lands on whichever side of the
+ # split the default happens to fall, which is a leak wearing a
+ # plausible timestamp.
+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
+
+ # The split is computed here rather than left to the runner, because the
+ # frozen context has to be captured *at* the split instant and the runner
+ # refuses a snapshot taken at any other one. Running the replay twice to
+ # discover the split would place every model call twice.
+ try:
+ split = split_by_time(findings, train_fraction=body.train_fraction)
+ except ValueError as exc:
+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
+
+ if body.context is None:
+ snapshot = ContextSnapshot(split_at=split.split_at)
+ else:
+ snapshot = capture_context(
+ split_at=split.split_at,
+ statements=body.context.statements,
+ priors=body.context.priors,
+ )
+
+ try:
+ normalizer = await fetch_normalized(
+ body.connector_id,
+ [dict(f.raw) for f in findings],
+ tenant_id=str(tenant_id),
+ )
+ except NormalizerUnavailable as exc:
+ raise HTTPException(
+ status_code=status.HTTP_502_BAD_GATEWAY,
+ detail=f"replay cannot reach the production normalizer: {exc}",
+ ) from exc
+
+ runner = ReplayRunner(
+ normalizer=normalizer,
+ tenant_id=str(tenant_id),
+ connector_id=body.connector_id,
+ )
+
+ try:
+ run = await runner.run(findings, snapshot=snapshot, train_fraction=body.train_fraction)
+ except ValueError as exc:
+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
+
+ logger.info(
+ "replay.run.completed",
+ tenant_id=str(tenant_id),
+ connector_id=body.connector_id,
+ findings=len(findings),
+ decisions=len(run.decisions),
+ writes_attempted=sum(run.writes_attempted.values()),
+ )
+ return ReplayResponse(
+ decisions=[d.as_dict() for d in run.decisions],
+ method=run.method(),
+ )
diff --git a/services/agents/app/main.py b/services/agents/app/main.py
index 5de9c6269..08a4682ed 100644
--- a/services/agents/app/main.py
+++ b/services/agents/app/main.py
@@ -16,6 +16,7 @@ from app.api.hunts import router as hunts_router
from app.api.investigate import router as investigate_router
from app.api.metrics import router as metrics_router
from app.api.playbooks import router as playbook_router
+from app.api.replay_router import router as replay_router
from app.api.router import router
from app.api.triage import router as triage_router
from app.core.telemetry import instrument_app
@@ -204,6 +205,9 @@ app.include_router(hunts_router) # prefix: /api/v1/hunts
app.include_router(hunt_search_router) # prefix: /api/v1/hunt (search + saved)
app.include_router(copilot_router) # prefix: /api/v1/copilot
app.include_router(explain_router) # prefix: /api/v1 (POST /explain — NDJSON stream)
+# Gap-closure Phase 1.4: POST /api/v1/replay/run. The runner shipped in 1.2
+# with no caller; this is what the API's evaluation job drives.
+app.include_router(replay_router, prefix="/api/v1")
@app.get("/health")
diff --git a/services/agents/app/replay/connector_normalizer.py b/services/agents/app/replay/connector_normalizer.py
index 628da6c55..c21a97cff 100644
--- a/services/agents/app/replay/connector_normalizer.py
+++ b/services/agents/app/replay/connector_normalizer.py
@@ -35,6 +35,7 @@ from __future__ import annotations
import hashlib
import json
import os
+import re
from collections.abc import Mapping, Sequence
from typing import Any
@@ -50,6 +51,27 @@ __all__ = ["PrenormalizedRows", "fetch_normalized", "row_key"]
_SERVICE_URL = os.getenv("CONNECTORS_SERVICE_URL", "http://connectors:8003")
_TIMEOUT_S = float(os.getenv("AISOC_REPLAY_NORMALIZE_TIMEOUT_S", "60"))
+#: The connectors service mounts its router under this prefix
+#: (``app/main.py``: ``include_router(router, prefix="/api/v1")``), and
+#: ``CONNECTORS_SERVICE_URL`` is the bare origin. The API's own caller in
+#: ``endpoints/connectors.py`` appends the same segment for the same reason.
+#:
+#: Connector ids are registry keys: lowercase letters, digits and underscores.
+#: The id reaches this module from a request body, and it is interpolated into
+#: a URL path, so it is validated against that shape before it gets there
+#: rather than encoded after. Encoding would make `../../admin` a literal
+#: segment that 404s; refusing says which value was wrong, and a value outside
+#: this shape could never name a real connector anyway.
+_CONNECTOR_ID_RE = re.compile(r"^[a-z0-9_]{1,64}$")
+
+#: This was wrong when the route shipped: the URL was built without it, so
+#: every normalize request would have 404'd. Nothing called this path until
+#: Phase 1.4 wired the CLI to it, which is the failure shape this program
+#: keeps finding - a mechanism that exists, is unit-tested, and has no caller
+#: on the path that needs it. ``test_the_normalize_url_matches_where_the_route_is_mounted``
+#: pins it against the connectors service's own mount.
+_API_PREFIX = "/api/v1"
+
def row_key(row: Mapping[str, Any]) -> str:
"""Stable identity for a vendor row, independent of dict ordering."""
@@ -96,6 +118,11 @@ async def fetch_normalized(
to measure, and continuing with partial coverage would publish a number
over whichever rows happened to succeed.
"""
+ if not _CONNECTOR_ID_RE.match(connector_id):
+ raise NormalizerUnavailable(
+ f"{connector_id!r} is not a connector id. Ids are lowercase letters, digits and "
+ f"underscores, and this one reaches a URL path, so it is refused rather than escaped."
+ )
if not rows:
return PrenormalizedRows({}, connector_id=connector_id)
@@ -107,7 +134,7 @@ async def fetch_normalized(
"and replay cannot reach the production normalizer"
)
- url = f"{base}/connectors/{connector_id}/normalize"
+ url = f"{base}{_API_PREFIX}/connectors/{connector_id}/normalize"
payload = {"rows": [dict(row) for row in rows]}
# A service token must declare the tenant it acts for; the connectors
# service refuses one that does not. The tenant comes from the caller's
diff --git a/services/agents/tests/test_replay_route.py b/services/agents/tests/test_replay_route.py
new file mode 100644
index 000000000..63502894f
--- /dev/null
+++ b/services/agents/tests/test_replay_route.py
@@ -0,0 +1,400 @@
+"""The replay route: the tenant comes from the credential, and nothing is written.
+
+Gap-closure Phase 1.4 gate.
+
+Phase 1.2 proved the runner splits, freezes and writes nothing. This proves
+the route in front of it resolves the tenant from the caller's credential
+rather than from the request, refuses a history it cannot read rather than
+grading a partial one, and reaches the production normalizer over HTTP rather
+than substituting a mapping of its own.
+
+The tenant assertions are the ones that matter. The tenant travels into the
+envelope handed to triage, so a caller-supplied tenant would decide which
+customer's business-context rules and institutional memory the graded verdict
+was produced under.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import httpx
+import pytest
+from app.api.replay_router import MAX_FINDINGS, ReplayRequest
+from app.api.replay_router import router as replay_router
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+_BASE = datetime(2026, 3, 1, tzinfo=UTC)
+_TENANT = "11111111-2222-3333-4444-555555555555"
+_SERVICE_TOKEN = "service-token-for-tests"
+
+_NOTABLE = {
+ "event_id": "ES-1",
+ "rule_id": "rule-42",
+ "search_name": "Suspicious PowerShell",
+ "urgency": "high",
+ "disposition": "disposition:1",
+ "src": "192.0.2.10",
+ "host": "WS-01",
+ "_time": "1772323200",
+}
+
+
+def _finding_payload(index: int, *, disposition: str = "false_positive") -> dict[str, Any]:
+ """One ``ClosedFinding.as_dict()`` row, as the actions service sends it."""
+ raw = {**_NOTABLE, "event_id": f"ES-{index:03d}"}
+ return {
+ "vendor": "splunk",
+ "finding_id": f"ES-{index:03d}",
+ "title": "Suspicious PowerShell",
+ "disposition": disposition,
+ "vendor_disposition": "disposition:1",
+ "closed_at": (_BASE + timedelta(hours=index)).isoformat(),
+ "closed_by": "a.analyst",
+ "reason": None,
+ "rule_id": "rule-42",
+ "severity": "high",
+ "raw": raw,
+ }
+
+
+def _normalized(raw: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "source": "splunk",
+ "external_id": raw.get("event_id", ""),
+ "title": raw.get("search_name") or "Splunk Notable Event",
+ "severity": "high",
+ "src_ip": raw.get("src"),
+ "hostname": raw.get("host"),
+ "raw_event": raw,
+ "created_at": raw.get("_time"),
+ }
+
+
+def _app() -> FastAPI:
+ """Only the replay router, mounted where ``app/main.py`` mounts it.
+
+ Importing ``app.main`` would pull the whole service in, including the hunt
+ scheduler's ``apscheduler``, which the agents CI job does not install
+ because no other test in this suite needs it. The behaviour under test is
+ the route's, and that it is reachable on the real application is asserted
+ two other ways: structurally by
+ ``test_the_router_is_included_by_the_service`` below, and live by
+ ``tests/e2e/test_replay_cli_end_to_end.py``, which drives the real
+ ``app.main`` through uvicorn.
+ """
+ app = FastAPI()
+ app.include_router(replay_router, prefix="/api/v1")
+ return app
+
+
+def client(monkeypatch: pytest.MonkeyPatch) -> TestClient:
+ monkeypatch.setenv("AISOC_DETERMINISTIC", "1")
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", _SERVICE_TOKEN)
+ monkeypatch.setenv("AISOC_AGENTS_SERVICE_TOKEN", "")
+ # No console secret configured, so only the service path is available and
+ # the tenant has to arrive on the header rather than in a token claim.
+ monkeypatch.delenv("SECRET_KEY", raising=False)
+ monkeypatch.delenv("AISOC_DEV_MODE", raising=False)
+ return TestClient(_app())
+
+
+def connectors(monkeypatch: pytest.MonkeyPatch) -> list[httpx.Request]:
+ """Stand in for the connectors service's ``POST /connectors/{id}/normalize``.
+
+ Patched at the transport rather than at ``fetch_normalized`` so the route
+ still builds the request, sends the headers and validates the response
+ shape. Patching the function would prove the route calls something.
+ """
+ seen: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request)
+ import json
+
+ rows = json.loads(request.content)["rows"]
+ return httpx.Response(200, json={"rows": [_normalized(row) for row in rows]})
+
+ real_client = httpx.AsyncClient
+
+ def _factory(*args: Any, **kwargs: Any) -> httpx.AsyncClient:
+ kwargs["transport"] = httpx.MockTransport(handler)
+ return real_client(*args, **kwargs)
+
+ monkeypatch.setattr("app.replay.connector_normalizer.httpx.AsyncClient", _factory)
+ return seen
+
+
+def _headers(tenant: str | None = _TENANT) -> dict[str, str]:
+ headers = {"Authorization": f"Bearer {_SERVICE_TOKEN}"}
+ if tenant is not None:
+ headers["X-AiSOC-Tenant-ID"] = tenant
+ return headers
+
+
+def _body(count: int = 20, **extra: Any) -> dict[str, Any]:
+ return {
+ "connector_id": "splunk",
+ "findings": [_finding_payload(i, disposition="true_positive" if i % 4 == 0 else "false_positive") for i in range(count)],
+ **extra,
+ }
+
+
+def test_a_replay_runs_the_test_window_and_reports_its_method(client: TestClient, connectors: list[httpx.Request]) -> None:
+ response = client.post("/api/v1/replay/run", json=_body(20), headers=_headers())
+
+ assert response.status_code == 200
+ body = response.json()
+ # 70/30 by time: 14 train, 6 test, and only the test window is replayed.
+ assert len(body["decisions"]) == 6
+ assert body["method"]["split"]["train_findings"] == 14
+ assert body["method"]["split"]["test_findings"] == 6
+ assert body["method"]["tenant_id"] == _TENANT
+ # What production *would* have written, per sink, intercepted rather than
+ # performed. A number rather than an absence: a replay that never reached
+ # the writeback branch and a replay whose writeback was suppressed look
+ # identical from outside, and only the counter tells them apart.
+ attempted = body["method"]["shadow_writes_attempted"]
+ assert set(attempted) == {
+ "cache_verdict",
+ "persist_auto_triage",
+ "record_outcome",
+ "write_back_disposition",
+ }
+ assert all(count == len(body["decisions"]) for count in attempted.values())
+ # The gaps a replayed envelope does not carry travel into every report.
+ assert body["method"]["envelope_limits"]
+
+
+def test_a_service_token_with_no_tenant_header_is_refused_rather_than_widened(client: TestClient, connectors: list[httpx.Request]) -> None:
+ """An absent scope is empty, never every scope.
+
+ Every cross-tenant leak this codebase has had took this shape: a scope
+ that was absent rather than narrow, and a read that treated absent as
+ "no filter".
+ """
+ response = client.post("/api/v1/replay/run", json=_body(10), headers=_headers(tenant=None))
+
+ assert response.status_code == 403
+ assert not connectors
+
+
+def test_no_credential_at_all_is_a_401(client: TestClient) -> None:
+ response = client.post("/api/v1/replay/run", json=_body(10))
+
+ assert response.status_code == 401
+
+
+def test_the_request_carries_no_tenant_field() -> None:
+ """The tenant is not something a caller can name here."""
+ assert not any("tenant" in name for name in ReplayRequest.model_fields)
+
+
+def test_the_tenant_the_credential_named_is_the_one_forwarded_to_connectors(client: TestClient, connectors: list[httpx.Request]) -> None:
+ other = "99999999-8888-7777-6666-555555555555"
+
+ client.post("/api/v1/replay/run", json=_body(10), headers=_headers(tenant=other))
+
+ assert connectors
+ assert connectors[0].headers["X-AiSOC-Tenant-ID"] == other
+
+
+def test_an_unparseable_close_time_is_refused_rather_than_defaulted(client: TestClient, connectors: list[httpx.Request]) -> None:
+ """A finding with an invented close time is a leak wearing a plausible timestamp.
+
+ It lands on whichever side of the train/test split the default happens to
+ fall, which can put a finding in the test window whose answer is already
+ inside the frozen context.
+ """
+ body = _body(10)
+ body["findings"][3]["closed_at"] = "the day before yesterday"
+
+ response = client.post("/api/v1/replay/run", json=body, headers=_headers())
+
+ assert response.status_code == 422
+ assert "closed_at" in response.json()["detail"]
+ assert not connectors
+
+
+def test_a_normalizer_that_cannot_be_reached_is_a_502_not_a_substituted_mapping(
+ client: TestClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Replay has no degraded mode here.
+
+ Grading the agent on a mapping written inside this service would measure a
+ pipeline nobody runs, and the output has the same shape as the real thing.
+ """
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(503, text="connectors is down")
+
+ real_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ "app.replay.connector_normalizer.httpx.AsyncClient",
+ lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),
+ )
+
+ response = client.post("/api/v1/replay/run", json=_body(10), headers=_headers())
+
+ assert response.status_code == 502
+ assert "normalizer" in response.json()["detail"]
+
+
+def test_a_window_above_the_ceiling_is_refused_rather_than_truncated(client: TestClient, connectors: list[httpx.Request]) -> None:
+ # Assert the constant is what the message quotes, rather than restating it.
+ assert MAX_FINDINGS == 2000
+ findings: list[dict[str, Any]] = [_finding_payload(i) for i in range(3)] * 700 # 2100 rows
+ body: dict[str, Any] = {"connector_id": "splunk", "findings": findings}
+
+ response = client.post("/api/v1/replay/run", json=body, headers=_headers())
+
+ assert response.status_code == 422
+ assert str(MAX_FINDINGS) in response.json()["detail"]
+
+
+def test_a_frozen_context_is_captured_at_the_split_the_runner_chose(client: TestClient, connectors: list[httpx.Request]) -> None:
+ """The snapshot has to be taken at the split instant or it is not a freeze.
+
+ A statement recorded after the split is dropped, and the count of drops
+ travels into the report so the freeze is not claimed to be tighter than
+ it is.
+ """
+ after_split = (_BASE + timedelta(hours=19)).isoformat()
+ before_split = (_BASE + timedelta(hours=2)).isoformat()
+ body = _body(
+ 20,
+ context={
+ "statements": [
+ {"statement": "finance bulk exports are expected", "created_at": before_split},
+ {"statement": "this alert was a true positive", "created_at": after_split},
+ ],
+ "priors": {},
+ },
+ )
+
+ response = client.post("/api/v1/replay/run", json=body, headers=_headers())
+
+ assert response.status_code == 200
+ frozen = response.json()["method"]["frozen_context"]
+ assert frozen["statements_frozen"] == 1
+ assert frozen["statements_dropped_after_split"] == 1
+
+
+def test_two_runs_over_one_history_produce_identical_decisions_through_the_route(
+ client: TestClient, connectors: list[httpx.Request]
+) -> None:
+ """The phase's acceptance bar, at this link.
+
+ Latency is excluded and nothing else is: it is a wall-clock measurement
+ and will never repeat. Every other field is a property of the input and
+ the code.
+ """
+ body = _body(20)
+
+ def _decisions() -> list[dict[str, Any]]:
+ response = client.post("/api/v1/replay/run", json=body, headers=_headers())
+ assert response.status_code == 200
+ rows = []
+ for decision in response.json()["decisions"]:
+ decision.pop("latency_ms")
+ rows.append(decision)
+ return rows
+
+ first = _decisions()
+ second = _decisions()
+ assert first == second
+ assert len(first) == 6
+
+
+def test_the_route_is_served_where_the_client_expects_it(client: TestClient) -> None:
+ """Read off the served OpenAPI document, not off ``app.routes``.
+
+ This FastAPI holds included routers lazily, so ``app.routes`` lists
+ wrappers and a membership test against it passes vacuously.
+ """
+ schema = client.get("/openapi.json").json()
+ assert "/api/v1/replay/run" in schema["paths"]
+
+
+def test_the_router_is_included_by_the_service() -> None:
+ """A router nobody includes is the failure this phase exists to close.
+
+ Parsed with ``ast`` rather than imported, for the reason ``_app`` records.
+ The live proof that the deployed application serves this route is
+ ``tests/e2e/test_replay_cli_end_to_end.py``, which drives it through four
+ real services.
+ """
+ import ast
+
+ main = ast.parse((Path(__file__).resolve().parents[1] / "app" / "main.py").read_text())
+ included = {
+ node.args[0].id
+ for node in ast.walk(main)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "include_router"
+ and node.args
+ and isinstance(node.args[0], ast.Name)
+ }
+ assert "replay_router" in included
+
+
+def test_a_console_token_carries_its_own_tenant(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The browser path does not use the service header at all.
+
+ Recorded because the two credential shapes resolve differently and only
+ one of them may name a tenant in a header.
+ """
+ import base64
+ import hashlib
+ import hmac
+ import json
+ import time
+
+ secret = "a-console-secret-at-least-32-characters-long"
+ monkeypatch.setenv("AISOC_DETERMINISTIC", "1")
+ monkeypatch.setenv("SECRET_KEY", secret)
+ monkeypatch.setenv("AISOC_SERVICE_TOKEN", _SERVICE_TOKEN)
+
+ def _segment(payload: dict[str, Any]) -> str:
+ return base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
+
+ tenant = str(uuid.uuid4())
+ header = _segment({"alg": "HS256", "typ": "JWT"})
+ claims = _segment({"sub": "analyst", "tenant_id": tenant, "type": "access", "exp": int(time.time()) + 600})
+ signature = hmac.new(secret.encode(), f"{header}.{claims}".encode(), hashlib.sha256).digest()
+ token = f"{header}.{claims}.{base64.urlsafe_b64encode(signature).decode().rstrip('=')}"
+
+ seen: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request)
+ rows = json.loads(request.content)["rows"]
+ return httpx.Response(200, json={"rows": [_normalized(row) for row in rows]})
+
+ real_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ "app.replay.connector_normalizer.httpx.AsyncClient",
+ lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),
+ )
+
+ response = TestClient(_app()).post(
+ "/api/v1/replay/run",
+ json=_body(10),
+ headers={"Authorization": f"Bearer {token}"},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["method"]["tenant_id"] == tenant
+ # Filtered to the normalize call rather than taking the first request
+ # seen: the patched client is process-wide and anything else the service
+ # does over httpx lands in the same list.
+ normalize_calls = [r for r in seen if r.url.path.endswith("/normalize")]
+ assert normalize_calls
+ assert normalize_calls[0].headers["X-AiSOC-Tenant-ID"] == tenant
diff --git a/services/agents/tests/test_replay_runner.py b/services/agents/tests/test_replay_runner.py
index 419f696fb..7158887e4 100644
--- a/services/agents/tests/test_replay_runner.py
+++ b/services/agents/tests/test_replay_runner.py
@@ -7,11 +7,17 @@ from __future__ import annotations
import uuid
from datetime import UTC, datetime, timedelta
+from pathlib import Path
from typing import Any
import httpx
import pytest
-from app.replay.connector_normalizer import PrenormalizedRows, fetch_normalized, row_key
+from app.replay.connector_normalizer import (
+ _API_PREFIX,
+ PrenormalizedRows,
+ fetch_normalized,
+ row_key,
+)
from app.replay.findings import HistoricalFinding
from app.replay.normalize import NormalizerUnavailable, to_fused_envelope
from app.replay.runner import DEFAULT_TRAIN_FRACTION, ReplayRunner, split_by_time
@@ -263,7 +269,13 @@ async def test_the_connectors_service_supplies_the_normalized_envelopes() -> Non
patch.setattr(httpx, "AsyncClient", _client)
lookup = await fetch_normalized("splunk", rows, tenant_id="t-1", service_url="http://connectors:8003", service_token="secret")
- assert seen["url"] == "http://connectors:8003/connectors/splunk/normalize"
+ # The connectors service mounts its router under /api/v1. This asserted
+ # the bare path when the route shipped, which codified the defect rather
+ # than catching it: nothing called `fetch_normalized` until Phase 1.4, so
+ # a 404 on every request was invisible.
+ # `test_the_normalize_url_matches_where_the_route_is_mounted` below reads
+ # the mount out of the connectors service instead of restating it here.
+ assert seen["url"] == "http://connectors:8003/api/v1/connectors/splunk/normalize"
assert seen["token"] == "Bearer secret"
assert seen["tenant"] == "t-1"
assert isinstance(lookup, PrenormalizedRows)
@@ -302,3 +314,79 @@ def test_a_row_outside_the_batch_is_refused_never_substituted() -> None:
with pytest.raises(NormalizerUnavailable, match="will not substitute"):
lookup.normalize({"event_id": "never-sent"})
+
+
+def test_the_normalize_url_matches_where_the_route_is_mounted() -> None:
+ """Both directions, against the connectors service's own source.
+
+ Gap-closure Phase 1.4. This service cannot import ``services/connectors``
+ (both package their code as top-level ``app``), so the mount prefix and
+ the route path are read out of that tree with ``ast`` rather than
+ restated here. A test that restates them is a test that agrees with
+ itself: the previous one asserted a URL missing the ``/api/v1`` segment
+ and passed for as long as nothing called the function.
+ """
+ import ast
+
+ connectors = Path(__file__).resolve().parents[3] / "services" / "connectors" / "app"
+
+ main = ast.parse((connectors / "main.py").read_text())
+ prefixes: set[str] = {
+ str(keyword.value.value)
+ for node in ast.walk(main)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "include_router"
+ for keyword in node.keywords
+ if keyword.arg == "prefix" and isinstance(keyword.value, ast.Constant) and isinstance(keyword.value.value, str)
+ }
+ assert prefixes, "could not read any router prefix out of the connectors service"
+
+ router = ast.parse((connectors / "api" / "router.py").read_text())
+ routes: set[str] = {
+ str(decorator.args[0].value)
+ for node in ast.walk(router)
+ if isinstance(node, ast.AsyncFunctionDef | ast.FunctionDef)
+ for decorator in node.decorator_list
+ if isinstance(decorator, ast.Call)
+ and isinstance(decorator.func, ast.Attribute)
+ and decorator.func.attr in {"post", "get"}
+ and decorator.args
+ and isinstance(decorator.args[0], ast.Constant)
+ and isinstance(decorator.args[0].value, str)
+ }
+ normalize_route = next(r for r in routes if r.endswith("/normalize"))
+
+ # What `fetch_normalized` would actually request, built the same way it
+ # builds it, against what the service actually serves.
+ served = {f"{prefix}{normalize_route}".replace("{connector_id}", "splunk") for prefix in prefixes}
+ requested = f"{_API_PREFIX}/connectors/splunk/normalize"
+ assert requested in served, f"{requested} is not served; the service mounts {sorted(served)}"
+
+
+ "connector_id",
+ [
+ "../../admin",
+ "splunk/../../v1/shutdown",
+ "http://169.254.169.254/latest/meta-data",
+ "splunk?x=1",
+ "SPLUNK",
+ "",
+ ],
+)
+async def test_a_connector_id_that_is_not_one_never_reaches_a_url(connector_id: str) -> None:
+ """The id arrives in a request body and is interpolated into a URL path.
+
+ Refused against the registry's own shape rather than escaped: encoding
+ would turn `../../admin` into a literal segment that 404s, which reads as
+ a missing connector, while refusing names the value that was wrong. A
+ string outside this shape could not name a real connector anyway.
+ """
+ with pytest.raises(NormalizerUnavailable, match="not a connector id"):
+ await fetch_normalized(
+ connector_id,
+ [dict(_NOTABLE)],
+ tenant_id="t-1",
+ service_url="http://connectors:8003",
+ service_token="secret",
+ )
diff --git a/services/api/app/_vendor/aisoc_benchmark/VENDORED.md b/services/api/app/_vendor/aisoc_benchmark/VENDORED.md
new file mode 100644
index 000000000..e055fcd1b
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/VENDORED.md
@@ -0,0 +1,42 @@
+# Vendored: `aisoc_benchmark`
+
+This directory is an **automatically maintained mirror** of
+`packages/aisoc-benchmark/aisoc_benchmark/`. Do not hand-edit files here.
+
+## Why it exists
+
+The replay evaluation job (gap-closure Phase 1.4) is orchestrated by
+`services/api`, because that service holds the credential vault and the tenant
+session and already proxies to `services/agents` and `services/actions`.
+Scoring is the one link in that chain with no round trip, since
+`aisoc-benchmark` is a distribution rather than a service.
+
+It still cannot be imported directly: the `aisoc-core-api` image is built with
+`./services/api` as its build context, so nothing under `packages/` is present
+at runtime. Same constraint that produced `_vendor/narrative.py`,
+`_vendor/llm_contract_rules.py` and `_vendor/nl_query/`.
+
+The alternative was a second scorer living in the API. `replay.py` opens by
+explaining why there is exactly one grader in this repository: a second one
+would eventually disagree with the first about what "hallucination rate"
+means, and both would keep calling themselves the same number.
+
+## How it stays in sync
+
+Run `python scripts/sync_vendored_benchmark.py` after changing anything under
+`packages/aisoc-benchmark/aisoc_benchmark/`. CI runs the same script in
+`--check` mode and fails the build when the two trees drift. The check runs in
+**both directions**: a module in the mirror with no source counterpart fails
+as well as a stale copy.
+
+## Files
+
+Every `*.py` file in the source package is mirrored byte for byte. At the time
+of writing that is `__init__.py`, `adapter.py`, `cli.py`, `metrics.py`,
+`replay.py` and `runner.py`. The list is derived from the source directory
+rather than recorded here, so a seventh module is picked up by existing rather
+than by being added to a list.
+
+The API imports `score_replay`, `format_replay_report` and `ReplayScore` from
+this mirror. `cli.py` and `runner.py` are mirrored for completeness and have
+no caller inside the API.
diff --git a/services/api/app/_vendor/aisoc_benchmark/__init__.py b/services/api/app/_vendor/aisoc_benchmark/__init__.py
new file mode 100644
index 000000000..a8f0ae43f
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/__init__.py
@@ -0,0 +1,56 @@
+"""Run the SOC agent benchmark against any agent, including one that is not ours.
+
+A benchmark a vendor cannot run against their own product is a self-report.
+This package is the boundary that makes it a standard instead: implement one
+method, get graded on the same corpus by the same code.
+"""
+
+from .adapter import AgentVerdict, BenchmarkIncident, HTTPAgent, SOCAgent
+from .metrics import (
+ BenchmarkResult,
+ IncidentScore,
+ aggregate,
+ extract_checkable_indicators,
+ score_incident,
+)
+from .replay import (
+ LATENCY_FIELDS,
+ LATENCY_LINE_PREFIX,
+ MALICIOUS,
+ MIN_MALICIOUS_FOR_HEADLINE,
+ CalibrationBin,
+ ClassScore,
+ ReplayScore,
+ SegmentScore,
+ format_replay_report,
+ score_replay,
+ strip_latency,
+)
+from .runner import format_report, load_corpus, run_benchmark
+
+__all__ = [
+ "LATENCY_FIELDS",
+ "LATENCY_LINE_PREFIX",
+ "MALICIOUS",
+ "MIN_MALICIOUS_FOR_HEADLINE",
+ "AgentVerdict",
+ "BenchmarkIncident",
+ "BenchmarkResult",
+ "CalibrationBin",
+ "ClassScore",
+ "HTTPAgent",
+ "IncidentScore",
+ "ReplayScore",
+ "SOCAgent",
+ "SegmentScore",
+ "aggregate",
+ "extract_checkable_indicators",
+ "format_report",
+ "load_corpus",
+ "run_benchmark",
+ "score_incident",
+ "format_replay_report",
+ "score_replay",
+ "strip_latency",
+]
+__version__ = "0.1.0"
diff --git a/services/api/app/_vendor/aisoc_benchmark/adapter.py b/services/api/app/_vendor/aisoc_benchmark/adapter.py
new file mode 100644
index 000000000..3873dd855
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/adapter.py
@@ -0,0 +1,199 @@
+"""The adapter protocol: what any SOC agent must implement to be graded.
+
+Pillar 4. A benchmark a vendor cannot run against their own product is a
+self-report. The existing harness grades the in-tree agent by importing it,
+which means the numbers are only ever about AiSOC — and a scoreboard with one
+entrant is a marketing page.
+
+This is the boundary that makes it a standard instead. An agent is anything
+that takes an incident and returns a verdict, so the protocol is deliberately
+small: four fields in, five fields out, no imports from this repository
+required. A vendor implements it in twenty lines against their own API and
+gets graded on the same corpus by the same code.
+
+Three deliberate exclusions:
+
+* **No tool interface.** How an agent reaches evidence is its business. The
+ benchmark supplies the incident and the telemetry corpus and grades the
+ answer; prescribing the tools would grade architecture rather than
+ outcomes, and would exclude every agent that does not look like ours.
+* **No prompt.** Same reason.
+* **No partial credit for effort.** An agent that called forty tools and
+ concluded wrongly scores as wrongly as one that guessed. Depth is reported
+ separately because it explains a score; it is not a score.
+
+What *is* prescribed is the shape of the answer, because the metrics depend
+on it — specifically on being able to tell a confident wrong answer from an
+abstention, and a cited indicator from an invented one.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+
+@dataclass(frozen=True)
+class BenchmarkIncident:
+ """One incident to investigate.
+
+ Supplied verbatim to the agent. ``telemetry`` is the events the agent may
+ reason over, so an agent with no live estate can still be graded — the
+ corpus is the estate.
+ """
+
+ incident_id: str
+ title: str
+ description: str
+ severity: str
+ raw_alert: dict[str, Any] = field(default_factory=dict)
+ telemetry: list[dict[str, Any]] = field(default_factory=list)
+
+ def as_dict(self) -> dict[str, Any]:
+ return {
+ "incident_id": self.incident_id,
+ "title": self.title,
+ "description": self.description,
+ "severity": self.severity,
+ "raw_alert": self.raw_alert,
+ "telemetry": self.telemetry,
+ }
+
+
+@dataclass
+class AgentVerdict:
+ """What an agent returns. Every field earns its place in a metric.
+
+ ``abstained`` exists because "I do not know" is a correct answer that a
+ forced-choice benchmark punishes, and punishing it trains agents to
+ guess. An abstention scores neither right nor wrong, and the abstention
+ rate is published alongside accuracy so the two cannot be traded off
+ invisibly.
+ """
+
+ #: malicious | suspicious | benign | unknown
+ disposition: str = "unknown"
+
+ #: 0.0-1.0. Used for calibration, not for credit: a confidently wrong
+ #: answer is worse than a hedged one and the metrics say so.
+ confidence: float = 0.0
+
+ #: ATT&CK technique ids the agent attributes to the incident.
+ techniques: list[str] = field(default_factory=list)
+
+ #: Concrete indicators the agent's reasoning relies on — addresses,
+ #: hashes, hostnames, accounts. Graded against the evidence it was given.
+ #: This is how hallucination is measured rather than estimated.
+ cited_indicators: list[str] = field(default_factory=list)
+
+ #: Actions the agent would take. Graded for containment appropriateness.
+ proposed_actions: list[str] = field(default_factory=list)
+
+ #: Free-text reasoning. Not scored directly; retained so a result can be
+ #: read by a human who disagrees with the score.
+ narrative: str = ""
+
+ #: True when the agent declined to reach a verdict.
+ abstained: bool = False
+
+ #: Optional, and reported rather than scored. An agent that reached the
+ #: right answer in one step is not worse than one that took twelve.
+ tool_calls: int = 0
+ distinct_tools: int = 0
+ latency_ms: int = 0
+ tokens: int = 0
+ usd_cost: float = 0.0
+
+ @classmethod
+ def from_dict(cls, payload: dict[str, Any]) -> AgentVerdict:
+ """Build from a JSON response, tolerating an incomplete one.
+
+ Missing fields default rather than raise: an adapter under
+ development should score badly, not crash the run. A crash is
+ indistinguishable from a harness bug, and the entrant will assume it
+ is ours.
+ """
+ return cls(
+ disposition=str(payload.get("disposition", "unknown")).lower(),
+ confidence=float(payload.get("confidence", 0.0) or 0.0),
+ techniques=[str(t).upper() for t in (payload.get("techniques") or [])],
+ cited_indicators=[str(i) for i in (payload.get("cited_indicators") or [])],
+ proposed_actions=[str(a) for a in (payload.get("proposed_actions") or [])],
+ narrative=str(payload.get("narrative", "")),
+ abstained=bool(payload.get("abstained", False)),
+ tool_calls=int(payload.get("tool_calls", 0) or 0),
+ distinct_tools=int(payload.get("distinct_tools", 0) or 0),
+ latency_ms=int(payload.get("latency_ms", 0) or 0),
+ tokens=int(payload.get("tokens", 0) or 0),
+ usd_cost=float(payload.get("usd_cost", 0.0) or 0.0),
+ )
+
+
+@runtime_checkable
+class SOCAgent(Protocol):
+ """Implement this to be graded.
+
+ One method. Everything else about the agent — model, tools, prompts,
+ architecture — is out of scope by design.
+ """
+
+ name: str
+ version: str
+
+ async def investigate(self, incident: BenchmarkIncident) -> AgentVerdict:
+ """Investigate one incident and return a verdict.
+
+ Must not raise. An agent that cannot reach a verdict should return
+ ``AgentVerdict(abstained=True)`` with a narrative explaining why: the
+ benchmark distinguishes "declined" from "crashed", and an exception
+ is scored as a crash.
+ """
+
+
+class HTTPAgent:
+ """Adapter for an agent behind an HTTP endpoint.
+
+ The path most entrants will take: point the benchmark at a URL, get
+ graded. Posts the incident as JSON and reads an ``AgentVerdict``-shaped
+ response.
+
+ A failed call is an abstention with the failure named, not an exception.
+ A harness that dies on one bad response grades nothing, and the entrant
+ reasonably assumes the fault is ours.
+ """
+
+ def __init__(
+ self,
+ url: str,
+ *,
+ name: str = "http-agent",
+ version: str = "unknown",
+ headers: dict[str, str] | None = None,
+ timeout_seconds: float = 120.0,
+ ) -> None:
+ self.url = url
+ self.name = name
+ self.version = version
+ self.headers = headers or {}
+ self.timeout_seconds = timeout_seconds
+
+ async def investigate(self, incident: BenchmarkIncident) -> AgentVerdict:
+ import httpx
+
+ try:
+ async with httpx.AsyncClient(timeout=self.timeout_seconds) as client:
+ response = await client.post(self.url, json=incident.as_dict(), headers=self.headers)
+ response.raise_for_status()
+ payload = response.json()
+ except Exception as exc:
+ return AgentVerdict(
+ abstained=True,
+ narrative=(f"adapter error: {type(exc).__name__}: {exc}. Scored as an abstention, not as a wrong answer."),
+ )
+
+ if not isinstance(payload, dict):
+ return AgentVerdict(
+ abstained=True,
+ narrative=f"adapter returned {type(payload).__name__}, expected an object",
+ )
+ return AgentVerdict.from_dict(payload)
diff --git a/services/api/app/_vendor/aisoc_benchmark/cli.py b/services/api/app/_vendor/aisoc_benchmark/cli.py
new file mode 100644
index 000000000..6b07b0947
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/cli.py
@@ -0,0 +1,84 @@
+"""``aisoc-benchmark`` — grade any SOC agent on the same corpus.
+
+ aisoc-benchmark run --agent-url https://your-agent/investigate \\
+ --corpus corpus/soc-agent-benchmark-v1.json
+
+The point of the command is that it does not import AiSOC. A vendor points
+it at their own endpoint and gets the same numbers computed by the same
+code, which is what separates a benchmark from a self-report.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from .adapter import HTTPAgent
+from .runner import CorpusError, format_report, load_corpus, run_benchmark
+
+DEFAULT_CORPUS = Path(__file__).resolve().parent.parent / "corpus" / "soc-agent-benchmark-v1.json"
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(prog="aisoc-benchmark", description=__doc__)
+ sub = parser.add_subparsers(dest="command", required=True)
+
+ run = sub.add_parser("run", help="Grade an agent over the corpus.")
+ run.add_argument("--agent-url", required=True, help="POST endpoint taking an incident.")
+ run.add_argument("--agent-name", default="http-agent")
+ run.add_argument("--agent-version", default="unknown")
+ run.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
+ run.add_argument("--header", action="append", default=[], metavar="K=V")
+ run.add_argument("--concurrency", type=int, default=4)
+ run.add_argument("--limit", type=int, default=None)
+ run.add_argument("--json", action="store_true", help="Emit JSON instead of a report.")
+ run.add_argument("--out", type=Path, help="Write the JSON result here as well.")
+
+ describe = sub.add_parser("corpus", help="Describe the corpus without running anything.")
+ describe.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS)
+
+ args = parser.parse_args(argv)
+
+ if args.command == "corpus":
+ try:
+ incidents, _, provenance = load_corpus(args.corpus)
+ except CorpusError as exc:
+ print(f"aisoc-benchmark: {exc}", file=sys.stderr)
+ return 1
+ print(f"{len(incidents)} incidents in {args.corpus}")
+ for source, count in sorted(provenance.items()):
+ print(f" {source:<20} {count}")
+ return 0
+
+ headers: dict[str, str] = {}
+ for item in args.header:
+ key, _, value = item.partition("=")
+ if key and value:
+ headers[key] = value
+
+ agent = HTTPAgent(
+ args.agent_url,
+ name=args.agent_name,
+ version=args.agent_version,
+ headers=headers,
+ )
+
+ try:
+ result = asyncio.run(run_benchmark(agent, args.corpus, concurrency=args.concurrency, limit=args.limit))
+ except CorpusError as exc:
+ print(f"aisoc-benchmark: {exc}", file=sys.stderr)
+ return 1
+
+ payload = result.as_dict()
+ if args.out:
+ args.out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+ print(json.dumps(payload, indent=2) if args.json else format_report(result))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/services/api/app/_vendor/aisoc_benchmark/metrics.py b/services/api/app/_vendor/aisoc_benchmark/metrics.py
new file mode 100644
index 000000000..77bb551ab
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/metrics.py
@@ -0,0 +1,291 @@
+"""Benchmark metrics: accuracy, hallucination, containment, cost.
+
+Pillar 4. The existing scoreboard publishes MITRE accuracy and three
+substrate self-consistency numbers. Three metrics the plan calls for are
+absent, and each is absent for the same reason: they are the ones that make
+an agent look worse.
+
+**Hallucination rate.** Measured, not estimated: the fraction of concrete
+indicators an agent cites that do not appear anywhere in the evidence it was
+given. An agent that invents a plausible IP address to justify a verdict is
+doing the single most dangerous thing an analyst-replacement can do, and no
+accuracy number reveals it.
+
+**Containment accuracy.** Whether the proposed action matches what the
+incident needed. An agent that reaches the right verdict and proposes
+isolating the wrong host has not helped.
+
+**Calibration.** Confidence is only useful if it tracks correctness.
+Published as the gap between mean confidence on correct answers and on wrong
+ones; a small or negative gap means the confidence score carries no
+information, and everything downstream that gates on a threshold is gating
+on noise.
+
+Abstention is treated as its own outcome throughout rather than folded into
+either accuracy or error. "I do not know" is a correct answer that a
+forced-choice benchmark punishes, and punishing it trains agents to guess.
+The abstention rate is published beside accuracy so the two cannot be traded
+off invisibly.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+#: Indicator shapes we can check against evidence. Free-text claims are not
+#: graded: "the process behaved suspiciously" is not checkable, and scoring
+#: it would measure phrasing.
+_INDICATOR_PATTERNS = (
+ re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), # IPv4
+ re.compile(r"\b[a-fA-F0-9]{64}\b"), # SHA-256
+ re.compile(r"\b[a-fA-F0-9]{40}\b"), # SHA-1
+ re.compile(r"\b[a-fA-F0-9]{32}\b"), # MD5
+ # Domain-ish, including every label. The single-label form this used to
+ # carry matched only `evil.example` out of `evil.example.com`, which was
+ # invisible while the pattern was used solely as a yes/no test and became
+ # visible the moment `extract_checkable_indicators` started returning the
+ # matched text. Anything the narrower form matched this one still matches,
+ # so `is_checkable_indicator` answers exactly as before.
+ re.compile(r"\b[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*\.[a-zA-Z]{2,}\b"),
+)
+
+
+def is_checkable_indicator(value: str) -> bool:
+ """Whether a cited indicator has a shape we can verify against evidence."""
+ text = (value or "").strip()
+ if not text:
+ return False
+ return any(pattern.search(text) for pattern in _INDICATOR_PATTERNS)
+
+
+def extract_checkable_indicators(text: str) -> list[str]:
+ """Pull every checkable indicator out of free-text reasoning.
+
+ Replay evaluation (gap-closure Phase 1.3) grades what the production
+ triage path wrote, and that path emits prose: findings and a confidence
+ basis, not a structured ``cited_indicators`` list. So the indicators have
+ to be read back out of the sentences.
+
+ The pattern set is :data:`_INDICATOR_PATTERNS`, unchanged, which is what
+ makes this an extension of the existing hallucination measurement rather
+ than a second one. Anything this returns is by construction something
+ :func:`is_checkable_indicator` accepts, so both paths grade the same
+ vocabulary.
+
+ Over-counting is the safe direction and is the one this errs in: a phrase
+ shaped like a domain that is really an English abbreviation will be
+ checked against the evidence and, if absent, counted as hallucinated. That
+ makes the published rate a ceiling rather than a flattering floor, and the
+ per-decision list travels with the report so a disputed count can be
+ re-derived by a reader who disagrees.
+
+ Ordered by first appearance and de-duplicated, so the same address cited
+ four times is one indicator rather than four.
+ """
+ found: list[str] = []
+ seen: set[str] = set()
+ for pattern in _INDICATOR_PATTERNS:
+ for match in pattern.finditer(text or ""):
+ value = match.group(0)
+ key = value.lower()
+ if key in seen:
+ continue
+ seen.add(key)
+ found.append(value)
+ return sorted(found, key=lambda v: ((text or "").find(v), v))
+
+
+def evidence_corpus(incident: Any) -> str:
+ """Everything the agent was given, flattened for containment checks."""
+ parts = [
+ str(getattr(incident, "title", "")),
+ str(getattr(incident, "description", "")),
+ repr(getattr(incident, "raw_alert", {})),
+ repr(getattr(incident, "telemetry", [])),
+ ]
+ return " ".join(parts).lower()
+
+
+@dataclass
+class IncidentScore:
+ """One incident's outcome. Kept per-incident so a published aggregate
+ can be traced back to the cases that produced it."""
+
+ incident_id: str
+ abstained: bool = False
+ disposition_correct: bool = False
+ technique_recall: float = 0.0
+ technique_precision: float = 0.0
+ hallucinated_indicators: list[str] = field(default_factory=list)
+ cited_checkable: int = 0
+ containment_correct: bool | None = None
+ confidence: float = 0.0
+ latency_ms: int = 0
+ tokens: int = 0
+ usd_cost: float = 0.0
+ distinct_tools: int = 0
+
+
+@dataclass
+class BenchmarkResult:
+ """Aggregate scores. Every field is defined so a reader can reproduce it."""
+
+ agent_name: str
+ agent_version: str
+ incidents: int = 0
+
+ # Accuracy, computed over answered incidents only. Dividing by the whole
+ # corpus would let an agent raise its score by abstaining more, which is
+ # the exact behaviour the abstention rate exists to expose.
+ disposition_accuracy: float = 0.0
+ technique_recall: float = 0.0
+ technique_precision: float = 0.0
+ containment_accuracy: float | None = None
+
+ # Honesty.
+ abstention_rate: float = 0.0
+ hallucination_rate: float = 0.0
+ hallucinated_total: int = 0
+ indicators_checked: int = 0
+
+ # Calibration: mean confidence when right minus mean confidence when
+ # wrong. Near zero means the confidence score carries no information.
+ calibration_gap: float = 0.0
+ mean_confidence_correct: float = 0.0
+ mean_confidence_incorrect: float = 0.0
+
+ # Cost and latency, reported not scored.
+ mean_latency_ms: float = 0.0
+ p95_latency_ms: float = 0.0
+ total_tokens: int = 0
+ total_usd: float = 0.0
+ mean_usd_per_incident: float = 0.0
+ mean_distinct_tools: float = 0.0
+
+ # Provenance. A corpus of synthetic incidents produces a synthetic
+ # number, and a scoreboard that does not say so is misleading even when
+ # every figure in it is arithmetically correct.
+ corpus_provenance: dict[str, int] = field(default_factory=dict)
+
+ per_incident: list[IncidentScore] = field(default_factory=list)
+
+ def as_dict(self, *, include_per_incident: bool = False) -> dict[str, Any]:
+ payload = asdict(self)
+ if not include_per_incident:
+ payload.pop("per_incident", None)
+ return payload
+
+
+def score_incident(
+ incident: Any,
+ verdict: Any,
+ expected: dict[str, Any],
+) -> IncidentScore:
+ """Grade one verdict against ground truth."""
+ score = IncidentScore(
+ incident_id=str(getattr(incident, "incident_id", "")),
+ abstained=bool(getattr(verdict, "abstained", False)),
+ confidence=float(getattr(verdict, "confidence", 0.0) or 0.0),
+ latency_ms=int(getattr(verdict, "latency_ms", 0) or 0),
+ tokens=int(getattr(verdict, "tokens", 0) or 0),
+ usd_cost=float(getattr(verdict, "usd_cost", 0.0) or 0.0),
+ distinct_tools=int(getattr(verdict, "distinct_tools", 0) or 0),
+ )
+
+ # Hallucination is graded even on abstention: an agent that declines to
+ # reach a verdict while citing an invented address has still invented it,
+ # and that is the behaviour worth catching.
+ corpus = evidence_corpus(incident)
+ for indicator in getattr(verdict, "cited_indicators", None) or []:
+ text = str(indicator).strip()
+ if not is_checkable_indicator(text):
+ continue
+ score.cited_checkable += 1
+ if text.lower() not in corpus:
+ score.hallucinated_indicators.append(text)
+
+ if score.abstained:
+ return score
+
+ expected_disposition = str(expected.get("disposition", "")).lower()
+ if expected_disposition:
+ score.disposition_correct = str(getattr(verdict, "disposition", "")).lower() == expected_disposition
+
+ expected_techniques = {str(t).upper() for t in (expected.get("techniques") or [])}
+ predicted = {str(t).upper() for t in (getattr(verdict, "techniques", None) or [])}
+ if expected_techniques:
+ hits = len(expected_techniques & predicted)
+ score.technique_recall = hits / len(expected_techniques)
+ score.technique_precision = hits / len(predicted) if predicted else 0.0
+
+ expected_actions = {str(a).lower() for a in (expected.get("actions") or [])}
+ if expected_actions:
+ proposed = {str(a).lower() for a in (getattr(verdict, "proposed_actions", None) or [])}
+ # Any overlap counts. Requiring an exact set would grade an agent on
+ # matching our action vocabulary rather than on containing the
+ # incident, and a vendor's verbs are legitimately their own.
+ score.containment_correct = bool(expected_actions & proposed)
+
+ return score
+
+
+def _percentile(values: list[float], fraction: float) -> float:
+ if not values:
+ return 0.0
+ ordered = sorted(values)
+ index = min(len(ordered) - 1, int(round(fraction * (len(ordered) - 1))))
+ return ordered[index]
+
+
+def aggregate(
+ agent_name: str,
+ agent_version: str,
+ scores: list[IncidentScore],
+ *,
+ corpus_provenance: dict[str, int] | None = None,
+) -> BenchmarkResult:
+ """Roll per-incident scores into the published figures."""
+ result = BenchmarkResult(
+ agent_name=agent_name,
+ agent_version=agent_version,
+ incidents=len(scores),
+ per_incident=scores,
+ corpus_provenance=dict(corpus_provenance or {}),
+ )
+ if not scores:
+ return result
+
+ answered = [s for s in scores if not s.abstained]
+ result.abstention_rate = (len(scores) - len(answered)) / len(scores)
+
+ if answered:
+ result.disposition_accuracy = sum(s.disposition_correct for s in answered) / len(answered)
+ result.technique_recall = sum(s.technique_recall for s in answered) / len(answered)
+ result.technique_precision = sum(s.technique_precision for s in answered) / len(answered)
+
+ containment = [s for s in answered if s.containment_correct is not None]
+ if containment:
+ result.containment_accuracy = sum(bool(s.containment_correct) for s in containment) / len(containment)
+
+ correct = [s.confidence for s in answered if s.disposition_correct]
+ wrong = [s.confidence for s in answered if not s.disposition_correct]
+ result.mean_confidence_correct = sum(correct) / len(correct) if correct else 0.0
+ result.mean_confidence_incorrect = sum(wrong) / len(wrong) if wrong else 0.0
+ result.calibration_gap = result.mean_confidence_correct - result.mean_confidence_incorrect
+
+ # Hallucination spans every incident, answered or not.
+ result.indicators_checked = sum(s.cited_checkable for s in scores)
+ result.hallucinated_total = sum(len(s.hallucinated_indicators) for s in scores)
+ result.hallucination_rate = result.hallucinated_total / result.indicators_checked if result.indicators_checked else 0.0
+
+ latencies = [float(s.latency_ms) for s in scores]
+ result.mean_latency_ms = sum(latencies) / len(latencies)
+ result.p95_latency_ms = _percentile(latencies, 0.95)
+ result.total_tokens = sum(s.tokens for s in scores)
+ result.total_usd = round(sum(s.usd_cost for s in scores), 6)
+ result.mean_usd_per_incident = round(result.total_usd / len(scores), 6)
+ result.mean_distinct_tools = sum(s.distinct_tools for s in scores) / len(scores)
+
+ return result
diff --git a/services/api/app/_vendor/aisoc_benchmark/replay.py b/services/api/app/_vendor/aisoc_benchmark/replay.py
new file mode 100644
index 000000000..2bd97c9f1
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/replay.py
@@ -0,0 +1,654 @@
+"""Scoring for replay evaluation: the agent against a customer's own analysts.
+
+Gap-closure Phase 1.3.
+
+This extends the benchmark package rather than standing beside it. The
+hallucination machinery is :func:`aisoc_benchmark.metrics.is_checkable_indicator`
+and :func:`~aisoc_benchmark.metrics.extract_checkable_indicators`, unchanged, so
+an indicator counted here is one the synthetic-corpus grader would also have
+counted. A second grader would eventually disagree with the first about what
+"hallucinated" means, and the published numbers would stop being comparable
+while still looking like they were.
+
+What replay measures that the synthetic corpus cannot
+------------------------------------------------------
+The synthetic corpus has balanced classes by construction. A real queue does
+not: most closed findings are false positives, and a handful are the ones that
+mattered. Three consequences shape this module.
+
+**Headline accuracy is withheld on a thin corpus.** Below
+:data:`MIN_MALICIOUS_FOR_HEADLINE` true positives the report prints the count
+and refuses the number. Accuracy over 200 findings of which three were
+malicious is 98% for an agent that calls everything benign, and publishing
+that figure would be actively misleading in the direction that sells.
+
+**Recall on malicious leads.** It is the number an operator is actually
+deciding on, and it is the one an imbalanced corpus hides. :class:`ReplayScore`
+puts it first and the renderers keep that order.
+
+**Every mean travels with the count it was computed over.** A precision of
+1.00 across two predictions is not a precision of 1.00 across two hundred, and
+a table that prints only the ratio has thrown away the distinction.
+
+Absent is not zero
+------------------
+A rate with no denominator is ``None`` and renders as "not measured". This is
+not fussiness: ``0.0`` in a precision column reads as "the agent got every one
+of these wrong", and "the agent never predicted this class" is a different
+fact with a different remedy.
+
+Determinism
+-----------
+Bootstrap confidence intervals resample, so they need a source of randomness,
+and a report that has to reproduce byte for byte cannot have an unseeded one.
+:data:`BOOTSTRAP_SEED` fixes it, :class:`ReplayScore` records the seed and the
+resample count it used, and :func:`score_replay` takes both as arguments so a
+caller who wants a different interval can ask for one and have the report say
+so.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from collections import Counter
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+from .metrics import extract_checkable_indicators
+
+__all__ = [
+ "ABSTENTION_VERDICTS",
+ "BOOTSTRAP_RESAMPLES",
+ "BOOTSTRAP_SEED",
+ "CALIBRATION_BINS",
+ "GRADED_DISPOSITIONS",
+ "LATENCY_FIELDS",
+ "LATENCY_LINE_PREFIX",
+ "MALICIOUS",
+ "MIN_MALICIOUS_FOR_HEADLINE",
+ "UNLABELED",
+ "CalibrationBin",
+ "ClassScore",
+ "ReplayScore",
+ "SegmentScore",
+ "format_replay_report",
+ "score_replay",
+ "strip_latency",
+]
+
+#: The one line of the rendered report that measures the host rather than the
+#: agent. Declared here, used by :func:`format_replay_report` to emit it and
+#: by :func:`strip_latency` to remove it, so the producer and the remover
+#: cannot drift into disagreeing about which line that is.
+LATENCY_LINE_PREFIX = "- Mean latency:"
+
+#: The matching fields in :meth:`ReplayScore.as_dict`, for the JSON export.
+LATENCY_FIELDS: tuple[str, ...] = ("mean_latency_ms", "p95_latency_ms")
+
+#: The canonical disposition that means "this was a real threat". Written once
+#: because "malicious recall" has to mean the same thing in the metric, the
+#: renderer and the promotion gate Phase 2 builds on top of it.
+MALICIOUS = "true_positive"
+
+#: The four canonical dispositions an analyst can close a finding as. Mirrors
+#: ``CANONICAL_DISPOSITIONS`` in ``services/actions``; this package does not
+#: import from a service, and ``scripts/check_replay_contract_parity.py``
+#: keeps the two in step in both directions.
+GRADED_DISPOSITIONS: tuple[str, ...] = (
+ MALICIOUS,
+ "benign_true_positive",
+ "false_positive",
+ "benign",
+)
+
+#: An analyst who declined to classify. Excluded from accuracy, never guessed.
+UNLABELED = "unlabeled"
+
+#: Agent outputs that are a refusal to decide rather than a decision. These are
+#: abstentions: scored as neither right nor wrong, and counted separately so
+#: accuracy and abstention cannot be traded off invisibly. ``escalate`` belongs
+#: here for the same reason ``needs_review`` does: both route the alert to a
+#: human, which is a decision about who decides, not about what happened.
+ABSTENTION_VERDICTS: frozenset[str] = frozenset({"needs_review", "escalate", "unknown", ""})
+
+#: Below this many malicious cases in the test window, no headline accuracy is
+#: printed. Set by the plan.
+MIN_MALICIOUS_FOR_HEADLINE = 30
+
+#: Reliability-diagram bin edges. Ten equal-width bins over [0, 1].
+CALIBRATION_BINS = 10
+
+BOOTSTRAP_RESAMPLES = 1000
+BOOTSTRAP_SEED = 20260926
+
+
+@dataclass(frozen=True)
+class ClassScore:
+ """Precision and recall for one disposition, with the counts behind them."""
+
+ label: str
+ #: Ground-truth occurrences. The denominator of recall.
+ support: int
+ #: Times the agent predicted this class. The denominator of precision.
+ predicted: int
+ correct: int
+ precision: float | None
+ recall: float | None
+ f1: float | None
+
+
+@dataclass(frozen=True)
+class CalibrationBin:
+ """One reliability bin: how often the agent was right when this sure."""
+
+ lower: float
+ upper: float
+ count: int
+ mean_confidence: float | None
+ accuracy: float | None
+
+
+@dataclass(frozen=True)
+class SegmentScore:
+ """A per-rule or per-source slice. Small slices are reported, not hidden."""
+
+ key: str
+ graded: int
+ correct: int
+ accuracy: float | None
+ malicious_support: int
+ malicious_recall: float | None
+
+
+@dataclass
+class ReplayScore:
+ """The report. Field order is the reading order the renderers follow."""
+
+ # ---- what was graded -------------------------------------------------
+ decisions: int = 0
+ labelled: int = 0
+ unlabeled: int = 0
+ errored: int = 0
+ graded: int = 0
+ abstained: int = 0
+
+ # ---- the number an operator is deciding on ---------------------------
+ malicious_support: int = 0
+ malicious_recall: float | None = None
+ malicious_recall_ci: tuple[float, float] | None = None
+ malicious_precision: float | None = None
+
+ # ---- headline, withheld on a thin corpus -----------------------------
+ headline_accuracy: float | None = None
+ headline_accuracy_ci: tuple[float, float] | None = None
+ headline_withheld_reason: str | None = None
+
+ # ---- the rest --------------------------------------------------------
+ abstention_rate: float | None = None
+ per_class: list[ClassScore] = field(default_factory=list)
+ confusion: dict[str, dict[str, int]] = field(default_factory=dict)
+ calibration: list[CalibrationBin] = field(default_factory=list)
+ expected_calibration_error: float | None = None
+ hallucination_rate: float | None = None
+ hallucinated_total: int = 0
+ indicators_checked: int = 0
+ hallucinated_examples: list[str] = field(default_factory=list)
+ per_rule: list[SegmentScore] = field(default_factory=list)
+ per_source: list[SegmentScore] = field(default_factory=list)
+
+ # ---- cost and latency, reported not scored ---------------------------
+ total_tokens: int = 0
+ measured_usd: float | None = None
+ estimated_usd: float | None = None
+ unpriced_calls: int = 0
+ mean_latency_ms: float | None = None
+ p95_latency_ms: float | None = None
+ models: list[str] = field(default_factory=list)
+
+ # ---- provenance ------------------------------------------------------
+ bootstrap_resamples: int = BOOTSTRAP_RESAMPLES
+ bootstrap_seed: int = BOOTSTRAP_SEED
+ min_malicious_for_headline: int = MIN_MALICIOUS_FOR_HEADLINE
+
+ def as_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+def _ratio(numerator: int, denominator: int) -> float | None:
+ """A rate, or ``None`` when there was nothing to divide by."""
+ return (numerator / denominator) if denominator else None
+
+
+def _f1(precision: float | None, recall: float | None) -> float | None:
+ if precision is None or recall is None or (precision + recall) == 0:
+ return None
+ return 2 * precision * recall / (precision + recall)
+
+
+def _percentile(values: list[float], fraction: float) -> float | None:
+ if not values:
+ return None
+ ordered = sorted(values)
+ index = min(len(ordered) - 1, int(round(fraction * (len(ordered) - 1))))
+ return ordered[index]
+
+
+def _bootstrap_ci(
+ outcomes: list[int],
+ *,
+ resamples: int,
+ seed: int,
+ confidence: float = 0.95,
+) -> tuple[float, float] | None:
+ """Percentile bootstrap over a list of 0/1 outcomes.
+
+ Returns ``None`` below five observations. An interval over three samples
+ is arithmetic, not evidence, and printing one invites a reader to treat a
+ handful of cases as a measurement.
+ """
+ if len(outcomes) < 5 or resamples <= 0:
+ return None
+ rng = random.Random(seed)
+ n = len(outcomes)
+ means: list[float] = []
+ for _ in range(resamples):
+ total = 0
+ for _ in range(n):
+ total += outcomes[rng.randrange(n)]
+ means.append(total / n)
+ means.sort()
+ tail = (1.0 - confidence) / 2.0
+ low = means[min(len(means) - 1, int(tail * len(means)))]
+ high = means[min(len(means) - 1, int((1.0 - tail) * len(means)))]
+ return (round(low, 6), round(high, 6))
+
+
+def _evidence_text(decision: dict[str, Any]) -> str:
+ """Everything the agent was given, flattened, for the hallucination check."""
+ return repr(decision.get("evidence") or {}).lower()
+
+
+def _reasoning_text(decision: dict[str, Any]) -> str:
+ parts = [*(decision.get("findings") or []), *(decision.get("confidence_basis") or [])]
+ return " ".join(str(p) for p in parts)
+
+
+def _segment(
+ decisions: list[dict[str, Any]],
+ key_name: str,
+) -> list[SegmentScore]:
+ """Group graded decisions by a field and score each group."""
+ groups: dict[str, list[dict[str, Any]]] = {}
+ for decision in decisions:
+ key = str(decision.get(key_name) or "unattributed")
+ groups.setdefault(key, []).append(decision)
+
+ scores: list[SegmentScore] = []
+ for key in sorted(groups):
+ rows = groups[key]
+ answered = [d for d in rows if not _is_abstention(d)]
+ correct = sum(1 for d in answered if _is_correct(d))
+ malicious = [d for d in rows if d.get("expected_disposition") == MALICIOUS]
+ malicious_hit = sum(1 for d in malicious if d.get("verdict") == MALICIOUS)
+ scores.append(
+ SegmentScore(
+ key=key,
+ graded=len(answered),
+ correct=correct,
+ accuracy=_ratio(correct, len(answered)),
+ malicious_support=len(malicious),
+ malicious_recall=_ratio(malicious_hit, len(malicious)),
+ )
+ )
+ return scores
+
+
+def _is_abstention(decision: dict[str, Any]) -> bool:
+ return str(decision.get("verdict") or "") in ABSTENTION_VERDICTS
+
+
+def _is_correct(decision: dict[str, Any]) -> bool:
+ return decision.get("verdict") == decision.get("expected_disposition")
+
+
+def score_replay(
+ decisions: list[dict[str, Any]],
+ *,
+ bootstrap_resamples: int = BOOTSTRAP_RESAMPLES,
+ bootstrap_seed: int = BOOTSTRAP_SEED,
+) -> ReplayScore:
+ """Grade a replay run's decisions against the analysts' own labels.
+
+ ``decisions`` are :meth:`app.replay.runner.ReplayDecision.as_dict`
+ payloads. Plain dicts rather than an imported type, because this package
+ must stay installable without a service on the path: a vendor grading
+ their own agent against their own history should not need ``services/``.
+
+ Three populations, kept apart on purpose:
+
+ * **errored** - triage refused the finding. Neither a verdict nor an
+ abstention. Folding these into abstentions would let a run that crashed
+ on every hard case look appropriately humble.
+ * **unlabeled** - the analyst declined to classify. Excluded from
+ accuracy entirely, and counted so a customer whose history is mostly
+ unlabeled sees that rather than a confident number over a remnant.
+ * **graded** - everything else, split into answered and abstained.
+ """
+ score = ReplayScore(
+ decisions=len(decisions),
+ bootstrap_resamples=bootstrap_resamples,
+ bootstrap_seed=bootstrap_seed,
+ )
+
+ errored = [d for d in decisions if d.get("error")]
+ usable = [d for d in decisions if not d.get("error")]
+ labelled = [d for d in usable if d.get("labelled") and d.get("expected_disposition") in GRADED_DISPOSITIONS]
+ score.errored = len(errored)
+ score.unlabeled = len(usable) - len(labelled)
+ score.labelled = len(labelled)
+
+ # Hallucination spans every decision that produced reasoning, answered or
+ # not, and errored or not: an agent that invented an address before
+ # failing has still invented it.
+ checked = 0
+ hallucinated: list[str] = []
+ for decision in decisions:
+ corpus = _evidence_text(decision)
+ for indicator in extract_checkable_indicators(_reasoning_text(decision)):
+ checked += 1
+ if indicator.lower() not in corpus:
+ hallucinated.append(indicator)
+ score.indicators_checked = checked
+ score.hallucinated_total = len(hallucinated)
+ score.hallucination_rate = _ratio(len(hallucinated), checked)
+ score.hallucinated_examples = sorted(set(hallucinated))[:20]
+
+ # Cost and latency describe the whole run, including the decisions that
+ # are not graded: they were still paid for.
+ score.total_tokens = sum(int(d.get("tokens") or 0) for d in decisions)
+ measured = [float(d["measured_usd"]) for d in decisions if d.get("measured_usd") is not None]
+ estimated = [float(d["estimated_usd"]) for d in decisions if d.get("estimated_usd") is not None]
+ score.measured_usd = round(sum(measured), 6) if measured else None
+ score.estimated_usd = round(sum(estimated), 6) if estimated else None
+ score.unpriced_calls = sum(int(d.get("unpriced_calls") or 0) for d in decisions)
+ latencies = [float(d.get("latency_ms") or 0.0) for d in decisions]
+ score.mean_latency_ms = (sum(latencies) / len(latencies)) if latencies else None
+ score.p95_latency_ms = _percentile(latencies, 0.95)
+ score.models = sorted({str(m) for d in decisions for m in (d.get("resolved_models") or [])})
+
+ if not labelled:
+ score.headline_withheld_reason = (
+ "No finding in the test window carried an analyst disposition this platform can name, so there is nothing to grade against."
+ )
+ return score
+
+ answered = [d for d in labelled if not _is_abstention(d)]
+ score.abstained = len(labelled) - len(answered)
+ score.graded = len(answered)
+ score.abstention_rate = _ratio(score.abstained, len(labelled))
+
+ # ---- confusion matrix, over labelled decisions including abstentions --
+ matrix: dict[str, dict[str, int]] = {
+ expected: dict.fromkeys([*GRADED_DISPOSITIONS, "abstained"], 0) for expected in GRADED_DISPOSITIONS
+ }
+ for decision in labelled:
+ expected = str(decision.get("expected_disposition"))
+ predicted = "abstained" if _is_abstention(decision) else str(decision.get("verdict") or "")
+ if predicted not in matrix[expected]:
+ # A verdict outside the taxonomy. Recorded under its own column
+ # rather than dropped, because a column nobody expected is a
+ # finding about the agent.
+ for row in matrix.values():
+ row.setdefault(predicted, 0)
+ matrix[expected][predicted] += 1
+ score.confusion = matrix
+
+ # ---- per class, precision over answered only -------------------------
+ support = Counter(str(d.get("expected_disposition")) for d in labelled)
+ predicted_counts = Counter(str(d.get("verdict") or "") for d in answered)
+ for label in GRADED_DISPOSITIONS:
+ correct = sum(1 for d in answered if d.get("verdict") == label and _is_correct(d))
+ precision = _ratio(correct, predicted_counts.get(label, 0))
+ recall = _ratio(correct, support.get(label, 0))
+ score.per_class.append(
+ ClassScore(
+ label=label,
+ support=support.get(label, 0),
+ predicted=predicted_counts.get(label, 0),
+ correct=correct,
+ precision=precision,
+ recall=recall,
+ f1=_f1(precision, recall),
+ )
+ )
+
+ malicious_rows = [d for d in labelled if d.get("expected_disposition") == MALICIOUS]
+ score.malicious_support = len(malicious_rows)
+ malicious_class = next(c for c in score.per_class if c.label == MALICIOUS)
+ score.malicious_recall = malicious_class.recall
+ score.malicious_precision = malicious_class.precision
+ # Recall's outcomes are per malicious case: did the agent call it
+ # malicious? An abstention counts against recall, because an alert routed
+ # to a human was not caught by the agent.
+ score.malicious_recall_ci = _bootstrap_ci(
+ [1 if d.get("verdict") == MALICIOUS else 0 for d in malicious_rows],
+ resamples=bootstrap_resamples,
+ seed=bootstrap_seed,
+ )
+
+ # ---- calibration ------------------------------------------------------
+ score.calibration, score.expected_calibration_error = _calibration(answered)
+
+ # ---- headline, withheld below the floor -------------------------------
+ if score.malicious_support < MIN_MALICIOUS_FOR_HEADLINE:
+ score.headline_withheld_reason = (
+ f"The test window holds {score.malicious_support} malicious case(s), below the "
+ f"{MIN_MALICIOUS_FOR_HEADLINE} this report requires before printing a headline accuracy. "
+ f"On a queue where almost everything is a false positive, an agent that calls everything "
+ f"benign scores well, so the figure would describe the queue rather than the agent. "
+ f"Per-class recall and the confusion matrix below are reported in full."
+ )
+ elif answered:
+ correct = sum(1 for d in answered if _is_correct(d))
+ score.headline_accuracy = correct / len(answered)
+ score.headline_accuracy_ci = _bootstrap_ci(
+ [1 if _is_correct(d) else 0 for d in answered],
+ resamples=bootstrap_resamples,
+ seed=bootstrap_seed,
+ )
+
+ score.per_rule = _segment(labelled, "rule_id")
+ score.per_source = _segment(labelled, "vendor")
+ return score
+
+
+def _pct(value: float | None) -> str:
+ """A rate as a percentage, or the words that mean there was no denominator.
+
+ "not measured" rather than ``0.0`` everywhere. A zero in a recall column
+ says the agent missed every malicious case; no denominator says it was
+ never asked. Printing the first when the second is true is the single most
+ misleading thing a report of this kind can do.
+ """
+ return "not measured" if value is None else f"{value * 100:.1f}%"
+
+
+def _interval(bounds: tuple[float, float] | None) -> str:
+ return "not measured" if bounds is None else f"{bounds[0] * 100:.1f}% to {bounds[1] * 100:.1f}%"
+
+
+def format_replay_report(score: ReplayScore, *, method: dict[str, Any] | None = None) -> str:
+ """Render the report as Markdown, malicious recall first.
+
+ Deterministic by construction: no timestamps, no host names, no dict
+ iteration that is not sorted. A report that reproduces byte for byte is
+ the phase's acceptance test, and the easiest way to fail it is a renderer
+ that prints "generated at".
+ """
+ lines: list[str] = ["# Replay evaluation", ""]
+
+ lines += [
+ "## What was graded",
+ "",
+ f"- Decisions replayed: {score.decisions}",
+ f"- Carried an analyst label: {score.labelled}",
+ f"- Unlabelled, excluded from accuracy: {score.unlabeled}",
+ f"- Refused by triage, scored as neither: {score.errored}",
+ f"- Answered: {score.graded}. Abstained: {score.abstained} ({_pct(score.abstention_rate)})",
+ "",
+ "## Recall on malicious",
+ "",
+ f"- Malicious cases in the test window: {score.malicious_support}",
+ f"- Recall: {_pct(score.malicious_recall)} (95% CI {_interval(score.malicious_recall_ci)})",
+ f"- Precision: {_pct(score.malicious_precision)}",
+ "",
+ "## Headline accuracy",
+ "",
+ ]
+ if score.headline_accuracy is None:
+ lines += [f"Withheld. {score.headline_withheld_reason}", ""]
+ else:
+ lines += [
+ f"{_pct(score.headline_accuracy)} over {score.graded} answered decisions (95% CI {_interval(score.headline_accuracy_ci)}).",
+ "",
+ ]
+
+ lines += ["## Per class", "", "| Disposition | Support | Predicted | Precision | Recall | F1 |", "|---|---|---|---|---|---|"]
+ for row in score.per_class:
+ lines.append(f"| {row.label} | {row.support} | {row.predicted} | {_pct(row.precision)} | {_pct(row.recall)} | {_pct(row.f1)} |")
+
+ lines += ["", "## Confusion matrix", "", "Rows are the analyst's label, columns the agent's verdict.", ""]
+ columns = sorted({column for row in score.confusion.values() for column in row})
+ lines.append("| analyst \\ agent | " + " | ".join(columns) + " |")
+ lines.append("|---" * (len(columns) + 1) + "|")
+ for expected in sorted(score.confusion):
+ cells = " | ".join(str(score.confusion[expected].get(column, 0)) for column in columns)
+ lines.append(f"| {expected} | {cells} |")
+
+ lines += [
+ "",
+ "## Calibration",
+ "",
+ f"Expected calibration error: {_pct(score.expected_calibration_error)}",
+ "",
+ "| Confidence bin | Decisions | Mean confidence | Accuracy |",
+ "|---|---|---|---|",
+ ]
+ for cbin in score.calibration:
+ lines.append(f"| {cbin.lower:.1f} to {cbin.upper:.1f} | {cbin.count} | {_pct(cbin.mean_confidence)} | {_pct(cbin.accuracy)} |")
+
+ lines += [
+ "",
+ "## Hallucination",
+ "",
+ f"- Checkable indicators cited: {score.indicators_checked}",
+ f"- Absent from the evidence: {score.hallucinated_total} ({_pct(score.hallucination_rate)})",
+ ]
+ if score.hallucinated_examples:
+ lines.append(f"- Examples: {', '.join(score.hallucinated_examples[:10])}")
+
+ breakdowns: tuple[tuple[str, list[SegmentScore]], ...] = (
+ ("Per rule", score.per_rule),
+ ("Per source", score.per_source),
+ )
+ for title, segments in breakdowns:
+ lines += ["", f"## {title}", "", "| Key | Answered | Accuracy | Malicious | Malicious recall |", "|---|---|---|---|---|"]
+ for segment in segments:
+ lines.append(
+ f"| {segment.key} | {segment.graded} | {_pct(segment.accuracy)} | "
+ f"{segment.malicious_support} | {_pct(segment.malicious_recall)} |"
+ )
+
+ measured = "not measured" if score.measured_usd is None else f"${score.measured_usd:.6f}"
+ estimated = "not measured" if score.estimated_usd is None else f"${score.estimated_usd:.6f}"
+ lines += [
+ "",
+ "## Cost and latency",
+ "",
+ f"- Tokens: {score.total_tokens}",
+ f"- Measured spend: {measured} (list-price estimate for unbilled calls: {estimated}; "
+ f"calls with no price at all: {score.unpriced_calls})",
+ f"{LATENCY_LINE_PREFIX} {'not measured' if score.mean_latency_ms is None else f'{score.mean_latency_ms:.0f} ms'}, "
+ f"p95 {'not measured' if score.p95_latency_ms is None else f'{score.p95_latency_ms:.0f} ms'}",
+ f"- Models: {', '.join(score.models) if score.models else 'none, no model call was placed'}",
+ "",
+ "## Method",
+ "",
+ f"- Confidence intervals: percentile bootstrap, {score.bootstrap_resamples} resamples, seed {score.bootstrap_seed}.",
+ f"- Headline accuracy requires at least {score.min_malicious_for_headline} malicious cases.",
+ ]
+ if method:
+ for key in sorted(method):
+ lines.append(f"- {key}: {json.dumps(method[key], sort_keys=True)}")
+ lines.append("")
+ return "\n".join(lines)
+
+
+def strip_latency(report: str) -> str:
+ """Return the report without the two wall-clock latency figures.
+
+ Everything else in this report is a property of the input and the code,
+ so two runs over one pinned window reproduce it byte for byte. Mean and
+ p95 latency are not: they measure the machine the replay ran on, and they
+ will differ between two runs on the same host, let alone two hosts.
+
+ So the reproducibility claim is stated precisely rather than loosely. The
+ report reproduces byte for byte **apart from these two figures**, and this
+ function is what produces the artefact that claim is about. It removes the
+ line :data:`LATENCY_LINE_PREFIX` names rather than matching on the word
+ "latency" anywhere, because a future section that discusses latency in
+ prose must not be silently deleted from an operator's export.
+
+ The line is replaced rather than dropped, so the surrounding structure and
+ the line count are unchanged and a diff of two reports points at content
+ rather than at an offset.
+ """
+ return "\n".join(
+ f"{LATENCY_LINE_PREFIX} excluded from this export (wall clock, not reproducible)" if line.startswith(LATENCY_LINE_PREFIX) else line
+ for line in report.split("\n")
+ )
+
+
+def _calibration(answered: list[dict[str, Any]]) -> tuple[list[CalibrationBin], float | None]:
+ """Reliability bins and expected calibration error.
+
+ ECE is the support-weighted mean gap between confidence and accuracy
+ across the bins. Empty bins contribute nothing and are still emitted, so
+ the diagram shows where an agent never expressed a confidence rather than
+ silently closing the gap.
+ """
+ if not answered:
+ return [], None
+
+ width = 1.0 / CALIBRATION_BINS
+ bins: list[CalibrationBin] = []
+ weighted_gap = 0.0
+ for index in range(CALIBRATION_BINS):
+ lower = index * width
+ upper = lower + width
+ # The last bin is closed at 1.0 so a confidence of exactly 1.0 lands
+ # somewhere rather than being dropped.
+ members = [
+ d
+ for d in answered
+ if lower <= float(d.get("confidence") or 0.0) < upper
+ or (index == CALIBRATION_BINS - 1 and float(d.get("confidence") or 0.0) == 1.0)
+ ]
+ if members:
+ mean_conf = sum(float(d.get("confidence") or 0.0) for d in members) / len(members)
+ accuracy = sum(1 for d in members if _is_correct(d)) / len(members)
+ weighted_gap += len(members) * abs(mean_conf - accuracy)
+ else:
+ mean_conf = None
+ accuracy = None
+ bins.append(
+ CalibrationBin(
+ lower=round(lower, 4),
+ upper=round(upper, 4),
+ count=len(members),
+ mean_confidence=(round(mean_conf, 6) if mean_conf is not None else None),
+ accuracy=(round(accuracy, 6) if accuracy is not None else None),
+ )
+ )
+ return bins, round(weighted_gap / len(answered), 6)
diff --git a/services/api/app/_vendor/aisoc_benchmark/runner.py b/services/api/app/_vendor/aisoc_benchmark/runner.py
new file mode 100644
index 000000000..b82c7ac8a
--- /dev/null
+++ b/services/api/app/_vendor/aisoc_benchmark/runner.py
@@ -0,0 +1,199 @@
+"""Load a corpus, run an agent over it, publish the numbers.
+
+The corpus carries **provenance per incident**, not as a footnote. A figure
+computed over synthetic incidents is a synthetic figure, and a scoreboard
+that omits that is misleading even when every number in it is arithmetically
+correct. ``BenchmarkResult.corpus_provenance`` travels with the result so a
+downstream consumer cannot drop it, and the runner refuses to emit an
+unlabelled corpus.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from .adapter import AgentVerdict, BenchmarkIncident
+from .metrics import BenchmarkResult, aggregate, score_incident
+
+logger = logging.getLogger("aisoc.benchmark")
+
+#: Every incident must declare where it came from. "synthetic" is an honest
+#: answer; absent is not, because the reader then assumes the generous one.
+VALID_PROVENANCE = frozenset(
+ {
+ "synthetic", # generated for this harness
+ "public-dataset", # CICIDS, AIT-LDS, MITRE Engenuity and similar
+ "real-anonymised", # a real incident, scrubbed
+ "adversary-emulation", # Caldera / Atomic Red Team output
+ }
+)
+
+#: One agent must not be able to hang the whole run.
+PER_INCIDENT_TIMEOUT_SECONDS = 300.0
+
+
+class CorpusError(ValueError):
+ pass
+
+
+def load_corpus(path: Path) -> tuple[list[BenchmarkIncident], list[dict[str, Any]], dict[str, int]]:
+ """Read a corpus file. Returns incidents, ground truth, and provenance counts."""
+ try:
+ raw = json.loads(Path(path).read_text(encoding="utf-8"))
+ except Exception as exc:
+ raise CorpusError(f"could not read corpus {path}: {exc}") from exc
+
+ records = raw.get("incidents") if isinstance(raw, dict) else raw
+ if not isinstance(records, list) or not records:
+ raise CorpusError(f"corpus {path} contains no incidents")
+
+ incidents: list[BenchmarkIncident] = []
+ truth: list[dict[str, Any]] = []
+ provenance: Counter[str] = Counter()
+
+ for index, record in enumerate(records):
+ if not isinstance(record, dict):
+ raise CorpusError(f"corpus entry {index} is not an object")
+
+ source = str(record.get("provenance", "")).strip().lower()
+ if source not in VALID_PROVENANCE:
+ raise CorpusError(
+ f"corpus entry {record.get('id', index)!r} declares provenance "
+ f"{source!r}; must be one of {sorted(VALID_PROVENANCE)}. A figure "
+ f"computed over synthetic incidents is a synthetic figure, and an "
+ f"unlabelled corpus lets a reader assume otherwise."
+ )
+ provenance[source] += 1
+
+ incidents.append(
+ BenchmarkIncident(
+ incident_id=str(record.get("id", f"incident-{index}")),
+ title=str(record.get("title", "")),
+ description=str(record.get("description", "")),
+ severity=str(record.get("severity", "medium")),
+ raw_alert=record.get("raw_alert") or {},
+ telemetry=record.get("telemetry") or [],
+ )
+ )
+ truth.append(
+ {
+ "disposition": record.get("expected_disposition") or record.get("disposition") or "",
+ "techniques": record.get("expected_techniques") or record.get("techniques") or [],
+ "actions": record.get("expected_actions") or [],
+ }
+ )
+
+ return incidents, truth, dict(provenance)
+
+
+async def run_benchmark(
+ agent: Any,
+ corpus_path: Path,
+ *,
+ concurrency: int = 4,
+ limit: int | None = None,
+) -> BenchmarkResult:
+ """Grade ``agent`` over the corpus.
+
+ An exception from the agent is scored as an abstention with the failure
+ named, never propagated. A harness that dies on one bad response grades
+ nothing, and the entrant will reasonably assume the fault is ours.
+ """
+ incidents, truth, provenance = load_corpus(corpus_path)
+ if limit:
+ incidents, truth = incidents[:limit], truth[:limit]
+ # Recount over the slice. Reporting the full corpus provenance
+ # alongside a truncated run overstates what was actually graded,
+ # which is the same class of error the labelling exists to prevent.
+ kept = {i.incident_id for i in incidents}
+ provenance = dict(
+ Counter(r["provenance"] for r in json.loads(Path(corpus_path).read_text(encoding="utf-8"))["incidents"] if r.get("id") in kept)
+ )
+
+ semaphore = asyncio.Semaphore(max(1, concurrency))
+
+ async def _one(incident: BenchmarkIncident, expected: dict[str, Any]):
+ async with semaphore:
+ started = time.monotonic()
+ try:
+ verdict = await asyncio.wait_for(agent.investigate(incident), timeout=PER_INCIDENT_TIMEOUT_SECONDS)
+ except TimeoutError:
+ verdict = AgentVerdict(
+ abstained=True,
+ narrative=f"agent exceeded {PER_INCIDENT_TIMEOUT_SECONDS:.0f}s",
+ )
+ except Exception as exc:
+ verdict = AgentVerdict(
+ abstained=True,
+ narrative=f"agent raised {type(exc).__name__}: {exc}",
+ )
+ # Measured here rather than trusted from the agent: self-reported
+ # latency is the one number an entrant has an incentive to shade.
+ if not verdict.latency_ms:
+ verdict.latency_ms = int((time.monotonic() - started) * 1000)
+ return score_incident(incident, verdict, expected)
+
+ scores = await asyncio.gather(*(_one(i, t) for i, t in zip(incidents, truth, strict=True)))
+
+ result = aggregate(
+ getattr(agent, "name", "unknown"),
+ getattr(agent, "version", "unknown"),
+ list(scores),
+ corpus_provenance=provenance,
+ )
+ logger.info(
+ "benchmark complete agent=%s incidents=%d accuracy=%.3f hallucination=%.3f abstention=%.3f",
+ result.agent_name,
+ result.incidents,
+ result.disposition_accuracy,
+ result.hallucination_rate,
+ result.abstention_rate,
+ )
+ return result
+
+
+def format_report(result: BenchmarkResult) -> str:
+ """Human-readable summary.
+
+ Provenance leads. The temptation is to open with the accuracy figure, and
+ a reader who sees 0.94 before they see "synthetic: 200" has already
+ formed an impression the caveat will not undo.
+ """
+ provenance = ", ".join(f"{k}: {v}" for k, v in sorted(result.corpus_provenance.items()))
+ lines = [
+ f"Agent: {result.agent_name} {result.agent_version}",
+ f"Corpus: {result.incidents} incidents ({provenance or 'unlabelled'})",
+ "",
+ "Accuracy (over answered incidents only — abstaining does not raise it)",
+ f" disposition accuracy {result.disposition_accuracy:.3f}",
+ f" technique recall {result.technique_recall:.3f}",
+ f" technique precision {result.technique_precision:.3f}",
+ ]
+ if result.containment_accuracy is not None:
+ lines.append(f" containment accuracy {result.containment_accuracy:.3f}")
+
+ lines += [
+ "",
+ "Honesty",
+ f" abstention rate {result.abstention_rate:.3f}",
+ f" hallucination rate {result.hallucination_rate:.3f} "
+ f"({result.hallucinated_total}/{result.indicators_checked} cited indicators "
+ f"absent from the evidence)",
+ f" calibration gap {result.calibration_gap:+.3f} "
+ f"(confidence when right minus when wrong; near zero means the score "
+ f"carries no information)",
+ "",
+ "Cost",
+ f" mean latency {result.mean_latency_ms:.0f} ms",
+ f" p95 latency {result.p95_latency_ms:.0f} ms",
+ f" total tokens {result.total_tokens}",
+ f" mean USD per incident {result.mean_usd_per_incident:.6f}",
+ f" mean distinct tools {result.mean_distinct_tools:.1f}",
+ ]
+ return "\n".join(lines)
diff --git a/services/api/app/api/v1/endpoints/evaluations.py b/services/api/app/api/v1/endpoints/evaluations.py
new file mode 100644
index 000000000..20f94460e
--- /dev/null
+++ b/services/api/app/api/v1/endpoints/evaluations.py
@@ -0,0 +1,432 @@
+"""Replay evaluation: start a run, poll it, read the report, export it.
+
+Gap-closure Phase 1.4.
+
+Not to be confused with ``endpoints/replay.py``
+----------------------------------------------
+That module publishes a redacted investigation ledger to a public share link
+at ``/r/{slug}``. It is a different noun that happens to share a word, and it
+is deliberately not extended here: a reader looking for share-link publishing
+should not be handed replay evaluation, and the two have opposite privacy
+postures. This one is tenant-scoped and authenticated throughout.
+
+The tenant never appears in a request
+--------------------------------------
+Every route takes ``user.tenant_id`` from the authenticated principal. There
+is no tenant field on any payload and no tenant path parameter. The connector
+is named by its row id, and that row is loaded with a tenant predicate, so
+naming another tenant's connector is a 404 rather than a read.
+
+Permissions
+-----------
+Starting a run needs ``connectors:write``, the same bar as testing a
+connector, because it does the same kind of thing: an outbound call to a
+customer's own SIEM using stored credentials. Reading a report needs
+``reports:read``, which every analyst role holds, because the report is the
+artefact the evaluation exists to produce.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any, Literal
+
+import structlog
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Query, status
+from fastapi.responses import PlainTextResponse, Response
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app._vendor.aisoc_benchmark.replay import MIN_MALICIOUS_FOR_HEADLINE
+from app.api.v1.deps import AuthUser, DBSession
+from app.db.database import AsyncSessionLocal
+from app.models.connector import Connector
+from app.services.replay_evaluation import report as report_export
+from app.services.replay_evaluation import store
+from app.services.replay_evaluation.job import (
+ BOOTSTRAP_RESAMPLES,
+ BOOTSTRAP_SEED,
+ ReplayRequest,
+ run_evaluation,
+)
+from app.services.replay_evaluation.vendors import (
+ REPLAYABLE_CONNECTORS,
+ UnsupportedConnector,
+ vendor_for,
+)
+
+logger = structlog.get_logger(__name__)
+
+router = APIRouter(prefix="/evaluations", tags=["evaluations"])
+
+#: Default window when the caller names neither end. Ninety days is long
+#: enough for most queues to hold the 30 malicious cases the headline needs
+#: and short enough that the estate being graded still resembles today's.
+DEFAULT_WINDOW_DAYS = 90
+
+#: Matches the agents service's own ceiling on one replay.
+MAX_FINDINGS = 2000
+
+
+class StartReplayRequest(BaseModel):
+ """Which connector, which window, and how the report should be computed."""
+
+ #: The saved connector row. Its type decides which reader runs and its
+ #: vault-encrypted credentials are what reaches the vendor. A connector id
+ #: belonging to another tenant resolves to nothing, because the row is
+ #: loaded with a tenant predicate.
+ connector_id: uuid.UUID
+ since: datetime | None = None
+ until: datetime | None = None
+ #: The time split. Findings closed at or before the split point are the
+ #: train window and fix the frozen context; only the later period is
+ #: replayed and graded.
+ train_fraction: float = Field(default=0.7, gt=0.0, lt=1.0)
+ limit: int = Field(default=1000, ge=1, le=MAX_FINDINGS)
+ #: Both travel onto the row and into the report. A caller who wants a
+ #: different interval can ask for one and the report will say so.
+ bootstrap_seed: int = BOOTSTRAP_SEED
+ bootstrap_resamples: int = Field(default=BOOTSTRAP_RESAMPLES, ge=0, le=20000)
+
+
+class EvaluationSummary(BaseModel):
+ """One run as the list and the poll return it."""
+
+ id: uuid.UUID
+ status: str
+ error: str | None = None
+ connector_id: str
+ vendor: str
+ window_start: datetime
+ window_end: datetime
+ train_fraction: float
+ bootstrap_seed: int
+ bootstrap_resamples: int
+
+ # Sample sizes, beside the headline rather than buried in the score, so a
+ # reader sees how much was graded at the same moment they see the number.
+ findings_read: int
+ findings_labelled: int
+ decisions_recorded: int
+ graded: int
+ malicious_support: int
+
+ #: ``None`` when the corpus was too thin. Never 0, which would read as
+ #: "the agent got every answer wrong". ``headline_withheld_reason``
+ #: carries the sentence explaining which it is.
+ headline_accuracy: float | None = None
+ headline_withheld_reason: str | None = None
+ malicious_recall: float | None = None
+
+ created_at: datetime
+ started_at: datetime | None = None
+ completed_at: datetime | None = None
+ is_terminal: bool
+
+
+class EvaluationDetail(EvaluationSummary):
+ """A completed run with its report and the provenance behind it."""
+
+ score: dict[str, Any] | None = None
+ method: dict[str, Any] | None = None
+ report_markdown: str | None = None
+
+
+class ReplayableConnectorInfo(BaseModel):
+ connector_type: str
+ vendor: str
+ label: str
+
+
+class CapabilitiesResponse(BaseModel):
+ """What this deployment can replay, so the console offers nothing it cannot.
+
+ ``min_malicious_for_headline`` travels to the client so the page can say
+ why a headline is withheld before a run is started, rather than only
+ after.
+ """
+
+ connectors: list[ReplayableConnectorInfo]
+ default_window_days: int
+ default_train_fraction: float
+ default_bootstrap_seed: int
+ default_bootstrap_resamples: int
+ min_malicious_for_headline: int
+ max_findings: int
+
+
+def _summary(row: store.EvaluationRow) -> EvaluationSummary:
+ return EvaluationSummary(
+ id=row.id,
+ status=row.status,
+ error=row.error,
+ connector_id=row.connector_id,
+ vendor=row.vendor,
+ window_start=row.window_start,
+ window_end=row.window_end,
+ train_fraction=row.train_fraction,
+ bootstrap_seed=row.bootstrap_seed,
+ bootstrap_resamples=row.bootstrap_resamples,
+ findings_read=row.findings_read,
+ findings_labelled=row.findings_labelled,
+ decisions_recorded=row.decisions_recorded,
+ graded=row.graded,
+ malicious_support=row.malicious_support,
+ headline_accuracy=row.headline_accuracy,
+ headline_withheld_reason=row.headline_withheld_reason,
+ malicious_recall=row.malicious_recall,
+ created_at=row.created_at,
+ started_at=row.started_at,
+ completed_at=row.completed_at,
+ is_terminal=row.is_terminal,
+ )
+
+
+def _detail(row: store.EvaluationRow) -> EvaluationDetail:
+ return EvaluationDetail(
+ **_summary(row).model_dump(),
+ score=row.score,
+ method=row.method,
+ report_markdown=row.report_markdown,
+ )
+
+
+async def _run_detached(request: ReplayRequest) -> None:
+ """Run the job on its own session.
+
+ The request's session is closed when the response is returned, so the
+ background task opens its own rather than using one that is about to go
+ away. ``run_evaluation`` never raises: every failure lands on the row.
+ """
+ async with AsyncSessionLocal() as session:
+ await run_evaluation(session, request)
+
+
+ "/replay/capabilities",
+ response_model=CapabilitiesResponse,
+ summary="Which connectors can be replayed on this deployment",
+)
+async def capabilities(user: AuthUser) -> CapabilitiesResponse:
+ user.require_permission("reports:read")
+ # MIN_MALICIOUS_FOR_HEADLINE comes from the vendored scorer, so the floor
+ # the console shows and the floor the renderer applies are the same
+ # number rather than two that happen to agree today.
+ return CapabilitiesResponse(
+ connectors=[
+ ReplayableConnectorInfo(connector_type=entry.connector_type, vendor=entry.vendor, label=entry.label)
+ for _, entry in sorted(REPLAYABLE_CONNECTORS.items())
+ ],
+ default_window_days=DEFAULT_WINDOW_DAYS,
+ default_train_fraction=0.7,
+ default_bootstrap_seed=BOOTSTRAP_SEED,
+ default_bootstrap_resamples=BOOTSTRAP_RESAMPLES,
+ min_malicious_for_headline=MIN_MALICIOUS_FOR_HEADLINE,
+ max_findings=MAX_FINDINGS,
+ )
+
+
+ "/replay",
+ response_model=EvaluationSummary,
+ status_code=status.HTTP_202_ACCEPTED,
+ summary="Start a replay evaluation against this tenant's own history",
+)
+async def start_replay(
+ body: StartReplayRequest,
+ background: BackgroundTasks,
+ db: DBSession,
+ user: AuthUser,
+) -> EvaluationSummary:
+ """Queue a run and return its id immediately.
+
+ 202 rather than 200: replaying a few hundred findings through a model is
+ not a request-response operation, and a route that blocked on it would
+ time out at whichever proxy is in front of it while the work continued.
+ """
+ user.require_permission("connectors:write")
+
+ until = body.until or datetime.now(UTC)
+ since = body.since or (until - timedelta(days=DEFAULT_WINDOW_DAYS))
+ if until <= since:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=f"until ({until.isoformat()}) must be after since ({since.isoformat()})",
+ )
+
+ connector = await _load_connector_row(db, tenant_id=user.tenant_id, connector_id=body.connector_id)
+ try:
+ vendor = vendor_for(connector.connector_type)
+ except UnsupportedConnector as exc:
+ raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
+
+ evaluation_id = await store.create_evaluation(
+ db,
+ tenant_id=user.tenant_id,
+ connector_id=str(body.connector_id),
+ vendor=vendor,
+ window_start=since,
+ window_end=until,
+ train_fraction=body.train_fraction,
+ bootstrap_seed=body.bootstrap_seed,
+ bootstrap_resamples=body.bootstrap_resamples,
+ requested_by=user.user_id,
+ )
+
+ background.add_task(
+ _run_detached,
+ ReplayRequest(
+ tenant_id=user.tenant_id,
+ evaluation_id=evaluation_id,
+ connector_row_id=body.connector_id,
+ connector_type=connector.connector_type,
+ vendor=vendor,
+ window_start=since,
+ window_end=until,
+ train_fraction=body.train_fraction,
+ limit=body.limit,
+ bootstrap_seed=body.bootstrap_seed,
+ bootstrap_resamples=body.bootstrap_resamples,
+ ),
+ )
+
+ row = await store.get_evaluation(db, tenant_id=user.tenant_id, evaluation_id=evaluation_id, detailed=False)
+ if row is None: # pragma: no cover - the insert committed one line above
+ raise HTTPException(status_code=500, detail="the evaluation row disappeared immediately after insert")
+ logger.info(
+ "replay.evaluation.queued",
+ evaluation_id=str(evaluation_id),
+ vendor=vendor,
+ tenant_id=str(user.tenant_id),
+ )
+ return _summary(row)
+
+
[email protected]("/replay", response_model=list[EvaluationSummary], summary="This tenant's replay evaluations")
+async def list_replays(
+ db: DBSession,
+ user: AuthUser,
+ limit: int = Query(default=50, ge=1, le=200),
+) -> list[EvaluationSummary]:
+ user.require_permission("reports:read")
+ rows = await store.list_evaluations(db, tenant_id=user.tenant_id, limit=limit)
+ return [_summary(row) for row in rows]
+
+
[email protected]("/replay/{evaluation_id}", response_model=EvaluationDetail, summary="One replay evaluation")
+async def get_replay(evaluation_id: uuid.UUID, db: DBSession, user: AuthUser) -> EvaluationDetail:
+ user.require_permission("reports:read")
+ return _detail(await _require_evaluation(db, tenant_id=user.tenant_id, evaluation_id=evaluation_id))
+
+
+ "/replay/{evaluation_id}/decisions",
+ response_model=list[dict],
+ summary="The decisions behind a report",
+)
+async def get_replay_decisions(
+ evaluation_id: uuid.UUID,
+ db: DBSession,
+ user: AuthUser,
+ limit: int = Query(default=2000, ge=1, le=MAX_FINDINGS),
+) -> list[dict[str, Any]]:
+ """Every replayed finding, so a disputed number can be re-derived.
+
+ Read separately from the report because these rows carry the evidence
+ handed to triage and are measured in megabytes, and the page that shows
+ the headline should not pay for them.
+ """
+ user.require_permission("reports:read")
+ await _require_evaluation(db, tenant_id=user.tenant_id, evaluation_id=evaluation_id)
+ return await store.list_decisions(db, tenant_id=user.tenant_id, evaluation_id=evaluation_id, limit=limit)
+
+
[email protected]("/replay/{evaluation_id}/export", summary="Export a report as JSON, Markdown or PDF")
+async def export_replay(
+ evaluation_id: uuid.UUID,
+ db: DBSession,
+ user: AuthUser,
+ format: Literal["json", "markdown", "pdf"] = Query(default="markdown"),
+ exclude_latency: bool = Query(
+ default=False,
+ description=(
+ "Replace the two wall-clock latency figures with a note. Everything else in the report "
+ "is a property of the input and the code, so two runs over one pinned window are then "
+ "byte-identical; latency measures the host and never will be."
+ ),
+ ),
+) -> Response:
+ """Serve the stored report, not a fresh render of it.
+
+ All three formats come from the artefact written when the run completed.
+ Re-rendering later would produce a document that no longer matches what
+ the operator read, and this phase's acceptance bar is a report that
+ reproduces byte for byte.
+ """
+ user.require_permission("reports:read")
+ row = await _require_evaluation(db, tenant_id=user.tenant_id, evaluation_id=evaluation_id)
+
+ if row.status != "completed" or not row.report_markdown:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=(f"this evaluation is '{row.status}' and has no report to export" + (f": {row.error}" if row.error else "")),
+ )
+
+ stem = f"aisoc-replay-{evaluation_id}"
+ if format == "json":
+ return Response(
+ content=report_export.report_json(
+ evaluation_id=str(evaluation_id),
+ score=row.score,
+ method=row.method,
+ exclude_latency=exclude_latency,
+ ),
+ media_type="application/json",
+ headers={"Content-Disposition": f'attachment; filename="{stem}.json"'},
+ )
+
+ markdown = report_export.report_markdown(row.report_markdown, exclude_latency=exclude_latency)
+ if format == "markdown":
+ return PlainTextResponse(
+ content=markdown,
+ media_type="text/markdown; charset=utf-8",
+ headers={"Content-Disposition": f'attachment; filename="{stem}.md"'},
+ )
+
+ try:
+ pdf = report_export.render_pdf(markdown)
+ except report_export.PdfUnavailableError as exc:
+ # 503 rather than 500: the report exists and the other two formats
+ # work, so this is a missing capability on this deployment rather
+ # than a failure of the evaluation.
+ raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc)) from exc
+ return Response(
+ content=pdf,
+ media_type="application/pdf",
+ headers={"Content-Disposition": f'attachment; filename="{stem}.pdf"'},
+ )
+
+
+async def _require_evaluation(db: AsyncSession, *, tenant_id: uuid.UUID, evaluation_id: uuid.UUID) -> store.EvaluationRow:
+ row = await store.get_evaluation(db, tenant_id=tenant_id, evaluation_id=evaluation_id)
+ if row is None:
+ # 404 rather than 403 for a row in another tenant. The query carries
+ # the tenant predicate, so "not yours" and "not there" are the same
+ # answer here and telling them apart would confirm the id exists.
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="evaluation not found")
+ return row
+
+
+async def _load_connector_row(db: AsyncSession, *, tenant_id: uuid.UUID, connector_id: uuid.UUID) -> Connector:
+ """The tenant's own connector row, or a 404.
+
+ The tenant predicate is in the WHERE clause rather than checked after the
+ fetch, so another tenant's connector id resolves to no row rather than to
+ a row this code then decides to refuse.
+ """
+ row = (await db.execute(select(Connector).where(Connector.id == connector_id, Connector.tenant_id == tenant_id))).scalar_one_or_none()
+ if row is None:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="connector not found")
+ return row
diff --git a/services/api/app/api/v1/router.py b/services/api/app/api/v1/router.py
index abc1b2d39..6b4936e08 100644
--- a/services/api/app/api/v1/router.py
+++ b/services/api/app/api/v1/router.py
@@ -30,6 +30,7 @@ from app.api.v1.endpoints import (
easm,
effective_permissions,
email_approval,
+ evaluations,
federated,
feedback,
fusion,
@@ -246,6 +247,11 @@ api_router.include_router(phishing.router)
# provider, and air-gapped mode permits local providers only.
api_router.include_router(sandbox.router)
+# Gap-closure Phase 1.4 — replay evaluation: triage measured against this
+# tenant's own analysts on their own closed findings. Distinct from
+# `replay.router` above, which publishes a redacted ledger to a share link.
+api_router.include_router(evaluations.router)
+
# Knowledge-base + RAG over org docs/runbooks (Tier 3)
api_router.include_router(knowledge_base.router)
diff --git a/services/api/app/services/replay_evaluation/__init__.py b/services/api/app/services/replay_evaluation/__init__.py
new file mode 100644
index 000000000..d9a07ffd3
--- /dev/null
+++ b/services/api/app/services/replay_evaluation/__init__.py
@@ -0,0 +1,36 @@
+"""Replay evaluation: triage measured against a tenant's own closed findings.
+
+Gap-closure Phase 1.4. Three modules, kept apart so two of them need no
+database and no network to test:
+
+``vendors``
+ Which connector types have a closed-finding reader, and how their stored
+ field names translate into the credential keys the readers expect. Pure.
+``store``
+ Reading and writing the two tenant-scoped tables migration 065 creates.
+``job``
+ The orchestration across ``services/actions``, ``services/agents`` and the
+ vendored scorer, and the only module that makes an outbound call.
+"""
+
+from __future__ import annotations
+
+from app.services.replay_evaluation.vendors import (
+ REPLAYABLE_CONNECTORS,
+ ReplayableConnector,
+ UnsupportedConnector,
+ credentials_for,
+ is_replayable,
+ replayable_connector_ids,
+ vendor_for,
+)
+
+__all__ = [
+ "REPLAYABLE_CONNECTORS",
+ "ReplayableConnector",
+ "UnsupportedConnector",
+ "credentials_for",
+ "is_replayable",
+ "replayable_connector_ids",
+ "vendor_for",
+]
diff --git a/services/api/app/services/replay_evaluation/job.py b/services/api/app/services/replay_evaluation/job.py
new file mode 100644
index 000000000..72451856d
--- /dev/null
+++ b/services/api/app/services/replay_evaluation/job.py
@@ -0,0 +1,306 @@
+"""Orchestrate one replay evaluation across three services, then score it.
+
+Gap-closure Phase 1.4.
+
+Why this lives in the API
+-------------------------
+No process can hold two of these services. ``services/actions`` owns the SIEM
+credential path and the history readers; ``services/agents`` owns triage;
+``services/connectors`` owns ``normalize()``. All three package their code as
+top-level ``app``, so a process that imported two of them would import two
+modules called ``app`` and get one of them.
+
+The API is the service that already does this. It holds the vault and the
+tenant session, it already proxies ``/cases/{id}/investigate`` to agents, and
+it already dispatches live actions to actions. So it reads the history from
+actions, hands it to agents for the shadow triage, and scores the result with
+the vendored benchmark package, which is the one link in the chain with no
+round trip.
+
+The chain, and where each link can fail
+---------------------------------------
+1. the connector row, tenant-scoped, and its vault-decrypted credentials
+2. ``POST {actions}/api/v1/replay/history`` for the closed findings
+3. ``POST {agents}/api/v1/replay/run`` for the shadow decisions, which itself
+ reaches the connectors service for the production ``normalize()``
+4. ``score_replay`` and ``format_replay_report``, in process
+5. one write of the report and the decisions behind it
+
+Every failure is terminal and carries its reason onto the row. There is no
+partial result: a run that read half a window and scored it would publish a
+number over whichever findings happened to arrive, and the report would look
+exactly like one that worked.
+
+Determinism
+-----------
+Nothing here contributes a timestamp, a random value or an unsorted iteration
+to the report. The seed and the resample count travel from the request onto
+the row and into the renderer's method section, so two runs over the same
+history with the deterministic model path produce the same bytes. The wall
+clock appears only in ``created_at`` / ``started_at`` / ``completed_at``, which
+are columns rather than report content.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+import httpx
+import structlog
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app._vendor.aisoc_benchmark.replay import (
+ BOOTSTRAP_RESAMPLES,
+ BOOTSTRAP_SEED,
+ format_replay_report,
+ score_replay,
+)
+from app.models.connector import Connector
+from app.security.credential_vault import CredentialVaultError, get_vault
+from app.services.actions_client import base_url as actions_base_url
+from app.services.actions_client import service_headers as actions_headers
+from app.services.replay_evaluation import store
+from app.services.replay_evaluation.vendors import (
+ UnsupportedConnector,
+ credentials_for,
+)
+
+log = structlog.get_logger(__name__)
+
+__all__ = [
+ "BOOTSTRAP_RESAMPLES",
+ "BOOTSTRAP_SEED",
+ "ReplayJobError",
+ "ReplayRequest",
+ "run_evaluation",
+]
+
+#: A window of a few hundred findings through a model is not a fast call. The
+#: ceiling is generous on purpose: a timeout that fires mid-run would record a
+#: failure for a job that was working.
+_HISTORY_TIMEOUT_S = 180.0
+_REPLAY_TIMEOUT_S = 1800.0
+
+
+class ReplayJobError(Exception):
+ """A link in the chain failed. The message is what lands on the row.
+
+ Carries no partial result by design. The caller records it as a terminal
+ failure with this reason, which is the difference between an operator
+ knowing their Splunk credentials expired and an operator reading a
+ confident report over eleven findings.
+ """
+
+
+@dataclass(frozen=True)
+class ReplayRequest:
+ """One evaluation as the route resolved it, tenant already from the credential."""
+
+ tenant_id: uuid.UUID
+ evaluation_id: uuid.UUID
+ connector_row_id: uuid.UUID
+ connector_type: str
+ vendor: str
+ window_start: datetime
+ window_end: datetime
+ train_fraction: float
+ limit: int
+ bootstrap_seed: int
+ bootstrap_resamples: int
+
+
+def _agents_base_url() -> str:
+ """Where the agents service lives.
+
+ Read exactly the way ``endpoints/cases.py`` reads it for the investigate
+ proxy, including both spellings, so a deployment that configured one of
+ them does not have to learn a third name for this call. Resolved per call
+ rather than at import so a test can point it at a local server.
+ """
+ raw = os.getenv("AGENTS_SERVICE_URL") or os.getenv("AGENTS_API_URL") or "http://agents:8084"
+ return raw.rstrip("/")
+
+
+def _agents_headers(tenant_id: uuid.UUID) -> dict[str, str]:
+ """Service credential plus the tenant this call acts for.
+
+ The agents service resolves a service token with no tenant header to an
+ **empty** scope and refuses, rather than widening to every tenant. So the
+ header is always sent, and it always carries the tenant the route took
+ from the caller's credential.
+ """
+ token = (os.getenv("AISOC_AGENTS_SERVICE_TOKEN") or os.getenv("AISOC_SERVICE_TOKEN") or "").strip()
+ headers = {"Accept": "application/json", "X-AiSOC-Tenant-ID": str(tenant_id)}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ return headers
+
+
+async def _load_connector(db: AsyncSession, request: ReplayRequest) -> tuple[str, dict[str, Any]]:
+ """The connector's type and its decrypted credentials, scoped to the tenant.
+
+ Returns the credential mapping the actions readers expect, not the raw
+ ``auth_config``: the translation lives in one place and this is where it
+ is applied.
+ """
+ row = (
+ await db.execute(
+ select(Connector).where(
+ Connector.id == request.connector_row_id,
+ Connector.tenant_id == request.tenant_id,
+ )
+ )
+ ).scalar_one_or_none()
+ if row is None:
+ raise ReplayJobError("the connector was not found for this tenant")
+
+ try:
+ decrypted = get_vault().decrypt_dict(row.auth_config or {})
+ except CredentialVaultError as exc:
+ raise ReplayJobError(
+ f"stored credentials for this connector could not be decrypted ({exc}); "
+ f"the vault key that wrote them is not in the current keyring"
+ ) from exc
+
+ try:
+ credentials = credentials_for(row.connector_type, decrypted, row.connector_config or {})
+ except UnsupportedConnector as exc:
+ raise ReplayJobError(str(exc)) from exc
+ return row.connector_type, credentials
+
+
+async def _read_history(request: ReplayRequest, credentials: dict[str, Any]) -> dict[str, Any]:
+ payload: dict[str, Any] = {
+ "vendor": request.vendor,
+ "credentials": credentials,
+ "since": request.window_start.isoformat(),
+ "until": request.window_end.isoformat(),
+ "limit": request.limit,
+ }
+ # The two vendor-specific knobs the readers take. Sent only when the
+ # connector stored one, so the reader keeps its own default otherwise.
+ if credentials.get("search_override"):
+ payload["search_override"] = credentials["search_override"]
+ if credentials.get("index"):
+ payload["index"] = credentials["index"]
+
+ url = f"{actions_base_url()}/api/v1/replay/history"
+ try:
+ async with httpx.AsyncClient(timeout=_HISTORY_TIMEOUT_S) as client:
+ response = await client.post(url, headers=actions_headers(), json=payload)
+ except httpx.HTTPError as exc:
+ raise ReplayJobError(f"the actions service is unreachable at {actions_base_url()}: {exc}") from exc
+
+ if response.status_code >= 400:
+ raise ReplayJobError(f"reading {request.vendor} history failed with HTTP {response.status_code}: {response.text[:400]}")
+ body = response.json()
+ if not isinstance(body, dict) or not isinstance(body.get("findings"), list):
+ raise ReplayJobError("the actions service returned an unexpected history shape")
+ return body
+
+
+async def _run_replay(request: ReplayRequest, connector_type: str, findings: list[dict[str, Any]]) -> dict[str, Any]:
+ url = f"{_agents_base_url()}/api/v1/replay/run"
+ payload = {
+ "connector_id": connector_type,
+ "findings": findings,
+ "train_fraction": request.train_fraction,
+ }
+ try:
+ async with httpx.AsyncClient(timeout=_REPLAY_TIMEOUT_S) as client:
+ response = await client.post(url, headers=_agents_headers(request.tenant_id), json=payload)
+ except httpx.HTTPError as exc:
+ raise ReplayJobError(f"the agents service is unreachable at {_agents_base_url()}: {exc}") from exc
+
+ if response.status_code >= 400:
+ raise ReplayJobError(f"the replay run failed with HTTP {response.status_code}: {response.text[:400]}")
+ body = response.json()
+ if not isinstance(body, dict) or not isinstance(body.get("decisions"), list):
+ raise ReplayJobError("the agents service returned an unexpected replay shape")
+ return body
+
+
+async def run_evaluation(db: AsyncSession, request: ReplayRequest) -> None:
+ """Drive the whole chain and write the result. Never raises to the caller.
+
+ Every exception becomes a ``failed`` row with its reason, because this
+ runs detached from the request that started it and an exception escaping
+ here would leave the row ``running`` forever with nothing to read.
+ """
+ try:
+ await store.mark_running(db, tenant_id=request.tenant_id, evaluation_id=request.evaluation_id)
+ connector_type, credentials = await _load_connector(db, request)
+
+ history = await _read_history(request, credentials)
+ findings = history["findings"]
+ if not findings:
+ raise ReplayJobError(
+ "no closed findings were returned for this window. Nothing was graded, and reporting "
+ "a result over zero findings would describe the window rather than the agent."
+ )
+
+ run = await _run_replay(request, connector_type, findings)
+ decisions = run["decisions"]
+ method = dict(run.get("method") or {})
+
+ score = score_replay(
+ decisions,
+ bootstrap_seed=request.bootstrap_seed,
+ bootstrap_resamples=request.bootstrap_resamples,
+ )
+ # The sample sizes the report's method section publishes. Added here
+ # rather than in the renderer so the console and the Markdown agree
+ # about what was read, which is the disagreement this repository keeps
+ # finding between two surfaces over the same rows.
+ method["history"] = {
+ "vendor": request.vendor,
+ "connector_type": connector_type,
+ "window_start": request.window_start.isoformat(),
+ "window_end": request.window_end.isoformat(),
+ "findings_read": int(history.get("count") or len(findings)),
+ "findings_labelled": int(history.get("labelled") or 0),
+ "findings_unlabeled": int(history.get("unlabeled") or 0),
+ }
+ report = format_replay_report(score, method=method)
+
+ await store.store_result(
+ db,
+ tenant_id=request.tenant_id,
+ evaluation_id=request.evaluation_id,
+ score=score.as_dict(),
+ method=method,
+ report_markdown=report,
+ decisions=decisions,
+ findings_read=int(history.get("count") or len(findings)),
+ findings_labelled=int(history.get("labelled") or 0),
+ )
+ log.info(
+ "replay.evaluation.completed",
+ evaluation_id=str(request.evaluation_id),
+ findings=len(findings),
+ decisions=len(decisions),
+ graded=score.graded,
+ )
+ except ReplayJobError as exc:
+ await _record_failure(db, request, str(exc))
+ except Exception as exc: # noqa: BLE001 - a detached job must terminate its own row
+ log.exception("replay.evaluation.unexpected_failure", evaluation_id=str(request.evaluation_id))
+ await _record_failure(db, request, f"{type(exc).__name__}: {exc}")
+
+
+async def _record_failure(db: AsyncSession, request: ReplayRequest, reason: str) -> None:
+ try:
+ await db.rollback()
+ await store.fail_evaluation(
+ db,
+ tenant_id=request.tenant_id,
+ evaluation_id=request.evaluation_id,
+ error=reason,
+ )
+ except Exception: # noqa: BLE001 - nothing left to do but say so
+ log.exception("replay.evaluation.failure_not_recorded", evaluation_id=str(request.evaluation_id))
diff --git a/services/api/app/services/replay_evaluation/report.py b/services/api/app/services/replay_evaluation/report.py
new file mode 100644
index 000000000..a24887481
--- /dev/null
+++ b/services/api/app/services/replay_evaluation/report.py
@@ -0,0 +1,254 @@
+"""Export a stored replay report as JSON, Markdown or PDF.
+
+Gap-closure Phase 1.4.
+
+There is one renderer, and it is not here
+-----------------------------------------
+``format_replay_report`` in the benchmark package produces the Markdown, and
+``ReplayScore.as_dict`` produces the JSON. Both shipped with Phase 1.3 and
+both are reused unchanged. This module does not render a report; it serves the
+one that was stored when the run completed, and turns that same Markdown into
+a PDF.
+
+That is deliberate. The phase's acceptance bar is that a report reproduces
+byte for byte, and a second renderer would eventually disagree with the first
+about a heading or a rounding while both still called themselves the report.
+
+Why the Markdown to HTML step is a small hand-written pass
+----------------------------------------------------------
+The Markdown being converted is not arbitrary: it comes from one function in
+this repository, and that function emits exactly headings, bullet lists,
+pipe tables and paragraphs. A general Markdown library would be a new pinned
+dependency to convert output whose full grammar is forty lines away.
+
+The pass escapes everything it does not itself emit. The report embeds
+connector-supplied values: rule ids, vendor names, and the hallucinated
+indicator examples, which are by definition strings a model produced from
+attacker-influenced evidence. Writing those into HTML unescaped would put
+model output into a document an operator opens.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import logging
+import re
+from typing import Any
+
+from app._vendor.aisoc_benchmark.replay import LATENCY_FIELDS, strip_latency
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+ "PDF_UNAVAILABLE_DETAIL",
+ "PdfUnavailableError",
+ "markdown_to_html",
+ "render_pdf",
+ "report_json",
+ "report_markdown",
+]
+
+#: Said to the operator when the native stack behind WeasyPrint is missing.
+#: Named as a constant so the route and the test assert the same sentence, and
+#: so it says what to do rather than reporting a failed export as an empty file.
+PDF_UNAVAILABLE_DETAIL = (
+ "PDF export needs the WeasyPrint native libraries (Pango, Cairo, GLib), which are "
+ "installed in the API container image but may be absent on a local checkout. "
+ "The JSON and Markdown exports carry the same report and need nothing extra."
+)
+
+#: A table row: leading and trailing pipes with cells between.
+_TABLE_ROW = re.compile(r"^\|(.+)\|$")
+#: A table's separator row, which carries no data and is not rendered.
+_TABLE_DIVIDER = re.compile(r"^\|[\s:|-]+\|$")
+
+
+class PdfUnavailableError(RuntimeError):
+ """WeasyPrint or its native stack is not installed."""
+
+
+def report_json(
+ *,
+ evaluation_id: str,
+ score: dict[str, Any] | None,
+ method: dict[str, Any] | None,
+ exclude_latency: bool = False,
+) -> str:
+ """The machine-readable export.
+
+ ``sort_keys`` because the export is compared byte for byte across runs and
+ Python's dict order would otherwise carry insertion order into the file.
+
+ ``exclude_latency`` replaces the two wall-clock figures with ``None``
+ rather than deleting the keys. A reader parsing the JSON should not have
+ to tell "this run measured no latency" from "this key is missing on some
+ exports", and ``None`` is already how every other unmeasured rate is
+ written in this schema.
+ """
+ payload_score = dict(score or {})
+ if exclude_latency:
+ for field in LATENCY_FIELDS:
+ if field in payload_score:
+ payload_score[field] = None
+ return json.dumps(
+ {
+ "evaluation_id": evaluation_id,
+ "score": payload_score,
+ "method": method or {},
+ "latency_excluded": exclude_latency,
+ },
+ indent=2,
+ sort_keys=True,
+ default=str,
+ )
+
+
+def report_markdown(stored: str, *, exclude_latency: bool = False) -> str:
+ """The stored report, optionally without its two wall-clock figures.
+
+ This is a filter over the artefact, not a re-render. ``strip_latency``
+ lives beside the renderer that emits the line, so the producer and the
+ remover cannot drift into disagreeing about which line it is.
+ """
+ return strip_latency(stored) if exclude_latency else stored
+
+
+def _cells(line: str) -> list[str]:
+ match = _TABLE_ROW.match(line)
+ if match is None: # pragma: no cover - callers test the pattern first
+ return []
+ return [cell.strip() for cell in match.group(1).split("|")]
+
+
+def _inline(text: str) -> str:
+ """Escape, then re-apply the only inline markup the report emits.
+
+ Order matters: escaping first means a value that happens to contain
+ ``**`` or a tag is rendered as text rather than as markup. The report's
+ own bold spans survive because they are re-applied to the escaped string.
+ """
+ escaped = html.escape(text)
+ return re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", escaped)
+
+
+def markdown_to_html(markdown: str) -> str:
+ """Convert the replay report's Markdown subset to an HTML fragment.
+
+ Handles the four constructs ``format_replay_report`` emits and nothing
+ else. Anything unrecognised becomes a paragraph, escaped, which is the
+ safe direction: an unhandled construct renders as its own source text
+ rather than as markup somebody else chose.
+ """
+ out: list[str] = []
+ table: list[list[str]] = []
+ bullets: list[str] = []
+
+ def flush_table() -> None:
+ if not table:
+ return
+ header, *body = table
+ out.append("<table>")
+ out.append("<thead><tr>" + "".join(f"<th>{_inline(c)}</th>" for c in header) + "</tr></thead>")
+ out.append("<tbody>")
+ for row in body:
+ out.append("<tr>" + "".join(f"<td>{_inline(c)}</td>" for c in row) + "</tr>")
+ out.append("</tbody></table>")
+ table.clear()
+
+ def flush_bullets() -> None:
+ if not bullets:
+ return
+ out.append("<ul>" + "".join(f"<li>{_inline(b)}</li>" for b in bullets) + "</ul>")
+ bullets.clear()
+
+ for raw_line in markdown.splitlines():
+ line = raw_line.rstrip()
+
+ if _TABLE_DIVIDER.match(line):
+ continue
+ if _TABLE_ROW.match(line):
+ flush_bullets()
+ table.append(_cells(line))
+ continue
+ flush_table()
+
+ if line.startswith("- "):
+ bullets.append(line[2:])
+ continue
+ flush_bullets()
+
+ if not line:
+ continue
+ if line.startswith("### "):
+ out.append(f"<h3>{_inline(line[4:])}</h3>")
+ elif line.startswith("## "):
+ out.append(f"<h2>{_inline(line[3:])}</h2>")
+ elif line.startswith("# "):
+ out.append(f"<h1>{_inline(line[2:])}</h1>")
+ else:
+ out.append(f"<p>{_inline(line)}</p>")
+
+ flush_table()
+ flush_bullets()
+ return "\n".join(out)
+
+
+def _document(body_html: str) -> str:
+ """Wrap the fragment in a print-ready page.
+
+ No generation timestamp and no host name anywhere. A report that prints
+ "generated at" is a report that cannot reproduce byte for byte, which is
+ this phase's acceptance bar.
+ """
+ return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>AiSOC replay evaluation</title>
+<style>
+ @page {{ margin: 18mm; }}
+ body {{
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
+ color: #0f172a;
+ font-size: 10.5pt;
+ line-height: 1.45;
+ }}
+ h1 {{ font-size: 20pt; margin: 0 0 4mm; }}
+ h2 {{ font-size: 13pt; margin: 7mm 0 2mm; border-bottom: 1px solid #cbd5e1; padding-bottom: 1mm; }}
+ h3 {{ font-size: 11pt; margin: 5mm 0 2mm; }}
+ p {{ margin: 0 0 2mm; }}
+ ul {{ margin: 0 0 3mm 5mm; padding: 0; }}
+ li {{ margin: 0 0 1mm; }}
+ table {{ border-collapse: collapse; width: 100%; margin: 0 0 4mm; font-size: 9pt; }}
+ th, td {{ border: 1px solid #cbd5e1; padding: 1.5mm 2mm; text-align: left; }}
+ th {{ background: #f1f5f9; font-weight: 600; }}
+</style>
+</head>
+<body>
+{body_html}
+</body>
+</html>"""
+
+
+def render_pdf(markdown: str) -> bytes:
+ """Render the stored report as a PDF.
+
+ Raises :class:`PdfUnavailableError` rather than returning an empty or
+ partial document, because a zero-byte PDF served with a 200 reads as a
+ corrupt report rather than as a missing library.
+ """
+ # Imported at call time, following ``app.services.digest_pdf``: WeasyPrint
+ # pulls a native stack that is present in the API image and routinely
+ # absent on a checkout, and a module-level import would take the whole
+ # service down over an export nobody asked for.
+ try:
+ from weasyprint import HTML # type: ignore[import-untyped]
+ except (ImportError, OSError) as exc:
+ # OSError as well as ImportError: WeasyPrint imports but fails to load
+ # its native libraries on a machine without them, and that surfaces
+ # here rather than as a missing module.
+ logger.warning("replay report PDF export unavailable: %s", exc)
+ raise PdfUnavailableError(PDF_UNAVAILABLE_DETAIL) from exc
+
+ return HTML(string=_document(markdown_to_html(markdown))).write_pdf()
diff --git a/services/api/app/services/replay_evaluation/store.py b/services/api/app/services/replay_evaluation/store.py
new file mode 100644
index 000000000..2f14b312a
--- /dev/null
+++ b/services/api/app/services/replay_evaluation/store.py
@@ -0,0 +1,382 @@
+"""Reading and writing replay evaluations, always bound to one tenant.
+
+Gap-closure Phase 1.4.
+
+Every statement here binds ``tenant_id`` as a query parameter, and every
+caller receives that tenant from the authenticated principal. The row-level
+security policies in migration 065 are defence in depth behind this, not
+instead of it: this repository enforces tenant isolation at the query layer
+and keeps RLS as the second layer, because the services connect as a role the
+policies apply to only after 061 and a query-layer predicate holds on any
+connection.
+
+Raw SQL rather than an ORM model, following ``app.services.sandbox.store``.
+The rows are written once by a background job and read back whole; mapping
+them would add a model to keep in parity with a migration for no reader that
+needs one.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+__all__ = [
+ "TERMINAL_STATUSES",
+ "EvaluationRow",
+ "create_evaluation",
+ "fail_evaluation",
+ "get_evaluation",
+ "list_decisions",
+ "list_evaluations",
+ "mark_running",
+ "store_result",
+]
+
+#: A run in one of these will never change again, so a poller can stop.
+TERMINAL_STATUSES = frozenset({"completed", "failed"})
+
+#: Columns the list view needs. The score and the report are deliberately
+#: absent: a list query should not drag a few megabytes of JSONB and prose per
+#: row across the wire to render a table of sample sizes.
+_SUMMARY_COLUMNS = """
+ id, tenant_id, status, error, connector_id, vendor,
+ window_start, window_end, train_fraction, bootstrap_seed, bootstrap_resamples,
+ findings_read, findings_labelled, decisions_recorded, graded, malicious_support,
+ headline_accuracy, headline_withheld_reason, malicious_recall,
+ requested_by, created_at, started_at, completed_at
+"""
+
+
+@dataclass(frozen=True)
+class EvaluationRow:
+ """One evaluation as the API returns it.
+
+ ``score``, ``method`` and ``report_markdown`` are ``None`` on a summary
+ read rather than empty, so a caller can tell "not loaded" from "the run
+ produced none", which are different states with different causes.
+ """
+
+ id: uuid.UUID
+ tenant_id: uuid.UUID
+ status: str
+ error: str | None
+ connector_id: str
+ vendor: str
+ window_start: datetime
+ window_end: datetime
+ train_fraction: float
+ bootstrap_seed: int
+ bootstrap_resamples: int
+ findings_read: int
+ findings_labelled: int
+ decisions_recorded: int
+ graded: int
+ malicious_support: int
+ headline_accuracy: float | None
+ headline_withheld_reason: str | None
+ malicious_recall: float | None
+ requested_by: uuid.UUID | None
+ created_at: datetime
+ started_at: datetime | None
+ completed_at: datetime | None
+ score: dict[str, Any] | None = None
+ method: dict[str, Any] | None = None
+ report_markdown: str | None = None
+
+ @property
+ def is_terminal(self) -> bool:
+ return self.status in TERMINAL_STATUSES
+
+
+def _row_to_evaluation(row: Any, *, detailed: bool = False) -> EvaluationRow:
+ return EvaluationRow(
+ id=row.id,
+ tenant_id=row.tenant_id,
+ status=row.status,
+ error=row.error,
+ connector_id=row.connector_id,
+ vendor=row.vendor,
+ window_start=row.window_start,
+ window_end=row.window_end,
+ train_fraction=float(row.train_fraction),
+ bootstrap_seed=int(row.bootstrap_seed),
+ bootstrap_resamples=int(row.bootstrap_resamples),
+ findings_read=int(row.findings_read),
+ findings_labelled=int(row.findings_labelled),
+ decisions_recorded=int(row.decisions_recorded),
+ graded=int(row.graded),
+ malicious_support=int(row.malicious_support),
+ headline_accuracy=row.headline_accuracy,
+ headline_withheld_reason=row.headline_withheld_reason,
+ malicious_recall=row.malicious_recall,
+ requested_by=row.requested_by,
+ created_at=row.created_at,
+ started_at=row.started_at,
+ completed_at=row.completed_at,
+ score=_as_json(row.score) if detailed else None,
+ method=_as_json(row.method) if detailed else None,
+ report_markdown=row.report_markdown if detailed else None,
+ )
+
+
+def _as_json(value: Any) -> dict[str, Any] | None:
+ """Read a JSONB column back as a dict.
+
+ asyncpg returns JSONB as a parsed object through SQLAlchemy's JSONB type,
+ but this module binds through ``text()`` and gets whatever the driver
+ hands back. A string is parsed; anything else is returned as-is.
+ """
+ if value is None:
+ return None
+ if isinstance(value, str):
+ try:
+ parsed = json.loads(value)
+ except ValueError:
+ return None
+ return parsed if isinstance(parsed, dict) else None
+ return value if isinstance(value, dict) else None
+
+
+async def create_evaluation(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ connector_id: str,
+ vendor: str,
+ window_start: datetime,
+ window_end: datetime,
+ train_fraction: float,
+ bootstrap_seed: int,
+ bootstrap_resamples: int,
+ requested_by: uuid.UUID | None,
+) -> uuid.UUID:
+ """Insert a queued run and return its id.
+
+ The row exists before any work starts, so a caller that polls immediately
+ gets ``queued`` rather than a 404 that looks like a lost request.
+ """
+ evaluation_id = uuid.uuid4()
+ await db.execute(
+ # `requested_by` is resolved through a subselect rather than bound
+ # straight into the column. The authenticated principal is not always
+ # a row in `users`: the dev-mode demo user has a fixed id that no
+ # migration seeds, and an API key is a row in `api_keys`. Binding the
+ # id directly turns either of those into a foreign-key violation, so
+ # starting an evaluation would 500 on a deployment where every other
+ # route works. An id with no user resolves to NULL, which is what the
+ # column's `ON DELETE SET NULL` already means: attribution is unknown,
+ # and the evaluation is still the tenant's.
+ text("""
+ INSERT INTO aisoc_replay_evaluations
+ (id, tenant_id, status, connector_id, vendor, window_start, window_end,
+ train_fraction, bootstrap_seed, bootstrap_resamples, requested_by)
+ VALUES
+ (:id, :tenant_id, 'queued', :connector_id, :vendor, :window_start, :window_end,
+ :train_fraction, :bootstrap_seed, :bootstrap_resamples,
+ (SELECT id FROM users WHERE id = :requested_by))
+ """).bindparams(
+ id=evaluation_id,
+ tenant_id=tenant_id,
+ connector_id=connector_id,
+ vendor=vendor,
+ window_start=window_start,
+ window_end=window_end,
+ train_fraction=train_fraction,
+ bootstrap_seed=bootstrap_seed,
+ bootstrap_resamples=bootstrap_resamples,
+ requested_by=requested_by,
+ )
+ )
+ await db.commit()
+ return evaluation_id
+
+
+async def mark_running(db: AsyncSession, *, tenant_id: uuid.UUID, evaluation_id: uuid.UUID) -> None:
+ await db.execute(
+ text("""
+ UPDATE aisoc_replay_evaluations
+ SET status = 'running', started_at = NOW()
+ WHERE id = :id AND tenant_id = :tenant_id
+ """).bindparams(id=evaluation_id, tenant_id=tenant_id)
+ )
+ await db.commit()
+
+
+async def fail_evaluation(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ evaluation_id: uuid.UUID,
+ error: str,
+) -> None:
+ """Record a terminal failure with the reason.
+
+ A run that crashed must not be left ``running``: a job that never
+ terminates is indistinguishable from a slow one, and a poller would wait
+ on it forever.
+ """
+ await db.execute(
+ text("""
+ UPDATE aisoc_replay_evaluations
+ SET status = 'failed', error = :error, completed_at = NOW()
+ WHERE id = :id AND tenant_id = :tenant_id
+ """).bindparams(id=evaluation_id, tenant_id=tenant_id, error=error[:4000])
+ )
+ await db.commit()
+
+
+async def store_result(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ evaluation_id: uuid.UUID,
+ score: dict[str, Any],
+ method: dict[str, Any],
+ report_markdown: str,
+ decisions: list[dict[str, Any]],
+ findings_read: int,
+ findings_labelled: int,
+) -> None:
+ """Write the report and the decisions behind it, in one transaction.
+
+ The report is stored as the renderer produced it. The phase's acceptance
+ bar is that a report reproduces byte for byte, and a report re-rendered
+ later by a newer renderer is a different artefact from the one the
+ operator read.
+ """
+ await db.execute(
+ text("""
+ UPDATE aisoc_replay_evaluations
+ SET status = 'completed',
+ completed_at = NOW(),
+ score = CAST(:score AS JSONB),
+ method = CAST(:method AS JSONB),
+ report_markdown = :report,
+ findings_read = :findings_read,
+ findings_labelled = :findings_labelled,
+ decisions_recorded = :decisions_recorded,
+ graded = :graded,
+ malicious_support = :malicious_support,
+ headline_accuracy = :headline_accuracy,
+ headline_withheld_reason = :withheld,
+ malicious_recall = :malicious_recall
+ WHERE id = :id AND tenant_id = :tenant_id
+ """).bindparams(
+ id=evaluation_id,
+ tenant_id=tenant_id,
+ score=json.dumps(score, sort_keys=True, default=str),
+ method=json.dumps(method, sort_keys=True, default=str),
+ report=report_markdown,
+ findings_read=findings_read,
+ findings_labelled=findings_labelled,
+ decisions_recorded=len(decisions),
+ graded=int(score.get("graded") or 0),
+ malicious_support=int(score.get("malicious_support") or 0),
+ # Left NULL when withheld. Never 0, which would read as "the agent
+ # got every answer wrong".
+ headline_accuracy=score.get("headline_accuracy"),
+ withheld=score.get("headline_withheld_reason"),
+ malicious_recall=score.get("malicious_recall"),
+ )
+ )
+
+ for decision in decisions:
+ await db.execute(
+ text("""
+ INSERT INTO aisoc_replay_decisions
+ (evaluation_id, tenant_id, finding_id, vendor, rule_id, closed_at,
+ expected_disposition, vendor_disposition, labelled,
+ verdict, verdict_raw, confidence, tier, decision, error)
+ VALUES
+ (:evaluation_id, :tenant_id, :finding_id, :vendor, :rule_id,
+ CAST(NULLIF(:closed_at, '') AS TIMESTAMPTZ),
+ :expected, :vendor_disposition, :labelled,
+ :verdict, :verdict_raw, :confidence, :tier, CAST(:decision AS JSONB), :error)
+ """).bindparams(
+ evaluation_id=evaluation_id,
+ tenant_id=tenant_id,
+ finding_id=str(decision.get("finding_id") or ""),
+ vendor=str(decision.get("vendor") or ""),
+ rule_id=decision.get("rule_id"),
+ closed_at=str(decision.get("closed_at") or ""),
+ expected=str(decision.get("expected_disposition") or ""),
+ vendor_disposition=str(decision.get("vendor_disposition") or ""),
+ labelled=bool(decision.get("labelled")),
+ verdict=decision.get("verdict"),
+ verdict_raw=decision.get("verdict_raw"),
+ confidence=float(decision.get("confidence") or 0.0),
+ tier=str(decision.get("tier") or ""),
+ decision=json.dumps(decision, sort_keys=True, default=str),
+ error=decision.get("error"),
+ )
+ )
+ await db.commit()
+
+
+async def get_evaluation(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ evaluation_id: uuid.UUID,
+ detailed: bool = True,
+) -> EvaluationRow | None:
+ columns = f"{_SUMMARY_COLUMNS}, score, method, report_markdown" if detailed else _SUMMARY_COLUMNS
+ row = (
+ await db.execute(
+ text(
+ f"SELECT {columns} FROM aisoc_replay_evaluations " # noqa: S608 - column list is a module constant
+ "WHERE id = :id AND tenant_id = :tenant_id"
+ ).bindparams(id=evaluation_id, tenant_id=tenant_id)
+ )
+ ).fetchone()
+ return _row_to_evaluation(row, detailed=detailed) if row else None
+
+
+async def list_evaluations(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ limit: int = 50,
+) -> list[EvaluationRow]:
+ rows = (
+ await db.execute(
+ text(
+ f"SELECT {_SUMMARY_COLUMNS} FROM aisoc_replay_evaluations " # noqa: S608 - column list is a module constant
+ "WHERE tenant_id = :tenant_id ORDER BY created_at DESC LIMIT :limit"
+ ).bindparams(tenant_id=tenant_id, limit=limit)
+ )
+ ).fetchall()
+ return [_row_to_evaluation(row) for row in rows]
+
+
+async def list_decisions(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ evaluation_id: uuid.UUID,
+ limit: int = 2000,
+) -> list[dict[str, Any]]:
+ """The decisions behind a report, oldest close time first.
+
+ Ordered by ``(closed_at, finding_id)`` rather than by insertion, because
+ close times collide when an analyst bulk-closes a queue and a sort that is
+ not total makes the page order depend on the database's row layout.
+ """
+ rows = (
+ await db.execute(
+ text("""
+ SELECT decision FROM aisoc_replay_decisions
+ WHERE evaluation_id = :evaluation_id AND tenant_id = :tenant_id
+ ORDER BY closed_at NULLS LAST, finding_id
+ LIMIT :limit
+ """).bindparams(evaluation_id=evaluation_id, tenant_id=tenant_id, limit=limit)
+ )
+ ).fetchall()
+ return [parsed for row in rows if (parsed := _as_json(row.decision)) is not None]
diff --git a/services/api/app/services/replay_evaluation/vendors.py b/services/api/app/services/replay_evaluation/vendors.py
new file mode 100644
index 000000000..ddc8e64b8
--- /dev/null
+++ b/services/api/app/services/replay_evaluation/vendors.py
@@ -0,0 +1,200 @@
+"""Which connectors can be replayed, and how their credentials reach the readers.
+
+Gap-closure Phase 1.4.
+
+Two vocabularies meet here and neither is wrong.
+
+A **connector** stores credentials under the field names its setup wizard
+collects: Splunk's is ``base_url``, QRadar's is ``console_url``, Elastic's is
+``base_url`` again. Those names are part of a saved row and a rendered form,
+and renaming them would invalidate every stored connector.
+
+A **client** in ``services/actions`` reads a flat, vendor-prefixed mapping:
+``splunk_url``, ``qradar_url``, ``elastic_url``. Those names are shared with
+the writeback path and with playbook parameters, and renaming them would
+break every playbook that names one.
+
+So there is a translation, and this module is the only place it is written
+down. It is pure: no database, no HTTP, no vault. Every mapping can therefore
+be driven in a test without a running anything, which matters because the
+failure mode is silent. A key that lands under the wrong name does not raise;
+the factory returns ``None`` and the read is refused as "no usable
+credentials", which reads to an operator as a configuration problem on their
+side.
+
+Why a table and not a convention
+--------------------------------
+Three of the five pairs would fit a rule like "prefix the field with the
+vendor". ``base_url`` to ``splunk_url`` does not, and neither does
+``sec_token`` to ``qradar_token``. A convention with exceptions is a table
+with extra steps, so it is a table.
+
+Defender is reached through the ``azure_defender`` connector
+------------------------------------------------------------
+There is no separate Defender XDR connector in this tree. ``azure_defender``
+collects exactly the three credentials the Defender reader needs, and its own
+description names the Defender products it covers, so it is the connector that
+maps to the ``defender`` reader rather than a sixth one being invented.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+__all__ = [
+ "REPLAYABLE_CONNECTORS",
+ "ReplayableConnector",
+ "UnsupportedConnector",
+ "credentials_for",
+ "is_replayable",
+ "replayable_connector_ids",
+ "vendor_for",
+]
+
+
+class UnsupportedConnector(Exception):
+ """This connector type has no closed-finding reader.
+
+ Raised rather than returning an empty history, because "AiSOC cannot read
+ this vendor's closed findings" and "this vendor's analysts closed nothing"
+ are different answers and only one of them is about the product.
+ """
+
+
+@dataclass(frozen=True)
+class ReplayableConnector:
+ """One connector type that can be replayed, and how to translate it."""
+
+ connector_type: str
+ #: The reader arm on ``services/actions``.
+ vendor: str
+ label: str
+ #: Connector ``auth_config`` field name to the credential key the client
+ #: factory in ``app.executors.siem`` reads.
+ auth_map: dict[str, str]
+ #: Connector ``connector_config`` field name to the same. Kept apart
+ #: because ``connector_config`` is not secret and is not vault-encrypted,
+ #: so a mapping that pulled from the wrong one would either leak a secret
+ #: into a non-secret column or fail to decrypt.
+ config_map: dict[str, str]
+
+
+#: The five vendors Phase 1.1 wrote readers for, keyed by the connector type
+#: a tenant actually saves. Anything absent here is refused by name.
+REPLAYABLE_CONNECTORS: dict[str, ReplayableConnector] = {
+ "splunk": ReplayableConnector(
+ connector_type="splunk",
+ vendor="splunk",
+ label="Splunk Enterprise Security",
+ auth_map={
+ "base_url": "splunk_url",
+ "token": "splunk_token",
+ "username": "splunk_username",
+ "password": "splunk_password",
+ },
+ # The saved search is where a deployment keeps its notables, and
+ # deployments rename it. Hardcoding one returns nothing on a site that
+ # did, which is a zero-row window rather than an error.
+ config_map={"ssl_verify": "splunk_verify_ssl", "saved_search": "search_override"},
+ ),
+ "microsoft_sentinel": ReplayableConnector(
+ connector_type="microsoft_sentinel",
+ vendor="sentinel",
+ label="Microsoft Sentinel",
+ auth_map={
+ "tenant_id": "sentinel_tenant_id",
+ "client_id": "sentinel_client_id",
+ "client_secret": "sentinel_client_secret",
+ "subscription_id": "sentinel_subscription_id",
+ "resource_group": "sentinel_resource_group",
+ "workspace": "sentinel_workspace_name",
+ },
+ config_map={},
+ ),
+ "elastic": ReplayableConnector(
+ connector_type="elastic",
+ vendor="elastic",
+ label="Elastic Security",
+ auth_map={
+ "base_url": "elastic_url",
+ "api_key": "elastic_api_key",
+ "username": "elastic_username",
+ "password": "elastic_password",
+ },
+ config_map={"index": "index"},
+ ),
+ "qradar": ReplayableConnector(
+ connector_type="qradar",
+ vendor="qradar",
+ label="IBM QRadar",
+ auth_map={"console_url": "qradar_url", "sec_token": "qradar_token"},
+ config_map={"verify_tls": "qradar_verify_ssl"},
+ ),
+ "azure_defender": ReplayableConnector(
+ connector_type="azure_defender",
+ vendor="defender",
+ label="Microsoft Defender XDR",
+ auth_map={
+ "tenant_id": "mde_tenant_id",
+ "client_id": "mde_client_id",
+ "client_secret": "mde_client_secret",
+ },
+ config_map={},
+ ),
+}
+
+
+def replayable_connector_ids() -> list[str]:
+ """Connector types with a closed-finding reader, sorted."""
+ return sorted(REPLAYABLE_CONNECTORS)
+
+
+def is_replayable(connector_type: str) -> bool:
+ return connector_type in REPLAYABLE_CONNECTORS
+
+
+def vendor_for(connector_type: str) -> str:
+ """The reader arm for a connector type, or refuse by name."""
+ entry = REPLAYABLE_CONNECTORS.get(connector_type)
+ if entry is None:
+ raise UnsupportedConnector(
+ f"'{connector_type}' has no closed-finding reader. Replay evaluation supports {', '.join(replayable_connector_ids())}."
+ )
+ return entry.vendor
+
+
+def credentials_for(
+ connector_type: str,
+ auth_config: dict[str, Any],
+ connector_config: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ """Translate one connector's stored settings into the reader's credential keys.
+
+ ``auth_config`` arrives already decrypted; this function never touches the
+ vault, so a test can drive every mapping with plain strings.
+
+ A field the connector did not store is omitted rather than set to ``None``.
+ The factories test truthiness, so an explicit ``None`` and an absent key
+ behave the same for them, but an omitted key keeps the forwarded payload to
+ what the tenant actually configured.
+ """
+ entry = REPLAYABLE_CONNECTORS.get(connector_type)
+ if entry is None:
+ raise UnsupportedConnector(
+ f"'{connector_type}' has no closed-finding reader. Replay evaluation supports {', '.join(replayable_connector_ids())}."
+ )
+
+ credentials: dict[str, Any] = {}
+ for source, target in entry.auth_map.items():
+ value = auth_config.get(source)
+ if value not in (None, ""):
+ credentials[target] = value
+ for source, target in entry.config_map.items():
+ value = (connector_config or {}).get(source)
+ # Booleans are copied even when False: ``ssl_verify: false`` is a
+ # deliberate choice for an internal CA, and dropping it would silently
+ # re-enable verification against a certificate that will not validate.
+ if isinstance(value, bool) or value not in (None, ""):
+ credentials[target] = value
+ return credentials
diff --git a/services/api/migrations/065_replay_evaluations.sql b/services/api/migrations/065_replay_evaluations.sql
new file mode 100644
index 000000000..bfbdd279e
--- /dev/null
+++ b/services/api/migrations/065_replay_evaluations.sql
@@ -0,0 +1,178 @@
+-- 065: replay evaluation jobs and the decisions behind their reports.
+--
+-- Gap-closure Phase 1.4. An operator asks "how would this have done on my
+-- alerts", the API reads their closed findings, replays them through the
+-- production triage path and grades the result. This is where the answer
+-- lives.
+--
+-- Note on the number: 064 is `064_sandbox_upload_policy.sql`, which landed
+-- from a different phase while this one was in flight. The plan's deviation
+-- log names 064 for this table because that was the next free number when it
+-- was written. It is not any more.
+--
+-- Why two tables
+-- --------------
+-- `aisoc_replay_evaluations` answers "what did the report say". It is read on
+-- every page load and every export, and it is small.
+--
+-- `aisoc_replay_decisions` answers "why did it say that". One row per
+-- replayed finding, holding the analyst's label, the agent's verdict and the
+-- evidence the agent was given. It is read when somebody disputes a number,
+-- which is rare and heavy. Folding them into one table would put a few
+-- megabytes of evidence behind every list query.
+--
+-- Why the report markdown is stored rather than re-rendered
+-- ---------------------------------------------------------
+-- The phase's acceptance bar is that a report reproduces byte for byte. A
+-- report re-rendered later by a newer renderer is a different artefact from
+-- the one the operator read, and the export would silently stop matching what
+-- the page showed when it was produced. `report_markdown` is the artefact.
+-- `score` is kept alongside it so the console can render structured views
+-- without parsing prose back out.
+--
+-- Why a status column and not a queue
+-- -----------------------------------
+-- The job runs in-process in the API, which is where the vault and the tenant
+-- session already are. `status` is how a caller polling `GET
+-- /evaluations/replay/{id}` tells a run still going from one that failed, and
+-- `error` is why. A run that crashed reads `failed` with a reason rather than
+-- staying `running` forever, because a job that never terminates is
+-- indistinguishable from a slow one.
+
+CREATE TABLE IF NOT EXISTS aisoc_replay_evaluations (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
+
+ -- 'queued' | 'running' | 'completed' | 'failed'
+ status TEXT NOT NULL DEFAULT 'queued',
+ error TEXT,
+
+ -- What was asked for.
+ connector_id TEXT NOT NULL,
+ vendor TEXT NOT NULL,
+ window_start TIMESTAMPTZ NOT NULL,
+ window_end TIMESTAMPTZ NOT NULL,
+ train_fraction DOUBLE PRECISION NOT NULL,
+ -- Both travel onto the row because both change the confidence intervals,
+ -- and a report whose seed is not recorded cannot be reproduced.
+ bootstrap_seed INTEGER NOT NULL,
+ bootstrap_resamples INTEGER NOT NULL,
+
+ -- Sample sizes, promoted out of the score JSON because the console shows
+ -- them beside the headline and a list query should not have to open a
+ -- JSONB document to say how much was graded.
+ findings_read INTEGER NOT NULL DEFAULT 0,
+ findings_labelled INTEGER NOT NULL DEFAULT 0,
+ decisions_recorded INTEGER NOT NULL DEFAULT 0,
+ graded INTEGER NOT NULL DEFAULT 0,
+ malicious_support INTEGER NOT NULL DEFAULT 0,
+
+ -- NULL when the corpus was too thin for a headline. Not 0: a zero in an
+ -- accuracy column says the agent got everything wrong, and "withheld" is
+ -- a different fact with a different remedy. `headline_withheld_reason`
+ -- carries the sentence the report prints.
+ headline_accuracy DOUBLE PRECISION,
+ headline_withheld_reason TEXT,
+ malicious_recall DOUBLE PRECISION,
+
+ -- The full `ReplayScore.as_dict()` and the method block the runner
+ -- produced, so the console can render every panel from one read.
+ score JSONB,
+ method JSONB,
+ -- The artefact the operator read, byte for byte.
+ report_markdown TEXT,
+
+ requested_by UUID REFERENCES users(id) ON DELETE SET NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ started_at TIMESTAMPTZ,
+ completed_at TIMESTAMPTZ
+);
+
+COMMENT ON TABLE aisoc_replay_evaluations IS
+ 'One replay evaluation: triage measured against a tenant''s own analysts on '
+ 'their own closed findings. The stored report_markdown is the artefact the '
+ 'operator read, not a re-render.';
+COMMENT ON COLUMN aisoc_replay_evaluations.headline_accuracy IS
+ 'NULL when withheld for a thin corpus. Never 0 to mean "not measured".';
+COMMENT ON COLUMN aisoc_replay_evaluations.bootstrap_seed IS
+ 'Seed the percentile bootstrap used. Recorded because a report that cannot '
+ 'name its seed cannot be reproduced.';
+
+CREATE TABLE IF NOT EXISTS aisoc_replay_decisions (
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+ evaluation_id UUID NOT NULL REFERENCES aisoc_replay_evaluations(id) ON DELETE CASCADE,
+ -- Denormalised so the RLS policy on this table stands on its own rather
+ -- than on a join. A policy that has to reach another table to decide is a
+ -- policy that returns nothing when that table is itself filtered.
+ tenant_id UUID NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
+
+ finding_id TEXT NOT NULL,
+ vendor TEXT NOT NULL,
+ rule_id TEXT,
+ closed_at TIMESTAMPTZ,
+
+ -- The analyst's label, canonical. 'unlabeled' means excluded from
+ -- accuracy, never guessed.
+ expected_disposition TEXT NOT NULL,
+ vendor_disposition TEXT NOT NULL DEFAULT '',
+ labelled BOOLEAN NOT NULL DEFAULT FALSE,
+
+ -- What triage said. `verdict_raw` is kept beside the canonical form
+ -- because a verdict the taxonomy does not recognise is a finding about
+ -- the product and must not be laundered into a canonical one.
+ verdict TEXT,
+ verdict_raw TEXT,
+ confidence DOUBLE PRECISION NOT NULL DEFAULT 0,
+ tier TEXT NOT NULL DEFAULT '',
+
+ -- Reasoning and the evidence it was produced from, so a disputed
+ -- hallucination score can be re-derived rather than re-argued.
+ decision JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+ -- Set when triage refused the finding outright, which is neither a
+ -- verdict nor an abstention and is scored as neither.
+ error TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+COMMENT ON TABLE aisoc_replay_decisions IS
+ 'One replayed finding: the analyst label, the agent verdict, and the evidence '
+ 'the agent was given. Kept so a disputed score can be re-derived.';
+
+CREATE INDEX IF NOT EXISTS idx_replay_evaluations_tenant_created
+ ON aisoc_replay_evaluations (tenant_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_replay_evaluations_tenant_status
+ ON aisoc_replay_evaluations (tenant_id, status);
+CREATE INDEX IF NOT EXISTS idx_replay_decisions_evaluation
+ ON aisoc_replay_decisions (evaluation_id);
+CREATE INDEX IF NOT EXISTS idx_replay_decisions_tenant
+ ON aisoc_replay_decisions (tenant_id, created_at DESC);
+
+-- Row-level security. The `OR current_tenant_id() IS NULL` arm is what lets
+-- the cross-tenant workers (retention purge, tenant deletion) reach these
+-- tables on a connection that binds no tenant; dropping it makes those
+-- silently see nothing. FORCE so the owner does not walk past, because the
+-- owner is the role that runs migrations.
+ALTER TABLE aisoc_replay_evaluations ENABLE ROW LEVEL SECURITY;
+ALTER TABLE aisoc_replay_evaluations FORCE ROW LEVEL SECURITY;
+DROP POLICY IF EXISTS aisoc_replay_evaluations_tenant ON aisoc_replay_evaluations;
+CREATE POLICY aisoc_replay_evaluations_tenant ON aisoc_replay_evaluations
+ USING (tenant_id = current_tenant_id() OR current_tenant_id() IS NULL);
+
+ALTER TABLE aisoc_replay_decisions ENABLE ROW LEVEL SECURITY;
+ALTER TABLE aisoc_replay_decisions FORCE ROW LEVEL SECURITY;
+DROP POLICY IF EXISTS aisoc_replay_decisions_tenant ON aisoc_replay_decisions;
+CREATE POLICY aisoc_replay_decisions_tenant ON aisoc_replay_decisions
+ USING (tenant_id = current_tenant_id() OR current_tenant_id() IS NULL);
+
+-- 061 set ALTER DEFAULT PRIVILEGES so new tables pick these up, but only for
+-- tables created by the role that ran it. Granting explicitly means a chain
+-- replayed by a different owner still leaves the runtime role able to work.
+DO $$
+BEGIN
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'aisoc_app') THEN
+ GRANT SELECT, INSERT, UPDATE, DELETE ON aisoc_replay_evaluations TO aisoc_app;
+ GRANT SELECT, INSERT, UPDATE, DELETE ON aisoc_replay_decisions TO aisoc_app;
+ END IF;
+END
+$$;
diff --git a/services/api/tests/test_replay_evaluation.py b/services/api/tests/test_replay_evaluation.py
new file mode 100644
index 000000000..9213d6a39
--- /dev/null
+++ b/services/api/tests/test_replay_evaluation.py
@@ -0,0 +1,637 @@
+"""Replay evaluation: the credential translation, the chain, the export, the routes.
+
+Gap-closure Phase 1.4 gate.
+
+Four properties, each asserted by name below:
+
+* a connector's stored field names reach the readers as the credential keys
+ the client factory actually reads, in both directions
+* a failure anywhere in the chain is terminal with a reason, never a partial
+ report over whichever findings arrived
+* the export serves the artefact that was stored, and the latency exclusion is
+ the only thing it changes
+* the tenant comes from the credential, and a run belonging to another tenant
+ is not reachable
+
+Nothing here needs a database or a network. The store is a recorded stub and
+the two service calls are a mock transport, so the orchestration can be driven
+through every branch.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+
+import httpx
+import pytest
+from app._vendor.aisoc_benchmark.replay import LATENCY_LINE_PREFIX, format_replay_report, score_replay
+from app.api.v1.deps import CurrentUser, get_current_user
+from app.api.v1.endpoints.evaluations import StartReplayRequest
+from app.api.v1.endpoints.evaluations import router as evaluations_router
+from app.services.replay_evaluation import job as job_module
+from app.services.replay_evaluation import report as report_export
+from app.services.replay_evaluation import store as store_module
+from app.services.replay_evaluation.vendors import (
+ REPLAYABLE_CONNECTORS,
+ UnsupportedConnector,
+ credentials_for,
+ replayable_connector_ids,
+ vendor_for,
+)
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+TENANT = uuid.UUID("aaaaaaaa-0000-0000-0000-00000000000a")
+OTHER_TENANT = uuid.UUID("bbbbbbbb-0000-0000-0000-00000000000b")
+USER = uuid.UUID("11111111-1111-1111-1111-111111111111")
+
+
+# ---------------------------------------------------------------------------
+# The credential translation
+# ---------------------------------------------------------------------------
+
+
+class TestCredentialTranslation:
+ """Two vocabularies meet, and a key under the wrong name fails silently.
+
+ The factory returns ``None`` rather than raising, so the symptom is "no
+ usable credentials", which reads to an operator as a problem on their
+ side. That makes this the mapping most worth pinning.
+ """
+
+ def test_splunk_fields_reach_the_keys_the_factory_reads(self) -> None:
+ credentials = credentials_for(
+ "splunk",
+ {"base_url": "https://splunk:8089", "token": "t"},
+ {"ssl_verify": False, "saved_search": "`my_notables`"},
+ )
+
+ assert credentials["splunk_url"] == "https://splunk:8089"
+ assert credentials["splunk_token"] == "t"
+ assert credentials["search_override"] == "`my_notables`"
+ # False is copied, not dropped. Disabling verification is a deliberate
+ # choice for an internal CA, and dropping it would silently re-enable
+ # verification against a certificate that will not validate.
+ assert credentials["splunk_verify_ssl"] is False
+
+ def test_every_mapped_target_is_a_key_some_factory_actually_reads(self) -> None:
+ """The direction that drifts.
+
+ ``services/actions`` declares the key names its client factory reads.
+ This asserts every target this table produces is in one of those sets,
+ so a factory that renames a key fails here rather than at a customer's
+ first evaluation.
+ """
+ # Mirrored rather than imported: this service cannot import
+ # ``services/actions``. The lists are the ``*_CLIENT_PARAM_KEYS``
+ # tuples in ``app/executors/siem.py`` plus the two reader-only knobs.
+ known = {
+ "splunk_url",
+ "splunk_host",
+ "splunk_token",
+ "splunk_username",
+ "splunk_password",
+ "splunk_verify_ssl",
+ "elastic_url",
+ "elastic_api_key",
+ "elastic_username",
+ "elastic_password",
+ "kibana_url",
+ "sentinel_tenant_id",
+ "sentinel_client_id",
+ "sentinel_client_secret",
+ "sentinel_subscription_id",
+ "sentinel_resource_group",
+ "sentinel_workspace_name",
+ "qradar_url",
+ "qradar_token",
+ "qradar_verify_ssl",
+ "mde_tenant_id",
+ "mde_client_id",
+ "mde_client_secret",
+ # Reader arguments rather than client-constructor keys.
+ "search_override",
+ "index",
+ }
+ for entry in REPLAYABLE_CONNECTORS.values():
+ for target in (*entry.auth_map.values(), *entry.config_map.values()):
+ assert target in known, f"{entry.connector_type} maps to unknown credential key {target}"
+
+ def test_a_field_the_tenant_did_not_store_is_omitted_not_nulled(self) -> None:
+ credentials = credentials_for("splunk", {"base_url": "https://splunk:8089"}, {})
+
+ assert "splunk_token" not in credentials
+ assert "splunk_username" not in credentials
+
+ def test_every_replayable_connector_names_a_reader_arm(self) -> None:
+ for connector_type in replayable_connector_ids():
+ assert vendor_for(connector_type) in {"splunk", "sentinel", "elastic", "qradar", "defender"}
+
+ def test_all_five_reader_arms_are_reachable_from_some_connector(self) -> None:
+ """The reverse direction: a reader nothing can reach is a reader nobody runs."""
+ reached = {entry.vendor for entry in REPLAYABLE_CONNECTORS.values()}
+ assert reached == {"splunk", "sentinel", "elastic", "qradar", "defender"}
+
+ def test_a_connector_with_no_reader_is_refused_by_name(self) -> None:
+ with pytest.raises(UnsupportedConnector) as excinfo:
+ credentials_for("crowdstrike", {"client_id": "x"}, {})
+
+ message = str(excinfo.value)
+ assert "crowdstrike" in message
+ # The refusal lists what *is* supported, so an operator has the next
+ # step rather than only the rejection.
+ assert "splunk" in message
+
+
+# ---------------------------------------------------------------------------
+# The export
+# ---------------------------------------------------------------------------
+
+
+def _decision(index: int, expected: str, verdict: str) -> dict[str, Any]:
+ return {
+ "finding_id": f"ES-{index:03d}",
+ "vendor": "splunk",
+ "rule_id": "rule-42",
+ "closed_at": "2026-03-01T00:00:00+00:00",
+ "expected_disposition": expected,
+ "vendor_disposition": "disposition:1",
+ "labelled": True,
+ "verdict": verdict,
+ "verdict_raw": verdict,
+ "confidence": 0.8,
+ "tier": "deterministic",
+ "findings": ["Observed 198.51.100.7 beaconing"],
+ "confidence_basis": [],
+ "evidence": {"src_ip": "198.51.100.7"},
+ "tool_calls": 0,
+ "resolved_models": [],
+ "tokens": 0,
+ "measured_usd": None,
+ "estimated_usd": None,
+ "unpriced_calls": 0,
+ "latency_ms": index,
+ "error": None,
+ }
+
+
+def _corpus() -> list[dict[str, Any]]:
+ rows = [_decision(i, "true_positive", "true_positive") for i in range(35)]
+ rows += [_decision(100 + i, "false_positive", "false_positive") for i in range(60)]
+ return rows
+
+
+class TestExport:
+ def test_the_markdown_export_is_the_stored_bytes_untouched(self) -> None:
+ stored = "# Replay evaluation\n\n- Decisions replayed: 3\n"
+
+ assert report_export.report_markdown(stored) == stored
+
+ def test_excluding_latency_changes_that_line_and_nothing_else(self) -> None:
+ """The precise form of the reproducibility claim.
+
+ Everything else in the report is a property of the input and the code.
+ The two latency figures measure the host, and they will differ between
+ two runs on the same machine.
+ """
+ report = format_replay_report(score_replay(_corpus()))
+ stripped = report_export.report_markdown(report, exclude_latency=True)
+
+ original = report.split("\n")
+ after = stripped.split("\n")
+ assert len(original) == len(after)
+ differing = [(a, b) for a, b in zip(original, after, strict=True) if a != b]
+ assert len(differing) == 1
+ assert differing[0][0].startswith(LATENCY_LINE_PREFIX)
+ assert differing[0][1].startswith(LATENCY_LINE_PREFIX)
+ assert "not reproducible" in differing[0][1]
+
+ def test_the_json_export_nulls_latency_rather_than_dropping_the_keys(self) -> None:
+ score = score_replay(_corpus()).as_dict()
+ assert score["mean_latency_ms"] is not None
+
+ payload = json.loads(report_export.report_json(evaluation_id="e", score=score, method={}, exclude_latency=True))
+
+ # Present and None, not absent. A reader should not have to tell "this
+ # run measured no latency" from "this key is missing on some exports".
+ assert payload["score"]["mean_latency_ms"] is None
+ assert payload["score"]["p95_latency_ms"] is None
+ assert payload["latency_excluded"] is True
+
+ def test_the_json_export_is_stable_across_two_renders(self) -> None:
+ score = score_replay(_corpus()).as_dict()
+
+ first = report_export.report_json(evaluation_id="e", score=score, method={"a": 1})
+ second = report_export.report_json(evaluation_id="e", score=score, method={"a": 1})
+
+ assert first == second
+
+ def test_model_output_in_the_report_is_escaped_before_it_becomes_html(self) -> None:
+ """The report embeds strings a model produced from attacker-influenced evidence.
+
+ Hallucinated indicator examples are model output by definition, and
+ they reach the PDF. Writing them into HTML unescaped would put that
+ output into a document an operator opens.
+ """
+ markdown = "## Hallucination\n\n- Examples: <script>alert(1)</script>, a&b\n"
+
+ html = report_export.markdown_to_html(markdown)
+
+ assert "<script>" not in html
+ assert "&lt;script&gt;" in html
+ assert "a&amp;b" in html
+
+ def test_the_html_has_no_generation_timestamp(self) -> None:
+ """A report that prints "generated at" cannot reproduce byte for byte."""
+ html = report_export.markdown_to_html("# Replay evaluation\n\n- Decisions replayed: 3\n")
+
+ assert "generated" not in html.lower()
+
+ def test_tables_survive_the_conversion_with_their_header(self) -> None:
+ markdown = "| Disposition | Support |\n|---|---|\n| true_positive | 31 |\n"
+
+ html = report_export.markdown_to_html(markdown)
+
+ assert "<th>Disposition</th>" in html
+ assert "<td>true_positive</td>" in html
+ # The divider row carries no data and must not become a row.
+ assert html.count("<tr>") == 2
+
+
+# ---------------------------------------------------------------------------
+# The chain
+# ---------------------------------------------------------------------------
+
+
+class _RecordingStore:
+ """Stands in for the two tables, recording what the job would have written."""
+
+ def __init__(self) -> None:
+ self.status: list[str] = []
+ self.failure: str | None = None
+ self.result: dict[str, Any] | None = None
+
+ async def mark_running(self, db: Any, *, tenant_id: uuid.UUID, evaluation_id: uuid.UUID) -> None:
+ self.status.append("running")
+
+ async def fail_evaluation(self, db: Any, *, tenant_id: uuid.UUID, evaluation_id: uuid.UUID, error: str) -> None:
+ self.status.append("failed")
+ self.failure = error
+
+ async def store_result(self, db: Any, **fields: Any) -> None:
+ self.status.append("completed")
+ self.result = fields
+
+
+class _StubSession:
+ """Just enough AsyncSession for the job's own calls."""
+
+ def __init__(self, connector: Any = None) -> None:
+ self._connector = connector
+
+ async def execute(self, *args: Any, **kwargs: Any) -> Any:
+ connector = self._connector
+
+ class _Result:
+ def scalar_one_or_none(self) -> Any:
+ return connector
+
+ return _Result()
+
+ async def rollback(self) -> None:
+ return None
+
+
+class _Connector:
+ def __init__(self) -> None:
+ self.connector_type = "splunk"
+ self.auth_config = {"base_url": "https://splunk:8089", "token": "t"}
+ self.connector_config = {"ssl_verify": True}
+
+
+def _request(**overrides: Any) -> job_module.ReplayRequest:
+ defaults: dict[str, Any] = {
+ "tenant_id": TENANT,
+ "evaluation_id": uuid.uuid4(),
+ "connector_row_id": uuid.uuid4(),
+ "connector_type": "splunk",
+ "vendor": "splunk",
+ "window_start": datetime(2026, 3, 1, tzinfo=UTC),
+ "window_end": datetime(2026, 6, 1, tzinfo=UTC),
+ "train_fraction": 0.7,
+ "limit": 1000,
+ "bootstrap_seed": 20260926,
+ "bootstrap_resamples": 50,
+ }
+ return job_module.ReplayRequest(**{**defaults, **overrides})
+
+
+def recording_store(monkeypatch: pytest.MonkeyPatch) -> _RecordingStore:
+ stub = _RecordingStore()
+ monkeypatch.setattr(job_module.store, "mark_running", stub.mark_running)
+ monkeypatch.setattr(job_module.store, "fail_evaluation", stub.fail_evaluation)
+ monkeypatch.setattr(job_module.store, "store_result", stub.store_result)
+ return stub
+
+
+def no_vault(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The vault is exercised by its own suite; here it passes the dict through."""
+
+ class _Vault:
+ def decrypt_dict(self, value: dict[str, Any]) -> dict[str, Any]:
+ return dict(value)
+
+ monkeypatch.setattr(job_module, "get_vault", lambda: _Vault())
+
+
+def _transport(monkeypatch: pytest.MonkeyPatch, handler: Any) -> None:
+ real_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ job_module.httpx,
+ "AsyncClient",
+ lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),
+ )
+
+
+class TestChain:
+ async def test_a_complete_run_scores_and_stores_the_report(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path.endswith("/replay/history"):
+ return httpx.Response(
+ 200,
+ json={"vendor": "splunk", "count": 95, "labelled": 95, "unlabeled": 0, "findings": [{"x": 1}]},
+ )
+ return httpx.Response(200, json={"decisions": _corpus(), "method": {"split": {"train_findings": 66}}})
+
+ _transport(monkeypatch, handler)
+
+ await job_module.run_evaluation(_StubSession(_Connector()), _request())
+
+ assert recording_store.status == ["running", "completed"]
+ result = recording_store.result
+ assert result is not None
+ assert result["report_markdown"].startswith("# Replay evaluation")
+ # The sample sizes the console shows beside the headline come from the
+ # history read, not from the decision count, so the two surfaces
+ # cannot disagree about what was read.
+ assert result["method"]["history"]["findings_read"] == 95
+ assert result["method"]["history"]["findings_labelled"] == 95
+ assert result["findings_read"] == 95
+
+ async def test_an_empty_window_is_a_failure_not_a_report_over_nothing(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ return httpx.Response(200, json={"vendor": "splunk", "count": 0, "labelled": 0, "findings": []})
+
+ _transport(monkeypatch, handler)
+
+ await job_module.run_evaluation(_StubSession(_Connector()), _request())
+
+ assert recording_store.status == ["running", "failed"]
+ assert "no closed findings" in (recording_store.failure or "")
+ assert recording_store.result is None
+
+ async def test_a_history_read_that_fails_is_terminal_with_the_reason(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path.endswith("/replay/history"):
+ return httpx.Response(502, json={"detail": "reading splunk history failed: ConnectError"})
+ raise AssertionError("the replay must not run when the history could not be read")
+
+ _transport(monkeypatch, handler)
+
+ await job_module.run_evaluation(_StubSession(_Connector()), _request())
+
+ assert recording_store.status == ["running", "failed"]
+ assert "502" in (recording_store.failure or "")
+
+ async def test_a_connector_in_another_tenant_is_not_reachable(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ """The row is loaded with a tenant predicate, so it resolves to nothing."""
+ _transport(monkeypatch, lambda request: httpx.Response(200, json={}))
+
+ await job_module.run_evaluation(_StubSession(connector=None), _request(tenant_id=OTHER_TENANT))
+
+ assert recording_store.status == ["running", "failed"]
+ assert "not found" in (recording_store.failure or "")
+
+ async def test_an_unexpected_exception_still_terminates_the_row(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ """A detached job that raises would otherwise leave the row running forever.
+
+ A job that never terminates is indistinguishable from a slow one, and
+ a poller would wait on it until it gave up.
+ """
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ raise RuntimeError("something nobody predicted")
+
+ _transport(monkeypatch, handler)
+
+ await job_module.run_evaluation(_StubSession(_Connector()), _request())
+
+ assert recording_store.status == ["running", "failed"]
+ assert recording_store.failure
+
+ async def test_the_agents_call_declares_the_tenant_it_acts_for(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ """A service token with no tenant header resolves to an empty scope.
+
+ So the header is always sent, and it always carries the tenant the
+ route took from the caller's credential.
+ """
+ seen: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.append(request)
+ if request.url.path.endswith("/replay/history"):
+ return httpx.Response(200, json={"vendor": "splunk", "count": 1, "labelled": 1, "findings": [{"x": 1}]})
+ return httpx.Response(200, json={"decisions": _corpus(), "method": {}})
+
+ _transport(monkeypatch, handler)
+
+ await job_module.run_evaluation(_StubSession(_Connector()), _request())
+
+ replay_calls = [r for r in seen if r.url.path.endswith("/replay/run")]
+ assert replay_calls
+ assert replay_calls[0].headers["X-AiSOC-Tenant-ID"] == str(TENANT)
+
+ async def test_the_credentials_forwarded_are_the_translated_ones(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ recording_store: _RecordingStore,
+ no_vault: None,
+ ) -> None:
+ seen: list[dict[str, Any]] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path.endswith("/replay/history"):
+ seen.append(json.loads(request.content))
+ return httpx.Response(200, json={"vendor": "splunk", "count": 1, "labelled": 1, "findings": [{"x": 1}]})
+ return httpx.Response(200, json={"decisions": _corpus(), "method": {}})
+
+ _transport(monkeypatch, handler)
+
+ await job_module.run_evaluation(_StubSession(_Connector()), _request())
+
+ assert seen
+ assert seen[0]["credentials"]["splunk_url"] == "https://splunk:8089"
+ assert "base_url" not in seen[0]["credentials"]
+
+
+# ---------------------------------------------------------------------------
+# The routes
+# ---------------------------------------------------------------------------
+
+
+def stub_app(monkeypatch: pytest.MonkeyPatch) -> FastAPI:
+ """Only the evaluations router, with a resolved user.
+
+ Same pattern as ``test_misp_push.py``: the production composition is
+ covered elsewhere, and that these routes refuse an anonymous caller is
+ asserted by ``scripts/check_route_auth.py``.
+ """
+ app = FastAPI()
+ app.include_router(evaluations_router, prefix="/api/v1")
+ app.dependency_overrides[get_current_user] = lambda: CurrentUser(
+ user_id=USER,
+ tenant_id=TENANT,
+ role="admin",
+ scopes=["*"],
+ )
+ return app
+
+
+def test_capabilities_names_only_connectors_with_a_reader(stub_app: FastAPI) -> None:
+ """The console must not offer a source this deployment cannot replay."""
+ from app.db.database import get_db
+
+ stub_app.dependency_overrides[get_db] = lambda: None
+ body = TestClient(stub_app).get("/api/v1/evaluations/replay/capabilities").json()
+
+ assert {c["connector_type"] for c in body["connectors"]} == set(replayable_connector_ids())
+ # The floor travels to the client so the page can say why a headline would
+ # be withheld before a run is started, not only after.
+ assert body["min_malicious_for_headline"] == 30
+
+
+def test_an_evaluation_in_another_tenant_is_a_404(stub_app: FastAPI, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Not a 403: telling the two apart would confirm the id exists."""
+ from app.db.database import get_db
+
+ stub_app.dependency_overrides[get_db] = _StubSession
+
+ async def _none(db: Any, **kwargs: Any) -> None:
+ return None
+
+ monkeypatch.setattr(store_module, "get_evaluation", _none)
+
+ response = TestClient(stub_app).get(f"/api/v1/evaluations/replay/{uuid.uuid4()}")
+
+ assert response.status_code == 404
+
+
+def test_exporting_a_run_that_has_no_report_is_a_409_with_the_reason(stub_app: FastAPI, monkeypatch: pytest.MonkeyPatch) -> None:
+ from app.db.database import get_db
+
+ stub_app.dependency_overrides[get_db] = _StubSession
+ evaluation_id = uuid.uuid4()
+
+ async def _failed(db: Any, **kwargs: Any) -> store_module.EvaluationRow:
+ return store_module.EvaluationRow(
+ id=evaluation_id,
+ tenant_id=TENANT,
+ status="failed",
+ error="the actions service is unreachable",
+ connector_id="c",
+ vendor="splunk",
+ window_start=datetime(2026, 3, 1, tzinfo=UTC),
+ window_end=datetime(2026, 6, 1, tzinfo=UTC),
+ train_fraction=0.7,
+ bootstrap_seed=1,
+ bootstrap_resamples=1,
+ findings_read=0,
+ findings_labelled=0,
+ decisions_recorded=0,
+ graded=0,
+ malicious_support=0,
+ headline_accuracy=None,
+ headline_withheld_reason=None,
+ malicious_recall=None,
+ requested_by=USER,
+ created_at=datetime(2026, 6, 1, tzinfo=UTC),
+ started_at=None,
+ completed_at=None,
+ )
+
+ monkeypatch.setattr(store_module, "get_evaluation", _failed)
+
+ response = TestClient(stub_app).get(f"/api/v1/evaluations/replay/{evaluation_id}/export")
+
+ assert response.status_code == 409
+ assert "unreachable" in response.json()["detail"]
+
+
+def test_no_route_on_this_surface_takes_a_tenant_from_the_caller() -> None:
+ """The tenant is never a path segment, a query parameter or a body field.
+
+ Read off the route objects rather than off the source, so a parameter
+ added through a dependency is covered too.
+ """
+ for route in evaluations_router.routes:
+ assert "tenant" not in getattr(route, "path", "")
+ dependant = getattr(route, "dependant", None)
+ if dependant is None:
+ continue
+ for parameter in [*dependant.query_params, *dependant.path_params]:
+ assert "tenant" not in parameter.name
+
+ for model in (StartReplayRequest,):
+ assert not any("tenant" in name for name in model.model_fields)
+
+
+def test_the_withheld_headline_is_null_rather_than_zero() -> None:
+ """A zero in an accuracy column says the agent got every answer wrong.
+
+ "The window held too few malicious cases to print one" is a different
+ fact with a different remedy, and the two must not render the same.
+ """
+ thin = [_decision(i, "true_positive", "true_positive") for i in range(5)]
+ score = score_replay(thin)
+
+ assert score.headline_accuracy is None
+ assert score.headline_withheld_reason
+ assert "below the 30" in score.headline_withheld_reason
diff --git a/tests/e2e/test_replay_cli_end_to_end.py b/tests/e2e/test_replay_cli_end_to_end.py
new file mode 100644
index 000000000..af40d82dc
--- /dev/null
+++ b/tests/e2e/test_replay_cli_end_to_end.py
@@ -0,0 +1,606 @@
+"""The CLI, three services and a mocked Splunk ES: the phase's acceptance bar.
+
+Gap-closure Phase 1.4.
+
+What this proves
+----------------
+Phase 1's "Done when" reads: *the CLI, run against a mocked Splunk ES holding
+200 recorded closed notables, produces a report that reproduces byte for byte
+on a second run with the deterministic model path.*
+
+Three of its four links were already proven by unit suites:
+
+* mocked Splunk to 200 closed findings, in
+ ``services/actions/tests/test_alert_history.py``
+* findings to decisions, reproducibly, in
+ ``services/agents/tests/test_replay_runner.py``
+* decisions to a byte-identical report, in
+ ``packages/aisoc-benchmark/tests/test_replay_metrics.py``
+
+The fourth is CLI to API to report, end to end, and it is what this file is.
+Three proven links are not the same claim as one end-to-end run.
+
+Why it spawns processes instead of importing
+--------------------------------------------
+No Python process can hold two of these services. ``services/actions`` owns
+the SIEM credential path, ``services/agents`` owns triage and
+``services/connectors`` owns ``normalize()``; all three package their code as
+top-level ``app``, so a process importing two of them would import two modules
+called ``app`` and get one of them. That is the same constraint that put the
+orchestration in the API rather than in the CLI, and it means an end-to-end
+run is four processes, a database and a mock vendor.
+
+They are started from the source tree with uvicorn rather than from images, so
+this proves the code in the working tree rather than whatever was last
+published.
+
+Exactly one field is excluded
+-----------------------------
+Mean and p95 latency measure the machine the replay ran on and will differ
+between two runs on one host. ``--exclude-latency`` asks the API for the
+report with those two figures replaced by a note; everything else is a
+property of the input and the code. The claim is therefore stated precisely:
+byte for byte apart from wall-clock latency. ``test_only_latency_differs``
+below asserts that the exclusion is the *only* thing standing between the two
+runs, by comparing the unexcluded reports too.
+
+Running it
+----------
+Needs a Postgres with the migration chain applied::
+
+ docker run -d --name aisoc-e2e-pg -e POSTGRES_USER=aisoc \\
+ -e POSTGRES_PASSWORD=aisoc -e POSTGRES_DB=aisoc -p 55433:5432 postgres:16
+ export AISOC_E2E_DATABASE_MIGRATION_URL=postgresql+asyncpg://aisoc:[email protected]:55433/aisoc
+ export AISOC_APP_DB_PASSWORD=aisoc_app
+ export AISOC_E2E_DATABASE_URL=postgresql+asyncpg://aisoc_app:[email protected]:55433/aisoc
+ python -m pytest tests/e2e/test_replay_cli_end_to_end.py
+
+Skips when no database is configured, so a local ``pytest tests/`` stays
+green. It **cannot** skip where it is supposed to run: ``integration.yml``
+sets ``AISOC_REPLAY_E2E_REQUIRED=1`` and an unreachable database is then a
+failure. A gate that quietly declines to run looks identical to one that
+passed, and this repository has shipped that exact shape more than once.
+"""
+
+from __future__ import annotations
+
+import http.server
+import json
+import os
+import socket
+import subprocess
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+#: What the four services connect as. This is deliberately **not** the owner:
+#: `061_runtime_app_role.sql` moves the deployment onto a DML-only `aisoc_app`
+#: that the row-level-security policies apply to, and a superuser ignores every
+#: policy in the schema even under `FORCE ROW LEVEL SECURITY`. Running the
+#: end-to-end proof as the owner would measure a role nothing ships.
+DSN = os.environ.get("AISOC_E2E_DATABASE_URL", "").strip()
+
+#: The owner. Used for the migration chain and for creating the administrator,
+#: both of which need privileges the runtime role deliberately lacks. Falls
+#: back to `DSN` so a single-role setup still runs.
+MIGRATION_DSN = os.environ.get("AISOC_E2E_DATABASE_MIGRATION_URL", "").strip() or DSN
+
+REQUIRED = os.environ.get("AISOC_REPLAY_E2E_REQUIRED", "").strip() not in ("", "0", "false")
+
+pytestmark = pytest.mark.skipif(
+ not DSN and not REQUIRED,
+ reason="needs a live Postgres with the migration chain applied (integration.yml)",
+)
+
+#: Long enough that a report reproducing is evidence rather than coincidence,
+#: and the number the phase's acceptance bar names.
+NOTABLE_COUNT = 200
+
+#: Fixed so the window, the split and therefore the report do not depend on
+#: the day the test runs. An unpinned window is a different window on a second
+#: run, which is a different report for a reason that has nothing to do with
+#: determinism.
+WINDOW_START = datetime(2026, 3, 1, tzinfo=UTC)
+WINDOW_END = datetime(2026, 6, 1, tzinfo=UTC)
+
+SERVICE_TOKEN = "replay-e2e-service-token"
+SECRET_KEY = "replay-e2e-secret-key-at-least-32-characters"
+
+#: A real administrator, created by the deployment's own bootstrap script and
+#: signed in through the real login route. The dev-mode auth bypass would have
+#: been one line shorter and would have proved less: it skips the permission
+#: checks these routes declare (`connectors:write` to start a run,
+#: `reports:read` to read one), so a run under it could not tell a correctly
+#: gated surface from an ungated one.
+ADMIN_EMAIL = "[email protected]"
+ADMIN_PASSWORD = "replay-e2e-Password-1234"
+
+
+# ---------------------------------------------------------------------------
+# The recorded Splunk ES history
+# ---------------------------------------------------------------------------
+
+
+def _notables() -> list[dict[str, str]]:
+ """200 closed notables, in the shape ``list_closed_notables`` selects.
+
+ **Synthetic.** Hand-built to the field list the reader's SPL asks for.
+ None of it came from a customer.
+
+ The dispositions are laid out so the *test* window (the later 30% of the
+ time split) holds more than the 30 malicious cases the report requires
+ before it will print a headline accuracy. That is deliberate: a run whose
+ headline is withheld would still reproduce byte for byte, and would prove
+ less, because the withheld branch prints far fewer numbers.
+ """
+ rows: list[dict[str, str]] = []
+ for index in range(NOTABLE_COUNT):
+ closed_at = WINDOW_START + timedelta(hours=index * 6)
+ # Two in three are true positives, which puts ~40 malicious cases in
+ # the 60-finding test window.
+ disposition = "disposition:1" if index % 3 != 2 else "disposition:3"
+ rows.append(
+ {
+ "event_id": f"ES-{index:04d}",
+ "rule_id": f"rule-{index % 7}",
+ "rule_name": "Suspicious PowerShell execution",
+ "search_name": "Suspicious PowerShell execution",
+ "urgency": "high" if index % 2 == 0 else "medium",
+ "disposition": disposition,
+ "review_time": str(int(closed_at.timestamp())),
+ "reviewer": f"analyst-{index % 4}",
+ "comment": "Reviewed and closed.",
+ "src": f"198.51.100.{index % 250}",
+ "host": f"WS-{index % 50:03d}",
+ "_time": str(int(closed_at.timestamp())),
+ }
+ )
+ return rows
+
+
+class _SplunkHandler(http.server.BaseHTTPRequestHandler):
+ """The two REST endpoints ``SplunkClient.run_search`` actually calls."""
+
+ rows: list[dict[str, str]] = []
+
+ def log_message(self, *args: object) -> None: # noqa: D102 - silence the default access log
+ return
+
+ def _json(self, payload: dict[str, object]) -> None:
+ body = json.dumps(payload).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler's naming
+ if self.path.startswith("/services/search/jobs"):
+ self.rfile.read(int(self.headers.get("Content-Length", 0) or 0))
+ self._json({"sid": "e2e-search-1"})
+ return
+ self.send_error(404)
+
+ def do_GET(self) -> None: # noqa: N802
+ if "/results" in self.path:
+ self._json({"results": self.rows})
+ return
+ self.send_error(404)
+
+
+# ---------------------------------------------------------------------------
+# Process plumbing
+# ---------------------------------------------------------------------------
+
+
+def _free_port() -> int:
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
+def _wait_for(url: str, *, timeout: float = 90.0, name: str = "") -> None:
+ """Poll until a service answers, or fail naming which one did not.
+
+ Any HTTP status counts as up, including a 404. The four services do not
+ agree on a health path, and waiting for one specific path would make this
+ a test of route naming rather than of whether the process is serving. An
+ ASGI app that returns 404 has started, parsed its configuration and bound
+ its port, which is the whole question here.
+
+ A timeout is reported as "this service never came up", never as a skip. A
+ harness that skips when a dependency is missing reports success for a run
+ that measured nothing.
+ """
+ deadline = time.monotonic() + timeout
+ last = ""
+ while time.monotonic() < deadline:
+ try:
+ with urllib.request.urlopen(url, timeout=2) as response: # noqa: S310 - fixed localhost URL
+ if response.status < 500:
+ return
+ except urllib.error.HTTPError:
+ # It answered. That is the signal.
+ return
+ except (urllib.error.URLError, OSError, TimeoutError) as exc:
+ last = str(exc)
+ time.sleep(0.3)
+ raise AssertionError(f"{name or url} did not become reachable within {timeout:.0f}s (last error: {last})")
+
+
+class _Service:
+ """One uvicorn process, started from the source tree."""
+
+ def __init__(self, name: str, directory: Path, port: int, env: dict[str, str]) -> None:
+ self.name = name
+ self.port = port
+ self.log = open( # noqa: SIM115 - held for the process lifetime, closed in stop()
+ REPO_ROOT / f".e2e-{name}.log", "w", encoding="utf-8"
+ )
+ self.process = subprocess.Popen( # noqa: S603 - fixed argv, no shell
+ [sys.executable, "-m", "uvicorn", "app.main:app", "--host", "127.0.0.1", "--port", str(port)],
+ cwd=str(directory),
+ env={**os.environ, "PYTHONPATH": ".", **env},
+ stdout=self.log,
+ stderr=subprocess.STDOUT,
+ )
+
+ @property
+ def url(self) -> str:
+ return f"http://127.0.0.1:{self.port}"
+
+ def wait(self, path: str = "/health") -> None:
+ if self.process.poll() is not None:
+ raise AssertionError(f"{self.name} exited immediately; see .e2e-{self.name}.log")
+ _wait_for(f"{self.url}{path}", name=self.name)
+
+ def stop(self) -> None:
+ self.process.terminate()
+ try:
+ self.process.wait(timeout=15)
+ except subprocess.TimeoutExpired: # pragma: no cover - only on a wedged process
+ self.process.kill()
+ self.log.close()
+
+
[email protected](scope="module")
+def splunk() -> Iterator[str]:
+ _SplunkHandler.rows = _notables()
+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _SplunkHandler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_address[1]}"
+ finally:
+ server.shutdown()
+ server.server_close()
+
+
[email protected](scope="module")
+def stack(splunk: str) -> Iterator[dict[str, str]]:
+ """actions, connectors, agents and the API, from the working tree."""
+ if not DSN:
+ raise AssertionError(
+ "AISOC_REPLAY_E2E_REQUIRED is set but AISOC_E2E_DATABASE_URL is not. "
+ "The end-to-end proof cannot run without a database, and reporting success "
+ "for a run that measured nothing is the failure this guard exists to prevent."
+ )
+
+ # DDL and user creation run as the owner; everything the services do runs
+ # as the runtime role. Collapsing the two is what let 92 RLS policies sit
+ # inert for months elsewhere in this repository.
+ owner_env = {
+ "PYTHONPATH": ".",
+ "DATABASE_URL": MIGRATION_DSN,
+ "DATABASE_MIGRATION_URL": MIGRATION_DSN,
+ "SECRET_KEY": SECRET_KEY,
+ }
+ migrate = subprocess.run( # noqa: S603 - fixed argv, no shell
+ [sys.executable, "-m", "app.scripts.run_migrations"],
+ cwd=str(REPO_ROOT / "services" / "api"),
+ env={**os.environ, **owner_env},
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ assert migrate.returncode == 0, f"migrations failed:\n{migrate.stdout}\n{migrate.stderr}"
+
+ admin = subprocess.run( # noqa: S603 - fixed argv, no shell
+ [sys.executable, "-m", "app.scripts.bootstrap_admin", "--password-stdin", "--reset-password"],
+ cwd=str(REPO_ROOT / "services" / "api"),
+ env={**os.environ, **owner_env, "AISOC_ADMIN_EMAIL": ADMIN_EMAIL},
+ input=ADMIN_PASSWORD,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ assert admin.returncode == 0, f"bootstrap_admin failed:\n{admin.stdout}\n{admin.stderr}"
+
+ ports = {name: _free_port() for name in ("actions", "connectors", "agents", "api")}
+ shared = {
+ "AISOC_DEV_MODE": "1",
+ "AISOC_SERVICE_TOKEN": SERVICE_TOKEN,
+ "SECRET_KEY": SECRET_KEY,
+ # The deterministic model path. No key, no network, and the same
+ # verdict for the same evidence every time.
+ "AISOC_DETERMINISTIC": "1",
+ "AISOC_CORS_ORIGINS": "http://127.0.0.1",
+ }
+
+ services = [
+ _Service(
+ "actions",
+ REPO_ROOT / "services" / "actions",
+ ports["actions"],
+ {**shared, "AISOC_ACTIONS_SERVICE_TOKEN": SERVICE_TOKEN},
+ ),
+ _Service(
+ "connectors",
+ REPO_ROOT / "services" / "connectors",
+ ports["connectors"],
+ {**shared, "AISOC_CONNECTORS_DISABLE_SCHEDULER": "1"},
+ ),
+ _Service(
+ "agents",
+ REPO_ROOT / "services" / "agents",
+ ports["agents"],
+ {**shared, "CONNECTORS_SERVICE_URL": f"http://127.0.0.1:{ports['connectors']}"},
+ ),
+ _Service(
+ "api",
+ REPO_ROOT / "services" / "api",
+ ports["api"],
+ {
+ **shared,
+ "DATABASE_URL": DSN,
+ "DATABASE_MIGRATION_URL": MIGRATION_DSN,
+ "AISOC_ACTIONS_BASE_URL": f"http://127.0.0.1:{ports['actions']}",
+ "AISOC_ACTIONS_SERVICE_TOKEN": SERVICE_TOKEN,
+ "AGENTS_SERVICE_URL": f"http://127.0.0.1:{ports['agents']}",
+ "CONNECTORS_SERVICE_URL": f"http://127.0.0.1:{ports['connectors']}",
+ },
+ ),
+ ]
+ try:
+ for service in services:
+ service.wait()
+ api_url = services[-1].url
+ yield {
+ "api": api_url,
+ "splunk": splunk,
+ "token": _sign_in(api_url),
+ }
+ finally:
+ for service in reversed(services):
+ service.stop()
+
+
+def _sign_in(api_url: str) -> str:
+ """A real access token from the real login route."""
+ request = urllib.request.Request( # noqa: S310 - fixed localhost URL
+ f"{api_url}/api/v1/auth/login",
+ method="POST",
+ data=json.dumps({"email": ADMIN_EMAIL, "password": ADMIN_PASSWORD}).encode(),
+ headers={"Content-Type": "application/json"},
+ )
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310
+ body = json.loads(response.read())
+ token = body.get("access_token")
+ assert token, f"login returned no access token: {body}"
+ return str(token)
+
+
+def _api(stack: dict[str, str], method: str, path: str, payload: dict | None = None) -> dict:
+ request = urllib.request.Request( # noqa: S310 - fixed localhost URL
+ f"{stack['api']}{path}",
+ method=method,
+ data=json.dumps(payload).encode() if payload is not None else None,
+ headers={"Content-Type": "application/json", "Authorization": f"Bearer {stack['token']}"},
+ )
+ with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310
+ return json.loads(response.read() or b"{}")
+
+
[email protected](scope="module")
+def connector_id(stack: dict[str, str]) -> str:
+ """A saved Splunk connector pointing at the mock, through the real API.
+
+ Created through ``POST /connectors`` rather than inserted, so the
+ credentials are encrypted by the same vault the job decrypts them with. A
+ row written straight into the table would prove the replay works against
+ plaintext nobody stores.
+ """
+ created = _api(
+ stack,
+ "POST",
+ "/api/v1/connectors",
+ {
+ "name": "Splunk ES (end-to-end mock)",
+ "connector_type": "splunk",
+ "category": "siem",
+ "auth_config": {"base_url": stack["splunk"], "token": "e2e-token"},
+ "connector_config": {"ssl_verify": False},
+ },
+ )
+ return str(created["id"])
+
+
+def _run_cli(stack: dict[str, str], connector: str, output: Path) -> str:
+ """One full `aisoc replay`, returning the report it wrote."""
+ result = subprocess.run( # noqa: S603 - fixed argv, no shell
+ [
+ sys.executable,
+ "-m",
+ "aisoc_cli.main",
+ "replay",
+ "--api-url",
+ stack["api"],
+ "--api-key",
+ stack["token"],
+ "--connector-id",
+ connector,
+ "--since",
+ WINDOW_START.isoformat(),
+ "--until",
+ WINDOW_END.isoformat(),
+ "--exclude-latency",
+ "--poll-interval",
+ "1",
+ "--timeout",
+ "900",
+ "--output",
+ str(output),
+ ],
+ cwd=str(REPO_ROOT),
+ env={**os.environ, "PYTHONPATH": str(REPO_ROOT / "packages" / "aisoc-cli" / "src")},
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ assert result.returncode == 0, f"aisoc replay failed:\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
+ return output.read_text()
+
+
[email protected](scope="module")
+def two_runs(stack: dict[str, str], connector_id: str, tmp_path_factory: pytest.TempPathFactory) -> list[str]:
+ """The same evaluation, twice, through the CLI.
+
+ Module-scoped because each run replays 60 findings through the full chain
+ and the whole point is to compare two of them, not to do it once per
+ assertion.
+ """
+ directory = tmp_path_factory.mktemp("replay-e2e")
+ return [
+ _run_cli(stack, connector_id, directory / "first.md"),
+ _run_cli(stack, connector_id, directory / "second.md"),
+ ]
+
+
+# ---------------------------------------------------------------------------
+# The acceptance bar
+# ---------------------------------------------------------------------------
+
+
+def test_the_report_reproduces_byte_for_byte(two_runs: list[str]) -> None:
+ """Phase 1's "Done when", end to end.
+
+ Two invocations of the CLI, against a mocked Splunk ES holding 200
+ recorded closed notables, through the API, the actions service, the
+ connectors service and the agents service, on the deterministic model
+ path.
+ """
+ first, second = two_runs
+
+ assert first == second
+ assert first.encode() == second.encode()
+
+
+def test_the_report_describes_the_history_it_was_given(two_runs: list[str]) -> None:
+ """A report that reproduces but measures nothing would pass the test above.
+
+ So the artefact is checked for the numbers it is supposed to carry: the
+ 200 findings read, a graded test window, and a headline over a corpus deep
+ enough for one.
+ """
+ report = two_runs[0]
+
+ assert report.startswith("# Replay evaluation")
+ assert "Decisions replayed: 60" in report
+ assert '"findings_read": 200' in report
+ # 70/30 by time over 200 findings.
+ assert '"test_findings": 60' in report
+ assert '"train_findings": 140' in report
+
+
+def test_the_headline_is_printed_rather_than_withheld(two_runs: list[str]) -> None:
+ """The deeper branch of the renderer, so the comparison covers more of it.
+
+ A withheld headline reproduces just as reliably and prints far fewer
+ numbers, so a run that was accidentally thin would weaken the proof
+ without failing it.
+ """
+ report = two_runs[0]
+
+ assert "## Headline accuracy" in report
+ assert "Withheld." not in report
+ assert "answered decisions" in report
+
+
+def test_only_latency_differs_between_the_two_runs(stack: dict[str, str], connector_id: str, two_runs: list[str]) -> None:
+ """The exclusion is the only thing standing between the runs, and it is one line.
+
+ Fetching both reports again *without* the exclusion and diffing them shows
+ exactly which lines are not reproducible. If that set is ever larger than
+ the single latency line, the claim has quietly stopped being true and this
+ fails rather than the byte comparison passing on a stripped artefact that
+ hid it.
+ """
+ runs = _api(stack, "GET", "/api/v1/evaluations/replay?limit=2")
+ assert len(runs) == 2, "expected exactly the two runs this module started"
+
+ full = []
+ for run in runs:
+ request = urllib.request.Request( # noqa: S310 - fixed localhost URL
+ f"{stack['api']}/api/v1/evaluations/replay/{run['id']}/export?format=markdown",
+ headers={"Authorization": f"Bearer {stack['token']}"},
+ )
+ with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310
+ full.append(response.read().decode())
+
+ differing = [(a, b) for a, b in zip(full[0].split("\n"), full[1].split("\n"), strict=True) if a != b]
+ assert len(differing) <= 1, f"more than latency differs between two runs: {differing}"
+ if differing:
+ assert differing[0][0].startswith("- Mean latency:")
+
+
+def test_the_run_wrote_nothing_through_triage(stack: dict[str, str]) -> None:
+ """Shadow mode, asserted on the live chain rather than on the runner alone.
+
+ The method block records what production *would* have written, per sink.
+ Those counters being present and non-zero is the evidence that the
+ writeback branch was reached and intercepted, rather than never reached.
+ """
+ runs = _api(stack, "GET", "/api/v1/evaluations/replay?limit=1")
+ detail = _api(stack, "GET", f"/api/v1/evaluations/replay/{runs[0]['id']}")
+
+ attempted = detail["method"]["shadow_writes_attempted"]
+ assert attempted, "no write was even attempted, so shadow mode proved nothing"
+ assert set(attempted) >= {"persist_auto_triage", "write_back_disposition"}
+
+
+def test_the_decisions_behind_the_report_were_stored(stack: dict[str, str]) -> None:
+ """A disputed number has to be re-derivable, which needs the rows."""
+ runs = _api(stack, "GET", "/api/v1/evaluations/replay?limit=1")
+ decisions = _api(stack, "GET", f"/api/v1/evaluations/replay/{runs[0]['id']}/decisions")
+
+ assert len(decisions) == 60
+ assert all(d["vendor"] == "splunk" for d in decisions)
+ assert {d["expected_disposition"] for d in decisions} <= {"true_positive", "false_positive", "unlabeled"}
+
+
+def test_the_json_export_reproduces_too(stack: dict[str, str]) -> None:
+ """The other machine-readable half of the same artefact."""
+ runs = _api(stack, "GET", "/api/v1/evaluations/replay?limit=2")
+
+ exports = []
+ for run in runs:
+ request = urllib.request.Request( # noqa: S310 - fixed localhost URL
+ f"{stack['api']}/api/v1/evaluations/replay/{run['id']}/export?format=json&exclude_latency=true",
+ headers={"Authorization": f"Bearer {stack['token']}"},
+ )
+ with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310
+ body = json.loads(response.read())
+ # The id is the one field that legitimately differs between two runs.
+ body.pop("evaluation_id")
+ exports.append(json.dumps(body, sort_keys=True))
+
+ assert exports[0] == exports[1]