Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Replay evaluation: the credential translation, the chain, the export, the routes.
Gap-closure Phase 1.4 gate.
Four properties, each asserted by name below:
* a connector's stored field names reach the readers as the credential keys
the client factory actually reads, in both directions
* a failure anywhere in the chain is terminal with a reason, never a partial
report over whichever findings arrived
* the export serves the artefact that was stored, and the latency exclusion is
the only thing it changes
* the tenant comes from the credential, and a run belonging to another tenant
is not reachable
Nothing here needs a database or a network. The store is a recorded stub and
the two service calls are a mock transport, so the orchestration can be driven
through every branch.
"""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime
from typing import Any
import httpx
import pytest
from app._vendor.aisoc_benchmark.replay import LATENCY_LINE_PREFIXES, format_replay_report, score_replay
from app.api.v1.deps import CurrentUser, get_current_user
from app.api.v1.endpoints.evaluations import StartReplayRequest
from app.api.v1.endpoints.evaluations import router as evaluations_router
from app.services.replay_evaluation import job as job_module
from app.services.replay_evaluation import report as report_export
from app.services.replay_evaluation import store as store_module
from app.services.replay_evaluation.vendors import (
REPLAYABLE_CONNECTORS,
UnsupportedConnector,
credentials_for,
replayable_connector_ids,
vendor_for,
)
from fastapi import FastAPI
from fastapi.testclient import TestClient
TENANT = uuid.UUID("aaaaaaaa-0000-0000-0000-00000000000a")
OTHER_TENANT = uuid.UUID("bbbbbbbb-0000-0000-0000-00000000000b")
USER = uuid.UUID("11111111-1111-1111-1111-111111111111")
# ---------------------------------------------------------------------------
# The credential translation
# ---------------------------------------------------------------------------
class TestCredentialTranslation:
"""Two vocabularies meet, and a key under the wrong name fails silently.
The factory returns ``None`` rather than raising, so the symptom is "no
usable credentials", which reads to an operator as a problem on their
side. That makes this the mapping most worth pinning.
"""
def test_splunk_fields_reach_the_keys_the_factory_reads(self) -> None:
credentials = credentials_for(
"splunk",
{"base_url": "https://splunk:8089", "token": "t"},
{"ssl_verify": False, "saved_search": "`my_notables`"},
)
assert credentials["splunk_url"] == "https://splunk:8089"
assert credentials["splunk_token"] == "t"
assert credentials["search_override"] == "`my_notables`"
# False is copied, not dropped. Disabling verification is a deliberate
# choice for an internal CA, and dropping it would silently re-enable
# verification against a certificate that will not validate.
assert credentials["splunk_verify_ssl"] is False
def test_every_mapped_target_is_a_key_some_factory_actually_reads(self) -> None:
"""The direction that drifts.
``services/actions`` declares the key names its client factory reads.
This asserts every target this table produces is in one of those sets,
so a factory that renames a key fails here rather than at a customer's
first evaluation.
"""
# Mirrored rather than imported: this service cannot import
# ``services/actions``. The lists are the ``*_CLIENT_PARAM_KEYS``
# tuples in ``app/executors/siem.py`` plus the two reader-only knobs.
known = {
"splunk_url",
"splunk_host",
"splunk_token",
"splunk_username",
"splunk_password",
"splunk_verify_ssl",
"elastic_url",
"elastic_api_key",
"elastic_username",
"elastic_password",
"kibana_url",
"sentinel_tenant_id",
"sentinel_client_id",
"sentinel_client_secret",
"sentinel_subscription_id",
"sentinel_resource_group",
"sentinel_workspace_name",
"qradar_url",
"qradar_token",
"qradar_verify_ssl",
"mde_tenant_id",
"mde_client_id",
"mde_client_secret",
# Reader arguments rather than client-constructor keys.
"search_override",
"index",
}
for entry in REPLAYABLE_CONNECTORS.values():
for target in (*entry.auth_map.values(), *entry.config_map.values()):
assert target in known, f"{entry.connector_type} maps to unknown credential key {target}"
def test_a_field_the_tenant_did_not_store_is_omitted_not_nulled(self) -> None:
credentials = credentials_for("splunk", {"base_url": "https://splunk:8089"}, {})
assert "splunk_token" not in credentials
assert "splunk_username" not in credentials
def test_every_replayable_connector_names_a_reader_arm(self) -> None:
for connector_type in replayable_connector_ids():
assert vendor_for(connector_type) in {"splunk", "sentinel", "elastic", "qradar", "defender"}
def test_all_five_reader_arms_are_reachable_from_some_connector(self) -> None:
"""The reverse direction: a reader nothing can reach is a reader nobody runs."""
reached = {entry.vendor for entry in REPLAYABLE_CONNECTORS.values()}
assert reached == {"splunk", "sentinel", "elastic", "qradar", "defender"}
def test_a_connector_with_no_reader_is_refused_by_name(self) -> None:
with pytest.raises(UnsupportedConnector) as excinfo:
credentials_for("crowdstrike", {"client_id": "x"}, {})
message = str(excinfo.value)
assert "crowdstrike" in message
# The refusal lists what *is* supported, so an operator has the next
# step rather than only the rejection.
assert "splunk" in message
# ---------------------------------------------------------------------------
# The export
# ---------------------------------------------------------------------------
def _decision(index: int, expected: str, verdict: str) -> dict[str, Any]:
return {
"finding_id": f"ES-{index:03d}",
"vendor": "splunk",
"rule_id": "rule-42",
"closed_at": "2026-03-01T00:00:00+00:00",
"expected_disposition": expected,
"vendor_disposition": "disposition:1",
"labelled": True,
"verdict": verdict,
"verdict_raw": verdict,
"confidence": 0.8,
"tier": "deterministic",
"findings": ["Observed 198.51.100.7 beaconing"],
"confidence_basis": [],
"evidence": {"src_ip": "198.51.100.7"},
"tool_calls": 0,
"resolved_models": [],
"tokens": 0,
"measured_usd": None,
"estimated_usd": None,
"unpriced_calls": 0,
"latency_ms": index,
"error": None,
}
def _corpus() -> list[dict[str, Any]]:
rows = [_decision(i, "true_positive", "true_positive") for i in range(35)]
rows += [_decision(100 + i, "false_positive", "false_positive") for i in range(60)]
return rows
class TestExport:
def test_the_markdown_export_is_the_stored_bytes_untouched(self) -> None:
stored = "# Replay evaluation\n\n- Decisions replayed: 3\n"
assert report_export.report_markdown(stored) == stored
def test_excluding_latency_changes_that_line_and_nothing_else(self) -> None:
"""The precise form of the reproducibility claim.
Everything else in the report is a property of the input and the code.
The two latency figures measure the host, and they will differ between
two runs on the same machine.
"""
report = format_replay_report(score_replay(_corpus()))
stripped = report_export.report_markdown(report, exclude_latency=True)
original = report.split("\n")
after = stripped.split("\n")
assert len(original) == len(after)
differing = [(a, b) for a, b in zip(original, after, strict=True) if a != b]
# A property, not a count. The report carries more than one
# host-measured line since the scorecard added "time to verdict",
# and what has to hold is that the export touches *only* lines the
# stripper declares -- not that there happens to be one of them.
assert differing, "the export changed nothing while claiming to exclude latency"
for before, line in differing:
assert before.startswith(LATENCY_LINE_PREFIXES), f"excluded a line it does not declare: {before!r}"
assert line.startswith(LATENCY_LINE_PREFIXES)
assert "not reproducible" in line
def test_the_json_export_nulls_latency_rather_than_dropping_the_keys(self) -> None:
score = score_replay(_corpus()).as_dict()
assert score["mean_latency_ms"] is not None
payload = json.loads(report_export.report_json(evaluation_id="e", score=score, method={}, exclude_latency=True))
# Present and None, not absent. A reader should not have to tell "this
# run measured no latency" from "this key is missing on some exports".
assert payload["score"]["mean_latency_ms"] is None
assert payload["score"]["p95_latency_ms"] is None
assert payload["latency_excluded"] is True
def test_the_json_export_is_stable_across_two_renders(self) -> None:
score = score_replay(_corpus()).as_dict()
first = report_export.report_json(evaluation_id="e", score=score, method={"a": 1})
second = report_export.report_json(evaluation_id="e", score=score, method={"a": 1})
assert first == second
def test_model_output_in_the_report_is_escaped_before_it_becomes_html(self) -> None:
"""The report embeds strings a model produced from attacker-influenced evidence.
Hallucinated indicator examples are model output by definition, and
they reach the PDF. Writing them into HTML unescaped would put that
output into a document an operator opens.
"""
markdown = "## Hallucination\n\n- Examples: <script>alert(1)</script>, a&b\n"
html = report_export.markdown_to_html(markdown)
assert "<script>" not in html
assert "<script>" in html
assert "a&b" in html
def test_the_html_has_no_generation_timestamp(self) -> None:
"""A report that prints "generated at" cannot reproduce byte for byte."""
html = report_export.markdown_to_html("# Replay evaluation\n\n- Decisions replayed: 3\n")
assert "generated" not in html.lower()
def test_tables_survive_the_conversion_with_their_header(self) -> None:
markdown = "| Disposition | Support |\n|---|---|\n| true_positive | 31 |\n"
html = report_export.markdown_to_html(markdown)
assert "<th>Disposition</th>" in html
assert "<td>true_positive</td>" in html
# The divider row carries no data and must not become a row.
assert html.count("<tr>") == 2
# ---------------------------------------------------------------------------
# The chain
# ---------------------------------------------------------------------------
class _RecordingStore:
"""Stands in for the two tables, recording what the job would have written."""
def __init__(self) -> None:
self.status: list[str] = []
self.failure: str | None = None
self.result: dict[str, Any] | None = None
async def mark_running(self, db: Any, *, tenant_id: uuid.UUID, evaluation_id: uuid.UUID) -> None:
self.status.append("running")
async def fail_evaluation(self, db: Any, *, tenant_id: uuid.UUID, evaluation_id: uuid.UUID, error: str) -> None:
self.status.append("failed")
self.failure = error
async def store_result(self, db: Any, **fields: Any) -> None:
self.status.append("completed")
self.result = fields
class _StubSession:
"""Just enough AsyncSession for the job's own calls."""
def __init__(self, connector: Any = None) -> None:
self._connector = connector
async def begin_nested(self) -> Any:
"""A SAVEPOINT. The job wraps each optional context read in one.
Modelled here rather than left off: a double that cannot do what
the real session can is how a defect reaches production with a
green suite behind it. The frozen-context reads need savepoints
because PostgreSQL aborts the whole transaction on a failed
statement, so catching the error is not enough to keep the later
write alive.
"""
class _Savepoint:
async def rollback(self) -> None:
return None
async def commit(self) -> None:
return None
return _Savepoint()
async def execute(self, *args: Any, **kwargs: Any) -> Any:
connector = self._connector
class _Result:
def scalar_one_or_none(self) -> Any:
return connector
return _Result()
async def rollback(self) -> None:
return None
class _Connector:
def __init__(self) -> None:
self.connector_type = "splunk"
self.auth_config = {"base_url": "https://splunk:8089", "token": "t"}
self.connector_config = {"ssl_verify": True}
def _request(**overrides: Any) -> job_module.ReplayRequest:
defaults: dict[str, Any] = {
"tenant_id": TENANT,
"evaluation_id": uuid.uuid4(),
"connector_row_id": uuid.uuid4(),
"connector_type": "splunk",
"vendor": "splunk",
"window_start": datetime(2026, 3, 1, tzinfo=UTC),
"window_end": datetime(2026, 6, 1, tzinfo=UTC),
"train_fraction": 0.7,
"limit": 1000,
"bootstrap_seed": 20260926,
"bootstrap_resamples": 50,
}
return job_module.ReplayRequest(**{**defaults, **overrides})
@pytest.fixture
def recording_store(monkeypatch: pytest.MonkeyPatch) -> _RecordingStore:
stub = _RecordingStore()
monkeypatch.setattr(job_module.store, "mark_running", stub.mark_running)
monkeypatch.setattr(job_module.store, "fail_evaluation", stub.fail_evaluation)
monkeypatch.setattr(job_module.store, "store_result", stub.store_result)
return stub
@pytest.fixture
def no_vault(monkeypatch: pytest.MonkeyPatch) -> None:
"""The vault is exercised by its own suite; here it passes the dict through."""
class _Vault:
def decrypt_dict(self, value: dict[str, Any]) -> dict[str, Any]:
return dict(value)
monkeypatch.setattr(job_module, "get_vault", lambda: _Vault())
def _transport(monkeypatch: pytest.MonkeyPatch, handler: Any) -> None:
real_client = httpx.AsyncClient
monkeypatch.setattr(
job_module.httpx,
"AsyncClient",
lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),
)
@pytest.mark.asyncio
class TestChain:
async def test_a_complete_run_scores_and_stores_the_report(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path.endswith("/replay/history"):
return httpx.Response(
200,
json={"vendor": "splunk", "count": 95, "labelled": 95, "unlabeled": 0, "findings": [{"x": 1}]},
)
return httpx.Response(200, json={"decisions": _corpus(), "method": {"split": {"train_findings": 66}}})
_transport(monkeypatch, handler)
await job_module.run_evaluation(_StubSession(_Connector()), _request())
assert recording_store.status == ["running", "completed"]
result = recording_store.result
assert result is not None
assert result["report_markdown"].startswith("# Replay evaluation")
# The sample sizes the console shows beside the headline come from the
# history read, not from the decision count, so the two surfaces
# cannot disagree about what was read.
assert result["method"]["history"]["findings_read"] == 95
assert result["method"]["history"]["findings_labelled"] == 95
assert result["findings_read"] == 95
async def test_an_empty_window_is_a_failure_not_a_report_over_nothing(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"vendor": "splunk", "count": 0, "labelled": 0, "findings": []})
_transport(monkeypatch, handler)
await job_module.run_evaluation(_StubSession(_Connector()), _request())
assert recording_store.status == ["running", "failed"]
assert "no closed findings" in (recording_store.failure or "")
assert recording_store.result is None
async def test_a_history_read_that_fails_is_terminal_with_the_reason(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path.endswith("/replay/history"):
return httpx.Response(502, json={"detail": "reading splunk history failed: ConnectError"})
raise AssertionError("the replay must not run when the history could not be read")
_transport(monkeypatch, handler)
await job_module.run_evaluation(_StubSession(_Connector()), _request())
assert recording_store.status == ["running", "failed"]
assert "502" in (recording_store.failure or "")
async def test_a_connector_in_another_tenant_is_not_reachable(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
"""The row is loaded with a tenant predicate, so it resolves to nothing."""
_transport(monkeypatch, lambda request: httpx.Response(200, json={}))
await job_module.run_evaluation(_StubSession(connector=None), _request(tenant_id=OTHER_TENANT))
assert recording_store.status == ["running", "failed"]
assert "not found" in (recording_store.failure or "")
async def test_an_unexpected_exception_still_terminates_the_row(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
"""A detached job that raises would otherwise leave the row running forever.
A job that never terminates is indistinguishable from a slow one, and
a poller would wait on it until it gave up.
"""
def handler(request: httpx.Request) -> httpx.Response:
raise RuntimeError("something nobody predicted")
_transport(monkeypatch, handler)
await job_module.run_evaluation(_StubSession(_Connector()), _request())
assert recording_store.status == ["running", "failed"]
assert recording_store.failure
async def test_the_agents_call_declares_the_tenant_it_acts_for(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
"""A service token with no tenant header resolves to an empty scope.
So the header is always sent, and it always carries the tenant the
route took from the caller's credential.
"""
seen: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
if request.url.path.endswith("/replay/history"):
return httpx.Response(200, json={"vendor": "splunk", "count": 1, "labelled": 1, "findings": [{"x": 1}]})
return httpx.Response(200, json={"decisions": _corpus(), "method": {}})
_transport(monkeypatch, handler)
await job_module.run_evaluation(_StubSession(_Connector()), _request())
replay_calls = [r for r in seen if r.url.path.endswith("/replay/run")]
assert replay_calls
assert replay_calls[0].headers["X-AiSOC-Tenant-ID"] == str(TENANT)
async def test_the_credentials_forwarded_are_the_translated_ones(
self,
monkeypatch: pytest.MonkeyPatch,
recording_store: _RecordingStore,
no_vault: None,
) -> None:
seen: list[dict[str, Any]] = []
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path.endswith("/replay/history"):
seen.append(json.loads(request.content))
return httpx.Response(200, json={"vendor": "splunk", "count": 1, "labelled": 1, "findings": [{"x": 1}]})
return httpx.Response(200, json={"decisions": _corpus(), "method": {}})
_transport(monkeypatch, handler)
await job_module.run_evaluation(_StubSession(_Connector()), _request())
assert seen
assert seen[0]["credentials"]["splunk_url"] == "https://splunk:8089"
assert "base_url" not in seen[0]["credentials"]
# ---------------------------------------------------------------------------
# The routes
# ---------------------------------------------------------------------------
@pytest.fixture
def stub_app(monkeypatch: pytest.MonkeyPatch) -> FastAPI:
"""Only the evaluations router, with a resolved user.
Same pattern as ``test_misp_push.py``: the production composition is
covered elsewhere, and that these routes refuse an anonymous caller is
asserted by ``scripts/check_route_auth.py``.
"""
app = FastAPI()
app.include_router(evaluations_router, prefix="/api/v1")
app.dependency_overrides[get_current_user] = lambda: CurrentUser(
user_id=USER,
tenant_id=TENANT,
email="[email protected]",
role="admin",
scopes=["*"],
)
return app
def test_capabilities_names_only_connectors_with_a_reader(stub_app: FastAPI) -> None:
"""The console must not offer a source this deployment cannot replay."""
from app.db.database import get_db
stub_app.dependency_overrides[get_db] = lambda: None
body = TestClient(stub_app).get("/api/v1/evaluations/replay/capabilities").json()
assert {c["connector_type"] for c in body["connectors"]} == set(replayable_connector_ids())
# The floor travels to the client so the page can say why a headline would
# be withheld before a run is started, not only after.
assert body["min_malicious_for_headline"] == 30
def test_an_evaluation_in_another_tenant_is_a_404(stub_app: FastAPI, monkeypatch: pytest.MonkeyPatch) -> None:
"""Not a 403: telling the two apart would confirm the id exists."""
from app.db.database import get_db
stub_app.dependency_overrides[get_db] = _StubSession
async def _none(db: Any, **kwargs: Any) -> None:
return None
monkeypatch.setattr(store_module, "get_evaluation", _none)
response = TestClient(stub_app).get(f"/api/v1/evaluations/replay/{uuid.uuid4()}")
assert response.status_code == 404
def test_exporting_a_run_that_has_no_report_is_a_409_with_the_reason(stub_app: FastAPI, monkeypatch: pytest.MonkeyPatch) -> None:
from app.db.database import get_db
stub_app.dependency_overrides[get_db] = _StubSession
evaluation_id = uuid.uuid4()
async def _failed(db: Any, **kwargs: Any) -> store_module.EvaluationRow:
return store_module.EvaluationRow(
id=evaluation_id,
tenant_id=TENANT,
status="failed",
error="the actions service is unreachable",
connector_id="c",
vendor="splunk",
window_start=datetime(2026, 3, 1, tzinfo=UTC),
window_end=datetime(2026, 6, 1, tzinfo=UTC),
train_fraction=0.7,
bootstrap_seed=1,
bootstrap_resamples=1,
findings_read=0,
findings_labelled=0,
decisions_recorded=0,
graded=0,
malicious_support=0,
headline_accuracy=None,
headline_withheld_reason=None,
malicious_recall=None,
requested_by=USER,
created_at=datetime(2026, 6, 1, tzinfo=UTC),
started_at=None,
completed_at=None,
)
monkeypatch.setattr(store_module, "get_evaluation", _failed)
response = TestClient(stub_app).get(f"/api/v1/evaluations/replay/{evaluation_id}/export")
assert response.status_code == 409
assert "unreachable" in response.json()["detail"]
def test_no_route_on_this_surface_takes_a_tenant_from_the_caller() -> None:
"""The tenant is never a path segment, a query parameter or a body field.
Read off the route objects rather than off the source, so a parameter
added through a dependency is covered too.
"""
for route in evaluations_router.routes:
assert "tenant" not in getattr(route, "path", "")
dependant = getattr(route, "dependant", None)
if dependant is None:
continue
for parameter in [*dependant.query_params, *dependant.path_params]:
assert "tenant" not in parameter.name
for model in (StartReplayRequest,):
assert not any("tenant" in name for name in model.model_fields)
def test_the_withheld_headline_is_null_rather_than_zero() -> None:
"""A zero in an accuracy column says the agent got every answer wrong.
"The window held too few malicious cases to print one" is a different
fact with a different remedy, and the two must not render the same.
"""
thin = [_decision(i, "true_positive", "true_positive") for i in range(5)]
score = score_replay(thin)
assert score.headline_accuracy is None
assert score.headline_withheld_reason
assert "below the 30" in score.headline_withheld_reason