Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(replay): a replay carries the context production triages with (fix pass 3.1) (#1186)

Beenu Aroraf99086847a41
diff --git a/services/agents/app/api/replay_router.py b/services/agents/app/api/replay_router.py
index 4d2098dde..fc7556bd5 100644
--- a/services/agents/app/api/replay_router.py
+++ b/services/agents/app/api/replay_router.py
@@ -31,6 +31,7 @@ an absence. Those counters travel into the report's method section.
from __future__ import annotations
+from datetime import UTC, datetime
from typing import Annotated, Any
import structlog
@@ -52,6 +53,7 @@ from app.security.tenant_scope import (
require_console_or_service_auth,
scoped_tenant_or_403,
)
+from app.workers.business_context import BusinessContextApplier, load_rules_from_yaml
logger = structlog.get_logger()
@@ -65,6 +67,59 @@ ScopedPrincipal = Annotated[TenantPrincipal, Depends(require_console_or_service_
MAX_FINDINGS = 2000
+def _business_context_applier(context: FrozenContext | None, *, split_at: datetime) -> BusinessContextApplier | None:
+ """The tenant's rules as an applier, or ``None`` when none should apply.
+
+ Four reasons to return ``None``, each of which would otherwise be a
+ quiet wrong answer rather than an error:
+
+ * no rule set was captured -- absent is not an empty rule set;
+ * the tenant disabled theirs, and a replay must not opt them back in;
+ * it was last edited **after** the split, so applying it would grade a
+ window using hindsight about that window;
+ * it does not parse, which the production worker also treats as "apply
+ none" rather than failing triage.
+
+ An undated rule set is kept, matching how ``capture_context`` treats an
+ undated statement: under-applying context is the defect this item
+ closes, so the lenient direction is the recorded one.
+ """
+ raw = (context.business_context if context else None) or {}
+ if not raw or not raw.get("enabled"):
+ return None
+
+ edited = raw.get("updated_at")
+ if isinstance(edited, str) and edited.strip():
+ try:
+ parsed = datetime.fromisoformat(edited.strip().replace("Z", "+00:00"))
+ except ValueError:
+ parsed = None
+ if parsed is not None:
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=UTC)
+ if parsed > split_at:
+ logger.info("replay.business_context.dropped_after_split", edited_at=edited)
+ return None
+
+ yaml_text = str(raw.get("yaml_text") or "")
+ if not yaml_text.strip():
+ return None
+ try:
+ rules = load_rules_from_yaml(yaml_text)
+ except Exception as exc: # noqa: BLE001 - a bad rule set applies none, as in the worker
+ logger.warning("replay.business_context.unparseable", error=str(exc))
+ return None
+ if not rules:
+ return None
+
+ applier = BusinessContextApplier()
+ # Seeded directly rather than left to read Postgres: the snapshot is the
+ # frozen copy, and a live read would reintroduce exactly the post-split
+ # edit this function just filtered out.
+ applier.seed_tenant_rules(rules)
+ return applier
+
+
class FrozenContext(BaseModel):
"""Organisation memory and outcome priors as the caller captured them.
@@ -89,6 +144,17 @@ class FrozenContext(BaseModel):
skills: list[dict[str, Any]] = Field(default_factory=list)
skills_under_test: list[dict[str, Any]] = Field(default_factory=list, max_length=8)
+ #: The tenant's authored business-context rule set, as stored: the YAML
+ #: verbatim, whether it is enabled, and when it was last edited.
+ #:
+ #: Production applies these on every alert
+ #: (`BusinessContextApplier` in `main.py`), so a replay that does not is
+ #: grading a different agent. They travel in the snapshot rather than
+ #: being read live for the same reason everything else here does: a rule
+ #: edited after the split must not decide a verdict about a window that
+ #: closed before it.
+ business_context: dict[str, Any] | None = None
+
class ReplayRequest(BaseModel):
"""One window of closed findings, and the connector whose mapping to use."""
@@ -176,6 +242,10 @@ async def run_replay(body: ReplayRequest, principal: ScopedPrincipal) -> ReplayR
normalizer=normalizer,
tenant_id=str(tenant_id),
connector_id=body.connector_id,
+ # Production triages with the tenant's rules applied (`main.py`), so
+ # a replay without them grades an agent nobody runs. Frozen at the
+ # split by `_business_context_applier`.
+ business_context=_business_context_applier(body.context, split_at=split.split_at),
)
try:
diff --git a/services/agents/app/workers/business_context.py b/services/agents/app/workers/business_context.py
index 4fa76e16a..3cf2a19cc 100644
--- a/services/agents/app/workers/business_context.py
+++ b/services/agents/app/workers/business_context.py
@@ -317,12 +317,17 @@ class BusinessContextApplier:
self._rules: list[BusinessContextRule] = []
self._mtime: float | None = None
self._tenant_rules: dict[str, tuple[float, list[BusinessContextRule]]] = {}
+ #: Set by `seed_tenant_rules` for the replay path. When present, these
+ #: replace the Postgres read entirely -- see that method.
+ self._seeded_rules: list[BusinessContextRule] | None = None
self._load()
# ── tenant rules (Postgres) ───────────────────────────────────────────
async def _load_tenant_rules(self, tenant_id: str) -> list[BusinessContextRule]:
"""Read and cache the tenant's authored rules. Never raises."""
+ if self._seeded_rules is not None:
+ return self._seeded_rules
cached = self._tenant_rules.get(tenant_id)
now = time.monotonic()
if cached is not None and now - cached[0] < _TENANT_CACHE_TTL_SECONDS:
@@ -390,6 +395,20 @@ class BusinessContextApplier:
"""Drop cached tenant rules. Used by tests."""
self._tenant_rules.clear()
+ def seed_tenant_rules(self, rules: list[BusinessContextRule]) -> None:
+ """Apply these rules to every tenant, instead of reading Postgres.
+
+ For the replay path (fix pass 3.1), which holds a rule set frozen at
+ the split point and must not re-read the live one: a read would
+ reintroduce an edit made after the window being graded, which is the
+ leakage the frozen context exists to prevent.
+
+ Seeded under a sentinel rather than per tenant id because a replay
+ runs one tenant at a time and the applier is constructed per run, so
+ there is no second tenant whose rules this could reach.
+ """
+ self._seeded_rules = list(rules)
+
def _load(self) -> None:
if not self._path:
self._rules = []
diff --git a/services/agents/tests/test_replay_business_context.py b/services/agents/tests/test_replay_business_context.py
new file mode 100644
index 000000000..c34299cf8
--- /dev/null
+++ b/services/agents/tests/test_replay_business_context.py
@@ -0,0 +1,103 @@
+"""A replay applies the tenant's business-context rules, as production does.
+
+Fix pass 3.1, agents half. `main.py` constructs a `BusinessContextApplier`
+whenever the feature is enabled -- which is the default -- and hands it to
+the triage worker, so every production verdict is reached with the tenant's
+rules applied. The replay route constructed `ReplayRunner` with no
+`business_context` at all, so a replay graded the agent in an estate with no
+crown jewels, no known-noisy hosts and no suppressions.
+
+That is not a neutral omission. Business-context rules mostly raise or lower
+severity on specific assets, so a replay without them scores a different
+agent than the one the tenant runs, and the report does not say so.
+
+The rules travel in the frozen context rather than being read live, for the
+same reason every other part of that snapshot does: a rule set edited after
+the split must not decide a verdict about a window that closed before it.
+`capture_context` applies the filter, so the route only has to hand the rows
+over with their recorded time intact.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from app.api.replay_router import FrozenContext, _business_context_applier
+
+#: The grammar the production worker parses: `id`, `when`, `then`. Written
+#: out rather than built by a helper so a change to the grammar breaks this
+#: test instead of silently passing an empty rule list.
+_YAML = """
+rules:
+ - id: crown-jewels
+ when:
+ host: db-prod-1
+ then:
+ set_severity: critical
+"""
+
+
+class TestTheRunnerGetsTheTenantsRules:
+ def test_an_enabled_rule_set_becomes_an_applier(self) -> None:
+ context = FrozenContext(
+ business_context={
+ "yaml_text": _YAML,
+ "enabled": True,
+ "updated_at": "2026-01-20T00:00:00+00:00",
+ }
+ )
+
+ applier = _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC))
+
+ assert applier is not None
+
+ def test_no_rule_set_means_no_applier(self) -> None:
+ """Absent is not the same as empty, and neither invents rules."""
+ assert _business_context_applier(FrozenContext(), split_at=datetime(2026, 3, 1, tzinfo=UTC)) is None
+
+ def test_a_disabled_rule_set_is_not_applied(self) -> None:
+ """A tenant who turned their rules off is not silently opted back in."""
+ context = FrozenContext(business_context={"yaml_text": _YAML, "enabled": False, "updated_at": "2026-01-20T00:00:00+00:00"})
+
+ assert _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC)) is None
+
+ def test_a_rule_set_edited_after_the_split_is_dropped(self) -> None:
+ """The same leakage rule every other part of the snapshot follows.
+
+ A rule authored with hindsight about the window being graded would
+ make the score a statement about the author, not the agent.
+ """
+ split = datetime(2026, 3, 1, tzinfo=UTC)
+ context = FrozenContext(
+ business_context={
+ "yaml_text": _YAML,
+ "enabled": True,
+ "updated_at": (split + timedelta(days=5)).isoformat(),
+ }
+ )
+
+ assert _business_context_applier(context, split_at=split) is None
+
+ def test_an_undated_rule_set_is_kept(self) -> None:
+ """Matches how `capture_context` treats an undated statement.
+
+ Dropping it would silently under-apply context, which is the error
+ this item exists to fix; keeping it is recorded behaviour rather
+ than an accident.
+ """
+ context = FrozenContext(business_context={"yaml_text": _YAML, "enabled": True})
+
+ assert _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC)) is not None
+
+ def test_unparseable_yaml_does_not_break_the_replay(self) -> None:
+ """A bad rule set applies none, exactly as the worker does."""
+ context = FrozenContext(business_context={"yaml_text": "rules: [[[", "enabled": True, "updated_at": "2026-01-20T00:00:00+00:00"})
+
+ assert _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC)) is None
+
+
[email protected]("field", ["statements", "priors", "business_context"])
+def test_the_frozen_context_model_carries_every_part(field: str) -> None:
+ """The wire contract the API half of 3.1 writes to."""
+ assert field in FrozenContext.model_fields
diff --git a/services/api/app/services/replay_evaluation/job.py b/services/api/app/services/replay_evaluation/job.py
index 96b3b6e0c..3410a6895 100644
--- a/services/api/app/services/replay_evaluation/job.py
+++ b/services/api/app/services/replay_evaluation/job.py
@@ -43,15 +43,18 @@ are columns rather than report content.
from __future__ import annotations
+import json
import os
import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import datetime
from typing import Any
import httpx
import structlog
-from sqlalchemy import select
+from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app._vendor.aisoc_benchmark.replay import (
@@ -86,6 +89,18 @@ __all__ = [
_HISTORY_TIMEOUT_S = 180.0
_REPLAY_TIMEOUT_S = 1800.0
+#: Caps on the frozen context. Production's triage prompt reads the same
+#: ceiling for statements (`analyst_feedback.active_statements`), and a prior
+#: list is bounded because a tenant with a long history would otherwise push
+#: a multi-megabyte body through a call that already carries the findings.
+_STATEMENT_LIMIT = 200
+_PRIOR_LIMIT = 2000
+
+#: Written by `app.services.human_priors` and by the agents-side
+#: `app.memory.outcomes`; both spell the key this way. The replay sends the
+#: bare signature, because that is what the agents side looks a prior up by.
+_OUTCOME_KEY_PREFIX = "outcome:"
+
class ReplayJobError(Exception):
"""A link in the chain failed. The message is what lands on the row.
@@ -218,22 +233,162 @@ async def _read_history(request: ReplayRequest, credentials: dict[str, Any]) ->
return body
-async def _run_replay(request: ReplayRequest, connector_type: str, findings: list[dict[str, Any]]) -> dict[str, Any]:
+@dataclass(frozen=True)
+class FrozenContextCapture:
+ """What production would have had, read at capture time with its dates.
+
+ The split filter is **not** applied here. Each row travels with its own
+ recorded time and `capture_context` on the agents side drops the late
+ ones, so "what counts as after the split" is decided in one place rather
+ than re-implemented either side of an HTTP call -- the drift this
+ repository keeps finding between two surfaces over the same rows.
+
+ `unread` names each part that could not be read. A replay must not die
+ because institutional memory was briefly unavailable, but it must not
+ quietly report a score produced with less context than it claims either,
+ so the gap is published in the method note.
+ """
+
+ statements: list[dict[str, Any]]
+ priors: dict[str, dict[str, Any]]
+ business_context: dict[str, Any] | None
+ unread: tuple[str, ...]
+
+ def as_payload(self, request: ReplayRequest) -> dict[str, Any]:
+ return {
+ "statements": self.statements,
+ "priors": self.priors,
+ "business_context": self.business_context,
+ "skills": list(request.skills or ()),
+ "skills_under_test": [request.skill_under_test] if request.skill_under_test else [],
+ }
+
+
+@asynccontextmanager
+async def _optional_read(db: AsyncSession, part: str, unread: list[str]) -> AsyncIterator[None]:
+ """Run one context read so a failure costs only that read.
+
+ The savepoint is the whole point. PostgreSQL aborts the **entire**
+ transaction on a failed statement and refuses everything after it with
+ `current transaction is aborted, commands ignored until end of
+ transaction block` -- so catching the exception and carrying on does not
+ isolate the failure, it poisons the write that stores the report. A
+ plain try/except here did exactly that: the replay scored correctly and
+ then died on its own `UPDATE`, which reads as a storage bug rather than
+ as a missing table.
+
+ `begin_nested` issues a SAVEPOINT and rolls back to it, which is the
+ only construct that confines a failed statement on this engine. Same
+ finding as `tests/test_audit_chain_survives_a_failed_write.py`.
+ """
+ savepoint = await db.begin_nested()
+ try:
+ yield
+ except Exception as exc: # noqa: BLE001 - fail soft, and say which part
+ await savepoint.rollback()
+ log.warning(f"replay.context.{part}_unread", error=str(exc))
+ unread.append(part)
+ else:
+ await savepoint.commit()
+
+
+def _jsonable(row: Any) -> dict[str, Any]:
+ """A row as JSON the agents service can parse.
+
+ Datetimes are rendered ISO rather than dropped: the recorded time is
+ exactly what the split filter reads on the other side, so losing it
+ would silently make every row undated and therefore kept.
+ """
+ out: dict[str, Any] = {}
+ for key, value in dict(row).items():
+ out[key] = value.isoformat() if hasattr(value, "isoformat") else value
+ return out
+
+
+async def _capture_frozen_context(db: AsyncSession, request: ReplayRequest) -> FrozenContextCapture:
+ """Organisation memory, outcome priors and the tenant's business rules.
+
+ Production triages every alert with all three
+ (`services/agents/app/workers/fused_alert_consumer.py` and
+ `main.py`), so a replay that omits them is not measuring production. The
+ direction of that error is the reason it matters: memory and priors
+ mostly *suppress*, so a bare replay re-raises alerts production would
+ have closed.
+ """
+ statements: list[dict[str, Any]] = []
+ priors: dict[str, dict[str, Any]] = {}
+ business_context: dict[str, Any] | None = None
+ unread: list[str] = []
+
+ async with _optional_read(db, "statements", unread):
+ rows = await db.execute(
+ text(
+ "SELECT statement, reason_code, scope, scope_value, observations, updated_at "
+ "FROM aisoc_context_statements "
+ "WHERE tenant_id = CAST(:t AS uuid) "
+ " AND (expires_at IS NULL OR expires_at > :split) "
+ "ORDER BY observations DESC, updated_at DESC "
+ "LIMIT :lim"
+ ).bindparams(t=str(request.tenant_id), split=request.window_end, lim=_STATEMENT_LIMIT)
+ )
+ statements = [_jsonable(row) for row in rows.mappings()]
+
+ async with _optional_read(db, "priors", unread):
+ rows = await db.execute(
+ text(
+ "SELECT key, value FROM aisoc_institutional_memory WHERE tenant_id = CAST(:t AS uuid) AND key LIKE :prefix LIMIT :lim"
+ ).bindparams(t=str(request.tenant_id), prefix=f"{_OUTCOME_KEY_PREFIX}%", lim=_PRIOR_LIMIT)
+ )
+ for row in rows.mappings():
+ key = str(row["key"])
+ value = row["value"]
+ if isinstance(value, str):
+ try:
+ value = json.loads(value)
+ except ValueError:
+ continue
+ if isinstance(value, dict):
+ priors[key[len(_OUTCOME_KEY_PREFIX) :]] = _jsonable(value)
+
+ async with _optional_read(db, "business_context", unread):
+ rows = await db.execute(
+ text(
+ "SELECT yaml_text, enabled, updated_at FROM aisoc_business_context_rule_sets WHERE tenant_id = CAST(:t AS uuid)"
+ ).bindparams(t=str(request.tenant_id))
+ )
+ row = next(iter(rows.mappings()), None)
+ if row is not None:
+ updated_at = row["updated_at"]
+ business_context = {
+ "yaml_text": row["yaml_text"] or "",
+ "enabled": bool(row["enabled"]),
+ "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at,
+ }
+
+ return FrozenContextCapture(
+ statements=statements,
+ priors=priors,
+ business_context=business_context,
+ unread=tuple(unread),
+ )
+
+
+async def _run_replay(
+ request: ReplayRequest,
+ connector_type: str,
+ findings: list[dict[str, Any]],
+ context: FrozenContextCapture,
+) -> dict[str, Any]:
url = f"{_agents_base_url()}/api/v1/replay/run"
+ # Always sent. Before fix-pass 3.1 the block went only when a caller
+ # passed skills, so the ordinary replay ran with an empty snapshot while
+ # the docs described one carrying organisation memory.
payload: dict[str, Any] = {
"connector_id": connector_type,
"findings": findings,
"train_fraction": request.train_fraction,
+ "context": context.as_payload(request),
}
- if request.skills is not None or request.skill_under_test is not None:
- # Sent only when a caller asked for it. An ordinary replay still sends
- # no context block at all, so its snapshot is the empty one documented
- # on `FrozenContext` rather than an empty-but-present one that would
- # read differently in the method note.
- payload["context"] = {
- "skills": list(request.skills or ()),
- "skills_under_test": [request.skill_under_test] if request.skill_under_test else [],
- }
try:
async with httpx.AsyncClient(timeout=_REPLAY_TIMEOUT_S) as client:
response = await client.post(url, headers=_agents_headers(request.tenant_id), json=payload)
@@ -267,9 +422,15 @@ async def run_evaluation(db: AsyncSession, request: ReplayRequest) -> None:
"a result over zero findings would describe the window rather than the agent."
)
- run = await _run_replay(request, connector_type, findings)
+ context = await _capture_frozen_context(db, request)
+ run = await _run_replay(request, connector_type, findings, context)
decisions = run["decisions"]
method = dict(run.get("method") or {})
+ if context.unread:
+ # Published rather than logged: a score produced with less context
+ # than the method section describes is the defect this capture
+ # exists to close, and a silent partial capture would recreate it.
+ method["frozen_context_unread"] = list(context.unread)
score = score_replay(
decisions,
diff --git a/services/api/tests/test_replay_evaluation.py b/services/api/tests/test_replay_evaluation.py
index 9213d6a39..c96952a9e 100644
--- a/services/api/tests/test_replay_evaluation.py
+++ b/services/api/tests/test_replay_evaluation.py
@@ -292,6 +292,26 @@ class _StubSession:
def __init__(self, connector: Any = None) -> None:
self._connector = connector
+ async def begin_nested(self) -> Any:
+ """A SAVEPOINT. The job wraps each optional context read in one.
+
+ Modelled here rather than left off: a double that cannot do what
+ the real session can is how a defect reaches production with a
+ green suite behind it. The frozen-context reads need savepoints
+ because PostgreSQL aborts the whole transaction on a failed
+ statement, so catching the error is not enough to keep the later
+ write alive.
+ """
+
+ class _Savepoint:
+ async def rollback(self) -> None:
+ return None
+
+ async def commit(self) -> None:
+ return None
+
+ return _Savepoint()
+
async def execute(self, *args: Any, **kwargs: Any) -> Any:
connector = self._connector
diff --git a/services/api/tests/test_replay_frozen_context.py b/services/api/tests/test_replay_frozen_context.py
new file mode 100644
index 000000000..f6d0396f8
--- /dev/null
+++ b/services/api/tests/test_replay_frozen_context.py
@@ -0,0 +1,343 @@
+"""The replay sends the context production triages with, frozen at the split.
+
+Fix pass 3.1. A replay grades the agent against closed findings, and the
+report it produces is read as "this is how triage would have done". That
+reading is only true if the replay runs with what production runs with.
+
+Two halves were missing, and both flatter or distort the result rather than
+erroring:
+
+* `job.py` sent a context block **only** when a caller passed tenant skills,
+ so an ordinary replay sent none at all. Organisation memory
+ (`aisoc_context_statements`), outcome priors
+ (`aisoc_institutional_memory`, `outcome:` keys) and the tenant's
+ business-context rule set were never captured, while production reads all
+ three on every alert. `replay.md`, `triage-context.md` and
+ `tenant-skills.md` all describe a replay that carries them.
+* The agents-side runner was constructed with no `business_context`, while
+ `main.py` constructs a `BusinessContextApplier` whenever the feature is
+ enabled -- which is by default.
+
+The direction of the error matters. Organisation memory and priors mostly
+*suppress*, so a replay without them re-raises alerts production would have
+closed and scores the agent against a world it does not run in.
+
+What is deliberately **not** sent: anything recorded after the split. The
+capture here reads rows with their recorded time attached and the agents
+side drops the late ones (`capture_context` in
+`services/agents/app/replay/shadow.py`), so the filtering happens in one
+place rather than being re-implemented on each side of the call.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime
+from typing import Any
+
+import httpx
+import pytest
+from app.services.replay_evaluation import job as job_module
+
+TENANT = uuid.UUID("11111111-1111-1111-1111-111111111111")
+
+
+class _RecordingStore:
+ def __init__(self) -> None:
+ self.failure: str | None = None
+ self.result: dict[str, Any] | None = None
+
+ async def mark_running(self, db: Any, **_: Any) -> None:
+ return None
+
+ async def fail_evaluation(self, db: Any, *, error: str, **_: Any) -> None:
+ self.failure = error
+
+ async def store_result(self, db: Any, **fields: Any) -> None:
+ self.result = fields
+
+
+class _Connector:
+ connector_type = "splunk"
+ auth_config = {"base_url": "https://splunk:8089", "token": "t"}
+ connector_config = {"ssl_verify": True}
+
+
+class _ContextSession:
+ """A session that answers the three context reads and the connector read.
+
+ Routed on the SQL text rather than on call order: the job is free to
+ reorder its reads, and a positional stub would then answer the wrong
+ query while still passing.
+ """
+
+ def __init__(self, *, statements: list[dict[str, Any]], priors: list[Any], rule_set: Any) -> None:
+ self._statements = statements
+ self._priors = priors
+ self._rule_set = rule_set
+ self.queries: list[str] = []
+ self.savepoints = 0
+ self.savepoint_rollbacks = 0
+
+ async def begin_nested(self) -> Any:
+ """A SAVEPOINT, recorded so a test can assert one was actually taken."""
+ self.savepoints += 1
+ session = self
+
+ class _Savepoint:
+ async def rollback(self) -> None:
+ session.savepoint_rollbacks += 1
+
+ async def commit(self) -> None:
+ return None
+
+ return _Savepoint()
+
+ async def execute(self, statement: Any, *args: Any, **kwargs: Any) -> Any:
+ sql = str(statement)
+ self.queries.append(sql)
+ rows: list[Any] = []
+ scalar: Any = None
+ if "aisoc_context_statements" in sql:
+ rows = self._statements
+ elif "aisoc_institutional_memory" in sql:
+ rows = self._priors
+ elif "aisoc_business_context_rule_sets" in sql:
+ rows = [self._rule_set] if self._rule_set is not None else []
+ else:
+ scalar = _Connector()
+
+ class _Result:
+ def scalar_one_or_none(self) -> Any:
+ return scalar
+
+ def mappings(self) -> Any:
+ return rows
+
+ def __iter__(self) -> Any:
+ return iter(rows)
+
+ return _Result()
+
+ async def rollback(self) -> None:
+ return None
+
+
+def _request(**overrides: Any) -> job_module.ReplayRequest:
+ defaults: dict[str, Any] = {
+ "tenant_id": TENANT,
+ "evaluation_id": uuid.uuid4(),
+ "connector_row_id": uuid.uuid4(),
+ "connector_type": "splunk",
+ "vendor": "splunk",
+ "window_start": datetime(2026, 3, 1, tzinfo=UTC),
+ "window_end": datetime(2026, 6, 1, tzinfo=UTC),
+ "train_fraction": 0.7,
+ "limit": 1000,
+ "bootstrap_seed": 20260926,
+ "bootstrap_resamples": 50,
+ }
+ return job_module.ReplayRequest(**{**defaults, **overrides})
+
+
+def no_vault(monkeypatch: pytest.MonkeyPatch) -> None:
+ class _Vault:
+ def decrypt_dict(self, value: dict[str, Any]) -> dict[str, Any]:
+ return dict(value)
+
+ monkeypatch.setattr(job_module, "get_vault", lambda: _Vault())
+
+
+def recording_store(monkeypatch: pytest.MonkeyPatch) -> _RecordingStore:
+ stub = _RecordingStore()
+ monkeypatch.setattr(job_module.store, "mark_running", stub.mark_running)
+ monkeypatch.setattr(job_module.store, "fail_evaluation", stub.fail_evaluation)
+ monkeypatch.setattr(job_module.store, "store_result", stub.store_result)
+ return stub
+
+
+def _capture_replay_payload(monkeypatch: pytest.MonkeyPatch, sink: dict[str, Any]) -> None:
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path.endswith("/replay/history"):
+ return httpx.Response(
+ 200,
+ json={"vendor": "splunk", "count": 2, "labelled": 2, "unlabeled": 0, "findings": [{"x": 1}, {"x": 2}]},
+ )
+ import json as _json
+
+ sink.update(_json.loads(request.content.decode()))
+ return httpx.Response(
+ 200,
+ json={
+ "decisions": [
+ {"finding_id": "a", "predicted": "malicious", "actual": "malicious", "split": "test"},
+ {"finding_id": "b", "predicted": "benign", "actual": "benign", "split": "test"},
+ ],
+ "method": {},
+ },
+ )
+
+ real_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ job_module.httpx,
+ "AsyncClient",
+ lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),
+ )
+
+
+def _session() -> _ContextSession:
+ return _ContextSession(
+ statements=[
+ {
+ "statement": "deploy-bot opens change tickets; its bulk edits are expected",
+ "reason_code": "expected_automation",
+ "scope": "actor",
+ "scope_value": "deploy-bot",
+ "observations": 12,
+ "updated_at": datetime(2026, 2, 1, tzinfo=UTC),
+ }
+ ],
+ priors=[
+ {
+ "key": "outcome:sig-benign-backup",
+ "value": {
+ "disposition": "benign",
+ "author": "human",
+ "confidence": 1.0,
+ "last_seen": "2026-02-10T00:00:00+00:00",
+ },
+ }
+ ],
+ rule_set={
+ "yaml_text": "rules:\n - name: crown jewels\n when: {host: db-prod-1}\n then: {severity: critical}\n",
+ "enabled": True,
+ "updated_at": datetime(2026, 1, 20, tzinfo=UTC),
+ },
+ )
+
+
+class TestTheReplayCarriesProductionContext:
+ async def test_organisation_memory_is_sent(
+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None
+ ) -> None:
+ """Statements in `aisoc_context_statements` reach the agents service.
+
+ Pre-fix this list was absent: the payload carried no `context` key at
+ all unless a caller passed skills.
+ """
+ sent: dict[str, Any] = {}
+ _capture_replay_payload(monkeypatch, sent)
+
+ await job_module.run_evaluation(_session(), _request())
+
+ assert recording_store.failure is None, recording_store.failure
+ statements = sent.get("context", {}).get("statements") or []
+ assert [s["statement"] for s in statements] == ["deploy-bot opens change tickets; its bulk edits are expected"]
+
+ async def test_outcome_priors_are_sent_keyed_by_signature(
+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None
+ ) -> None:
+ """Priors travel under the bare signature, not the `outcome:` storage key.
+
+ The agents side looks a prior up by signature; sending the storage
+ key would put every prior somewhere nothing reads, which is the
+ silent-miss shape `human_priors.py` documents.
+ """
+ sent: dict[str, Any] = {}
+ _capture_replay_payload(monkeypatch, sent)
+
+ await job_module.run_evaluation(_session(), _request())
+
+ priors = sent.get("context", {}).get("priors") or {}
+ assert list(priors) == ["sig-benign-backup"]
+ assert priors["sig-benign-backup"]["author"] == "human"
+
+ async def test_the_business_context_rule_set_is_sent(
+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None
+ ) -> None:
+ """Production applies the tenant's rules on every alert; a replay must too."""
+ sent: dict[str, Any] = {}
+ _capture_replay_payload(monkeypatch, sent)
+
+ await job_module.run_evaluation(_session(), _request())
+
+ rules = sent.get("context", {}).get("business_context") or {}
+ assert rules.get("enabled") is True
+ assert "crown jewels" in (rules.get("yaml_text") or "")
+ # Carries its own recorded time so the agents side can drop a rule set
+ # edited after the split, the same way it drops a late statement.
+ assert rules.get("updated_at")
+
+ async def test_a_context_block_is_sent_even_with_no_skills(
+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None
+ ) -> None:
+ """The ordinary replay path is the one that was running bare."""
+ sent: dict[str, Any] = {}
+ _capture_replay_payload(monkeypatch, sent)
+
+ await job_module.run_evaluation(_session(), _request())
+
+ assert "context" in sent
+ assert sent["context"].get("skills") == []
+
+ async def test_a_context_read_that_fails_does_not_fail_the_replay(
+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None
+ ) -> None:
+ """Fail-soft, and say so in the method note rather than silently.
+
+ A replay that dies because institutional memory was briefly
+ unreadable is worse than one that runs with less context and records
+ which part it could not read.
+ """
+ sent: dict[str, Any] = {}
+ _capture_replay_payload(monkeypatch, sent)
+
+ class _Broken(_ContextSession):
+ async def execute(self, statement: Any, *args: Any, **kwargs: Any) -> Any:
+ if "aisoc_context_statements" in str(statement):
+ raise RuntimeError("relation does not exist")
+ return await super().execute(statement, *args, **kwargs)
+
+ session = _Broken(statements=[], priors=[], rule_set=None)
+ await job_module.run_evaluation(session, _request())
+
+ assert recording_store.failure is None, recording_store.failure
+ assert recording_store.result is not None
+ unread = (recording_store.result.get("method") or {}).get("frozen_context_unread") or []
+ assert "statements" in unread
+
+ async def test_a_failed_read_is_confined_to_a_savepoint(
+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None
+ ) -> None:
+ """Catching the exception is not enough on PostgreSQL.
+
+ A failed statement aborts the **whole** transaction and the engine
+ refuses everything after it with `current transaction is aborted,
+ commands ignored until end of transaction block` -- so a plain
+ try/except around a context read does not isolate the failure, it
+ poisons the write that stores the report. The replay then scores
+ correctly and dies on its own `UPDATE`, which reads as a storage
+ bug rather than as a missing table.
+
+ That is not hypothetical: it is what the live four-service replay
+ job reported before this was confined, while every offline test
+ passed, because a stub session has no transaction to poison.
+ """
+ sent: dict[str, Any] = {}
+ _capture_replay_payload(monkeypatch, sent)
+
+ class _Broken(_ContextSession):
+ async def execute(self, statement: Any, *args: Any, **kwargs: Any) -> Any:
+ if "aisoc_institutional_memory" in str(statement):
+ raise RuntimeError("relation does not exist")
+ return await super().execute(statement, *args, **kwargs)
+
+ session = _Broken(statements=[], priors=[], rule_set=None)
+ await job_module.run_evaluation(session, _request())
+
+ assert session.savepoints >= 3, "each optional read must take its own savepoint"
+ assert session.savepoint_rollbacks == 1, "only the failed read should have rolled back"
+ assert recording_store.failure is None, recording_store.failure