Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(replay): a replay carries the context production triages with (fix pass 3.1) (#1186)
Beenu Aroraf99086847a41
diff --git a/services/agents/app/api/replay_router.py b/services/agents/app/api/replay_router.pyindex 4d2098dde..fc7556bd5 100644--- a/services/agents/app/api/replay_router.py+++ b/services/agents/app/api/replay_router.py@@ -31,6 +31,7 @@ an absence. Those counters travel into the report's method section.from __future__ import annotations+from datetime import UTC, datetimefrom typing import Annotated, Anyimport structlog@@ -52,6 +53,7 @@ from app.security.tenant_scope import (require_console_or_service_auth,scoped_tenant_or_403,)+from app.workers.business_context import BusinessContextApplier, load_rules_from_yamllogger = structlog.get_logger()@@ -65,6 +67,59 @@ ScopedPrincipal = Annotated[TenantPrincipal, Depends(require_console_or_service_MAX_FINDINGS = 2000+def _business_context_applier(context: FrozenContext | None, *, split_at: datetime) -> BusinessContextApplier | None:+ """The tenant's rules as an applier, or ``None`` when none should apply.++ Four reasons to return ``None``, each of which would otherwise be a+ quiet wrong answer rather than an error:++ * no rule set was captured -- absent is not an empty rule set;+ * the tenant disabled theirs, and a replay must not opt them back in;+ * it was last edited **after** the split, so applying it would grade a+ window using hindsight about that window;+ * it does not parse, which the production worker also treats as "apply+ none" rather than failing triage.++ An undated rule set is kept, matching how ``capture_context`` treats an+ undated statement: under-applying context is the defect this item+ closes, so the lenient direction is the recorded one.+ """+ raw = (context.business_context if context else None) or {}+ if not raw or not raw.get("enabled"):+ return None++ edited = raw.get("updated_at")+ if isinstance(edited, str) and edited.strip():+ try:+ parsed = datetime.fromisoformat(edited.strip().replace("Z", "+00:00"))+ except ValueError:+ parsed = None+ if parsed is not None:+ if parsed.tzinfo is None:+ parsed = parsed.replace(tzinfo=UTC)+ if parsed > split_at:+ logger.info("replay.business_context.dropped_after_split", edited_at=edited)+ return None++ yaml_text = str(raw.get("yaml_text") or "")+ if not yaml_text.strip():+ return None+ try:+ rules = load_rules_from_yaml(yaml_text)+ except Exception as exc: # noqa: BLE001 - a bad rule set applies none, as in the worker+ logger.warning("replay.business_context.unparseable", error=str(exc))+ return None+ if not rules:+ return None++ applier = BusinessContextApplier()+ # Seeded directly rather than left to read Postgres: the snapshot is the+ # frozen copy, and a live read would reintroduce exactly the post-split+ # edit this function just filtered out.+ applier.seed_tenant_rules(rules)+ return applier++class FrozenContext(BaseModel):"""Organisation memory and outcome priors as the caller captured them.@@ -89,6 +144,17 @@ class FrozenContext(BaseModel):skills: list[dict[str, Any]] = Field(default_factory=list)skills_under_test: list[dict[str, Any]] = Field(default_factory=list, max_length=8)+ #: The tenant's authored business-context rule set, as stored: the YAML+ #: verbatim, whether it is enabled, and when it was last edited.+ #:+ #: Production applies these on every alert+ #: (`BusinessContextApplier` in `main.py`), so a replay that does not is+ #: grading a different agent. They travel in the snapshot rather than+ #: being read live for the same reason everything else here does: a rule+ #: edited after the split must not decide a verdict about a window that+ #: closed before it.+ business_context: dict[str, Any] | None = None+class ReplayRequest(BaseModel):"""One window of closed findings, and the connector whose mapping to use."""@@ -176,6 +242,10 @@ async def run_replay(body: ReplayRequest, principal: ScopedPrincipal) -> ReplayRnormalizer=normalizer,tenant_id=str(tenant_id),connector_id=body.connector_id,+ # Production triages with the tenant's rules applied (`main.py`), so+ # a replay without them grades an agent nobody runs. Frozen at the+ # split by `_business_context_applier`.+ business_context=_business_context_applier(body.context, split_at=split.split_at),)try:diff --git a/services/agents/app/workers/business_context.py b/services/agents/app/workers/business_context.pyindex 4fa76e16a..3cf2a19cc 100644--- a/services/agents/app/workers/business_context.py+++ b/services/agents/app/workers/business_context.py@@ -317,12 +317,17 @@ class BusinessContextApplier:self._rules: list[BusinessContextRule] = []self._mtime: float | None = Noneself._tenant_rules: dict[str, tuple[float, list[BusinessContextRule]]] = {}+ #: Set by `seed_tenant_rules` for the replay path. When present, these+ #: replace the Postgres read entirely -- see that method.+ self._seeded_rules: list[BusinessContextRule] | None = Noneself._load()# ── tenant rules (Postgres) ───────────────────────────────────────────async def _load_tenant_rules(self, tenant_id: str) -> list[BusinessContextRule]:"""Read and cache the tenant's authored rules. Never raises."""+ if self._seeded_rules is not None:+ return self._seeded_rulescached = self._tenant_rules.get(tenant_id)now = time.monotonic()if cached is not None and now - cached[0] < _TENANT_CACHE_TTL_SECONDS:@@ -390,6 +395,20 @@ class BusinessContextApplier:"""Drop cached tenant rules. Used by tests."""self._tenant_rules.clear()+ def seed_tenant_rules(self, rules: list[BusinessContextRule]) -> None:+ """Apply these rules to every tenant, instead of reading Postgres.++ For the replay path (fix pass 3.1), which holds a rule set frozen at+ the split point and must not re-read the live one: a read would+ reintroduce an edit made after the window being graded, which is the+ leakage the frozen context exists to prevent.++ Seeded under a sentinel rather than per tenant id because a replay+ runs one tenant at a time and the applier is constructed per run, so+ there is no second tenant whose rules this could reach.+ """+ self._seeded_rules = list(rules)+def _load(self) -> None:if not self._path:self._rules = []diff --git a/services/agents/tests/test_replay_business_context.py b/services/agents/tests/test_replay_business_context.pynew file mode 100644index 000000000..c34299cf8--- /dev/null+++ b/services/agents/tests/test_replay_business_context.py@@ -0,0 +1,103 @@+"""A replay applies the tenant's business-context rules, as production does.++Fix pass 3.1, agents half. `main.py` constructs a `BusinessContextApplier`+whenever the feature is enabled -- which is the default -- and hands it to+the triage worker, so every production verdict is reached with the tenant's+rules applied. The replay route constructed `ReplayRunner` with no+`business_context` at all, so a replay graded the agent in an estate with no+crown jewels, no known-noisy hosts and no suppressions.++That is not a neutral omission. Business-context rules mostly raise or lower+severity on specific assets, so a replay without them scores a different+agent than the one the tenant runs, and the report does not say so.++The rules travel in the frozen context rather than being read live, for the+same reason every other part of that snapshot does: a rule set edited after+the split must not decide a verdict about a window that closed before it.+`capture_context` applies the filter, so the route only has to hand the rows+over with their recorded time intact.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta++import pytest+from app.api.replay_router import FrozenContext, _business_context_applier++#: The grammar the production worker parses: `id`, `when`, `then`. Written+#: out rather than built by a helper so a change to the grammar breaks this+#: test instead of silently passing an empty rule list.+_YAML = """+rules:+ - id: crown-jewels+ when:+ host: db-prod-1+ then:+ set_severity: critical+"""+++class TestTheRunnerGetsTheTenantsRules:+ def test_an_enabled_rule_set_becomes_an_applier(self) -> None:+ context = FrozenContext(+ business_context={+ "yaml_text": _YAML,+ "enabled": True,+ "updated_at": "2026-01-20T00:00:00+00:00",+ }+ )++ applier = _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC))++ assert applier is not None++ def test_no_rule_set_means_no_applier(self) -> None:+ """Absent is not the same as empty, and neither invents rules."""+ assert _business_context_applier(FrozenContext(), split_at=datetime(2026, 3, 1, tzinfo=UTC)) is None++ def test_a_disabled_rule_set_is_not_applied(self) -> None:+ """A tenant who turned their rules off is not silently opted back in."""+ context = FrozenContext(business_context={"yaml_text": _YAML, "enabled": False, "updated_at": "2026-01-20T00:00:00+00:00"})++ assert _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC)) is None++ def test_a_rule_set_edited_after_the_split_is_dropped(self) -> None:+ """The same leakage rule every other part of the snapshot follows.++ A rule authored with hindsight about the window being graded would+ make the score a statement about the author, not the agent.+ """+ split = datetime(2026, 3, 1, tzinfo=UTC)+ context = FrozenContext(+ business_context={+ "yaml_text": _YAML,+ "enabled": True,+ "updated_at": (split + timedelta(days=5)).isoformat(),+ }+ )++ assert _business_context_applier(context, split_at=split) is None++ def test_an_undated_rule_set_is_kept(self) -> None:+ """Matches how `capture_context` treats an undated statement.++ Dropping it would silently under-apply context, which is the error+ this item exists to fix; keeping it is recorded behaviour rather+ than an accident.+ """+ context = FrozenContext(business_context={"yaml_text": _YAML, "enabled": True})++ assert _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC)) is not None++ def test_unparseable_yaml_does_not_break_the_replay(self) -> None:+ """A bad rule set applies none, exactly as the worker does."""+ context = FrozenContext(business_context={"yaml_text": "rules: [[[", "enabled": True, "updated_at": "2026-01-20T00:00:00+00:00"})++ assert _business_context_applier(context, split_at=datetime(2026, 3, 1, tzinfo=UTC)) is None++[email protected]("field", ["statements", "priors", "business_context"])+def test_the_frozen_context_model_carries_every_part(field: str) -> None:+ """The wire contract the API half of 3.1 writes to."""+ assert field in FrozenContext.model_fieldsdiff --git a/services/api/app/services/replay_evaluation/job.py b/services/api/app/services/replay_evaluation/job.pyindex 96b3b6e0c..3410a6895 100644--- a/services/api/app/services/replay_evaluation/job.py+++ b/services/api/app/services/replay_evaluation/job.py@@ -43,15 +43,18 @@ are columns rather than report content.from __future__ import annotations+import jsonimport osimport uuid+from collections.abc import AsyncIterator+from contextlib import asynccontextmanagerfrom dataclasses import dataclassfrom datetime import datetimefrom typing import Anyimport httpximport structlog-from sqlalchemy import select+from sqlalchemy import select, textfrom sqlalchemy.ext.asyncio import AsyncSessionfrom app._vendor.aisoc_benchmark.replay import (@@ -86,6 +89,18 @@ __all__ = [_HISTORY_TIMEOUT_S = 180.0_REPLAY_TIMEOUT_S = 1800.0+#: Caps on the frozen context. Production's triage prompt reads the same+#: ceiling for statements (`analyst_feedback.active_statements`), and a prior+#: list is bounded because a tenant with a long history would otherwise push+#: a multi-megabyte body through a call that already carries the findings.+_STATEMENT_LIMIT = 200+_PRIOR_LIMIT = 2000++#: Written by `app.services.human_priors` and by the agents-side+#: `app.memory.outcomes`; both spell the key this way. The replay sends the+#: bare signature, because that is what the agents side looks a prior up by.+_OUTCOME_KEY_PREFIX = "outcome:"+class ReplayJobError(Exception):"""A link in the chain failed. The message is what lands on the row.@@ -218,22 +233,162 @@ async def _read_history(request: ReplayRequest, credentials: dict[str, Any]) ->return body-async def _run_replay(request: ReplayRequest, connector_type: str, findings: list[dict[str, Any]]) -> dict[str, Any]:+@dataclass(frozen=True)+class FrozenContextCapture:+ """What production would have had, read at capture time with its dates.++ The split filter is **not** applied here. Each row travels with its own+ recorded time and `capture_context` on the agents side drops the late+ ones, so "what counts as after the split" is decided in one place rather+ than re-implemented either side of an HTTP call -- the drift this+ repository keeps finding between two surfaces over the same rows.++ `unread` names each part that could not be read. A replay must not die+ because institutional memory was briefly unavailable, but it must not+ quietly report a score produced with less context than it claims either,+ so the gap is published in the method note.+ """++ statements: list[dict[str, Any]]+ priors: dict[str, dict[str, Any]]+ business_context: dict[str, Any] | None+ unread: tuple[str, ...]++ def as_payload(self, request: ReplayRequest) -> dict[str, Any]:+ return {+ "statements": self.statements,+ "priors": self.priors,+ "business_context": self.business_context,+ "skills": list(request.skills or ()),+ "skills_under_test": [request.skill_under_test] if request.skill_under_test else [],+ }+++@asynccontextmanager+async def _optional_read(db: AsyncSession, part: str, unread: list[str]) -> AsyncIterator[None]:+ """Run one context read so a failure costs only that read.++ The savepoint is the whole point. PostgreSQL aborts the **entire**+ transaction on a failed statement and refuses everything after it with+ `current transaction is aborted, commands ignored until end of+ transaction block` -- so catching the exception and carrying on does not+ isolate the failure, it poisons the write that stores the report. A+ plain try/except here did exactly that: the replay scored correctly and+ then died on its own `UPDATE`, which reads as a storage bug rather than+ as a missing table.++ `begin_nested` issues a SAVEPOINT and rolls back to it, which is the+ only construct that confines a failed statement on this engine. Same+ finding as `tests/test_audit_chain_survives_a_failed_write.py`.+ """+ savepoint = await db.begin_nested()+ try:+ yield+ except Exception as exc: # noqa: BLE001 - fail soft, and say which part+ await savepoint.rollback()+ log.warning(f"replay.context.{part}_unread", error=str(exc))+ unread.append(part)+ else:+ await savepoint.commit()+++def _jsonable(row: Any) -> dict[str, Any]:+ """A row as JSON the agents service can parse.++ Datetimes are rendered ISO rather than dropped: the recorded time is+ exactly what the split filter reads on the other side, so losing it+ would silently make every row undated and therefore kept.+ """+ out: dict[str, Any] = {}+ for key, value in dict(row).items():+ out[key] = value.isoformat() if hasattr(value, "isoformat") else value+ return out+++async def _capture_frozen_context(db: AsyncSession, request: ReplayRequest) -> FrozenContextCapture:+ """Organisation memory, outcome priors and the tenant's business rules.++ Production triages every alert with all three+ (`services/agents/app/workers/fused_alert_consumer.py` and+ `main.py`), so a replay that omits them is not measuring production. The+ direction of that error is the reason it matters: memory and priors+ mostly *suppress*, so a bare replay re-raises alerts production would+ have closed.+ """+ statements: list[dict[str, Any]] = []+ priors: dict[str, dict[str, Any]] = {}+ business_context: dict[str, Any] | None = None+ unread: list[str] = []++ async with _optional_read(db, "statements", unread):+ rows = await db.execute(+ text(+ "SELECT statement, reason_code, scope, scope_value, observations, updated_at "+ "FROM aisoc_context_statements "+ "WHERE tenant_id = CAST(:t AS uuid) "+ " AND (expires_at IS NULL OR expires_at > :split) "+ "ORDER BY observations DESC, updated_at DESC "+ "LIMIT :lim"+ ).bindparams(t=str(request.tenant_id), split=request.window_end, lim=_STATEMENT_LIMIT)+ )+ statements = [_jsonable(row) for row in rows.mappings()]++ async with _optional_read(db, "priors", unread):+ rows = await db.execute(+ text(+ "SELECT key, value FROM aisoc_institutional_memory WHERE tenant_id = CAST(:t AS uuid) AND key LIKE :prefix LIMIT :lim"+ ).bindparams(t=str(request.tenant_id), prefix=f"{_OUTCOME_KEY_PREFIX}%", lim=_PRIOR_LIMIT)+ )+ for row in rows.mappings():+ key = str(row["key"])+ value = row["value"]+ if isinstance(value, str):+ try:+ value = json.loads(value)+ except ValueError:+ continue+ if isinstance(value, dict):+ priors[key[len(_OUTCOME_KEY_PREFIX) :]] = _jsonable(value)++ async with _optional_read(db, "business_context", unread):+ rows = await db.execute(+ text(+ "SELECT yaml_text, enabled, updated_at FROM aisoc_business_context_rule_sets WHERE tenant_id = CAST(:t AS uuid)"+ ).bindparams(t=str(request.tenant_id))+ )+ row = next(iter(rows.mappings()), None)+ if row is not None:+ updated_at = row["updated_at"]+ business_context = {+ "yaml_text": row["yaml_text"] or "",+ "enabled": bool(row["enabled"]),+ "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at,+ }++ return FrozenContextCapture(+ statements=statements,+ priors=priors,+ business_context=business_context,+ unread=tuple(unread),+ )+++async def _run_replay(+ request: ReplayRequest,+ connector_type: str,+ findings: list[dict[str, Any]],+ context: FrozenContextCapture,+) -> dict[str, Any]:url = f"{_agents_base_url()}/api/v1/replay/run"+ # Always sent. Before fix-pass 3.1 the block went only when a caller+ # passed skills, so the ordinary replay ran with an empty snapshot while+ # the docs described one carrying organisation memory.payload: dict[str, Any] = {"connector_id": connector_type,"findings": findings,"train_fraction": request.train_fraction,+ "context": context.as_payload(request),}- if request.skills is not None or request.skill_under_test is not None:- # Sent only when a caller asked for it. An ordinary replay still sends- # no context block at all, so its snapshot is the empty one documented- # on `FrozenContext` rather than an empty-but-present one that would- # read differently in the method note.- payload["context"] = {- "skills": list(request.skills or ()),- "skills_under_test": [request.skill_under_test] if request.skill_under_test else [],- }try:async with httpx.AsyncClient(timeout=_REPLAY_TIMEOUT_S) as client:response = await client.post(url, headers=_agents_headers(request.tenant_id), json=payload)@@ -267,9 +422,15 @@ async def run_evaluation(db: AsyncSession, request: ReplayRequest) -> None:"a result over zero findings would describe the window rather than the agent.")- run = await _run_replay(request, connector_type, findings)+ context = await _capture_frozen_context(db, request)+ run = await _run_replay(request, connector_type, findings, context)decisions = run["decisions"]method = dict(run.get("method") or {})+ if context.unread:+ # Published rather than logged: a score produced with less context+ # than the method section describes is the defect this capture+ # exists to close, and a silent partial capture would recreate it.+ method["frozen_context_unread"] = list(context.unread)score = score_replay(decisions,diff --git a/services/api/tests/test_replay_evaluation.py b/services/api/tests/test_replay_evaluation.pyindex 9213d6a39..c96952a9e 100644--- a/services/api/tests/test_replay_evaluation.py+++ b/services/api/tests/test_replay_evaluation.py@@ -292,6 +292,26 @@ class _StubSession:def __init__(self, connector: Any = None) -> None:self._connector = connector+ async def begin_nested(self) -> Any:+ """A SAVEPOINT. The job wraps each optional context read in one.++ Modelled here rather than left off: a double that cannot do what+ the real session can is how a defect reaches production with a+ green suite behind it. The frozen-context reads need savepoints+ because PostgreSQL aborts the whole transaction on a failed+ statement, so catching the error is not enough to keep the later+ write alive.+ """++ class _Savepoint:+ async def rollback(self) -> None:+ return None++ async def commit(self) -> None:+ return None++ return _Savepoint()+async def execute(self, *args: Any, **kwargs: Any) -> Any:connector = self._connectordiff --git a/services/api/tests/test_replay_frozen_context.py b/services/api/tests/test_replay_frozen_context.pynew file mode 100644index 000000000..f6d0396f8--- /dev/null+++ b/services/api/tests/test_replay_frozen_context.py@@ -0,0 +1,343 @@+"""The replay sends the context production triages with, frozen at the split.++Fix pass 3.1. A replay grades the agent against closed findings, and the+report it produces is read as "this is how triage would have done". That+reading is only true if the replay runs with what production runs with.++Two halves were missing, and both flatter or distort the result rather than+erroring:++* `job.py` sent a context block **only** when a caller passed tenant skills,+ so an ordinary replay sent none at all. Organisation memory+ (`aisoc_context_statements`), outcome priors+ (`aisoc_institutional_memory`, `outcome:` keys) and the tenant's+ business-context rule set were never captured, while production reads all+ three on every alert. `replay.md`, `triage-context.md` and+ `tenant-skills.md` all describe a replay that carries them.+* The agents-side runner was constructed with no `business_context`, while+ `main.py` constructs a `BusinessContextApplier` whenever the feature is+ enabled -- which is by default.++The direction of the error matters. Organisation memory and priors mostly+*suppress*, so a replay without them re-raises alerts production would have+closed and scores the agent against a world it does not run in.++What is deliberately **not** sent: anything recorded after the split. The+capture here reads rows with their recorded time attached and the agents+side drops the late ones (`capture_context` in+`services/agents/app/replay/shadow.py`), so the filtering happens in one+place rather than being re-implemented on each side of the call.+"""++from __future__ import annotations++import uuid+from datetime import UTC, datetime+from typing import Any++import httpx+import pytest+from app.services.replay_evaluation import job as job_module++TENANT = uuid.UUID("11111111-1111-1111-1111-111111111111")+++class _RecordingStore:+ def __init__(self) -> None:+ self.failure: str | None = None+ self.result: dict[str, Any] | None = None++ async def mark_running(self, db: Any, **_: Any) -> None:+ return None++ async def fail_evaluation(self, db: Any, *, error: str, **_: Any) -> None:+ self.failure = error++ async def store_result(self, db: Any, **fields: Any) -> None:+ self.result = fields+++class _Connector:+ connector_type = "splunk"+ auth_config = {"base_url": "https://splunk:8089", "token": "t"}+ connector_config = {"ssl_verify": True}+++class _ContextSession:+ """A session that answers the three context reads and the connector read.++ Routed on the SQL text rather than on call order: the job is free to+ reorder its reads, and a positional stub would then answer the wrong+ query while still passing.+ """++ def __init__(self, *, statements: list[dict[str, Any]], priors: list[Any], rule_set: Any) -> None:+ self._statements = statements+ self._priors = priors+ self._rule_set = rule_set+ self.queries: list[str] = []+ self.savepoints = 0+ self.savepoint_rollbacks = 0++ async def begin_nested(self) -> Any:+ """A SAVEPOINT, recorded so a test can assert one was actually taken."""+ self.savepoints += 1+ session = self++ class _Savepoint:+ async def rollback(self) -> None:+ session.savepoint_rollbacks += 1++ async def commit(self) -> None:+ return None++ return _Savepoint()++ async def execute(self, statement: Any, *args: Any, **kwargs: Any) -> Any:+ sql = str(statement)+ self.queries.append(sql)+ rows: list[Any] = []+ scalar: Any = None+ if "aisoc_context_statements" in sql:+ rows = self._statements+ elif "aisoc_institutional_memory" in sql:+ rows = self._priors+ elif "aisoc_business_context_rule_sets" in sql:+ rows = [self._rule_set] if self._rule_set is not None else []+ else:+ scalar = _Connector()++ class _Result:+ def scalar_one_or_none(self) -> Any:+ return scalar++ def mappings(self) -> Any:+ return rows++ def __iter__(self) -> Any:+ return iter(rows)++ return _Result()++ async def rollback(self) -> None:+ return None+++def _request(**overrides: Any) -> job_module.ReplayRequest:+ defaults: dict[str, Any] = {+ "tenant_id": TENANT,+ "evaluation_id": uuid.uuid4(),+ "connector_row_id": uuid.uuid4(),+ "connector_type": "splunk",+ "vendor": "splunk",+ "window_start": datetime(2026, 3, 1, tzinfo=UTC),+ "window_end": datetime(2026, 6, 1, tzinfo=UTC),+ "train_fraction": 0.7,+ "limit": 1000,+ "bootstrap_seed": 20260926,+ "bootstrap_resamples": 50,+ }+ return job_module.ReplayRequest(**{**defaults, **overrides})+++def no_vault(monkeypatch: pytest.MonkeyPatch) -> None:+ class _Vault:+ def decrypt_dict(self, value: dict[str, Any]) -> dict[str, Any]:+ return dict(value)++ monkeypatch.setattr(job_module, "get_vault", lambda: _Vault())+++def recording_store(monkeypatch: pytest.MonkeyPatch) -> _RecordingStore:+ stub = _RecordingStore()+ monkeypatch.setattr(job_module.store, "mark_running", stub.mark_running)+ monkeypatch.setattr(job_module.store, "fail_evaluation", stub.fail_evaluation)+ monkeypatch.setattr(job_module.store, "store_result", stub.store_result)+ return stub+++def _capture_replay_payload(monkeypatch: pytest.MonkeyPatch, sink: dict[str, Any]) -> None:+ def handler(request: httpx.Request) -> httpx.Response:+ if request.url.path.endswith("/replay/history"):+ return httpx.Response(+ 200,+ json={"vendor": "splunk", "count": 2, "labelled": 2, "unlabeled": 0, "findings": [{"x": 1}, {"x": 2}]},+ )+ import json as _json++ sink.update(_json.loads(request.content.decode()))+ return httpx.Response(+ 200,+ json={+ "decisions": [+ {"finding_id": "a", "predicted": "malicious", "actual": "malicious", "split": "test"},+ {"finding_id": "b", "predicted": "benign", "actual": "benign", "split": "test"},+ ],+ "method": {},+ },+ )++ real_client = httpx.AsyncClient+ monkeypatch.setattr(+ job_module.httpx,+ "AsyncClient",+ lambda *a, **kw: real_client(*a, **{**kw, "transport": httpx.MockTransport(handler)}),+ )+++def _session() -> _ContextSession:+ return _ContextSession(+ statements=[+ {+ "statement": "deploy-bot opens change tickets; its bulk edits are expected",+ "reason_code": "expected_automation",+ "scope": "actor",+ "scope_value": "deploy-bot",+ "observations": 12,+ "updated_at": datetime(2026, 2, 1, tzinfo=UTC),+ }+ ],+ priors=[+ {+ "key": "outcome:sig-benign-backup",+ "value": {+ "disposition": "benign",+ "author": "human",+ "confidence": 1.0,+ "last_seen": "2026-02-10T00:00:00+00:00",+ },+ }+ ],+ rule_set={+ "yaml_text": "rules:\n - name: crown jewels\n when: {host: db-prod-1}\n then: {severity: critical}\n",+ "enabled": True,+ "updated_at": datetime(2026, 1, 20, tzinfo=UTC),+ },+ )+++class TestTheReplayCarriesProductionContext:+ async def test_organisation_memory_is_sent(+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None+ ) -> None:+ """Statements in `aisoc_context_statements` reach the agents service.++ Pre-fix this list was absent: the payload carried no `context` key at+ all unless a caller passed skills.+ """+ sent: dict[str, Any] = {}+ _capture_replay_payload(monkeypatch, sent)++ await job_module.run_evaluation(_session(), _request())++ assert recording_store.failure is None, recording_store.failure+ statements = sent.get("context", {}).get("statements") or []+ assert [s["statement"] for s in statements] == ["deploy-bot opens change tickets; its bulk edits are expected"]++ async def test_outcome_priors_are_sent_keyed_by_signature(+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None+ ) -> None:+ """Priors travel under the bare signature, not the `outcome:` storage key.++ The agents side looks a prior up by signature; sending the storage+ key would put every prior somewhere nothing reads, which is the+ silent-miss shape `human_priors.py` documents.+ """+ sent: dict[str, Any] = {}+ _capture_replay_payload(monkeypatch, sent)++ await job_module.run_evaluation(_session(), _request())++ priors = sent.get("context", {}).get("priors") or {}+ assert list(priors) == ["sig-benign-backup"]+ assert priors["sig-benign-backup"]["author"] == "human"++ async def test_the_business_context_rule_set_is_sent(+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None+ ) -> None:+ """Production applies the tenant's rules on every alert; a replay must too."""+ sent: dict[str, Any] = {}+ _capture_replay_payload(monkeypatch, sent)++ await job_module.run_evaluation(_session(), _request())++ rules = sent.get("context", {}).get("business_context") or {}+ assert rules.get("enabled") is True+ assert "crown jewels" in (rules.get("yaml_text") or "")+ # Carries its own recorded time so the agents side can drop a rule set+ # edited after the split, the same way it drops a late statement.+ assert rules.get("updated_at")++ async def test_a_context_block_is_sent_even_with_no_skills(+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None+ ) -> None:+ """The ordinary replay path is the one that was running bare."""+ sent: dict[str, Any] = {}+ _capture_replay_payload(monkeypatch, sent)++ await job_module.run_evaluation(_session(), _request())++ assert "context" in sent+ assert sent["context"].get("skills") == []++ async def test_a_context_read_that_fails_does_not_fail_the_replay(+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None+ ) -> None:+ """Fail-soft, and say so in the method note rather than silently.++ A replay that dies because institutional memory was briefly+ unreadable is worse than one that runs with less context and records+ which part it could not read.+ """+ sent: dict[str, Any] = {}+ _capture_replay_payload(monkeypatch, sent)++ class _Broken(_ContextSession):+ async def execute(self, statement: Any, *args: Any, **kwargs: Any) -> Any:+ if "aisoc_context_statements" in str(statement):+ raise RuntimeError("relation does not exist")+ return await super().execute(statement, *args, **kwargs)++ session = _Broken(statements=[], priors=[], rule_set=None)+ await job_module.run_evaluation(session, _request())++ assert recording_store.failure is None, recording_store.failure+ assert recording_store.result is not None+ unread = (recording_store.result.get("method") or {}).get("frozen_context_unread") or []+ assert "statements" in unread++ async def test_a_failed_read_is_confined_to_a_savepoint(+ self, monkeypatch: pytest.MonkeyPatch, recording_store: _RecordingStore, no_vault: None+ ) -> None:+ """Catching the exception is not enough on PostgreSQL.++ A failed statement aborts the **whole** transaction and the engine+ refuses everything after it with `current transaction is aborted,+ commands ignored until end of transaction block` -- so a plain+ try/except around a context read does not isolate the failure, it+ poisons the write that stores the report. The replay then scores+ correctly and dies on its own `UPDATE`, which reads as a storage+ bug rather than as a missing table.++ That is not hypothetical: it is what the live four-service replay+ job reported before this was confined, while every offline test+ passed, because a stub session has no transaction to poison.+ """+ sent: dict[str, Any] = {}+ _capture_replay_payload(monkeypatch, sent)++ class _Broken(_ContextSession):+ async def execute(self, statement: Any, *args: Any, **kwargs: Any) -> Any:+ if "aisoc_institutional_memory" in str(statement):+ raise RuntimeError("relation does not exist")+ return await super().execute(statement, *args, **kwargs)++ session = _Broken(statements=[], priors=[], rule_set=None)+ await job_module.run_evaluation(session, _request())++ assert session.savepoints >= 3, "each optional read must take its own savepoint"+ assert session.savepoint_rollbacks == 1, "only the failed read should have rolled back"+ assert recording_store.failure is None, recording_store.failure