Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix: durable fusion→agents auto-triage spine (#568, #571, #569, #570) (#572)
Beenu Arorac090b297eca5
diff --git a/services/actions/app/executors/siem.py b/services/actions/app/executors/siem.pyindex 57ee0fb0d..8b5a51260 100644--- a/services/actions/app/executors/siem.py+++ b/services/actions/app/executors/siem.py@@ -77,7 +77,9 @@ class SearchSIEMExecutor(BaseExecutor):splunk = _splunk_client(request.parameters)if splunk:try:- results = await splunk.run_search(query=query, max_results=max_results)+ # SplunkClient.run_search takes `max_count` (issue #570); the old+ # `max_results=` kwarg raised TypeError on every live Splunk search.+ results = await splunk.run_search(query=query, max_count=max_results)return ActionResult(action_id=request.id,status=ActionStatus.COMPLETED,diff --git a/services/actions/tests/test_search_siem_executor.py b/services/actions/tests/test_search_siem_executor.pynew file mode 100644index 000000000..58c702e2d--- /dev/null+++ b/services/actions/tests/test_search_siem_executor.py@@ -0,0 +1,57 @@+"""Issue #570 — SearchSIEMExecutor must call SplunkClient with the right kwarg.++`SplunkClient.run_search` takes `max_count`; the executor previously passed+`max_results=`, raising TypeError on every live Splunk search (silently+returning a FAILED action). These tests pin the contract so it can't regress.+"""++from __future__ import annotations++import inspect+from uuid import uuid4++import pytest+from app.clients.splunk_client import SplunkClient+from app.executors import siem+from app.executors.siem import SearchSIEMExecutor+from app.models.action import ActionRequest, ActionStatus, ActionType+++def test_run_search_signature_uses_max_count_not_max_results():+ params = inspect.signature(SplunkClient.run_search).parameters+ assert "max_count" in params+ assert "max_results" not in params+++class _FakeSplunk:+ def __init__(self) -> None:+ self.calls: list[dict] = []++ async def run_search(self, **kwargs): # noqa: ANN003 — mirror the real signature loosely+ self.calls.append(kwargs)+ return [{"_raw": "evt-1"}, {"_raw": "evt-2"}]+++def _request() -> ActionRequest:+ return ActionRequest(+ incident_id=uuid4(),+ tenant_id=uuid4(),+ action_type=ActionType.SEARCH_SIEM,+ target="splunk",+ parameters={"query": "index=main sourcetype=WinEventLog", "max_results": 25},+ )+++async def test_executor_calls_run_search_with_max_count(monkeypatch):+ fake = _FakeSplunk()+ monkeypatch.setattr(siem, "_splunk_client", lambda params: fake)++ result = await SearchSIEMExecutor().execute(_request())++ assert result.status is ActionStatus.COMPLETED+ assert len(fake.calls) == 1+ kwargs = fake.calls[0]+ assert kwargs.get("max_count") == 25 # correct kwarg + value threaded through+ assert "max_results" not in kwargs # the broken kwarg is gone+ assert result.output["result_count"] == 2diff --git a/services/agents/app/agents/auto_triage_agent.py b/services/agents/app/agents/auto_triage_agent.pyindex 5bdc0b294..068a18c4d 100644--- a/services/agents/app/agents/auto_triage_agent.py+++ b/services/agents/app/agents/auto_triage_agent.py@@ -44,6 +44,18 @@ logger = structlog.get_logger()AUTO_CLOSE_THRESHOLD: float = float(os.getenv("AISOC_AUTO_CLOSE_THRESHOLD", "0.85"))++class AutoTriageError(RuntimeError):+ """Raised when the LLM auto-triage call or its response parsing fails.++ Issue #571: the old code swallowed LLM/parse exceptions and returned a+ RUNNING state with a null verdict, so the worker's deterministic fallback+ (`_llm_triage`'s except branch) was unreachable and a null verdict could be+ recorded as "completed". Raising a typed error lets the caller fall back to+ deterministic triage or mark the alert ``needs_review`` — never null.+ """++_metrics: dict[str, Any] = {"auto_resolved_count": 0,"escalated_count": 0,@@ -233,11 +245,13 @@ async def run_auto_triage(state: InvestigationState) -> InvestigationState:raw_text = response.contentresult = _parse_llm_response(raw_text)except Exception as exc:- logger.error("Auto-triage LLM call failed, escalating", error=str(exc))- state.add_finding(f"Auto-triage LLM error: {exc} — escalating to manual triage")- _metrics["escalated_count"] += 1+ # Issue #571: do NOT swallow + return a null-verdict RUNNING state.+ # Raise a typed error so the caller falls back to deterministic triage+ # (or marks the alert needs_review) instead of completing with no verdict.+ logger.error("Auto-triage LLM call failed", error=str(exc))+ state.add_finding(f"Auto-triage LLM error: {exc}")_metrics["total_processed"] += 1- return state+ raise AutoTriageError(str(exc)) from excelapsed_ms = round((time.monotonic() - t0) * 1000)diff --git a/services/agents/app/agents/investigation_agent.py b/services/agents/app/agents/investigation_agent.pyindex 11bc94a4c..8d142bd68 100644--- a/services/agents/app/agents/investigation_agent.py+++ b/services/agents/app/agents/investigation_agent.py@@ -11,7 +11,9 @@ from typing import Anyimport httpximport structlog+from app.agents.dispositions import BENIGN, BENIGN_TRUE_POSITIVE, FALSE_POSITIVE, NEEDS_REVIEWfrom app.confidence import score_investigation+from app.investigator.splunk_evidence import collect_splunk_evidencefrom app.models.state import ActionRisk, AgentStatus, InvestigationState, ProposedActionfrom app.tools.mitre import lookup_technique@@ -37,6 +39,11 @@ async def run_investigation(state: InvestigationState) -> InvestigationState:state.iteration_count += 1+ # --- Pull surrounding SIEM evidence for Splunk-originated alerts (#570) ---+ # Bounded, read-only, credential-vault-backed. Failure => missing evidence+ # (handled after scoring below), never a silent dismissal.+ splunk_evidence = await collect_splunk_evidence(state)+# Analyze enrichment results for threat patternsmalicious_iocs = {k: v for k, v in state.ioc_enrichments.items() if v.get("threat_classification") in ("malicious", "suspicious")}@@ -104,6 +111,14 @@ async def run_investigation(state: InvestigationState) -> InvestigationState:)confidence, basis, verdict = score_investigation(state)++ # #570: never dismiss a Splunk-originated alert on MISSING evidence. If the+ # bounded evidence query failed/timed out (applicable but not queried), a+ # dismissive verdict is downgraded to needs_review so a human decides.+ if splunk_evidence.applicable and not splunk_evidence.queried and verdict in (FALSE_POSITIVE, BENIGN, BENIGN_TRUE_POSITIVE):+ state.add_finding("Splunk evidence unavailable — cannot confirm a benign/false-positive verdict; escalating to needs_review.")+ verdict = NEEDS_REVIEW+state.confidence = confidencestate.confidence_basis = basisstate.verdict = verdictdiff --git a/services/agents/app/api/router.py b/services/agents/app/api/router.pyindex ec55bf1cf..d7e82b9df 100644--- a/services/agents/app/api/router.py+++ b/services/agents/app/api/router.py@@ -11,12 +11,14 @@ from uuid import UUID, uuid4from fastapi import APIRouter, BackgroundTasks, HTTPExceptionfrom pydantic import BaseModel-from app.graph.workflow import investigation_graph+from app.graph.runner import run_full_investigationfrom app.models.state import AgentTask, InvestigationStaterouter = APIRouter()-# In-memory run store (replace with Redis/DB in production)+# In-memory run store for status polling. The durable record of every step is+# the Postgres Investigation Ledger (written by the shared graph runner) — this+# dict is only the fast local status cache for GET /investigations/{run_id}._runs: dict[str, dict] = {}@@ -35,12 +37,20 @@ class InvestigationResponse(BaseModel):async def _run_investigation(run_id: str, state: InvestigationState) -> None:- """Run investigation in background and store results."""+ """Run investigation in background and store results.++ Uses the SAME durable graph runner as the Kafka auto-triage worker+ (issue #569), so manual and automated investigations share one+ orchestration implementation and both persist every step to the ledger.+ """try:- initial_state = state.to_dict()- result = await investigation_graph.ainvoke(initial_state)- _runs[run_id] = {"status": "completed", "result": result, "completed_at": datetime.utcnow().isoformat()}- except Exception as exc:+ result = await run_full_investigation(state)+ _runs[run_id] = {+ "status": "completed",+ "result": result.to_dict(),+ "completed_at": datetime.utcnow().isoformat(),+ }+ except Exception as exc: # noqa: BLE001 — surface failure via the status cache_runs[run_id] = {"status": "failed", "error": str(exc)}diff --git a/services/agents/app/graph/runner.py b/services/agents/app/graph/runner.pynew file mode 100644index 000000000..b7810f770--- /dev/null+++ b/services/agents/app/graph/runner.py@@ -0,0 +1,153 @@+"""Shared, durable investigation-graph runner (issue #569).++Both the Kafka auto-triage worker and the manual investigations API run the+*same* orchestration through here, so escalated alerts flow through one+implementation instead of two divergent ones:++* the manual API runs the FULL graph (auto-triage → triage → enrichment →+ investigation → attack-path) via :func:`run_full_investigation`;+* the auto-triage worker, having already produced a governed verdict, runs the+ post-triage escalation subgraph via :func:`run_escalation`.++Every node transition is recorded to the Investigation Ledger (append-only,+``ON CONFLICT (run_id, seq) DO NOTHING``), so a restart mid-investigation+re-runs idempotently — it never loses or duplicates the run. A wall-clock+budget bounds the run; a token budget is enforced upstream by the+``CostGovernor`` and a tool-call budget by the ReAct loop's ``max_iters``.+"""++from __future__ import annotations++import asyncio+import os+from dataclasses import dataclass++import structlog++from app.agents.dispositions import NEEDS_REVIEW+from app.investigator import ledger as ledger_module+from app.models.state import AgentStatus, InvestigationState++logger = structlog.get_logger()+++@dataclass(frozen=True)+class InvestigationBudget:+ """Resource envelope for one investigation (issue #569)."""++ max_seconds: float = 120.0+ max_tokens: int = 20_000+ max_tool_calls: int = 8+++def default_budget() -> InvestigationBudget:+ return InvestigationBudget(+ max_seconds=float(os.getenv("AISOC_INVESTIGATION_MAX_SECONDS", "120")),+ max_tokens=int(os.getenv("AISOC_MAX_TOKENS_PER_ALERT", "20000")),+ max_tool_calls=int(os.getenv("AISOC_INVESTIGATION_MAX_TOOL_CALLS", "8")),+ )+++async def _run(+ graph, # noqa: ANN001 — a compiled LangGraph+ state: InvestigationState,+ *,+ budget: InvestigationBudget | None,+ persist: bool,+ seq_start: int,+) -> InvestigationState:+ budget = budget or default_budget()+ tenant_ref = str(state.tenant_id)+ state_dict = state.to_dict()+ merged: dict = dict(state_dict)++ tenant_uuid = await ledger_module.resolve_tenant(tenant_ref) if persist else None+ if persist:+ # Idempotent — start_run ON CONFLICT DO NOTHING (the worker may have+ # already opened this run when it persisted the triage verdict).+ await ledger_module.start_run(+ run_id=state.run_id,+ case_id=str(state.incident_id),+ tenant_ref=tenant_ref,+ alert_summary=state.alert_summary or "",+ raw_alert=state.raw_alert or {},+ model_used="graph:investigation",+ )++ seq = seq_start+ timed_out = False+ try:+ async with asyncio.timeout(budget.max_seconds):+ async for step in graph.astream(state_dict):+ if not isinstance(step, dict):+ continue+ for node, out in step.items():+ seq += 1+ if isinstance(out, dict):+ merged.update(out)+ if tenant_uuid is not None:+ await ledger_module.record_event(+ run_id=state.run_id,+ tenant_id=tenant_uuid,+ seq=seq,+ kind="graph_step",+ agent=str(node),+ summary=f"graph node '{node}' completed",+ payload={"node": str(node)},+ )+ except TimeoutError:+ timed_out = True+ logger.warning("investigation.budget_timeout", run_id=str(state.run_id), max_seconds=budget.max_seconds)++ try:+ result = InvestigationState.model_validate(merged)+ except Exception as exc: # noqa: BLE001 — never let a mapping error lose the run+ logger.warning("investigation.state_rebuild_failed", run_id=str(state.run_id), error=str(exc))+ result = state++ if timed_out:+ result.add_finding(f"Investigation budget ({budget.max_seconds}s) exceeded — partial results, escalating.")+ if not result.verdict:+ result.verdict = NEEDS_REVIEW+ if result.status is AgentStatus.COMPLETED:+ result.status = AgentStatus.RUNNING++ if persist:+ await ledger_module.complete_run(+ run_id=result.run_id,+ tenant_id=tenant_uuid or result.tenant_id,+ status="completed",+ error=result.error,+ iterations=result.iteration_count,+ )+ return result+++async def run_full_investigation(+ state: InvestigationState,+ *,+ budget: InvestigationBudget | None = None,+ persist: bool = True,+) -> InvestigationState:+ """Run the FULL graph (auto-triage → … → attack-path). Manual API entry."""+ from app.graph.workflow import investigation_graph # noqa: PLC0415 — avoid import cycle++ return await _run(investigation_graph, state, budget=budget, persist=persist, seq_start=0)+++async def run_escalation(+ state: InvestigationState,+ *,+ budget: InvestigationBudget | None = None,+ persist: bool = True,+ seq_start: int = 1,+) -> InvestigationState:+ """Run the post-triage escalation subgraph (triage → … → attack-path).++ Used by the auto-triage worker after it has produced (and persisted) a+ governed verdict. ``seq_start`` defaults to 1 so the recorded graph steps+ follow the worker's ``triage_verdict`` event (seq 1) without colliding.+ """+ from app.graph.workflow import escalation_graph # noqa: PLC0415 — avoid import cycle++ return await _run(escalation_graph, state, budget=budget, persist=persist, seq_start=seq_start)diff --git a/services/agents/app/graph/workflow.py b/services/agents/app/graph/workflow.pyindex fc9acdda6..826ea002f 100644--- a/services/agents/app/graph/workflow.py+++ b/services/agents/app/graph/workflow.py@@ -14,7 +14,7 @@ import structlogfrom langgraph.graph import END, StateGraphfrom app.agents.attack_path_agent import run_attack_path-from app.agents.auto_triage_agent import run_auto_triage+from app.agents.auto_triage_agent import AutoTriageError, run_auto_triagefrom app.agents.enrichment_agent import run_enrichmentfrom app.agents.investigation_agent import run_investigationfrom app.agents.triage_agent import run_triage@@ -36,7 +36,15 @@ def _from_dict(d: dict) -> InvestigationState:async def auto_triage_node(state: dict) -> dict:s = _from_dict(state)- s = await run_auto_triage(s)+ try:+ s = await run_auto_triage(s)+ except AutoTriageError as exc:+ # LLM/parse failure (issue #571): never terminate on a null verdict —+ # escalate through the full pipeline (deterministic triage runs next).+ logger.warning("graph.auto_triage_failed_escalating", error=str(exc), incident_id=str(s.incident_id))+ s.add_finding(f"Auto-triage LLM unavailable ({exc}) — escalating to full pipeline")+ if s.status is AgentStatus.COMPLETED:+ s.status = AgentStatus.RUNNINGreturn s.to_dict()@@ -113,5 +121,30 @@ def build_investigation_graph() -> StateGraph:return graph.compile()-# Module-level compiled graph (singleton)+def build_escalation_graph() -> StateGraph:+ """The post-triage escalation pipeline (issue #569).++ Shares the SAME node implementations as the full graph but skips the+ auto-triage entry node — used by the Kafka auto-triage worker, which has+ already produced a governed verdict, to route escalated alerts (TP /+ low-confidence / needs_review) through enrichment → investigation →+ attack-path without re-triaging.++ Flow: triage ──► enrichment ──► investigation ──► attack_path ──► END+ """+ graph = StateGraph(dict)+ graph.add_node("triage", triage_node)+ graph.add_node("enrichment", enrichment_node)+ graph.add_node("investigation", investigation_node)+ graph.add_node("attack_path", attack_path_node)+ graph.set_entry_point("triage")+ graph.add_edge("triage", "enrichment")+ graph.add_edge("enrichment", "investigation")+ graph.add_edge("investigation", "attack_path")+ graph.add_edge("attack_path", END)+ return graph.compile()+++# Module-level compiled graphs (singletons)investigation_graph = build_investigation_graph()+escalation_graph = build_escalation_graph()diff --git a/services/agents/app/investigator/ledger.py b/services/agents/app/investigator/ledger.pyindex cb52ac63b..dae564f1b 100644--- a/services/agents/app/investigator/ledger.py+++ b/services/agents/app/investigator/ledger.py@@ -36,6 +36,14 @@ import structloglogger = structlog.get_logger()+class LedgerPersistError(RuntimeError):+ """Raised by the durable write paths (e.g. :func:`persist_auto_triage`) when+ a real database error occurs, so the caller can retry / dead-letter instead+ of silently losing the outcome (issue #571). A *missing* database (no+ ``DATABASE_URL``) is NOT an error — those paths no-op and return ``False``.+ """++_POOL: asyncpg.Pool | None = None@@ -99,6 +107,22 @@ async def _resolve_tenant_id(conn: asyncpg.Connection, tenant_ref: str) -> uuid.return row["id"] if row else None+async def resolve_tenant(tenant_ref: str) -> uuid.UUID | None:+ """Resolve a tenant ref (UUID string / slug / name) to its UUID, or None.++ Best-effort (no DB configured or lookup failure ⇒ None). Used by the shared+ graph runner to attribute per-node ledger events (issue #569)."""+ pool = await get_pool()+ if pool is None:+ return None+ try:+ async with pool.acquire() as conn:+ return await _resolve_tenant_id(conn, tenant_ref)+ except Exception as exc: # noqa: BLE001 — best-effort+ logger.debug("ledger.resolve_tenant_failed", tenant_ref=tenant_ref, error=str(exc))+ return None++async def _set_rls_context(conn: asyncpg.Connection, tenant_id: uuid.UUID) -> None:"""Match the API service's set_rls_context — required so the audit-logimmutability trigger and tenant policies allow our INSERTs."""@@ -139,6 +163,7 @@ async def start_run(model_used, status, started_at, created_at)VALUES($1, $2, $3, $4, $5::jsonb, $6, 'running', now(), now())+ ON CONFLICT (id) DO NOTHING""",run_id,tenant_id,@@ -319,3 +344,152 @@ async def complete_run(run_id=str(run_id),error=str(exc),)+++def _coerce_uuid(value: Any) -> uuid.UUID | None:+ try:+ return uuid.UUID(str(value))+ except (ValueError, TypeError):+ return None+++async def persist_auto_triage(+ *,+ run_id: uuid.UUID,+ alert_id: str | None,+ tenant_ref: str,+ alert_summary: str,+ raw_alert: dict[str, Any] | None,+ tier: str,+ verdict: str,+ confidence: float,+ rationale: str,+ findings: list[str] | None = None,+ proposed_actions: list[dict[str, Any]] | None = None,+ auto_closed: bool = False,+ iterations: int = 0,+ tokens: int = 0,+ cost_usd: float = 0.0,+) -> bool:+ """Durably record an auto-triage outcome (issue #571) in ONE transaction:++ * upsert the ``investigation_runs`` row (idempotent on ``id``),+ * append the verdict as an immutable ``investigation_events`` row (seq 1),+ * mark the run completed, and+ * write the disposition/verdict + rationale + recommendations back onto the+ ``alerts`` row so the alert API surfaces the automated verdict.++ Idempotent: a replay reuses the deterministic ``run_id`` (ON CONFLICT DO+ NOTHING on the run + the ``(run_id, seq)`` event key), so it never+ duplicates ledger rows or outcomes.++ Returns ``True`` when written, ``False`` when skipped (no database+ configured, or an unknown tenant — neither is retryable). Raises+ :class:`LedgerPersistError` on a real DB error so the caller can retry /+ dead-letter. A completed run therefore always carries a non-null verdict.+ """+ pool = await get_pool()+ if pool is None:+ return False++ verdict = (verdict or "").strip() or "needs_review"+ recommendations = proposed_actions or []+ alert_uuid = _coerce_uuid(alert_id)++ try:+ async with pool.acquire() as conn:+ tenant_id = await _resolve_tenant_id(conn, tenant_ref)+ if tenant_id is None:+ logger.debug("ledger.auto_triage_skip", reason="unknown_tenant", tenant_ref=tenant_ref)+ return False+ await _set_rls_context(conn, tenant_id)+ async with conn.transaction():+ await conn.execute(+ """+ INSERT INTO investigation_runs+ (id, tenant_id, case_id, alert_summary, raw_alert,+ model_used, status, started_at, created_at)+ VALUES+ ($1, $2, $3, $4, $5::jsonb, $6, 'running', now(), now())+ ON CONFLICT (id) DO NOTHING+ """,+ run_id,+ tenant_id,+ str(alert_id or ""),+ (alert_summary or "")[:8000] or None,+ json.dumps(raw_alert or {}),+ f"kafka:auto_triage:{tier}",+ )+ await conn.execute(+ """+ INSERT INTO investigation_events+ (id, run_id, tenant_id, seq, ts, kind, agent, summary,+ payload, duration_ms, created_at)+ VALUES+ ($1, $2, $3, 1, now(), 'triage_verdict', $4, $5, $6::jsonb, 0, now())+ ON CONFLICT (run_id, seq) DO NOTHING+ """,+ uuid.uuid4(),+ run_id,+ tenant_id,+ f"auto_triage:{tier}",+ f"verdict={verdict} confidence={confidence:.2f}"[:8000],+ json.dumps(+ {+ "verdict": verdict,+ "confidence": confidence,+ "rationale": rationale,+ "findings": findings or [],+ "proposed_actions": recommendations,+ "auto_closed": auto_closed,+ }+ ),+ )+ await conn.execute(+ """+ UPDATE investigation_runs+ SET status = 'completed', iterations = $2,+ total_tokens = $3, total_cost_usd = $4, completed_at = now()+ WHERE id = $1+ """,+ run_id,+ iterations,+ tokens,+ cost_usd,+ )+ if alert_uuid is not None:+ # Surface the automated verdict on the alert row. Status is+ # only advanced to 'resolved' when auto-triage auto-closed a+ # benign/FP alert at high confidence; otherwise the alert+ # stays open for escalation / human review.+ await conn.execute(+ """+ UPDATE alerts+ SET disposition = $3,+ ai_score = $4,+ ai_summary = $5,+ ai_recommendations = $6::jsonb,+ status = CASE WHEN $7 THEN 'resolved' ELSE status END,+ resolved_at = CASE WHEN $7 THEN now() ELSE resolved_at END,+ updated_at = now()+ WHERE id = $1 AND tenant_id = $2+ """,+ alert_uuid,+ tenant_id,+ verdict[:50],+ float(confidence),+ (rationale or "")[:8000] or None,+ json.dumps(recommendations),+ auto_closed,+ )+ logger.info(+ "ledger.auto_triage_persisted",+ run_id=str(run_id),+ verdict=verdict,+ auto_closed=auto_closed,+ alert_id=str(alert_id or ""),+ )+ return True+ except Exception as exc: # noqa: BLE001 — re-raised as a typed, retryable error+ logger.warning("ledger.auto_triage_persist_failed", run_id=str(run_id), error=str(exc))+ raise LedgerPersistError(str(exc)) from excdiff --git a/services/agents/app/investigator/splunk_evidence.py b/services/agents/app/investigator/splunk_evidence.pynew file mode 100644index 000000000..b6b60b44d--- /dev/null+++ b/services/agents/app/investigator/splunk_evidence.py@@ -0,0 +1,325 @@+"""Bounded, read-only Splunk evidence tool for the investigation graph (#570).++The auto-investigation used to reason only over enrichment already present on+the alert — it never pulled the surrounding events from the SIEM that raised+it. This tool closes that gap for Splunk-originated alerts:++* resolve the originating connector instance from the alert's provenance+ (``connector_id`` — issue #568), never from the LLM;+* obtain credentials from the CredentialVault, never from a prompt/log/ledger;+* build a TIME-BOUNDED, allow-listed SPL pivot around the alert's user / host /+ IP / hash (values are quote-escaped; the query shape is fixed);+* enforce a maximum time range, execution timeout, and result/field caps;+* cite the (redacted) results + a content hash in the investigation findings+ and the ledger;+* treat any failure/timeout as *missing evidence* so the caller escalates to+ needs_review instead of dismissing the alert as a false positive.++Everything is best-effort and self-contained (a minimal Splunk REST client) so+it runs in the agents service without importing the actions service.+"""++from __future__ import annotations++import hashlib+import json+import os+import uuid+from dataclasses import dataclass, field+from datetime import datetime, timedelta+from typing import Any++import httpx+import structlog++from app.investigator import ledger as ledger_module+from app.models.state import InvestigationState+from app.security.credential_vault import get_vault++logger = structlog.get_logger()++_MAX_RESULTS = int(os.getenv("AISOC_SPLUNK_EVIDENCE_MAX_RESULTS", "50"))+_MAX_WINDOW_HOURS = float(os.getenv("AISOC_SPLUNK_EVIDENCE_MAX_WINDOW_HOURS", "24"))+_WINDOW_HOURS = min(float(os.getenv("AISOC_SPLUNK_EVIDENCE_WINDOW_HOURS", "6")), _MAX_WINDOW_HOURS)+_TIMEOUT_S = float(os.getenv("AISOC_SPLUNK_EVIDENCE_TIMEOUT_S", "20"))+_MAX_FIELD_LEN = 500 # per-field redaction cap (byte-ish bound on cited evidence)+++@dataclass(frozen=True)+class SplunkCreds:+ base_url: str+ token: str+ ssl_verify: bool = True+ index: str = "main"+++@dataclass+class SplunkEvidenceResult:+ """Outcome of an evidence attempt.++ ``applicable`` is True only when a Splunk connector was actually resolved+ for the alert, so a *missing evidence* escalation (issue #570) is never+ triggered for non-Splunk alerts. ``queried`` is True only on a successful+ bounded query.+ """++ applicable: bool+ queried: bool+ result_count: int = 0+ error: str | None = None+ query: str | None = None+ findings: list[str] = field(default_factory=list)+++def _spl_quote(value: str) -> str:+ """Quote-escape a value for safe insertion into an SPL double-quoted string."""+ return value.replace("\\", "\\\\").replace('"', '\\"')+++def _safe_index(index: str) -> str:+ """Allow-list the index token (letters/digits/_-* only)."""+ cleaned = "".join(ch for ch in (index or "") if ch.isalnum() or ch in "_-*")+ return cleaned or "main"+++def build_evidence_spl(raw_alert: dict[str, Any], *, index: str = "main", max_results: int = _MAX_RESULTS) -> str | None:+ """Build a bounded, allow-listed SPL pivot around the alert's entities.++ Returns ``None`` when the alert has no pivotable entity (so we never run an+ unbounded ``search index=*`` fishing query).+ """+ filters: list[str] = []+ username = _first_str(raw_alert, "username", "user")+ if username:+ filters.append(f'user="{_spl_quote(username)}"')+ host = _first_str(raw_alert, "hostname", "host")+ if host:+ filters.append(f'host="{_spl_quote(host)}"')+ ip = _first_str(raw_alert, "src_ip", "dst_ip")+ if ip:+ q = _spl_quote(ip)+ filters.append(f'(src_ip="{q}" OR dest_ip="{q}" OR src="{q}" OR dest="{q}")')+ file_hash = _first_str(raw_alert, "file_hash")+ if file_hash:+ q = _spl_quote(file_hash)+ filters.append(f'(file_hash="{q}" OR hash="{q}")')++ if not filters:+ return None+ where = " OR ".join(filters)+ return f"search index={_safe_index(index)} ({where}) | head {int(max_results)}"+++def _first_str(obj: dict[str, Any], *keys: str) -> str | None:+ for k in keys:+ v = obj.get(k)+ if isinstance(v, str) and v.strip():+ return v.strip()+ return None+++def _time_window(raw_alert: dict[str, Any]) -> tuple[str, str]:+ """Return (earliest, latest) Splunk time tokens, bounded to ±_WINDOW_HOURS."""+ et = raw_alert.get("event_time")+ if not et:+ raw_event = raw_alert.get("raw_event")+ if isinstance(raw_event, dict):+ et = raw_event.get("time")+ if isinstance(et, str) and et:+ try:+ dt = datetime.fromisoformat(et.replace("Z", "+00:00"))+ start = dt - timedelta(hours=_WINDOW_HOURS)+ end = dt + timedelta(hours=_WINDOW_HOURS)+ return (str(int(start.timestamp())), str(int(end.timestamp())))+ except ValueError:+ pass+ return (f"-{int(_WINDOW_HOURS)}h", "now")+++def _redact_samples(results: list[dict[str, Any]]) -> list[str]:+ """Compact, length-capped one-liners for a few results (bounded citation)."""+ out: list[str] = []+ for row in results:+ if not isinstance(row, dict):+ continue+ parts = []+ for k in ("_time", "host", "source", "sourcetype", "user", "action"):+ v = row.get(k)+ if v:+ parts.append(f"{k}={str(v)[:80]}")+ line = " ".join(parts) or json.dumps(row, default=str)+ out.append(line[:_MAX_FIELD_LEN])+ return out+++class SplunkEvidenceClient:+ """Minimal, bounded, read-only Splunk oneshot-search client."""++ def __init__(self, creds: SplunkCreds, *, timeout_s: float = _TIMEOUT_S, max_results: int = _MAX_RESULTS) -> None:+ self._base = creds.base_url.rstrip("/")+ self._token = creds.token+ self._verify = creds.ssl_verify+ self._timeout = timeout_s+ self._max_results = max_results++ async def search(self, spl: str, *, earliest: str, latest: str) -> list[dict[str, Any]]:+ async with httpx.AsyncClient(timeout=self._timeout, verify=self._verify) as client:+ resp = await client.post(+ f"{self._base}/services/search/jobs",+ headers={+ "Authorization": f"Bearer {self._token}",+ "Content-Type": "application/x-www-form-urlencoded",+ },+ data={+ "search": spl,+ "output_mode": "json",+ "exec_mode": "oneshot",+ "earliest_time": earliest,+ "latest_time": latest,+ "count": str(self._max_results),+ },+ )+ resp.raise_for_status()+ data = resp.json()+ results = data.get("results", []) if isinstance(data, dict) else []+ return results[: self._max_results]+++async def resolve_splunk_credentials(state: InvestigationState) -> SplunkCreds | None:+ """Resolve + decrypt the originating Splunk connector's credentials.++ Keyed on the alert's ``connector_id`` provenance (issue #568) scoped to the+ tenant, and only for a connector whose ``connector_type`` is ``splunk``.+ Returns ``None`` (not raise) when the alert isn't Splunk-originated, no DB+ is configured, or the vault can't decrypt — the caller then simply skips+ the evidence step. Credentials are never logged.+ """+ raw = state.raw_alert or {}+ connector_id = raw.get("connector_id")+ if not connector_id:+ return None+ pool = await ledger_module.get_pool()+ if pool is None:+ return None+ try:+ cid = uuid.UUID(str(connector_id))+ except (ValueError, TypeError):+ return None+ try:+ async with pool.acquire() as conn:+ await conn.execute("SELECT set_config('app.tenant_id', $1, true)", str(state.tenant_id))+ row = await conn.fetchrow(+ """+ SELECT auth_config, connector_config+ FROM connectors+ WHERE id = $1 AND tenant_id = $2 AND connector_type = 'splunk'+ """,+ cid,+ state.tenant_id,+ )+ if row is None:+ return None+ auth = _as_dict(row["auth_config"])+ cfg = _as_dict(row["connector_config"])+ vault = get_vault()+ if vault is not None:+ auth = vault.decrypt_dict(auth)+ merged = {**cfg, **auth}+ base_url = str(merged.get("base_url") or "").strip()+ token = str(merged.get("token") or "").strip()+ if not base_url or not token:+ return None+ return SplunkCreds(+ base_url=base_url,+ token=token,+ ssl_verify=bool(merged.get("ssl_verify", True)),+ index=str(merged.get("saved_search") or "main") if str(merged.get("saved_search") or "").startswith("index=") else "main",+ )+ except Exception as exc: # noqa: BLE001 — best-effort; never leak creds in the log+ logger.debug("splunk_evidence.resolve_failed", error=type(exc).__name__)+ return None+++def _as_dict(value: Any) -> dict[str, Any]:+ if isinstance(value, dict):+ return value+ if isinstance(value, str) and value.strip():+ try:+ parsed = json.loads(value)+ return parsed if isinstance(parsed, dict) else {}+ except (ValueError, TypeError):+ return {}+ return {}+++def _make_client(creds: SplunkCreds) -> SplunkEvidenceClient:+ return SplunkEvidenceClient(creds)+++async def collect_splunk_evidence(state: InvestigationState) -> SplunkEvidenceResult:+ """Run one bounded Splunk evidence query for a Splunk-originated alert.++ Adds redacted, hashed evidence to the investigation findings (cited in the+ verdict rationale) and records it to the ledger. On any failure/timeout it+ returns ``queried=False`` with ``applicable=True`` so the caller escalates+ to needs_review rather than dismissing the alert.+ """+ creds = await resolve_splunk_credentials(state)+ if creds is None:+ return SplunkEvidenceResult(applicable=False, queried=False)++ raw = state.raw_alert or {}+ spl = build_evidence_spl(raw, index=creds.index, max_results=_MAX_RESULTS)+ if spl is None:+ state.add_finding("Splunk evidence: no pivotable entity (user/host/ip/hash) on the alert — skipped.")+ return SplunkEvidenceResult(applicable=True, queried=False, error="no_pivot")++ earliest, latest = _time_window(raw)+ try:+ results = await _make_client(creds).search(spl, earliest=earliest, latest=latest)+ except Exception as exc: # noqa: BLE001 — treat as missing evidence, escalate safely+ logger.warning("splunk_evidence.query_failed", error=type(exc).__name__)+ state.add_finding(f"Splunk evidence query failed ({type(exc).__name__}) — treating as missing evidence, escalating.")+ return SplunkEvidenceResult(applicable=True, queried=False, error=str(exc), query=spl)++ count = len(results)+ digest = hashlib.sha256(json.dumps(results, sort_keys=True, default=str).encode("utf-8")).hexdigest()[:16]+ samples = _redact_samples(results[:3])++ findings = [+ f"Splunk evidence: {count} surrounding event(s) via bounded query "+ f"[head {_MAX_RESULTS}, window ±{int(_WINDOW_HOURS)}h] digest={digest}.",+ ]+ findings.extend(f"Splunk sample: {s}" for s in samples)+ for f in findings:+ state.add_finding(f)+ state.confidence_basis.append(f"splunk_evidence_count={count} digest={digest}")++ await _record_evidence_ledger(state, query=spl, count=count, digest=digest)++ return SplunkEvidenceResult(+ applicable=True,+ queried=True,+ result_count=count,+ query=spl,+ findings=findings,+ )+++async def _record_evidence_ledger(state: InvestigationState, *, query: str, count: int, digest: str) -> None:+ """Best-effort ledger event citing the query + result count + digest.++ Records ONLY the query, count, and content hash — never credentials.+ """+ try:+ await ledger_module.record_event(+ run_id=state.run_id,+ tenant_id=state.tenant_id,+ seq=900, # high seq bucket for tool evidence; ON CONFLICT dedups replays+ kind="splunk_evidence",+ agent="splunk_evidence_tool",+ summary=f"Splunk evidence: {count} events (digest {digest})",+ payload={"query": query, "result_count": count, "digest": digest},+ )+ except Exception as exc: # noqa: BLE001 — audit is best-effort+ logger.debug("splunk_evidence.ledger_noop", error=type(exc).__name__)diff --git a/services/agents/app/workers/fused_alert_consumer.py b/services/agents/app/workers/fused_alert_consumer.pyindex aab753e01..1a8f39cff 100644--- a/services/agents/app/workers/fused_alert_consumer.py+++ b/services/agents/app/workers/fused_alert_consumer.py@@ -31,17 +31,22 @@ never dies on one poison alert.from __future__ import annotations+import asyncio+import contextlibimport jsonimport osimport uuid+from datetime import UTC, datetimefrom typing import Anyimport structlogfrom app.agents.auto_triage_agent import run_auto_triage+from app.agents.dispositions import NEEDS_REVIEWfrom app.agents.triage_agent import run_triagefrom app.core.cost_governor import Decision, get_governorfrom app.core.cost_telemetry import CostTracker+from app.graph.runner import default_budget, run_escalationfrom app.investigator import ledger as ledger_modulefrom app.llm.factory import llm_overridefrom app.models.state import AgentStatus, InvestigationState@@ -53,6 +58,16 @@ logger = structlog.get_logger()_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "tryaisoc.com/agents/auto-triage")+# Idempotency key component (issue #571): the deterministic run_id is derived+# from (canonical alert id, workflow version), so a replayed fused message+# reuses the same run_id and cannot duplicate ledger runs/outcomes. Bump this+# when the triage workflow changes in a way that should re-run past alerts.+WORKFLOW_VERSION = "auto-triage-v1"++# Bounded retries before dead-lettering a poison alert (issue #571).+_MAX_ATTEMPTS = int(os.getenv("AISOC_AGENT_MAX_ATTEMPTS", "3"))+_RETRY_BACKOFF_S = float(os.getenv("AISOC_AGENT_RETRY_BACKOFF_S", "0.5"))+_METRICS = {"triaged": 0,"deduplicated": 0,@@ -60,10 +75,21 @@ _METRICS = {"llm": 0,"bc_suppressed": 0,"bc_mutated": 0,+ "needs_review_fallback": 0,+ "escalated": 0,+ "persist_retries": 0,+ "dead_lettered": 0,"errors": 0,}+def _escalation_enabled() -> bool:+ """Route non-auto-closed alerts through the full investigation graph+ (issue #569). On by default; set ``AISOC_AGENT_ESCALATE_TO_GRAPH=0`` to+ keep triage-only (e.g. in CI / low-resource deployments)."""+ return os.getenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "1").strip().lower() not in {"0", "false", "no", "off"}++def _coerce_uuid(value: Any, *, fallback: str) -> uuid.UUID:try:return uuid.UUID(str(value))@@ -71,6 +97,15 @@ def _coerce_uuid(value: Any, *, fallback: str) -> uuid.UUID:return uuid.uuid5(_NAMESPACE, str(value or fallback))+def _rationale_of(state: InvestigationState) -> str:+ """Human-readable rationale for the verdict, for the ledger + alerts row."""+ if state.confidence_basis:+ return " · ".join(str(b) for b in state.confidence_basis)[:8000]+ if state.findings:+ return " · ".join(str(f) for f in state.findings)[:8000]+ return f"Auto-triage verdict={state.verdict} confidence={state.confidence:.2f}"++def build_state(message: dict[str, Any]) -> InvestigationState | None:"""Map an ``aisoc.alerts.fused`` message to a seeded InvestigationState."""if not isinstance(message, dict):@@ -80,11 +115,15 @@ def build_state(message: dict[str, Any]) -> InvestigationState | None:return Nonetenant = _coerce_uuid(message.get("tenant_id") or alert.get("tenant_id"), fallback="default")incident = _coerce_uuid(message.get("incident_id") or message.get("id") or alert.get("id"), fallback=str(tenant))+ # Canonical alert row id (issue #568): the fused envelope now carries the+ # durable alerts.id as `alert_row_id`, falling back to the (also canonical)+ # message id. This is what the verdict is persisted against.+ alert_row_id = str(message.get("alert_row_id") or message.get("id") or alert.get("id") or "")summary = str(alert.get("title") or "").strip()if message.get("narrative"):summary = f"{summary} — {message['narrative']}"[:1000] if summary else str(message["narrative"])[:1000]raw_alert = {- "id": str(message.get("id") or alert.get("id") or ""),+ "id": alert_row_id,"severity": alert.get("severity"),"src_ip": alert.get("src_ip"),"dst_ip": alert.get("dst_ip"),@@ -98,8 +137,18 @@ def build_state(message: dict[str, Any]) -> InvestigationState | None:"raw_event": alert.get("raw_event", {}),"confidence": message.get("confidence_score"),"fusion_decision": message.get("fusion_decision"),+ # Connector provenance (issue #568) carried through the graph so the+ # investigation (e.g. the Splunk evidence tool, #570) can resolve the+ # originating connector instance.+ "connector_id": alert.get("connector_id"),+ "connector_type": alert.get("connector_type"),+ "source_event_ids": alert.get("source_event_ids", []),}+ # Deterministic, replay-stable run id (issue #571) — same alert + workflow+ # version ⇒ same run, so replays are idempotent in the ledger.+ run_id = uuid.uuid5(_NAMESPACE, f"{alert_row_id}:{WORKFLOW_VERSION}") if alert_row_id else uuid.uuid4()return InvestigationState(+ run_id=run_id,incident_id=incident,tenant_id=tenant,alert_summary=summary,@@ -117,12 +166,17 @@ class FusedAlertTriageWorker:bootstrap_servers: str,topic: str = "aisoc.alerts.fused",group_id: str = "aisoc-agents-triage",+ dlq_topic: str | None = None,+ max_attempts: int = _MAX_ATTEMPTS,business_context: BusinessContextApplier | None = None,) -> None:self._bootstrap = bootstrap_serversself._topic = topicself._group_id = group_id+ self._dlq_topic = dlq_topic or os.getenv("KAFKA_TOPIC_ALERTS_FUSED_DLQ", f"{topic}.dlq")+ self._max_attempts = max(1, max_attempts)self._consumer: Any | None = None+ self._producer: Any | None = None # lazily created for the DLQself._running = False# Phase B4 — environment-specific noise reduction applied post-fusion →# pre-triage. None = disabled (no rules file / flag off).@@ -148,24 +202,99 @@ class FusedAlertTriageWorker:async for msg in self._consumer:if not self._running:break+ # Commit ONLY after a durable outcome (verdict + ledger written)+ # or after the poison alert is dead-lettered — so a crash between+ # inference and persistence replays the alert instead of losing+ # it, and offsets advance only on durable completion (issue #571).+ await self._process_with_retry(msg.value)try:- await self.triage(msg.value)- except Exception as exc: # noqa: BLE001 — one poison alert must not kill the loop- _METRICS["errors"] += 1- logger.error("auto_triage_worker.error", error=str(exc), exc_info=True)- else:- try:- await self._consumer.commit()- except Exception as commit_exc: # noqa: BLE001 — reprocess on restart- logger.warning("auto_triage_worker.commit_failed", error=str(commit_exc))+ await self._consumer.commit()+ except Exception as commit_exc: # noqa: BLE001 — reprocess on restart+ logger.warning("auto_triage_worker.commit_failed", error=str(commit_exc))finally:await self._consumer.stop()+ async def _process_with_retry(self, message: Any) -> bool:+ """Triage one alert with bounded retries; dead-letter on exhaustion.++ Returns True on durable success, False if the message was dead-lettered.+ Either way the caller commits (a poison alert must not wedge the loop).+ """+ last_error: str = ""+ stage = "triage"+ for attempt in range(1, self._max_attempts + 1):+ try:+ await self.triage(message)+ return True+ except ledger_module.LedgerPersistError as exc:+ stage = "persist"+ last_error = str(exc)+ _METRICS["persist_retries"] += 1+ logger.warning("auto_triage_worker.persist_retry", attempt=attempt, error=last_error)+ except Exception as exc: # noqa: BLE001 — retry, then dead-letter+ stage = "triage"+ last_error = str(exc)+ logger.warning("auto_triage_worker.triage_retry", attempt=attempt, error=last_error)+ if attempt < self._max_attempts:+ await asyncio.sleep(self._retry_backoff(attempt))+ _METRICS["errors"] += 1+ await self._send_to_dlq(message, attempts=self._max_attempts, stage=stage, error=last_error)+ return False++ def _retry_backoff(self, attempt: int) -> float:+ return _RETRY_BACKOFF_S * attempt++ async def _ensure_producer(self) -> Any | None:+ if self._producer is not None:+ return self._producer+ try:+ from aiokafka import AIOKafkaProducer # noqa: PLC0415 — optional dep, runtime only++ self._producer = AIOKafkaProducer(+ bootstrap_servers=self._bootstrap,+ value_serializer=lambda v: json.dumps(v).encode("utf-8"),+ )+ await self._producer.start()+ return self._producer+ except Exception as exc: # noqa: BLE001 — DLQ is best-effort+ logger.error("auto_triage_worker.dlq_producer_failed", error=str(exc))+ self._producer = None+ return None++ async def _send_to_dlq(self, message: Any, *, attempts: int, stage: str, error: str) -> None:+ """Publish a poison alert to the DLQ with alert id, attempts, and stage."""+ alert_id = ""+ if isinstance(message, dict):+ alert = message.get("alert") if isinstance(message.get("alert"), dict) else {}+ alert_id = str(message.get("alert_row_id") or message.get("id") or alert.get("id") or "")+ record = {+ "alert_id": alert_id,+ "attempts": attempts,+ "failure_stage": stage,+ "error": (error or "")[:2000],+ "dead_lettered_at": datetime.now(UTC).isoformat(),+ "original": message if isinstance(message, dict) else {"raw": str(message)[:2000]},+ }+ producer = await self._ensure_producer()+ if producer is None:+ logger.error("auto_triage_worker.dlq_unavailable", alert_id=alert_id, stage=stage)+ return+ try:+ await producer.send(self._dlq_topic, value=record)+ _METRICS["dead_lettered"] += 1+ logger.error("auto_triage_worker.dead_lettered", alert_id=alert_id, stage=stage, attempts=attempts)+ except Exception as exc: # noqa: BLE001 — never wedge the loop on a DLQ failure+ logger.error("auto_triage_worker.dlq_send_failed", alert_id=alert_id, error=str(exc))+async def stop(self) -> None:self._running = Falseif self._consumer is not None:await self._consumer.stop()self._consumer = None+ if self._producer is not None:+ with contextlib.suppress(Exception):+ await self._producer.stop()+ self._producer = Nonelogger.info("auto_triage_worker.stopped", metrics=dict(_METRICS))async def triage(self, message: dict[str, Any]) -> dict[str, Any] | None:@@ -228,6 +357,15 @@ class FusedAlertTriageWorker:tokens = tracker.total_tokensverdict = state.verdictconfidence = state.confidence+ # Never complete with a null/empty verdict (issue #571): if both the+ # LLM and deterministic paths somehow left it unset, fail safe to+ # needs_review so the alert is escalated to a human, not auto-closed.+ if not verdict:+ verdict = state.verdict = NEEDS_REVIEW+ if state.status is AgentStatus.COMPLETED:+ state.status = AgentStatus.RUNNING+ _METRICS["needs_review_fallback"] += 1+ state.add_finding("Triage produced no verdict — defaulting to needs_review (escalated).")# Close the cost-governor loop: cache the verdict (so an alert flood# dedups instead of re-paying) and account the spend (so per-tenant# budgets + the circuit breaker actually fire). Best-effort.@@ -245,6 +383,14 @@ class FusedAlertTriageWorker:_METRICS["triaged"] += 1await self._record(state, tier=tier, verdict=verdict, confidence=confidence, tokens=tokens, cost_usd=cost_usd)+ # Issue #569: route escalations (anything NOT auto-closed — TP,+ # low-confidence, needs_review) through the full investigation graph.+ # High-confidence FP/BTP already terminated (status COMPLETED) and+ # skip enrichment. Best-effort: the verdict is already durable, so an+ # enrichment/investigation failure never fails the triage outcome.+ if state.status is not AgentStatus.COMPLETED:+ await self._maybe_escalate(state)+return {"run_id": str(state.run_id),"incident_id": str(state.incident_id),@@ -258,6 +404,23 @@ class FusedAlertTriageWorker:"proposed_actions": [{"action_type": a.action_type, "requires_approval": a.requires_approval} for a in state.proposed_actions],}+ async def _maybe_escalate(self, state: InvestigationState) -> None:+ """Run the full investigation graph for an escalated alert (issue #569).++ Shares the same graph runner as the manual investigations API. Every+ node is recorded to the ledger under the deterministic run_id, so a+ restart re-runs idempotently. Best-effort + budgeted — a failure or+ timeout leaves the durable verdict intact and the alert escalated.+ """+ if not _escalation_enabled():+ return+ try:+ async with CostTracker(run_id=str(state.run_id), tenant_id=str(state.tenant_id)):+ await run_escalation(state, budget=default_budget(), seq_start=1)+ _METRICS["escalated"] += 1+ except Exception as exc: # noqa: BLE001 — escalation is best-effort over a durable verdict+ logger.warning("auto_triage_worker.escalation_failed", run_id=str(state.run_id), error=str(exc))+async def _resolve_tenant_llm(self, tenant_id: uuid.UUID) -> Any:"""Resolve the tenant's LLM config, or None to force deterministic triage."""try:@@ -288,27 +451,42 @@ class FusedAlertTriageWorker:tokens: int = 0,cost_usd: float = 0.0,) -> None:- """Best-effort ledger write. No DB => no-op (never raises)."""- try:- await ledger_module.start_run(- run_id=state.run_id,- case_id=str(state.incident_id),- tenant_ref=str(state.tenant_id),- alert_summary=state.alert_summary or "",- raw_alert=state.raw_alert or {},- model_used=f"kafka:auto_triage:{tier}",- )- await ledger_module.complete_run(- run_id=state.run_id,- tenant_id=state.tenant_id,- status="completed",- error=None,- iterations=state.iteration_count,- total_tokens=tokens,- total_cost_usd=cost_usd,- )- except Exception as exc: # noqa: BLE001 — audit is best-effort- logger.debug("auto_triage_worker.ledger_noop", error=str(exc), verdict=verdict, confidence=confidence)+ """Durably persist the triage outcome (issue #571).++ Writes the verdict/confidence/rationale/recommendations to BOTH the+ ledger and the ``alerts`` row in one transaction. No DB configured =>+ no-op (returns without raising). A real DB error raises+ :class:`LedgerPersistError` so the caller retries / dead-letters — the+ Kafka offset is not committed until this succeeds.+ """+ rationale = _rationale_of(state)+ recommendations = [+ {+ "action_type": a.action_type,+ "description": a.description,+ "risk_level": a.risk_level.value if hasattr(a.risk_level, "value") else str(a.risk_level),+ "requires_approval": a.requires_approval,+ "target": a.target,+ }+ for a in state.proposed_actions+ ]+ await ledger_module.persist_auto_triage(+ run_id=state.run_id,+ alert_id=(state.raw_alert or {}).get("id"),+ tenant_ref=str(state.tenant_id),+ alert_summary=state.alert_summary or "",+ raw_alert=state.raw_alert or {},+ tier=tier,+ verdict=str(verdict) if verdict else NEEDS_REVIEW,+ confidence=float(confidence or 0.0),+ rationale=rationale,+ findings=list(state.findings),+ proposed_actions=recommendations,+ auto_closed=state.status is AgentStatus.COMPLETED,+ iterations=state.iteration_count,+ tokens=tokens,+ cost_usd=cost_usd,+ )@staticmethoddef get_metrics() -> dict[str, int]:diff --git a/services/agents/tests/test_auto_triage_dispositions.py b/services/agents/tests/test_auto_triage_dispositions.pyindex 76fa38f54..df6b1c8db 100644--- a/services/agents/tests/test_auto_triage_dispositions.py+++ b/services/agents/tests/test_auto_triage_dispositions.py@@ -126,6 +126,30 @@ async def test_malicious_true_positive_escalates(monkeypatch):assert out.status != AgentStatus.COMPLETED+async def test_llm_timeout_raises_typed_error_not_null_verdict(monkeypatch):+ # Issue #571: an LLM failure must raise AutoTriageError (so the worker falls+ # back to deterministic), NOT silently return a null-verdict RUNNING state.+ async def _boom(_llm, _messages):+ raise TimeoutError("llm timed out")++ monkeypatch.setattr(ata, "make_chat_model", lambda *a, **k: object())+ monkeypatch.setattr(ata, "safe_ainvoke", _boom)+ with pytest.raises(ata.AutoTriageError):+ await ata.run_auto_triage(_state("Beaconing", {"severity": "critical"}))+++async def test_malformed_json_raises_typed_error(monkeypatch):+ async def _bad(_llm, _messages):+ return SimpleNamespace(content="the model rambled with no json")++ monkeypatch.setattr(ata, "make_chat_model", lambda *a, **k: object())+ monkeypatch.setattr(ata, "safe_ainvoke", _bad)+ with pytest.raises(ata.AutoTriageError):+ await ata.run_auto_triage(_state("Suspicious", {"severity": "high"}))++@pytest.mark.asyncioasync def test_btp_metric_is_tracked_separately_from_fp(monkeypatch):ata._metrics.update({"fp_count": 0, "btp_count": 0, "benign_count": 0, "tp_count": 0, "total_processed": 0, "confidence_sum": 0.0})diff --git a/services/agents/tests/test_fused_alert_worker.py b/services/agents/tests/test_fused_alert_worker.pyindex 769e25de0..f8b1e7638 100644--- a/services/agents/tests/test_fused_alert_worker.py+++ b/services/agents/tests/test_fused_alert_worker.py@@ -48,14 +48,22 @@ def _fused(**alert_overrides) -> dict:@pytest.fixture(autouse=True)def _no_ledger_db(monkeypatch):- # Ledger writes are best-effort; make them explicit no-ops in tests.+ # No DB in unit tests: the durable persist path no-ops. Stub it explicitly+ # so a stray DATABASE_URL in the CI env can't make tests hit a real DB.async def _noop(*args, **kwargs): # noqa: ANN002, ANN003return None+ async def _persist_noop(*args, **kwargs): # noqa: ANN002, ANN003+ return False+monkeypatch.setattr(worker_mod.ledger_module, "start_run", _noop)monkeypatch.setattr(worker_mod.ledger_module, "complete_run", _noop)+ monkeypatch.setattr(worker_mod.ledger_module, "persist_auto_triage", _persist_noop)# Force the deterministic path unless a test overrides (no LLM key in CI).monkeypatch.setenv("AISOC_DETERMINISTIC", "0")+ # Escalation runs the full graph (external calls); off by default in unit+ # tests — the dedicated escalation tests enable it and stub run_escalation.+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "0")def _worker() -> FusedAlertTriageWorker:@@ -132,6 +140,145 @@ async def test_circuit_open_forces_deterministic(monkeypatch):assert result["tier"] == "deterministic"+# ── #571: durable verdict, typed fallback, idempotency, DLQ ──────────────────+++async def test_llm_failure_falls_back_to_deterministic_non_null_verdict(monkeypatch):+ # An LLM error must NOT produce a null verdict: the worker falls back to+ # deterministic triage (which always yields a verdict).+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))+ monkeypatch.setenv("AISOC_DETERMINISTIC", "0")++ async def _raise(_state):+ raise RuntimeError("llm timeout")++ monkeypatch.setattr(worker_mod, "run_auto_triage", _raise)+ result = await _worker().triage(_fused())+ assert result["tier"] == "deterministic"+ assert result["verdict"] is not None+++async def test_null_verdict_defaults_to_needs_review(monkeypatch):+ # Even if a triage path leaves the verdict unset, we never complete null —+ # fail safe to needs_review (escalated, not auto-closed).+ monkeypatch.setenv("AISOC_DETERMINISTIC", "1")+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))++ async def _null(state):+ state.verdict = None+ return state++ monkeypatch.setattr(worker_mod, "run_triage", _null)+ result = await _worker().triage(_fused())+ assert result["verdict"] == "needs_review"+++async def test_deterministic_run_id_is_idempotent():+ # Same alert + workflow version ⇒ same run_id, so replays don't duplicate.+ a = build_state(_fused())+ b = build_state(_fused())+ assert a.run_id == b.run_id+ # A different alert id yields a different run.+ c = build_state(_fused() | {"id": "44444444-4444-4444-4444-444444444444", "alert_row_id": "44444444-4444-4444-4444-444444444444"})+ assert c.run_id != a.run_id+++async def test_persist_failure_is_retried_then_dead_lettered(monkeypatch):+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))++ async def _boom(*args, **kwargs): # noqa: ANN002, ANN003+ raise worker_mod.ledger_module.LedgerPersistError("db down")++ monkeypatch.setattr(worker_mod.ledger_module, "persist_auto_triage", _boom)++ sent: list[tuple[str, dict]] = []++ class _FakeProducer:+ async def send(self, topic, value): # noqa: ANN001+ sent.append((topic, value))++ worker = FusedAlertTriageWorker(bootstrap_servers="unused", max_attempts=2)+ monkeypatch.setattr(worker, "_retry_backoff", lambda attempt: 0.0)++ async def _fake_producer():+ return _FakeProducer()++ monkeypatch.setattr(worker, "_ensure_producer", _fake_producer)++ ok = await worker._process_with_retry(_fused())+ assert ok is False # exhausted retries → dead-lettered+ assert len(sent) == 1+ topic, record = sent[0]+ assert topic.endswith(".dlq")+ assert record["failure_stage"] == "persist"+ assert record["attempts"] == 2+ assert record["alert_id"] == "22222222-2222-2222-2222-222222222222"+++async def test_successful_triage_returns_true_and_commits(monkeypatch):+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))+ worker = FusedAlertTriageWorker(bootstrap_servers="unused", max_attempts=2)+ # persist_auto_triage is stubbed to no-op by the autouse fixture.+ assert await worker._process_with_retry(_fused()) is True+++# ── #569: escalation into the full investigation graph ───────────────────────+++async def test_non_auto_closed_alert_escalates_to_full_graph(monkeypatch):+ # A critical cred-dump triages to a non-benign verdict (not auto-closed) →+ # it must enter the full investigation graph.+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "1")+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))+ called: dict = {}++ async def _fake_escalation(state, **kwargs): # noqa: ANN003+ called["state"] = state+ return state++ monkeypatch.setattr(worker_mod, "run_escalation", _fake_escalation)+ result = await _worker().triage(_fused())+ assert result["verdict"] not in ("false_positive", "benign", "benign_true_positive")+ assert "state" in called # escalation ran+++async def test_high_confidence_auto_closed_does_not_escalate(monkeypatch):+ # A high-confidence FP that auto-closed must terminate WITHOUT enrichment.+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "1")+ monkeypatch.setenv("AISOC_DETERMINISTIC", "1")+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))++ async def _auto_closed(state):+ state.verdict = "false_positive"+ state.status = worker_mod.AgentStatus.COMPLETED+ return state++ monkeypatch.setattr(worker_mod, "run_triage", _auto_closed)+ calls = {"n": 0}++ async def _fake_escalation(state, **kwargs): # noqa: ANN003+ calls["n"] += 1+ return state++ monkeypatch.setattr(worker_mod, "run_escalation", _fake_escalation)+ await _worker().triage(_fused())+ assert calls["n"] == 0+++async def test_escalation_can_be_disabled_by_flag(monkeypatch):+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "0")+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))+ calls = {"n": 0}++ async def _fake_escalation(state, **kwargs): # noqa: ANN003+ calls["n"] += 1+ return state++ monkeypatch.setattr(worker_mod, "run_escalation", _fake_escalation)+ await _worker().triage(_fused())+ assert calls["n"] == 0++# ── fail-soft ─────────────────────────────────────────────────────────────────@@ -156,6 +303,8 @@ def _fake_llm_config(*, allowed: bool):def __init__(self) -> None:self.allowed = allowedself.api_key = "sk-test" if allowed else None+ self.base_url = None+ self.model = "gpt-4o-mini"async def _resolve(_tenant): # noqa: ANN001return _Cfg()diff --git a/services/agents/tests/test_graph_runner.py b/services/agents/tests/test_graph_runner.pynew file mode 100644index 000000000..53efe0177--- /dev/null+++ b/services/agents/tests/test_graph_runner.py@@ -0,0 +1,100 @@+"""#569 — the shared, durable investigation-graph runner.++Proves the runner records one ledger event per graph node (under the run's+deterministic id, with a seq offset so it follows the worker's triage_verdict+event) and that a wall-clock budget bounds the run and fails safe to+needs_review on timeout. A tiny fake graph stands in for the compiled+LangGraph so the runner logic is tested in isolation (no external calls).+"""++from __future__ import annotations++import asyncio+import uuid+from uuid import uuid4++import pytest+from app.graph import runner as runner_mod+from app.graph.runner import InvestigationBudget, _run+from app.models.state import InvestigationState++pytestmark = pytest.mark.asyncio+++class _FakeGraph:+ def __init__(self, steps: list[dict], *, delay: float = 0.0) -> None:+ self._steps = steps+ self._delay = delay++ async def astream(self, _state_dict): # noqa: ANN001+ for step in self._steps:+ if self._delay:+ await asyncio.sleep(self._delay)+ yield step+++def _state() -> InvestigationState:+ return InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="cred dump")+++async def test_runner_records_one_ledger_event_per_node(monkeypatch):+ events: list[dict] = []++ async def _rec(**kwargs): # noqa: ANN003+ events.append(kwargs)+ return uuid4()++ async def _resolve(_ref): # noqa: ANN001+ return uuid4()++ async def _noop(**kwargs): # noqa: ANN003+ return None++ monkeypatch.setattr(runner_mod.ledger_module, "record_event", _rec)+ monkeypatch.setattr(runner_mod.ledger_module, "resolve_tenant", _resolve)+ monkeypatch.setattr(runner_mod.ledger_module, "start_run", _noop)+ monkeypatch.setattr(runner_mod.ledger_module, "complete_run", _noop)++ state = _state()+ graph = _FakeGraph(+ [+ {"triage": {"verdict": "true_positive"}},+ {"enrichment": {"findings": ["enriched"]}},+ {"attack_path": {"status": "completed"}},+ ]+ )+ await _run(graph, state, budget=InvestigationBudget(max_seconds=5), persist=True, seq_start=1)++ assert [e["agent"] for e in events] == ["triage", "enrichment", "attack_path"]+ # seq follows the worker's triage_verdict event (seq 1) without colliding.+ assert [e["seq"] for e in events] == [2, 3, 4]+ assert all(e["kind"] == "graph_step" for e in events)+++async def test_runner_budget_timeout_fails_safe_to_needs_review():+ state = _state()+ # Each step sleeps 0.2s; a 0.05s budget times out before completion.+ graph = _FakeGraph([{"triage": {}}, {"enrichment": {}}], delay=0.2)+ result = await _run(graph, state, budget=InvestigationBudget(max_seconds=0.05), persist=False, seq_start=0)+ assert result.verdict == "needs_review"+ assert any("budget" in f.lower() for f in result.findings)+++async def test_runner_merges_node_outputs_into_final_state(monkeypatch):+ async def _resolve(_ref): # noqa: ANN001+ return None # persist path disabled (no tenant) — pure merge test++ monkeypatch.setattr(runner_mod.ledger_module, "resolve_tenant", _resolve)++ state = _state()+ graph = _FakeGraph(+ [+ {"triage": {"verdict": "true_positive", "confidence": 0.7}},+ {"enrichment": {"findings": ["ioc malicious"]}},+ ]+ )+ result = await _run(graph, state, budget=InvestigationBudget(max_seconds=5), persist=False, seq_start=0)+ assert result.verdict == "true_positive"+ assert result.confidence == 0.7+ assert "ioc malicious" in result.findings+ assert isinstance(result.run_id, uuid.UUID)diff --git a/services/agents/tests/test_splunk_evidence.py b/services/agents/tests/test_splunk_evidence.pynew file mode 100644index 000000000..f0ba0398e--- /dev/null+++ b/services/agents/tests/test_splunk_evidence.py@@ -0,0 +1,217 @@+"""#570 — bounded, read-only Splunk evidence tool for the investigation graph.++Verifies the query is time/entity-bounded and allow-listed, credentials come+from the vault (never a prompt) and never leak into findings/queries, a+failure/timeout is treated as *missing evidence* (escalates to needs_review),+and results are cited (redacted + hashed) in the investigation. The Splunk REST+call is mocked (fake httpx client) so it runs in agents CI with no live server.+"""++from __future__ import annotations++from uuid import uuid4++import pytest+from app.agents import investigation_agent as ia+from app.agents.dispositions import FALSE_POSITIVE, NEEDS_REVIEW+from app.investigator import splunk_evidence as se+from app.investigator.splunk_evidence import (+ SplunkCreds,+ SplunkEvidenceClient,+ SplunkEvidenceResult,+ build_evidence_spl,+ collect_splunk_evidence,+)+from app.models.state import AgentStatus, InvestigationState++pytestmark = pytest.mark.asyncio++TOKEN = "s3cr3t-splunk-token-should-never-leak"+++def _state(**raw) -> InvestigationState:+ base = {+ "id": "22222222-2222-2222-2222-222222222222",+ "connector_id": "00000000-0000-0000-0000-0000000000c1",+ "connector_type": "splunk",+ "username": "svc-backup",+ "hostname": "WIN-DC01",+ "src_ip": "10.20.30.40",+ "event_time": "2026-07-12T08:00:00Z",+ }+ base.update(raw)+ return InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="cred dump", raw_alert=base, status=AgentStatus.RUNNING)+++# ── SPL builder (pure) ───────────────────────────────────────────────────────+++async def test_build_evidence_spl_is_time_and_entity_bounded():+ spl = build_evidence_spl(+ {"username": "svc-backup", "hostname": "WIN-DC01", "src_ip": "10.0.0.5", "file_hash": "a" * 64},+ index="main",+ max_results=50,+ )+ assert spl is not None+ assert spl.startswith("search index=main (")+ assert 'user="svc-backup"' in spl+ assert 'host="WIN-DC01"' in spl+ assert 'src_ip="10.0.0.5"' in spl+ assert "| head 50" in spl # bounded result count+++async def test_build_evidence_spl_none_without_pivot():+ assert build_evidence_spl({"severity": "high"}) is None+++async def test_spl_values_are_quote_escaped():+ # An adversary-controlled username with a quote must not break out of the+ # SPL string (allow-listed construction).+ spl = build_evidence_spl({"username": 'a" OR host="*'})+ assert spl is not None+ assert '\\"' in spl # the inner quote is escaped+ assert 'user="a\\" OR host=\\"*"' in spl+++async def test_time_window_is_bounded():+ earliest, latest = se._time_window({"event_time": "2026-07-12T08:00:00Z"})+ # epoch strings, 2 * window hours apart+ span_hours = (int(latest) - int(earliest)) / 3600+ assert span_hours == pytest.approx(2 * se._WINDOW_HOURS, abs=0.01)+++# ── credential resolution + collection ───────────────────────────────────────+++async def test_collect_not_applicable_without_connector():+ # No DB in tests => resolve returns None => not a Splunk alert => no-op.+ result = await collect_splunk_evidence(_state(connector_id=None))+ assert result.applicable is False+ assert result.queried is False+++async def test_collect_success_cites_redacted_hashed_evidence(monkeypatch):+ async def _creds(_state):+ return SplunkCreds(base_url="https://splunk.local:8089", token=TOKEN, ssl_verify=True, index="main")++ class _FakeClient:+ def __init__(self, creds, **kwargs): # noqa: ANN001, ANN003+ self._creds = creds++ async def search(self, spl, *, earliest, latest): # noqa: ANN001+ return [{"_time": "2026-07-12T07:59:00Z", "host": "WIN-DC01", "user": "svc-backup", "action": "lsass_access"}]++ monkeypatch.setattr(se, "resolve_splunk_credentials", _creds)+ monkeypatch.setattr(se, "_make_client", lambda creds: _FakeClient(creds))++ state = _state()+ result = await collect_splunk_evidence(state)++ assert result.applicable is True+ assert result.queried is True+ assert result.result_count == 1+ # Evidence cited in findings with a content hash…+ joined = "\n".join(state.findings)+ assert "Splunk evidence: 1 surrounding event" in joined+ assert "digest=" in joined+ # …and the credential NEVER appears in findings or the query.+ assert TOKEN not in joined+ assert result.query is not None and TOKEN not in result.query+++async def test_collect_query_failure_is_missing_evidence(monkeypatch):+ async def _creds(_state):+ return SplunkCreds(base_url="https://splunk.local:8089", token=TOKEN)++ class _BoomClient:+ def __init__(self, creds, **kwargs): # noqa: ANN001, ANN003+ pass++ async def search(self, *a, **k): # noqa: ANN002, ANN003+ raise TimeoutError("splunk timed out")++ monkeypatch.setattr(se, "resolve_splunk_credentials", _creds)+ monkeypatch.setattr(se, "_make_client", lambda creds: _BoomClient(creds))++ state = _state()+ result = await collect_splunk_evidence(state)+ assert result.applicable is True+ assert result.queried is False+ assert result.error is not None+ assert any("missing evidence" in f.lower() for f in state.findings)+ assert TOKEN not in "\n".join(state.findings)+++# ── bounded REST client (mocked httpx) ───────────────────────────────────────+++async def test_client_issues_bounded_oneshot_request(monkeypatch):+ captured: dict = {}++ class _FakeResp:+ def raise_for_status(self):+ return None++ def json(self):+ return {"results": [{"_time": "t", "host": "h"}, {"_time": "t2", "host": "h2"}]}++ class _FakeAsyncClient:+ def __init__(self, *args, **kwargs):+ captured["init"] = kwargs++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ return None++ async def post(self, url, headers=None, data=None): # noqa: ANN001+ captured["url"] = url+ captured["headers"] = headers+ captured["data"] = data+ return _FakeResp()++ monkeypatch.setattr(se.httpx, "AsyncClient", _FakeAsyncClient)++ creds = SplunkCreds(base_url="https://splunk.local:8089", token=TOKEN, ssl_verify=False)+ client = SplunkEvidenceClient(creds, max_results=50)+ results = await client.search('search index=main (host="h") | head 50', earliest="-6h", latest="now")++ assert len(results) == 2+ assert captured["url"].endswith("/services/search/jobs")+ assert captured["data"]["exec_mode"] == "oneshot" # read-only oneshot+ assert captured["data"]["count"] == "50" # bounded+ assert captured["data"]["earliest_time"] == "-6h"+ assert captured["data"]["latest_time"] == "now"+ # verify=False threaded to httpx; token in the header only (never returned).+ assert captured["init"].get("verify") is False+ assert captured["headers"]["Authorization"] == f"Bearer {TOKEN}"+++# ── investigation-agent escalation on missing evidence ───────────────────────+++async def test_investigation_escalates_to_needs_review_on_missing_evidence(monkeypatch):+ async def _missing(_state):+ return SplunkEvidenceResult(applicable=True, queried=False, error="timeout")++ monkeypatch.setattr(ia, "collect_splunk_evidence", _missing)+ # Force the base scorer to a dismissive verdict; the #570 guard must upgrade it.+ monkeypatch.setattr(ia, "score_investigation", lambda state: (0.9, ["scored fp"], FALSE_POSITIVE))++ state = InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="x", raw_alert={"connector_type": "splunk"})+ result = await ia.run_investigation(state)+ assert result.verdict == NEEDS_REVIEW+++async def test_investigation_keeps_verdict_when_evidence_present(monkeypatch):+ async def _ok(_state):+ return SplunkEvidenceResult(applicable=True, queried=True, result_count=3)++ monkeypatch.setattr(ia, "collect_splunk_evidence", _ok)+ monkeypatch.setattr(ia, "score_investigation", lambda state: (0.9, ["scored fp"], FALSE_POSITIVE))++ state = InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="x", raw_alert={"connector_type": "splunk"})+ result = await ia.run_investigation(state)+ # Evidence was successfully gathered → the scorer's verdict stands.+ assert result.verdict == FALSE_POSITIVEdiff --git a/services/api/app/models/alert.py b/services/api/app/models/alert.pyindex 82569aede..e93bfaf60 100644--- a/services/api/app/models/alert.py+++ b/services/api/app/models/alert.py@@ -33,6 +33,10 @@ class Alert(Base):connector_type: Mapped[str | None] = mapped_column(String(100), nullable=True)source_event_ids: Mapped[list] = mapped_column(JSONB, default=list)ocsf_class_uid: Mapped[int | None] = mapped_column(Integer, nullable=True)+ # Detection-rule / Splunk saved-search provenance (issue #568) — explicit+ # identifiers so downstream never parses them out of the title/source.+ rule_id: Mapped[str | None] = mapped_column(Text, nullable=True)+ rule_name: Mapped[str | None] = mapped_column(Text, nullable=True)# Analyst verdict / disposition (set via feedback endpoint)disposition: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)diff --git a/services/api/migrations/047_alerts_rule_provenance.sql b/services/api/migrations/047_alerts_rule_provenance.sqlnew file mode 100644index 000000000..f56d0f492--- /dev/null+++ b/services/api/migrations/047_alerts_rule_provenance.sql@@ -0,0 +1,16 @@+-- Issue #568: preserve source-rule provenance on the alert row.+--+-- Fusion previously encoded the firing detection rule only inside the+-- human-readable `source` ("detection:<id>") and `title` strings, so the+-- auto-triage worker and investigation graph had to parse them back out.+-- These columns carry the rule identifier + name (and, for vendor-asserted+-- Splunk notables, the saved-search name) as first-class fields.+--+-- Idempotent + transactional so the create_all lineage and repeated compose+-- installs stay green.+BEGIN;++ALTER TABLE alerts ADD COLUMN IF NOT EXISTS rule_id TEXT;+ALTER TABLE alerts ADD COLUMN IF NOT EXISTS rule_name TEXT;++COMMIT;diff --git a/services/fusion/app/models/alert.py b/services/fusion/app/models/alert.pyindex b81306a37..e2bcfd70e 100644--- a/services/fusion/app/models/alert.py+++ b/services/fusion/app/models/alert.py@@ -10,10 +10,18 @@ import jsonfrom datetime import datetimefrom enum import Enumfrom typing import Any-from uuid import UUID, uuid4+from uuid import UUID, uuid4, uuid5from pydantic import BaseModel, Field+# Fixed namespace for deriving the canonical, replay-stable alert UUID from an+# alert's dedup fingerprint (issue #568). Using uuid5 over a constant namespace+# means the same logical alert (same tenant + fingerprint) always resolves to+# the same UUID across RawAlert → FusedAlert → Kafka → Postgres → ledger → API,+# so a replayed source event maps to exactly one alert row instead of minting a+# fresh random id on every pass.+ALERT_ID_NAMESPACE = UUID("a15c0c00-0000-5568-a150-c000000a1e77")+class AlertSeverity(str, Enum):CRITICAL = "critical"@@ -68,9 +76,30 @@ class RawAlert(BaseModel):tags: list[str] = Field(default_factory=list)risk_score: float = 0.0+ # Provenance (issue #568) — carried first-class end-to-end so downstream+ # (auto-triage, the investigation graph, the Splunk evidence tool) can+ # resolve the originating connector and source events WITHOUT parsing the+ # human-readable title/source strings.+ connector_id: UUID | None = None+ connector_type: str | None = None+ source_event_ids: list[str] = Field(default_factory=list)+ ocsf_class_uid: int | None = None+ rule_id: str | None = None # detection rule / Splunk saved-search identifier+ rule_name: str | None = None+created_at: datetime = Field(default_factory=datetime.utcnow)event_time: datetime | None = None+ def deterministic_id(self) -> UUID:+ """Replay-stable canonical UUID derived from the dedup fingerprint.++ Same tenant + fingerprint ⇒ same UUID on every pass, so replays resolve+ to exactly one alert row (issue #568). This is intentionally aligned+ with :meth:`fingerprint` (the dedup key) — two events that dedup to the+ same row also share the same canonical id.+ """+ return uuid5(ALERT_ID_NAMESPACE, f"{self.tenant_id}:{self.fingerprint()}")+def fingerprint(self) -> str:"""Generate a stable deduplication fingerprint."""fields = {diff --git a/services/fusion/app/services/alert_sink.py b/services/fusion/app/services/alert_sink.pyindex a4cc3537d..abb336041 100644--- a/services/fusion/app/services/alert_sink.py+++ b/services/fusion/app/services/alert_sink.py@@ -21,6 +21,8 @@ Design constraints:from __future__ import annotationsimport json+from dataclasses import dataclass+from enum import Enumimport asyncpgimport structlog@@ -29,30 +31,69 @@ from app.models.alert import FusedAlert, FusionDecisionlogger = structlog.get_logger()++class PersistOutcome(str, Enum):+ """Distinguishable outcomes of a persist attempt (issue #568).++ The old sink collapsed "inserted", "already-existed", "DB down", and+ "insert errored" all into ``None``, so the caller could not tell a real+ duplicate from an outage it should retry. Each state is now explicit.+ """++ INSERTED = "inserted" # a new row was written; alert_id is the row id+ DUPLICATE = "duplicate" # already existed (dedup hit or id conflict)+ UNAVAILABLE = "unavailable" # DB unreachable — retryable, NOT a duplicate+ FAILED = "failed" # insert errored (bad tenant, SQL error) — observable+++@dataclass(frozen=True)+class PersistResult:+ """Outcome of :meth:`AlertSink.persist` plus the canonical alert id."""++ outcome: PersistOutcome+ alert_id: str | None = None++ @property+ def persisted(self) -> bool:+ return self.outcome is PersistOutcome.INSERTED++ @property+ def durable(self) -> bool:+ """True when the alert is known to exist in the store (new or dup)."""+ return self.outcome in (PersistOutcome.INSERTED, PersistOutcome.DUPLICATE)++_INSERT_SQL = """INSERT INTO alerts (- tenant_id, title, description, severity, status,+ id, tenant_id, title, description, severity, status,mitre_tactics, mitre_techniques, iocs, entities, raw_event,dedup_hash, confidence, confidence_label, confidence_rationale,- narrative, anomaly_score, event_time+ narrative, anomaly_score, event_time,+ connector_id, connector_type, source_event_ids, ocsf_class_uid,+ rule_id, rule_name)SELECT- $1, $2, $3, $4, 'new',- $5::jsonb, $6::jsonb, $7::jsonb, $8::jsonb, $9::jsonb,- $10::text, $11, $12, $13::jsonb,- $14, $15, COALESCE($16, NOW())+ $1, $2, $3, $4, $5, 'new',+ $6::jsonb, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb,+ $11::text, $12, $13, $14::jsonb,+ $15, $16, COALESCE($17, NOW()),+ $18, $19, $20::jsonb, $21,+ $22, $23WHERE NOT EXISTS (- SELECT 1 FROM alerts WHERE tenant_id = $1 AND dedup_hash = $10::text+ SELECT 1 FROM alerts WHERE tenant_id = $2 AND dedup_hash = $11::text)+ON CONFLICT (id) DO NOTHINGRETURNING id"""-# NB: $10 (dedup_hash) is cast to ::text in BOTH the INSERT target and the-# WHERE. `dedup_hash` is VARCHAR(64), and varchar comparisons resolve through-# text operators — so `WHERE dedup_hash = $10` deduces $10 as text while the-# INSERT target deduces it as varchar. asyncpg's prepare cannot unify the two-# ("inconsistent types deduced for parameter $10: text versus character-# varying") and the persist fails silently, so no alert row is ever written.-# Pinning both uses to ::text makes the deduction unambiguous.+# Two idempotency guards, complementary (issue #568):+# * WHERE NOT EXISTS (tenant_id, dedup_hash) — content dedup; also protects+# legacy rows written before ids were deterministic (random id, same hash).+# * ON CONFLICT (id) DO NOTHING — exact-replay idempotency on the canonical+# deterministic UUID (RawAlert.deterministic_id()).+# NB: $11 (dedup_hash) is cast ::text in BOTH the INSERT target and the WHERE.+# `dedup_hash` is VARCHAR(64); a varchar/text mismatch makes asyncpg's prepare+# fail to unify parameter $11 ("inconsistent types deduced"), silently dropping+# every write. Pinning both uses to ::text makes the deduction unambiguous.def _asyncpg_dsn(url: str) -> str:@@ -115,18 +156,23 @@ class AlertSink:await self.start()return self._pool- async def persist(self, fused: FusedAlert) -> str | None:- """Insert one fused alert. Returns the new row id, or ``None`` when- skipped (duplicate decision, dedup hit, or DB unavailable)."""+ async def persist(self, fused: FusedAlert) -> PersistResult:+ """Insert one fused alert, returning a structured :class:`PersistResult`.++ The canonical alert id is ``fused.id`` (deterministic — issue #568), so+ even a duplicate / outage carries the id the row resolves to once the+ DB is reachable.+ """+ canonical_id = str(fused.id)if fused.fusion_decision == FusionDecision.DUPLICATE:- return None+ return PersistResult(PersistOutcome.DUPLICATE, canonical_id)pool = await self._ensure_pool()if pool is None:if not self._connect_failed_logged:logger.error("alert_sink.unavailable_dropping_persistence")self._connect_failed_logged = True- return None+ return PersistResult(PersistOutcome.UNAVAILABLE, canonical_id)self._connect_failed_logged = Falsealert = fused.alert@@ -134,6 +180,7 @@ class AlertSink:async with pool.acquire() as conn:row = await conn.fetchrow(_INSERT_SQL,+ fused.id,alert.tenant_id,alert.title[:500],alert.description or None,@@ -150,15 +197,22 @@ class AlertSink:fused.narrative,fused.anomaly_score,alert.event_time,+ alert.connector_id,+ alert.connector_type,+ json.dumps(alert.source_event_ids),+ alert.ocsf_class_uid,+ alert.rule_id,+ alert.rule_name,)if row is None:logger.debug("alert_sink.dedup_skip", fingerprint=alert.fingerprint())- return None- return str(row["id"])+ return PersistResult(PersistOutcome.DUPLICATE, canonical_id)+ return PersistResult(PersistOutcome.INSERTED, str(row["id"]))except asyncpg.ForeignKeyViolationError:# Unknown tenant — a mis-provisioned connector, not a pipeline bug.+ # Distinct from a duplicate so it is observable, not silently masked.logger.warning("alert_sink.unknown_tenant", tenant_id=str(alert.tenant_id))- return None+ return PersistResult(PersistOutcome.FAILED, None)except Exception as exc: # noqa: BLE001 — one bad row must not wedge the consumerlogger.error("alert_sink.persist_failed", error=str(exc))- return None+ return PersistResult(PersistOutcome.FAILED, None)diff --git a/services/fusion/app/services/detection_engine.py b/services/fusion/app/services/detection_engine.pyindex f9ff7d14a..405c59511 100644--- a/services/fusion/app/services/detection_engine.py+++ b/services/fusion/app/services/detection_engine.py@@ -38,6 +38,7 @@ import structlogfrom app.models.alert import AlertSeverity, RawAlertfrom app.services.detection_matcher import matches+from app.services.provenance import extract_provenancelogger = structlog.get_logger()@@ -136,6 +137,7 @@ class DetectionEngine:tenant_id = uuid.UUID(str(tenant_raw))except (ValueError, TypeError):return None+ connector_id, connector_type, class_uid = extract_provenance(message, ocsf)return RawAlert(tenant_id=tenant_id,source=f"detection:{hit.rule_id}",@@ -148,6 +150,11 @@ class DetectionEngine:username=_get(ocsf, "actor", "user", "name"),mitre_techniques=hit.mitre,raw_event=ocsf,+ connector_id=connector_id,+ connector_type=connector_type,+ ocsf_class_uid=class_uid,+ rule_id=hit.rule_id,+ rule_name=hit.name,)diff --git a/services/fusion/app/services/promoter.py b/services/fusion/app/services/promoter.pyindex ea0230f5d..9ced99e3e 100644--- a/services/fusion/app/services/promoter.py+++ b/services/fusion/app/services/promoter.py@@ -34,6 +34,7 @@ from typing import Anyimport structlogfrom app.models.alert import AlertSeverity, RawAlert+from app.services.provenance import extract_provenancelogger = structlog.get_logger()@@ -161,6 +162,7 @@ def promote_normalized_event(message: dict[str, Any]) -> RawAlert | None:severity = _SEVERITY_BY_ID.get(severity_id if isinstance(severity_id, int) else 0, AlertSeverity.MEDIUM)tactics, techniques = _mitre(ocsf)+ connector_id, connector_type, class_uid = extract_provenance(message, ocsf)return RawAlert(tenant_id=tenant_id,@@ -177,4 +179,7 @@ def promote_normalized_event(message: dict[str, Any]) -> RawAlert | None:mitre_techniques=techniques,raw_event=ocsf,event_time=_event_time(ocsf),+ connector_id=connector_id,+ connector_type=connector_type,+ ocsf_class_uid=class_uid,)diff --git a/services/fusion/app/services/provenance.py b/services/fusion/app/services/provenance.pynew file mode 100644index 000000000..3c9cf2c8e--- /dev/null+++ b/services/fusion/app/services/provenance.py@@ -0,0 +1,62 @@+"""Extract connector / OCSF provenance from an ingest ``raw_events`` envelope.++Issue #568: the fusion spine dropped the connector instance id, connector type,+and OCSF class on the floor when promoting/detecting alerts, so every persisted+row rendered as source "unknown" and downstream agents could not resolve the+originating connector (needed by the Splunk evidence tool in #570). This helper+recovers that provenance from the envelope the Go ingest service publishes.++The ingest ``NormalizedEvent`` carries ``connector_id`` at the top level and,+inside ``ocsf_event``, both ``source_connector_id`` and ``metadata.product``.+``connector_type`` is not always on the wire, so we fall back to the OCSF+product vendor/name for a human-meaningful provenance label; the canonical+connector *instance* id is what downstream resolution actually keys on.+"""++from __future__ import annotations++import uuid+from typing import Any+++def _parse_uuid(value: Any) -> uuid.UUID | None:+ if value in (None, ""):+ return None+ try:+ return uuid.UUID(str(value))+ except (ValueError, TypeError):+ return None+++def _product_label(ocsf: dict[str, Any]) -> str | None:+ meta = ocsf.get("metadata") if isinstance(ocsf, dict) else None+ product = meta.get("product") if isinstance(meta, dict) else None+ if not isinstance(product, dict):+ return None+ vendor = product.get("vendor_name")+ name = product.get("name")+ parts = [p for p in (vendor, name) if isinstance(p, str) and p]+ return " ".join(parts) or None+++def extract_provenance(+ message: dict[str, Any],+ ocsf: dict[str, Any] | None = None,+) -> tuple[uuid.UUID | None, str | None, int | None]:+ """Return ``(connector_id, connector_type, ocsf_class_uid)`` for a message.++ Never raises — a malformed envelope yields ``(None, None, None)`` so the+ consumer keeps flowing.+ """+ ocsf = ocsf if isinstance(ocsf, dict) else (message.get("ocsf_event") if isinstance(message, dict) else None)+ ocsf = ocsf if isinstance(ocsf, dict) else {}++ connector_id = _parse_uuid(message.get("connector_id")) or _parse_uuid(ocsf.get("source_connector_id"))+ connector_type = message.get("connector_type") if isinstance(message.get("connector_type"), str) else None+ if not connector_type:+ connector_type = _product_label(ocsf)++ class_uid_raw = ocsf.get("class_uid")+ class_uid = class_uid_raw if isinstance(class_uid_raw, int) else None++ return connector_id, connector_type, class_uiddiff --git a/services/fusion/app/services/windowed_detection.py b/services/fusion/app/services/windowed_detection.pyindex b393b1ef5..d868ae936 100644--- a/services/fusion/app/services/windowed_detection.py+++ b/services/fusion/app/services/windowed_detection.py@@ -30,6 +30,7 @@ import structlogfrom app.models.alert import AlertSeverity, RawAlertfrom app.services.detection_engine import DetectionHitfrom app.services.detection_matcher import matches+from app.services.provenance import extract_provenancelogger = structlog.get_logger()@@ -180,6 +181,7 @@ class WindowedDetectionEngine:except (ValueError, TypeError):return Nonefields = self._fields(message)+ connector_id, connector_type, class_uid = extract_provenance(message, ocsf)return RawAlert(tenant_id=tenant_id,source=f"detection:{hit.rule_id}",@@ -191,4 +193,9 @@ class WindowedDetectionEngine:username=fields.get("user"),mitre_techniques=hit.mitre,raw_event=ocsf,+ connector_id=connector_id,+ connector_type=connector_type,+ ocsf_class_uid=class_uid,+ rule_id=hit.rule_id,+ rule_name=hit.name,)diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.pyindex 813f0ec4e..d18925353 100644--- a/services/fusion/app/workers/consumer.py+++ b/services/fusion/app/workers/consumer.py@@ -13,7 +13,7 @@ from aiokafka import AIOKafkaConsumer, AIOKafkaProducerfrom app.core.config import settingsfrom app.models.alert import FusionDecision, RawAlert-from app.services.alert_sink import AlertSink+from app.services.alert_sink import AlertSink, PersistOutcome, PersistResultfrom app.services.detection_engine import DetectionEnginefrom app.services.dlq import DeadLetter, DeadLetterQueue, LoggingDLQ, safe_recordfrom app.services.event_schema import validate_event@@ -33,6 +33,8 @@ _METRICS = {"promoted": 0,"not_promoted": 0,"persisted": 0,+ "persist_unavailable": 0,+ "persist_failed": 0,"laked": 0,"detected": 0,"dead_lettered": 0,@@ -273,7 +275,19 @@ class FusionWorker:await self._fuse_and_persist(alert, source_event_id=validation.source_event_id)async def _fuse_and_persist(self, alert: RawAlert, *, source_event_id: str | None = None) -> None:- """Run one RawAlert through fusion, publish it, and persist it."""+ """Run one RawAlert through fusion, persist it, then publish it.++ Issue #568: the canonical, replay-stable alert id and source-event+ provenance are stamped on the RawAlert *before* fusion, so ``FusedAlert.id``+ (derived from ``alert.id``) matches the persisted ``alerts.id`` row.+ Persist runs *before* publish so the published envelope carries the+ durable row id + a truthful persistence outcome downstream can trust.+ """+ # Stamp source-event provenance + canonical id (issue #568).+ if source_event_id and source_event_id not in alert.source_event_ids:+ alert.source_event_ids.append(source_event_id)+ alert.id = alert.deterministic_id()+fused = await self._engine.process(alert)_METRICS["processed"] += 1@@ -284,17 +298,28 @@ class FusionWorker:else:_METRICS["new_incidents"] += 1- # Publish fused alert (even duplicates, so downstream can track)+ # Persist FIRST (Phase 3.1 sink). Fail-soft + idempotent — see+ # app/services/alert_sink.py. Duplicates are handled inside.+ result = PersistResult(PersistOutcome.UNAVAILABLE, str(fused.id))+ if self._sink is not None:+ result = await self._sink.persist(fused)+ if result.outcome is PersistOutcome.INSERTED:+ _METRICS["persisted"] += 1+ elif result.outcome is PersistOutcome.UNAVAILABLE:+ _METRICS["persist_unavailable"] += 1+ elif result.outcome is PersistOutcome.FAILED:+ _METRICS["persist_failed"] += 1++ # Publish fused alert (even duplicates, so downstream can track),+ # carrying the durable row id + persistence outcome (issue #568).+ envelope = fused.model_dump(mode="json")+ envelope["alert_row_id"] = result.alert_id+ envelope["persist_outcome"] = result.outcome.valueawait self._producer.send(settings.kafka_topic_alerts_fused,- value=fused.model_dump(mode="json"),+ value=envelope,)- # Persist to the alert store (Phase 3.1). Fail-soft + idempotent —- # see app/services/alert_sink.py. Duplicates are skipped inside.- if self._sink is not None and await self._sink.persist(fused) is not None:- _METRICS["persisted"] += 1-@staticmethoddef get_metrics() -> dict:return dict(_METRICS)diff --git a/services/fusion/tests/test_alert_sink.py b/services/fusion/tests/test_alert_sink.pyindex 250eb3575..65fa4c1d9 100644--- a/services/fusion/tests/test_alert_sink.py+++ b/services/fusion/tests/test_alert_sink.py@@ -8,6 +8,7 @@ not a mock.from __future__ import annotations+import jsonimport uuidfrom contextlib import asynccontextmanager@@ -18,9 +19,16 @@ from app.models.alert import (FusionDecision,RawAlert,)-from app.services.alert_sink import AlertSink, _asyncpg_dsn, _entities, _iocs+from app.services.alert_sink import (+ AlertSink,+ PersistOutcome,+ _asyncpg_dsn,+ _entities,+ _iocs,+)TENANT = uuid.UUID("00000000-0000-0000-0000-000000000001")+CONNECTOR = uuid.UUID("00000000-0000-0000-0000-0000000000c1")def _fused(decision: FusionDecision = FusionDecision.NEW_ALERT) -> FusedAlert:@@ -33,7 +41,15 @@ def _fused(decision: FusionDecision = FusionDecision.NEW_ALERT) -> FusedAlert:hostname="WIN-DC01",username="svc-backup",mitre_techniques=["T1003"],+ connector_id=CONNECTOR,+ connector_type="crowdstrike_falcon",+ source_event_ids=["evt-1"],+ ocsf_class_uid=2004,+ rule_id="edr-cred-dump-1",+ rule_name="Credential dumping via LSASS access",)+ # Canonical, replay-stable id (issue #568) — the sink inserts it explicitly.+ raw.id = raw.deterministic_id()return FusedAlert(id=raw.id,tenant_id=TENANT,@@ -83,54 +99,80 @@ async def test_duplicate_decision_is_never_persisted():sink = AlertSink("postgresql://x")sink._pool = _StubPool(row={"id": uuid.uuid4()}) # would succeed if calledresult = await sink.persist(_fused(FusionDecision.DUPLICATE))- assert result is None+ assert result.outcome is PersistOutcome.DUPLICATEassert sink._pool.conn.calls == []@pytest.mark.asyncio-async def test_new_alert_is_inserted_with_fingerprint():- row_id = uuid.uuid4()- pool = _StubPool(row={"id": row_id})+async def test_new_alert_is_inserted_with_canonical_id_and_provenance():+ fused = _fused()+ # The sink inserts the canonical id explicitly; the DB echoes it back.+ pool = _StubPool(row={"id": fused.id})sink = AlertSink("postgresql://x")sink._pool = pool- fused = _fused()result = await sink.persist(fused)- assert result == str(row_id)+ assert result.outcome is PersistOutcome.INSERTED+ assert result.alert_id == str(fused.id)+ assert result.persisted is True(sql, args) = pool.conn.calls[0]assert "INSERT INTO alerts" in sql- assert "WHERE NOT EXISTS" in sql # idempotency guard- assert args[0] == TENANT- assert args[9] == fused.alert.fingerprint() # dedup_hash+ assert "WHERE NOT EXISTS" in sql # content dedup guard+ assert "ON CONFLICT (id) DO NOTHING" in sql # exact-replay idempotency+ # Positional args follow the _INSERT_SQL column order.+ assert args[0] == fused.id # explicit canonical id+ assert args[1] == TENANT+ assert args[10] == fused.alert.fingerprint() # dedup_hash+ assert args[17] == CONNECTOR # connector_id+ assert args[18] == "crowdstrike_falcon" # connector_type+ assert json.loads(args[19]) == ["evt-1"] # source_event_ids+ assert args[20] == 2004 # ocsf_class_uid+ assert args[21] == "edr-cred-dump-1" # rule_id@pytest.mark.asyncio-async def test_dedup_hit_returns_none():- pool = _StubPool(row=None) # WHERE NOT EXISTS filtered the insert out+async def test_dedup_hit_returns_duplicate():+ pool = _StubPool(row=None) # WHERE NOT EXISTS / ON CONFLICT filtered it outsink = AlertSink("postgresql://x")sink._pool = pool- assert await sink.persist(_fused()) is None+ result = await sink.persist(_fused())+ assert result.outcome is PersistOutcome.DUPLICATE+ assert result.alert_id == str(_fused().id) # canonical id still known+ assert result.durable is True@pytest.mark.asyncio-async def test_db_unavailable_degrades_without_raising(monkeypatch):+async def test_db_unavailable_is_distinct_from_duplicate(monkeypatch):sink = AlertSink("postgresql://nope:[email protected]:1/none")async def _fail_start():sink._pool = Nonemonkeypatch.setattr(sink, "start", _fail_start)- # Must not raise — fail-soft is the contract.- assert await sink.persist(_fused()) is None+ result = await sink.persist(_fused())+ # Fail-soft (no raise) but observably UNAVAILABLE — retryable, NOT a dup.+ assert result.outcome is PersistOutcome.UNAVAILABLE+ assert result.durable is False@pytest.mark.asyncio-async def test_persist_exception_is_swallowed():+async def test_persist_exception_is_failed_not_duplicate():class _BoomPool(_StubPool):def acquire(self):raise RuntimeError("connection reset")sink = AlertSink("postgresql://x")sink._pool = _BoomPool(row=None)- assert await sink.persist(_fused()) is None+ result = await sink.persist(_fused())+ assert result.outcome is PersistOutcome.FAILED+ assert result.durable is False+++def test_deterministic_id_is_replay_stable():+ # Same tenant + content ⇒ same canonical id across independent builds.+ a = _fused().alert+ b = _fused().alert+ assert a.id == b.id == a.deterministic_id()+ # And it is a proper uuid5 (not the random default_factory).+ assert a.id != uuid.uuid4()diff --git a/services/fusion/tests/test_consumer_dlq.py b/services/fusion/tests/test_consumer_dlq.pyindex a347db1c7..d157d041c 100644--- a/services/fusion/tests/test_consumer_dlq.py+++ b/services/fusion/tests/test_consumer_dlq.py@@ -43,6 +43,7 @@ def _worker_with_dlq() -> tuple[FusionWorker, InMemoryDLQ]:engine = SimpleNamespace(process=AsyncMock(return_value=SimpleNamespace(+ id="fused-1",fusion_decision=FusionDecision.NEW_ALERT,model_dump=lambda mode="json": {"id": "fused-1"},)diff --git a/services/fusion/tests/test_promoter.py b/services/fusion/tests/test_promoter.pyindex 3fb7a08fd..8d86358f0 100644--- a/services/fusion/tests/test_promoter.py+++ b/services/fusion/tests/test_promoter.py@@ -14,6 +14,7 @@ from app.models.alert import AlertSeverityfrom app.services.promoter import promote_normalized_event, should_promoteTENANT = "00000000-0000-0000-0000-000000000001"+CONNECTOR = "00000000-0000-0000-0000-0000000000c1"def _message(ocsf_overrides: dict | None = None) -> dict:@@ -44,7 +45,8 @@ def _message(ocsf_overrides: dict | None = None) -> dict:ocsf.update(ocsf_overrides)return {"id": str(uuid.uuid4()),- "connector_id": "conn-1",+ "connector_id": CONNECTOR,+ "connector_type": "crowdstrike_falcon","tenant_id": TENANT,"ocsf_event": ocsf,"normalization_version": "1.1.0",@@ -87,6 +89,30 @@ class TestPromoteNormalizedEvent:assert alert.mitre_tactics == ["Credential Access"]assert alert.event_time is not Noneassert alert.raw_event["class_uid"] == 2001+ # Provenance carried first-class (issue #568)+ assert str(alert.connector_id) == CONNECTOR+ assert alert.connector_type == "crowdstrike_falcon"+ assert alert.ocsf_class_uid == 2001++ def test_provenance_falls_back_to_ocsf_when_envelope_sparse(self):+ # No top-level connector_id/type — recover from the OCSF payload.+ msg = _message()+ del msg["connector_id"]+ del msg["connector_type"]+ msg["ocsf_event"]["source_connector_id"] = CONNECTOR+ alert = promote_normalized_event(msg)+ assert alert is not None+ assert str(alert.connector_id) == CONNECTOR+ # connector_type falls back to the OCSF product vendor+name label.+ assert alert.connector_type == "CrowdStrike Falcon"++ def test_canonical_id_is_deterministic_across_promotions(self):+ # The stamped id happens in the worker, but the derivation is on the+ # model and must be stable for identical content (issue #568).+ a = promote_normalized_event(_message())+ b = promote_normalized_event(_message())+ assert a is not None and b is not None+ assert a.deterministic_id() == b.deterministic_id()def test_severity_ladder(self):for sev_id, expected in [