Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix: durable fusion→agents auto-triage spine (#568, #571, #569, #570) (#572)

Beenu Arorac090b297eca5
diff --git a/services/actions/app/executors/siem.py b/services/actions/app/executors/siem.py
index 57ee0fb0d..8b5a51260 100644
--- a/services/actions/app/executors/siem.py
+++ b/services/actions/app/executors/siem.py
@@ -77,7 +77,9 @@ class SearchSIEMExecutor(BaseExecutor):
splunk = _splunk_client(request.parameters)
if splunk:
try:
- results = await splunk.run_search(query=query, max_results=max_results)
+ # SplunkClient.run_search takes `max_count` (issue #570); the old
+ # `max_results=` kwarg raised TypeError on every live Splunk search.
+ results = await splunk.run_search(query=query, max_count=max_results)
return ActionResult(
action_id=request.id,
status=ActionStatus.COMPLETED,
diff --git a/services/actions/tests/test_search_siem_executor.py b/services/actions/tests/test_search_siem_executor.py
new file mode 100644
index 000000000..58c702e2d
--- /dev/null
+++ b/services/actions/tests/test_search_siem_executor.py
@@ -0,0 +1,57 @@
+"""Issue #570 — SearchSIEMExecutor must call SplunkClient with the right kwarg.
+
+`SplunkClient.run_search` takes `max_count`; the executor previously passed
+`max_results=`, raising TypeError on every live Splunk search (silently
+returning a FAILED action). These tests pin the contract so it can't regress.
+"""
+
+from __future__ import annotations
+
+import inspect
+from uuid import uuid4
+
+import pytest
+from app.clients.splunk_client import SplunkClient
+from app.executors import siem
+from app.executors.siem import SearchSIEMExecutor
+from app.models.action import ActionRequest, ActionStatus, ActionType
+
+
+def test_run_search_signature_uses_max_count_not_max_results():
+ params = inspect.signature(SplunkClient.run_search).parameters
+ assert "max_count" in params
+ assert "max_results" not in params
+
+
+class _FakeSplunk:
+ def __init__(self) -> None:
+ self.calls: list[dict] = []
+
+ async def run_search(self, **kwargs): # noqa: ANN003 — mirror the real signature loosely
+ self.calls.append(kwargs)
+ return [{"_raw": "evt-1"}, {"_raw": "evt-2"}]
+
+
+def _request() -> ActionRequest:
+ return ActionRequest(
+ incident_id=uuid4(),
+ tenant_id=uuid4(),
+ action_type=ActionType.SEARCH_SIEM,
+ target="splunk",
+ parameters={"query": "index=main sourcetype=WinEventLog", "max_results": 25},
+ )
+
+
+async def test_executor_calls_run_search_with_max_count(monkeypatch):
+ fake = _FakeSplunk()
+ monkeypatch.setattr(siem, "_splunk_client", lambda params: fake)
+
+ result = await SearchSIEMExecutor().execute(_request())
+
+ assert result.status is ActionStatus.COMPLETED
+ assert len(fake.calls) == 1
+ kwargs = fake.calls[0]
+ assert kwargs.get("max_count") == 25 # correct kwarg + value threaded through
+ assert "max_results" not in kwargs # the broken kwarg is gone
+ assert result.output["result_count"] == 2
diff --git a/services/agents/app/agents/auto_triage_agent.py b/services/agents/app/agents/auto_triage_agent.py
index 5bdc0b294..068a18c4d 100644
--- a/services/agents/app/agents/auto_triage_agent.py
+++ b/services/agents/app/agents/auto_triage_agent.py
@@ -44,6 +44,18 @@ logger = structlog.get_logger()
AUTO_CLOSE_THRESHOLD: float = float(os.getenv("AISOC_AUTO_CLOSE_THRESHOLD", "0.85"))
+
+class AutoTriageError(RuntimeError):
+ """Raised when the LLM auto-triage call or its response parsing fails.
+
+ Issue #571: the old code swallowed LLM/parse exceptions and returned a
+ RUNNING state with a null verdict, so the worker's deterministic fallback
+ (`_llm_triage`'s except branch) was unreachable and a null verdict could be
+ recorded as "completed". Raising a typed error lets the caller fall back to
+ deterministic triage or mark the alert ``needs_review`` — never null.
+ """
+
+
_metrics: dict[str, Any] = {
"auto_resolved_count": 0,
"escalated_count": 0,
@@ -233,11 +245,13 @@ async def run_auto_triage(state: InvestigationState) -> InvestigationState:
raw_text = response.content
result = _parse_llm_response(raw_text)
except Exception as exc:
- logger.error("Auto-triage LLM call failed, escalating", error=str(exc))
- state.add_finding(f"Auto-triage LLM error: {exc} — escalating to manual triage")
- _metrics["escalated_count"] += 1
+ # Issue #571: do NOT swallow + return a null-verdict RUNNING state.
+ # Raise a typed error so the caller falls back to deterministic triage
+ # (or marks the alert needs_review) instead of completing with no verdict.
+ logger.error("Auto-triage LLM call failed", error=str(exc))
+ state.add_finding(f"Auto-triage LLM error: {exc}")
_metrics["total_processed"] += 1
- return state
+ raise AutoTriageError(str(exc)) from exc
elapsed_ms = round((time.monotonic() - t0) * 1000)
diff --git a/services/agents/app/agents/investigation_agent.py b/services/agents/app/agents/investigation_agent.py
index 11bc94a4c..8d142bd68 100644
--- a/services/agents/app/agents/investigation_agent.py
+++ b/services/agents/app/agents/investigation_agent.py
@@ -11,7 +11,9 @@ from typing import Any
import httpx
import structlog
+from app.agents.dispositions import BENIGN, BENIGN_TRUE_POSITIVE, FALSE_POSITIVE, NEEDS_REVIEW
from app.confidence import score_investigation
+from app.investigator.splunk_evidence import collect_splunk_evidence
from app.models.state import ActionRisk, AgentStatus, InvestigationState, ProposedAction
from app.tools.mitre import lookup_technique
@@ -37,6 +39,11 @@ async def run_investigation(state: InvestigationState) -> InvestigationState:
state.iteration_count += 1
+ # --- Pull surrounding SIEM evidence for Splunk-originated alerts (#570) ---
+ # Bounded, read-only, credential-vault-backed. Failure => missing evidence
+ # (handled after scoring below), never a silent dismissal.
+ splunk_evidence = await collect_splunk_evidence(state)
+
# Analyze enrichment results for threat patterns
malicious_iocs = {k: v for k, v in state.ioc_enrichments.items() if v.get("threat_classification") in ("malicious", "suspicious")}
@@ -104,6 +111,14 @@ async def run_investigation(state: InvestigationState) -> InvestigationState:
)
confidence, basis, verdict = score_investigation(state)
+
+ # #570: never dismiss a Splunk-originated alert on MISSING evidence. If the
+ # bounded evidence query failed/timed out (applicable but not queried), a
+ # dismissive verdict is downgraded to needs_review so a human decides.
+ if splunk_evidence.applicable and not splunk_evidence.queried and verdict in (FALSE_POSITIVE, BENIGN, BENIGN_TRUE_POSITIVE):
+ state.add_finding("Splunk evidence unavailable — cannot confirm a benign/false-positive verdict; escalating to needs_review.")
+ verdict = NEEDS_REVIEW
+
state.confidence = confidence
state.confidence_basis = basis
state.verdict = verdict
diff --git a/services/agents/app/api/router.py b/services/agents/app/api/router.py
index ec55bf1cf..d7e82b9df 100644
--- a/services/agents/app/api/router.py
+++ b/services/agents/app/api/router.py
@@ -11,12 +11,14 @@ from uuid import UUID, uuid4
from fastapi import APIRouter, BackgroundTasks, HTTPException
from pydantic import BaseModel
-from app.graph.workflow import investigation_graph
+from app.graph.runner import run_full_investigation
from app.models.state import AgentTask, InvestigationState
router = APIRouter()
-# In-memory run store (replace with Redis/DB in production)
+# In-memory run store for status polling. The durable record of every step is
+# the Postgres Investigation Ledger (written by the shared graph runner) — this
+# dict is only the fast local status cache for GET /investigations/{run_id}.
_runs: dict[str, dict] = {}
@@ -35,12 +37,20 @@ class InvestigationResponse(BaseModel):
async def _run_investigation(run_id: str, state: InvestigationState) -> None:
- """Run investigation in background and store results."""
+ """Run investigation in background and store results.
+
+ Uses the SAME durable graph runner as the Kafka auto-triage worker
+ (issue #569), so manual and automated investigations share one
+ orchestration implementation and both persist every step to the ledger.
+ """
try:
- initial_state = state.to_dict()
- result = await investigation_graph.ainvoke(initial_state)
- _runs[run_id] = {"status": "completed", "result": result, "completed_at": datetime.utcnow().isoformat()}
- except Exception as exc:
+ result = await run_full_investigation(state)
+ _runs[run_id] = {
+ "status": "completed",
+ "result": result.to_dict(),
+ "completed_at": datetime.utcnow().isoformat(),
+ }
+ except Exception as exc: # noqa: BLE001 — surface failure via the status cache
_runs[run_id] = {"status": "failed", "error": str(exc)}
diff --git a/services/agents/app/graph/runner.py b/services/agents/app/graph/runner.py
new file mode 100644
index 000000000..b7810f770
--- /dev/null
+++ b/services/agents/app/graph/runner.py
@@ -0,0 +1,153 @@
+"""Shared, durable investigation-graph runner (issue #569).
+
+Both the Kafka auto-triage worker and the manual investigations API run the
+*same* orchestration through here, so escalated alerts flow through one
+implementation instead of two divergent ones:
+
+* the manual API runs the FULL graph (auto-triage → triage → enrichment →
+ investigation → attack-path) via :func:`run_full_investigation`;
+* the auto-triage worker, having already produced a governed verdict, runs the
+ post-triage escalation subgraph via :func:`run_escalation`.
+
+Every node transition is recorded to the Investigation Ledger (append-only,
+``ON CONFLICT (run_id, seq) DO NOTHING``), so a restart mid-investigation
+re-runs idempotently — it never loses or duplicates the run. A wall-clock
+budget bounds the run; a token budget is enforced upstream by the
+``CostGovernor`` and a tool-call budget by the ReAct loop's ``max_iters``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from dataclasses import dataclass
+
+import structlog
+
+from app.agents.dispositions import NEEDS_REVIEW
+from app.investigator import ledger as ledger_module
+from app.models.state import AgentStatus, InvestigationState
+
+logger = structlog.get_logger()
+
+
+@dataclass(frozen=True)
+class InvestigationBudget:
+ """Resource envelope for one investigation (issue #569)."""
+
+ max_seconds: float = 120.0
+ max_tokens: int = 20_000
+ max_tool_calls: int = 8
+
+
+def default_budget() -> InvestigationBudget:
+ return InvestigationBudget(
+ max_seconds=float(os.getenv("AISOC_INVESTIGATION_MAX_SECONDS", "120")),
+ max_tokens=int(os.getenv("AISOC_MAX_TOKENS_PER_ALERT", "20000")),
+ max_tool_calls=int(os.getenv("AISOC_INVESTIGATION_MAX_TOOL_CALLS", "8")),
+ )
+
+
+async def _run(
+ graph, # noqa: ANN001 — a compiled LangGraph
+ state: InvestigationState,
+ *,
+ budget: InvestigationBudget | None,
+ persist: bool,
+ seq_start: int,
+) -> InvestigationState:
+ budget = budget or default_budget()
+ tenant_ref = str(state.tenant_id)
+ state_dict = state.to_dict()
+ merged: dict = dict(state_dict)
+
+ tenant_uuid = await ledger_module.resolve_tenant(tenant_ref) if persist else None
+ if persist:
+ # Idempotent — start_run ON CONFLICT DO NOTHING (the worker may have
+ # already opened this run when it persisted the triage verdict).
+ await ledger_module.start_run(
+ run_id=state.run_id,
+ case_id=str(state.incident_id),
+ tenant_ref=tenant_ref,
+ alert_summary=state.alert_summary or "",
+ raw_alert=state.raw_alert or {},
+ model_used="graph:investigation",
+ )
+
+ seq = seq_start
+ timed_out = False
+ try:
+ async with asyncio.timeout(budget.max_seconds):
+ async for step in graph.astream(state_dict):
+ if not isinstance(step, dict):
+ continue
+ for node, out in step.items():
+ seq += 1
+ if isinstance(out, dict):
+ merged.update(out)
+ if tenant_uuid is not None:
+ await ledger_module.record_event(
+ run_id=state.run_id,
+ tenant_id=tenant_uuid,
+ seq=seq,
+ kind="graph_step",
+ agent=str(node),
+ summary=f"graph node '{node}' completed",
+ payload={"node": str(node)},
+ )
+ except TimeoutError:
+ timed_out = True
+ logger.warning("investigation.budget_timeout", run_id=str(state.run_id), max_seconds=budget.max_seconds)
+
+ try:
+ result = InvestigationState.model_validate(merged)
+ except Exception as exc: # noqa: BLE001 — never let a mapping error lose the run
+ logger.warning("investigation.state_rebuild_failed", run_id=str(state.run_id), error=str(exc))
+ result = state
+
+ if timed_out:
+ result.add_finding(f"Investigation budget ({budget.max_seconds}s) exceeded — partial results, escalating.")
+ if not result.verdict:
+ result.verdict = NEEDS_REVIEW
+ if result.status is AgentStatus.COMPLETED:
+ result.status = AgentStatus.RUNNING
+
+ if persist:
+ await ledger_module.complete_run(
+ run_id=result.run_id,
+ tenant_id=tenant_uuid or result.tenant_id,
+ status="completed",
+ error=result.error,
+ iterations=result.iteration_count,
+ )
+ return result
+
+
+async def run_full_investigation(
+ state: InvestigationState,
+ *,
+ budget: InvestigationBudget | None = None,
+ persist: bool = True,
+) -> InvestigationState:
+ """Run the FULL graph (auto-triage → … → attack-path). Manual API entry."""
+ from app.graph.workflow import investigation_graph # noqa: PLC0415 — avoid import cycle
+
+ return await _run(investigation_graph, state, budget=budget, persist=persist, seq_start=0)
+
+
+async def run_escalation(
+ state: InvestigationState,
+ *,
+ budget: InvestigationBudget | None = None,
+ persist: bool = True,
+ seq_start: int = 1,
+) -> InvestigationState:
+ """Run the post-triage escalation subgraph (triage → … → attack-path).
+
+ Used by the auto-triage worker after it has produced (and persisted) a
+ governed verdict. ``seq_start`` defaults to 1 so the recorded graph steps
+ follow the worker's ``triage_verdict`` event (seq 1) without colliding.
+ """
+ from app.graph.workflow import escalation_graph # noqa: PLC0415 — avoid import cycle
+
+ return await _run(escalation_graph, state, budget=budget, persist=persist, seq_start=seq_start)
diff --git a/services/agents/app/graph/workflow.py b/services/agents/app/graph/workflow.py
index fc9acdda6..826ea002f 100644
--- a/services/agents/app/graph/workflow.py
+++ b/services/agents/app/graph/workflow.py
@@ -14,7 +14,7 @@ import structlog
from langgraph.graph import END, StateGraph
from app.agents.attack_path_agent import run_attack_path
-from app.agents.auto_triage_agent import run_auto_triage
+from app.agents.auto_triage_agent import AutoTriageError, run_auto_triage
from app.agents.enrichment_agent import run_enrichment
from app.agents.investigation_agent import run_investigation
from app.agents.triage_agent import run_triage
@@ -36,7 +36,15 @@ def _from_dict(d: dict) -> InvestigationState:
async def auto_triage_node(state: dict) -> dict:
s = _from_dict(state)
- s = await run_auto_triage(s)
+ try:
+ s = await run_auto_triage(s)
+ except AutoTriageError as exc:
+ # LLM/parse failure (issue #571): never terminate on a null verdict —
+ # escalate through the full pipeline (deterministic triage runs next).
+ logger.warning("graph.auto_triage_failed_escalating", error=str(exc), incident_id=str(s.incident_id))
+ s.add_finding(f"Auto-triage LLM unavailable ({exc}) — escalating to full pipeline")
+ if s.status is AgentStatus.COMPLETED:
+ s.status = AgentStatus.RUNNING
return s.to_dict()
@@ -113,5 +121,30 @@ def build_investigation_graph() -> StateGraph:
return graph.compile()
-# Module-level compiled graph (singleton)
+def build_escalation_graph() -> StateGraph:
+ """The post-triage escalation pipeline (issue #569).
+
+ Shares the SAME node implementations as the full graph but skips the
+ auto-triage entry node — used by the Kafka auto-triage worker, which has
+ already produced a governed verdict, to route escalated alerts (TP /
+ low-confidence / needs_review) through enrichment → investigation →
+ attack-path without re-triaging.
+
+ Flow: triage ──► enrichment ──► investigation ──► attack_path ──► END
+ """
+ graph = StateGraph(dict)
+ graph.add_node("triage", triage_node)
+ graph.add_node("enrichment", enrichment_node)
+ graph.add_node("investigation", investigation_node)
+ graph.add_node("attack_path", attack_path_node)
+ graph.set_entry_point("triage")
+ graph.add_edge("triage", "enrichment")
+ graph.add_edge("enrichment", "investigation")
+ graph.add_edge("investigation", "attack_path")
+ graph.add_edge("attack_path", END)
+ return graph.compile()
+
+
+# Module-level compiled graphs (singletons)
investigation_graph = build_investigation_graph()
+escalation_graph = build_escalation_graph()
diff --git a/services/agents/app/investigator/ledger.py b/services/agents/app/investigator/ledger.py
index cb52ac63b..dae564f1b 100644
--- a/services/agents/app/investigator/ledger.py
+++ b/services/agents/app/investigator/ledger.py
@@ -36,6 +36,14 @@ import structlog
logger = structlog.get_logger()
+class LedgerPersistError(RuntimeError):
+ """Raised by the durable write paths (e.g. :func:`persist_auto_triage`) when
+ a real database error occurs, so the caller can retry / dead-letter instead
+ of silently losing the outcome (issue #571). A *missing* database (no
+ ``DATABASE_URL``) is NOT an error — those paths no-op and return ``False``.
+ """
+
+
_POOL: asyncpg.Pool | None = None
@@ -99,6 +107,22 @@ async def _resolve_tenant_id(conn: asyncpg.Connection, tenant_ref: str) -> uuid.
return row["id"] if row else None
+async def resolve_tenant(tenant_ref: str) -> uuid.UUID | None:
+ """Resolve a tenant ref (UUID string / slug / name) to its UUID, or None.
+
+ Best-effort (no DB configured or lookup failure ⇒ None). Used by the shared
+ graph runner to attribute per-node ledger events (issue #569)."""
+ pool = await get_pool()
+ if pool is None:
+ return None
+ try:
+ async with pool.acquire() as conn:
+ return await _resolve_tenant_id(conn, tenant_ref)
+ except Exception as exc: # noqa: BLE001 — best-effort
+ logger.debug("ledger.resolve_tenant_failed", tenant_ref=tenant_ref, error=str(exc))
+ return None
+
+
async def _set_rls_context(conn: asyncpg.Connection, tenant_id: uuid.UUID) -> None:
"""Match the API service's set_rls_context — required so the audit-log
immutability trigger and tenant policies allow our INSERTs."""
@@ -139,6 +163,7 @@ async def start_run(
model_used, status, started_at, created_at)
VALUES
($1, $2, $3, $4, $5::jsonb, $6, 'running', now(), now())
+ ON CONFLICT (id) DO NOTHING
""",
run_id,
tenant_id,
@@ -319,3 +344,152 @@ async def complete_run(
run_id=str(run_id),
error=str(exc),
)
+
+
+def _coerce_uuid(value: Any) -> uuid.UUID | None:
+ try:
+ return uuid.UUID(str(value))
+ except (ValueError, TypeError):
+ return None
+
+
+async def persist_auto_triage(
+ *,
+ run_id: uuid.UUID,
+ alert_id: str | None,
+ tenant_ref: str,
+ alert_summary: str,
+ raw_alert: dict[str, Any] | None,
+ tier: str,
+ verdict: str,
+ confidence: float,
+ rationale: str,
+ findings: list[str] | None = None,
+ proposed_actions: list[dict[str, Any]] | None = None,
+ auto_closed: bool = False,
+ iterations: int = 0,
+ tokens: int = 0,
+ cost_usd: float = 0.0,
+) -> bool:
+ """Durably record an auto-triage outcome (issue #571) in ONE transaction:
+
+ * upsert the ``investigation_runs`` row (idempotent on ``id``),
+ * append the verdict as an immutable ``investigation_events`` row (seq 1),
+ * mark the run completed, and
+ * write the disposition/verdict + rationale + recommendations back onto the
+ ``alerts`` row so the alert API surfaces the automated verdict.
+
+ Idempotent: a replay reuses the deterministic ``run_id`` (ON CONFLICT DO
+ NOTHING on the run + the ``(run_id, seq)`` event key), so it never
+ duplicates ledger rows or outcomes.
+
+ Returns ``True`` when written, ``False`` when skipped (no database
+ configured, or an unknown tenant — neither is retryable). Raises
+ :class:`LedgerPersistError` on a real DB error so the caller can retry /
+ dead-letter. A completed run therefore always carries a non-null verdict.
+ """
+ pool = await get_pool()
+ if pool is None:
+ return False
+
+ verdict = (verdict or "").strip() or "needs_review"
+ recommendations = proposed_actions or []
+ alert_uuid = _coerce_uuid(alert_id)
+
+ try:
+ async with pool.acquire() as conn:
+ tenant_id = await _resolve_tenant_id(conn, tenant_ref)
+ if tenant_id is None:
+ logger.debug("ledger.auto_triage_skip", reason="unknown_tenant", tenant_ref=tenant_ref)
+ return False
+ await _set_rls_context(conn, tenant_id)
+ async with conn.transaction():
+ await conn.execute(
+ """
+ INSERT INTO investigation_runs
+ (id, tenant_id, case_id, alert_summary, raw_alert,
+ model_used, status, started_at, created_at)
+ VALUES
+ ($1, $2, $3, $4, $5::jsonb, $6, 'running', now(), now())
+ ON CONFLICT (id) DO NOTHING
+ """,
+ run_id,
+ tenant_id,
+ str(alert_id or ""),
+ (alert_summary or "")[:8000] or None,
+ json.dumps(raw_alert or {}),
+ f"kafka:auto_triage:{tier}",
+ )
+ await conn.execute(
+ """
+ INSERT INTO investigation_events
+ (id, run_id, tenant_id, seq, ts, kind, agent, summary,
+ payload, duration_ms, created_at)
+ VALUES
+ ($1, $2, $3, 1, now(), 'triage_verdict', $4, $5, $6::jsonb, 0, now())
+ ON CONFLICT (run_id, seq) DO NOTHING
+ """,
+ uuid.uuid4(),
+ run_id,
+ tenant_id,
+ f"auto_triage:{tier}",
+ f"verdict={verdict} confidence={confidence:.2f}"[:8000],
+ json.dumps(
+ {
+ "verdict": verdict,
+ "confidence": confidence,
+ "rationale": rationale,
+ "findings": findings or [],
+ "proposed_actions": recommendations,
+ "auto_closed": auto_closed,
+ }
+ ),
+ )
+ await conn.execute(
+ """
+ UPDATE investigation_runs
+ SET status = 'completed', iterations = $2,
+ total_tokens = $3, total_cost_usd = $4, completed_at = now()
+ WHERE id = $1
+ """,
+ run_id,
+ iterations,
+ tokens,
+ cost_usd,
+ )
+ if alert_uuid is not None:
+ # Surface the automated verdict on the alert row. Status is
+ # only advanced to 'resolved' when auto-triage auto-closed a
+ # benign/FP alert at high confidence; otherwise the alert
+ # stays open for escalation / human review.
+ await conn.execute(
+ """
+ UPDATE alerts
+ SET disposition = $3,
+ ai_score = $4,
+ ai_summary = $5,
+ ai_recommendations = $6::jsonb,
+ status = CASE WHEN $7 THEN 'resolved' ELSE status END,
+ resolved_at = CASE WHEN $7 THEN now() ELSE resolved_at END,
+ updated_at = now()
+ WHERE id = $1 AND tenant_id = $2
+ """,
+ alert_uuid,
+ tenant_id,
+ verdict[:50],
+ float(confidence),
+ (rationale or "")[:8000] or None,
+ json.dumps(recommendations),
+ auto_closed,
+ )
+ logger.info(
+ "ledger.auto_triage_persisted",
+ run_id=str(run_id),
+ verdict=verdict,
+ auto_closed=auto_closed,
+ alert_id=str(alert_id or ""),
+ )
+ return True
+ except Exception as exc: # noqa: BLE001 — re-raised as a typed, retryable error
+ logger.warning("ledger.auto_triage_persist_failed", run_id=str(run_id), error=str(exc))
+ raise LedgerPersistError(str(exc)) from exc
diff --git a/services/agents/app/investigator/splunk_evidence.py b/services/agents/app/investigator/splunk_evidence.py
new file mode 100644
index 000000000..b6b60b44d
--- /dev/null
+++ b/services/agents/app/investigator/splunk_evidence.py
@@ -0,0 +1,325 @@
+"""Bounded, read-only Splunk evidence tool for the investigation graph (#570).
+
+The auto-investigation used to reason only over enrichment already present on
+the alert — it never pulled the surrounding events from the SIEM that raised
+it. This tool closes that gap for Splunk-originated alerts:
+
+* resolve the originating connector instance from the alert's provenance
+ (``connector_id`` — issue #568), never from the LLM;
+* obtain credentials from the CredentialVault, never from a prompt/log/ledger;
+* build a TIME-BOUNDED, allow-listed SPL pivot around the alert's user / host /
+ IP / hash (values are quote-escaped; the query shape is fixed);
+* enforce a maximum time range, execution timeout, and result/field caps;
+* cite the (redacted) results + a content hash in the investigation findings
+ and the ledger;
+* treat any failure/timeout as *missing evidence* so the caller escalates to
+ needs_review instead of dismissing the alert as a false positive.
+
+Everything is best-effort and self-contained (a minimal Splunk REST client) so
+it runs in the agents service without importing the actions service.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import Any
+
+import httpx
+import structlog
+
+from app.investigator import ledger as ledger_module
+from app.models.state import InvestigationState
+from app.security.credential_vault import get_vault
+
+logger = structlog.get_logger()
+
+_MAX_RESULTS = int(os.getenv("AISOC_SPLUNK_EVIDENCE_MAX_RESULTS", "50"))
+_MAX_WINDOW_HOURS = float(os.getenv("AISOC_SPLUNK_EVIDENCE_MAX_WINDOW_HOURS", "24"))
+_WINDOW_HOURS = min(float(os.getenv("AISOC_SPLUNK_EVIDENCE_WINDOW_HOURS", "6")), _MAX_WINDOW_HOURS)
+_TIMEOUT_S = float(os.getenv("AISOC_SPLUNK_EVIDENCE_TIMEOUT_S", "20"))
+_MAX_FIELD_LEN = 500 # per-field redaction cap (byte-ish bound on cited evidence)
+
+
+@dataclass(frozen=True)
+class SplunkCreds:
+ base_url: str
+ token: str
+ ssl_verify: bool = True
+ index: str = "main"
+
+
+@dataclass
+class SplunkEvidenceResult:
+ """Outcome of an evidence attempt.
+
+ ``applicable`` is True only when a Splunk connector was actually resolved
+ for the alert, so a *missing evidence* escalation (issue #570) is never
+ triggered for non-Splunk alerts. ``queried`` is True only on a successful
+ bounded query.
+ """
+
+ applicable: bool
+ queried: bool
+ result_count: int = 0
+ error: str | None = None
+ query: str | None = None
+ findings: list[str] = field(default_factory=list)
+
+
+def _spl_quote(value: str) -> str:
+ """Quote-escape a value for safe insertion into an SPL double-quoted string."""
+ return value.replace("\\", "\\\\").replace('"', '\\"')
+
+
+def _safe_index(index: str) -> str:
+ """Allow-list the index token (letters/digits/_-* only)."""
+ cleaned = "".join(ch for ch in (index or "") if ch.isalnum() or ch in "_-*")
+ return cleaned or "main"
+
+
+def build_evidence_spl(raw_alert: dict[str, Any], *, index: str = "main", max_results: int = _MAX_RESULTS) -> str | None:
+ """Build a bounded, allow-listed SPL pivot around the alert's entities.
+
+ Returns ``None`` when the alert has no pivotable entity (so we never run an
+ unbounded ``search index=*`` fishing query).
+ """
+ filters: list[str] = []
+ username = _first_str(raw_alert, "username", "user")
+ if username:
+ filters.append(f'user="{_spl_quote(username)}"')
+ host = _first_str(raw_alert, "hostname", "host")
+ if host:
+ filters.append(f'host="{_spl_quote(host)}"')
+ ip = _first_str(raw_alert, "src_ip", "dst_ip")
+ if ip:
+ q = _spl_quote(ip)
+ filters.append(f'(src_ip="{q}" OR dest_ip="{q}" OR src="{q}" OR dest="{q}")')
+ file_hash = _first_str(raw_alert, "file_hash")
+ if file_hash:
+ q = _spl_quote(file_hash)
+ filters.append(f'(file_hash="{q}" OR hash="{q}")')
+
+ if not filters:
+ return None
+ where = " OR ".join(filters)
+ return f"search index={_safe_index(index)} ({where}) | head {int(max_results)}"
+
+
+def _first_str(obj: dict[str, Any], *keys: str) -> str | None:
+ for k in keys:
+ v = obj.get(k)
+ if isinstance(v, str) and v.strip():
+ return v.strip()
+ return None
+
+
+def _time_window(raw_alert: dict[str, Any]) -> tuple[str, str]:
+ """Return (earliest, latest) Splunk time tokens, bounded to ±_WINDOW_HOURS."""
+ et = raw_alert.get("event_time")
+ if not et:
+ raw_event = raw_alert.get("raw_event")
+ if isinstance(raw_event, dict):
+ et = raw_event.get("time")
+ if isinstance(et, str) and et:
+ try:
+ dt = datetime.fromisoformat(et.replace("Z", "+00:00"))
+ start = dt - timedelta(hours=_WINDOW_HOURS)
+ end = dt + timedelta(hours=_WINDOW_HOURS)
+ return (str(int(start.timestamp())), str(int(end.timestamp())))
+ except ValueError:
+ pass
+ return (f"-{int(_WINDOW_HOURS)}h", "now")
+
+
+def _redact_samples(results: list[dict[str, Any]]) -> list[str]:
+ """Compact, length-capped one-liners for a few results (bounded citation)."""
+ out: list[str] = []
+ for row in results:
+ if not isinstance(row, dict):
+ continue
+ parts = []
+ for k in ("_time", "host", "source", "sourcetype", "user", "action"):
+ v = row.get(k)
+ if v:
+ parts.append(f"{k}={str(v)[:80]}")
+ line = " ".join(parts) or json.dumps(row, default=str)
+ out.append(line[:_MAX_FIELD_LEN])
+ return out
+
+
+class SplunkEvidenceClient:
+ """Minimal, bounded, read-only Splunk oneshot-search client."""
+
+ def __init__(self, creds: SplunkCreds, *, timeout_s: float = _TIMEOUT_S, max_results: int = _MAX_RESULTS) -> None:
+ self._base = creds.base_url.rstrip("/")
+ self._token = creds.token
+ self._verify = creds.ssl_verify
+ self._timeout = timeout_s
+ self._max_results = max_results
+
+ async def search(self, spl: str, *, earliest: str, latest: str) -> list[dict[str, Any]]:
+ async with httpx.AsyncClient(timeout=self._timeout, verify=self._verify) as client:
+ resp = await client.post(
+ f"{self._base}/services/search/jobs",
+ headers={
+ "Authorization": f"Bearer {self._token}",
+ "Content-Type": "application/x-www-form-urlencoded",
+ },
+ data={
+ "search": spl,
+ "output_mode": "json",
+ "exec_mode": "oneshot",
+ "earliest_time": earliest,
+ "latest_time": latest,
+ "count": str(self._max_results),
+ },
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ results = data.get("results", []) if isinstance(data, dict) else []
+ return results[: self._max_results]
+
+
+async def resolve_splunk_credentials(state: InvestigationState) -> SplunkCreds | None:
+ """Resolve + decrypt the originating Splunk connector's credentials.
+
+ Keyed on the alert's ``connector_id`` provenance (issue #568) scoped to the
+ tenant, and only for a connector whose ``connector_type`` is ``splunk``.
+ Returns ``None`` (not raise) when the alert isn't Splunk-originated, no DB
+ is configured, or the vault can't decrypt — the caller then simply skips
+ the evidence step. Credentials are never logged.
+ """
+ raw = state.raw_alert or {}
+ connector_id = raw.get("connector_id")
+ if not connector_id:
+ return None
+ pool = await ledger_module.get_pool()
+ if pool is None:
+ return None
+ try:
+ cid = uuid.UUID(str(connector_id))
+ except (ValueError, TypeError):
+ return None
+ try:
+ async with pool.acquire() as conn:
+ await conn.execute("SELECT set_config('app.tenant_id', $1, true)", str(state.tenant_id))
+ row = await conn.fetchrow(
+ """
+ SELECT auth_config, connector_config
+ FROM connectors
+ WHERE id = $1 AND tenant_id = $2 AND connector_type = 'splunk'
+ """,
+ cid,
+ state.tenant_id,
+ )
+ if row is None:
+ return None
+ auth = _as_dict(row["auth_config"])
+ cfg = _as_dict(row["connector_config"])
+ vault = get_vault()
+ if vault is not None:
+ auth = vault.decrypt_dict(auth)
+ merged = {**cfg, **auth}
+ base_url = str(merged.get("base_url") or "").strip()
+ token = str(merged.get("token") or "").strip()
+ if not base_url or not token:
+ return None
+ return SplunkCreds(
+ base_url=base_url,
+ token=token,
+ ssl_verify=bool(merged.get("ssl_verify", True)),
+ index=str(merged.get("saved_search") or "main") if str(merged.get("saved_search") or "").startswith("index=") else "main",
+ )
+ except Exception as exc: # noqa: BLE001 — best-effort; never leak creds in the log
+ logger.debug("splunk_evidence.resolve_failed", error=type(exc).__name__)
+ return None
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+ if isinstance(value, dict):
+ return value
+ if isinstance(value, str) and value.strip():
+ try:
+ parsed = json.loads(value)
+ return parsed if isinstance(parsed, dict) else {}
+ except (ValueError, TypeError):
+ return {}
+ return {}
+
+
+def _make_client(creds: SplunkCreds) -> SplunkEvidenceClient:
+ return SplunkEvidenceClient(creds)
+
+
+async def collect_splunk_evidence(state: InvestigationState) -> SplunkEvidenceResult:
+ """Run one bounded Splunk evidence query for a Splunk-originated alert.
+
+ Adds redacted, hashed evidence to the investigation findings (cited in the
+ verdict rationale) and records it to the ledger. On any failure/timeout it
+ returns ``queried=False`` with ``applicable=True`` so the caller escalates
+ to needs_review rather than dismissing the alert.
+ """
+ creds = await resolve_splunk_credentials(state)
+ if creds is None:
+ return SplunkEvidenceResult(applicable=False, queried=False)
+
+ raw = state.raw_alert or {}
+ spl = build_evidence_spl(raw, index=creds.index, max_results=_MAX_RESULTS)
+ if spl is None:
+ state.add_finding("Splunk evidence: no pivotable entity (user/host/ip/hash) on the alert — skipped.")
+ return SplunkEvidenceResult(applicable=True, queried=False, error="no_pivot")
+
+ earliest, latest = _time_window(raw)
+ try:
+ results = await _make_client(creds).search(spl, earliest=earliest, latest=latest)
+ except Exception as exc: # noqa: BLE001 — treat as missing evidence, escalate safely
+ logger.warning("splunk_evidence.query_failed", error=type(exc).__name__)
+ state.add_finding(f"Splunk evidence query failed ({type(exc).__name__}) — treating as missing evidence, escalating.")
+ return SplunkEvidenceResult(applicable=True, queried=False, error=str(exc), query=spl)
+
+ count = len(results)
+ digest = hashlib.sha256(json.dumps(results, sort_keys=True, default=str).encode("utf-8")).hexdigest()[:16]
+ samples = _redact_samples(results[:3])
+
+ findings = [
+ f"Splunk evidence: {count} surrounding event(s) via bounded query "
+ f"[head {_MAX_RESULTS}, window ±{int(_WINDOW_HOURS)}h] digest={digest}.",
+ ]
+ findings.extend(f"Splunk sample: {s}" for s in samples)
+ for f in findings:
+ state.add_finding(f)
+ state.confidence_basis.append(f"splunk_evidence_count={count} digest={digest}")
+
+ await _record_evidence_ledger(state, query=spl, count=count, digest=digest)
+
+ return SplunkEvidenceResult(
+ applicable=True,
+ queried=True,
+ result_count=count,
+ query=spl,
+ findings=findings,
+ )
+
+
+async def _record_evidence_ledger(state: InvestigationState, *, query: str, count: int, digest: str) -> None:
+ """Best-effort ledger event citing the query + result count + digest.
+
+ Records ONLY the query, count, and content hash — never credentials.
+ """
+ try:
+ await ledger_module.record_event(
+ run_id=state.run_id,
+ tenant_id=state.tenant_id,
+ seq=900, # high seq bucket for tool evidence; ON CONFLICT dedups replays
+ kind="splunk_evidence",
+ agent="splunk_evidence_tool",
+ summary=f"Splunk evidence: {count} events (digest {digest})",
+ payload={"query": query, "result_count": count, "digest": digest},
+ )
+ except Exception as exc: # noqa: BLE001 — audit is best-effort
+ logger.debug("splunk_evidence.ledger_noop", error=type(exc).__name__)
diff --git a/services/agents/app/workers/fused_alert_consumer.py b/services/agents/app/workers/fused_alert_consumer.py
index aab753e01..1a8f39cff 100644
--- a/services/agents/app/workers/fused_alert_consumer.py
+++ b/services/agents/app/workers/fused_alert_consumer.py
@@ -31,17 +31,22 @@ never dies on one poison alert.
from __future__ import annotations
+import asyncio
+import contextlib
import json
import os
import uuid
+from datetime import UTC, datetime
from typing import Any
import structlog
from app.agents.auto_triage_agent import run_auto_triage
+from app.agents.dispositions import NEEDS_REVIEW
from app.agents.triage_agent import run_triage
from app.core.cost_governor import Decision, get_governor
from app.core.cost_telemetry import CostTracker
+from app.graph.runner import default_budget, run_escalation
from app.investigator import ledger as ledger_module
from app.llm.factory import llm_override
from app.models.state import AgentStatus, InvestigationState
@@ -53,6 +58,16 @@ logger = structlog.get_logger()
_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "tryaisoc.com/agents/auto-triage")
+# Idempotency key component (issue #571): the deterministic run_id is derived
+# from (canonical alert id, workflow version), so a replayed fused message
+# reuses the same run_id and cannot duplicate ledger runs/outcomes. Bump this
+# when the triage workflow changes in a way that should re-run past alerts.
+WORKFLOW_VERSION = "auto-triage-v1"
+
+# Bounded retries before dead-lettering a poison alert (issue #571).
+_MAX_ATTEMPTS = int(os.getenv("AISOC_AGENT_MAX_ATTEMPTS", "3"))
+_RETRY_BACKOFF_S = float(os.getenv("AISOC_AGENT_RETRY_BACKOFF_S", "0.5"))
+
_METRICS = {
"triaged": 0,
"deduplicated": 0,
@@ -60,10 +75,21 @@ _METRICS = {
"llm": 0,
"bc_suppressed": 0,
"bc_mutated": 0,
+ "needs_review_fallback": 0,
+ "escalated": 0,
+ "persist_retries": 0,
+ "dead_lettered": 0,
"errors": 0,
}
+def _escalation_enabled() -> bool:
+ """Route non-auto-closed alerts through the full investigation graph
+ (issue #569). On by default; set ``AISOC_AGENT_ESCALATE_TO_GRAPH=0`` to
+ keep triage-only (e.g. in CI / low-resource deployments)."""
+ return os.getenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "1").strip().lower() not in {"0", "false", "no", "off"}
+
+
def _coerce_uuid(value: Any, *, fallback: str) -> uuid.UUID:
try:
return uuid.UUID(str(value))
@@ -71,6 +97,15 @@ def _coerce_uuid(value: Any, *, fallback: str) -> uuid.UUID:
return uuid.uuid5(_NAMESPACE, str(value or fallback))
+def _rationale_of(state: InvestigationState) -> str:
+ """Human-readable rationale for the verdict, for the ledger + alerts row."""
+ if state.confidence_basis:
+ return " · ".join(str(b) for b in state.confidence_basis)[:8000]
+ if state.findings:
+ return " · ".join(str(f) for f in state.findings)[:8000]
+ return f"Auto-triage verdict={state.verdict} confidence={state.confidence:.2f}"
+
+
def build_state(message: dict[str, Any]) -> InvestigationState | None:
"""Map an ``aisoc.alerts.fused`` message to a seeded InvestigationState."""
if not isinstance(message, dict):
@@ -80,11 +115,15 @@ def build_state(message: dict[str, Any]) -> InvestigationState | None:
return None
tenant = _coerce_uuid(message.get("tenant_id") or alert.get("tenant_id"), fallback="default")
incident = _coerce_uuid(message.get("incident_id") or message.get("id") or alert.get("id"), fallback=str(tenant))
+ # Canonical alert row id (issue #568): the fused envelope now carries the
+ # durable alerts.id as `alert_row_id`, falling back to the (also canonical)
+ # message id. This is what the verdict is persisted against.
+ alert_row_id = str(message.get("alert_row_id") or message.get("id") or alert.get("id") or "")
summary = str(alert.get("title") or "").strip()
if message.get("narrative"):
summary = f"{summary} — {message['narrative']}"[:1000] if summary else str(message["narrative"])[:1000]
raw_alert = {
- "id": str(message.get("id") or alert.get("id") or ""),
+ "id": alert_row_id,
"severity": alert.get("severity"),
"src_ip": alert.get("src_ip"),
"dst_ip": alert.get("dst_ip"),
@@ -98,8 +137,18 @@ def build_state(message: dict[str, Any]) -> InvestigationState | None:
"raw_event": alert.get("raw_event", {}),
"confidence": message.get("confidence_score"),
"fusion_decision": message.get("fusion_decision"),
+ # Connector provenance (issue #568) carried through the graph so the
+ # investigation (e.g. the Splunk evidence tool, #570) can resolve the
+ # originating connector instance.
+ "connector_id": alert.get("connector_id"),
+ "connector_type": alert.get("connector_type"),
+ "source_event_ids": alert.get("source_event_ids", []),
}
+ # Deterministic, replay-stable run id (issue #571) — same alert + workflow
+ # version ⇒ same run, so replays are idempotent in the ledger.
+ run_id = uuid.uuid5(_NAMESPACE, f"{alert_row_id}:{WORKFLOW_VERSION}") if alert_row_id else uuid.uuid4()
return InvestigationState(
+ run_id=run_id,
incident_id=incident,
tenant_id=tenant,
alert_summary=summary,
@@ -117,12 +166,17 @@ class FusedAlertTriageWorker:
bootstrap_servers: str,
topic: str = "aisoc.alerts.fused",
group_id: str = "aisoc-agents-triage",
+ dlq_topic: str | None = None,
+ max_attempts: int = _MAX_ATTEMPTS,
business_context: BusinessContextApplier | None = None,
) -> None:
self._bootstrap = bootstrap_servers
self._topic = topic
self._group_id = group_id
+ self._dlq_topic = dlq_topic or os.getenv("KAFKA_TOPIC_ALERTS_FUSED_DLQ", f"{topic}.dlq")
+ self._max_attempts = max(1, max_attempts)
self._consumer: Any | None = None
+ self._producer: Any | None = None # lazily created for the DLQ
self._running = False
# Phase B4 — environment-specific noise reduction applied post-fusion →
# pre-triage. None = disabled (no rules file / flag off).
@@ -148,24 +202,99 @@ class FusedAlertTriageWorker:
async for msg in self._consumer:
if not self._running:
break
+ # Commit ONLY after a durable outcome (verdict + ledger written)
+ # or after the poison alert is dead-lettered — so a crash between
+ # inference and persistence replays the alert instead of losing
+ # it, and offsets advance only on durable completion (issue #571).
+ await self._process_with_retry(msg.value)
try:
- await self.triage(msg.value)
- except Exception as exc: # noqa: BLE001 — one poison alert must not kill the loop
- _METRICS["errors"] += 1
- logger.error("auto_triage_worker.error", error=str(exc), exc_info=True)
- else:
- try:
- await self._consumer.commit()
- except Exception as commit_exc: # noqa: BLE001 — reprocess on restart
- logger.warning("auto_triage_worker.commit_failed", error=str(commit_exc))
+ await self._consumer.commit()
+ except Exception as commit_exc: # noqa: BLE001 — reprocess on restart
+ logger.warning("auto_triage_worker.commit_failed", error=str(commit_exc))
finally:
await self._consumer.stop()
+ async def _process_with_retry(self, message: Any) -> bool:
+ """Triage one alert with bounded retries; dead-letter on exhaustion.
+
+ Returns True on durable success, False if the message was dead-lettered.
+ Either way the caller commits (a poison alert must not wedge the loop).
+ """
+ last_error: str = ""
+ stage = "triage"
+ for attempt in range(1, self._max_attempts + 1):
+ try:
+ await self.triage(message)
+ return True
+ except ledger_module.LedgerPersistError as exc:
+ stage = "persist"
+ last_error = str(exc)
+ _METRICS["persist_retries"] += 1
+ logger.warning("auto_triage_worker.persist_retry", attempt=attempt, error=last_error)
+ except Exception as exc: # noqa: BLE001 — retry, then dead-letter
+ stage = "triage"
+ last_error = str(exc)
+ logger.warning("auto_triage_worker.triage_retry", attempt=attempt, error=last_error)
+ if attempt < self._max_attempts:
+ await asyncio.sleep(self._retry_backoff(attempt))
+ _METRICS["errors"] += 1
+ await self._send_to_dlq(message, attempts=self._max_attempts, stage=stage, error=last_error)
+ return False
+
+ def _retry_backoff(self, attempt: int) -> float:
+ return _RETRY_BACKOFF_S * attempt
+
+ async def _ensure_producer(self) -> Any | None:
+ if self._producer is not None:
+ return self._producer
+ try:
+ from aiokafka import AIOKafkaProducer # noqa: PLC0415 — optional dep, runtime only
+
+ self._producer = AIOKafkaProducer(
+ bootstrap_servers=self._bootstrap,
+ value_serializer=lambda v: json.dumps(v).encode("utf-8"),
+ )
+ await self._producer.start()
+ return self._producer
+ except Exception as exc: # noqa: BLE001 — DLQ is best-effort
+ logger.error("auto_triage_worker.dlq_producer_failed", error=str(exc))
+ self._producer = None
+ return None
+
+ async def _send_to_dlq(self, message: Any, *, attempts: int, stage: str, error: str) -> None:
+ """Publish a poison alert to the DLQ with alert id, attempts, and stage."""
+ alert_id = ""
+ if isinstance(message, dict):
+ alert = message.get("alert") if isinstance(message.get("alert"), dict) else {}
+ alert_id = str(message.get("alert_row_id") or message.get("id") or alert.get("id") or "")
+ record = {
+ "alert_id": alert_id,
+ "attempts": attempts,
+ "failure_stage": stage,
+ "error": (error or "")[:2000],
+ "dead_lettered_at": datetime.now(UTC).isoformat(),
+ "original": message if isinstance(message, dict) else {"raw": str(message)[:2000]},
+ }
+ producer = await self._ensure_producer()
+ if producer is None:
+ logger.error("auto_triage_worker.dlq_unavailable", alert_id=alert_id, stage=stage)
+ return
+ try:
+ await producer.send(self._dlq_topic, value=record)
+ _METRICS["dead_lettered"] += 1
+ logger.error("auto_triage_worker.dead_lettered", alert_id=alert_id, stage=stage, attempts=attempts)
+ except Exception as exc: # noqa: BLE001 — never wedge the loop on a DLQ failure
+ logger.error("auto_triage_worker.dlq_send_failed", alert_id=alert_id, error=str(exc))
+
async def stop(self) -> None:
self._running = False
if self._consumer is not None:
await self._consumer.stop()
self._consumer = None
+ if self._producer is not None:
+ with contextlib.suppress(Exception):
+ await self._producer.stop()
+ self._producer = None
logger.info("auto_triage_worker.stopped", metrics=dict(_METRICS))
async def triage(self, message: dict[str, Any]) -> dict[str, Any] | None:
@@ -228,6 +357,15 @@ class FusedAlertTriageWorker:
tokens = tracker.total_tokens
verdict = state.verdict
confidence = state.confidence
+ # Never complete with a null/empty verdict (issue #571): if both the
+ # LLM and deterministic paths somehow left it unset, fail safe to
+ # needs_review so the alert is escalated to a human, not auto-closed.
+ if not verdict:
+ verdict = state.verdict = NEEDS_REVIEW
+ if state.status is AgentStatus.COMPLETED:
+ state.status = AgentStatus.RUNNING
+ _METRICS["needs_review_fallback"] += 1
+ state.add_finding("Triage produced no verdict — defaulting to needs_review (escalated).")
# Close the cost-governor loop: cache the verdict (so an alert flood
# dedups instead of re-paying) and account the spend (so per-tenant
# budgets + the circuit breaker actually fire). Best-effort.
@@ -245,6 +383,14 @@ class FusedAlertTriageWorker:
_METRICS["triaged"] += 1
await self._record(state, tier=tier, verdict=verdict, confidence=confidence, tokens=tokens, cost_usd=cost_usd)
+ # Issue #569: route escalations (anything NOT auto-closed — TP,
+ # low-confidence, needs_review) through the full investigation graph.
+ # High-confidence FP/BTP already terminated (status COMPLETED) and
+ # skip enrichment. Best-effort: the verdict is already durable, so an
+ # enrichment/investigation failure never fails the triage outcome.
+ if state.status is not AgentStatus.COMPLETED:
+ await self._maybe_escalate(state)
+
return {
"run_id": str(state.run_id),
"incident_id": str(state.incident_id),
@@ -258,6 +404,23 @@ class FusedAlertTriageWorker:
"proposed_actions": [{"action_type": a.action_type, "requires_approval": a.requires_approval} for a in state.proposed_actions],
}
+ async def _maybe_escalate(self, state: InvestigationState) -> None:
+ """Run the full investigation graph for an escalated alert (issue #569).
+
+ Shares the same graph runner as the manual investigations API. Every
+ node is recorded to the ledger under the deterministic run_id, so a
+ restart re-runs idempotently. Best-effort + budgeted — a failure or
+ timeout leaves the durable verdict intact and the alert escalated.
+ """
+ if not _escalation_enabled():
+ return
+ try:
+ async with CostTracker(run_id=str(state.run_id), tenant_id=str(state.tenant_id)):
+ await run_escalation(state, budget=default_budget(), seq_start=1)
+ _METRICS["escalated"] += 1
+ except Exception as exc: # noqa: BLE001 — escalation is best-effort over a durable verdict
+ logger.warning("auto_triage_worker.escalation_failed", run_id=str(state.run_id), error=str(exc))
+
async def _resolve_tenant_llm(self, tenant_id: uuid.UUID) -> Any:
"""Resolve the tenant's LLM config, or None to force deterministic triage."""
try:
@@ -288,27 +451,42 @@ class FusedAlertTriageWorker:
tokens: int = 0,
cost_usd: float = 0.0,
) -> None:
- """Best-effort ledger write. No DB => no-op (never raises)."""
- try:
- await ledger_module.start_run(
- run_id=state.run_id,
- case_id=str(state.incident_id),
- tenant_ref=str(state.tenant_id),
- alert_summary=state.alert_summary or "",
- raw_alert=state.raw_alert or {},
- model_used=f"kafka:auto_triage:{tier}",
- )
- await ledger_module.complete_run(
- run_id=state.run_id,
- tenant_id=state.tenant_id,
- status="completed",
- error=None,
- iterations=state.iteration_count,
- total_tokens=tokens,
- total_cost_usd=cost_usd,
- )
- except Exception as exc: # noqa: BLE001 — audit is best-effort
- logger.debug("auto_triage_worker.ledger_noop", error=str(exc), verdict=verdict, confidence=confidence)
+ """Durably persist the triage outcome (issue #571).
+
+ Writes the verdict/confidence/rationale/recommendations to BOTH the
+ ledger and the ``alerts`` row in one transaction. No DB configured =>
+ no-op (returns without raising). A real DB error raises
+ :class:`LedgerPersistError` so the caller retries / dead-letters — the
+ Kafka offset is not committed until this succeeds.
+ """
+ rationale = _rationale_of(state)
+ recommendations = [
+ {
+ "action_type": a.action_type,
+ "description": a.description,
+ "risk_level": a.risk_level.value if hasattr(a.risk_level, "value") else str(a.risk_level),
+ "requires_approval": a.requires_approval,
+ "target": a.target,
+ }
+ for a in state.proposed_actions
+ ]
+ await ledger_module.persist_auto_triage(
+ run_id=state.run_id,
+ alert_id=(state.raw_alert or {}).get("id"),
+ tenant_ref=str(state.tenant_id),
+ alert_summary=state.alert_summary or "",
+ raw_alert=state.raw_alert or {},
+ tier=tier,
+ verdict=str(verdict) if verdict else NEEDS_REVIEW,
+ confidence=float(confidence or 0.0),
+ rationale=rationale,
+ findings=list(state.findings),
+ proposed_actions=recommendations,
+ auto_closed=state.status is AgentStatus.COMPLETED,
+ iterations=state.iteration_count,
+ tokens=tokens,
+ cost_usd=cost_usd,
+ )
@staticmethod
def get_metrics() -> dict[str, int]:
diff --git a/services/agents/tests/test_auto_triage_dispositions.py b/services/agents/tests/test_auto_triage_dispositions.py
index 76fa38f54..df6b1c8db 100644
--- a/services/agents/tests/test_auto_triage_dispositions.py
+++ b/services/agents/tests/test_auto_triage_dispositions.py
@@ -126,6 +126,30 @@ async def test_malicious_true_positive_escalates(monkeypatch):
assert out.status != AgentStatus.COMPLETED
+async def test_llm_timeout_raises_typed_error_not_null_verdict(monkeypatch):
+ # Issue #571: an LLM failure must raise AutoTriageError (so the worker falls
+ # back to deterministic), NOT silently return a null-verdict RUNNING state.
+ async def _boom(_llm, _messages):
+ raise TimeoutError("llm timed out")
+
+ monkeypatch.setattr(ata, "make_chat_model", lambda *a, **k: object())
+ monkeypatch.setattr(ata, "safe_ainvoke", _boom)
+ with pytest.raises(ata.AutoTriageError):
+ await ata.run_auto_triage(_state("Beaconing", {"severity": "critical"}))
+
+
+async def test_malformed_json_raises_typed_error(monkeypatch):
+ async def _bad(_llm, _messages):
+ return SimpleNamespace(content="the model rambled with no json")
+
+ monkeypatch.setattr(ata, "make_chat_model", lambda *a, **k: object())
+ monkeypatch.setattr(ata, "safe_ainvoke", _bad)
+ with pytest.raises(ata.AutoTriageError):
+ await ata.run_auto_triage(_state("Suspicious", {"severity": "high"}))
+
+
@pytest.mark.asyncio
async def test_btp_metric_is_tracked_separately_from_fp(monkeypatch):
ata._metrics.update({"fp_count": 0, "btp_count": 0, "benign_count": 0, "tp_count": 0, "total_processed": 0, "confidence_sum": 0.0})
diff --git a/services/agents/tests/test_fused_alert_worker.py b/services/agents/tests/test_fused_alert_worker.py
index 769e25de0..f8b1e7638 100644
--- a/services/agents/tests/test_fused_alert_worker.py
+++ b/services/agents/tests/test_fused_alert_worker.py
@@ -48,14 +48,22 @@ def _fused(**alert_overrides) -> dict:
@pytest.fixture(autouse=True)
def _no_ledger_db(monkeypatch):
- # Ledger writes are best-effort; make them explicit no-ops in tests.
+ # No DB in unit tests: the durable persist path no-ops. Stub it explicitly
+ # so a stray DATABASE_URL in the CI env can't make tests hit a real DB.
async def _noop(*args, **kwargs): # noqa: ANN002, ANN003
return None
+ async def _persist_noop(*args, **kwargs): # noqa: ANN002, ANN003
+ return False
+
monkeypatch.setattr(worker_mod.ledger_module, "start_run", _noop)
monkeypatch.setattr(worker_mod.ledger_module, "complete_run", _noop)
+ monkeypatch.setattr(worker_mod.ledger_module, "persist_auto_triage", _persist_noop)
# Force the deterministic path unless a test overrides (no LLM key in CI).
monkeypatch.setenv("AISOC_DETERMINISTIC", "0")
+ # Escalation runs the full graph (external calls); off by default in unit
+ # tests — the dedicated escalation tests enable it and stub run_escalation.
+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "0")
def _worker() -> FusedAlertTriageWorker:
@@ -132,6 +140,145 @@ async def test_circuit_open_forces_deterministic(monkeypatch):
assert result["tier"] == "deterministic"
+# ── #571: durable verdict, typed fallback, idempotency, DLQ ──────────────────
+
+
+async def test_llm_failure_falls_back_to_deterministic_non_null_verdict(monkeypatch):
+ # An LLM error must NOT produce a null verdict: the worker falls back to
+ # deterministic triage (which always yields a verdict).
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))
+ monkeypatch.setenv("AISOC_DETERMINISTIC", "0")
+
+ async def _raise(_state):
+ raise RuntimeError("llm timeout")
+
+ monkeypatch.setattr(worker_mod, "run_auto_triage", _raise)
+ result = await _worker().triage(_fused())
+ assert result["tier"] == "deterministic"
+ assert result["verdict"] is not None
+
+
+async def test_null_verdict_defaults_to_needs_review(monkeypatch):
+ # Even if a triage path leaves the verdict unset, we never complete null —
+ # fail safe to needs_review (escalated, not auto-closed).
+ monkeypatch.setenv("AISOC_DETERMINISTIC", "1")
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))
+
+ async def _null(state):
+ state.verdict = None
+ return state
+
+ monkeypatch.setattr(worker_mod, "run_triage", _null)
+ result = await _worker().triage(_fused())
+ assert result["verdict"] == "needs_review"
+
+
+async def test_deterministic_run_id_is_idempotent():
+ # Same alert + workflow version ⇒ same run_id, so replays don't duplicate.
+ a = build_state(_fused())
+ b = build_state(_fused())
+ assert a.run_id == b.run_id
+ # A different alert id yields a different run.
+ c = build_state(_fused() | {"id": "44444444-4444-4444-4444-444444444444", "alert_row_id": "44444444-4444-4444-4444-444444444444"})
+ assert c.run_id != a.run_id
+
+
+async def test_persist_failure_is_retried_then_dead_lettered(monkeypatch):
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))
+
+ async def _boom(*args, **kwargs): # noqa: ANN002, ANN003
+ raise worker_mod.ledger_module.LedgerPersistError("db down")
+
+ monkeypatch.setattr(worker_mod.ledger_module, "persist_auto_triage", _boom)
+
+ sent: list[tuple[str, dict]] = []
+
+ class _FakeProducer:
+ async def send(self, topic, value): # noqa: ANN001
+ sent.append((topic, value))
+
+ worker = FusedAlertTriageWorker(bootstrap_servers="unused", max_attempts=2)
+ monkeypatch.setattr(worker, "_retry_backoff", lambda attempt: 0.0)
+
+ async def _fake_producer():
+ return _FakeProducer()
+
+ monkeypatch.setattr(worker, "_ensure_producer", _fake_producer)
+
+ ok = await worker._process_with_retry(_fused())
+ assert ok is False # exhausted retries → dead-lettered
+ assert len(sent) == 1
+ topic, record = sent[0]
+ assert topic.endswith(".dlq")
+ assert record["failure_stage"] == "persist"
+ assert record["attempts"] == 2
+ assert record["alert_id"] == "22222222-2222-2222-2222-222222222222"
+
+
+async def test_successful_triage_returns_true_and_commits(monkeypatch):
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))
+ worker = FusedAlertTriageWorker(bootstrap_servers="unused", max_attempts=2)
+ # persist_auto_triage is stubbed to no-op by the autouse fixture.
+ assert await worker._process_with_retry(_fused()) is True
+
+
+# ── #569: escalation into the full investigation graph ───────────────────────
+
+
+async def test_non_auto_closed_alert_escalates_to_full_graph(monkeypatch):
+ # A critical cred-dump triages to a non-benign verdict (not auto-closed) →
+ # it must enter the full investigation graph.
+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "1")
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))
+ called: dict = {}
+
+ async def _fake_escalation(state, **kwargs): # noqa: ANN003
+ called["state"] = state
+ return state
+
+ monkeypatch.setattr(worker_mod, "run_escalation", _fake_escalation)
+ result = await _worker().triage(_fused())
+ assert result["verdict"] not in ("false_positive", "benign", "benign_true_positive")
+ assert "state" in called # escalation ran
+
+
+async def test_high_confidence_auto_closed_does_not_escalate(monkeypatch):
+ # A high-confidence FP that auto-closed must terminate WITHOUT enrichment.
+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "1")
+ monkeypatch.setenv("AISOC_DETERMINISTIC", "1")
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))
+
+ async def _auto_closed(state):
+ state.verdict = "false_positive"
+ state.status = worker_mod.AgentStatus.COMPLETED
+ return state
+
+ monkeypatch.setattr(worker_mod, "run_triage", _auto_closed)
+ calls = {"n": 0}
+
+ async def _fake_escalation(state, **kwargs): # noqa: ANN003
+ calls["n"] += 1
+ return state
+
+ monkeypatch.setattr(worker_mod, "run_escalation", _fake_escalation)
+ await _worker().triage(_fused())
+ assert calls["n"] == 0
+
+
+async def test_escalation_can_be_disabled_by_flag(monkeypatch):
+ monkeypatch.setenv("AISOC_AGENT_ESCALATE_TO_GRAPH", "0")
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=False))
+ calls = {"n": 0}
+
+ async def _fake_escalation(state, **kwargs): # noqa: ANN003
+ calls["n"] += 1
+ return state
+
+ monkeypatch.setattr(worker_mod, "run_escalation", _fake_escalation)
+ await _worker().triage(_fused())
+ assert calls["n"] == 0
+
+
# ── fail-soft ─────────────────────────────────────────────────────────────────
@@ -156,6 +303,8 @@ def _fake_llm_config(*, allowed: bool):
def __init__(self) -> None:
self.allowed = allowed
self.api_key = "sk-test" if allowed else None
+ self.base_url = None
+ self.model = "gpt-4o-mini"
async def _resolve(_tenant): # noqa: ANN001
return _Cfg()
diff --git a/services/agents/tests/test_graph_runner.py b/services/agents/tests/test_graph_runner.py
new file mode 100644
index 000000000..53efe0177
--- /dev/null
+++ b/services/agents/tests/test_graph_runner.py
@@ -0,0 +1,100 @@
+"""#569 — the shared, durable investigation-graph runner.
+
+Proves the runner records one ledger event per graph node (under the run's
+deterministic id, with a seq offset so it follows the worker's triage_verdict
+event) and that a wall-clock budget bounds the run and fails safe to
+needs_review on timeout. A tiny fake graph stands in for the compiled
+LangGraph so the runner logic is tested in isolation (no external calls).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from uuid import uuid4
+
+import pytest
+from app.graph import runner as runner_mod
+from app.graph.runner import InvestigationBudget, _run
+from app.models.state import InvestigationState
+
+pytestmark = pytest.mark.asyncio
+
+
+class _FakeGraph:
+ def __init__(self, steps: list[dict], *, delay: float = 0.0) -> None:
+ self._steps = steps
+ self._delay = delay
+
+ async def astream(self, _state_dict): # noqa: ANN001
+ for step in self._steps:
+ if self._delay:
+ await asyncio.sleep(self._delay)
+ yield step
+
+
+def _state() -> InvestigationState:
+ return InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="cred dump")
+
+
+async def test_runner_records_one_ledger_event_per_node(monkeypatch):
+ events: list[dict] = []
+
+ async def _rec(**kwargs): # noqa: ANN003
+ events.append(kwargs)
+ return uuid4()
+
+ async def _resolve(_ref): # noqa: ANN001
+ return uuid4()
+
+ async def _noop(**kwargs): # noqa: ANN003
+ return None
+
+ monkeypatch.setattr(runner_mod.ledger_module, "record_event", _rec)
+ monkeypatch.setattr(runner_mod.ledger_module, "resolve_tenant", _resolve)
+ monkeypatch.setattr(runner_mod.ledger_module, "start_run", _noop)
+ monkeypatch.setattr(runner_mod.ledger_module, "complete_run", _noop)
+
+ state = _state()
+ graph = _FakeGraph(
+ [
+ {"triage": {"verdict": "true_positive"}},
+ {"enrichment": {"findings": ["enriched"]}},
+ {"attack_path": {"status": "completed"}},
+ ]
+ )
+ await _run(graph, state, budget=InvestigationBudget(max_seconds=5), persist=True, seq_start=1)
+
+ assert [e["agent"] for e in events] == ["triage", "enrichment", "attack_path"]
+ # seq follows the worker's triage_verdict event (seq 1) without colliding.
+ assert [e["seq"] for e in events] == [2, 3, 4]
+ assert all(e["kind"] == "graph_step" for e in events)
+
+
+async def test_runner_budget_timeout_fails_safe_to_needs_review():
+ state = _state()
+ # Each step sleeps 0.2s; a 0.05s budget times out before completion.
+ graph = _FakeGraph([{"triage": {}}, {"enrichment": {}}], delay=0.2)
+ result = await _run(graph, state, budget=InvestigationBudget(max_seconds=0.05), persist=False, seq_start=0)
+ assert result.verdict == "needs_review"
+ assert any("budget" in f.lower() for f in result.findings)
+
+
+async def test_runner_merges_node_outputs_into_final_state(monkeypatch):
+ async def _resolve(_ref): # noqa: ANN001
+ return None # persist path disabled (no tenant) — pure merge test
+
+ monkeypatch.setattr(runner_mod.ledger_module, "resolve_tenant", _resolve)
+
+ state = _state()
+ graph = _FakeGraph(
+ [
+ {"triage": {"verdict": "true_positive", "confidence": 0.7}},
+ {"enrichment": {"findings": ["ioc malicious"]}},
+ ]
+ )
+ result = await _run(graph, state, budget=InvestigationBudget(max_seconds=5), persist=False, seq_start=0)
+ assert result.verdict == "true_positive"
+ assert result.confidence == 0.7
+ assert "ioc malicious" in result.findings
+ assert isinstance(result.run_id, uuid.UUID)
diff --git a/services/agents/tests/test_splunk_evidence.py b/services/agents/tests/test_splunk_evidence.py
new file mode 100644
index 000000000..f0ba0398e
--- /dev/null
+++ b/services/agents/tests/test_splunk_evidence.py
@@ -0,0 +1,217 @@
+"""#570 — bounded, read-only Splunk evidence tool for the investigation graph.
+
+Verifies the query is time/entity-bounded and allow-listed, credentials come
+from the vault (never a prompt) and never leak into findings/queries, a
+failure/timeout is treated as *missing evidence* (escalates to needs_review),
+and results are cited (redacted + hashed) in the investigation. The Splunk REST
+call is mocked (fake httpx client) so it runs in agents CI with no live server.
+"""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+import pytest
+from app.agents import investigation_agent as ia
+from app.agents.dispositions import FALSE_POSITIVE, NEEDS_REVIEW
+from app.investigator import splunk_evidence as se
+from app.investigator.splunk_evidence import (
+ SplunkCreds,
+ SplunkEvidenceClient,
+ SplunkEvidenceResult,
+ build_evidence_spl,
+ collect_splunk_evidence,
+)
+from app.models.state import AgentStatus, InvestigationState
+
+pytestmark = pytest.mark.asyncio
+
+TOKEN = "s3cr3t-splunk-token-should-never-leak"
+
+
+def _state(**raw) -> InvestigationState:
+ base = {
+ "id": "22222222-2222-2222-2222-222222222222",
+ "connector_id": "00000000-0000-0000-0000-0000000000c1",
+ "connector_type": "splunk",
+ "username": "svc-backup",
+ "hostname": "WIN-DC01",
+ "src_ip": "10.20.30.40",
+ "event_time": "2026-07-12T08:00:00Z",
+ }
+ base.update(raw)
+ return InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="cred dump", raw_alert=base, status=AgentStatus.RUNNING)
+
+
+# ── SPL builder (pure) ───────────────────────────────────────────────────────
+
+
+async def test_build_evidence_spl_is_time_and_entity_bounded():
+ spl = build_evidence_spl(
+ {"username": "svc-backup", "hostname": "WIN-DC01", "src_ip": "10.0.0.5", "file_hash": "a" * 64},
+ index="main",
+ max_results=50,
+ )
+ assert spl is not None
+ assert spl.startswith("search index=main (")
+ assert 'user="svc-backup"' in spl
+ assert 'host="WIN-DC01"' in spl
+ assert 'src_ip="10.0.0.5"' in spl
+ assert "| head 50" in spl # bounded result count
+
+
+async def test_build_evidence_spl_none_without_pivot():
+ assert build_evidence_spl({"severity": "high"}) is None
+
+
+async def test_spl_values_are_quote_escaped():
+ # An adversary-controlled username with a quote must not break out of the
+ # SPL string (allow-listed construction).
+ spl = build_evidence_spl({"username": 'a" OR host="*'})
+ assert spl is not None
+ assert '\\"' in spl # the inner quote is escaped
+ assert 'user="a\\" OR host=\\"*"' in spl
+
+
+async def test_time_window_is_bounded():
+ earliest, latest = se._time_window({"event_time": "2026-07-12T08:00:00Z"})
+ # epoch strings, 2 * window hours apart
+ span_hours = (int(latest) - int(earliest)) / 3600
+ assert span_hours == pytest.approx(2 * se._WINDOW_HOURS, abs=0.01)
+
+
+# ── credential resolution + collection ───────────────────────────────────────
+
+
+async def test_collect_not_applicable_without_connector():
+ # No DB in tests => resolve returns None => not a Splunk alert => no-op.
+ result = await collect_splunk_evidence(_state(connector_id=None))
+ assert result.applicable is False
+ assert result.queried is False
+
+
+async def test_collect_success_cites_redacted_hashed_evidence(monkeypatch):
+ async def _creds(_state):
+ return SplunkCreds(base_url="https://splunk.local:8089", token=TOKEN, ssl_verify=True, index="main")
+
+ class _FakeClient:
+ def __init__(self, creds, **kwargs): # noqa: ANN001, ANN003
+ self._creds = creds
+
+ async def search(self, spl, *, earliest, latest): # noqa: ANN001
+ return [{"_time": "2026-07-12T07:59:00Z", "host": "WIN-DC01", "user": "svc-backup", "action": "lsass_access"}]
+
+ monkeypatch.setattr(se, "resolve_splunk_credentials", _creds)
+ monkeypatch.setattr(se, "_make_client", lambda creds: _FakeClient(creds))
+
+ state = _state()
+ result = await collect_splunk_evidence(state)
+
+ assert result.applicable is True
+ assert result.queried is True
+ assert result.result_count == 1
+ # Evidence cited in findings with a content hash…
+ joined = "\n".join(state.findings)
+ assert "Splunk evidence: 1 surrounding event" in joined
+ assert "digest=" in joined
+ # …and the credential NEVER appears in findings or the query.
+ assert TOKEN not in joined
+ assert result.query is not None and TOKEN not in result.query
+
+
+async def test_collect_query_failure_is_missing_evidence(monkeypatch):
+ async def _creds(_state):
+ return SplunkCreds(base_url="https://splunk.local:8089", token=TOKEN)
+
+ class _BoomClient:
+ def __init__(self, creds, **kwargs): # noqa: ANN001, ANN003
+ pass
+
+ async def search(self, *a, **k): # noqa: ANN002, ANN003
+ raise TimeoutError("splunk timed out")
+
+ monkeypatch.setattr(se, "resolve_splunk_credentials", _creds)
+ monkeypatch.setattr(se, "_make_client", lambda creds: _BoomClient(creds))
+
+ state = _state()
+ result = await collect_splunk_evidence(state)
+ assert result.applicable is True
+ assert result.queried is False
+ assert result.error is not None
+ assert any("missing evidence" in f.lower() for f in state.findings)
+ assert TOKEN not in "\n".join(state.findings)
+
+
+# ── bounded REST client (mocked httpx) ───────────────────────────────────────
+
+
+async def test_client_issues_bounded_oneshot_request(monkeypatch):
+ captured: dict = {}
+
+ class _FakeResp:
+ def raise_for_status(self):
+ return None
+
+ def json(self):
+ return {"results": [{"_time": "t", "host": "h"}, {"_time": "t2", "host": "h2"}]}
+
+ class _FakeAsyncClient:
+ def __init__(self, *args, **kwargs):
+ captured["init"] = kwargs
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *args):
+ return None
+
+ async def post(self, url, headers=None, data=None): # noqa: ANN001
+ captured["url"] = url
+ captured["headers"] = headers
+ captured["data"] = data
+ return _FakeResp()
+
+ monkeypatch.setattr(se.httpx, "AsyncClient", _FakeAsyncClient)
+
+ creds = SplunkCreds(base_url="https://splunk.local:8089", token=TOKEN, ssl_verify=False)
+ client = SplunkEvidenceClient(creds, max_results=50)
+ results = await client.search('search index=main (host="h") | head 50', earliest="-6h", latest="now")
+
+ assert len(results) == 2
+ assert captured["url"].endswith("/services/search/jobs")
+ assert captured["data"]["exec_mode"] == "oneshot" # read-only oneshot
+ assert captured["data"]["count"] == "50" # bounded
+ assert captured["data"]["earliest_time"] == "-6h"
+ assert captured["data"]["latest_time"] == "now"
+ # verify=False threaded to httpx; token in the header only (never returned).
+ assert captured["init"].get("verify") is False
+ assert captured["headers"]["Authorization"] == f"Bearer {TOKEN}"
+
+
+# ── investigation-agent escalation on missing evidence ───────────────────────
+
+
+async def test_investigation_escalates_to_needs_review_on_missing_evidence(monkeypatch):
+ async def _missing(_state):
+ return SplunkEvidenceResult(applicable=True, queried=False, error="timeout")
+
+ monkeypatch.setattr(ia, "collect_splunk_evidence", _missing)
+ # Force the base scorer to a dismissive verdict; the #570 guard must upgrade it.
+ monkeypatch.setattr(ia, "score_investigation", lambda state: (0.9, ["scored fp"], FALSE_POSITIVE))
+
+ state = InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="x", raw_alert={"connector_type": "splunk"})
+ result = await ia.run_investigation(state)
+ assert result.verdict == NEEDS_REVIEW
+
+
+async def test_investigation_keeps_verdict_when_evidence_present(monkeypatch):
+ async def _ok(_state):
+ return SplunkEvidenceResult(applicable=True, queried=True, result_count=3)
+
+ monkeypatch.setattr(ia, "collect_splunk_evidence", _ok)
+ monkeypatch.setattr(ia, "score_investigation", lambda state: (0.9, ["scored fp"], FALSE_POSITIVE))
+
+ state = InvestigationState(incident_id=uuid4(), tenant_id=uuid4(), alert_summary="x", raw_alert={"connector_type": "splunk"})
+ result = await ia.run_investigation(state)
+ # Evidence was successfully gathered → the scorer's verdict stands.
+ assert result.verdict == FALSE_POSITIVE
diff --git a/services/api/app/models/alert.py b/services/api/app/models/alert.py
index 82569aede..e93bfaf60 100644
--- a/services/api/app/models/alert.py
+++ b/services/api/app/models/alert.py
@@ -33,6 +33,10 @@ class Alert(Base):
connector_type: Mapped[str | None] = mapped_column(String(100), nullable=True)
source_event_ids: Mapped[list] = mapped_column(JSONB, default=list)
ocsf_class_uid: Mapped[int | None] = mapped_column(Integer, nullable=True)
+ # Detection-rule / Splunk saved-search provenance (issue #568) — explicit
+ # identifiers so downstream never parses them out of the title/source.
+ rule_id: Mapped[str | None] = mapped_column(Text, nullable=True)
+ rule_name: Mapped[str | None] = mapped_column(Text, nullable=True)
# Analyst verdict / disposition (set via feedback endpoint)
disposition: Mapped[str | None] = mapped_column(String(50), nullable=True, index=True)
diff --git a/services/api/migrations/047_alerts_rule_provenance.sql b/services/api/migrations/047_alerts_rule_provenance.sql
new file mode 100644
index 000000000..f56d0f492
--- /dev/null
+++ b/services/api/migrations/047_alerts_rule_provenance.sql
@@ -0,0 +1,16 @@
+-- Issue #568: preserve source-rule provenance on the alert row.
+--
+-- Fusion previously encoded the firing detection rule only inside the
+-- human-readable `source` ("detection:<id>") and `title` strings, so the
+-- auto-triage worker and investigation graph had to parse them back out.
+-- These columns carry the rule identifier + name (and, for vendor-asserted
+-- Splunk notables, the saved-search name) as first-class fields.
+--
+-- Idempotent + transactional so the create_all lineage and repeated compose
+-- installs stay green.
+BEGIN;
+
+ALTER TABLE alerts ADD COLUMN IF NOT EXISTS rule_id TEXT;
+ALTER TABLE alerts ADD COLUMN IF NOT EXISTS rule_name TEXT;
+
+COMMIT;
diff --git a/services/fusion/app/models/alert.py b/services/fusion/app/models/alert.py
index b81306a37..e2bcfd70e 100644
--- a/services/fusion/app/models/alert.py
+++ b/services/fusion/app/models/alert.py
@@ -10,10 +10,18 @@ import json
from datetime import datetime
from enum import Enum
from typing import Any
-from uuid import UUID, uuid4
+from uuid import UUID, uuid4, uuid5
from pydantic import BaseModel, Field
+# Fixed namespace for deriving the canonical, replay-stable alert UUID from an
+# alert's dedup fingerprint (issue #568). Using uuid5 over a constant namespace
+# means the same logical alert (same tenant + fingerprint) always resolves to
+# the same UUID across RawAlert → FusedAlert → Kafka → Postgres → ledger → API,
+# so a replayed source event maps to exactly one alert row instead of minting a
+# fresh random id on every pass.
+ALERT_ID_NAMESPACE = UUID("a15c0c00-0000-5568-a150-c000000a1e77")
+
class AlertSeverity(str, Enum):
CRITICAL = "critical"
@@ -68,9 +76,30 @@ class RawAlert(BaseModel):
tags: list[str] = Field(default_factory=list)
risk_score: float = 0.0
+ # Provenance (issue #568) — carried first-class end-to-end so downstream
+ # (auto-triage, the investigation graph, the Splunk evidence tool) can
+ # resolve the originating connector and source events WITHOUT parsing the
+ # human-readable title/source strings.
+ connector_id: UUID | None = None
+ connector_type: str | None = None
+ source_event_ids: list[str] = Field(default_factory=list)
+ ocsf_class_uid: int | None = None
+ rule_id: str | None = None # detection rule / Splunk saved-search identifier
+ rule_name: str | None = None
+
created_at: datetime = Field(default_factory=datetime.utcnow)
event_time: datetime | None = None
+ def deterministic_id(self) -> UUID:
+ """Replay-stable canonical UUID derived from the dedup fingerprint.
+
+ Same tenant + fingerprint ⇒ same UUID on every pass, so replays resolve
+ to exactly one alert row (issue #568). This is intentionally aligned
+ with :meth:`fingerprint` (the dedup key) — two events that dedup to the
+ same row also share the same canonical id.
+ """
+ return uuid5(ALERT_ID_NAMESPACE, f"{self.tenant_id}:{self.fingerprint()}")
+
def fingerprint(self) -> str:
"""Generate a stable deduplication fingerprint."""
fields = {
diff --git a/services/fusion/app/services/alert_sink.py b/services/fusion/app/services/alert_sink.py
index a4cc3537d..abb336041 100644
--- a/services/fusion/app/services/alert_sink.py
+++ b/services/fusion/app/services/alert_sink.py
@@ -21,6 +21,8 @@ Design constraints:
from __future__ import annotations
import json
+from dataclasses import dataclass
+from enum import Enum
import asyncpg
import structlog
@@ -29,30 +31,69 @@ from app.models.alert import FusedAlert, FusionDecision
logger = structlog.get_logger()
+
+class PersistOutcome(str, Enum):
+ """Distinguishable outcomes of a persist attempt (issue #568).
+
+ The old sink collapsed "inserted", "already-existed", "DB down", and
+ "insert errored" all into ``None``, so the caller could not tell a real
+ duplicate from an outage it should retry. Each state is now explicit.
+ """
+
+ INSERTED = "inserted" # a new row was written; alert_id is the row id
+ DUPLICATE = "duplicate" # already existed (dedup hit or id conflict)
+ UNAVAILABLE = "unavailable" # DB unreachable — retryable, NOT a duplicate
+ FAILED = "failed" # insert errored (bad tenant, SQL error) — observable
+
+
+@dataclass(frozen=True)
+class PersistResult:
+ """Outcome of :meth:`AlertSink.persist` plus the canonical alert id."""
+
+ outcome: PersistOutcome
+ alert_id: str | None = None
+
+ @property
+ def persisted(self) -> bool:
+ return self.outcome is PersistOutcome.INSERTED
+
+ @property
+ def durable(self) -> bool:
+ """True when the alert is known to exist in the store (new or dup)."""
+ return self.outcome in (PersistOutcome.INSERTED, PersistOutcome.DUPLICATE)
+
+
_INSERT_SQL = """
INSERT INTO alerts (
- tenant_id, title, description, severity, status,
+ id, tenant_id, title, description, severity, status,
mitre_tactics, mitre_techniques, iocs, entities, raw_event,
dedup_hash, confidence, confidence_label, confidence_rationale,
- narrative, anomaly_score, event_time
+ narrative, anomaly_score, event_time,
+ connector_id, connector_type, source_event_ids, ocsf_class_uid,
+ rule_id, rule_name
)
SELECT
- $1, $2, $3, $4, 'new',
- $5::jsonb, $6::jsonb, $7::jsonb, $8::jsonb, $9::jsonb,
- $10::text, $11, $12, $13::jsonb,
- $14, $15, COALESCE($16, NOW())
+ $1, $2, $3, $4, $5, 'new',
+ $6::jsonb, $7::jsonb, $8::jsonb, $9::jsonb, $10::jsonb,
+ $11::text, $12, $13, $14::jsonb,
+ $15, $16, COALESCE($17, NOW()),
+ $18, $19, $20::jsonb, $21,
+ $22, $23
WHERE NOT EXISTS (
- SELECT 1 FROM alerts WHERE tenant_id = $1 AND dedup_hash = $10::text
+ SELECT 1 FROM alerts WHERE tenant_id = $2 AND dedup_hash = $11::text
)
+ON CONFLICT (id) DO NOTHING
RETURNING id
"""
-# NB: $10 (dedup_hash) is cast to ::text in BOTH the INSERT target and the
-# WHERE. `dedup_hash` is VARCHAR(64), and varchar comparisons resolve through
-# text operators — so `WHERE dedup_hash = $10` deduces $10 as text while the
-# INSERT target deduces it as varchar. asyncpg's prepare cannot unify the two
-# ("inconsistent types deduced for parameter $10: text versus character
-# varying") and the persist fails silently, so no alert row is ever written.
-# Pinning both uses to ::text makes the deduction unambiguous.
+# Two idempotency guards, complementary (issue #568):
+# * WHERE NOT EXISTS (tenant_id, dedup_hash) — content dedup; also protects
+# legacy rows written before ids were deterministic (random id, same hash).
+# * ON CONFLICT (id) DO NOTHING — exact-replay idempotency on the canonical
+# deterministic UUID (RawAlert.deterministic_id()).
+# NB: $11 (dedup_hash) is cast ::text in BOTH the INSERT target and the WHERE.
+# `dedup_hash` is VARCHAR(64); a varchar/text mismatch makes asyncpg's prepare
+# fail to unify parameter $11 ("inconsistent types deduced"), silently dropping
+# every write. Pinning both uses to ::text makes the deduction unambiguous.
def _asyncpg_dsn(url: str) -> str:
@@ -115,18 +156,23 @@ class AlertSink:
await self.start()
return self._pool
- async def persist(self, fused: FusedAlert) -> str | None:
- """Insert one fused alert. Returns the new row id, or ``None`` when
- skipped (duplicate decision, dedup hit, or DB unavailable)."""
+ async def persist(self, fused: FusedAlert) -> PersistResult:
+ """Insert one fused alert, returning a structured :class:`PersistResult`.
+
+ The canonical alert id is ``fused.id`` (deterministic — issue #568), so
+ even a duplicate / outage carries the id the row resolves to once the
+ DB is reachable.
+ """
+ canonical_id = str(fused.id)
if fused.fusion_decision == FusionDecision.DUPLICATE:
- return None
+ return PersistResult(PersistOutcome.DUPLICATE, canonical_id)
pool = await self._ensure_pool()
if pool is None:
if not self._connect_failed_logged:
logger.error("alert_sink.unavailable_dropping_persistence")
self._connect_failed_logged = True
- return None
+ return PersistResult(PersistOutcome.UNAVAILABLE, canonical_id)
self._connect_failed_logged = False
alert = fused.alert
@@ -134,6 +180,7 @@ class AlertSink:
async with pool.acquire() as conn:
row = await conn.fetchrow(
_INSERT_SQL,
+ fused.id,
alert.tenant_id,
alert.title[:500],
alert.description or None,
@@ -150,15 +197,22 @@ class AlertSink:
fused.narrative,
fused.anomaly_score,
alert.event_time,
+ alert.connector_id,
+ alert.connector_type,
+ json.dumps(alert.source_event_ids),
+ alert.ocsf_class_uid,
+ alert.rule_id,
+ alert.rule_name,
)
if row is None:
logger.debug("alert_sink.dedup_skip", fingerprint=alert.fingerprint())
- return None
- return str(row["id"])
+ return PersistResult(PersistOutcome.DUPLICATE, canonical_id)
+ return PersistResult(PersistOutcome.INSERTED, str(row["id"]))
except asyncpg.ForeignKeyViolationError:
# Unknown tenant — a mis-provisioned connector, not a pipeline bug.
+ # Distinct from a duplicate so it is observable, not silently masked.
logger.warning("alert_sink.unknown_tenant", tenant_id=str(alert.tenant_id))
- return None
+ return PersistResult(PersistOutcome.FAILED, None)
except Exception as exc: # noqa: BLE001 — one bad row must not wedge the consumer
logger.error("alert_sink.persist_failed", error=str(exc))
- return None
+ return PersistResult(PersistOutcome.FAILED, None)
diff --git a/services/fusion/app/services/detection_engine.py b/services/fusion/app/services/detection_engine.py
index f9ff7d14a..405c59511 100644
--- a/services/fusion/app/services/detection_engine.py
+++ b/services/fusion/app/services/detection_engine.py
@@ -38,6 +38,7 @@ import structlog
from app.models.alert import AlertSeverity, RawAlert
from app.services.detection_matcher import matches
+from app.services.provenance import extract_provenance
logger = structlog.get_logger()
@@ -136,6 +137,7 @@ class DetectionEngine:
tenant_id = uuid.UUID(str(tenant_raw))
except (ValueError, TypeError):
return None
+ connector_id, connector_type, class_uid = extract_provenance(message, ocsf)
return RawAlert(
tenant_id=tenant_id,
source=f"detection:{hit.rule_id}",
@@ -148,6 +150,11 @@ class DetectionEngine:
username=_get(ocsf, "actor", "user", "name"),
mitre_techniques=hit.mitre,
raw_event=ocsf,
+ connector_id=connector_id,
+ connector_type=connector_type,
+ ocsf_class_uid=class_uid,
+ rule_id=hit.rule_id,
+ rule_name=hit.name,
)
diff --git a/services/fusion/app/services/promoter.py b/services/fusion/app/services/promoter.py
index ea0230f5d..9ced99e3e 100644
--- a/services/fusion/app/services/promoter.py
+++ b/services/fusion/app/services/promoter.py
@@ -34,6 +34,7 @@ from typing import Any
import structlog
from app.models.alert import AlertSeverity, RawAlert
+from app.services.provenance import extract_provenance
logger = structlog.get_logger()
@@ -161,6 +162,7 @@ def promote_normalized_event(message: dict[str, Any]) -> RawAlert | None:
severity = _SEVERITY_BY_ID.get(severity_id if isinstance(severity_id, int) else 0, AlertSeverity.MEDIUM)
tactics, techniques = _mitre(ocsf)
+ connector_id, connector_type, class_uid = extract_provenance(message, ocsf)
return RawAlert(
tenant_id=tenant_id,
@@ -177,4 +179,7 @@ def promote_normalized_event(message: dict[str, Any]) -> RawAlert | None:
mitre_techniques=techniques,
raw_event=ocsf,
event_time=_event_time(ocsf),
+ connector_id=connector_id,
+ connector_type=connector_type,
+ ocsf_class_uid=class_uid,
)
diff --git a/services/fusion/app/services/provenance.py b/services/fusion/app/services/provenance.py
new file mode 100644
index 000000000..3c9cf2c8e
--- /dev/null
+++ b/services/fusion/app/services/provenance.py
@@ -0,0 +1,62 @@
+"""Extract connector / OCSF provenance from an ingest ``raw_events`` envelope.
+
+Issue #568: the fusion spine dropped the connector instance id, connector type,
+and OCSF class on the floor when promoting/detecting alerts, so every persisted
+row rendered as source "unknown" and downstream agents could not resolve the
+originating connector (needed by the Splunk evidence tool in #570). This helper
+recovers that provenance from the envelope the Go ingest service publishes.
+
+The ingest ``NormalizedEvent`` carries ``connector_id`` at the top level and,
+inside ``ocsf_event``, both ``source_connector_id`` and ``metadata.product``.
+``connector_type`` is not always on the wire, so we fall back to the OCSF
+product vendor/name for a human-meaningful provenance label; the canonical
+connector *instance* id is what downstream resolution actually keys on.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+
+def _parse_uuid(value: Any) -> uuid.UUID | None:
+ if value in (None, ""):
+ return None
+ try:
+ return uuid.UUID(str(value))
+ except (ValueError, TypeError):
+ return None
+
+
+def _product_label(ocsf: dict[str, Any]) -> str | None:
+ meta = ocsf.get("metadata") if isinstance(ocsf, dict) else None
+ product = meta.get("product") if isinstance(meta, dict) else None
+ if not isinstance(product, dict):
+ return None
+ vendor = product.get("vendor_name")
+ name = product.get("name")
+ parts = [p for p in (vendor, name) if isinstance(p, str) and p]
+ return " ".join(parts) or None
+
+
+def extract_provenance(
+ message: dict[str, Any],
+ ocsf: dict[str, Any] | None = None,
+) -> tuple[uuid.UUID | None, str | None, int | None]:
+ """Return ``(connector_id, connector_type, ocsf_class_uid)`` for a message.
+
+ Never raises — a malformed envelope yields ``(None, None, None)`` so the
+ consumer keeps flowing.
+ """
+ ocsf = ocsf if isinstance(ocsf, dict) else (message.get("ocsf_event") if isinstance(message, dict) else None)
+ ocsf = ocsf if isinstance(ocsf, dict) else {}
+
+ connector_id = _parse_uuid(message.get("connector_id")) or _parse_uuid(ocsf.get("source_connector_id"))
+ connector_type = message.get("connector_type") if isinstance(message.get("connector_type"), str) else None
+ if not connector_type:
+ connector_type = _product_label(ocsf)
+
+ class_uid_raw = ocsf.get("class_uid")
+ class_uid = class_uid_raw if isinstance(class_uid_raw, int) else None
+
+ return connector_id, connector_type, class_uid
diff --git a/services/fusion/app/services/windowed_detection.py b/services/fusion/app/services/windowed_detection.py
index b393b1ef5..d868ae936 100644
--- a/services/fusion/app/services/windowed_detection.py
+++ b/services/fusion/app/services/windowed_detection.py
@@ -30,6 +30,7 @@ import structlog
from app.models.alert import AlertSeverity, RawAlert
from app.services.detection_engine import DetectionHit
from app.services.detection_matcher import matches
+from app.services.provenance import extract_provenance
logger = structlog.get_logger()
@@ -180,6 +181,7 @@ class WindowedDetectionEngine:
except (ValueError, TypeError):
return None
fields = self._fields(message)
+ connector_id, connector_type, class_uid = extract_provenance(message, ocsf)
return RawAlert(
tenant_id=tenant_id,
source=f"detection:{hit.rule_id}",
@@ -191,4 +193,9 @@ class WindowedDetectionEngine:
username=fields.get("user"),
mitre_techniques=hit.mitre,
raw_event=ocsf,
+ connector_id=connector_id,
+ connector_type=connector_type,
+ ocsf_class_uid=class_uid,
+ rule_id=hit.rule_id,
+ rule_name=hit.name,
)
diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.py
index 813f0ec4e..d18925353 100644
--- a/services/fusion/app/workers/consumer.py
+++ b/services/fusion/app/workers/consumer.py
@@ -13,7 +13,7 @@ from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
from app.core.config import settings
from app.models.alert import FusionDecision, RawAlert
-from app.services.alert_sink import AlertSink
+from app.services.alert_sink import AlertSink, PersistOutcome, PersistResult
from app.services.detection_engine import DetectionEngine
from app.services.dlq import DeadLetter, DeadLetterQueue, LoggingDLQ, safe_record
from app.services.event_schema import validate_event
@@ -33,6 +33,8 @@ _METRICS = {
"promoted": 0,
"not_promoted": 0,
"persisted": 0,
+ "persist_unavailable": 0,
+ "persist_failed": 0,
"laked": 0,
"detected": 0,
"dead_lettered": 0,
@@ -273,7 +275,19 @@ class FusionWorker:
await self._fuse_and_persist(alert, source_event_id=validation.source_event_id)
async def _fuse_and_persist(self, alert: RawAlert, *, source_event_id: str | None = None) -> None:
- """Run one RawAlert through fusion, publish it, and persist it."""
+ """Run one RawAlert through fusion, persist it, then publish it.
+
+ Issue #568: the canonical, replay-stable alert id and source-event
+ provenance are stamped on the RawAlert *before* fusion, so ``FusedAlert.id``
+ (derived from ``alert.id``) matches the persisted ``alerts.id`` row.
+ Persist runs *before* publish so the published envelope carries the
+ durable row id + a truthful persistence outcome downstream can trust.
+ """
+ # Stamp source-event provenance + canonical id (issue #568).
+ if source_event_id and source_event_id not in alert.source_event_ids:
+ alert.source_event_ids.append(source_event_id)
+ alert.id = alert.deterministic_id()
+
fused = await self._engine.process(alert)
_METRICS["processed"] += 1
@@ -284,17 +298,28 @@ class FusionWorker:
else:
_METRICS["new_incidents"] += 1
- # Publish fused alert (even duplicates, so downstream can track)
+ # Persist FIRST (Phase 3.1 sink). Fail-soft + idempotent — see
+ # app/services/alert_sink.py. Duplicates are handled inside.
+ result = PersistResult(PersistOutcome.UNAVAILABLE, str(fused.id))
+ if self._sink is not None:
+ result = await self._sink.persist(fused)
+ if result.outcome is PersistOutcome.INSERTED:
+ _METRICS["persisted"] += 1
+ elif result.outcome is PersistOutcome.UNAVAILABLE:
+ _METRICS["persist_unavailable"] += 1
+ elif result.outcome is PersistOutcome.FAILED:
+ _METRICS["persist_failed"] += 1
+
+ # Publish fused alert (even duplicates, so downstream can track),
+ # carrying the durable row id + persistence outcome (issue #568).
+ envelope = fused.model_dump(mode="json")
+ envelope["alert_row_id"] = result.alert_id
+ envelope["persist_outcome"] = result.outcome.value
await self._producer.send(
settings.kafka_topic_alerts_fused,
- value=fused.model_dump(mode="json"),
+ value=envelope,
)
- # Persist to the alert store (Phase 3.1). Fail-soft + idempotent —
- # see app/services/alert_sink.py. Duplicates are skipped inside.
- if self._sink is not None and await self._sink.persist(fused) is not None:
- _METRICS["persisted"] += 1
-
@staticmethod
def get_metrics() -> dict:
return dict(_METRICS)
diff --git a/services/fusion/tests/test_alert_sink.py b/services/fusion/tests/test_alert_sink.py
index 250eb3575..65fa4c1d9 100644
--- a/services/fusion/tests/test_alert_sink.py
+++ b/services/fusion/tests/test_alert_sink.py
@@ -8,6 +8,7 @@ not a mock.
from __future__ import annotations
+import json
import uuid
from contextlib import asynccontextmanager
@@ -18,9 +19,16 @@ from app.models.alert import (
FusionDecision,
RawAlert,
)
-from app.services.alert_sink import AlertSink, _asyncpg_dsn, _entities, _iocs
+from app.services.alert_sink import (
+ AlertSink,
+ PersistOutcome,
+ _asyncpg_dsn,
+ _entities,
+ _iocs,
+)
TENANT = uuid.UUID("00000000-0000-0000-0000-000000000001")
+CONNECTOR = uuid.UUID("00000000-0000-0000-0000-0000000000c1")
def _fused(decision: FusionDecision = FusionDecision.NEW_ALERT) -> FusedAlert:
@@ -33,7 +41,15 @@ def _fused(decision: FusionDecision = FusionDecision.NEW_ALERT) -> FusedAlert:
hostname="WIN-DC01",
username="svc-backup",
mitre_techniques=["T1003"],
+ connector_id=CONNECTOR,
+ connector_type="crowdstrike_falcon",
+ source_event_ids=["evt-1"],
+ ocsf_class_uid=2004,
+ rule_id="edr-cred-dump-1",
+ rule_name="Credential dumping via LSASS access",
)
+ # Canonical, replay-stable id (issue #568) — the sink inserts it explicitly.
+ raw.id = raw.deterministic_id()
return FusedAlert(
id=raw.id,
tenant_id=TENANT,
@@ -83,54 +99,80 @@ async def test_duplicate_decision_is_never_persisted():
sink = AlertSink("postgresql://x")
sink._pool = _StubPool(row={"id": uuid.uuid4()}) # would succeed if called
result = await sink.persist(_fused(FusionDecision.DUPLICATE))
- assert result is None
+ assert result.outcome is PersistOutcome.DUPLICATE
assert sink._pool.conn.calls == []
@pytest.mark.asyncio
-async def test_new_alert_is_inserted_with_fingerprint():
- row_id = uuid.uuid4()
- pool = _StubPool(row={"id": row_id})
+async def test_new_alert_is_inserted_with_canonical_id_and_provenance():
+ fused = _fused()
+ # The sink inserts the canonical id explicitly; the DB echoes it back.
+ pool = _StubPool(row={"id": fused.id})
sink = AlertSink("postgresql://x")
sink._pool = pool
- fused = _fused()
result = await sink.persist(fused)
- assert result == str(row_id)
+ assert result.outcome is PersistOutcome.INSERTED
+ assert result.alert_id == str(fused.id)
+ assert result.persisted is True
(sql, args) = pool.conn.calls[0]
assert "INSERT INTO alerts" in sql
- assert "WHERE NOT EXISTS" in sql # idempotency guard
- assert args[0] == TENANT
- assert args[9] == fused.alert.fingerprint() # dedup_hash
+ assert "WHERE NOT EXISTS" in sql # content dedup guard
+ assert "ON CONFLICT (id) DO NOTHING" in sql # exact-replay idempotency
+ # Positional args follow the _INSERT_SQL column order.
+ assert args[0] == fused.id # explicit canonical id
+ assert args[1] == TENANT
+ assert args[10] == fused.alert.fingerprint() # dedup_hash
+ assert args[17] == CONNECTOR # connector_id
+ assert args[18] == "crowdstrike_falcon" # connector_type
+ assert json.loads(args[19]) == ["evt-1"] # source_event_ids
+ assert args[20] == 2004 # ocsf_class_uid
+ assert args[21] == "edr-cred-dump-1" # rule_id
@pytest.mark.asyncio
-async def test_dedup_hit_returns_none():
- pool = _StubPool(row=None) # WHERE NOT EXISTS filtered the insert out
+async def test_dedup_hit_returns_duplicate():
+ pool = _StubPool(row=None) # WHERE NOT EXISTS / ON CONFLICT filtered it out
sink = AlertSink("postgresql://x")
sink._pool = pool
- assert await sink.persist(_fused()) is None
+ result = await sink.persist(_fused())
+ assert result.outcome is PersistOutcome.DUPLICATE
+ assert result.alert_id == str(_fused().id) # canonical id still known
+ assert result.durable is True
@pytest.mark.asyncio
-async def test_db_unavailable_degrades_without_raising(monkeypatch):
+async def test_db_unavailable_is_distinct_from_duplicate(monkeypatch):
sink = AlertSink("postgresql://nope:[email protected]:1/none")
async def _fail_start():
sink._pool = None
monkeypatch.setattr(sink, "start", _fail_start)
- # Must not raise — fail-soft is the contract.
- assert await sink.persist(_fused()) is None
+ result = await sink.persist(_fused())
+ # Fail-soft (no raise) but observably UNAVAILABLE — retryable, NOT a dup.
+ assert result.outcome is PersistOutcome.UNAVAILABLE
+ assert result.durable is False
@pytest.mark.asyncio
-async def test_persist_exception_is_swallowed():
+async def test_persist_exception_is_failed_not_duplicate():
class _BoomPool(_StubPool):
def acquire(self):
raise RuntimeError("connection reset")
sink = AlertSink("postgresql://x")
sink._pool = _BoomPool(row=None)
- assert await sink.persist(_fused()) is None
+ result = await sink.persist(_fused())
+ assert result.outcome is PersistOutcome.FAILED
+ assert result.durable is False
+
+
+def test_deterministic_id_is_replay_stable():
+ # Same tenant + content ⇒ same canonical id across independent builds.
+ a = _fused().alert
+ b = _fused().alert
+ assert a.id == b.id == a.deterministic_id()
+ # And it is a proper uuid5 (not the random default_factory).
+ assert a.id != uuid.uuid4()
diff --git a/services/fusion/tests/test_consumer_dlq.py b/services/fusion/tests/test_consumer_dlq.py
index a347db1c7..d157d041c 100644
--- a/services/fusion/tests/test_consumer_dlq.py
+++ b/services/fusion/tests/test_consumer_dlq.py
@@ -43,6 +43,7 @@ def _worker_with_dlq() -> tuple[FusionWorker, InMemoryDLQ]:
engine = SimpleNamespace(
process=AsyncMock(
return_value=SimpleNamespace(
+ id="fused-1",
fusion_decision=FusionDecision.NEW_ALERT,
model_dump=lambda mode="json": {"id": "fused-1"},
)
diff --git a/services/fusion/tests/test_promoter.py b/services/fusion/tests/test_promoter.py
index 3fb7a08fd..8d86358f0 100644
--- a/services/fusion/tests/test_promoter.py
+++ b/services/fusion/tests/test_promoter.py
@@ -14,6 +14,7 @@ from app.models.alert import AlertSeverity
from app.services.promoter import promote_normalized_event, should_promote
TENANT = "00000000-0000-0000-0000-000000000001"
+CONNECTOR = "00000000-0000-0000-0000-0000000000c1"
def _message(ocsf_overrides: dict | None = None) -> dict:
@@ -44,7 +45,8 @@ def _message(ocsf_overrides: dict | None = None) -> dict:
ocsf.update(ocsf_overrides)
return {
"id": str(uuid.uuid4()),
- "connector_id": "conn-1",
+ "connector_id": CONNECTOR,
+ "connector_type": "crowdstrike_falcon",
"tenant_id": TENANT,
"ocsf_event": ocsf,
"normalization_version": "1.1.0",
@@ -87,6 +89,30 @@ class TestPromoteNormalizedEvent:
assert alert.mitre_tactics == ["Credential Access"]
assert alert.event_time is not None
assert alert.raw_event["class_uid"] == 2001
+ # Provenance carried first-class (issue #568)
+ assert str(alert.connector_id) == CONNECTOR
+ assert alert.connector_type == "crowdstrike_falcon"
+ assert alert.ocsf_class_uid == 2001
+
+ def test_provenance_falls_back_to_ocsf_when_envelope_sparse(self):
+ # No top-level connector_id/type — recover from the OCSF payload.
+ msg = _message()
+ del msg["connector_id"]
+ del msg["connector_type"]
+ msg["ocsf_event"]["source_connector_id"] = CONNECTOR
+ alert = promote_normalized_event(msg)
+ assert alert is not None
+ assert str(alert.connector_id) == CONNECTOR
+ # connector_type falls back to the OCSF product vendor+name label.
+ assert alert.connector_type == "CrowdStrike Falcon"
+
+ def test_canonical_id_is_deterministic_across_promotions(self):
+ # The stamped id happens in the worker, but the derivation is on the
+ # model and must be stable for identical content (issue #568).
+ a = promote_normalized_event(_message())
+ b = promote_normalized_event(_message())
+ assert a is not None and b is not None
+ assert a.deterministic_id() == b.deterministic_id()
def test_severity_ladder(self):
for sev_id, expected in [