Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(agents): fail closed when LLM triage errors on a severe alert (#1249)

antigravitysoham-eng156d24ca056c
diff --git a/services/agents/app/workers/fused_alert_consumer.py b/services/agents/app/workers/fused_alert_consumer.py
index 21b66d999..b9a0af14b 100644
--- a/services/agents/app/workers/fused_alert_consumer.py
+++ b/services/agents/app/workers/fused_alert_consumer.py
@@ -102,6 +102,7 @@ _METRICS = {
"outcome_written": 0,
"outcome_suppressed": 0,
"ungrounded_demoted": 0,
+ "llm_failed_held_for_review": 0,
"persist_retries": 0,
"dead_lettered": 0,
"approvals_raised": 0,
@@ -287,6 +288,36 @@ def build_state(message: dict[str, Any]) -> InvestigationState | None:
)
+# Severities a failed model call may not close. The deterministic scorer is a
+# fallback for when the model is unreachable, and on a laptop CPU or a gateway
+# with the wrong base URL that is every call. Observed: a critical
+# "ransomware canary files encrypted" alert, vssadmin deleting shadow copies,
+# came back `likely_benign` from the fallback after the LLM timed out.
+_FAIL_CLOSED_SEVERITIES = frozenset({"critical", "high"})
+
+
+def _hold_severe_for_review_after_llm_failure(state: InvestigationState) -> InvestigationState:
+ """Fail closed: an LLM error is never the reason a severe alert closes.
+
+ Only auto-closeable verdicts on high or critical alerts change, and only on
+ the path where the model was asked and failed. A deployment that runs
+ deterministic triage on purpose keeps today's behaviour.
+ """
+ severity = str((state.raw_alert or {}).get("severity") or "").strip().lower()
+ if severity not in _FAIL_CLOSED_SEVERITIES:
+ return state
+ verdict = state.verdict
+ if normalize_disposition(str(verdict or ""), default=NEEDS_REVIEW) not in AUTO_CLOSEABLE_DISPOSITIONS:
+ return state
+ state.add_finding(
+ f"LLM triage failed, so the deterministic verdict '{verdict}' on a {severity}-severity "
+ "alert is held for review instead of closing it."
+ )
+ state.verdict = NEEDS_REVIEW
+ _METRICS["llm_failed_held_for_review"] += 1
+ return state
+
+
class FusedAlertTriageWorker:
"""Consumes ``aisoc.alerts.fused`` and auto-triages each alert (copilot)."""
@@ -1143,7 +1174,7 @@ class FusedAlertTriageWorker:
logger.warning("auto_triage_worker.llm_failed_fallback", error=str(exc))
state = await run_triage(state)
_METRICS["deterministic"] += 1
- return state, "deterministic"
+ return _hold_severe_for_review_after_llm_failure(state), "deterministic"
def _apply_groundedness_gate(
self,
diff --git a/services/agents/tests/test_alert_triggered_playbooks.py b/services/agents/tests/test_alert_triggered_playbooks.py
index 4a96dd464..7278631a2 100644
--- a/services/agents/tests/test_alert_triggered_playbooks.py
+++ b/services/agents/tests/test_alert_triggered_playbooks.py
@@ -209,14 +209,28 @@ class TestTheWiring:
assert "alert_trigger.run_for_alert(" in source, "no playbook can start from an alert, which is the defect 5.1 exists to fix"
def test_it_runs_after_triage_not_before(self) -> None:
- """A playbook's conditions read the verdict and the confidence."""
+ """A playbook's conditions read the verdict and the confidence.
+
+ Located inside `triage` by AST rather than by `str.index` over the
+ whole module. The substring form pinned whichever occurrence came
+ first in the file, so adding *any* helper above `triage` that also
+ reads `state.verdict` moved the anchor into the helper and failed a
+ correctly-ordered worker — which is what a fail-closed LLM-error
+ guard did. The property under test is the order of these statements
+ within the function that runs them, so that is what this reads.
+ """
+ import ast
import pathlib
- source = pathlib.Path(__file__).resolve().parents[1].joinpath("app/workers/fused_alert_consumer.py").read_text()
- trigger_at = source.index("alert_trigger.run_for_alert(")
- verdict_at = source.index("verdict = state.verdict")
- assert trigger_at < verdict_at
- assert source.index("tokens = tracker.total_tokens") < trigger_at
+ path = pathlib.Path(__file__).resolve().parents[1].joinpath("app/workers/fused_alert_consumer.py")
+ tree = ast.parse(path.read_text())
+ triage = next(n for n in ast.walk(tree) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "triage")
+ body = "\n".join(path.read_text().splitlines()[triage.lineno - 1 : triage.end_lineno])
+
+ trigger_at = body.index("alert_trigger.run_for_alert(")
+ verdict_at = body.index("verdict = state.verdict")
+ assert trigger_at < verdict_at, "the playbook trigger must run before the verdict is read back"
+ assert body.index("tokens = tracker.total_tokens") < trigger_at
class TestTheTriggerEventMatchesTheCorpus:
diff --git a/services/agents/tests/test_fused_alert_worker.py b/services/agents/tests/test_fused_alert_worker.py
index f8b1e7638..61f74db5e 100644
--- a/services/agents/tests/test_fused_alert_worker.py
+++ b/services/agents/tests/test_fused_alert_worker.py
@@ -321,3 +321,41 @@ class _FakeGovernor:
def check(self, tenant_id, fingerprint): # noqa: ANN001, ARG002
return self._decision
+
+
+async def _llm_raises_and_fallback_says(monkeypatch, verdict: str):
+ monkeypatch.setattr(worker_mod, "resolve_llm_config", _fake_llm_config(allowed=True))
+ monkeypatch.setenv("AISOC_DETERMINISTIC", "0")
+
+ async def _raise(_state):
+ raise RuntimeError("litellm.Timeout: Connection timed out. Timeout passed=60.0")
+
+ async def _fallback(state):
+ state.verdict = verdict
+ return state
+
+ monkeypatch.setattr(worker_mod, "run_auto_triage", _raise)
+ monkeypatch.setattr(worker_mod, "run_triage", _fallback)
+
+
[email protected]("severity", ["critical", "high", "CRITICAL"])
+async def test_llm_failure_never_closes_a_severe_alert(monkeypatch, severity):
+ # A failed model call is not evidence. The fallback said likely_benign on a
+ # critical ransomware alert once; it must be held for review instead.
+ await _llm_raises_and_fallback_says(monkeypatch, "likely_benign")
+ result = await _worker().triage(_fused(severity=severity))
+ assert result["tier"] == "deterministic"
+ assert result["verdict"] == "needs_review"
+
+
[email protected]("severity", ["low", "medium", "info"])
+async def test_llm_failure_on_a_minor_alert_keeps_the_fallback_verdict(monkeypatch, severity):
+ await _llm_raises_and_fallback_says(monkeypatch, "benign")
+ result = await _worker().triage(_fused(severity=severity))
+ assert result["verdict"] == "benign"
+
+
+async def test_llm_failure_does_not_downgrade_a_true_positive(monkeypatch):
+ await _llm_raises_and_fallback_says(monkeypatch, "true_positive")
+ result = await _worker().triage(_fused(severity="critical"))
+ assert result["verdict"] == "true_positive"