Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Tool selection, evidence completeness and time to verdict.
Gap-closure wave 3. None of the three had a single hit anywhere in the
tree before this: `run_evals.py` graded outcomes, and an agent that
reaches the right verdict by reading the alert title and nothing else
scores identically to one that investigated.
"""
from __future__ import annotations
from app.eval.agent_quality import AgentRun, score_agent_quality
def _run(incident: str, **kw) -> AgentRun: # noqa: ANN003
return AgentRun(incident_id=incident, verdict=kw.pop("verdict", "true_positive"), **kw)
class TestToolSelection:
def test_calling_every_expected_tool_scores_full(self) -> None:
score = score_agent_quality(
[_run("a", tools_called=("search_siem", "get_host"), expected_tools=frozenset({"search_siem", "get_host"}))]
)
assert score.tool_selection.value == 1.0
def test_partial_credit_rather_than_all_or_nothing(self) -> None:
"""Three of four required tools is most of the way there, and
grading it the same as zero would hide real progress."""
score = score_agent_quality([_run("a", tools_called=("a", "b", "c"), expected_tools=frozenset({"a", "b", "c", "d"}))])
assert score.tool_selection.value == 0.75
def test_calling_nothing_scores_zero_not_unmeasured(self) -> None:
"""An agent that called no tools against an incident that needed
three genuinely scored zero. That is a measurement."""
score = score_agent_quality([_run("a", tools_called=(), expected_tools=frozenset({"a", "b", "c"}))])
assert score.tool_selection.measured
assert score.tool_selection.value == 0.0
def test_no_expected_tools_anywhere_is_unmeasured_not_zero(self) -> None:
"""The distinction the injection metrics had to learn: a corpus
with no answer key cannot grade, and rendering that as 0% says
the agent failed when nobody asked it anything."""
score = score_agent_quality([_run("a", tools_called=("x",))])
assert not score.tool_selection.measured
assert "expected tool" in (score.tool_selection.reason or "")
def test_extra_tools_are_counted_separately_not_penalised(self) -> None:
"""Exploring is legitimate. It is tracked because a rising count
beside flat completeness is an agent thrashing, which is worth
seeing — but it is not a scoring penalty."""
score = score_agent_quality([_run("a", tools_called=("a", "unrelated"), expected_tools=frozenset({"a"}))])
assert score.tool_selection.value == 1.0
assert score.unexpected_tool_runs == 1
class TestEvidenceCompleteness:
def test_citing_all_the_evidence_scores_full(self) -> None:
score = score_agent_quality([_run("a", evidence_cited=("e1", "e2"), expected_evidence=frozenset({"e1", "e2"}))])
assert score.evidence_completeness.value == 1.0
def test_missing_the_decisive_event_shows_up(self) -> None:
score = score_agent_quality([_run("a", evidence_cited=("e1",), expected_evidence=frozenset({"e1", "e2"}))])
assert score.evidence_completeness.value == 0.5
def test_it_is_independent_of_the_verdict(self) -> None:
"""The whole reason this metric exists: a correct verdict on
half the evidence is one unusual alert away from being wrong."""
score = score_agent_quality([_run("a", verdict="true_positive", evidence_cited=(), expected_evidence=frozenset({"e1"}))])
assert score.evidence_completeness.value == 0.0
class TestTimeToVerdict:
def test_median_and_p95_are_reported(self) -> None:
runs = [_run(f"i{n}", seconds_to_verdict=float(n)) for n in range(1, 21)]
score = score_agent_quality(runs)
assert score.timed_runs == 20
median = score.median_seconds_to_verdict
p95 = score.p95_seconds_to_verdict
assert median is not None and p95 is not None, "timings were recorded but no percentile came back"
assert p95 >= median
def test_untimed_runs_are_absent_not_instant(self) -> None:
score = score_agent_quality([_run("a")])
assert score.timed_runs == 0
assert score.median_seconds_to_verdict is None
class TestTheScoreCarriesItsDenominator:
def test_every_measure_reports_what_it_divided_by(self) -> None:
score = score_agent_quality([_run("a", tools_called=("x",), expected_tools=frozenset({"x", "y"}))])
payload = score.as_dict()
assert payload["tool_selection_accuracy"]["denominator"] == 2
assert payload["runs"] == 1
def test_an_empty_run_set_refuses_rather_than_scoring_zero(self) -> None:
score = score_agent_quality([])
assert not score.tool_selection.measured
assert not score.evidence_completeness.measured