Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Wave 6 — groundedness / hallucination eval axis."""
from __future__ import annotations
from app.confidence.groundedness import score_groundedness
def test_fully_grounded_output_scores_one():
evidence = "Beacon from 45.9.148.99 to c2.evil.example matching T1071. CVE-2024-1234 present."
output = "Malicious beacon: 45.9.148.99 contacting c2.evil.example (T1071); exploited CVE-2024-1234."
r = score_groundedness(output, evidence)
assert r.score == 1.0
assert r.has_hallucination is False
def test_hallucinated_indicator_is_flagged():
evidence = "Beacon from 45.9.148.99 observed."
# 203.0.113.7 is NOT in the evidence -> hallucination.
output = "Attacker used 45.9.148.99 and pivoted to 203.0.113.7."
r = score_groundedness(output, evidence)
assert r.score == 0.5
assert "203.0.113.7" in r.hallucinated
assert "45.9.148.99" in r.grounded
assert r.has_hallucination is True
def test_output_with_no_indicators_is_grounded():
r = score_groundedness("The activity appears benign and expected.", "some evidence")
assert r.score == 1.0
assert r.has_hallucination is False
def test_fabricated_cve_and_hash_flagged():
evidence = "Suspicious process launch on host web-01."
output = "Linked to CVE-2021-44228 and hash " + "a" * 64
r = score_groundedness(output, evidence)
assert r.score == 0.0
assert any("cve-2021-44228" == h for h in r.hallucinated)