Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
{
"schema_version": 1,
"description": "82-pair evaluation set for the deterministic NL → ES|QL translator (Milestone 5B). Each entry is a natural-language security question plus the structured intents we expect the parser to extract. We score on (a) syntactic validity of the emitted ES|QL (parsed by services/agents/app/nl_query/grammar.py) and (b) semantic match of the parsed intents against the gold IR. The 2026-06-28 expansion (Phase 4.6) added 32 cases covering: geo-by-country-name (`from iran`, `from china`, …), quoted-string LIKE filters against `message`, MIN/MAX numeric aggregations, multi-axis group-bys (host+user, ip+port, user+country), and longer-tail SOC hosts (`db01`, `app-prod-03`, `edge1.corp.local`).",
"index_pattern_default": "logs-*",
"time_range_hours_default": 24,
"cases": [
{
"id": "auth-001",
"question": "failed logins by user in the last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "auth-002",
"question": "successful authentications by user last 6 hours",
"expected": {
"filters": [["event.outcome", "==", "success"], ["event.category", "==", "authentication"]],
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "auth-003",
"question": "unique users with failed logons in the last hour",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"group_by": [],
"aggregations": [["DISTINCT_COUNT", "user.name", "unique_user_name"]],
"time_range_hours": 1
}
},
{
"id": "auth-004",
"question": "count failed authentications by user and host last 12 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"group_by": ["user.name", "host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 12
}
},
{
"id": "auth-005",
"question": "top 10 users by failed login count last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 10,
"time_range_hours": 24
}
},
{
"id": "auth-006",
"question": "denied logins from 10.0.0.5 in the last day",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"], ["source.ip", "==", "10.0.0.5"]],
"time_range_hours": 24
}
},
{
"id": "auth-007",
"question": "ssh logins by user last week",
"expected": {
"filters": [["event.category", "==", "authentication"]],
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 168
}
},
{
"id": "auth-008",
"question": "how many authentication events in the last 6 hours",
"expected": {
"filters": [["event.category", "==", "authentication"]],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "auth-009",
"question": "top 5 hosts with the most failed logins last 12 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 5,
"time_range_hours": 12
}
},
{
"id": "auth-010",
"question": "successful logons by user account in the last 30 days",
"expected": {
"filters": [["event.outcome", "==", "success"], ["event.category", "==", "authentication"]],
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 720
}
},
{
"id": "net-001",
"question": "denied dns queries by domain last 30 days",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "network"]],
"group_by": ["destination.domain"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 720
}
},
{
"id": "net-002",
"question": "average bytes per host last 6 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["AVG", "network.bytes", "avg_network_bytes"]],
"time_range_hours": 6
}
},
{
"id": "net-003",
"question": "sum of bytes by source ip last 30 days",
"expected": {
"group_by": ["source.ip"],
"aggregations": [["SUM", "network.bytes", "sum_network_bytes"]],
"time_range_hours": 720
}
},
{
"id": "net-004",
"question": "max bytes by destination ip in the last day",
"expected": {
"group_by": ["destination.ip"],
"aggregations": [["MAX", "network.bytes", "max_network_bytes"]],
"time_range_hours": 24
}
},
{
"id": "net-005",
"question": "blocked firewall events by source ip last 12 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "network"]],
"group_by": ["source.ip"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 12
}
},
{
"id": "net-006",
"question": "top 20 destination ips by event count last 6 hours",
"expected": {
"group_by": ["destination.ip"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 20,
"time_range_hours": 6
}
},
{
"id": "net-007",
"question": "unique source ips in the last hour",
"expected": {
"aggregations": [["DISTINCT_COUNT", "source.ip", "unique_source_ip"]],
"time_range_hours": 1
}
},
{
"id": "net-008",
"question": "count network events from 192.168.1.10 last 24 hours",
"expected": {
"filters": [["event.category", "==", "network"], ["source.ip", "==", "192.168.1.10"]],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "net-009",
"question": "total bytes by destination port last 6 hours",
"expected": {
"group_by": ["destination.port"],
"aggregations": [["SUM", "network.bytes", "sum_network_bytes"]],
"time_range_hours": 6
}
},
{
"id": "net-010",
"question": "top 10 domains by query count last week",
"expected": {
"group_by": ["destination.domain"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 10,
"time_range_hours": 168
}
},
{
"id": "proc-001",
"question": "process executions by host last 6 hours",
"expected": {
"filters": [["event.category", "==", "process"]],
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "proc-002",
"question": "top 10 processes by execution count last 12 hours",
"expected": {
"group_by": ["process.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 10,
"time_range_hours": 12
}
},
{
"id": "proc-003",
"question": "unique processes by host in the last day",
"expected": {
"group_by": ["host.name"],
"aggregations": [["DISTINCT_COUNT", "process.name", "unique_process_name"]],
"time_range_hours": 24
}
},
{
"id": "proc-004",
"question": "count process events where process is powershell.exe last 6 hours",
"expected": {
"filters": [["event.category", "==", "process"], ["process.name", "==", "powershell.exe"]],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "proc-005",
"question": "executions by user and process last 12 hours",
"expected": {
"group_by": ["user.name", "process.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 12
}
},
{
"id": "file-001",
"question": "file events by host in the last day",
"expected": {
"filters": [["event.category", "==", "file"]],
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "file-002",
"question": "filesystem events by user last 6 hours",
"expected": {
"filters": [["event.category", "==", "file"]],
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "file-003",
"question": "top 10 hosts by file event count last 12 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 10,
"time_range_hours": 12
}
},
{
"id": "host-001",
"question": "events from host web01 in the last hour",
"expected": {
"filters": [["host.name", "==", "web01"]],
"time_range_hours": 1
}
},
{
"id": "host-002",
"question": "count events by host last 24 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "host-003",
"question": "top 10 hosts by event count last 6 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 10,
"time_range_hours": 6
}
},
{
"id": "host-004",
"question": "events where hostname is db-prod-01 last 12 hours",
"expected": {
"filters": [["host.name", "==", "db-prod-01"]],
"time_range_hours": 12
}
},
{
"id": "host-005",
"question": "show last 100 events from host web02",
"expected": {
"filters": [["host.name", "==", "web02"]],
"limit": 100
}
},
{
"id": "user-001",
"question": "events for user alice last 24 hours",
"expected": {
"filters": [["user.name", "==", "alice"]],
"time_range_hours": 24
}
},
{
"id": "user-002",
"question": "count events by user in the last day",
"expected": {
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "user-003",
"question": "unique hosts by user last 6 hours",
"expected": {
"group_by": ["user.name"],
"aggregations": [["DISTINCT_COUNT", "host.name", "unique_host_name"]],
"time_range_hours": 6
}
},
{
"id": "user-004",
"question": "top 5 users by event count last 12 hours",
"expected": {
"group_by": ["user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 5,
"time_range_hours": 12
}
},
{
"id": "user-005",
"question": "events where account equals admin last 24 hours",
"expected": {
"filters": [["user.name", "==", "admin"]],
"time_range_hours": 24
}
},
{
"id": "outcome-001",
"question": "failed events by category last 6 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"]],
"group_by": ["event.category"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "outcome-002",
"question": "successful events by host last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "success"]],
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "outcome-003",
"question": "denied events by source ip last 12 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"]],
"group_by": ["source.ip"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 12
}
},
{
"id": "outcome-004",
"question": "rejected events from 172.16.0.5 in the last hour",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["source.ip", "==", "172.16.0.5"]],
"time_range_hours": 1
}
},
{
"id": "limit-001",
"question": "show last 50 events",
"expected": {
"limit": 50
}
},
{
"id": "limit-002",
"question": "limit 25 events from web01",
"expected": {
"filters": [["host.name", "==", "web01"]],
"limit": 25
}
},
{
"id": "limit-003",
"question": "first 200 failed logins last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"limit": 200,
"time_range_hours": 24
}
},
{
"id": "time-001",
"question": "events in the last 15 minutes",
"expected": {
"time_range_hours": 1
}
},
{
"id": "time-002",
"question": "failed logins in the past 2 weeks",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"time_range_hours": 336
}
},
{
"id": "time-003",
"question": "events in the previous 3 days",
"expected": {
"time_range_hours": 72
}
},
{
"id": "time-004",
"question": "events in the last month",
"expected": {
"time_range_hours": 720
}
},
{
"id": "geo-001",
"question": "failed logins by country last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
"group_by": ["source.geo.country_iso_code"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "geo-002",
"question": "attacks from iran in the last 24 hours",
"expected": {
"filters": [["source.geo.country_iso_code", "==", "IR"]],
"time_range_hours": 24
}
},
{
"id": "geo-003",
"question": "traffic from china by host last 6 hours",
"expected": {
"filters": [["source.geo.country_iso_code", "==", "CN"]],
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "geo-004",
"question": "logins from russia last 12 hours",
"expected": {
"filters": [["source.geo.country_iso_code", "==", "RU"], ["event.category", "==", "authentication"]],
"time_range_hours": 12
}
},
{
"id": "geo-005",
"question": "sessions originating from north korea last week",
"expected": {
"filters": [["source.geo.country_iso_code", "==", "KP"]],
"time_range_hours": 168
}
},
{
"id": "geo-006",
"question": "connections from ukraine in the last hour",
"expected": {
"filters": [["source.geo.country_iso_code", "==", "UA"]],
"time_range_hours": 1
}
},
{
"id": "geo-007",
"question": "scans coming from brazil last 6 hours",
"expected": {
"filters": [["source.geo.country_iso_code", "==", "BR"]],
"time_range_hours": 6
}
},
{
"id": "msg-001",
"question": "events containing \"powershell\" last 6 hours",
"expected": {
"filters": [["message", "LIKE", "powershell"]],
"time_range_hours": 6
}
},
{
"id": "msg-002",
"question": "events containing \"invoke-mimikatz\" last 24 hours",
"expected": {
"filters": [["message", "LIKE", "invoke-mimikatz"]],
"time_range_hours": 24
}
},
{
"id": "msg-003",
"question": "events containing \"net user /add\" last 12 hours",
"expected": {
"filters": [["message", "LIKE", "net user /add"]],
"time_range_hours": 12
}
},
{
"id": "num-001",
"question": "min bytes by host last 6 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["MIN", "network.bytes", "min_network_bytes"]],
"time_range_hours": 6
}
},
{
"id": "num-002",
"question": "average bytes by destination port last 24 hours",
"expected": {
"group_by": ["destination.port"],
"aggregations": [["AVG", "network.bytes", "avg_network_bytes"]],
"time_range_hours": 24
}
},
{
"id": "num-003",
"question": "max bytes per host last 12 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["MAX", "network.bytes", "max_network_bytes"]],
"time_range_hours": 12
}
},
{
"id": "num-004",
"question": "sum of bytes by destination ip last week",
"expected": {
"group_by": ["destination.ip"],
"aggregations": [["SUM", "network.bytes", "sum_network_bytes"]],
"time_range_hours": 168
}
},
{
"id": "num-005",
"question": "average bytes by source ip in the past day",
"expected": {
"group_by": ["source.ip"],
"aggregations": [["AVG", "network.bytes", "avg_network_bytes"]],
"time_range_hours": 24
}
},
{
"id": "group-001",
"question": "count events by host and user last 24 hours",
"expected": {
"group_by": ["host.name", "user.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "group-002",
"question": "count network events by source ip and destination port last 6 hours",
"expected": {
"filters": [["event.category", "==", "network"]],
"group_by": ["source.ip", "destination.port"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
},
{
"id": "group-003",
"question": "count authentications by user and country last 12 hours",
"expected": {
"filters": [["event.category", "==", "authentication"]],
"group_by": ["user.name", "source.geo.country_iso_code"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 12
}
},
{
"id": "host-006",
"question": "events for host db01 last 6 hours",
"expected": {
"filters": [["host.name", "==", "db01"]],
"time_range_hours": 6
}
},
{
"id": "host-007",
"question": "events on host app-prod-03 in the past hour",
"expected": {
"filters": [["host.name", "==", "app-prod-03"]],
"time_range_hours": 1
}
},
{
"id": "host-008",
"question": "events from edge1.corp.local in the last day",
"expected": {
"filters": [["host.name", "==", "edge1.corp.local"]],
"time_range_hours": 24
}
},
{
"id": "host-009",
"question": "top 20 hosts by event count last 6 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 20,
"time_range_hours": 6
}
},
{
"id": "user-006",
"question": "events by user bob in the last day",
"expected": {
"filters": [["user.name", "==", "bob"]],
"time_range_hours": 24
}
},
{
"id": "user-007",
"question": "events for user svc-deploy last 12 hours",
"expected": {
"filters": [["user.name", "==", "svc-deploy"]],
"time_range_hours": 12
}
},
{
"id": "user-008",
"question": "unique users by host last 24 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["DISTINCT_COUNT", "user.name", "unique_user_name"]],
"time_range_hours": 24
}
},
{
"id": "proc-006",
"question": "process events on host web01 last 6 hours",
"expected": {
"filters": [["event.category", "==", "process"], ["host.name", "==", "web01"]],
"time_range_hours": 6
}
},
{
"id": "proc-007",
"question": "top 5 processes by execution count last hour",
"expected": {
"group_by": ["process.name"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 5,
"time_range_hours": 1
}
},
{
"id": "net-011",
"question": "blocked dns queries by domain last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"], ["event.category", "==", "network"]],
"group_by": ["destination.domain"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "net-012",
"question": "top 10 source ips by event count last 12 hours",
"expected": {
"group_by": ["source.ip"],
"aggregations": [["COUNT", null, "event_count"]],
"limit": 10,
"time_range_hours": 12
}
},
{
"id": "net-013",
"question": "unique destination ips by host last 6 hours",
"expected": {
"group_by": ["host.name"],
"aggregations": [["DISTINCT_COUNT", "destination.ip", "unique_destination_ip"]],
"time_range_hours": 6
}
},
{
"id": "outcome-005",
"question": "blocked events by host last 24 hours",
"expected": {
"filters": [["event.outcome", "==", "failure"]],
"group_by": ["host.name"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 24
}
},
{
"id": "outcome-006",
"question": "allowed events by source ip last 6 hours",
"expected": {
"filters": [["event.outcome", "==", "success"]],
"group_by": ["source.ip"],
"aggregations": [["COUNT", null, "event_count"]],
"time_range_hours": 6
}
}
]
}