Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

22.3 KB
Raw Blame
{
  "schema_version": 1,
  "description": "82-pair evaluation set for the deterministic NL → ES|QL translator (Milestone 5B). Each entry is a natural-language security question plus the structured intents we expect the parser to extract. We score on (a) syntactic validity of the emitted ES|QL (parsed by services/agents/app/nl_query/grammar.py) and (b) semantic match of the parsed intents against the gold IR. The 2026-06-28 expansion (Phase 4.6) added 32 cases covering: geo-by-country-name (`from iran`, `from china`, …), quoted-string LIKE filters against `message`, MIN/MAX numeric aggregations, multi-axis group-bys (host+user, ip+port, user+country), and longer-tail SOC hosts (`db01`, `app-prod-03`, `edge1.corp.local`).",
  "index_pattern_default": "logs-*",
  "time_range_hours_default": 24,
  "cases": [
    {
      "id": "auth-001",
      "question": "failed logins by user in the last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "auth-002",
      "question": "successful authentications by user last 6 hours",
      "expected": {
        "filters": [["event.outcome", "==", "success"], ["event.category", "==", "authentication"]],
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "auth-003",
      "question": "unique users with failed logons in the last hour",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "group_by": [],
        "aggregations": [["DISTINCT_COUNT", "user.name", "unique_user_name"]],
        "time_range_hours": 1
      }
    },
    {
      "id": "auth-004",
      "question": "count failed authentications by user and host last 12 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "group_by": ["user.name", "host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "auth-005",
      "question": "top 10 users by failed login count last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 10,
        "time_range_hours": 24
      }
    },
    {
      "id": "auth-006",
      "question": "denied logins from 10.0.0.5 in the last day",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"], ["source.ip", "==", "10.0.0.5"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "auth-007",
      "question": "ssh logins by user last week",
      "expected": {
        "filters": [["event.category", "==", "authentication"]],
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 168
      }
    },
    {
      "id": "auth-008",
      "question": "how many authentication events in the last 6 hours",
      "expected": {
        "filters": [["event.category", "==", "authentication"]],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "auth-009",
      "question": "top 5 hosts with the most failed logins last 12 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 5,
        "time_range_hours": 12
      }
    },
    {
      "id": "auth-010",
      "question": "successful logons by user account in the last 30 days",
      "expected": {
        "filters": [["event.outcome", "==", "success"], ["event.category", "==", "authentication"]],
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 720
      }
    },
    {
      "id": "net-001",
      "question": "denied dns queries by domain last 30 days",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "network"]],
        "group_by": ["destination.domain"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 720
      }
    },
    {
      "id": "net-002",
      "question": "average bytes per host last 6 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["AVG", "network.bytes", "avg_network_bytes"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "net-003",
      "question": "sum of bytes by source ip last 30 days",
      "expected": {
        "group_by": ["source.ip"],
        "aggregations": [["SUM", "network.bytes", "sum_network_bytes"]],
        "time_range_hours": 720
      }
    },
    {
      "id": "net-004",
      "question": "max bytes by destination ip in the last day",
      "expected": {
        "group_by": ["destination.ip"],
        "aggregations": [["MAX", "network.bytes", "max_network_bytes"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "net-005",
      "question": "blocked firewall events by source ip last 12 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "network"]],
        "group_by": ["source.ip"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "net-006",
      "question": "top 20 destination ips by event count last 6 hours",
      "expected": {
        "group_by": ["destination.ip"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 20,
        "time_range_hours": 6
      }
    },
    {
      "id": "net-007",
      "question": "unique source ips in the last hour",
      "expected": {
        "aggregations": [["DISTINCT_COUNT", "source.ip", "unique_source_ip"]],
        "time_range_hours": 1
      }
    },
    {
      "id": "net-008",
      "question": "count network events from 192.168.1.10 last 24 hours",
      "expected": {
        "filters": [["event.category", "==", "network"], ["source.ip", "==", "192.168.1.10"]],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "net-009",
      "question": "total bytes by destination port last 6 hours",
      "expected": {
        "group_by": ["destination.port"],
        "aggregations": [["SUM", "network.bytes", "sum_network_bytes"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "net-010",
      "question": "top 10 domains by query count last week",
      "expected": {
        "group_by": ["destination.domain"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 10,
        "time_range_hours": 168
      }
    },
    {
      "id": "proc-001",
      "question": "process executions by host last 6 hours",
      "expected": {
        "filters": [["event.category", "==", "process"]],
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "proc-002",
      "question": "top 10 processes by execution count last 12 hours",
      "expected": {
        "group_by": ["process.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 10,
        "time_range_hours": 12
      }
    },
    {
      "id": "proc-003",
      "question": "unique processes by host in the last day",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["DISTINCT_COUNT", "process.name", "unique_process_name"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "proc-004",
      "question": "count process events where process is powershell.exe last 6 hours",
      "expected": {
        "filters": [["event.category", "==", "process"], ["process.name", "==", "powershell.exe"]],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "proc-005",
      "question": "executions by user and process last 12 hours",
      "expected": {
        "group_by": ["user.name", "process.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "file-001",
      "question": "file events by host in the last day",
      "expected": {
        "filters": [["event.category", "==", "file"]],
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "file-002",
      "question": "filesystem events by user last 6 hours",
      "expected": {
        "filters": [["event.category", "==", "file"]],
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "file-003",
      "question": "top 10 hosts by file event count last 12 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 10,
        "time_range_hours": 12
      }
    },
    {
      "id": "host-001",
      "question": "events from host web01 in the last hour",
      "expected": {
        "filters": [["host.name", "==", "web01"]],
        "time_range_hours": 1
      }
    },
    {
      "id": "host-002",
      "question": "count events by host last 24 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "host-003",
      "question": "top 10 hosts by event count last 6 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 10,
        "time_range_hours": 6
      }
    },
    {
      "id": "host-004",
      "question": "events where hostname is db-prod-01 last 12 hours",
      "expected": {
        "filters": [["host.name", "==", "db-prod-01"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "host-005",
      "question": "show last 100 events from host web02",
      "expected": {
        "filters": [["host.name", "==", "web02"]],
        "limit": 100
      }
    },
    {
      "id": "user-001",
      "question": "events for user alice last 24 hours",
      "expected": {
        "filters": [["user.name", "==", "alice"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "user-002",
      "question": "count events by user in the last day",
      "expected": {
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "user-003",
      "question": "unique hosts by user last 6 hours",
      "expected": {
        "group_by": ["user.name"],
        "aggregations": [["DISTINCT_COUNT", "host.name", "unique_host_name"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "user-004",
      "question": "top 5 users by event count last 12 hours",
      "expected": {
        "group_by": ["user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 5,
        "time_range_hours": 12
      }
    },
    {
      "id": "user-005",
      "question": "events where account equals admin last 24 hours",
      "expected": {
        "filters": [["user.name", "==", "admin"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "outcome-001",
      "question": "failed events by category last 6 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"]],
        "group_by": ["event.category"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "outcome-002",
      "question": "successful events by host last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "success"]],
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "outcome-003",
      "question": "denied events by source ip last 12 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"]],
        "group_by": ["source.ip"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "outcome-004",
      "question": "rejected events from 172.16.0.5 in the last hour",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["source.ip", "==", "172.16.0.5"]],
        "time_range_hours": 1
      }
    },
    {
      "id": "limit-001",
      "question": "show last 50 events",
      "expected": {
        "limit": 50
      }
    },
    {
      "id": "limit-002",
      "question": "limit 25 events from web01",
      "expected": {
        "filters": [["host.name", "==", "web01"]],
        "limit": 25
      }
    },
    {
      "id": "limit-003",
      "question": "first 200 failed logins last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "limit": 200,
        "time_range_hours": 24
      }
    },
    {
      "id": "time-001",
      "question": "events in the last 15 minutes",
      "expected": {
        "time_range_hours": 1
      }
    },
    {
      "id": "time-002",
      "question": "failed logins in the past 2 weeks",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "time_range_hours": 336
      }
    },
    {
      "id": "time-003",
      "question": "events in the previous 3 days",
      "expected": {
        "time_range_hours": 72
      }
    },
    {
      "id": "time-004",
      "question": "events in the last month",
      "expected": {
        "time_range_hours": 720
      }
    },
    {
      "id": "geo-001",
      "question": "failed logins by country last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "authentication"]],
        "group_by": ["source.geo.country_iso_code"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "geo-002",
      "question": "attacks from iran in the last 24 hours",
      "expected": {
        "filters": [["source.geo.country_iso_code", "==", "IR"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "geo-003",
      "question": "traffic from china by host last 6 hours",
      "expected": {
        "filters": [["source.geo.country_iso_code", "==", "CN"]],
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "geo-004",
      "question": "logins from russia last 12 hours",
      "expected": {
        "filters": [["source.geo.country_iso_code", "==", "RU"], ["event.category", "==", "authentication"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "geo-005",
      "question": "sessions originating from north korea last week",
      "expected": {
        "filters": [["source.geo.country_iso_code", "==", "KP"]],
        "time_range_hours": 168
      }
    },
    {
      "id": "geo-006",
      "question": "connections from ukraine in the last hour",
      "expected": {
        "filters": [["source.geo.country_iso_code", "==", "UA"]],
        "time_range_hours": 1
      }
    },
    {
      "id": "geo-007",
      "question": "scans coming from brazil last 6 hours",
      "expected": {
        "filters": [["source.geo.country_iso_code", "==", "BR"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "msg-001",
      "question": "events containing \"powershell\" last 6 hours",
      "expected": {
        "filters": [["message", "LIKE", "powershell"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "msg-002",
      "question": "events containing \"invoke-mimikatz\" last 24 hours",
      "expected": {
        "filters": [["message", "LIKE", "invoke-mimikatz"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "msg-003",
      "question": "events containing \"net user /add\" last 12 hours",
      "expected": {
        "filters": [["message", "LIKE", "net user /add"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "num-001",
      "question": "min bytes by host last 6 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["MIN", "network.bytes", "min_network_bytes"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "num-002",
      "question": "average bytes by destination port last 24 hours",
      "expected": {
        "group_by": ["destination.port"],
        "aggregations": [["AVG", "network.bytes", "avg_network_bytes"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "num-003",
      "question": "max bytes per host last 12 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["MAX", "network.bytes", "max_network_bytes"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "num-004",
      "question": "sum of bytes by destination ip last week",
      "expected": {
        "group_by": ["destination.ip"],
        "aggregations": [["SUM", "network.bytes", "sum_network_bytes"]],
        "time_range_hours": 168
      }
    },
    {
      "id": "num-005",
      "question": "average bytes by source ip in the past day",
      "expected": {
        "group_by": ["source.ip"],
        "aggregations": [["AVG", "network.bytes", "avg_network_bytes"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "group-001",
      "question": "count events by host and user last 24 hours",
      "expected": {
        "group_by": ["host.name", "user.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "group-002",
      "question": "count network events by source ip and destination port last 6 hours",
      "expected": {
        "filters": [["event.category", "==", "network"]],
        "group_by": ["source.ip", "destination.port"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "group-003",
      "question": "count authentications by user and country last 12 hours",
      "expected": {
        "filters": [["event.category", "==", "authentication"]],
        "group_by": ["user.name", "source.geo.country_iso_code"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "host-006",
      "question": "events for host db01 last 6 hours",
      "expected": {
        "filters": [["host.name", "==", "db01"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "host-007",
      "question": "events on host app-prod-03 in the past hour",
      "expected": {
        "filters": [["host.name", "==", "app-prod-03"]],
        "time_range_hours": 1
      }
    },
    {
      "id": "host-008",
      "question": "events from edge1.corp.local in the last day",
      "expected": {
        "filters": [["host.name", "==", "edge1.corp.local"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "host-009",
      "question": "top 20 hosts by event count last 6 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 20,
        "time_range_hours": 6
      }
    },
    {
      "id": "user-006",
      "question": "events by user bob in the last day",
      "expected": {
        "filters": [["user.name", "==", "bob"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "user-007",
      "question": "events for user svc-deploy last 12 hours",
      "expected": {
        "filters": [["user.name", "==", "svc-deploy"]],
        "time_range_hours": 12
      }
    },
    {
      "id": "user-008",
      "question": "unique users by host last 24 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["DISTINCT_COUNT", "user.name", "unique_user_name"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "proc-006",
      "question": "process events on host web01 last 6 hours",
      "expected": {
        "filters": [["event.category", "==", "process"], ["host.name", "==", "web01"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "proc-007",
      "question": "top 5 processes by execution count last hour",
      "expected": {
        "group_by": ["process.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 5,
        "time_range_hours": 1
      }
    },
    {
      "id": "net-011",
      "question": "blocked dns queries by domain last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"], ["event.category", "==", "network"]],
        "group_by": ["destination.domain"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "net-012",
      "question": "top 10 source ips by event count last 12 hours",
      "expected": {
        "group_by": ["source.ip"],
        "aggregations": [["COUNT", null, "event_count"]],
        "limit": 10,
        "time_range_hours": 12
      }
    },
    {
      "id": "net-013",
      "question": "unique destination ips by host last 6 hours",
      "expected": {
        "group_by": ["host.name"],
        "aggregations": [["DISTINCT_COUNT", "destination.ip", "unique_destination_ip"]],
        "time_range_hours": 6
      }
    },
    {
      "id": "outcome-005",
      "question": "blocked events by host last 24 hours",
      "expected": {
        "filters": [["event.outcome", "==", "failure"]],
        "group_by": ["host.name"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 24
      }
    },
    {
      "id": "outcome-006",
      "question": "allowed events by source ip last 6 hours",
      "expected": {
        "filters": [["event.outcome", "==", "success"]],
        "group_by": ["source.ip"],
        "aggregations": [["COUNT", null, "event_count"]],
        "time_range_hours": 6
      }
    }
  ]
}