Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

test(ai-estate): recorder classification, coalescing, mirrors and the graph projection

Beenu Arorad2bd682f90cb
diff --git a/services/fusion/tests/test_ai_asset_recorder.py b/services/fusion/tests/test_ai_asset_recorder.py
new file mode 100644
index 000000000..c6bacffa2
--- /dev/null
+++ b/services/fusion/tests/test_ai_asset_recorder.py
@@ -0,0 +1,377 @@
+"""The AI-asset recorder: classification, coalescing, and the mirror.
+
+Depth plan 6.1.
+
+Every payload below is the shape ``services/ingest`` publishes — a
+``NormalizedEvent`` envelope with ``ocsf_event`` inside — built from the field
+maps in ``services/ingest/internal/normalizer/templates/ai-runtime.yaml`` and
+``ai-finding.yaml`` rather than from the recorder's own expectations. A fixture
+written from the thing under test proves only that it agrees with itself, and
+the recurring defect here is a test double more capable than the real source.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from typing import Any
+
+import pytest
+from app.services.ai_asset_recorder import (
+ ALL_KINDS,
+ CODING_AGENT_PROCESSES,
+ AiAssetRecorder,
+ _Sighting,
+ _score,
+ classify,
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+
+# The conftest's collection hook adds the asyncio marker too late for
+# pytest-asyncio 1.x to see it, so the async cases below carry it explicitly.
+# Explicit rather than the module-scope `pytestmark` the sibling suites use,
+# because this file mixes sync and async cases and a module mark warns on
+# every sync one.
+
+TENANT = "11111111-1111-1111-1111-111111111111"
+
+
+def runtime_event(**overrides: Any) -> dict[str, Any]:
+ """An `ai-runtime` event as the normalizer emits it.
+
+ Field placement follows `ai-runtime.yaml`: `agent_id` lands on
+ `actor.process.name`, `on_behalf_of` on `actor.user.name`, `model` on
+ `metadata.product.feature.name`, `server_name` on `resource.name`.
+ """
+ ocsf: dict[str, Any] = {
+ "class_uid": 6003,
+ "class_name": "API Activity",
+ "source_type": "ai-runtime",
+ "activity_name": "search_siem",
+ "severity_id": 1,
+ "actor": {"process": {"name": "agent-7f2", "path": "/usr/local/bin/claude"}, "user": {"name": "[email protected]"}},
+ "src_endpoint": {"hostname": "ada-mbp", "ip": "198.51.100.14"},
+ "metadata": {"product": {"vendor_name": "AiSOC", "name": "AI Runtime", "feature": {"name": "claude-sonnet-4"}}},
+ "cloud": {"provider": "anthropic"},
+ "resource": {"name": "github-mcp"},
+ "finding": {"evidence": {"prompt_tokens": 1200, "completion_tokens": 300}},
+ }
+ ocsf.update(overrides)
+ return {"tenant_id": TENANT, "ocsf_event": ocsf}
+
+
+def process_event(name: str, cmd_line: str = "", hostname: str = "ada-mbp") -> dict[str, Any]:
+ """An OCSF Process Activity event, the class EDR telemetry normalizes to."""
+ return {
+ "tenant_id": TENANT,
+ "ocsf_event": {
+ "class_uid": 1007,
+ "class_name": "Process Activity",
+ "source_type": "crowdstrike_falcon",
+ "process": {"name": name, "cmd_line": cmd_line},
+ "device": {"name": hostname},
+ "actor": {"user": {"name": "[email protected]"}},
+ },
+ }
+
+
+# ---------------------------------------------------------------------------
+# Classification
+# ---------------------------------------------------------------------------
+
+
+def test_runtime_event_yields_a_coding_agent_linked_to_person_and_device():
+ sightings = classify(runtime_event())
+ agents = [s for s in sightings if s.kind == "coding_agent"]
+ assert len(agents) == 1
+ agent = agents[0]
+ assert agent.external_id == "agent-7f2"
+ assert agent.vendor == "Anthropic"
+ assert agent.principal == "[email protected]"
+ assert agent.device == "ada-mbp"
+ assert agent.attributes["models"] == ["claude-sonnet-4"]
+ assert agent.attributes["tools"] == ["search_siem"]
+
+
+def test_runtime_event_also_yields_the_mcp_server_it_called():
+ kinds = {s.kind for s in classify(runtime_event())}
+ assert "mcp_server" in kinds
+
+
+def test_an_unrecognised_agent_binary_is_an_ai_app_not_a_coding_agent():
+ """The honest answer, not the flattering one.
+
+ An agent we cannot place is an `ai_app`. Calling it a coding agent would
+ put a vendor name on a row we have no evidence for.
+ """
+ event = runtime_event(
+ actor={"process": {"name": "svc-42", "path": "/opt/inhouse/summariser"}, "user": {"name": "[email protected]"}}
+ )
+ agents = [s for s in classify(event) if s.kind in ("coding_agent", "ai_app")]
+ assert [s.kind for s in agents] == ["ai_app"]
+ assert agents[0].vendor == "anthropic"
+
+
+def test_process_activity_finds_a_coding_agent_with_a_device_and_no_model():
+ """Endpoint discovery sees what runtime telemetry cannot, and vice versa.
+
+ This is why `discovery_source` is stored: an analyst reading a row with no
+ model needs to know whether it is missing or unknowable.
+ """
+ sightings = classify(process_event("claude", "claude --resume"))
+ assert len(sightings) == 1
+ assert sightings[0].kind == "coding_agent"
+ assert sightings[0].discovery_source == "endpoint"
+ assert sightings[0].device == "ada-mbp"
+ assert "models" not in sightings[0].attributes
+
+
[email protected]("flag", ["--dangerously-skip-permissions", "--yolo", "--auto-approve", "--no-confirm"])
+def test_permission_prompts_disabled_is_read_from_the_command_line(flag: str):
+ sightings = classify(process_event("claude", f"claude {flag} -p 'refactor'"))
+ assert sightings[0].attributes["permission_prompts_disabled"] is True
+
+
+def test_an_ordinary_process_is_not_an_ai_asset():
+ """The negative control for the match table.
+
+ A substring heuristic on "agent" would put every EDR sensor on the
+ inventory, so the table is explicit and this asserts it stays that way.
+ """
+ assert classify(process_event("falcon-sensor")) == []
+ assert classify(process_event("chrome")) == []
+
+
+def test_a_process_name_that_is_not_class_1007_is_ignored():
+ """A file event naming a binary is not evidence the binary ran."""
+ event = process_event("claude")
+ event["ocsf_event"]["class_uid"] = 1001
+ assert classify(event) == []
+
+
+def test_provider_audit_yields_a_key_with_no_device():
+ event = {
+ "tenant_id": TENANT,
+ "ocsf_event": {
+ "class_uid": 3001,
+ "source_type": "llm_usage",
+ "activity_name": "openai.api_key.created",
+ "actor": {"user": {"name": "[email protected]"}},
+ "metadata": {"product": {"vendor_name": "OpenAI", "name": "AI / LLM Usage Audit"}},
+ "finding": {"uid": "key_abc123"},
+ },
+ }
+ sightings = classify(event)
+ assert [s.kind for s in sightings] == ["provider_key"]
+ assert sightings[0].device is None
+ assert sightings[0].discovery_source == "provider_audit"
+
+
+def test_oauth_consent_yields_a_grant_with_its_scopes():
+ event = {
+ "tenant_id": TENANT,
+ "ocsf_event": {
+ "class_uid": 3001,
+ "source_type": "llm_usage",
+ "activity_name": "oauth.consent.granted",
+ "actor": {"user": {"name": "[email protected]"}},
+ "metadata": {"product": {"vendor_name": "Google Workspace"}},
+ "finding": {"uid": "notetaker.example", "evidence": {"scopes": ["https://mail.google.com/"]}},
+ },
+ }
+ sightings = classify(event)
+ assert [s.kind for s in sightings] == ["oauth_grant"]
+ assert sightings[0].attributes["scopes"] == ["https://mail.google.com/"]
+
+
+def test_an_event_without_a_tenant_is_dropped_rather_than_guessed():
+ event = runtime_event()
+ event.pop("tenant_id")
+ event["ocsf_event"].pop("tenant_uid", None)
+ assert classify(event) == []
+
+
+# ---------------------------------------------------------------------------
+# Coalescing and the write
+# ---------------------------------------------------------------------------
+
+
+class _FakeConn:
+ def __init__(self, sink: list[tuple]) -> None:
+ self._sink = sink
+
+ async def execute(self, _sql: str, *args: Any) -> None:
+ self._sink.append(args)
+
+
+class _FakePool:
+ """A pool double that records the arguments it was asked to write.
+
+ Deliberately records rather than answers: a double that returns a row for
+ any query is how a table nobody writes passes its tests.
+ """
+
+ def __init__(self) -> None:
+ self.writes: list[tuple] = []
+
+ def acquire(self): # noqa: ANN201
+ sink = self.writes
+
+ class _Ctx:
+ async def __aenter__(self): # noqa: ANN202
+ return _FakeConn(sink)
+
+ async def __aexit__(self, *_exc: Any) -> None:
+ return None
+
+ return _Ctx()
+
+
+async def test_chatty_agent_coalesces_into_one_row_per_flush():
+ """Thirty tool calls must be one upsert, not thirty.
+
+ Without this the recorder puts a database write on the hot path of every
+ AI event in the deployment, to maintain a `last_seen_at` nobody reads at
+ that resolution.
+ """
+ pool = _FakePool()
+ recorder = AiAssetRecorder(lambda: pool)
+ for i in range(30):
+ await recorder.observe(runtime_event(activity_name=f"tool_{i % 3}"))
+ written = await recorder.flush()
+
+ # One coding agent + one MCP server.
+ assert written == 2
+ assert len(pool.writes) == 2
+ counts = {row[1]: row[13] for row in pool.writes}
+ assert counts["coding_agent"] == 30
+ assert counts["mcp_server"] == 30
+
+
+async def test_coalescing_unions_tools_rather_than_overwriting_them():
+ pool = _FakePool()
+ recorder = AiAssetRecorder(lambda: pool)
+ await recorder.observe(runtime_event(activity_name="search_siem"))
+ await recorder.observe(runtime_event(activity_name="execute_command"))
+ await recorder.flush()
+ agent = next(row for row in pool.writes if row[1] == "coding_agent")
+ assert '"execute_command"' in agent[8] and '"search_siem"' in agent[8]
+
+
+async def test_a_later_sighting_without_a_person_does_not_erase_one_with():
+ """Merging keeps the link. The SQL keeps it too, via COALESCE."""
+ pool = _FakePool()
+ recorder = AiAssetRecorder(lambda: pool)
+ await recorder.observe(runtime_event())
+ anonymous = runtime_event(actor={"process": {"name": "agent-7f2", "path": "/usr/local/bin/claude"}})
+ await recorder.observe(anonymous)
+ await recorder.flush()
+ agent = next(row for row in pool.writes if row[1] == "coding_agent")
+ assert agent[6] == "[email protected]"
+
+
+async def test_no_pool_drops_and_counts_rather_than_raising():
+ """An inventory must never wedge alert promotion.
+
+ The drop is counted and logged so a quiet inventory is diagnosable rather
+ than mysterious — an empty page with no signal is the failure mode this
+ avoids.
+ """
+ recorder = AiAssetRecorder(lambda: None)
+ await recorder.observe(runtime_event())
+ assert await recorder.flush() == 0
+ assert recorder.metrics["dropped"] == 2
+
+
+async def test_a_failing_pool_is_survived():
+ class _Exploding:
+ def acquire(self): # noqa: ANN201
+ raise RuntimeError("connection refused")
+
+ recorder = AiAssetRecorder(lambda: _Exploding())
+ await recorder.observe(runtime_event())
+ assert await recorder.flush() == 0
+ assert recorder.metrics["dropped"] == 2
+
+
+# ---------------------------------------------------------------------------
+# The mirror
+# ---------------------------------------------------------------------------
+
+
+def test_recorder_kinds_match_the_api_models_vocabulary():
+ """Two services, one meaning.
+
+ `services/fusion` and `services/api` both package their code as a
+ top-level `app` and cannot import each other, so the kind vocabulary is
+ mirrored. A mirror nothing compares is just two copies that will drift,
+ which is why this reads the other file rather than trusting the comment
+ that says they match.
+ """
+ model = REPO_ROOT / "services/api/app/models/ai_asset.py"
+ text = model.read_text(encoding="utf-8")
+ for kind in ALL_KINDS:
+ assert f'"{kind}"' in text, f"{kind} is in the recorder and not in the API model"
+
+
+def test_risk_scoring_agrees_with_the_api_service_on_shared_cases():
+ """The two risk implementations must return the same number.
+
+ The console reads the stored score; the sanction route recomputes it.
+ If they disagree, approving an asset changes its risk for a reason that
+ is not the approval.
+ """
+ api_module = REPO_ROOT / "services/api/app/services/ai_inventory.py"
+ spec = importlib.util.spec_from_file_location("_ai_inventory_mirror", api_module)
+ assert spec and spec.loader
+ mirror = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mirror)
+
+ cases = [
+ {"kind": "coding_agent", "principal": None, "device": None, "attributes": {}},
+ {"kind": "coding_agent", "principal": "[email protected]", "device": "mbp", "attributes": {"tools": ["execute_command"]}},
+ {"kind": "provider_key", "principal": "[email protected]", "device": None, "attributes": {}},
+ {
+ "kind": "oauth_grant",
+ "principal": "[email protected]",
+ "device": None,
+ "attributes": {"scopes": ["https://mail.google.com/"]},
+ },
+ {
+ "kind": "coding_agent",
+ "principal": "[email protected]",
+ "device": "mbp",
+ "attributes": {"permission_prompts_disabled": True, "prompt_secret_kinds": ["aws_access_key"]},
+ },
+ ]
+ for case in cases:
+ sighting = _Sighting(
+ tenant_id=TENANT,
+ kind=case["kind"],
+ external_id="x",
+ device=case["device"],
+ principal=case["principal"],
+ attributes=dict(case["attributes"]),
+ )
+ fusion_score, fusion_reasons = _score(sighting)
+ api_score, api_reasons = mirror.score_risk(
+ kind=case["kind"],
+ principal=case["principal"],
+ device=case["device"],
+ sanctioned=None,
+ attributes=case["attributes"],
+ effective_permissions=None,
+ )
+ assert fusion_score == api_score, case
+ assert sorted(fusion_reasons) == sorted(api_reasons), case
+
+
+def test_every_coding_agent_vendor_is_named():
+ """A vendor of "" would render as a blank column that looks like data."""
+ assert all(vendor for vendor in CODING_AGENT_PROCESSES.values())
diff --git a/services/ingest/internal/graph/ai_estate_test.go b/services/ingest/internal/graph/ai_estate_test.go
new file mode 100644
index 000000000..46767ac32
--- /dev/null
+++ b/services/ingest/internal/graph/ai_estate_test.go
@@ -0,0 +1,192 @@
+package graph
+
+import (
+ "os"
+ "strings"
+ "testing"
+)
+
+// The AI estate projection (depth plan 6.1).
+//
+// Every payload here is built from the field map in
+// internal/normalizer/templates/ai-runtime.yaml rather than from the
+// extractor's expectations: agent_id lands on actor.process.name,
+// on_behalf_of on actor.user.name, server_name on resource.name.
+
+func aiRuntimeEvent() map[string]interface{} {
+ return map[string]interface{}{
+ "class_uid": float64(6003),
+ "source_type": "ai-runtime",
+ "actor": map[string]interface{}{
+ "process": map[string]interface{}{"name": "agent-7f2", "path": "/usr/local/bin/claude"},
+ "user": map[string]interface{}{"name": "[email protected]"},
+ },
+ "src_endpoint": map[string]interface{}{"hostname": "ada-mbp"},
+ "metadata": map[string]interface{}{
+ "product": map[string]interface{}{"feature": map[string]interface{}{"name": "claude-sonnet-4"}},
+ },
+ "resource": map[string]interface{}{"name": "github-mcp"},
+ }
+}
+
+func findNode(ev *Event, label NodeLabel) *Node {
+ for i := range ev.Nodes {
+ if ev.Nodes[i].Label == label {
+ return &ev.Nodes[i]
+ }
+ }
+ return nil
+}
+
+func hasEdge(ev *Event, relType RelType, from, to NodeLabel) bool {
+ for _, e := range ev.Edges {
+ if e.Type == relType && e.FromLabel == from && e.ToLabel == to {
+ return true
+ }
+ }
+ return false
+}
+
+func TestAIRuntimeEventProducesAnAssetLinkedToPersonAndDevice(t *testing.T) {
+ ev := ExtractFromOCSF("evt-1", "tenant-a", "ai_gateway", aiRuntimeEvent())
+ if ev == nil {
+ t.Fatal("expected a graph projection, got nil")
+ }
+ asset := findNode(ev, NodeAIAsset)
+ if asset == nil {
+ t.Fatal("no AIAsset node emitted")
+ }
+ if got := asset.Properties["kind"]; got != "coding_agent" {
+ t.Errorf("kind = %v, want coding_agent", got)
+ }
+ if got := asset.Properties["vendor"]; got != "Anthropic" {
+ t.Errorf("vendor = %v, want Anthropic", got)
+ }
+ if got := asset.Properties["model"]; got != "claude-sonnet-4" {
+ t.Errorf("model = %v, want claude-sonnet-4", got)
+ }
+ // The two links depth 6.1 asks for.
+ if !hasEdge(ev, RelOperates, NodeUser, NodeAIAsset) {
+ t.Error("no (:User)-[:OPERATES]->(:AIAsset) edge")
+ }
+ if !hasEdge(ev, RelHosts, NodeEndpoint, NodeAIAsset) {
+ t.Error("no (:Endpoint)-[:HOSTS]->(:AIAsset) edge")
+ }
+}
+
+func TestUnrecognisedAgentIsAnAIAppRatherThanACodingAgent(t *testing.T) {
+ // The honest answer. Claiming a vendor we have no evidence for would put
+ // a confident, wrong name on an inventory row somebody acts on.
+ ocsf := aiRuntimeEvent()
+ ocsf["actor"] = map[string]interface{}{
+ "process": map[string]interface{}{"name": "svc-42", "path": "/opt/inhouse/summariser"},
+ }
+ ev := ExtractFromOCSF("evt-2", "tenant-a", "ai_gateway", ocsf)
+ asset := findNode(ev, NodeAIAsset)
+ if asset == nil {
+ t.Fatal("no AIAsset node emitted")
+ }
+ if got := asset.Properties["kind"]; got != "ai_app" {
+ t.Errorf("kind = %v, want ai_app", got)
+ }
+}
+
+func TestAnAssetWithNoPersonEmitsNoOperatesEdge(t *testing.T) {
+ // Absent rather than guessed: an unlinked agent is the finding, and an
+ // edge to a placeholder user would hide it.
+ ocsf := aiRuntimeEvent()
+ ocsf["actor"] = map[string]interface{}{
+ "process": map[string]interface{}{"name": "agent-7f2", "path": "/usr/local/bin/claude"},
+ }
+ ev := ExtractFromOCSF("evt-3", "tenant-a", "ai_gateway", ocsf)
+ if hasEdge(ev, RelOperates, NodeUser, NodeAIAsset) {
+ t.Error("emitted an OPERATES edge for an event that named no person")
+ }
+}
+
+func TestProcessActivityFindsACodingAgentOnADevice(t *testing.T) {
+ ocsf := map[string]interface{}{
+ "class_uid": float64(1007),
+ "source_type": "crowdstrike_falcon",
+ "process": map[string]interface{}{"name": "C:\\Program Files\\aider\\Aider.exe"},
+ "device": map[string]interface{}{"name": "win-dev-02"},
+ }
+ ev := ExtractFromOCSF("evt-4", "tenant-a", "crowdstrike_falcon", ocsf)
+ asset := findNode(ev, NodeAIAsset)
+ if asset == nil {
+ t.Fatal("no AIAsset node emitted from process activity")
+ }
+ if got := asset.Properties["external_id"]; got != "aider" {
+ t.Errorf("external_id = %v, want aider (basename, lowercased, .exe stripped)", got)
+ }
+ if !hasEdge(ev, RelHosts, NodeEndpoint, NodeAIAsset) {
+ t.Error("no HOSTS edge from the device")
+ }
+}
+
+func TestOrdinaryProcessActivityIsNotAnAIAsset(t *testing.T) {
+ // The negative control for the match table. A substring heuristic on
+ // "agent" would put every EDR sensor on the AI inventory.
+ for _, name := range []string{"falcon-sensor", "chrome", "osqueryd", "MsMpEng.exe"} {
+ ocsf := map[string]interface{}{
+ "class_uid": float64(1007),
+ "process": map[string]interface{}{"name": name},
+ "device": map[string]interface{}{"name": "win-dev-02"},
+ }
+ ev := ExtractFromOCSF("evt-5", "tenant-a", "crowdstrike_falcon", ocsf)
+ if ev != nil && findNode(ev, NodeAIAsset) != nil {
+ t.Errorf("%s was classified as an AI asset", name)
+ }
+ }
+}
+
+func TestANonProcessClassNamingAnAgentIsIgnored(t *testing.T) {
+ // A file event naming a binary is not evidence the binary ran.
+ ocsf := map[string]interface{}{
+ "class_uid": float64(1001),
+ "process": map[string]interface{}{"name": "claude"},
+ "device": map[string]interface{}{"name": "win-dev-02"},
+ }
+ ev := ExtractFromOCSF("evt-6", "tenant-a", "crowdstrike_falcon", ocsf)
+ if ev != nil && findNode(ev, NodeAIAsset) != nil {
+ t.Error("a file-system event was classified as a running AI asset")
+ }
+}
+
+func TestAIAssetIsInTheCanonicalLabelEnumeration(t *testing.T) {
+ // Declared and never enumerated is drift a schema gate can miss: the
+ // constant exists, the YAML lists it, and AllNodeLabels — the list every
+ // consumer reads — does not.
+ for _, l := range AllNodeLabels {
+ if l == NodeAIAsset {
+ return
+ }
+ }
+ t.Error("NodeAIAsset is not in AllNodeLabels")
+}
+
+func TestAIAssetIsNotGlobal(t *testing.T) {
+ // An agent is somebody's. A global label is exempted from tenant-scoped
+ // reads and from an offboarding purge, both of which would be wrong here.
+ for _, l := range GlobalNodeLabels {
+ if l == NodeAIAsset {
+ t.Fatal("AIAsset is listed as a global reference label; it is tenant-scoped")
+ }
+ }
+}
+
+func TestAIAgentVocabularyMatchesTheFusionRecorder(t *testing.T) {
+ // Two services, one meaning, and neither can import the other. A mirror
+ // nothing compares is two copies that will drift, so this reads the
+ // Python file rather than trusting the comment that says they match.
+ data, err := os.ReadFile("../../../fusion/app/services/ai_asset_recorder.py")
+ if err != nil {
+ t.Skipf("fusion recorder not readable from here: %v", err)
+ }
+ recorder := string(data)
+ for name := range aiAgentProcesses {
+ if !strings.Contains(recorder, "\""+name+"\"") {
+ t.Errorf("%q is in the Go match table and not in CODING_AGENT_PROCESSES", name)
+ }
+ }
+}