Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

11.0 KB
Raw Blame
"""A first-run tenant gets a wizard, and sample data that proves something.

The gap
-------
A brand-new operator signed in and landed on `/dashboard`: every tile
zero, every panel an honest empty state, and nothing saying what to do
next. The empty states were correct — that work was already done — but
correct and *useful* are different things, and "0 connected sources" is
not a button.

Two properties this file exists to protect
-------------------------------------------
**Sample data must not mark setup complete.** Somebody who has only
looked at samples still has nothing connected, and a checklist that
congratulates them is the flattering fiction this product spends most of
its effort avoiding.

**Sample data must go through the real pipeline.** Inserting rows would
be easier and would prove nothing: a console full of seeded alerts looks
identical whether ingest, fusion and triage work or are completely
broken. Running the real path means a first-run operator who sees alerts
has also seen the product work.
"""

from __future__ import annotations

import pathlib

import pytest
from app.services import sample_data

ENDPOINT = pathlib.Path(__file__).resolve().parents[1] / "app/api/v1/endpoints/onboarding.py"


class TestTheScenarioSet:
    def test_it_is_not_all_critical(self) -> None:
        """A first run where everything is critical teaches an operator
        nothing about how the product separates signal from routine."""
        severities = {s.severity for s in sample_data.SCENARIOS}
        assert len(severities) >= 3, f"only {severities} represented"
        assert "low" in severities or "info" in severities, (
            "nothing in the sample set is routine, so the console cannot show what a benign verdict looks like"
        )

    def test_every_scenario_says_why_it_is_in_the_set(self) -> None:
        for scenario in sample_data.SCENARIOS:
            assert scenario.rationale, f"{scenario.key} has no rationale"
            assert len(scenario.rationale) > 30

    def test_addresses_are_documentation_ranges(self) -> None:
        """RFC 5737 and RFC 2606. Sample data that uses a real IP or a
        registrable domain is a sample that can point at somebody."""
        import ipaddress

        blob = repr([s.fields for s in sample_data.SCENARIOS])
        for octets in ("192.0.2.", "198.51.100.", "203.0.113."):
            blob = blob.replace(octets, "DOC.")
        found = [
            token
            for token in blob.replace("'", " ").replace(",", " ").split()
            if token.count(".") == 3 and token.replace(".", "").isdigit()
        ]
        for addr in found:
            parsed = ipaddress.ip_address(addr)
            assert not parsed.is_global, (
                f"{addr} is a globally routable address in sample data, so a sample alert points at a real host somebody owns"
            )

    def test_domains_are_reserved(self) -> None:
        blob = repr([s.fields for s in sample_data.SCENARIOS])
        assert "@example.com" in blob
        for risky in ("@gmail", "@outlook", "@acme.io", "@corp.net"):
            assert risky not in blob


class TestTheBatch:
    def test_events_are_spread_over_time(self) -> None:
        """Five events at one instant look like a bug, and leave the
        volume chart and the correlation window with nothing to show."""
        events = sample_data.build_events()
        stamps = {e["created_at"] for e in events}
        assert len(stamps) == len(events)

    def test_every_event_carries_a_distinct_vendor_id(self) -> None:
        """The CloudTrail lesson: without a distinct `external_id` and
        title, a v5 alert id derived from content makes every event
        deduplicate onto one row."""
        events = sample_data.build_events()
        assert len({e["external_id"] for e in events}) == len(events)
        assert len({e["title"] for e in events}) == len(events)

    def test_the_raw_event_is_marked_too(self) -> None:
        """So a reader inspecting the lake — not just the alert — can
        still tell these from real telemetry."""
        for event in sample_data.build_events():
            assert event["raw_event"]["aisoc_sample_data"] is True
            assert event["raw_event"]["aisoc_sample_scenario"]

    def test_the_connector_id_is_dedicated_not_borrowed(self) -> None:
        """Borrowing a vendor id would make sample data indistinguishable
        from that vendor's real data, and impossible to delete safely."""
        assert sample_data.SAMPLE_CONNECTOR == "aisoc_sample"
        for event in sample_data.build_events():
            assert event["source"] == "aisoc_sample"


class TestItUsesTheRealPipeline:
    def test_it_posts_to_ingest_rather_than_writing_rows(self) -> None:
        source = pathlib.Path(sample_data.__file__).read_text() if hasattr(sample_data, "__file__") else ""
        assert "/v1/ingest/batch" in source, (
            "sample data no longer goes through ingest, so it proves nothing about whether the pipeline works"
        )
        for row_write in ("INSERT INTO", "session.add", "db.add"):
            assert row_write not in source, (
                f"sample data writes rows directly ({row_write}), which looks identical whether the pipeline works or is completely broken"
            )

    @pytest.mark.asyncio
    async def test_no_service_token_is_an_actionable_error(self, monkeypatch) -> None:  # noqa: ANN001
        monkeypatch.delenv("AISOC_SERVICE_TOKEN", raising=False)
        monkeypatch.delenv("AISOC_INGEST_TOKEN", raising=False)
        with pytest.raises(sample_data.SampleDataError) as excinfo:
            await sample_data.load(tenant_id="00000000-0000-0000-0000-000000000001")
        message = str(excinfo.value)
        assert "AISOC_SERVICE_TOKEN" in message
        assert "make up" in message, "the error does not say how to fix it"


class TestTheEndpointContract:
    def test_sample_data_does_not_clear_first_run(self) -> None:
        """The property that matters most.

        Somebody who has only looked at samples still has nothing
        connected. A checklist that tells them they are set up is a lie
        the rest of this product works hard not to tell.
        """
        source = ENDPOINT.read_text()
        block = source[source.index("return OnboardingStatus(") :]
        assert "first_run=connectors == 0 and real_alerts == 0" in block, (
            "first_run is no longer computed from real sources only, so loading sample data would mark setup complete"
        )

    def test_real_alerts_exclude_the_sample_connector(self) -> None:
        source = ENDPOINT.read_text()
        assert "real_alerts = alerts - sample_alerts" in source

    def test_it_refuses_to_seed_a_working_estate(self) -> None:
        """A sample alert in a live queue is indistinguishable from a real
        one at a glance, and an analyst dismissing a genuine alert because
        they assumed it was sample data is the worse outcome."""
        source = ENDPOINT.read_text()
        assert "HTTP_409_CONFLICT" in source
        assert "already has" in source

    def test_status_is_derived_not_stored(self) -> None:
        """A `tenant.onboarded` flag drifts the moment somebody connects a
        source through the API or deletes their last one."""
        source = ENDPOINT.read_text()
        assert "SELECT COUNT(*) FROM connectors" in source
        assert "SELECT COUNT(*) FROM alerts" in source

    def test_a_missing_table_does_not_500_the_wizard(self) -> None:
        """A half-migrated deployment is exactly when somebody reaches for
        the thing that is supposed to help them fix it."""
        source = ENDPOINT.read_text()
        counter = source[source.index("async def _count") : source.index("@router.get")]
        assert "except Exception" in counter
        assert "return 0" in counter

    def test_an_unreachable_ingest_is_502_not_500(self) -> None:
        """The API is fine and something it depends on is not, and the
        status code should say which."""
        source = ENDPOINT.read_text()
        assert "HTTP_502_BAD_GATEWAY" in source

    def test_every_step_explains_why_it_matters(self) -> None:
        """A checklist that does not say why is a chore list."""
        source = ENDPOINT.read_text()
        assert source.count("why=(") + source.count('why="') >= 4


class TestTheIngestProfileExists:
    def test_aisoc_sample_is_mapped(self) -> None:
        """Without a profile the batch hits the lenient fallback, every
        event gets the same generic title and no vendor id, and all five
        deduplicate onto one alert — the exact CloudTrail defect."""
        normalizer = pathlib.Path(__file__).resolve().parents[3] / "services/ingest/internal/normalizer/normalizer.go"
        source = normalizer.read_text()
        assert '"aisoc_sample": {' in source, "aisoc_sample has no ingest profile, so all five sample events collapse into one alert"
        block = source[source.index('"aisoc_sample": {') :][:900]
        assert '"title":       "message"' in block or '"title":"message"' in block
        assert "external_id" in block
        assert "Sample data" in block, (
            "the profile does not name the sample vendor, so a reader cannot tell sample alerts from real ones in the source column"
        )


class TestTheModelStep:
    """The wizard tells a first-run operator where AI triage is running.

    Added because nothing did. A tenant could finish setup without ever
    learning that the bundled model is on CPU, that their GPU could be used,
    or that their own provider is three fields away -- and the first two are
    not reachable from the console at all, since switching the bundled Ollama
    onto a GPU means restarting that container with different compose
    arguments.
    """

    def test_the_step_exists(self) -> None:
        source = ENDPOINT.read_text()
        assert 'key="model"' in source

    def test_it_is_not_blocking(self) -> None:
        """`done=True` out of the box, deliberately: a local model ships and
        runs, so a tenant on CPU is configured, not unfinished. A step that
        showed red until somebody bought a GPU would be a chore invented by
        the checklist."""
        source = ENDPOINT.read_text()
        block = source[source.index('key="model"') :]
        block = block[: block.index("SetupStep(", 10)] if "SetupStep(" in block[10:] else block
        assert "done=True" in block

    def test_it_is_derived_from_the_tenants_own_row(self) -> None:
        """Same discipline as every other step: no stored flag to drift. A
        credential that exists but is disabled must not read as configured."""
        source = ENDPOINT.read_text()
        assert "tenant_llm_credentials" in source
        assert "enabled IS TRUE" in source

    def test_it_explains_why_like_the_others(self) -> None:
        """The existing gate counts `why=` occurrences; this asserts the new
        step carries one rather than relying on the count rising."""
        source = ENDPOINT.read_text()
        block = source[source.index('key="model"') :]
        block = block[: block.index('key="try"')]
        assert "why=(" in block or 'why="' in block