Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

18.4 KB
Raw Blame
"""
Tests for the detection-confidence + explainability scorer.

Wave 1 contract:

* every fused alert leaves the engine with a ``confidence_label`` of
  high / medium / low **and** a non-empty ``confidence_rationale``
  containing the factors that produced the score;
* the rationale is reproducible — the same input always produces the same
  score and the same ordering of factors;
* "high confidence" requires multiple positive signals (severity alone is
  not enough to clear the bar) so the analyst can trust the label;
* "low confidence" reliably surfaces the weak / suspicious / informational
  alerts so the queue stays clean.
"""

from __future__ import annotations

from uuid import uuid4

import pytest
from app.models.alert import (
    AlertSeverity,
    ConfidenceLabel,
    FusedAlert,
    FusionDecision,
    RawAlert,
)
from app.services.confidence import (
    HIGH_THRESHOLD,
    LOW_THRESHOLD,
    WEIGHT_ASSET_CRITICALITY,
    WEIGHT_IDENTITY_PRIVILEGE,
    ConfidenceScorer,
)

# ─── Helpers ─────────────────────────────────────────────────────────────────


def _alert(
    *,
    severity: AlertSeverity = AlertSeverity.MEDIUM,
    techniques: list[str] | None = None,
    risk_score: float = 0.0,
    src_ip: str | None = None,
    dst_ip: str | None = None,
    hostname: str | None = None,
    username: str | None = None,
    file_hash: str | None = None,
    domain: str | None = None,
    url: str | None = None,
) -> RawAlert:
    return RawAlert(
        tenant_id=uuid4(),
        source="unit-test",
        title="Test alert",
        severity=severity,
        mitre_techniques=techniques or [],
        risk_score=risk_score,
        src_ip=src_ip,
        dst_ip=dst_ip,
        hostname=hostname,
        username=username,
        file_hash=file_hash,
        domain=domain,
        url=url,
    )


def _fused(
    alert: RawAlert,
    *,
    anomaly_score: float = 0.0,
    priority_score: float = 0.0,
    enrichments: dict | None = None,
) -> FusedAlert:
    return FusedAlert(
        id=alert.id,
        tenant_id=alert.tenant_id,
        incident_id=uuid4(),
        fusion_decision=FusionDecision.NEW_INCIDENT,
        alert=alert,
        anomaly_score=anomaly_score,
        priority_score=priority_score,
        enrichments=enrichments or {},
    )


@pytest.fixture
def scorer() -> ConfidenceScorer:
    return ConfidenceScorer()


# ─── Rationale shape ─────────────────────────────────────────────────────────


def test_score_attaches_complete_rationale(scorer: ConfidenceScorer) -> None:
    """Every alert leaves the scorer with a non-empty rationale chain
    that names every weighted factor — no silent drops."""
    fused = _fused(_alert())
    scored = scorer.score(fused)
    factors = {f.factor for f in scored.confidence_rationale}
    assert factors == {
        "severity",
        "ml_anomaly",
        "ml_priority",
        "mitre_coverage",
        "threat_intel",
        "upstream_risk",
        "ioc_density",
        "asset_criticality",
        # Depth 2 wave 2, 1.1. Present on *every* alert, including ones with
        # no posture behind them, where it reads "unknown". That is the
        # whole point: a factor that appears only when the answer is known
        # cannot tell an analyst that the platform could not answer.
        "identity_privilege",
    }


def test_rationale_is_sorted_by_impact(scorer: ConfidenceScorer) -> None:
    """The UI renders the rationale top-down, so the most impactful
    factor must come first."""
    fused = _fused(
        _alert(
            severity=AlertSeverity.CRITICAL,
            techniques=["T1078", "T1098", "T1556"],
            src_ip="10.0.0.1",
            hostname="prod-1",
            username="alice",
        ),
        anomaly_score=0.95,
        priority_score=0.95,
        enrichments={"misp": {"hit": True}, "kev": {"matches": ["CVE-2024-1234"]}},
    )
    scored = scorer.score(fused)
    impacts = [abs(f.contribution * f.weight) for f in scored.confidence_rationale]
    assert impacts == sorted(impacts, reverse=True)


def test_rationale_weights_sum_to_one(scorer: ConfidenceScorer) -> None:
    """Weights are the model's *relative importance*; they must sum to 1.0
    so that a 100%-positive contribution maps to a full +0.5 delta."""
    scored = scorer.score(_fused(_alert()))
    total = sum(f.weight for f in scored.confidence_rationale)
    assert total == pytest.approx(1.0, abs=1e-6)


def test_score_is_deterministic(scorer: ConfidenceScorer) -> None:
    """Same input → identical score and rationale ordering. The rationale
    is reproducible audit evidence."""
    alert = _alert(severity=AlertSeverity.HIGH, techniques=["T1078"])
    a = scorer.score(_fused(alert, anomaly_score=0.7, priority_score=0.6))
    b = scorer.score(_fused(alert, anomaly_score=0.7, priority_score=0.6))
    assert a.confidence_score == b.confidence_score
    assert a.confidence_label == b.confidence_label
    assert [f.factor for f in a.confidence_rationale] == [f.factor for f in b.confidence_rationale]


# ─── Banding ─────────────────────────────────────────────────────────────────


def test_high_confidence_requires_multiple_positive_signals(scorer: ConfidenceScorer) -> None:
    """Severity alone must NOT clear the HIGH threshold — the whole point
    of the explainability surface is that "critical" without supporting
    evidence is a noisy detection, not a confident one."""
    bare = _fused(_alert(severity=AlertSeverity.CRITICAL))
    scored = scorer.score(bare)
    assert scored.confidence_label != ConfidenceLabel.HIGH


def test_high_confidence_with_strong_corroborating_signals(scorer: ConfidenceScorer) -> None:
    """Critical severity + strong ML scores + MITRE coverage + TI hit + IOCs
    must produce HIGH confidence. This is the "ship it" path."""
    fused = _fused(
        _alert(
            severity=AlertSeverity.CRITICAL,
            techniques=["T1078", "T1098", "T1556"],
            risk_score=0.9,
            src_ip="10.0.0.1",
            dst_ip="8.8.8.8",
            hostname="prod-1",
            username="alice",
            file_hash="abc123",
        ),
        anomaly_score=0.92,
        priority_score=0.95,
        enrichments={
            "misp": {"hit": True},
            "otx": {"hit": True},
            "kev": {"matches": ["CVE-2024-1234"]},
        },
    )
    scored = scorer.score(fused)
    assert scored.confidence_label == ConfidenceLabel.HIGH
    assert scored.confidence_score >= HIGH_THRESHOLD


def test_low_confidence_for_info_severity_with_no_signals(scorer: ConfidenceScorer) -> None:
    """An info-severity alert with no MITRE coverage, no TI hit, no IOCs,
    and zero ML scores belongs in the low-confidence band — that's the
    "suppress or downgrade" lane."""
    fused = _fused(_alert(severity=AlertSeverity.INFO))
    scored = scorer.score(fused)
    assert scored.confidence_label == ConfidenceLabel.LOW
    assert scored.confidence_score < LOW_THRESHOLD


def test_medium_confidence_for_balanced_signal(scorer: ConfidenceScorer) -> None:
    """High severity + two techniques + a couple IOCs + reasonable ML
    scores but **no TI hit** must land in MEDIUM. This is the most
    realistic real-world detection — looks suspicious but missing the
    last bit of corroborating evidence to auto-action."""
    fused = _fused(
        _alert(
            severity=AlertSeverity.HIGH,
            techniques=["T1078", "T1098"],
            src_ip="10.0.0.1",
            hostname="prod-1",
        ),
        anomaly_score=0.6,
        priority_score=0.55,
    )
    scored = scorer.score(fused)
    assert scored.confidence_label == ConfidenceLabel.MEDIUM
    assert LOW_THRESHOLD <= scored.confidence_score < HIGH_THRESHOLD


# ─── Individual factors ──────────────────────────────────────────────────────


def test_threat_intel_hit_increases_score(scorer: ConfidenceScorer) -> None:
    """A TI hit must monotonically increase confidence relative to the
    same alert without one — that's the whole point of enrichment."""
    base = _alert(severity=AlertSeverity.HIGH, techniques=["T1078"], src_ip="10.0.0.1")

    no_ti = scorer.score(_fused(base, anomaly_score=0.6, priority_score=0.6))
    with_ti = scorer.score(
        _fused(
            base,
            anomaly_score=0.6,
            priority_score=0.6,
            enrichments={"misp": {"hit": True}},
        )
    )
    assert with_ti.confidence_score > no_ti.confidence_score


def test_mitre_coverage_increases_score(scorer: ConfidenceScorer) -> None:
    """More MITRE techniques → more confident detection. Three techniques
    must beat zero, all else equal."""
    sparse = _fused(
        _alert(severity=AlertSeverity.HIGH, src_ip="10.0.0.1"),
        anomaly_score=0.5,
        priority_score=0.5,
    )
    rich = _fused(
        _alert(
            severity=AlertSeverity.HIGH,
            techniques=["T1078", "T1098", "T1556"],
            src_ip="10.0.0.1",
        ),
        anomaly_score=0.5,
        priority_score=0.5,
    )
    assert scorer.score(rich).confidence_score > scorer.score(sparse).confidence_score


def test_ioc_density_increases_score(scorer: ConfidenceScorer) -> None:
    """More populated IOC fields → more concrete signal → higher score.
    Five IOC fields must beat zero, all else equal."""
    sparse = _fused(_alert(severity=AlertSeverity.MEDIUM, techniques=["T1078"]))
    rich = _fused(
        _alert(
            severity=AlertSeverity.MEDIUM,
            techniques=["T1078"],
            src_ip="10.0.0.1",
            dst_ip="8.8.8.8",
            hostname="prod-1",
            username="alice",
            file_hash="abc123",
        )
    )
    assert scorer.score(rich).confidence_score > scorer.score(sparse).confidence_score


# ─── Feature flag (break-glass) ──────────────────────────────────────────────


def test_disabled_scorer_passes_alert_through_unchanged() -> None:
    """When ``AISOC_FEATURE_CONFIDENCE`` is off the scorer must be a no-op:
    the alert keeps the FusedAlert defaults (MEDIUM / 0.5 / []) so the
    frontend degrades gracefully and downstream consumers never see a
    half-populated rationale."""
    disabled = ConfidenceScorer(enabled=False)
    assert disabled.enabled is False

    fused = _fused(
        _alert(severity=AlertSeverity.CRITICAL, techniques=["T1078"]),
        anomaly_score=0.95,
        priority_score=0.95,
        enrichments={"misp": {"hit": True}},
    )
    scored = disabled.score(fused)

    # Same instance back; we explicitly do NOT compute a rationale when off.
    assert scored is fused
    assert scored.confidence_label == ConfidenceLabel.MEDIUM
    assert scored.confidence_score == 0.5
    assert scored.confidence_rationale == []


def test_score_clamped_to_unit_interval(scorer: ConfidenceScorer) -> None:
    """Score must always live in [0, 1] no matter how many signals fire."""
    extreme = _fused(
        _alert(
            severity=AlertSeverity.CRITICAL,
            techniques=["T1078", "T1098", "T1556", "T1110", "T1003"],
            risk_score=1.0,
            src_ip="10.0.0.1",
            dst_ip="8.8.8.8",
            hostname="prod-1",
            username="alice",
            file_hash="abc123",
            domain="evil.example",
            url="https://evil.example/payload",
        ),
        anomaly_score=1.0,
        priority_score=1.0,
        enrichments={
            "misp": {"hit": True},
            "otx": {"hit": True},
            "kev": {"matches": ["CVE-2024-1234"]},
            "virustotal": {"hit": True},
        },
    )
    scored = scorer.score(extreme)
    assert 0.0 <= scored.confidence_score <= 1.0


# ─── Asset criticality (business-context plan, PR-3) ─────────────────────────


def test_asset_criticality_reads_the_fuse_time_stamp(scorer: ConfidenceScorer) -> None:
    """A critical asset pushes the score up; the factor names the match."""
    fused = _fused(_alert())
    fused.enrichments["asset_context"] = {"criticality": "critical", "matched": "prod-dc-01"}
    scored = scorer.score(fused)
    factor = next(f for f in scored.confidence_rationale if f.factor == "asset_criticality")
    assert factor.contribution == 1.0
    assert factor.value == "critical"

    baseline = scorer.score(_fused(_alert()))
    assert scored.confidence_score > baseline.confidence_score


def test_no_inventory_match_is_unknown_not_zero_evidence(scorer: ConfidenceScorer) -> None:
    """Absent stamp: contribution exactly 0.0 and the row SAYS unknown —
    "we do not know what this host is" is a fact an analyst needs, so the
    factor is rendered rather than omitted."""
    scored = scorer.score(_fused(_alert()))
    factor = next(f for f in scored.confidence_rationale if f.factor == "asset_criticality")
    assert factor.contribution == 0.0
    assert factor.value == "unknown (no inventory match)"


def test_low_criticality_pulls_down_but_is_bounded(scorer: ConfidenceScorer) -> None:
    fused = _fused(_alert())
    fused.enrichments["asset_context"] = {"criticality": "low", "matched": "kiosk-7"}
    scored = scorer.score(fused)
    factor = next(f for f in scored.confidence_rationale if f.factor == "asset_criticality")
    assert factor.contribution == -0.3
    # Bounded: weight 0.0564, so the full swing is ±0.028 on the score. The
    # weight was 0.06 until 1.1 added `identity_privilege` and scaled the
    # eight pre-existing factors by 0.94 to keep the total at exactly 1.0.
    assert factor.weight == WEIGHT_ASSET_CRITICALITY
    assert factor.weight == pytest.approx(0.06 * 0.94)


# ─── Actor privilege at event time (depth 2 wave 2, 1.1) ─────────────────────


def _privilege_factor(scorer: ConfidenceScorer, stamp: object):
    fused = _fused(_alert())
    fused.enrichments["identity_privilege"] = stamp
    scored = scorer.score(fused)
    return next(f for f in scored.confidence_rationale if f.factor == "identity_privilege")


def test_absent_privilege_stamp_is_unknown_and_contributes_nothing(scorer: ConfidenceScorer) -> None:
    """No stamp must read as unknown in words, not as a bare 0.00.

    A zero with no explanation is indistinguishable from a measured neutral,
    and the two call for different actions: one is an ordinary account, the
    other is a provider whose posture has never been collected.
    """
    fused = _fused(_alert())
    scored = scorer.score(fused)
    factor = next(f for f in scored.confidence_rationale if f.factor == "identity_privilege")
    assert factor.contribution == 0.0
    assert "unknown" in factor.value


def test_unknown_tier_is_not_treated_as_standard(scorer: ConfidenceScorer) -> None:
    """`standard` is mildly negative; `unknown` is exactly neutral.

    Collapsing them would make an uncollected provider look like a estate
    full of ordinary users, which is the most reassuring possible rendering
    of a blind spot.
    """
    unknown = _privilege_factor(scorer, {"tier": "unknown", "measured": False})
    standard = _privilege_factor(scorer, {"tier": "standard", "measured": True, "tags_set": []})
    assert unknown.contribution == 0.0
    assert standard.contribution < 0.0


def test_tier_orders_the_contribution(scorer: ConfidenceScorer) -> None:
    tiers = ["standard", "elevated", "admin", "domain_admin"]
    contributions = [_privilege_factor(scorer, {"tier": t, "measured": True, "tags_set": []}).contribution for t in tiers]
    assert contributions == sorted(contributions)


def test_tags_add_but_are_capped(scorer: ConfidenceScorer) -> None:
    """Four tags must not swamp the tier, and the contribution stays in [-1, 1]."""
    none_set = _privilege_factor(scorer, {"tier": "admin", "measured": True, "tags_set": []})
    one_set = _privilege_factor(scorer, {"tier": "admin", "measured": True, "tags_set": ["can_escalate"]})
    all_set = _privilege_factor(
        scorer,
        {
            "tier": "admin",
            "measured": True,
            "tags_set": ["can_escalate", "data_access", "third_party", "cross_account"],
        },
    )
    assert none_set.contribution < one_set.contribution < all_set.contribution
    assert all_set.contribution <= 1.0
    # The cap is what stops tags from making the tier irrelevant: an admin
    # with four tags must still outrank a domain admin with none only
    # because of the tags, never because the tier stopped mattering.
    assert all_set.contribution - none_set.contribution <= 0.6 + 1e-9


def test_tags_dict_is_accepted_when_tags_set_is_missing(scorer: ConfidenceScorer) -> None:
    """The stamp has two shapes and only one of them is fusion's own.

    `tags_set` is written by the fuse-time resolver; `tags` is what the API
    stores and what a replayed or imported alert carries. Reading only the
    first silently scored every replayed alert as tagless.
    """
    factor = _privilege_factor(
        scorer,
        {
            "tier": "admin",
            "measured": True,
            "tags": {"can_escalate": True, "data_access": None, "third_party": False, "cross_account": True},
        },
    )
    baseline = _privilege_factor(scorer, {"tier": "admin", "measured": True, "tags_set": []})
    assert factor.contribution > baseline.contribution
    assert "can_escalate" in factor.value and "cross_account" in factor.value
    # None and False are both "not a reason to escalate" here, and neither
    # may be counted as a set tag.
    assert "third_party" not in factor.value
    assert "data_access" not in factor.value


def test_partial_snapshot_is_said_out_loud(scorer: ConfidenceScorer) -> None:
    factor = _privilege_factor(
        scorer,
        {"tier": "admin", "measured": True, "tags_set": [], "snapshot_partial": True},
    )
    assert "incomplete" in factor.value


def test_privilege_weight_is_bounded_like_asset_criticality(scorer: ConfidenceScorer) -> None:
    """Context about the alert must not outweigh evidence that it happened."""
    factor = _privilege_factor(scorer, {"tier": "domain_admin", "measured": True, "tags_set": []})
    assert factor.weight == WEIGHT_IDENTITY_PRIVILEGE
    assert WEIGHT_IDENTITY_PRIVILEGE <= 0.06