Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Per-principal behavioural baselines, and the three ways they refuse.
Depth plan 3.4. The assertions below are about behaviour under the
conditions that actually occur — a principal with no history, one with
enough, a service account whose variance has collapsed, a tenant boundary,
and Redis being unavailable — rather than about a helper returning a float.
Three of them are the ones worth having, because each pins a refusal that a
plausible-looking implementation would get wrong in the unsafe direction:
* **A short history contributes no key.** The first event a principal ever
produces is 100% of their history, so every value looks rare. Reporting
that would make every new account fire every novelty rule.
* **A degenerate distribution scores `None`, not `0.0`.** Service accounts
converge on a constant and their standard deviation collapses, so a
z-score of zero would make the most automated account in the estate the
least suspicious one it is possible to be.
* **An unreadable store contributes no key.** Same reason as the first-seen
store: absent means the clause does not match, which is quiet.
"""
from __future__ import annotations
import importlib.util
from pathlib import Path
from typing import Any
import pytest
from app.services.baselines import (
DERIVED_FIELDS,
MIN_PEER_MEMBERS,
MIN_VOLUME_SAMPLES,
OAUTH_APP_FIELDS,
PEER_GROUP_FIELDS,
PRINCIPAL_FIELDS,
RARITY_FIELDS,
BaselineStore,
resolve_principal,
welford_add,
z_score,
)
_REPO = Path(__file__).resolve().parents[3]
class _FakeRedis:
"""The hash and set operations the store actually calls."""
def __init__(self) -> None:
self.hashes: dict[str, dict[str, str]] = {}
self.sets: dict[str, set[str]] = {}
async def hincrby(self, key: str, field: str, amount: int = 1) -> int:
bucket = self.hashes.setdefault(key, {})
value = int(bucket.get(field, "0")) + amount
bucket[field] = str(value)
return value
async def hget(self, key: str, field: str) -> Any:
return self.hashes.get(key, {}).get(field)
async def hset(self, key: str, field: str, value: str) -> int:
self.hashes.setdefault(key, {})[field] = value
return 1
async def hgetall(self, key: str) -> dict[str, str]:
return dict(self.hashes.get(key, {}))
async def hlen(self, key: str) -> int:
return len(self.hashes.get(key, {}))
async def sadd(self, key: str, member: str) -> int:
self.sets.setdefault(key, set()).add(member)
return 1
async def scard(self, key: str) -> int:
return len(self.sets.get(key, set()))
async def expire(self, key: str, ttl: int) -> bool:
return True
class _NoRedis:
def __getattr__(self, _name: str) -> Any:
async def _boom(*_args: Any, **_kwargs: Any) -> Any:
raise ConnectionError("redis is down")
return _boom
async def _warm(store: BaselineStore, tenant: str, event: dict[str, Any], times: int) -> None:
"""Give a principal enough history that rarity is reportable."""
for _ in range(times):
await store.derived_fields(tenant, event, now=1000.0)
# -- rarity -----------------------------------------------------------------
@pytest.mark.asyncio
async def test_a_principal_with_too_little_history_gets_no_rarity() -> None:
"""Every value is novel on day one; saying so would fire on everything."""
store = BaselineStore(_FakeRedis())
fields = await store.derived_fields("t-1", {"user": "alice", "event_name": "GetObject"}, now=1000.0)
assert "baseline_action_rarity" not in fields
@pytest.mark.asyncio
async def test_a_routine_action_scores_near_zero_and_a_novel_one_near_one() -> None:
store = BaselineStore(_FakeRedis(), min_observations=20)
routine = {"user": "alice", "event_name": "GetObject"}
await _warm(store, "t-1", routine, 40)
common = await store.derived_fields("t-1", routine, now=1000.0)
assert common["baseline_action_rarity"] < 0.05
novel = await store.derived_fields("t-1", {"user": "alice", "event_name": "CreateAccessKey"}, now=1000.0)
# 1 sighting out of 42 observations: rare, but bounded by how much
# history exists rather than asserted as a flat 1.0.
assert 0.9 < novel["baseline_action_rarity"] < 1.0
@pytest.mark.asyncio
async def test_rarity_is_per_principal_not_per_tenant() -> None:
"""Alice's routine is not Bob's, which is the whole point of a baseline."""
store = BaselineStore(_FakeRedis(), min_observations=20)
await _warm(store, "t-1", {"user": "alice", "event_name": "CreateAccessKey"}, 40)
await _warm(store, "t-1", {"user": "bob", "event_name": "GetObject"}, 40)
for_bob = await store.derived_fields("t-1", {"user": "bob", "event_name": "CreateAccessKey"}, now=1000.0)
assert for_bob["baseline_action_rarity"] > 0.9
@pytest.mark.asyncio
async def test_tenants_do_not_share_baselines() -> None:
store = BaselineStore(_FakeRedis(), min_observations=20)
await _warm(store, "t-1", {"user": "alice", "event_name": "GetObject"}, 40)
other = await store.derived_fields("t-2", {"user": "alice", "event_name": "GetObject"}, now=1000.0)
assert "baseline_action_rarity" not in other
@pytest.mark.asyncio
async def test_the_value_set_per_principal_is_bounded() -> None:
"""An attacker-chosen resource name must not be a memory primitive."""
store = BaselineStore(_FakeRedis(), min_observations=1)
for index in range(700):
await store.derived_fields("t-1", {"user": "alice", "resource": f"obj-{index}"}, now=1000.0)
key = "aisoc:bl:t-1:alice:baseline_resource_rarity"
assert len(store._redis.hashes[key]) <= 514 # cap, plus the total field
# -- volume -----------------------------------------------------------------
@pytest.mark.asyncio
async def test_the_hour_in_progress_is_not_scored() -> None:
"""A partial count against complete ones calls every event at :01 odd."""
store = BaselineStore(_FakeRedis())
first = await store.derived_fields("t-1", {"user": "alice"}, now=3600.0)
assert "baseline_principal_volume_z" not in first
@pytest.mark.asyncio
async def test_a_volume_spike_scores_once_the_hour_completes() -> None:
store = BaselineStore(_FakeRedis())
hour = 3600.0
# Quiet hours of 8-12 events. The counts must *vary*: a principal who
# does exactly ten things an hour has zero variance, and the degenerate
# guard then refuses to score them at all — correctly, and it is what
# the first draft of this fixture accidentally measured.
for index in range(MIN_VOLUME_SAMPLES + 2):
for _ in range(8 + (index % 5)):
await store.derived_fields("t-1", {"user": "alice"}, now=hour * (index + 1))
# A loud hour, then the next hour's first event scores the loud one.
for _ in range(400):
await store.derived_fields("t-1", {"user": "alice"}, now=hour * 40)
scored = await store.derived_fields("t-1", {"user": "alice"}, now=hour * 41)
assert scored["baseline_principal_volume_z"] > 3.0
@pytest.mark.asyncio
async def test_an_oauth_app_is_baselined_separately_from_the_user() -> None:
"""A rogue grant acts as a user, so the user's own volume looks normal."""
store = BaselineStore(_FakeRedis())
event = {"user": "alice", "oauth_app": "backup-tool"}
hour = 3600.0
for index in range(MIN_VOLUME_SAMPLES + 2):
await store.derived_fields("t-1", event, now=hour * (index + 1))
key = "aisoc:bl:t-1:backup-tool:vol:oauth_app"
assert key in store._redis.hashes
# -- peer groups ------------------------------------------------------------
@pytest.mark.asyncio
async def test_a_group_smaller_than_the_floor_yields_no_peer_signal() -> None:
"""A group of one is that principal's own baseline wearing a group name."""
store = BaselineStore(_FakeRedis())
fields = await store.derived_fields("t-1", {"user": "alice", "department": "finance"}, now=1000.0)
assert "baseline_peer_deviation" not in fields
@pytest.mark.asyncio
async def test_the_peer_floor_counts_members_not_observations() -> None:
"""One noisy principal must not satisfy the minimum on its own."""
store = BaselineStore(_FakeRedis())
for _ in range(MIN_PEER_MEMBERS * 10):
await store.derived_fields("t-1", {"user": "alice", "department": "finance"}, now=1000.0)
fields = await store.derived_fields("t-1", {"user": "alice", "department": "finance"}, now=1000.0)
assert "baseline_peer_deviation" not in fields
# -- the refusals -----------------------------------------------------------
def test_a_degenerate_distribution_is_unknown_and_not_normal() -> None:
"""A service account's variance collapses; 0.0 would read as all-clear."""
stats: dict[str, float] = {}
for _ in range(50):
stats = welford_add(stats, 10.0)
assert z_score(stats, 9999.0, min_samples=5) is None
def test_a_short_history_is_unknown() -> None:
stats: dict[str, float] = {}
for value in (1.0, 2.0, 3.0):
stats = welford_add(stats, value)
assert z_score(stats, 100.0, min_samples=12) is None
def test_welford_mutates_nothing() -> None:
"""The ueba copy edited in place, so no baseline there ever advanced."""
stats = {"count": 1.0, "mean": 5.0, "m2": 0.0}
before = dict(stats)
welford_add(stats, 9.0)
assert stats == before
@pytest.mark.asyncio
async def test_an_unreadable_store_contributes_no_key() -> None:
store = BaselineStore(_NoRedis())
assert await store.derived_fields("t-1", {"user": "alice", "event_name": "x"}, now=1000.0) == {}
@pytest.mark.asyncio
async def test_an_event_naming_no_principal_contributes_no_key() -> None:
"""A baseline with no subject is not a baseline."""
store = BaselineStore(_FakeRedis())
assert await store.derived_fields("t-1", {"event_name": "GetObject"}, now=1000.0) == {}
@pytest.mark.asyncio
async def test_a_tenantless_event_is_not_recorded() -> None:
"""Without a tenant the key is shared, which is a cross-tenant read."""
redis = _FakeRedis()
store = BaselineStore(redis)
assert await store.derived_fields("", {"user": "alice", "event_name": "x"}, now=1000.0) == {}
assert redis.hashes == {}
def test_a_principal_is_resolved_case_insensitively() -> None:
assert resolve_principal({"user": " Alice "}) == "alice"
assert resolve_principal({"actor_email": "[email protected]"}) == "[email protected]"
assert resolve_principal({"event_name": "GetObject"}) is None
# -- typed signals ----------------------------------------------------------
@pytest.mark.asyncio
async def test_each_observation_is_a_typed_signal() -> None:
"""3.4 requires a typed signal, not an alert — so the kind must survive."""
store = BaselineStore(_FakeRedis(), min_observations=20)
routine = {"user": "alice", "event_name": "GetObject"}
await _warm(store, "t-1", routine, 40)
signals = await store.signals("t-1", routine, now=1000.0)
assert signals
assert {s.kind for s in signals} <= {"rarity", "volume", "peer"}
assert all(s.subject == "alice" for s in signals)
# -- drift ------------------------------------------------------------------
def test_the_reachability_gate_mirrors_this_module() -> None:
"""A new baseline field must be added in both places or it is invisible.
The direction that bites is the one the matcher's operator list got
wrong (deviation D9): the gate kept its own copy, the module grew a
field, and the gate went on reporting rules unreachable that were not.
Asserting equality rather than containment catches both directions.
"""
spec = importlib.util.spec_from_file_location("cdf_baselines", _REPO / "scripts" / "check_detection_fields.py")
assert spec and spec.loader
gate = importlib.util.module_from_spec(spec)
spec.loader.exec_module(gate)
assert gate._BASELINE_DERIVED_FIELDS == DERIVED_FIELDS
def test_every_rarity_field_is_declared_as_derived() -> None:
assert set(RARITY_FIELDS) <= DERIVED_FIELDS
def test_every_subject_chain_names_a_field_some_connector_emits() -> None:
"""A baseline over a field nothing emits is a signal that cannot fire.
This is the test that earns its place. The first draft of
`RARITY_FIELDS` gave `baseline_client_family_rarity` the chain
`client_family` / `user_agent_family` / `client_app`, and **no connector
emits any of the three** — so that rarity would have been published,
counted, and structurally incapable of being computed on any
deployment. Worse than a missing feature, because the field name reads
as working.
The namespace comes from the same static analysis
`scripts/check_detection_fields.py` performs, so the two cannot
disagree about what a connector emits.
"""
spec = importlib.util.spec_from_file_location("cdf_emit", _REPO / "scripts" / "check_detection_fields.py")
assert spec and spec.loader
gate = importlib.util.module_from_spec(spec)
spec.loader.exec_module(gate)
emitted = gate._connector_namespace()
chains: dict[str, tuple[str, ...]] = {
**RARITY_FIELDS,
"__principal__": PRINCIPAL_FIELDS,
"__oauth_app__": OAUTH_APP_FIELDS,
"__peer_group__": PEER_GROUP_FIELDS,
}
unreachable = {name: chain for name, chain in chains.items() if not (set(chain) & emitted)}
assert not unreachable, f"no connector emits any candidate for: {sorted(unreachable)}"
def test_the_narrowed_family_no_longer_claims_unrelated_fields() -> None:
"""`^active_` swept in a systemd unit state and an ITSM lookup.
Both were reported as blocked on a baseline that would never have
helped either, so the family read "2 rules need 3.4" when the answer
was zero. Pinned so the convenient-looking broad pattern cannot return.
"""
spec = importlib.util.spec_from_file_location("cdf_family", _REPO / "scripts" / "check_detection_fields.py")
assert spec and spec.loader
gate = importlib.util.module_from_spec(spec)
spec.loader.exec_module(gate)
assert gate._family("active_state") != "behavioural baseline"
assert gate._family("active_p0_or_p1_incident") == "external system context"
for field in DERIVED_FIELDS:
assert gate._family(field) == "behavioural baseline"