Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""
Unit tests for playbook engine correctness fixes.
These tests pin down behavior of the helper functions and ``PlaybookEngine.run``
that were silently broken before ``fix/playbook-engine-correctness``:
1. Expression-string conditions were silently always-true (only field/op/value
were evaluated).
2. ``gt``/``lt`` crashed on non-numeric values via ``float(value or 0)``.
3. ``contains`` couldn't express list-membership (``severity in [...]``).
4. Context flattening used ``setdefault``, so later steps couldn't refine
values set earlier (e.g. enrichment overwriting a placeholder ``user_id``).
5. ``_resolve_field`` couldn't traverse list indices (``entities.0.name``).
Tests avoid any network I/O by stubbing ``_emit`` so the realtime POST never
fires.
"""
from __future__ import annotations
import asyncio
from typing import Any
from unittest.mock import AsyncMock
import pytest
from app.playbook import engine as engine_mod
from app.playbook.engine import (
PlaybookEngine,
RunStatus,
StepStatus,
_evaluate_condition,
_evaluate_expression,
_resolve_field,
_to_float,
)
from app.playbook.models import Playbook, PlaybookStep, StepCondition, StepType
# ---------------------------------------------------------------------------
# _resolve_field
# ---------------------------------------------------------------------------
class TestResolveField:
def test_returns_none_for_empty_field(self) -> None:
assert _resolve_field({"a": 1}, "") is None
def test_top_level_key(self) -> None:
assert _resolve_field({"severity": "high"}, "severity") == "high"
def test_nested_dict_path(self) -> None:
ctx = {"alert": {"severity": "high"}}
assert _resolve_field(ctx, "alert.severity") == "high"
def test_missing_key_returns_none(self) -> None:
assert _resolve_field({"alert": {}}, "alert.severity") is None
def test_traverses_list_index(self) -> None:
ctx = {"entities": [{"name": "alice"}, {"name": "bob"}]}
assert _resolve_field(ctx, "entities.0.name") == "alice"
assert _resolve_field(ctx, "entities.1.name") == "bob"
def test_negative_list_index(self) -> None:
ctx = {"entities": ["a", "b", "c"]}
assert _resolve_field(ctx, "entities.-1") == "c"
def test_out_of_range_list_index_returns_none(self) -> None:
ctx = {"entities": ["a"]}
assert _resolve_field(ctx, "entities.5") is None
def test_non_numeric_index_on_list_returns_none(self) -> None:
ctx = {"entities": ["a", "b"]}
assert _resolve_field(ctx, "entities.name") is None
def test_traversal_into_scalar_returns_none(self) -> None:
# Can't drill into a string with a dot-path.
assert _resolve_field({"name": "alice"}, "name.first") is None
# ---------------------------------------------------------------------------
# _to_float
# ---------------------------------------------------------------------------
class TestToFloat:
@pytest.mark.parametrize(
"value,expected",
[
(None, 0.0),
("", 0.0),
(0, 0.0),
(42, 42.0),
(3.14, 3.14),
("7", 7.0),
("7.5", 7.5),
(True, 1.0),
(False, 0.0),
],
)
def test_coerces(self, value: Any, expected: float) -> None:
assert _to_float(value) == expected
@pytest.mark.parametrize("value", ["abc", "1.2.3", [1, 2], {"a": 1}, object()])
def test_returns_none_for_non_numeric(self, value: Any) -> None:
assert _to_float(value) is None
# ---------------------------------------------------------------------------
# _evaluate_condition — structured form
# ---------------------------------------------------------------------------
class TestEvaluateConditionStructured:
def test_empty_condition_passes(self) -> None:
# No field, no expression → treated as "no gate".
assert _evaluate_condition(StepCondition(), {}) is True
def test_eq(self) -> None:
cond = StepCondition(field="severity", operator="eq", value="high")
assert _evaluate_condition(cond, {"severity": "high"}) is True
assert _evaluate_condition(cond, {"severity": "low"}) is False
def test_ne(self) -> None:
cond = StepCondition(field="severity", operator="ne", value="low")
assert _evaluate_condition(cond, {"severity": "high"}) is True
assert _evaluate_condition(cond, {"severity": "low"}) is False
def test_exists_true_for_present_value(self) -> None:
cond = StepCondition(field="user_id", operator="exists")
assert _evaluate_condition(cond, {"user_id": "u1"}) is True
def test_exists_false_for_missing_value(self) -> None:
cond = StepCondition(field="user_id", operator="exists")
assert _evaluate_condition(cond, {}) is False
def test_contains_substring(self) -> None:
cond = StepCondition(field="message", operator="contains", value="error")
assert _evaluate_condition(cond, {"message": "fatal error: boom"}) is True
assert _evaluate_condition(cond, {"message": "all good"}) is False
def test_contains_supports_list_membership(self) -> None:
# This is the SOAR-classic case: ``severity in ["high","critical"]``.
# Pre-fix this returned False because ``["high","critical"] in
# str(value)`` raised TypeError or always coerced to False.
cond = StepCondition(field="severity", operator="contains", value=["high", "critical"])
assert _evaluate_condition(cond, {"severity": "high"}) is True
assert _evaluate_condition(cond, {"severity": "critical"}) is True
assert _evaluate_condition(cond, {"severity": "low"}) is False
def test_contains_missing_field_is_false(self) -> None:
cond = StepCondition(field="severity", operator="contains", value=["high"])
assert _evaluate_condition(cond, {}) is False
def test_gt_lt_numeric(self) -> None:
gt = StepCondition(field="score", operator="gt", value=50)
lt = StepCondition(field="score", operator="lt", value=50)
assert _evaluate_condition(gt, {"score": 80}) is True
assert _evaluate_condition(gt, {"score": 20}) is False
assert _evaluate_condition(lt, {"score": 20}) is True
assert _evaluate_condition(lt, {"score": 80}) is False
def test_gt_lt_does_not_crash_on_non_numeric(self) -> None:
# Pre-fix: ``float("abc" or 0)`` → ValueError. Now: False.
cond = StepCondition(field="score", operator="gt", value=10)
assert _evaluate_condition(cond, {"score": "not-a-number"}) is False
cond = StepCondition(field="score", operator="lt", value="also-not-a-number")
assert _evaluate_condition(cond, {"score": 50}) is False
def test_gt_lt_missing_field_treated_as_zero(self) -> None:
# Historical "missing means zero" semantics for numeric comparisons.
cond = StepCondition(field="score", operator="gt", value=-1)
assert _evaluate_condition(cond, {}) is True
cond = StepCondition(field="score", operator="lt", value=1)
assert _evaluate_condition(cond, {}) is True
def test_unknown_operator_is_false(self) -> None:
# Pydantic's Literal blocks construction, so bypass with model_construct.
cond = StepCondition.model_construct(field="x", operator="weird", value=1)
assert _evaluate_condition(cond, {"x": 1}) is False
# ---------------------------------------------------------------------------
# _evaluate_condition — expression form (the silently-always-true regression)
# ---------------------------------------------------------------------------
class TestEvaluateConditionExpressionForm:
def test_expression_takes_precedence_over_blank_field(self) -> None:
# Pre-fix this branch was never taken — expression was ignored entirely
# and the (empty field, eq, None) form silently evaluated to True
# because ``None == None``.
cond = StepCondition(expression="severity == 'high'")
assert _evaluate_condition(cond, {"severity": "high"}) is True
assert _evaluate_condition(cond, {"severity": "low"}) is False
def test_expression_false_path_actually_fires(self) -> None:
cond = StepCondition(expression="score > 100")
assert _evaluate_condition(cond, {"score": 5}) is False
# ---------------------------------------------------------------------------
# _evaluate_expression
# ---------------------------------------------------------------------------
class TestEvaluateExpression:
@pytest.mark.parametrize(
"expr,ctx,expected",
[
("severity == 'high'", {"severity": "high"}, True),
('severity == "critical"', {"severity": "critical"}, True),
("severity != 'low'", {"severity": "high"}, True),
("score > 10", {"score": 50}, True),
("score >= 50", {"score": 50}, True),
("score <= 50", {"score": 50}, True),
("score < 10", {"score": 50}, False),
("severity in ['high','critical']", {"severity": "critical"}, True),
("severity in ['high','critical']", {"severity": "low"}, False),
("severity not in ['low','info']", {"severity": "high"}, True),
("severity not in ['low','info']", {"severity": "low"}, False),
],
)
def test_common_operators(self, expr: str, ctx: dict, expected: bool) -> None:
assert _evaluate_expression(expr, ctx) is expected
def test_is_null_sugar(self) -> None:
assert _evaluate_expression("user_id is null", {}) is True
assert _evaluate_expression("user_id is null", {"user_id": "u1"}) is False
def test_is_not_null_sugar(self) -> None:
assert _evaluate_expression("user_id is not null", {"user_id": "u1"}) is True
assert _evaluate_expression("user_id is not null", {}) is False
def test_equality_against_null_literal(self) -> None:
assert _evaluate_expression("user_id == null", {}) is True
assert _evaluate_expression("user_id != null", {"user_id": "u1"}) is True
def test_dot_path_resolution(self) -> None:
ctx = {"alert": {"severity": "high"}}
assert _evaluate_expression("alert.severity == 'high'", ctx) is True
def test_list_index_resolution(self) -> None:
ctx = {"entities": [{"name": "alice"}]}
assert _evaluate_expression("entities.0.name == 'alice'", ctx) is True
def test_numeric_literal_int_and_float(self) -> None:
assert _evaluate_expression("score == 42", {"score": 42}) is True
assert _evaluate_expression("score == 3.14", {"score": 3.14}) is True
def test_bool_literal(self) -> None:
assert _evaluate_expression("flag == true", {"flag": True}) is True
assert _evaluate_expression("flag == false", {"flag": False}) is True
def test_malformed_returns_false(self) -> None:
assert _evaluate_expression("this is not an expression", {}) is False
assert _evaluate_expression("", {}) is False
assert _evaluate_expression(None, {}) is False # type: ignore[arg-type]
def test_gt_on_non_numeric_returns_false(self) -> None:
assert _evaluate_expression("severity > 'low'", {"severity": "high"}) is False
# ---------------------------------------------------------------------------
# PlaybookEngine.run — context-flattening override behavior
# ---------------------------------------------------------------------------
def _make_playbook(steps: list[PlaybookStep]) -> Playbook:
return Playbook(name="test-pb", steps=steps)
@pytest.fixture(autouse=True)
def _silence_realtime(monkeypatch: pytest.MonkeyPatch) -> None:
"""Stub out ``_emit`` so engine.run() never tries to POST to the realtime
service during tests."""
monkeypatch.setattr(engine_mod, "_emit", AsyncMock(return_value=None))
@pytest.fixture
def patch_handlers():
"""Helper to temporarily inject handlers into ``_HANDLERS``."""
saved: dict = {}
def _install(mapping: dict) -> None:
for k, v in mapping.items():
# `setdefault`, not assignment: a second install of the same key
# inside one test would otherwise record the *first injected*
# handler as the original, and teardown would write that test
# double permanently into the process-wide `_HANDLERS`. Nothing
# installs twice today; the registry is shared by every test in
# the process, so it should not depend on nobody starting.
saved.setdefault(k, engine_mod._HANDLERS.get(k))
engine_mod._HANDLERS[k] = v
yield _install
for k, v in saved.items():
if v is None:
engine_mod._HANDLERS.pop(k, None)
else:
engine_mod._HANDLERS[k] = v
class TestEngineContextFlattening:
@pytest.mark.asyncio
async def test_later_step_can_override_earlier_flattened_value(self, patch_handlers) -> None:
"""An enrichment step should be able to refine a placeholder value set
by an earlier step. Pre-fix, ``setdefault`` froze the first writer."""
async def enrich_handler(step: PlaybookStep, ctx: dict, http: Any) -> dict:
# Second step "resolves" the user_id placeholder.
return {"user_id": "u-canonical-123"}
# First step writes a placeholder via its handler.
async def placeholder_handler(step: PlaybookStep, ctx: dict, http: Any) -> dict:
return {"user_id": "u-placeholder"}
patch_handlers(
{
StepType.HTTP: placeholder_handler,
StepType.ENRICH: enrich_handler,
}
)
pb = _make_playbook(
[
PlaybookStep(id="s1", name="placeholder", type=StepType.HTTP),
PlaybookStep(id="s2", name="enrich", type=StepType.ENRICH),
]
)
run = await PlaybookEngine().run(pb, trigger_context={})
assert run.status == RunStatus.COMPLETED
# The second step's value wins.
assert run.context["user_id"] == "u-canonical-123"
# Namespaced keys preserve per-step history.
assert run.context["_step_s1"]["user_id"] == "u-placeholder"
assert run.context["_step_s2"]["user_id"] == "u-canonical-123"
@pytest.mark.asyncio
async def test_underscore_keys_are_not_auto_flattened(self, patch_handlers) -> None:
async def handler(step: PlaybookStep, ctx: dict, http: Any) -> dict:
return {"public": "yes", "_private": "no"}
patch_handlers({StepType.HTTP: handler})
pb = _make_playbook([PlaybookStep(id="s1", name="h", type=StepType.HTTP)])
run = await PlaybookEngine().run(pb, trigger_context={})
assert run.context["public"] == "yes"
assert "_private" not in run.context # bookkeeping keys stay namespaced
assert run.context["_step_s1"]["_private"] == "no"
class TestEngineConditionGate:
"""Round-trip the expression-form fix through the engine to confirm a
falsy expression actually skips a step (previously it silently always
passed)."""
@pytest.mark.asyncio
async def test_expression_condition_skips_step_when_false(self, patch_handlers) -> None:
called = {"hit": False}
async def handler(step: PlaybookStep, ctx: dict, http: Any) -> dict:
called["hit"] = True
# `notify` is dispatched through the action registry since depth
# 5.1, and the engine reads `executed` rather than "the handler
# returned" — a double that answers `{"ok": True}` would be a
# double more capable than the real report.
return {"ok": True, "executed": True}
patch_handlers({StepType.NOTIFY: handler})
pb = _make_playbook(
[
PlaybookStep(
id="s1",
name="notify-on-critical",
type=StepType.NOTIFY,
condition=StepCondition(expression="severity == 'critical'"),
),
]
)
run = await PlaybookEngine().run(pb, trigger_context={"severity": "low"})
assert called["hit"] is False
assert run.step_results[0]["status"] == StepStatus.SKIPPED
@pytest.mark.asyncio
async def test_expression_condition_runs_step_when_true(self, patch_handlers) -> None:
called = {"hit": False}
async def handler(step: PlaybookStep, ctx: dict, http: Any) -> dict:
called["hit"] = True
# `notify` is dispatched through the action registry since depth
# 5.1, and the engine reads `executed` rather than "the handler
# returned" — a double that answers `{"ok": True}` would be a
# double more capable than the real report.
return {"ok": True, "executed": True}
patch_handlers({StepType.NOTIFY: handler})
pb = _make_playbook(
[
PlaybookStep(
id="s1",
name="notify-on-critical",
type=StepType.NOTIFY,
condition=StepCondition(expression="severity == 'critical'"),
),
]
)
run = await PlaybookEngine().run(pb, trigger_context={"severity": "critical"})
assert called["hit"] is True
assert run.step_results[0]["status"] == StepStatus.SUCCESS
# ---------------------------------------------------------------------------
# PlaybookEngine.run — step types the engine cannot run must not report success
# ---------------------------------------------------------------------------
#
# Twelve of the twenty-two StepType members had no entry in ``_HANDLERS``.
# The run loop answered those with ``{"skipped": True}`` and left
# ``step_status`` at its SUCCESS default, so a playbook containing them ran to
# RunStatus.COMPLETED having done nothing that it said it did.
#
# ``approval`` is the one that matters most: it is a human decision point, it
# appears in 14 steps across the shipped packs, and it passed on its own —
# the run continued straight into the action an analyst was meant to
# authorise. "Unverifiable means not autonomous" applies to a gate that
# cannot be evaluated as much as to an action that cannot be verified.
class TestUnimplementedStepTypesFailClosed:
@pytest.mark.asyncio
async def test_an_approval_gate_the_engine_cannot_honour_halts_the_run(self) -> None:
"""Fails against the pre-change tree, which reported SUCCESS/COMPLETED."""
pb = _make_playbook(
[
PlaybookStep(id="gate", name="Analyst approves isolation", type=StepType.APPROVAL),
PlaybookStep(id="act", name="Isolate the host", type=StepType.ISOLATE_HOST),
]
)
run = await PlaybookEngine().run(pb, trigger_context={})
assert run.step_results[0]["status"] == StepStatus.FAILED
assert run.step_results[0]["result"]["unimplemented"] is True
assert "approval" in run.step_results[0]["result"]["error"]
assert run.status == RunStatus.FAILED
# The whole point: the step behind the gate must not have run.
assert len(run.step_results) == 1, "execution continued past a gate that was never granted"
@pytest.mark.asyncio
async def test_no_step_type_reports_success_without_a_handler(self) -> None:
"""Every StepType with no handler, not just the one that hurt most."""
# Not in the handler table is not the same as not runnable. Three
# step types are answered in the run loop because they need the run
# itself (`_CONTROL_FLOW`), and `condition` branches in the loop
# body. Deriving from `_HANDLERS` alone made this test pick `wait`
# and assert that a working step reports `unimplemented`.
unimplemented = [
st for st in StepType if st not in engine_mod._HANDLERS and st not in engine_mod._CONTROL_FLOW and st != StepType.CONDITION
]
assert unimplemented, "fixture assumption broken: expected some StepType to lack a handler"
for step_type in unimplemented:
pb = _make_playbook([PlaybookStep(id="s1", name=f"step {step_type.value}", type=step_type)])
run = await PlaybookEngine().run(pb, trigger_context={})
result = run.step_results[0]
assert result["status"] == StepStatus.FAILED, f"{step_type.value} reported {result['status']} with no handler"
assert result["result"].get("unimplemented") is True
assert run.status == RunStatus.FAILED
@pytest.mark.asyncio
async def test_a_missing_handler_is_not_retried(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""A handler that does not exist will not exist next attempt either;
retrying just delays the failure by up to 2**retry_max seconds.
This test used to name ``RUN_AV_SCAN`` as its missing handler. That
stopped being true the moment the response verbs were wired into
``_HANDLERS`` through ``RESPONSE_STEP_TYPES``, so the step it built
went down the *retry* branch and was attempted four times over
fourteen seconds — the exact behaviour the docstring forbids — while
the test reported green. It could not notice, because its only
behavioural assertion was ``_elapsed_ms == 0``: ``t0`` is reset at the
top of every attempt, so that field is the duration of the final
attempt alone and never of the step, and it reads 0 both when the
engine skips the retry loop (which writes a literal 0) and when a
handler simply fails in under half a millisecond. The two cases the
test exists to tell apart were indistinguishable to it, which left a
rounded wall-clock reading as the only thing standing between the
suite and a regression.
So: take the step type from the registry rather than naming one, and
assert the property directly by recording the backoff sleeps.
"""
# Not in the handler table is not the same as not runnable. Three
# step types are answered in the run loop because they need the run
# itself (`_CONTROL_FLOW`), and `condition` branches in the loop
# body. Deriving from `_HANDLERS` alone made this test pick `wait`
# and assert that a working step reports `unimplemented`.
unimplemented = [
st for st in StepType if st not in engine_mod._HANDLERS and st not in engine_mod._CONTROL_FLOW and st != StepType.CONDITION
]
assert unimplemented, "fixture assumption broken: expected some StepType to lack a handler"
step_type = unimplemented[0]
# Delegate to a zero-delay sleep so a regression fails in
# milliseconds instead of sitting in 2+4+8s of real backoff.
slept: list[float] = []
real_sleep = asyncio.sleep
async def _recording_sleep(delay: float, *args: Any, **kwargs: Any) -> Any:
slept.append(delay)
return await real_sleep(0, *args, **kwargs)
monkeypatch.setattr(asyncio, "sleep", _recording_sleep)
pb = _make_playbook([PlaybookStep(id="s1", name="scan", type=step_type, retry_max=3)])
run = await PlaybookEngine().run(pb, trigger_context={})
result = run.step_results[0]["result"]
assert run.step_results[0]["status"] == StepStatus.FAILED
# `unimplemented` is written only by the branch that skips the retry
# loop, so this is what says which path ran — `_elapsed_ms` never did.
assert result["unimplemented"] is True
assert result["executed"] is False
assert not slept, f"a missing handler was retried with {slept} of backoff; retry_max was {3}"
@pytest.mark.asyncio
async def test_on_failure_continue_lets_the_run_proceed_without_calling_it_completed(self) -> None:
"""``continue`` decides whether the run keeps going, not what it is called.
346 of the 380 steps in the shipped packs carry ``on_failure:
continue``, so treating it as "report this run as completed" would put
a green tick over a containment that never happened — which is the
same fake success, moved up one level from the step to the run.
"""
pb = _make_playbook(
[
PlaybookStep(id="s1", name="best-effort scan", type=StepType.RUN_AV_SCAN, on_failure="continue"),
PlaybookStep(id="s2", name="second step", type=StepType.RUN_SCRIPT, on_failure="continue"),
]
)
run = await PlaybookEngine().run(pb, trigger_context={})
assert len(run.step_results) == 2, "the author said continue; both steps must be attempted"
assert all(r["status"] == StepStatus.FAILED for r in run.step_results)
assert run.status == RunStatus.FAILED
assert run.error and "2 of 2 steps failed" in run.error
@pytest.mark.asyncio
async def test_a_dry_run_says_which_steps_would_fail(self) -> None:
"""A dry run exists to tell the author what would happen. Reporting a
bare ``dry_run: true`` for a step that cannot execute would imply it
is fine."""
pb = _make_playbook(
[
PlaybookStep(id="s1", name="notify", type=StepType.NOTIFY),
PlaybookStep(id="s2", name="gate", type=StepType.APPROVAL),
]
)
run = await PlaybookEngine().run(pb, trigger_context={}, dry_run=True)
assert run.status == RunStatus.COMPLETED
assert run.step_results[0]["result"].get("unimplemented") is None
assert run.step_results[1]["result"]["unimplemented"] is True
assert run.step_results[1]["result"]["would_fail"] is True