ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.
Python active-directorycloud-securitydefensive-validationfastapigood-first-issuemitre-attackopsecreactred-teamreportingsecurity
"""
Step 2 - Unit Tests: Strategy Engine
Tests for the autonomous engagement orchestration layer:
- ModuleOutcome / RoundResult / EngagementResult (models)
- OutcomeKnowledgeBase (learning, success rates, per-vendor tracking)
- OperatorNotifier (message collection, callback dispatch)
- ConstitutionEnforcer (module blocking, scope check, authorization)
- StrategyEngine._check_goal_achieved (goal detection logic)
All tests use mock dependencies - no real LLM, network, or module execution.
Run: pytest tests/unit/test_strategy_engine.py -v
"""
from __future__ import annotations
import sys
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
_ROOT = Path(__file__).parent.parent.parent.resolve()
if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
def _c_live_strategy_engine_with(
coordinator: Any,
plan: tuple[dict[str, Any], ...],
):
from ares.db.execution_lifecycle import TrustedPrincipal
from ares.strategy.engine import StrategyEngine
engine = StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())
engine._configure_c_live_test_plan(
coordinator=coordinator,
principal=TrustedPrincipal(
"00000000-0000-4000-8000-000000000021",
"00000000-0000-4000-8000-000000000021",
),
idempotency_key="00000000-0000-4000-8000-000000000048",
plan=plan,
)
return engine
def _c_live_strategy_outcome(disposition: Any, *, status: str = "success") -> Any:
from ares.db.execution_lifecycle import FixedResult
return SimpleNamespace(
disposition=disposition,
identity=None,
lifecycle_result=FixedResult.APPLIED,
module_result=SimpleNamespace(status=status, findings=[]),
terminal_committed=disposition.value == "terminal",
)
@pytest.mark.asyncio
async def test_c_live_strategy_single_module_uses_coordinator():
from ares.core.execution_admission import DispatchDispositionV1
coordinator = SimpleNamespace(
execute_module=AsyncMock(
return_value=_c_live_strategy_outcome(DispatchDispositionV1.TERMINAL)
)
)
engine = _c_live_strategy_engine_with(
coordinator,
({"module_id": "ad.test", "params": {"target": "10.0.0.1"}},),
)
result = await engine.run_autonomous_engagement(SimpleNamespace(id="campaign-21"))
assert result.final_status == "terminal"
assert result.modules_succeeded == ["ad.test"]
coordinator.execute_module.assert_awaited_once()
request = coordinator.execute_module.await_args.args[1]
assert request.ingress_code == "strategy"
assert request.module_id == "ad.test"
@pytest.mark.asyncio
async def test_c_live_strategy_child_revalidates_after_principal_revocation():
from ares.core.execution_admission import DispatchDispositionV1
coordinator = SimpleNamespace(
execute_module=AsyncMock(
side_effect=[
_c_live_strategy_outcome(DispatchDispositionV1.TERMINAL),
_c_live_strategy_outcome(DispatchDispositionV1.NON_DISPATCHABLE),
]
)
)
engine = _c_live_strategy_engine_with(
coordinator,
(
{"module_id": "ad.first", "params": {}},
{"module_id": "ad.revoked", "params": {}},
),
)
result = await engine.run_autonomous_engagement(SimpleNamespace(id="campaign-48"))
assert result.final_status == "non_dispatchable"
assert coordinator.execute_module.await_count == 2
@pytest.mark.parametrize(
"helper_name",
["coverage-predictor", "edr-context", "ai-planner"],
ids=["coverage-predictor", "edr-context", "ai-planner"],
)
@pytest.mark.asyncio
async def test_c_live_strategy_helper_is_disabled_without_network(helper_name):
from ares.strategy.engine import StrategyEngine
engine = StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())
campaign = MagicMock()
if helper_name == "coverage-predictor":
value = await engine._run_coverage_predictor(campaign)
assert value["detection_score"] == 0.0
elif helper_name == "edr-context":
value = await engine._get_edr_context(campaign)
assert value["viable_techniques"] == []
else:
value = await engine._run_ai_planner(campaign, "goal", "local", "", False, {})
assert value["execution_plan"] == []
engine._engine.run_module.assert_not_called()
@pytest.mark.asyncio
async def test_c_live_strategy_parent_child_partial_completion_is_durable_and_not_redispatched():
from ares.core.execution_admission import DispatchDispositionV1
effects: dict[str, int] = {}
calls = 0
async def execute_module(principal, request, campaign):
nonlocal calls
del principal, campaign
calls += 1
if calls == 2:
return _c_live_strategy_outcome(DispatchDispositionV1.NON_DISPATCHABLE)
if effects.get(request.module_id):
return _c_live_strategy_outcome(DispatchDispositionV1.REPLAYED)
effects[request.module_id] = 1
return _c_live_strategy_outcome(DispatchDispositionV1.TERMINAL)
coordinator = SimpleNamespace(execute_module=execute_module)
engine = _c_live_strategy_engine_with(
coordinator,
(
{"module_id": "ad.first", "params": {}},
{"module_id": "ad.second", "params": {}},
),
)
campaign = SimpleNamespace(id="campaign-74")
first = await engine.run_autonomous_engagement(campaign)
second = await engine.run_autonomous_engagement(campaign)
assert first.final_status == "non_dispatchable"
assert second.final_status == "terminal"
assert effects == {"ad.first": 1, "ad.second": 1}
@pytest.mark.asyncio
async def test_c_live_test_plan_is_complete_before_first_module_effect():
from ares.core.execution_admission import DispatchDispositionV1
original_params = {"target": "10.0.0.75"}
engine_holder: dict[str, Any] = {}
async def execute_module(principal, request, campaign):
del principal, campaign
frozen = engine_holder["engine"]._c_live_test_plan_json
assert frozen is not None
assert '"ad.first"' in frozen
assert '"ad.second"' in frozen
if request.module_id == "ad.first":
assert request.raw_parameters == {"target": "10.0.0.75"}
return _c_live_strategy_outcome(DispatchDispositionV1.TERMINAL)
engine = _c_live_strategy_engine_with(
SimpleNamespace(execute_module=execute_module),
(
{"module_id": "ad.first", "params": original_params},
{"module_id": "ad.second", "params": {}},
),
)
engine_holder["engine"] = engine
original_params["target"] = "mutated-after-freeze"
result = await engine.run_autonomous_engagement(SimpleNamespace(id="campaign-75"))
assert result.final_status == "terminal"
# ═══════════════════════════════════════════════════════════════════════════════
# 1. Data Models
# ═══════════════════════════════════════════════════════════════════════════════
class TestModels:
"""Test strategy data model construction and defaults."""
def test_module_outcome_defaults(self):
from ares.strategy.models import ModuleOutcome
o = ModuleOutcome(module_id="ad.kerberoast", success=True)
assert o.module_id == "ad.kerberoast"
assert o.success is True
assert o.quality == 0.0
assert o.edr_vendor == "unknown"
assert o.findings_count == 0
assert o.timestamp > 0
def test_module_outcome_full(self):
from ares.strategy.models import ModuleOutcome
o = ModuleOutcome(
module_id="ad.dcsync", success=True, quality=0.95,
evidence="krbtgt hash", edr_vendor="crowdstrike",
bypass_used="amsi-patch", findings_count=3,
)
assert o.quality == 0.95
assert o.edr_vendor == "crowdstrike"
assert o.bypass_used == "amsi-patch"
def test_round_result_defaults(self):
from ares.strategy.models import RoundResult
r = RoundResult(
round_num=1, plan_confidence=0.85,
detection_score=0.3, modules_executed=["ad.kerberoast"],
outcomes=[],
)
assert r.goal_achieved is False
assert r.stopped_reason == ""
def test_engagement_result(self):
from ares.strategy.models import EngagementResult
e = EngagementResult(
goal="domain_admin", total_rounds=5, final_status="goal_achieved",
rounds=[], final_detection_score=0.45,
modules_succeeded=["ad.kerberoast", "ad.dcsync"],
modules_failed=["ad.adcs"],
knowledge_updates=12, elapsed_seconds=300.5,
)
assert e.total_rounds == 5
assert len(e.modules_succeeded) == 2
def test_detection_spike_error(self):
from ares.strategy.models import DetectionSpikeError
exc = DetectionSpikeError("Spike detected", spike=0.25, round_num=3)
assert exc.spike == 0.25
assert exc.round_num == 3
assert "Spike" in str(exc)
# ═══════════════════════════════════════════════════════════════════════════════
# 2. OutcomeKnowledgeBase
# ═══════════════════════════════════════════════════════════════════════════════
class TestKnowledgeBase:
"""Test per-session learning and success rate tracking."""
@pytest.fixture
def kb(self):
from ares.strategy.knowledge_base import OutcomeKnowledgeBase
return OutcomeKnowledgeBase()
def test_empty_kb_returns_empty_rates(self, kb):
rates = kb.get_success_rates()
assert rates == {}
def test_single_success_recorded(self, kb):
kb.record_outcome("ad.kerberoast", success=True, quality=1.0)
rates = kb.get_success_rates()
assert "ad.kerberoast" in rates
assert rates["ad.kerberoast"] == 1.0
def test_mixed_outcomes_rate(self, kb):
kb.record_outcome("ad.kerberoast", success=True, quality=1.0)
kb.record_outcome("ad.kerberoast", success=False, quality=0.0)
kb.record_outcome("ad.kerberoast", success=True, quality=0.8)
rates = kb.get_success_rates()
# total_quality = 1.0 + 0.0 + 0.8 = 1.8 / 3 attempts = 0.6
assert abs(rates["ad.kerberoast"] - 0.6) < 0.01
def test_per_vendor_tracking(self, kb):
kb.record_outcome("amsi-patch", success=True, edr_vendor="crowdstrike")
kb.record_outcome("amsi-patch", success=False, edr_vendor="defender")
kb.record_outcome("amsi-patch", success=True, edr_vendor="crowdstrike")
effective_cs = kb.get_effective_techniques("crowdstrike")
effective_def = kb.get_effective_techniques("defender")
assert "amsi-patch" in effective_cs
assert "amsi-patch" not in effective_def
def test_get_effective_unknown_vendor(self, kb):
kb.record_outcome("mod1", success=True, edr_vendor="sentinelone")
result = kb.get_effective_techniques("nonexistent")
assert result == []
def test_bypass_technique_tracking(self, kb):
kb.record_outcome("ad.dcsync", success=True, bypass_used="amsi-patch",
edr_vendor="crowdstrike")
kb.record_outcome("ad.dcsync", success=True, bypass_used="etw-blind",
edr_vendor="crowdstrike")
key = ("ad.dcsync", "crowdstrike")
assert "amsi-patch" in kb._rates[key]["bypass_techniques"]
assert "etw-blind" in kb._rates[key]["bypass_techniques"]
def test_multiple_modules_independent(self, kb):
kb.record_outcome("mod_a", success=True, quality=1.0)
kb.record_outcome("mod_b", success=False, quality=0.0)
rates = kb.get_success_rates()
assert rates["mod_a"] == 1.0
assert rates["mod_b"] == 0.0
def test_quality_weighted_rate(self, kb):
"""Quality weighting should affect the rate."""
kb.record_outcome("mod", success=True, quality=0.5)
kb.record_outcome("mod", success=True, quality=0.5)
rates = kb.get_success_rates()
# total_quality=1.0, attempts=2 → 0.5
assert rates["mod"] == 0.5
# ═══════════════════════════════════════════════════════════════════════════════
# 3. OperatorNotifier
# ═══════════════════════════════════════════════════════════════════════════════
class TestNotifier:
"""Test notification collection and callback dispatch."""
@pytest.mark.asyncio
async def test_messages_collected(self):
from ares.strategy.notifier import OperatorNotifier
n = OperatorNotifier()
await n.send("plan_ready", {"round": 1, "confidence": 0.9})
await n.send("detection_threshold_exceeded", {"score": 0.7})
assert len(n.messages) == 2
assert n.messages[0]["event"] == "plan_ready"
assert n.messages[1]["event"] == "detection_threshold_exceeded"
@pytest.mark.asyncio
async def test_callback_invoked(self):
from ares.strategy.notifier import OperatorNotifier
received = []
n = OperatorNotifier(notify_fn=lambda msg: received.append(msg))
await n.send("test_event", {"data": 123})
assert len(received) == 1
assert received[0]["event"] == "test_event"
assert received[0]["data"] == 123
@pytest.mark.asyncio
async def test_async_callback(self):
from ares.strategy.notifier import OperatorNotifier
received = []
async def callback(msg):
received.append(msg)
n = OperatorNotifier(notify_fn=callback)
await n.send("async_test", {"value": "ok"})
assert len(received) == 1
@pytest.mark.asyncio
async def test_callback_exception_does_not_crash(self):
from ares.strategy.notifier import OperatorNotifier
def bad_callback(msg):
raise RuntimeError("callback exploded")
n = OperatorNotifier(notify_fn=bad_callback)
await n.send("test", {}) # must not raise
assert len(n.messages) == 1 # message still collected
@pytest.mark.asyncio
async def test_timestamp_present(self):
from ares.strategy.notifier import OperatorNotifier
n = OperatorNotifier()
await n.send("test", {})
assert "ts" in n.messages[0]
assert n.messages[0]["ts"] > 0
# ═══════════════════════════════════════════════════════════════════════════════
# 4. ConstitutionEnforcer
# ═══════════════════════════════════════════════════════════════════════════════
class TestConstitutionEnforcer:
"""Test hard enforcement layer between LLM output and module execution."""
def _make_plan(self, stages):
"""Build a mock AIPlan."""
@dataclass
class FakePlan:
reasoning: str = "test"
stages: list = None
confidence: float = 0.9
warnings: list = None
def __post_init__(self):
if self.stages is None:
self.stages = []
if self.warnings is None:
self.warnings = []
return FakePlan(stages=stages)
def _make_campaign(self, scope_cidr="10.0.0.0/8"):
from ares.core.campaign import Campaign, NoiseProfile, ScopeEntry
return Campaign(
name="Test", scope=[ScopeEntry(cidr=scope_cidr)],
noise_profile=NoiseProfile.NORMAL,
)
def test_clean_plan_passes_through(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer(
authorizations=["ad.kerberoast"],
)
plan = self._make_plan([
{"name": "recon", "modules": ["ad.kerberoast", "ad.enum_users"]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) == 0
assert len(cleaned.stages) == 1
assert "ad.kerberoast" in cleaned.stages[0]["modules"]
def test_forbidden_module_removed(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer(
forbidden_modules={"ad.dcsync"},
)
plan = self._make_plan([
{"name": "attack", "modules": ["ad.kerberoast", "ad.dcsync"]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) == 1
assert violations[0].module_id == "ad.dcsync"
assert violations[0].severity == "HARD"
assert "ad.dcsync" not in cleaned.stages[0]["modules"]
assert "ad.kerberoast" in cleaned.stages[0]["modules"]
def test_unauthorized_high_risk_module_blocked(self):
from ares.strategy.enforcer import ALWAYS_REQUIRE_AUTH, ConstitutionEnforcer
enforcer = ConstitutionEnforcer(authorizations=[])
# Pick a module from ALWAYS_REQUIRE_AUTH
auth_module = sorted(ALWAYS_REQUIRE_AUTH)[0]
plan = self._make_plan([
{"name": "escalate", "modules": [auth_module]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) >= 1
assert any(v.module_id == auth_module for v in violations)
def test_authorized_high_risk_module_passes(self):
from ares.strategy.enforcer import ALWAYS_REQUIRE_AUTH, ConstitutionEnforcer
auth_module = sorted(ALWAYS_REQUIRE_AUTH)[0]
enforcer = ConstitutionEnforcer(
authorizations=[auth_module],
)
plan = self._make_plan([
{"name": "escalate", "modules": [auth_module]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) == 0
assert auth_module in cleaned.stages[0]["modules"]
def test_out_of_scope_target_blocked(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer()
plan = self._make_plan([
{"name": "lateral", "modules": ["lateral.smb_relay"],
"params": {"lateral.smb_relay": {"target": "192.168.1.1"}}},
])
campaign = self._make_campaign("10.0.0.0/8")
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) >= 1
assert any("out of campaign scope" in v.reason for v in violations)
def test_in_scope_target_passes(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer()
plan = self._make_plan([
{"name": "lateral", "modules": ["lateral.smb_relay"],
"params": {"lateral.smb_relay": {"target": "10.0.0.5"}}},
])
campaign = self._make_campaign("10.0.0.0/8")
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) == 0
def test_persistence_blocked_without_flag(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer(allow_persistence=False)
plan = self._make_plan([
{"name": "persist", "modules": ["persistence.scheduled_task"]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) >= 1
assert any("persistence" in v.reason.lower() for v in violations)
def test_persistence_allowed_with_flag(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer(
allow_persistence=True,
authorizations=["persistence.scheduled_task"],
)
plan = self._make_plan([
{"name": "persist", "modules": ["persistence.scheduled_task"]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(violations) == 0
def test_all_modules_removed_yields_empty_stages(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer(
forbidden_modules={"mod_a", "mod_b"},
)
plan = self._make_plan([
{"name": "attack", "modules": ["mod_a", "mod_b"]},
])
campaign = self._make_campaign()
cleaned, violations = enforcer.enforce(plan, campaign)
assert len(cleaned.stages) == 0
assert len(violations) == 2
def test_describe_returns_config(self):
from ares.strategy.enforcer import ConstitutionEnforcer
enforcer = ConstitutionEnforcer(
authorizations=["ad.dcsync"],
forbidden_modules={"evil.module"},
)
desc = enforcer.describe()
assert "ad.dcsync" in desc["authorized_modules"]
assert "evil.module" in desc["forbidden_modules"]
# ═══════════════════════════════════════════════════════════════════════════════
# 5. Goal Achievement Detection
# ═══════════════════════════════════════════════════════════════════════════════
class TestGoalAchieved:
"""Test _check_goal_achieved logic with mock campaigns."""
def _make_engine(self):
from ares.strategy.engine import StrategyEngine
mock_ares = MagicMock()
mock_settings = MagicMock()
return StrategyEngine(ares_engine=mock_ares, settings=mock_settings)
def _make_campaign_with_findings(self, titles):
campaign = MagicMock()
findings = []
for t in titles:
f = MagicMock()
f.title = t
findings.append(f)
campaign.findings = findings
return campaign
def test_domain_admin_achieved_via_dcsync(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings([
"Port Scan Complete",
"DCSync Attack Successful - krbtgt hash obtained",
])
assert engine._check_goal_achieved(campaign, "domain_admin") is True
def test_domain_admin_not_achieved(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings([
"Port Scan Complete",
"Kerberoast: 3 hashes captured",
])
assert engine._check_goal_achieved(campaign, "domain_admin") is False
def test_cloud_admin_achieved(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings([
"AWS Admin Role Assumed",
])
assert engine._check_goal_achieved(campaign, "cloud_admin") is True
def test_data_exfil_achieved(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings([
"Sensitive File Found: credentials.xlsx",
])
assert engine._check_goal_achieved(campaign, "data_exfil") is True
def test_unknown_goal_returns_false(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings(["DCSync"])
assert engine._check_goal_achieved(campaign, "nonexistent_goal") is False
def test_empty_findings_returns_false(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings([])
assert engine._check_goal_achieved(campaign, "domain_admin") is False
def test_case_insensitive_matching(self):
engine = self._make_engine()
campaign = self._make_campaign_with_findings([
"dcsync attack completed", # lowercase
])
assert engine._check_goal_achieved(campaign, "domain_admin") is True
def test_dict_findings_supported(self):
"""_check_goal_achieved should handle dict findings (from DB)."""
engine = self._make_engine()
campaign = MagicMock()
campaign.findings = [
{"title": "DCSync Attack - Domain Admin Achieved", "validated": True, "false_positive": False},
]
mock_vault = MagicMock()
mock_vault.all.return_value = [
{"cred_type": "ntlm", "privilege": "domain_admin", "username": "krbtgt"}
]
campaign.vault = mock_vault
assert engine._check_goal_achieved(campaign, "domain_admin") is True
# ═══════════════════════════════════════════════════════════════════════════════
# 6. Strategy Module Re-exports
# ═══════════════════════════════════════════════════════════════════════════════
class TestReExports:
"""Verify strategy/__init__.py re-exports all public symbols."""
def test_all_exports_importable(self):
from ares.strategy import (
DetectionSpikeError,
EngagementResult,
ModuleOutcome,
OperatorNotifier,
OutcomeKnowledgeBase,
RoundResult,
StrategyEngine,
)
assert StrategyEngine is not None
assert ModuleOutcome is not None
assert RoundResult is not None
assert EngagementResult is not None
assert DetectionSpikeError is not None
assert OutcomeKnowledgeBase is not None
assert OperatorNotifier is not None