ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.
Python active-directorycloud-securitydefensive-validationfastapigood-first-issuemitre-attackopsecreactred-teamreportingsecurity
fix(security): Gate 5 - goal indicator vault verification (SUCCESS_MECHANISM_AUDIT.md)
Mafifrizifb4dc3fa3768
diff --git a/ares/strategy/engine.py b/ares/strategy/engine.pyindex 6e0684b..d2c2d9b 100644--- a/ares/strategy/engine.py+++ b/ares/strategy/engine.py@@ -891,9 +891,20 @@ class StrategyEngine:]return "\n".join(lines)- def _check_goal_achieved(self, campaign: "Any", goal: str) -> bool:- """Check if the engagement goal has been achieved via campaign artifacts."""- findings = getattr(campaign, "findings", [])+ def _check_goal_achieved(+ self,+ campaign: "Any",+ goal: str,+ confirmed_findings: list[Any] | None = None,+ ) -> bool:+ """+ Check if the engagement goal has been achieved via verified campaign artifacts.++ Gate 5 Enforcement:+ - Layer 1: Title matching ONLY against confirmed findings (passed Gate 3).+ - Layer 2: For privilege goals (domain_admin, credential_access, root_access, full_compromise),+ supporting credentials must actually exist in the Campaign Vault.+ """goal_indicators = {"domain_admin": ["DCSync", "Domain Admin", "krbtgt", "ntlm_hash"],"cloud_admin": ["Global Admin", "AWS Admin", "Storage Admin", "GCP Owner"],@@ -905,9 +916,112 @@ class StrategyEngine:if not indicators:return False- for finding in findings:+ # ── Layer 1: Filter strictly to confirmed findings ─────────────────+ if confirmed_findings is not None:+ active_findings = confirmed_findings+ else:+ all_findings = getattr(campaign, "findings", []) or []+ active_findings = []+ for f in all_findings:+ if isinstance(f, dict):+ is_val = bool(f.get("validated", False))+ is_fp = bool(f.get("false_positive", False))+ else:+ val_attr = getattr(f, "validated", False)+ fp_attr = getattr(f, "false_positive", False)+ is_val = bool(val_attr)+ is_fp = bool(fp_attr) if isinstance(fp_attr, bool) else False+ if is_val and not is_fp:+ active_findings.append(f)++ title_match = False+ matching_title = ""+ for finding in active_findings:title = (getattr(finding, "title", "") or(finding.get("title", "") if isinstance(finding, dict) else ""))if any(ind.lower() in title.lower() for ind in indicators):- return True+ title_match = True+ matching_title = title+ break++ if not title_match:+ return False++ # ── Layer 2: Privilege goals require verified credentials in vault ─+ privilege_goals = {"domain_admin", "credential_access", "root_access", "full_compromise"}+ if goal in privilege_goals:+ vault = (+ getattr(campaign, "vault", None)+ or getattr(campaign, "_vault", None)+ or (+ getattr(self._engine, "_runtime_states", None)+ and getattr(self._engine._runtime_states.get(getattr(campaign, "id", "")), "vault", None)+ )+ or getattr(self, "_vault", None)+ )++ has_creds = self._verify_vault_evidence(vault, goal)+ if not has_creds:+ logger.info(+ "goal_not_achieved_no_vault_evidence",+ goal_type=goal,+ title_matches=True,+ matching_title=matching_title,+ vault_has_creds=False,+ )+ return False++ return True++ def _verify_vault_evidence(self, vault: Any, goal: str) -> bool:+ """Verify that vault contains credentials supporting the goal."""+ if not vault:+ return False++ # Try domain_admins() method if present+ if goal in ("domain_admin", "full_compromise"):+ if hasattr(vault, "domain_admins") and callable(vault.domain_admins):+ da_creds = vault.domain_admins()+ if da_creds:+ return True++ # Extract credentials collection from vault+ creds: list[Any] = []+ if hasattr(vault, "all") and callable(vault.all):+ creds = vault.all()+ elif hasattr(vault, "_store") and isinstance(vault._store, dict):+ creds = list(vault._store.values())+ elif hasattr(vault, "has_credentials") and callable(vault.has_credentials):+ return bool(vault.has_credentials(min_privilege=goal))+ elif isinstance(vault, dict):+ creds = vault.get("credentials") or list(vault.values())+ elif isinstance(vault, (list, tuple)):+ creds = list(vault)++ if not creds:+ return False++ # Check privilege level of credentials+ for c in creds:+ if isinstance(c, dict):+ priv = str(c.get("privilege", "")).lower()+ user = str(c.get("username", "")).lower()+ ctype = str(c.get("cred_type", "")).lower()+ else:+ p_attr = getattr(c, "privilege", "")+ priv = p_attr.value.lower() if hasattr(p_attr, "value") else str(p_attr or "").lower()+ user = str(getattr(c, "username", "") or "").lower()+ ct_attr = getattr(c, "cred_type", "")+ ctype = ct_attr.value.lower() if hasattr(ct_attr, "value") else str(ct_attr or "").lower()++ if goal in ("domain_admin", "full_compromise"):+ if priv in ("domain_admin", "enterprise_admin") or "krbtgt" in user:+ return True+ # Fallback: any valid non-unknown credential if privilege not set+ if ctype and ctype not in ("unknown", ""):+ return True+ else:+ if (ctype and ctype not in ("unknown", "")) or (priv and priv not in ("unknown", "")):+ return True+return Falsediff --git a/tests/unit/test_honesty_gates.py b/tests/unit/test_honesty_gates.pynew file mode 100644index 0000000..9989c70--- /dev/null+++ b/tests/unit/test_honesty_gates.py@@ -0,0 +1,402 @@+"""+Tests for Gate 3 (Engine Validator Enforcement) and Gate 5 (Goal Indicator Verifier).+Verifies fail-closed enforcement preventing phantom exploits, fake findings, and premature goal achievement.+"""+from __future__ import annotations++import pytest+from unittest.mock import MagicMock, AsyncMock++from ares.core.config import AresSettings+from ares.core.campaign import Campaign, Finding, Severity, ScopeEntry+from ares.core.engine import AresEngine, EngineModuleResult, ModuleStatus+from ares.core.validator import (+ FindingValidator,+ ValidationCheck,+ ValidationStage,+ ValidationResult,+ build_default_validator,+)+from ares.strategy.engine import StrategyEngine+from ares.credential.vault import CredentialVault, Credential, CredentialType, PrivilegeLevel+++# ═══════════════════════════════════════════════════════════════════════════════+# Helper Fixtures & Factories+# ═══════════════════════════════════════════════════════════════════════════════++from ares.core.campaign import Campaign, Finding, Severity, ScopeEntry, NoiseProfile++def _make_campaign() -> Campaign:+ return Campaign(+ name="honesty-gate-test",+ client="test-client",+ operator="test-operator",+ scope=[ScopeEntry(cidr="10.0.0.0/24")],+ targets=["10.0.0.5"],+ noise_profile=NoiseProfile.NORMAL,+ )++def _make_settings() -> AresSettings:+ return AresSettings(+ environment="test",+ encryption_master_key="0" * 64,+ jwt_secret_key="0" * 32,+ )+++# ═══════════════════════════════════════════════════════════════════════════════+# GATE 3: Engine Validator Fail-Closed Enforcement Tests+# ═══════════════════════════════════════════════════════════════════════════════++class TestGate3ValidatorEnforcement:+ """Verifies that AresEngine enforces validator results fail-closed."""++ @pytest.mark.asyncio+ async def test_high_confidence_finding_confirmed(self):+ """Valid finding with high confidence (>= 0.4) is confirmed."""+ validator = FindingValidator()+ validator.register("test.valid", [+ ValidationCheck(+ stage=ValidationStage.EXPLOITABLE,+ name="pass_check",+ check=AsyncMock(return_value=(True, 0.9, "verified")),+ )+ ])+ engine = AresEngine(settings=_make_settings(), validator=validator)+ campaign = _make_campaign()++ finding = Finding(+ title="Verified Vulnerability",+ description="High confidence evidence",+ severity=Severity.HIGH,+ module_id="test.valid",+ )++ val_res = await validator.validate(finding, {})+ assert val_res.should_report is True+ assert val_res.confidence >= 0.4+ assert finding.validated is True+ assert finding.false_positive is False++ @pytest.mark.asyncio+ async def test_low_confidence_finding_rejected_by_validator(self):+ """Finding with confidence < 0.4 is rejected and marked as false positive."""+ validator = FindingValidator()+ validator.register("test.fake", [+ ValidationCheck(+ stage=ValidationStage.EXPLOITABLE,+ name="fail_check",+ check=AsyncMock(return_value=(False, 0.2, "insufficient evidence")),+ )+ ])+ engine = AresEngine(settings=_make_settings(), validator=validator)++ finding = Finding(+ title="Phantom Exploit Finding",+ description="No real evidence behind this",+ severity=Severity.CRITICAL,+ module_id="test.fake",+ )++ val_res = await validator.validate(finding, {})+ assert val_res.should_report is False+ assert val_res.confidence < 0.4+ assert finding.false_positive is True+ assert finding.validated is False++ @pytest.mark.asyncio+ async def test_unregistered_module_finding_passes_with_default_confidence(self):+ """Finding from an unregistered module passes with default 0.6 confidence (0.6 >= 0.4)."""+ validator = FindingValidator() # empty registry+ engine = AresEngine(settings=_make_settings(), validator=validator)++ finding = Finding(+ title="Unregistered Module Finding",+ description="Module not in registry yet",+ severity=Severity.MEDIUM,+ module_id="community.new_scanner",+ )++ val_res = await validator.validate(finding, {})+ assert val_res.confidence == 0.6+ assert val_res.should_report is True+ assert finding.validated is True+ assert finding.false_positive is False++ @pytest.mark.asyncio+ async def test_lateral_evidence_target_only_rejected(self):+ """Lateral movement finding with only target host (MOD-021 pattern) scores 0.2 and is rejected."""+ validator = build_default_validator()++ rdp_finding = Finding(+ title="Lateral Movement: RDP -> 10.0.0.5",+ description="Port 3389 was open, claiming lateral movement",+ severity=Severity.HIGH,+ module_id="lateral.rdp",+ host="10.0.0.5",+ evidence={"target": "10.0.0.5"}, # target only! No command output, no session+ )++ val_res = await validator.validate(rdp_finding, {})+ assert val_res.confidence == 0.2+ assert val_res.passed is False+ assert val_res.should_report is False+ assert rdp_finding.false_positive is True+ assert rdp_finding.validated is False++ @pytest.mark.asyncio+ async def test_lateral_evidence_target_plus_execution_confirmed(self):+ """Lateral movement finding with target + command output scores 0.8 and is confirmed."""+ validator = build_default_validator()++ psexec_finding = Finding(+ title="Lateral Movement: lateral.psexec -> 10.0.0.5",+ description="Moved laterally and executed whoami",+ severity=Severity.CRITICAL,+ module_id="lateral.psexec",+ host="10.0.0.5",+ evidence={+ "target": "10.0.0.5",+ "command_output": "nt authority\\system",+ "technique": "psexec",+ },+ )++ val_res = await validator.validate(psexec_finding, {})+ assert val_res.confidence == 0.8+ assert val_res.passed is True+ assert val_res.should_report is True+ assert psexec_finding.validated is True+ assert psexec_finding.false_positive is False++ @pytest.mark.asyncio+ async def test_engine_execution_filters_unvalidated_findings(self):+ """Engine execution loop rejects findings where validator returned confidence < 0.4."""+ from ares.modules.base import ModuleResult+ from unittest.mock import patch++ validator = FindingValidator()+ validator.register("test.mixed", [+ ValidationCheck(+ stage=ValidationStage.EXPLOITABLE,+ name="selective_check",+ check=lambda finding, context: (+ AsyncMock(return_value=(True, 0.9, "good"))()+ if "Valid" in finding.title+ else AsyncMock(return_value=(False, 0.1, "bad"))()+ ),+ )+ ])+ engine = AresEngine(settings=_make_settings(), validator=validator)+ campaign = _make_campaign()++ f_good = Finding(title="Valid Finding", description="Passes", severity=Severity.HIGH, module_id="test.mixed")+ f_bad = Finding(title="Fake Finding", description="Fails", severity=Severity.CRITICAL, module_id="test.mixed")++ # Mock the dispatch and execution to reach the engine validation loop directly+ from ares.core.execution_admission import _mint_test_dispatch_context+ from ares.modules.ad.kerberoast import KerberoastModule++ engine.load_modules()+ # Register a mock module that produces both findings+ async def fake_exec(self_unused, ctx):+ return ModuleResult(+ status="success",+ findings=[f_good, f_bad],+ raw={"test": True},+ module_id="test.mixed",+ )++ with patch.object(KerberoastModule, "execute", fake_exec):+ # Temporarily register test.mixed validator on kerberoast+ validator.register("ad.kerberoast", validator._registry["test.mixed"])+ result = await engine.run_module(+ "ad.kerberoast",+ campaign,+ {"dc": "10.0.0.5", "domain": "corp.local", "username": "u", "password": "p", "target_user": "t"},+ actor_role="team_lead",+ dispatch_context=_mint_test_dispatch_context(engine, campaign.id, "ad.kerberoast"),+ )++ # Gate 3 enforcement: only f_good is confirmed, f_bad is rejected+ assert result.status == ModuleStatus.DONE+ assert len(result.findings) == 1+ assert result.findings[0].title == "Valid Finding"+ assert result.findings[0].validated is True+ assert f_bad.false_positive is True+ assert f_bad.validated is False+++# ═══════════════════════════════════════════════════════════════════════════════+# GATE 5: Goal Indicator Verifier Tests+# ═══════════════════════════════════════════════════════════════════════════════++class TestGate5GoalVerifier:+ """Verifies that StrategyEngine._check_goal_achieved requires confirmed findings and vault evidence."""++ def _make_strategy_engine(self):+ return StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())++ def test_goal_not_achieved_if_finding_not_confirmed(self):+ """Title matches 'DCSync' but finding is unvalidated or marked false positive -> goal fails."""+ engine = self._make_strategy_engine()+ campaign = _make_campaign()++ # Finding with matching title, but false_positive=True, validated=False+ f = Finding(+ title="DCSync Attack Successful",+ description="Fake DCSync",+ severity=Severity.CRITICAL,+ )+ f.false_positive = True+ f.validated = False+ campaign.findings = [f]++ # Even with vault credentials present, unvalidated finding must block goal+ mock_vault = MagicMock()+ mock_vault.all.return_value = [+ MagicMock(cred_type="ntlm", privilege="domain_admin", username="krbtgt")+ ]+ object.__setattr__(campaign, "_vault", mock_vault)++ assert engine._check_goal_achieved(campaign, "domain_admin") is False++ def test_goal_not_achieved_if_confirmed_finding_but_vault_empty(self):+ """Finding is confirmed, but vault has no credentials -> privilege goal fails."""+ engine = self._make_strategy_engine()+ campaign = _make_campaign()++ # Finding passed Gate 3+ f = Finding(+ title="DCSync Attack - krbtgt hash obtained",+ description="Real DCSync finding",+ severity=Severity.CRITICAL,+ )+ f.validated = True+ f.false_positive = False+ campaign.findings = [f]++ # Vault is empty+ mock_vault = MagicMock()+ mock_vault.all.return_value = []+ mock_vault.domain_admins.return_value = []+ object.__setattr__(campaign, "_vault", mock_vault)++ assert engine._check_goal_achieved(campaign, "domain_admin") is False++ def test_goal_achieved_when_confirmed_finding_and_vault_creds_exist(self):+ """Finding is confirmed AND vault has supporting credentials -> goal succeeds."""+ engine = self._make_strategy_engine()+ campaign = _make_campaign()++ f = Finding(+ title="DCSync Attack - krbtgt hash obtained",+ description="Real DCSync finding",+ severity=Severity.CRITICAL,+ )+ f.validated = True+ f.false_positive = False+ campaign.findings = [f]++ # Real CredentialVault with a stored Domain Admin credential+ vault = CredentialVault(encryption_key="test-key-32-chars-long-secret!!")+ cred = Credential(+ username="krbtgt",+ domain="CORP",+ cred_type=CredentialType.NTLM,+ privilege=PrivilegeLevel.DOMAIN_ADMIN,+ )+ vault.store(cred, "aad3b435b51404eeaad3b435b51404ee:31d6cfe0d16ae931b73c59d7e0c089c0")+ object.__setattr__(campaign, "_vault", vault)++ assert engine._check_goal_achieved(campaign, "domain_admin") is True++ def test_non_privilege_goal_does_not_require_vault_credentials(self):+ """Non-privilege goals like data_exfil only require confirmed findings, not vault credentials."""+ engine = self._make_strategy_engine()+ campaign = _make_campaign()++ f = Finding(+ title="Sensitive File Exfiltrated: financials.xlsx",+ description="Data exfil confirmed",+ severity=Severity.HIGH,+ )+ f.validated = True+ f.false_positive = False+ campaign.findings = [f]+ # No vault attached at all+ assert engine._check_goal_achieved(campaign, "data_exfil") is True+++# ═══════════════════════════════════════════════════════════════════════════════+# INTEGRATION: MOD-005 and MOD-021 Simulation Tests+# ═══════════════════════════════════════════════════════════════════════════════++class TestGate3And5IntegrationSimulations:+ """End-to-end integration scenarios verifying defense against MOD-005 and MOD-021 failure modes."""++ @pytest.mark.asyncio+ async def test_mod_005_ghost_forge_simulation(self):+ """+ Simulation of MOD-005 (ghost_forge):+ Module produces CRITICAL 'DCSync Success' finding without network I/O or valid evidence.+ 1. Gate 3 rejects the finding (confidence too low / no execution evidence).+ 2. Finding is excluded from confirmed and marked false positive.+ 3. Gate 5 verifies that domain_admin goal is NOT achieved.+ """+ validator = build_default_validator()+ strategy_engine = StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())+ campaign = _make_campaign()++ # Simulated ghost_forge finding with empty evidence (no captured hashes, no I/O)+ fake_finding = Finding(+ title="DCSync Attack - krbtgt hash obtained",+ description="Claimed DCSync without real network execution",+ severity=Severity.CRITICAL,+ module_id="ad.kerberoast", # registered AD validator+ evidence={"hash_count": 0}, # 0 hashes captured!+ )++ # Gate 3: validate+ val_res = await validator.validate(fake_finding, {})+ assert val_res.confidence < 0.4+ assert val_res.should_report is False+ assert fake_finding.false_positive is True+ assert fake_finding.validated is False++ # Attempt to achieve goal with this rejected finding+ campaign.findings = [fake_finding]+ assert strategy_engine._check_goal_achieved(campaign, "domain_admin") is False++ @pytest.mark.asyncio+ async def test_mod_021_rdp_port_open_simulation(self):+ """+ Simulation of MOD-021 (RDP port open false lateral movement):+ Module produces finding 'Lateral Movement via RDP' simply because port 3389 was open.+ 1. Gate 3 blocks finding (target only = 0.2 < 0.4).+ 2. Gate 5 rejects goal achievement because finding is unvalidated and vault has no creds.+ """+ validator = build_default_validator()+ strategy_engine = StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())+ campaign = _make_campaign()++ fake_lateral_finding = Finding(+ title="Lateral Movement via RDP: 10.0.0.5",+ description="Port 3389 open - no session established",+ severity=Severity.HIGH,+ module_id="lateral.rdp",+ host="10.0.0.5",+ evidence={"target": "10.0.0.5", "port": 3389},+ )++ # Gate 3: validate+ val_res = await validator.validate(fake_lateral_finding, {})+ assert val_res.confidence == 0.2+ assert val_res.should_report is False+ assert fake_lateral_finding.false_positive is True+ assert fake_lateral_finding.validated is False++ # Gate 5: goal not achieved+ campaign.findings = [fake_lateral_finding]+ assert strategy_engine._check_goal_achieved(campaign, "domain_admin") is Falsediff --git a/tests/unit/test_strategic_modules.py b/tests/unit/test_strategic_modules.pyindex 48bc391..482033a 100644--- a/tests/unit/test_strategic_modules.py+++ b/tests/unit/test_strategic_modules.py@@ -912,12 +912,18 @@ class TestStrategyEngine:assert messages[0]["data"] == 42def test_check_goal_achieved_domain_admin(self):+ from unittest.mock import MagicMockfrom ares.strategy import StrategyEngineengine = StrategyEngine(ares_engine=None, settings=None)campaign = _mock_campaign()- # Simulate a DCSync finding- mock_finding = type("F", (), {"title": "DCSync - krbtgt hash obtained"})()+ # Simulate a DCSync finding that passed Gate 3+ mock_finding = type("F", (), {"title": "DCSync - krbtgt hash obtained", "validated": True, "false_positive": False})()campaign.findings = [mock_finding]+ mock_vault = MagicMock()+ mock_vault.all.return_value = [+ MagicMock(cred_type="ntlm", privilege="domain_admin", username="krbtgt")+ ]+ object.__setattr__(campaign, "_vault", mock_vault)assert engine._check_goal_achieved(campaign, "domain_admin") is Truedef test_check_goal_not_achieved_empty(self):diff --git a/tests/unit/test_strategy_engine.py b/tests/unit/test_strategy_engine.pyindex 99474cf..09ae147 100644--- a/tests/unit/test_strategy_engine.py+++ b/tests/unit/test_strategy_engine.py@@ -616,8 +616,13 @@ class TestGoalAchieved:engine = self._make_engine()campaign = MagicMock()campaign.findings = [- {"title": "DCSync Attack - Domain Admin Achieved"},+ {"title": "DCSync Attack - Domain Admin Achieved", "validated": True, "false_positive": False},]+ mock_vault = MagicMock()+ mock_vault.all.return_value = [+ {"cred_type": "ntlm", "privilege": "domain_admin", "username": "krbtgt"}+ ]+ campaign.vault = mock_vaultassert engine._check_goal_achieved(campaign, "domain_admin") is True