Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

fix(security): Gate 5 - goal indicator vault verification (SUCCESS_MECHANISM_AUDIT.md)

Mafifrizifb4dc3fa3768
diff --git a/ares/strategy/engine.py b/ares/strategy/engine.py
index 6e0684b..d2c2d9b 100644
--- a/ares/strategy/engine.py
+++ b/ares/strategy/engine.py
@@ -891,9 +891,20 @@ class StrategyEngine:
]
return "\n".join(lines)
- def _check_goal_achieved(self, campaign: "Any", goal: str) -> bool:
- """Check if the engagement goal has been achieved via campaign artifacts."""
- findings = getattr(campaign, "findings", [])
+ def _check_goal_achieved(
+ self,
+ campaign: "Any",
+ goal: str,
+ confirmed_findings: list[Any] | None = None,
+ ) -> bool:
+ """
+ Check if the engagement goal has been achieved via verified campaign artifacts.
+
+ Gate 5 Enforcement:
+ - Layer 1: Title matching ONLY against confirmed findings (passed Gate 3).
+ - Layer 2: For privilege goals (domain_admin, credential_access, root_access, full_compromise),
+ supporting credentials must actually exist in the Campaign Vault.
+ """
goal_indicators = {
"domain_admin": ["DCSync", "Domain Admin", "krbtgt", "ntlm_hash"],
"cloud_admin": ["Global Admin", "AWS Admin", "Storage Admin", "GCP Owner"],
@@ -905,9 +916,112 @@ class StrategyEngine:
if not indicators:
return False
- for finding in findings:
+ # ── Layer 1: Filter strictly to confirmed findings ─────────────────
+ if confirmed_findings is not None:
+ active_findings = confirmed_findings
+ else:
+ all_findings = getattr(campaign, "findings", []) or []
+ active_findings = []
+ for f in all_findings:
+ if isinstance(f, dict):
+ is_val = bool(f.get("validated", False))
+ is_fp = bool(f.get("false_positive", False))
+ else:
+ val_attr = getattr(f, "validated", False)
+ fp_attr = getattr(f, "false_positive", False)
+ is_val = bool(val_attr)
+ is_fp = bool(fp_attr) if isinstance(fp_attr, bool) else False
+ if is_val and not is_fp:
+ active_findings.append(f)
+
+ title_match = False
+ matching_title = ""
+ for finding in active_findings:
title = (getattr(finding, "title", "") or
(finding.get("title", "") if isinstance(finding, dict) else ""))
if any(ind.lower() in title.lower() for ind in indicators):
- return True
+ title_match = True
+ matching_title = title
+ break
+
+ if not title_match:
+ return False
+
+ # ── Layer 2: Privilege goals require verified credentials in vault ─
+ privilege_goals = {"domain_admin", "credential_access", "root_access", "full_compromise"}
+ if goal in privilege_goals:
+ vault = (
+ getattr(campaign, "vault", None)
+ or getattr(campaign, "_vault", None)
+ or (
+ getattr(self._engine, "_runtime_states", None)
+ and getattr(self._engine._runtime_states.get(getattr(campaign, "id", "")), "vault", None)
+ )
+ or getattr(self, "_vault", None)
+ )
+
+ has_creds = self._verify_vault_evidence(vault, goal)
+ if not has_creds:
+ logger.info(
+ "goal_not_achieved_no_vault_evidence",
+ goal_type=goal,
+ title_matches=True,
+ matching_title=matching_title,
+ vault_has_creds=False,
+ )
+ return False
+
+ return True
+
+ def _verify_vault_evidence(self, vault: Any, goal: str) -> bool:
+ """Verify that vault contains credentials supporting the goal."""
+ if not vault:
+ return False
+
+ # Try domain_admins() method if present
+ if goal in ("domain_admin", "full_compromise"):
+ if hasattr(vault, "domain_admins") and callable(vault.domain_admins):
+ da_creds = vault.domain_admins()
+ if da_creds:
+ return True
+
+ # Extract credentials collection from vault
+ creds: list[Any] = []
+ if hasattr(vault, "all") and callable(vault.all):
+ creds = vault.all()
+ elif hasattr(vault, "_store") and isinstance(vault._store, dict):
+ creds = list(vault._store.values())
+ elif hasattr(vault, "has_credentials") and callable(vault.has_credentials):
+ return bool(vault.has_credentials(min_privilege=goal))
+ elif isinstance(vault, dict):
+ creds = vault.get("credentials") or list(vault.values())
+ elif isinstance(vault, (list, tuple)):
+ creds = list(vault)
+
+ if not creds:
+ return False
+
+ # Check privilege level of credentials
+ for c in creds:
+ if isinstance(c, dict):
+ priv = str(c.get("privilege", "")).lower()
+ user = str(c.get("username", "")).lower()
+ ctype = str(c.get("cred_type", "")).lower()
+ else:
+ p_attr = getattr(c, "privilege", "")
+ priv = p_attr.value.lower() if hasattr(p_attr, "value") else str(p_attr or "").lower()
+ user = str(getattr(c, "username", "") or "").lower()
+ ct_attr = getattr(c, "cred_type", "")
+ ctype = ct_attr.value.lower() if hasattr(ct_attr, "value") else str(ct_attr or "").lower()
+
+ if goal in ("domain_admin", "full_compromise"):
+ if priv in ("domain_admin", "enterprise_admin") or "krbtgt" in user:
+ return True
+ # Fallback: any valid non-unknown credential if privilege not set
+ if ctype and ctype not in ("unknown", ""):
+ return True
+ else:
+ if (ctype and ctype not in ("unknown", "")) or (priv and priv not in ("unknown", "")):
+ return True
+
return False
diff --git a/tests/unit/test_honesty_gates.py b/tests/unit/test_honesty_gates.py
new file mode 100644
index 0000000..9989c70
--- /dev/null
+++ b/tests/unit/test_honesty_gates.py
@@ -0,0 +1,402 @@
+"""
+Tests for Gate 3 (Engine Validator Enforcement) and Gate 5 (Goal Indicator Verifier).
+Verifies fail-closed enforcement preventing phantom exploits, fake findings, and premature goal achievement.
+"""
+from __future__ import annotations
+
+import pytest
+from unittest.mock import MagicMock, AsyncMock
+
+from ares.core.config import AresSettings
+from ares.core.campaign import Campaign, Finding, Severity, ScopeEntry
+from ares.core.engine import AresEngine, EngineModuleResult, ModuleStatus
+from ares.core.validator import (
+ FindingValidator,
+ ValidationCheck,
+ ValidationStage,
+ ValidationResult,
+ build_default_validator,
+)
+from ares.strategy.engine import StrategyEngine
+from ares.credential.vault import CredentialVault, Credential, CredentialType, PrivilegeLevel
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# Helper Fixtures & Factories
+# ═══════════════════════════════════════════════════════════════════════════════
+
+from ares.core.campaign import Campaign, Finding, Severity, ScopeEntry, NoiseProfile
+
+def _make_campaign() -> Campaign:
+ return Campaign(
+ name="honesty-gate-test",
+ client="test-client",
+ operator="test-operator",
+ scope=[ScopeEntry(cidr="10.0.0.0/24")],
+ targets=["10.0.0.5"],
+ noise_profile=NoiseProfile.NORMAL,
+ )
+
+def _make_settings() -> AresSettings:
+ return AresSettings(
+ environment="test",
+ encryption_master_key="0" * 64,
+ jwt_secret_key="0" * 32,
+ )
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# GATE 3: Engine Validator Fail-Closed Enforcement Tests
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class TestGate3ValidatorEnforcement:
+ """Verifies that AresEngine enforces validator results fail-closed."""
+
+ @pytest.mark.asyncio
+ async def test_high_confidence_finding_confirmed(self):
+ """Valid finding with high confidence (>= 0.4) is confirmed."""
+ validator = FindingValidator()
+ validator.register("test.valid", [
+ ValidationCheck(
+ stage=ValidationStage.EXPLOITABLE,
+ name="pass_check",
+ check=AsyncMock(return_value=(True, 0.9, "verified")),
+ )
+ ])
+ engine = AresEngine(settings=_make_settings(), validator=validator)
+ campaign = _make_campaign()
+
+ finding = Finding(
+ title="Verified Vulnerability",
+ description="High confidence evidence",
+ severity=Severity.HIGH,
+ module_id="test.valid",
+ )
+
+ val_res = await validator.validate(finding, {})
+ assert val_res.should_report is True
+ assert val_res.confidence >= 0.4
+ assert finding.validated is True
+ assert finding.false_positive is False
+
+ @pytest.mark.asyncio
+ async def test_low_confidence_finding_rejected_by_validator(self):
+ """Finding with confidence < 0.4 is rejected and marked as false positive."""
+ validator = FindingValidator()
+ validator.register("test.fake", [
+ ValidationCheck(
+ stage=ValidationStage.EXPLOITABLE,
+ name="fail_check",
+ check=AsyncMock(return_value=(False, 0.2, "insufficient evidence")),
+ )
+ ])
+ engine = AresEngine(settings=_make_settings(), validator=validator)
+
+ finding = Finding(
+ title="Phantom Exploit Finding",
+ description="No real evidence behind this",
+ severity=Severity.CRITICAL,
+ module_id="test.fake",
+ )
+
+ val_res = await validator.validate(finding, {})
+ assert val_res.should_report is False
+ assert val_res.confidence < 0.4
+ assert finding.false_positive is True
+ assert finding.validated is False
+
+ @pytest.mark.asyncio
+ async def test_unregistered_module_finding_passes_with_default_confidence(self):
+ """Finding from an unregistered module passes with default 0.6 confidence (0.6 >= 0.4)."""
+ validator = FindingValidator() # empty registry
+ engine = AresEngine(settings=_make_settings(), validator=validator)
+
+ finding = Finding(
+ title="Unregistered Module Finding",
+ description="Module not in registry yet",
+ severity=Severity.MEDIUM,
+ module_id="community.new_scanner",
+ )
+
+ val_res = await validator.validate(finding, {})
+ assert val_res.confidence == 0.6
+ assert val_res.should_report is True
+ assert finding.validated is True
+ assert finding.false_positive is False
+
+ @pytest.mark.asyncio
+ async def test_lateral_evidence_target_only_rejected(self):
+ """Lateral movement finding with only target host (MOD-021 pattern) scores 0.2 and is rejected."""
+ validator = build_default_validator()
+
+ rdp_finding = Finding(
+ title="Lateral Movement: RDP -> 10.0.0.5",
+ description="Port 3389 was open, claiming lateral movement",
+ severity=Severity.HIGH,
+ module_id="lateral.rdp",
+ host="10.0.0.5",
+ evidence={"target": "10.0.0.5"}, # target only! No command output, no session
+ )
+
+ val_res = await validator.validate(rdp_finding, {})
+ assert val_res.confidence == 0.2
+ assert val_res.passed is False
+ assert val_res.should_report is False
+ assert rdp_finding.false_positive is True
+ assert rdp_finding.validated is False
+
+ @pytest.mark.asyncio
+ async def test_lateral_evidence_target_plus_execution_confirmed(self):
+ """Lateral movement finding with target + command output scores 0.8 and is confirmed."""
+ validator = build_default_validator()
+
+ psexec_finding = Finding(
+ title="Lateral Movement: lateral.psexec -> 10.0.0.5",
+ description="Moved laterally and executed whoami",
+ severity=Severity.CRITICAL,
+ module_id="lateral.psexec",
+ host="10.0.0.5",
+ evidence={
+ "target": "10.0.0.5",
+ "command_output": "nt authority\\system",
+ "technique": "psexec",
+ },
+ )
+
+ val_res = await validator.validate(psexec_finding, {})
+ assert val_res.confidence == 0.8
+ assert val_res.passed is True
+ assert val_res.should_report is True
+ assert psexec_finding.validated is True
+ assert psexec_finding.false_positive is False
+
+ @pytest.mark.asyncio
+ async def test_engine_execution_filters_unvalidated_findings(self):
+ """Engine execution loop rejects findings where validator returned confidence < 0.4."""
+ from ares.modules.base import ModuleResult
+ from unittest.mock import patch
+
+ validator = FindingValidator()
+ validator.register("test.mixed", [
+ ValidationCheck(
+ stage=ValidationStage.EXPLOITABLE,
+ name="selective_check",
+ check=lambda finding, context: (
+ AsyncMock(return_value=(True, 0.9, "good"))()
+ if "Valid" in finding.title
+ else AsyncMock(return_value=(False, 0.1, "bad"))()
+ ),
+ )
+ ])
+ engine = AresEngine(settings=_make_settings(), validator=validator)
+ campaign = _make_campaign()
+
+ f_good = Finding(title="Valid Finding", description="Passes", severity=Severity.HIGH, module_id="test.mixed")
+ f_bad = Finding(title="Fake Finding", description="Fails", severity=Severity.CRITICAL, module_id="test.mixed")
+
+ # Mock the dispatch and execution to reach the engine validation loop directly
+ from ares.core.execution_admission import _mint_test_dispatch_context
+ from ares.modules.ad.kerberoast import KerberoastModule
+
+ engine.load_modules()
+ # Register a mock module that produces both findings
+ async def fake_exec(self_unused, ctx):
+ return ModuleResult(
+ status="success",
+ findings=[f_good, f_bad],
+ raw={"test": True},
+ module_id="test.mixed",
+ )
+
+ with patch.object(KerberoastModule, "execute", fake_exec):
+ # Temporarily register test.mixed validator on kerberoast
+ validator.register("ad.kerberoast", validator._registry["test.mixed"])
+ result = await engine.run_module(
+ "ad.kerberoast",
+ campaign,
+ {"dc": "10.0.0.5", "domain": "corp.local", "username": "u", "password": "p", "target_user": "t"},
+ actor_role="team_lead",
+ dispatch_context=_mint_test_dispatch_context(engine, campaign.id, "ad.kerberoast"),
+ )
+
+ # Gate 3 enforcement: only f_good is confirmed, f_bad is rejected
+ assert result.status == ModuleStatus.DONE
+ assert len(result.findings) == 1
+ assert result.findings[0].title == "Valid Finding"
+ assert result.findings[0].validated is True
+ assert f_bad.false_positive is True
+ assert f_bad.validated is False
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# GATE 5: Goal Indicator Verifier Tests
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class TestGate5GoalVerifier:
+ """Verifies that StrategyEngine._check_goal_achieved requires confirmed findings and vault evidence."""
+
+ def _make_strategy_engine(self):
+ return StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())
+
+ def test_goal_not_achieved_if_finding_not_confirmed(self):
+ """Title matches 'DCSync' but finding is unvalidated or marked false positive -> goal fails."""
+ engine = self._make_strategy_engine()
+ campaign = _make_campaign()
+
+ # Finding with matching title, but false_positive=True, validated=False
+ f = Finding(
+ title="DCSync Attack Successful",
+ description="Fake DCSync",
+ severity=Severity.CRITICAL,
+ )
+ f.false_positive = True
+ f.validated = False
+ campaign.findings = [f]
+
+ # Even with vault credentials present, unvalidated finding must block goal
+ mock_vault = MagicMock()
+ mock_vault.all.return_value = [
+ MagicMock(cred_type="ntlm", privilege="domain_admin", username="krbtgt")
+ ]
+ object.__setattr__(campaign, "_vault", mock_vault)
+
+ assert engine._check_goal_achieved(campaign, "domain_admin") is False
+
+ def test_goal_not_achieved_if_confirmed_finding_but_vault_empty(self):
+ """Finding is confirmed, but vault has no credentials -> privilege goal fails."""
+ engine = self._make_strategy_engine()
+ campaign = _make_campaign()
+
+ # Finding passed Gate 3
+ f = Finding(
+ title="DCSync Attack - krbtgt hash obtained",
+ description="Real DCSync finding",
+ severity=Severity.CRITICAL,
+ )
+ f.validated = True
+ f.false_positive = False
+ campaign.findings = [f]
+
+ # Vault is empty
+ mock_vault = MagicMock()
+ mock_vault.all.return_value = []
+ mock_vault.domain_admins.return_value = []
+ object.__setattr__(campaign, "_vault", mock_vault)
+
+ assert engine._check_goal_achieved(campaign, "domain_admin") is False
+
+ def test_goal_achieved_when_confirmed_finding_and_vault_creds_exist(self):
+ """Finding is confirmed AND vault has supporting credentials -> goal succeeds."""
+ engine = self._make_strategy_engine()
+ campaign = _make_campaign()
+
+ f = Finding(
+ title="DCSync Attack - krbtgt hash obtained",
+ description="Real DCSync finding",
+ severity=Severity.CRITICAL,
+ )
+ f.validated = True
+ f.false_positive = False
+ campaign.findings = [f]
+
+ # Real CredentialVault with a stored Domain Admin credential
+ vault = CredentialVault(encryption_key="test-key-32-chars-long-secret!!")
+ cred = Credential(
+ username="krbtgt",
+ domain="CORP",
+ cred_type=CredentialType.NTLM,
+ privilege=PrivilegeLevel.DOMAIN_ADMIN,
+ )
+ vault.store(cred, "aad3b435b51404eeaad3b435b51404ee:31d6cfe0d16ae931b73c59d7e0c089c0")
+ object.__setattr__(campaign, "_vault", vault)
+
+ assert engine._check_goal_achieved(campaign, "domain_admin") is True
+
+ def test_non_privilege_goal_does_not_require_vault_credentials(self):
+ """Non-privilege goals like data_exfil only require confirmed findings, not vault credentials."""
+ engine = self._make_strategy_engine()
+ campaign = _make_campaign()
+
+ f = Finding(
+ title="Sensitive File Exfiltrated: financials.xlsx",
+ description="Data exfil confirmed",
+ severity=Severity.HIGH,
+ )
+ f.validated = True
+ f.false_positive = False
+ campaign.findings = [f]
+ # No vault attached at all
+ assert engine._check_goal_achieved(campaign, "data_exfil") is True
+
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# INTEGRATION: MOD-005 and MOD-021 Simulation Tests
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class TestGate3And5IntegrationSimulations:
+ """End-to-end integration scenarios verifying defense against MOD-005 and MOD-021 failure modes."""
+
+ @pytest.mark.asyncio
+ async def test_mod_005_ghost_forge_simulation(self):
+ """
+ Simulation of MOD-005 (ghost_forge):
+ Module produces CRITICAL 'DCSync Success' finding without network I/O or valid evidence.
+ 1. Gate 3 rejects the finding (confidence too low / no execution evidence).
+ 2. Finding is excluded from confirmed and marked false positive.
+ 3. Gate 5 verifies that domain_admin goal is NOT achieved.
+ """
+ validator = build_default_validator()
+ strategy_engine = StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())
+ campaign = _make_campaign()
+
+ # Simulated ghost_forge finding with empty evidence (no captured hashes, no I/O)
+ fake_finding = Finding(
+ title="DCSync Attack - krbtgt hash obtained",
+ description="Claimed DCSync without real network execution",
+ severity=Severity.CRITICAL,
+ module_id="ad.kerberoast", # registered AD validator
+ evidence={"hash_count": 0}, # 0 hashes captured!
+ )
+
+ # Gate 3: validate
+ val_res = await validator.validate(fake_finding, {})
+ assert val_res.confidence < 0.4
+ assert val_res.should_report is False
+ assert fake_finding.false_positive is True
+ assert fake_finding.validated is False
+
+ # Attempt to achieve goal with this rejected finding
+ campaign.findings = [fake_finding]
+ assert strategy_engine._check_goal_achieved(campaign, "domain_admin") is False
+
+ @pytest.mark.asyncio
+ async def test_mod_021_rdp_port_open_simulation(self):
+ """
+ Simulation of MOD-021 (RDP port open false lateral movement):
+ Module produces finding 'Lateral Movement via RDP' simply because port 3389 was open.
+ 1. Gate 3 blocks finding (target only = 0.2 < 0.4).
+ 2. Gate 5 rejects goal achievement because finding is unvalidated and vault has no creds.
+ """
+ validator = build_default_validator()
+ strategy_engine = StrategyEngine(ares_engine=MagicMock(), settings=MagicMock())
+ campaign = _make_campaign()
+
+ fake_lateral_finding = Finding(
+ title="Lateral Movement via RDP: 10.0.0.5",
+ description="Port 3389 open - no session established",
+ severity=Severity.HIGH,
+ module_id="lateral.rdp",
+ host="10.0.0.5",
+ evidence={"target": "10.0.0.5", "port": 3389},
+ )
+
+ # Gate 3: validate
+ val_res = await validator.validate(fake_lateral_finding, {})
+ assert val_res.confidence == 0.2
+ assert val_res.should_report is False
+ assert fake_lateral_finding.false_positive is True
+ assert fake_lateral_finding.validated is False
+
+ # Gate 5: goal not achieved
+ campaign.findings = [fake_lateral_finding]
+ assert strategy_engine._check_goal_achieved(campaign, "domain_admin") is False
diff --git a/tests/unit/test_strategic_modules.py b/tests/unit/test_strategic_modules.py
index 48bc391..482033a 100644
--- a/tests/unit/test_strategic_modules.py
+++ b/tests/unit/test_strategic_modules.py
@@ -912,12 +912,18 @@ class TestStrategyEngine:
assert messages[0]["data"] == 42
def test_check_goal_achieved_domain_admin(self):
+ from unittest.mock import MagicMock
from ares.strategy import StrategyEngine
engine = StrategyEngine(ares_engine=None, settings=None)
campaign = _mock_campaign()
- # Simulate a DCSync finding
- mock_finding = type("F", (), {"title": "DCSync - krbtgt hash obtained"})()
+ # Simulate a DCSync finding that passed Gate 3
+ mock_finding = type("F", (), {"title": "DCSync - krbtgt hash obtained", "validated": True, "false_positive": False})()
campaign.findings = [mock_finding]
+ mock_vault = MagicMock()
+ mock_vault.all.return_value = [
+ MagicMock(cred_type="ntlm", privilege="domain_admin", username="krbtgt")
+ ]
+ object.__setattr__(campaign, "_vault", mock_vault)
assert engine._check_goal_achieved(campaign, "domain_admin") is True
def test_check_goal_not_achieved_empty(self):
diff --git a/tests/unit/test_strategy_engine.py b/tests/unit/test_strategy_engine.py
index 99474cf..09ae147 100644
--- a/tests/unit/test_strategy_engine.py
+++ b/tests/unit/test_strategy_engine.py
@@ -616,8 +616,13 @@ class TestGoalAchieved:
engine = self._make_engine()
campaign = MagicMock()
campaign.findings = [
- {"title": "DCSync Attack - Domain Admin Achieved"},
+ {"title": "DCSync Attack - Domain Admin Achieved", "validated": True, "false_positive": False},
]
+ mock_vault = MagicMock()
+ mock_vault.all.return_value = [
+ {"cred_type": "ntlm", "privilege": "domain_admin", "username": "krbtgt"}
+ ]
+ campaign.vault = mock_vault
assert engine._check_goal_achieved(campaign, "domain_admin") is True