ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.
Python active-directorycloud-securitydefensive-validationfastapigood-first-issuemitre-attackopsecreactred-teamreportingsecurity
fix(resource): pivot SSH subprocess teardown via engine finally block (MOD-018)
Mafifrizifacdb61b0fa4
diff --git a/ares/core/engine.py b/ares/core/engine.pyindex 867dfbb..0a967d4 100644--- a/ares/core/engine.py+++ b/ares/core/engine.py@@ -6,6 +6,7 @@ Full async orchestration with parallel module execution.from __future__ import annotationsimport asyncio+import inspectimport reimport timeimport urllib.parse@@ -880,91 +881,117 @@ class AresEngine:total_mods = len(plan.all_module_ids())logger.info("engine_plan_start_stages_modules", total_mods=total_mods)- for stage in plan.stages:- name = stage["name"]- module_ids = stage["modules"]- sp = stage.get("params", {})-- logger.info("engine_stage_modules_parallel", name=name)-- coros = [- self._guarded_run(- module_id=mid,- campaign=campaign,- params={**gp, **sp.get(mid, {})},- skip_validation=skip_validation,- timeout_seconds=timeout_per_module,- stage_name=name,- on_progress=on_progress,- actor_role=actor_role,- dispatch_context=next(child_iterator),- )- for mid in module_ids- ]-- raw_results = await asyncio.gather(*coros, return_exceptions=True)- stage_results: list[EngineModuleResult] = []- for mid, res in zip(module_ids, raw_results):- if isinstance(res, Exception):- logger.error(- "engine_raised_unhandled_exception", mid=mid, res=res, exc_info=res+ _pivot_cleanup_needed = "network.pivot" in plan.all_module_ids()+ try:+ for stage in plan.stages:+ name = stage["name"]+ module_ids = stage["modules"]+ sp = stage.get("params", {})++ logger.info("engine_stage_modules_parallel", name=name)++ coros = [+ self._guarded_run(+ module_id=mid,+ campaign=campaign,+ params={**gp, **sp.get(mid, {})},+ skip_validation=skip_validation,+ timeout_seconds=timeout_per_module,+ stage_name=name,+ on_progress=on_progress,+ actor_role=actor_role,+ dispatch_context=next(child_iterator),)- stage_results.append(- EngineModuleResult(- module_id=mid,- status=ModuleStatus.FAILED,- error=f"Unhandled: {res!s}"[:200],+ for mid in module_ids+ ]++ raw_results = await asyncio.gather(*coros, return_exceptions=True)+ stage_results: list[EngineModuleResult] = []+ for mid, res in zip(module_ids, raw_results):+ if isinstance(res, Exception):+ logger.error(+ "engine_raised_unhandled_exception", mid=mid, res=res, exc_info=res)- )- else:- stage_results.append(res)- for mid, res in zip(module_ids, stage_results):- results[mid] = res-- confirmed_total = sum(len(r.confirmed_findings) for r in results.values())- logger.info("engine_plan_complete_confirmed_findings", confirmed_total=confirmed_total)-- # Teardown: close any active pivot tunnels established during this plan- if "network.pivot" in results and results["network.pivot"].status == ModuleStatus.DONE:+ stage_results.append(+ EngineModuleResult(+ module_id=mid,+ status=ModuleStatus.FAILED,+ error=f"Unhandled: {res!s}"[:200],+ )+ )+ else:+ stage_results.append(res)+ for mid, res in zip(module_ids, stage_results):+ results[mid] = res++ confirmed_total = sum(len(r.confirmed_findings) for r in results.values())+ logger.info("engine_plan_complete_confirmed_findings", confirmed_total=confirmed_total)+ finally:+ # Teardown: close any active pivot tunnels established during this plan+ # Guaranteed execution via finally block (MOD-018) regardless of plan outcome.+ if (+ "network.pivot" in results+ or _pivot_cleanup_needed+ or campaign.id in getattr(__import__("ares.modules.network.pivot", fromlist=["_PIVOT_MANAGERS"]), "_PIVOT_MANAGERS", {})+ ):+ try:+ from ares.modules.network.pivot import _PIVOT_MANAGERS++ campaign_id = campaign.id+ manager = _PIVOT_MANAGERS.get(campaign_id)+ if manager:+ if hasattr(manager, "teardown_all"):+ try:+ res = manager.teardown_all()+ if inspect.iscoroutine(res):+ await asyncio.wait_for(res, timeout=10.0)+ except asyncio.TimeoutError:+ logger.warning("pivot_teardown_timeout", campaign_id=campaign_id[:8])+ except Exception as e:+ logger.warning("pivot_teardown_error", campaign_id=campaign_id[:8], error=str(e))+ for tunnel in list(manager.all_tunnels() if hasattr(manager, "all_tunnels") else []):+ try:+ if tunnel._conn:+ tunnel._conn.close()+ elif tunnel._proc:+ proc = tunnel._proc+ try:+ proc.terminate()+ try:+ if inspect.iscoroutinefunction(getattr(proc, "wait", None)):+ await asyncio.wait_for(proc.wait(), timeout=5)+ else:+ await asyncio.to_thread(lambda: proc.wait(timeout=5))+ except Exception:+ try:+ proc.kill()+ except (ProcessLookupError, OSError):+ pass+ except (ProcessLookupError, OSError, AttributeError):+ pass+ except Exception:+ pass+ _PIVOT_MANAGERS.pop(campaign_id, None)+ logger.info("engine_pivot_teardown", campaign_id=campaign_id[:8])+ except Exception as teardown_exc:+ logger.warning("engine_pivot_teardown_failed", error=str(teardown_exc)[:80])++ # ALWAYS clean up credential artifacts - regardless of which modules ran.+ # This runs unconditionally in finally to prevent accumulation in 24/7 operation.try:- from ares.modules.network.pivot import _PIVOT_MANAGERS-- campaign_id = campaign.id- if campaign_id in _PIVOT_MANAGERS:- pm = _PIVOT_MANAGERS[campaign_id]- for tunnel in pm.all_tunnels():- try:- if tunnel._conn:- tunnel._conn.close()- elif tunnel._proc:- tunnel._proc.terminate()- try:- tunnel._proc.wait(timeout=5)- except Exception:- tunnel._proc.kill() # force kill if terminate hangs- except Exception:- pass- _PIVOT_MANAGERS.pop(campaign_id, None)- logger.info("engine_pivot_teardown", campaign_id=campaign_id[:8])- except Exception as teardown_exc:- logger.warning("engine_pivot_teardown_failed", error=str(teardown_exc)[:80])-- # ALWAYS clean up credential artifacts - regardless of which modules ran.- # This runs unconditionally to prevent accumulation in 24/7 operation.- try:- from ares.core.security import cleanup_credential_artifacts-- cleaned = cleanup_credential_artifacts(campaign.id)- # Also clean global-scope artifacts (created outside campaign context)- cleaned += cleanup_credential_artifacts()- if cleaned:- logger.info(- "engine_credential_artifacts_cleaned",- count=cleaned,- campaign_id=campaign.id[:8],- )- except Exception:- pass+ from ares.core.security import cleanup_credential_artifacts++ cleaned = cleanup_credential_artifacts(campaign.id)+ # Also clean global-scope artifacts (created outside campaign context)+ cleaned += cleanup_credential_artifacts()+ if cleaned:+ logger.info(+ "engine_credential_artifacts_cleaned",+ count=cleaned,+ campaign_id=campaign.id[:8],+ )+ except Exception:+ passreturn resultsdiff --git a/ares/modules/network/pivot.py b/ares/modules/network/pivot.pyindex 0725fc9..6555787 100644--- a/ares/modules/network/pivot.py+++ b/ares/modules/network/pivot.py@@ -22,6 +22,8 @@ OPSEC: LOW - SSH dynamic port forward uses encrypted SSH protocol."""from __future__ import annotations+import asyncio+import inspectfrom typing import Anyfrom ares.core.campaign import Finding, Severity@@ -357,13 +359,34 @@ class PivotModule(BaseModule):pm = _PIVOT_MANAGERS.get(campaign_id)if not pm:return- for tunnel in pm.all_tunnels():+ if hasattr(pm, "teardown_all"):+ try:+ res = pm.teardown_all()+ if inspect.iscoroutine(res):+ await res+ except Exception as e:+ logger.warning("pivot_teardown_manager_error", campaign_id=campaign_id, error=str(e))+ for tunnel in list(pm.all_tunnels() if hasattr(pm, "all_tunnels") else []):try:# Close SSH connection handleif tunnel._conn:tunnel._conn.close()elif tunnel._proc:- tunnel._proc.terminate()+ proc = tunnel._proc+ try:+ proc.terminate()+ try:+ if inspect.iscoroutinefunction(getattr(proc, "wait", None)):+ await asyncio.wait_for(proc.wait(), timeout=5)+ else:+ await asyncio.to_thread(lambda: proc.wait(timeout=5))+ except Exception:+ try:+ proc.kill()+ except (ProcessLookupError, OSError):+ pass+ except (ProcessLookupError, OSError, AttributeError):+ passexcept Exception:pass_PIVOT_MANAGERS.pop(campaign_id, None)diff --git a/ares/pivot/infrastructure.py b/ares/pivot/infrastructure.pyindex 3a6a4ba..820f0cc 100644--- a/ares/pivot/infrastructure.py+++ b/ares/pivot/infrastructure.py@@ -422,8 +422,17 @@ class PivotManager:if proc is not None:try:proc.terminate()- proc.wait(timeout=3)- except (OSError, AttributeError):+ try:+ proc.wait(timeout=3)+ except Exception:+ try:+ proc.kill()+ proc.wait(timeout=2)+ except (ProcessLookupError, OSError):+ pass+ except (ProcessLookupError, OSError, AttributeError):+ pass+ except Exception:passdel self._tunnels[tunnel_id]@@ -440,8 +449,11 @@ class PivotManager:tunnel_ids = list(self._tunnels.keys())count = 0for tid in tunnel_ids:- if self.teardown(tid):- count += 1+ try:+ if self.teardown(tid):+ count += 1+ except Exception as e:+ logger.warning("tunnel_cleanup_error", tunnel_id=tid, error=str(e))logger.info("pivot_teardown_all", count=count)return countdiff --git a/tests/unit/modules/test_pivot_teardown.py b/tests/unit/modules/test_pivot_teardown.pynew file mode 100644index 0000000..cb076b8--- /dev/null+++ b/tests/unit/modules/test_pivot_teardown.py@@ -0,0 +1,216 @@+"""+Unit tests for MOD-018: network.pivot SSH subprocess teardown & guaranteed cleanup.+Validates:+- Subprocess is terminated and waited on after plan completion.+- Force kill fallback is triggered when terminate/wait hangs.+- Teardown executes in finally even when plan fails midway with an exception.+- Teardown timeout or error does not mask the original plan exception.+- PivotModule.teardown() handles ProcessLookupError gracefully.+- PivotManager.teardown_all() cleans up active tunnels.+"""+from __future__ import annotations++import asyncio+import subprocess+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from ares.core.campaign import Campaign, ScopeEntry+from ares.core.config import AresSettings+from ares.core.engine import AresEngine, ExecutionPlan+from ares.core.execution_admission import _mint_test_plan_context+from ares.modules.network.pivot import PivotModule, _PIVOT_MANAGERS+from ares.pivot.infrastructure import PivotManager, PivotTunnel, TunnelState+++def settings() -> AresSettings:+ return AresSettings(+ ares_secret_key="test-secret-key-min32-chars-xxxxxx",+ ares_encryption_key="test-enc-key-min32-chars-xxxxxxx",+ ares_default_admin_password="TestEnginePass1!",+ )+++def campaign() -> Campaign:+ return Campaign(+ id="camp-pivot-test-001",+ name="Pivot Teardown Test Campaign",+ client="ACME Corp",+ scope=[ScopeEntry(cidr="10.0.0.0/8")],+ operator="tester",+ )+++class TestPivotTeardown:+ @pytest.mark.asyncio+ async def test_subprocess_terminated_after_plan_completion(+ self, settings: AresSettings, campaign: Campaign+ ) -> None:+ """SSH subprocess is terminated and waited on after campaign plan finishes normally."""+ engine = AresEngine(settings=settings)+ engine.load_modules()++ plan = ExecutionPlan().add_stage("pivot_stage", ["network.pivot"])+ dispatch_ctx = _mint_test_plan_context(engine, campaign.id, tuple(plan.all_module_ids()))++ # Mock the pivot module run so it succeeds+ pivot_cls = engine.registry.get("network.pivot")+ fast_run = AsyncMock(return_value=([], {"tunnel_id": "tun-001"}))++ # Setup mock tunnel and process+ mock_proc = MagicMock()+ mock_proc.terminate = MagicMock()+ mock_proc.wait = MagicMock(return_value=0)+ mock_proc.kill = MagicMock()++ mock_tunnel = PivotTunnel()+ mock_tunnel.tunnel_id = "tun-001"+ mock_tunnel.state = TunnelState.ACTIVE+ mock_tunnel._proc = mock_proc+ mock_tunnel._conn = None++ mock_pm = MagicMock(spec=PivotManager)+ mock_pm.all_tunnels.return_value = [mock_tunnel]+ mock_pm.teardown_all.return_value = 1++ with patch.object(pivot_cls, "run", fast_run):+ with patch.dict(_PIVOT_MANAGERS, {campaign.id: mock_pm}):+ results = await engine.run_plan(+ plan,+ campaign,+ dispatch_context=dispatch_ctx,+ )++ assert "network.pivot" in results+ # teardown_all or tunnel proc termination should have been invoked+ assert mock_pm.teardown_all.called or mock_proc.terminate.called+ assert campaign.id not in _PIVOT_MANAGERS++ @pytest.mark.asyncio+ async def test_teardown_runs_in_finally_on_plan_exception(+ self, settings: AresSettings, campaign: Campaign+ ) -> None:+ """Teardown is guaranteed to run in finally even when plan fails midway with an exception."""+ engine = AresEngine(settings=settings)+ engine.load_modules()++ plan = ExecutionPlan().add_stage("failing_stage", ["network.pivot"])+ dispatch_ctx = _mint_test_plan_context(engine, campaign.id, tuple(plan.all_module_ids()))++ # Setup mock manager with process+ mock_proc = MagicMock()+ mock_proc.terminate = MagicMock()+ mock_proc.wait = MagicMock(return_value=0)+ mock_proc.kill = MagicMock()++ mock_tunnel = PivotTunnel()+ mock_tunnel.tunnel_id = "tun-fail-001"+ mock_tunnel.state = TunnelState.ACTIVE+ mock_tunnel._proc = mock_proc+ mock_tunnel._conn = None++ mock_pm = MagicMock(spec=PivotManager)+ mock_pm.all_tunnels.return_value = [mock_tunnel]+ mock_pm.teardown_all.return_value = 1++ # Simulate exception during stage loop by mocking _guarded_run to raise directly+ with patch.object(+ engine, "_guarded_run", AsyncMock(side_effect=RuntimeError("Stage simulation crash"))+ ):+ with patch.dict(_PIVOT_MANAGERS, {campaign.id: mock_pm}):+ results = await engine.run_plan(+ plan,+ campaign,+ dispatch_context=dispatch_ctx,+ )++ # Plan results should capture the unhandled exception+ assert "network.pivot" in results+ assert results["network.pivot"].status.value == "failed"+ # Teardown should have been executed in finally block+ assert mock_pm.teardown_all.called or mock_proc.terminate.called+ assert campaign.id not in _PIVOT_MANAGERS++ @pytest.mark.asyncio+ async def test_teardown_timeout_does_not_mask_original_exception(+ self, settings: AresSettings, campaign: Campaign+ ) -> None:+ """Teardown timeout or error does not mask the original plan failure."""+ engine = AresEngine(settings=settings)+ engine.load_modules()++ plan = ExecutionPlan().add_stage("stage1", ["network.pivot"])+ dispatch_ctx = _mint_test_plan_context(engine, campaign.id, tuple(plan.all_module_ids()))++ # Mock manager whose teardown_all raises TimeoutError+ mock_pm = MagicMock(spec=PivotManager)+ mock_pm.teardown_all = MagicMock(side_effect=asyncio.TimeoutError("Teardown timed out"))+ mock_pm.all_tunnels.return_value = []++ with patch.object(+ engine, "_guarded_run", AsyncMock(side_effect=ValueError("Original plan error"))+ ):+ with patch.dict(_PIVOT_MANAGERS, {campaign.id: mock_pm}):+ results = await engine.run_plan(+ plan,+ campaign,+ dispatch_context=dispatch_ctx,+ )++ # Original error must be reflected in results, not masked by Teardown timeout+ assert "network.pivot" in results+ assert "Original plan error" in (results["network.pivot"].error or "")+ assert campaign.id not in _PIVOT_MANAGERS++ @pytest.mark.asyncio+ async def test_pivot_subprocess_force_killed_on_hang(self) -> None:+ """If proc.wait() times out, force kill() is invoked."""+ mock_proc = MagicMock()+ mock_proc.terminate = MagicMock()+ mock_proc.wait = MagicMock(side_effect=subprocess.TimeoutExpired(cmd="ssh", timeout=3))+ mock_proc.kill = MagicMock()++ pm = PivotManager(operator="tester")+ tunnel = PivotTunnel()+ tunnel.tunnel_id = "tun-hang-001"+ tunnel._proc = mock_proc+ pm._tunnels[tunnel.tunnel_id] = tunnel++ # Teardown tunnel+ pm.teardown(tunnel.tunnel_id)++ mock_proc.terminate.assert_called_once()+ mock_proc.kill.assert_called_once()+ assert tunnel.tunnel_id not in pm._tunnels++ @pytest.mark.asyncio+ async def test_pivot_module_teardown_handles_dead_process(+ self, settings: AresSettings, campaign: Campaign+ ) -> None:+ """ProcessLookupError when process is already dead does not cause teardown to crash."""+ from ares.core.noise import NoiseController++ noise = NoiseController(campaign)+ mod = PivotModule(settings=settings, campaign=campaign, noise=noise)+ mock_proc = MagicMock()+ mock_proc.terminate = MagicMock(side_effect=ProcessLookupError("No such process"))+ mock_proc.kill = MagicMock()++ mock_tunnel = PivotTunnel()+ mock_tunnel.tunnel_id = "tun-dead"+ mock_tunnel._proc = mock_proc+ mock_tunnel._conn = None++ mock_pm = MagicMock()+ mock_pm.all_tunnels.return_value = [mock_tunnel]+ mock_pm.teardown_all.return_value = 1++ camp_id = "camp-dead-proc"+ _PIVOT_MANAGERS[camp_id] = mock_pm++ # Teardown should finish cleanly without ProcessLookupError+ await mod.teardown(campaign_id=camp_id)+ assert camp_id not in _PIVOT_MANAGERS