Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

fix(resource): pivot SSH subprocess teardown via engine finally block (MOD-018)

Mafifrizifacdb61b0fa4
diff --git a/ares/core/engine.py b/ares/core/engine.py
index 867dfbb..0a967d4 100644
--- a/ares/core/engine.py
+++ b/ares/core/engine.py
@@ -6,6 +6,7 @@ Full async orchestration with parallel module execution.
from __future__ import annotations
import asyncio
+import inspect
import re
import time
import urllib.parse
@@ -880,91 +881,117 @@ class AresEngine:
total_mods = len(plan.all_module_ids())
logger.info("engine_plan_start_stages_modules", total_mods=total_mods)
- for stage in plan.stages:
- name = stage["name"]
- module_ids = stage["modules"]
- sp = stage.get("params", {})
-
- logger.info("engine_stage_modules_parallel", name=name)
-
- coros = [
- self._guarded_run(
- module_id=mid,
- campaign=campaign,
- params={**gp, **sp.get(mid, {})},
- skip_validation=skip_validation,
- timeout_seconds=timeout_per_module,
- stage_name=name,
- on_progress=on_progress,
- actor_role=actor_role,
- dispatch_context=next(child_iterator),
- )
- for mid in module_ids
- ]
-
- raw_results = await asyncio.gather(*coros, return_exceptions=True)
- stage_results: list[EngineModuleResult] = []
- for mid, res in zip(module_ids, raw_results):
- if isinstance(res, Exception):
- logger.error(
- "engine_raised_unhandled_exception", mid=mid, res=res, exc_info=res
+ _pivot_cleanup_needed = "network.pivot" in plan.all_module_ids()
+ try:
+ for stage in plan.stages:
+ name = stage["name"]
+ module_ids = stage["modules"]
+ sp = stage.get("params", {})
+
+ logger.info("engine_stage_modules_parallel", name=name)
+
+ coros = [
+ self._guarded_run(
+ module_id=mid,
+ campaign=campaign,
+ params={**gp, **sp.get(mid, {})},
+ skip_validation=skip_validation,
+ timeout_seconds=timeout_per_module,
+ stage_name=name,
+ on_progress=on_progress,
+ actor_role=actor_role,
+ dispatch_context=next(child_iterator),
)
- stage_results.append(
- EngineModuleResult(
- module_id=mid,
- status=ModuleStatus.FAILED,
- error=f"Unhandled: {res!s}"[:200],
+ for mid in module_ids
+ ]
+
+ raw_results = await asyncio.gather(*coros, return_exceptions=True)
+ stage_results: list[EngineModuleResult] = []
+ for mid, res in zip(module_ids, raw_results):
+ if isinstance(res, Exception):
+ logger.error(
+ "engine_raised_unhandled_exception", mid=mid, res=res, exc_info=res
)
- )
- else:
- stage_results.append(res)
- for mid, res in zip(module_ids, stage_results):
- results[mid] = res
-
- confirmed_total = sum(len(r.confirmed_findings) for r in results.values())
- logger.info("engine_plan_complete_confirmed_findings", confirmed_total=confirmed_total)
-
- # Teardown: close any active pivot tunnels established during this plan
- if "network.pivot" in results and results["network.pivot"].status == ModuleStatus.DONE:
+ stage_results.append(
+ EngineModuleResult(
+ module_id=mid,
+ status=ModuleStatus.FAILED,
+ error=f"Unhandled: {res!s}"[:200],
+ )
+ )
+ else:
+ stage_results.append(res)
+ for mid, res in zip(module_ids, stage_results):
+ results[mid] = res
+
+ confirmed_total = sum(len(r.confirmed_findings) for r in results.values())
+ logger.info("engine_plan_complete_confirmed_findings", confirmed_total=confirmed_total)
+ finally:
+ # Teardown: close any active pivot tunnels established during this plan
+ # Guaranteed execution via finally block (MOD-018) regardless of plan outcome.
+ if (
+ "network.pivot" in results
+ or _pivot_cleanup_needed
+ or campaign.id in getattr(__import__("ares.modules.network.pivot", fromlist=["_PIVOT_MANAGERS"]), "_PIVOT_MANAGERS", {})
+ ):
+ try:
+ from ares.modules.network.pivot import _PIVOT_MANAGERS
+
+ campaign_id = campaign.id
+ manager = _PIVOT_MANAGERS.get(campaign_id)
+ if manager:
+ if hasattr(manager, "teardown_all"):
+ try:
+ res = manager.teardown_all()
+ if inspect.iscoroutine(res):
+ await asyncio.wait_for(res, timeout=10.0)
+ except asyncio.TimeoutError:
+ logger.warning("pivot_teardown_timeout", campaign_id=campaign_id[:8])
+ except Exception as e:
+ logger.warning("pivot_teardown_error", campaign_id=campaign_id[:8], error=str(e))
+ for tunnel in list(manager.all_tunnels() if hasattr(manager, "all_tunnels") else []):
+ try:
+ if tunnel._conn:
+ tunnel._conn.close()
+ elif tunnel._proc:
+ proc = tunnel._proc
+ try:
+ proc.terminate()
+ try:
+ if inspect.iscoroutinefunction(getattr(proc, "wait", None)):
+ await asyncio.wait_for(proc.wait(), timeout=5)
+ else:
+ await asyncio.to_thread(lambda: proc.wait(timeout=5))
+ except Exception:
+ try:
+ proc.kill()
+ except (ProcessLookupError, OSError):
+ pass
+ except (ProcessLookupError, OSError, AttributeError):
+ pass
+ except Exception:
+ pass
+ _PIVOT_MANAGERS.pop(campaign_id, None)
+ logger.info("engine_pivot_teardown", campaign_id=campaign_id[:8])
+ except Exception as teardown_exc:
+ logger.warning("engine_pivot_teardown_failed", error=str(teardown_exc)[:80])
+
+ # ALWAYS clean up credential artifacts - regardless of which modules ran.
+ # This runs unconditionally in finally to prevent accumulation in 24/7 operation.
try:
- from ares.modules.network.pivot import _PIVOT_MANAGERS
-
- campaign_id = campaign.id
- if campaign_id in _PIVOT_MANAGERS:
- pm = _PIVOT_MANAGERS[campaign_id]
- for tunnel in pm.all_tunnels():
- try:
- if tunnel._conn:
- tunnel._conn.close()
- elif tunnel._proc:
- tunnel._proc.terminate()
- try:
- tunnel._proc.wait(timeout=5)
- except Exception:
- tunnel._proc.kill() # force kill if terminate hangs
- except Exception:
- pass
- _PIVOT_MANAGERS.pop(campaign_id, None)
- logger.info("engine_pivot_teardown", campaign_id=campaign_id[:8])
- except Exception as teardown_exc:
- logger.warning("engine_pivot_teardown_failed", error=str(teardown_exc)[:80])
-
- # ALWAYS clean up credential artifacts - regardless of which modules ran.
- # This runs unconditionally to prevent accumulation in 24/7 operation.
- try:
- from ares.core.security import cleanup_credential_artifacts
-
- cleaned = cleanup_credential_artifacts(campaign.id)
- # Also clean global-scope artifacts (created outside campaign context)
- cleaned += cleanup_credential_artifacts()
- if cleaned:
- logger.info(
- "engine_credential_artifacts_cleaned",
- count=cleaned,
- campaign_id=campaign.id[:8],
- )
- except Exception:
- pass
+ from ares.core.security import cleanup_credential_artifacts
+
+ cleaned = cleanup_credential_artifacts(campaign.id)
+ # Also clean global-scope artifacts (created outside campaign context)
+ cleaned += cleanup_credential_artifacts()
+ if cleaned:
+ logger.info(
+ "engine_credential_artifacts_cleaned",
+ count=cleaned,
+ campaign_id=campaign.id[:8],
+ )
+ except Exception:
+ pass
return results
diff --git a/ares/modules/network/pivot.py b/ares/modules/network/pivot.py
index 0725fc9..6555787 100644
--- a/ares/modules/network/pivot.py
+++ b/ares/modules/network/pivot.py
@@ -22,6 +22,8 @@ OPSEC: LOW - SSH dynamic port forward uses encrypted SSH protocol.
"""
from __future__ import annotations
+import asyncio
+import inspect
from typing import Any
from ares.core.campaign import Finding, Severity
@@ -357,13 +359,34 @@ class PivotModule(BaseModule):
pm = _PIVOT_MANAGERS.get(campaign_id)
if not pm:
return
- for tunnel in pm.all_tunnels():
+ if hasattr(pm, "teardown_all"):
+ try:
+ res = pm.teardown_all()
+ if inspect.iscoroutine(res):
+ await res
+ except Exception as e:
+ logger.warning("pivot_teardown_manager_error", campaign_id=campaign_id, error=str(e))
+ for tunnel in list(pm.all_tunnels() if hasattr(pm, "all_tunnels") else []):
try:
# Close SSH connection handle
if tunnel._conn:
tunnel._conn.close()
elif tunnel._proc:
- tunnel._proc.terminate()
+ proc = tunnel._proc
+ try:
+ proc.terminate()
+ try:
+ if inspect.iscoroutinefunction(getattr(proc, "wait", None)):
+ await asyncio.wait_for(proc.wait(), timeout=5)
+ else:
+ await asyncio.to_thread(lambda: proc.wait(timeout=5))
+ except Exception:
+ try:
+ proc.kill()
+ except (ProcessLookupError, OSError):
+ pass
+ except (ProcessLookupError, OSError, AttributeError):
+ pass
except Exception:
pass
_PIVOT_MANAGERS.pop(campaign_id, None)
diff --git a/ares/pivot/infrastructure.py b/ares/pivot/infrastructure.py
index 3a6a4ba..820f0cc 100644
--- a/ares/pivot/infrastructure.py
+++ b/ares/pivot/infrastructure.py
@@ -422,8 +422,17 @@ class PivotManager:
if proc is not None:
try:
proc.terminate()
- proc.wait(timeout=3)
- except (OSError, AttributeError):
+ try:
+ proc.wait(timeout=3)
+ except Exception:
+ try:
+ proc.kill()
+ proc.wait(timeout=2)
+ except (ProcessLookupError, OSError):
+ pass
+ except (ProcessLookupError, OSError, AttributeError):
+ pass
+ except Exception:
pass
del self._tunnels[tunnel_id]
@@ -440,8 +449,11 @@ class PivotManager:
tunnel_ids = list(self._tunnels.keys())
count = 0
for tid in tunnel_ids:
- if self.teardown(tid):
- count += 1
+ try:
+ if self.teardown(tid):
+ count += 1
+ except Exception as e:
+ logger.warning("tunnel_cleanup_error", tunnel_id=tid, error=str(e))
logger.info("pivot_teardown_all", count=count)
return count
diff --git a/tests/unit/modules/test_pivot_teardown.py b/tests/unit/modules/test_pivot_teardown.py
new file mode 100644
index 0000000..cb076b8
--- /dev/null
+++ b/tests/unit/modules/test_pivot_teardown.py
@@ -0,0 +1,216 @@
+"""
+Unit tests for MOD-018: network.pivot SSH subprocess teardown & guaranteed cleanup.
+Validates:
+- Subprocess is terminated and waited on after plan completion.
+- Force kill fallback is triggered when terminate/wait hangs.
+- Teardown executes in finally even when plan fails midway with an exception.
+- Teardown timeout or error does not mask the original plan exception.
+- PivotModule.teardown() handles ProcessLookupError gracefully.
+- PivotManager.teardown_all() cleans up active tunnels.
+"""
+from __future__ import annotations
+
+import asyncio
+import subprocess
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from ares.core.campaign import Campaign, ScopeEntry
+from ares.core.config import AresSettings
+from ares.core.engine import AresEngine, ExecutionPlan
+from ares.core.execution_admission import _mint_test_plan_context
+from ares.modules.network.pivot import PivotModule, _PIVOT_MANAGERS
+from ares.pivot.infrastructure import PivotManager, PivotTunnel, TunnelState
+
+
+def settings() -> AresSettings:
+ return AresSettings(
+ ares_secret_key="test-secret-key-min32-chars-xxxxxx",
+ ares_encryption_key="test-enc-key-min32-chars-xxxxxxx",
+ ares_default_admin_password="TestEnginePass1!",
+ )
+
+
+def campaign() -> Campaign:
+ return Campaign(
+ id="camp-pivot-test-001",
+ name="Pivot Teardown Test Campaign",
+ client="ACME Corp",
+ scope=[ScopeEntry(cidr="10.0.0.0/8")],
+ operator="tester",
+ )
+
+
+class TestPivotTeardown:
+ @pytest.mark.asyncio
+ async def test_subprocess_terminated_after_plan_completion(
+ self, settings: AresSettings, campaign: Campaign
+ ) -> None:
+ """SSH subprocess is terminated and waited on after campaign plan finishes normally."""
+ engine = AresEngine(settings=settings)
+ engine.load_modules()
+
+ plan = ExecutionPlan().add_stage("pivot_stage", ["network.pivot"])
+ dispatch_ctx = _mint_test_plan_context(engine, campaign.id, tuple(plan.all_module_ids()))
+
+ # Mock the pivot module run so it succeeds
+ pivot_cls = engine.registry.get("network.pivot")
+ fast_run = AsyncMock(return_value=([], {"tunnel_id": "tun-001"}))
+
+ # Setup mock tunnel and process
+ mock_proc = MagicMock()
+ mock_proc.terminate = MagicMock()
+ mock_proc.wait = MagicMock(return_value=0)
+ mock_proc.kill = MagicMock()
+
+ mock_tunnel = PivotTunnel()
+ mock_tunnel.tunnel_id = "tun-001"
+ mock_tunnel.state = TunnelState.ACTIVE
+ mock_tunnel._proc = mock_proc
+ mock_tunnel._conn = None
+
+ mock_pm = MagicMock(spec=PivotManager)
+ mock_pm.all_tunnels.return_value = [mock_tunnel]
+ mock_pm.teardown_all.return_value = 1
+
+ with patch.object(pivot_cls, "run", fast_run):
+ with patch.dict(_PIVOT_MANAGERS, {campaign.id: mock_pm}):
+ results = await engine.run_plan(
+ plan,
+ campaign,
+ dispatch_context=dispatch_ctx,
+ )
+
+ assert "network.pivot" in results
+ # teardown_all or tunnel proc termination should have been invoked
+ assert mock_pm.teardown_all.called or mock_proc.terminate.called
+ assert campaign.id not in _PIVOT_MANAGERS
+
+ @pytest.mark.asyncio
+ async def test_teardown_runs_in_finally_on_plan_exception(
+ self, settings: AresSettings, campaign: Campaign
+ ) -> None:
+ """Teardown is guaranteed to run in finally even when plan fails midway with an exception."""
+ engine = AresEngine(settings=settings)
+ engine.load_modules()
+
+ plan = ExecutionPlan().add_stage("failing_stage", ["network.pivot"])
+ dispatch_ctx = _mint_test_plan_context(engine, campaign.id, tuple(plan.all_module_ids()))
+
+ # Setup mock manager with process
+ mock_proc = MagicMock()
+ mock_proc.terminate = MagicMock()
+ mock_proc.wait = MagicMock(return_value=0)
+ mock_proc.kill = MagicMock()
+
+ mock_tunnel = PivotTunnel()
+ mock_tunnel.tunnel_id = "tun-fail-001"
+ mock_tunnel.state = TunnelState.ACTIVE
+ mock_tunnel._proc = mock_proc
+ mock_tunnel._conn = None
+
+ mock_pm = MagicMock(spec=PivotManager)
+ mock_pm.all_tunnels.return_value = [mock_tunnel]
+ mock_pm.teardown_all.return_value = 1
+
+ # Simulate exception during stage loop by mocking _guarded_run to raise directly
+ with patch.object(
+ engine, "_guarded_run", AsyncMock(side_effect=RuntimeError("Stage simulation crash"))
+ ):
+ with patch.dict(_PIVOT_MANAGERS, {campaign.id: mock_pm}):
+ results = await engine.run_plan(
+ plan,
+ campaign,
+ dispatch_context=dispatch_ctx,
+ )
+
+ # Plan results should capture the unhandled exception
+ assert "network.pivot" in results
+ assert results["network.pivot"].status.value == "failed"
+ # Teardown should have been executed in finally block
+ assert mock_pm.teardown_all.called or mock_proc.terminate.called
+ assert campaign.id not in _PIVOT_MANAGERS
+
+ @pytest.mark.asyncio
+ async def test_teardown_timeout_does_not_mask_original_exception(
+ self, settings: AresSettings, campaign: Campaign
+ ) -> None:
+ """Teardown timeout or error does not mask the original plan failure."""
+ engine = AresEngine(settings=settings)
+ engine.load_modules()
+
+ plan = ExecutionPlan().add_stage("stage1", ["network.pivot"])
+ dispatch_ctx = _mint_test_plan_context(engine, campaign.id, tuple(plan.all_module_ids()))
+
+ # Mock manager whose teardown_all raises TimeoutError
+ mock_pm = MagicMock(spec=PivotManager)
+ mock_pm.teardown_all = MagicMock(side_effect=asyncio.TimeoutError("Teardown timed out"))
+ mock_pm.all_tunnels.return_value = []
+
+ with patch.object(
+ engine, "_guarded_run", AsyncMock(side_effect=ValueError("Original plan error"))
+ ):
+ with patch.dict(_PIVOT_MANAGERS, {campaign.id: mock_pm}):
+ results = await engine.run_plan(
+ plan,
+ campaign,
+ dispatch_context=dispatch_ctx,
+ )
+
+ # Original error must be reflected in results, not masked by Teardown timeout
+ assert "network.pivot" in results
+ assert "Original plan error" in (results["network.pivot"].error or "")
+ assert campaign.id not in _PIVOT_MANAGERS
+
+ @pytest.mark.asyncio
+ async def test_pivot_subprocess_force_killed_on_hang(self) -> None:
+ """If proc.wait() times out, force kill() is invoked."""
+ mock_proc = MagicMock()
+ mock_proc.terminate = MagicMock()
+ mock_proc.wait = MagicMock(side_effect=subprocess.TimeoutExpired(cmd="ssh", timeout=3))
+ mock_proc.kill = MagicMock()
+
+ pm = PivotManager(operator="tester")
+ tunnel = PivotTunnel()
+ tunnel.tunnel_id = "tun-hang-001"
+ tunnel._proc = mock_proc
+ pm._tunnels[tunnel.tunnel_id] = tunnel
+
+ # Teardown tunnel
+ pm.teardown(tunnel.tunnel_id)
+
+ mock_proc.terminate.assert_called_once()
+ mock_proc.kill.assert_called_once()
+ assert tunnel.tunnel_id not in pm._tunnels
+
+ @pytest.mark.asyncio
+ async def test_pivot_module_teardown_handles_dead_process(
+ self, settings: AresSettings, campaign: Campaign
+ ) -> None:
+ """ProcessLookupError when process is already dead does not cause teardown to crash."""
+ from ares.core.noise import NoiseController
+
+ noise = NoiseController(campaign)
+ mod = PivotModule(settings=settings, campaign=campaign, noise=noise)
+ mock_proc = MagicMock()
+ mock_proc.terminate = MagicMock(side_effect=ProcessLookupError("No such process"))
+ mock_proc.kill = MagicMock()
+
+ mock_tunnel = PivotTunnel()
+ mock_tunnel.tunnel_id = "tun-dead"
+ mock_tunnel._proc = mock_proc
+ mock_tunnel._conn = None
+
+ mock_pm = MagicMock()
+ mock_pm.all_tunnels.return_value = [mock_tunnel]
+ mock_pm.teardown_all.return_value = 1
+
+ camp_id = "camp-dead-proc"
+ _PIVOT_MANAGERS[camp_id] = mock_pm
+
+ # Teardown should finish cleanly without ProcessLookupError
+ await mod.teardown(campaign_id=camp_id)
+ assert camp_id not in _PIVOT_MANAGERS