Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(actions): prove response actions took effect, and govern them per tenant (#664)
Beenu Arora9ca49afa52fa
diff --git a/services/actions/app/api/live_actions_router.py b/services/actions/app/api/live_actions_router.pyindex be2ddc3d8..fd15ba130 100644--- a/services/actions/app/api/live_actions_router.py+++ b/services/actions/app/api/live_actions_router.py@@ -32,7 +32,7 @@ from __future__ import annotationsfrom typing import Literalimport structlog-from fastapi import APIRouter+from fastapi import APIRouter, Dependsfrom pydantic import BaseModel, Fieldfrom app.live_actions import (@@ -44,9 +44,20 @@ from app.live_actions import (list_descriptors,list_vendors_for_capability,)+from app.security.authz import require_service_authlogger = structlog.get_logger(__name__)-router = APIRouter(prefix="/live-actions", tags=["live-actions"])+# Router-level auth. `/dispatch` executes real vendor actions — isolating a+# host, disabling an account, blocking an IP — and was reachable by anyone who+# could reach the port, while the legacy `app.api.router` next to it had+# required a service token on every route since it was written. Applying the+# dependency to the router rather than per-route means a route added later is+# protected by default instead of by remembering to decorate it.+router = APIRouter(+ prefix="/live-actions",+ tags=["live-actions"],+ dependencies=[Depends(require_service_auth)],+)class LiveActionDiscoveryResponse(BaseModel):diff --git a/services/actions/app/clients/crowdstrike_rtr.py b/services/actions/app/clients/crowdstrike_rtr.pyindex 084814284..6467c59be 100644--- a/services/actions/app/clients/crowdstrike_rtr.py+++ b/services/actions/app/clients/crowdstrike_rtr.py@@ -66,6 +66,39 @@ class CrowdStrikeRTRClient:resources = resp.json().get("resources", [])return resources[0] if resources else None+ async def get_containment_status(self, device_id: str) -> str | None:+ """Read a device's actual containment state.++ Returns CrowdStrike's ``status`` string — ``"contained"``,+ ``"containment_pending"``, ``"lift_containment_pending"`` or+ ``"normal"`` — or ``None`` when the device cannot be read.++ This exists so post-action verification can prove containment took+ effect. Resolving a hostname to a device id (``get_device_id``) only+ proves the host exists, so a verifier built on it would certify an+ uncontained host as verified.+ """+ async with httpx.AsyncClient(timeout=20.0) as client:+ await self._ensure_token(client)+ resp = await client.get(+ f"{self._base_url}/devices/entities/devices/v2",+ headers=self._auth_headers(),+ params={"ids": device_id},+ )+ if resp.status_code == 401:+ await self._authenticate(client)+ resp = await client.get(+ f"{self._base_url}/devices/entities/devices/v2",+ headers=self._auth_headers(),+ params={"ids": device_id},+ )+ resp.raise_for_status()+ resources = resp.json().get("resources", [])+ if not resources:+ return None+ status = resources[0].get("status")+ return str(status) if status is not None else None+async def contain_host(self, device_id: str) -> dict[str, Any]:"""Put a host into network containment via RTR."""async with httpx.AsyncClient(timeout=30.0) as client:diff --git a/services/actions/app/executors/endpoint.py b/services/actions/app/executors/endpoint.pyindex b597714bf..f3c0cb74e 100644--- a/services/actions/app/executors/endpoint.py+++ b/services/actions/app/executors/endpoint.py@@ -39,7 +39,7 @@ from app.clients.crowdstrike_rtr import CrowdStrikeRTRClientfrom app.clients.defender_client import DefenderClientfrom app.clients.sentinelone_client import SentinelOneClientfrom app.executors.base import _SIM_FUNNEL_CTA, BaseExecutor-from app.models.action import ActionRequest, ActionResult, ActionStatus, BlastRadius+from app.models.action import ActionRequest, ActionResult, ActionStatus, ActionType, BlastRadiuslogger = structlog.get_logger()@@ -249,10 +249,27 @@ class IsolateHostExecutor(BaseExecutor):)async def rollback(self, result: ActionResult) -> bool:+ """De-isolate the host by actually calling the EDR.++ This used to log "Rolling back isolate_host (de-isolating)" and return+ True without contacting any vendor, so an operator who clicked rollback+ was told the host was released while it stayed contained.+ `reverse_action` holds the real lift-containment calls for CrowdStrike,+ Defender and SentinelOne; it reports `simulated` when credentials are+ absent rather than claiming success.+ """+ # Imported inline, not at module scope: app.services.rollback imports+ # the vendor client factories from this module, so a top-level import+ # here would be circular.+ from app.services.rollback import reverse_via_rollback_service # noqa: PLC0415+hostname = result.rollback_data.get("hostname")- vendor = result.rollback_data.get("vendor")- logger.info("Rolling back isolate_host (de-isolating)", hostname=hostname, vendor=vendor)- return True+ return await reverse_via_rollback_service(+ ActionType.ISOLATE_HOST,+ hostname,+ result.rollback_data,+ logger,+ )class QuarantineFileExecutor(BaseExecutor):diff --git a/services/actions/app/executors/identity.py b/services/actions/app/executors/identity.pyindex f7addf99d..fd48afacd 100644--- a/services/actions/app/executors/identity.py+++ b/services/actions/app/executors/identity.py@@ -42,7 +42,7 @@ from app.clients.azure_entra_client import AzureEntraClientfrom app.clients.google_workspace_client import GoogleWorkspaceClientfrom app.clients.okta_client import OktaClientfrom app.executors.base import _SIM_FUNNEL_CTA, BaseExecutor-from app.models.action import ActionRequest, ActionResult, ActionStatus, BlastRadius+from app.models.action import ActionRequest, ActionResult, ActionStatus, ActionType, BlastRadiuslogger = structlog.get_logger()@@ -403,10 +403,19 @@ class SuspendSessionExecutor(BaseExecutor):)async def rollback(self, result: ActionResult) -> bool:+ """Un-suspend the session by actually calling the identity provider."""+ # Imported inline, not at module scope: app.services.rollback imports+ # the vendor client factories from this module, so a top-level import+ # here would be circular.+ from app.services.rollback import reverse_via_rollback_service # noqa: PLC0415+user_id = result.rollback_data.get("user_id")- vendor = result.rollback_data.get("vendor")- logger.info("Rolling back suspend_session (un-suspending)", user=user_id, vendor=vendor)- return True+ return await reverse_via_rollback_service(+ ActionType.SUSPEND_SESSION,+ user_id,+ result.rollback_data,+ logger,+ )class ForceMFAExecutor(BaseExecutor):diff --git a/services/actions/app/executors/network.py b/services/actions/app/executors/network.pyindex 21f07de87..146f0afee 100644--- a/services/actions/app/executors/network.py+++ b/services/actions/app/executors/network.py@@ -277,6 +277,40 @@ class BlockIPExecutor(BaseExecutor):return True+async def read_back_blocked_ip(ip: str, params: dict) -> bool | None:+ """Re-read the enforcing rule set to confirm an IP block is actually in place.++ Used by post-action verification. Returns True when a rule covering ``ip``+ is present, False when the rule set is readable and the IP is absent (the+ block silently did not take, or was removed), and None when we cannot tell.++ Only AWS security groups expose a read-back today. The PAN-OS, FortiGate+ and Cloudflare arms return None rather than a guess, because reporting+ VERIFIED without a confirming query is the exact failure this is meant to+ prevent.+ """+ aws = _aws_client(params)+ if aws is None:+ return None+ try:+ rules = await aws.describe_rules(params.get("aws_sg_id"))+ except Exception as exc: # noqa: BLE001 — indeterminate, never a false VERIFIED+ logger.warning("block_ip.read_back_failed", ip=ip, error=str(exc))+ return None+ if not rules:+ # An empty list also means "boto3 unavailable" in this client, so it is+ # not safe to read as "the rule is absent".+ return None+ needle = f"{ip}/32" if "/" not in ip else ip+ for rule in rules:+ if rule.get("IsEgress"):+ continue+ cidr = rule.get("CidrIpv4") or rule.get("CidrIpv6") or ""+ if cidr in {needle, ip}:+ return True+ return False++class AllowIPExecutor(BaseExecutor):"""Allows an IP address through an AWS Security Group (removes a block rule).diff --git a/services/actions/app/live_actions/dispatcher.py b/services/actions/app/live_actions/dispatcher.pyindex a29175056..eba932bd4 100644--- a/services/actions/app/live_actions/dispatcher.py+++ b/services/actions/app/live_actions/dispatcher.py@@ -31,14 +31,24 @@ so individual executors never have to think about them:from __future__ import annotationsimport os+from typing import Anyfrom uuid import uuid4import structlogfrom app.models.action import ActionRequest, ActionType-from app.services.autonomy_safety import AutonomyDecision, AutonomyMode, decide+from app.services.autonomy_safety import (+ ACTION_BLAST_RADIUS,+ AutonomyDecision,+ AutonomyMode,+ BlastRadius,+ decide,+ rollback_capability,+)from app.services.credential_resolver import resolve_paramsfrom app.services.maturity import MaturityTier+from app.services.tenant_policy import TenantPolicy, resolve_tenant_policy+from app.services.verification import PostActionVerifier, VerificationOutcomefrom . import registryfrom .models import LiveActionRequest, LiveActionResult, LiveActionStatus@@ -76,7 +86,19 @@ def _action_type_for(request: LiveActionRequest, executor: object) -> ActionTypereturn None-def _govern(request: LiveActionRequest, action_type: ActionType) -> AutonomyDecision:+async def _govern(request: LiveActionRequest, action_type: ActionType) -> AutonomyDecision:+ """Resolve the autonomy verdict from the *tenant's* policy.++ This used to be `decide(action_request, tier=configured_tier())`: one+ deployment-wide environment variable for every tenant, with the+ `whitelisted` flag left at its default of False so the L4 break-glass path+ for HIGH-blast-radius actions was unreachable. A tenant who selected L2 in+ the console got whatever the operator had exported.++ Now the tier, the per-action overrides and the whitelist all come from the+ tenant's stored policy, falling back to the environment only when there is+ no database to read.+ """action_request = ActionRequest(incident_id=request.case_id or uuid4(),tenant_id=request.tenant_id or uuid4(),@@ -85,7 +107,109 @@ def _govern(request: LiveActionRequest, action_type: ActionType) -> AutonomyDeciparameters=request.params,requested_by=request.requested_by,)- return decide(action_request, tier=configured_tier())+ policy = await resolve_tenant_policy(request.tenant_id)++ # A tenant override is a deliberate, explicit instruction for one action+ # type, so it outranks the tier ladder in both directions.+ override = policy.action_overrides.get(action_type.value, {}) or {}+ if override.get("block"):+ return _override_decision(+ action_type,+ policy,+ AutonomyMode.BLOCKED,+ f"action type '{action_type.value}' is blocked by tenant policy",+ )+ if override.get("force_auto"):+ return _override_decision(+ action_type,+ policy,+ AutonomyMode.AUTO,+ f"action type '{action_type.value}' is force-auto by tenant policy",+ )++ return decide(+ action_request,+ tier=policy.tier,+ whitelisted=policy.is_whitelisted(action_type.value, request.target),+ )+++def _override_decision(+ action_type: ActionType,+ policy: TenantPolicy,+ mode: AutonomyMode,+ reason: str,+) -> AutonomyDecision:+ """Build a decision for an explicit per-action tenant override."""+ blast = ACTION_BLAST_RADIUS.get(action_type, BlastRadius.HIGH)+ return AutonomyDecision(+ mode=mode,+ blast_radius=blast,+ tier=policy.tier,+ rollback=rollback_capability(action_type),+ # A force-auto override still earns verification at MEDIUM blast and+ # above: choosing to skip the approval queue is not the same as+ # choosing to skip the proof that the action took effect.+ requires_verification=(mode is AutonomyMode.AUTO and blast is not BlastRadius.MINIMAL),+ reason=f"{reason} (source: {policy.source})",+ )+++async def _verify_effect(+ request: LiveActionRequest,+ result: LiveActionResult,+ action_type: ActionType,+ log: Any,+) -> LiveActionResult:+ """Re-query the vendor and record whether the action actually took effect.++ Three outcomes, all recorded honestly on ``result.details``:++ * ``verified`` — a confirming query ran and the effect is present.+ * ``failed`` — a confirming query ran and the effect is absent. The+ action reported success and did not take, which is a genuine alarm, so+ the result status is downgraded to FAILED rather than left COMPLETED.+ * ``unverified`` — no probe exists for this action/vendor, or credentials+ were absent. Reported as-is; never upgraded to a confirmation.++ A verifier crash can never fail the action itself — the action already+ happened, and losing the record of it would be worse than not verifying.+ """+ verifier = PostActionVerifier()+ try:+ # `request.params` already carries the resolved vendor credentials by+ # this point: auth_config was translated and cleared before dispatch.+ outcome = await verifier.verify(action_type, request.target or "", dict(request.params))+ except Exception as exc: # noqa: BLE001 — verification must not mask a completed action+ log.warning("live_action.verification_crashed", error=str(exc))+ details = dict(result.details)+ details["verification"] = "unverified"+ details["verification_reason"] = f"verifier raised {type(exc).__name__}"+ return result.model_copy(update={"details": details})++ details = dict(result.details)+ details["verification"] = outcome.outcome.value+ details["verification_reason"] = outcome.reason+ log.info(+ "live_action.verified",+ verification=outcome.outcome.value,+ reason=outcome.reason,+ )++ if outcome.outcome is VerificationOutcome.FAILED:+ # The vendor accepted the call and the effect is not there. Saying+ # SUCCEEDED here is how a SOC ends up believing a host is contained+ # when it is not.+ return result.model_copy(+ update={+ "status": LiveActionStatus.FAILED,+ "details": details,+ "error": result.error or f"action reported success but verification failed: {outcome.reason}",+ "summary": f"{result.summary} — VERIFICATION FAILED: {outcome.reason}",+ }+ )++ return result.model_copy(update={"details": details})def _not_executed(request: LiveActionRequest, status: LiveActionStatus, decision: AutonomyDecision) -> LiveActionResult:@@ -145,7 +269,7 @@ async def dispatch(request: LiveActionRequest) -> LiveActionResult:decision: AutonomyDecision | None = Noneaction_type = _action_type_for(request, executor)if action_type is not None and not request.dry_run:- decision = _govern(request, action_type)+ decision = await _govern(request, action_type)log = log.bind(autonomy_mode=decision.mode.value, blast=decision.blast_radius.value)if decision.mode is AutonomyMode.BLOCKED:log.warning("live_action.blocked_by_policy", reason=decision.reason)@@ -193,6 +317,28 @@ async def dispatch(request: LiveActionRequest) -> LiveActionResult:details.setdefault("autonomy_reason", decision.reason)result = result.model_copy(update={"details": details})+ # ── Post-action verification ──────────────────────────────────────────+ #+ # The policy layer has always computed `requires_verification` for any+ # auto-executed action at MEDIUM blast or above, and this dispatcher then+ # discarded it: the field was never read, and PostActionVerifier had no+ # caller anywhere outside its own test. So the product could show that a+ # containment call was accepted, and never that containment took effect —+ # which is the single capability the market treats as the dividing line+ # between recommending and responding.+ #+ # Verification runs only for a real execution that actually succeeded. A+ # dry run and a SIMULATED result have nothing to verify, and a failed+ # execution is already an honest negative.+ if (+ decision is not None+ and decision.requires_verification+ and not request.dry_run+ and result.status is LiveActionStatus.SUCCEEDED+ and action_type is not None+ ):+ result = await _verify_effect(request, result, action_type, log)+log.info("live_action.completed",status=result.status.value,diff --git a/services/actions/app/services/rollback.py b/services/actions/app/services/rollback.pyindex 0b41e2cca..d1f1544dc 100644--- a/services/actions/app/services/rollback.py+++ b/services/actions/app/services/rollback.py@@ -173,3 +173,40 @@ async def reverse_action(action_type: ActionType, target: str, params: dict[str,except Exception as exc: # noqa: BLE001 — a failed reverse must be reported, not hiddenlogger.error("rollback.reverse_failed", action=action_type.value, target=target, error=str(exc))return RollbackResult(True, False, False, None, f"reverse call failed: {exc}")+++async def reverse_via_rollback_service(+ action_type: ActionType,+ target: str | None,+ rollback_data: dict[str, Any] | None,+ log: Any = None,+) -> bool:+ """Adapter so an executor's ``rollback()`` can use the real reverse calls.++ Every executor in ``endpoint.py``, ``identity.py`` and ``siem.py`` used to+ log an intent and ``return True`` without contacting the vendor, which told+ an operator the action had been undone when nothing had happened. The real+ reverse calls already existed here and had no caller outside their own+ test; this bridges the two.++ Returns True when the reverse really ran or was an honest credential-less+ simulation, and False when a reverse exists and genuinely failed — so a+ caller can distinguish "undone" from "could not undo".+ """+ result = await reverse_action(action_type, target or "", dict(rollback_data or {}))+ emit = (log or logger).info if result.ok else (log or logger).error+ emit(+ "rollback.executor_reverse",+ action=action_type.value,+ target=target,+ vendor=result.vendor,+ reversed=result.reversed_,+ simulated=result.simulated,+ supported=result.supported,+ reason=result.reason,+ )+ # An unsupported reverse is not a failure of this call — there is simply+ # nothing to undo for that action type (e.g. a notification).+ if not result.supported:+ return True+ return result.okdiff --git a/services/actions/app/services/tenant_policy.py b/services/actions/app/services/tenant_policy.pynew file mode 100644index 000000000..5b6e3fde8--- /dev/null+++ b/services/actions/app/services/tenant_policy.py@@ -0,0 +1,220 @@+"""Per-tenant autonomy policy, read from the store the product already writes.++The console lets a tenant choose their L0–L4 automation tier, plus per-action+overrides and a HIGH-blast-radius whitelist. All three persist in Postgres+(`remediation_maturity`, `remediation_whitelist`) behind+`/api/v1/remediation/config`.++The live-action dispatcher never read any of it. It resolved autonomy from a+single deployment-wide `AISOC_MATURITY_TIER` environment variable, so every+tenant in a multi-tenant install shared one posture and a tenant who chose L2+in the UI got whatever the operator had exported. `evaluate_gate` — the+function that does understand per-tenant tiers, overrides and whitelists — had+no callers and no tests anywhere in the tree.++This module closes that gap. It reads the tenant's real policy and is the+source of truth `_govern` consults.++Safety posture when the policy cannot be read:++* **No database configured** — fall back to `AISOC_MATURITY_TIER`. That is an+ explicit operator choice and the documented single-tenant behaviour.+* **Database configured but unreadable** — fall back to a conservative floor+ (L1, notify-only), never to the environment value. If we know a per-tenant+ policy exists and cannot read it, inheriting a possibly-more-permissive+ default is how an install silently auto-executes containment the tenant+ never authorised.+"""++from __future__ import annotations++import os+import time+from dataclasses import dataclass, field+from datetime import UTC, datetime+from typing import Any+from uuid import UUID++import structlog++from app.services.maturity import MaturityTier++logger = structlog.get_logger(__name__)++#: Tier used when a configured database cannot be read. Notify-only: the+#: platform still surfaces everything, it just does not act unilaterally.+UNREADABLE_POLICY_FLOOR = MaturityTier.L1_NOTIFY++#: Policy changes are rare and the live-action path is latency-sensitive, so a+#: short TTL keeps the common case to zero database round trips while still+#: picking up a tier change within seconds.+_CACHE_TTL_SECONDS = 30.0++_cache: dict[str, tuple[float, TenantPolicy]] = {}+++@dataclass(frozen=True)+class TenantPolicy:+ """A tenant's resolved autonomy posture."""++ tier: MaturityTier+ action_overrides: dict[str, Any] = field(default_factory=dict)+ whitelist: list[dict[str, Any]] = field(default_factory=list)+ #: True when this came from the tenant's stored policy, False when it came+ #: from the environment default or the conservative floor. Surfaced in the+ #: audit trail so an operator can tell a real tenant choice from a default.+ from_store: bool = False+ source: str = "environment"++ def is_whitelisted(self, action_type: str, target: str | None) -> bool:+ """True when an unexpired whitelist entry covers this action/target.++ `autonomy_safety.decide` accepts a `whitelisted` flag that gates the+ L4 break-glass path for HIGH-blast-radius actions. The dispatcher never+ passed it, so it was always False and that path was unreachable.+ """+ now = datetime.now(UTC)+ for entry in self.whitelist:+ if entry.get("action_type") != action_type:+ continue+ expires_at = entry.get("expires_at")+ if expires_at is not None and _as_utc(expires_at) <= now:+ continue+ constraints = entry.get("constraints") or {}+ targets = constraints.get("targets")+ # No target constraint means the entry covers the action type+ # broadly; an explicit list must actually contain this target.+ if targets and target not in targets:+ continue+ return True+ return False+++def _as_utc(value: Any) -> datetime:+ """Coerce a stored timestamp to an aware UTC datetime."""+ if isinstance(value, datetime):+ return value if value.tzinfo else value.replace(tzinfo=UTC)+ return datetime.fromisoformat(str(value).replace("Z", "+00:00"))+++def _env_tier() -> MaturityTier:+ """Deployment-wide default. Conservative when unset or unparseable."""+ raw = os.environ.get("AISOC_MATURITY_TIER", "").strip().upper()+ if not raw:+ return MaturityTier.L1_NOTIFY+ for tier in MaturityTier:+ if raw in {tier.name, tier.name.split("_")[0], str(tier.value)}:+ return tier+ logger.warning("tenant_policy.bad_tier_env", value=raw)+ return MaturityTier.L1_NOTIFY+++def _dsn() -> str | None:+ """Resolve an asyncpg-compatible DSN, or None when no database is set.++ Accepts the SQLAlchemy form the API uses (`postgresql+asyncpg://`) as well+ as the plain pgx form `services/ingest` uses, because both spellings are+ already present in deployed environments.+ """+ raw = os.environ.get("DATABASE_DSN") or os.environ.get("DATABASE_URL") or ""+ raw = raw.strip()+ if not raw:+ return None+ return raw.replace("postgresql+asyncpg://", "postgresql://").replace("postgres+asyncpg://", "postgresql://")+++async def _load_from_store(tenant_id: str, dsn: str) -> TenantPolicy | None:+ """Read the tenant's tier, overrides and live whitelist entries.++ Returns None when the tenant has no stored policy row, which is different+ from a read failure: a tenant who has never configured anything should get+ the deployment default, not the conservative floor.+ """+ import asyncpg # noqa: PLC0415 — optional at import time; only needed with a DSN++ conn = await asyncpg.connect(dsn, timeout=5.0)+ try:+ row = await conn.fetchrow(+ "SELECT maturity_tier, action_overrides FROM remediation_maturity WHERE tenant_id = $1",+ UUID(tenant_id),+ )+ if row is None:+ return None+ whitelist_rows = await conn.fetch(+ """+ SELECT action_type, blast_radius, constraints, expires_at+ FROM remediation_whitelist+ WHERE tenant_id = $1 AND (expires_at IS NULL OR expires_at > now())+ """,+ UUID(tenant_id),+ )+ finally:+ await conn.close()++ overrides = row["action_overrides"]+ if isinstance(overrides, str):+ import json # noqa: PLC0415 — JSONB arrives as str on some driver versions++ overrides = json.loads(overrides)++ whitelist: list[dict[str, Any]] = []+ for wl in whitelist_rows:+ constraints = wl["constraints"]+ if isinstance(constraints, str):+ import json # noqa: PLC0415++ constraints = json.loads(constraints)+ whitelist.append(+ {+ "action_type": wl["action_type"],+ "blast_radius": wl["blast_radius"],+ "constraints": constraints or {},+ "expires_at": wl["expires_at"],+ }+ )++ return TenantPolicy(+ tier=MaturityTier(int(row["maturity_tier"])),+ action_overrides=overrides or {},+ whitelist=whitelist,+ from_store=True,+ source="tenant_policy",+ )+++async def resolve_tenant_policy(tenant_id: str | UUID | None) -> TenantPolicy:+ """Resolve the autonomy policy that governs this tenant's actions."""+ dsn = _dsn()+ if tenant_id is None or dsn is None:+ # Single-tenant or no database: the environment value is the operator's+ # explicit, documented choice.+ return TenantPolicy(tier=_env_tier(), source="environment")++ key = str(tenant_id)+ cached = _cache.get(key)+ now = time.monotonic()+ if cached is not None and now - cached[0] < _CACHE_TTL_SECONDS:+ return cached[1]++ try:+ policy = await _load_from_store(key, dsn)+ except Exception as exc: # noqa: BLE001 — never inherit a permissive default on error+ logger.error(+ "tenant_policy.read_failed",+ tenant_id=key,+ error=str(exc),+ falling_back_to=UNREADABLE_POLICY_FLOOR.name,+ )+ return TenantPolicy(tier=UNREADABLE_POLICY_FLOOR, source="unreadable_policy_floor")++ if policy is None:+ logger.info("tenant_policy.no_row", tenant_id=key)+ policy = TenantPolicy(tier=_env_tier(), source="environment")++ _cache[key] = (now, policy)+ return policy+++def clear_cache() -> None:+ """Drop the policy cache. Used by tests and after a policy update."""+ _cache.clear()diff --git a/services/actions/app/services/verification.py b/services/actions/app/services/verification.pyindex eb0870bab..181c4a2a2 100644--- a/services/actions/app/services/verification.py+++ b/services/actions/app/services/verification.py@@ -55,24 +55,53 @@ class VerificationResult:Probe = Callable[[str, dict[str, Any]], Awaitable[bool | None]]+#: CrowdStrike device states that mean containment is actually in force.+#: ``containment_pending`` is deliberately excluded: the request was accepted+#: but the host is not contained yet, which is precisely the "the API returned+#: 200" condition this module exists to distinguish from a real effect.+_CONTAINED_STATES = {"contained"}++async def _probe_isolate(target: str, params: dict[str, Any]) -> bool | None:- """Confirm host isolation by re-querying the EDR.+ """Confirm host isolation by reading the EDR's containment state.++ This previously returned ``bool(device_id)`` — i.e. "does this hostname+ resolve to a device". That is true of every host in the fleet, contained or+ not, so it would have certified an uncontained host as VERIFIED. Now it+ reads the device's actual ``status``.- Today only CrowdStrike exposes enough to re-query (the device must resolve);- absent CrowdStrike credentials we return None → UNVERIFIED (honest).+ Absent CrowdStrike credentials we return None → UNVERIFIED (honest)."""cs = _cs_client(params)if cs is None:return Nonedevice_id = await cs.get_device_id(target)- # A resolvable device id is a real (if partial) confirmation the target- # exists and the containment call targeted a real host. Fuller state- # verification lands when the client exposes a containment-status read.- return bool(device_id)+ if not device_id:+ # The host cannot be found at all, so containment cannot be confirmed.+ # Indeterminate rather than FAILED: a renamed or decommissioned host is+ # not the same fact as "containment did not take".+ return None+ status = await cs.get_containment_status(device_id)+ if status is None:+ return None+ return status.lower() in _CONTAINED_STATES+++async def _probe_block_ip(target: str, params: dict[str, Any]) -> bool | None:+ """Confirm an IP block by re-reading the enforcing rule set.++ Only AWS security groups are readable today; every other vendor arm of+ ``BlockIPExecutor`` has no read-back, so this reports indeterminate rather+ than inventing a confirmation.+ """+ from app.executors.network import read_back_blocked_ip # noqa: PLC0415++ return await read_back_blocked_ip(target, params)_DEFAULT_PROBES: dict[ActionType, Probe] = {ActionType.ISOLATE_HOST: _probe_isolate,+ ActionType.BLOCK_IP: _probe_block_ip,}diff --git a/services/actions/tests/test_dispatch_tenant_governance.py b/services/actions/tests/test_dispatch_tenant_governance.pynew file mode 100644index 000000000..810f52014--- /dev/null+++ b/services/actions/tests/test_dispatch_tenant_governance.py@@ -0,0 +1,169 @@+"""The dispatcher must govern by the tenant's policy, not a global env var.++Companion to `test_tenant_policy.py`, which covers resolution in isolation.+These go through `dispatch()` because the defect was the wiring: the resolution+logic for per-tenant tiers existed in `maturity.evaluate_gate` and had no+caller anywhere in the tree.+"""++from __future__ import annotations++import uuid++import pytest+from app.live_actions import dispatcher as dispatcher_mod+from app.live_actions import registry+from app.live_actions.dispatcher import dispatch+from app.live_actions.models import LiveActionRequest, LiveActionResult, LiveActionStatus+from app.models.action import ActionType+from app.services.maturity import MaturityTier+from app.services.tenant_policy import TenantPolicy++pytestmark = pytest.mark.asyncio+++class _OkExecutor:+ _legacy_action_type = ActionType.ISOLATE_HOST++ async def execute(self, request: LiveActionRequest) -> LiveActionResult:+ return LiveActionResult(+ request_id=request.request_id,+ status=LiveActionStatus.SUCCEEDED,+ capability=request.capability,+ vendor_id=request.vendor_id,+ summary="isolated",+ )+++def _request(**overrides) -> LiveActionRequest:+ payload = {+ "request_id": uuid.uuid4(),+ "capability": "isolate_host",+ "vendor_id": "crowdstrike",+ "target": "WIN-DC01",+ "dry_run": False,+ "tenant_id": uuid.uuid4(),+ "requested_by": "[email protected]",+ }+ payload.update(overrides)+ return LiveActionRequest(**payload)++[email protected](autouse=True)+def _executor_and_no_verify(monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr(registry, "get_executor", lambda vendor, cap: _OkExecutor())++ # Keep these tests about governance: a stub verifier so a real probe never+ # runs and the assertions are about the autonomy verdict only.+ class _V:+ async def verify(self, action_type, target, params): # noqa: ANN001+ from app.services.verification import VerificationOutcome, VerificationResult++ return VerificationResult(VerificationOutcome.UNVERIFIED, action_type, target, reason="stub")++ monkeypatch.setattr(dispatcher_mod, "PostActionVerifier", lambda: _V())+++def _policy(monkeypatch: pytest.MonkeyPatch, policy: TenantPolicy):+ async def _resolve(tenant_id): # noqa: ANN001, ANN202+ return policy++ monkeypatch.setattr(dispatcher_mod, "resolve_tenant_policy", _resolve)+++async def test_an_observe_only_tenant_does_not_auto_execute(+ monkeypatch: pytest.MonkeyPatch,+):+ """L0 means the platform watches. Isolating a host is not watching."""+ _policy(monkeypatch, TenantPolicy(tier=MaturityTier.L0_OBSERVE, from_store=True))+ result = await dispatch(_request())+ assert result.status is not LiveActionStatus.SUCCEEDED+++async def test_a_tenant_blocklist_entry_blocks_the_action(+ monkeypatch: pytest.MonkeyPatch,+):+ _policy(+ monkeypatch,+ TenantPolicy(+ tier=MaturityTier.L4_AUTOMATE,+ action_overrides={"isolate_host": {"block": True}},+ from_store=True,+ source="tenant_policy",+ ),+ )+ result = await dispatch(_request())+ assert result.status is LiveActionStatus.BLOCKED+ assert "blocked by tenant policy" in result.summary+++async def test_force_auto_override_still_requires_verification(+ monkeypatch: pytest.MonkeyPatch,+):+ """Skipping the approval queue is not the same as skipping the proof.++ A tenant who force-autos host isolation has chosen to act without a human+ in the loop, which makes read-back confirmation more important, not less.+ """+ _policy(+ monkeypatch,+ TenantPolicy(+ tier=MaturityTier.L2_CONTAIN,+ action_overrides={"isolate_host": {"force_auto": True}},+ from_store=True,+ source="tenant_policy",+ ),+ )+ result = await dispatch(_request())+ assert result.status is LiveActionStatus.SUCCEEDED+ assert result.details["verification"] == "unverified"+++async def test_whitelist_is_passed_through_to_the_autonomy_decision(+ monkeypatch: pytest.MonkeyPatch,+):+ """`decide(whitelisted=...)` was never passed, so L4 break-glass was dead.++ Asserted by observing that two otherwise identical L4 dispatches differ+ only by the presence of a whitelist entry.+ """+ seen: list[bool] = []+ real_decide = dispatcher_mod.decide++ def _spy(action_request, *, tier, whitelisted=False, **kw): # noqa: ANN001, ANN003+ seen.append(whitelisted)+ return real_decide(action_request, tier=tier, whitelisted=whitelisted, **kw)++ monkeypatch.setattr(dispatcher_mod, "decide", _spy)++ _policy(monkeypatch, TenantPolicy(tier=MaturityTier.L4_AUTOMATE, from_store=True))+ await dispatch(_request())++ _policy(+ monkeypatch,+ TenantPolicy(+ tier=MaturityTier.L4_AUTOMATE,+ whitelist=[{"action_type": "isolate_host", "constraints": {}, "expires_at": None}],+ from_store=True,+ ),+ )+ await dispatch(_request())++ assert seen == [False, True]+++async def test_the_policy_source_is_recorded_for_the_audit_trail(+ monkeypatch: pytest.MonkeyPatch,+):+ """An operator must be able to tell a tenant choice from a default."""+ _policy(+ monkeypatch,+ TenantPolicy(+ tier=MaturityTier.L4_AUTOMATE,+ action_overrides={"isolate_host": {"force_auto": True}},+ from_store=True,+ source="tenant_policy",+ ),+ )+ result = await dispatch(_request())+ assert "tenant_policy" in result.details["autonomy_reason"]diff --git a/services/actions/tests/test_dispatch_verification.py b/services/actions/tests/test_dispatch_verification.pynew file mode 100644index 000000000..61a5d6fa6--- /dev/null+++ b/services/actions/tests/test_dispatch_verification.py@@ -0,0 +1,212 @@+"""Verification must run on the live dispatch path, not just in its own test.++The market treats outcome verification as the line between recommending and+responding: proving the target's state actually changed, not that an API call+was accepted. AiSOC had the verifier and did not call it.++`AutonomyDecision.requires_verification` was computed for every auto-executed+action at MEDIUM blast or above and then dropped when the dispatcher assembled+its result. `PostActionVerifier` had no caller anywhere outside+`test_verification.py`, so the product could report that a containment call+succeeded and never that containment took effect.++These tests assert the wiring, which is the part that was missing. They are+deliberately written against `dispatch()` rather than the verifier, because a+passing verifier test is exactly what made this look covered.+"""++from __future__ import annotations++import uuid++import pytest+from app.live_actions import dispatcher as dispatcher_mod+from app.live_actions import registry+from app.live_actions.dispatcher import dispatch+from app.live_actions.models import LiveActionRequest, LiveActionResult, LiveActionStatus+from app.models.action import ActionType+from app.services.autonomy_safety import (+ AutonomyDecision,+ AutonomyMode,+ BlastRadius,+ RollbackCapability,+)+from app.services.maturity import MaturityTier+from app.services.verification import VerificationOutcome, VerificationResult++pytestmark = pytest.mark.asyncio+++class _OkExecutor:+ """Minimal executor that reports success without touching a vendor."""++ _legacy_action_type = ActionType.ISOLATE_HOST++ async def execute(self, request: LiveActionRequest) -> LiveActionResult:+ return LiveActionResult(+ request_id=request.request_id,+ status=LiveActionStatus.SUCCEEDED,+ capability=request.capability,+ vendor_id=request.vendor_id,+ summary="isolated",+ )+++def _decision(blast: BlastRadius, *, requires_verification: bool) -> AutonomyDecision:+ return AutonomyDecision(+ mode=AutonomyMode.AUTO,+ blast_radius=blast,+ tier=MaturityTier.L3_REMEDIATE,+ rollback=RollbackCapability.REVERSIBLE,+ requires_verification=requires_verification,+ reason="test",+ )+++def _stub_govern(blast: BlastRadius, *, requires_verification: bool):+ """`_govern` is async (it reads the tenant's stored policy), so the stub is too."""++ async def _govern(request, action_type): # noqa: ANN001, ANN202+ return _decision(blast, requires_verification=requires_verification)++ return _govern+++def _request(**overrides) -> LiveActionRequest:+ payload = {+ "request_id": uuid.uuid4(),+ "capability": "isolate_host",+ "vendor_id": "crowdstrike",+ "target": "WIN-DC01",+ "dry_run": False,+ "tenant_id": uuid.uuid4(),+ "requested_by": "[email protected]",+ }+ payload.update(overrides)+ return LiveActionRequest(**payload)++[email protected](autouse=True)+def _wire(monkeypatch: pytest.MonkeyPatch):+ """Executor always succeeds; autonomy always demands verification."""+ monkeypatch.setattr(registry, "get_executor", lambda vendor, cap: _OkExecutor())+ monkeypatch.setattr(+ dispatcher_mod,+ "_govern",+ _stub_govern(BlastRadius.MEDIUM, requires_verification=True),+ )+++def _stub_verifier(monkeypatch: pytest.MonkeyPatch, outcome: VerificationOutcome):+ class _V:+ async def verify(self, action_type, target, params): # noqa: ANN001+ return VerificationResult(outcome, action_type, target, reason=f"stub {outcome.value}")++ monkeypatch.setattr(dispatcher_mod, "PostActionVerifier", lambda: _V())+++async def test_verified_effect_is_recorded_on_the_result(monkeypatch: pytest.MonkeyPatch):+ _stub_verifier(monkeypatch, VerificationOutcome.VERIFIED)+ result = await dispatch(_request())+ assert result.status is LiveActionStatus.SUCCEEDED+ assert result.details["verification"] == "verified"+++async def test_unverified_effect_does_not_become_a_confirmation(monkeypatch: pytest.MonkeyPatch):+ """No probe or no credentials must stay honest, not upgrade to verified."""+ _stub_verifier(monkeypatch, VerificationOutcome.UNVERIFIED)+ result = await dispatch(_request())+ assert result.details["verification"] == "unverified"+ assert result.status is LiveActionStatus.SUCCEEDED+++async def test_action_that_reported_success_but_did_not_take_is_downgraded(+ monkeypatch: pytest.MonkeyPatch,+):+ """The case that matters: vendor said 200, the effect is absent.++ Leaving this COMPLETED is how a SOC comes to believe a host is contained+ when it is not.+ """+ _stub_verifier(monkeypatch, VerificationOutcome.FAILED)+ result = await dispatch(_request())+ assert result.status is LiveActionStatus.FAILED+ assert result.details["verification"] == "failed"+ assert result.error and "verification failed" in result.error+++async def test_a_dry_run_is_not_verified(monkeypatch: pytest.MonkeyPatch):+ """Nothing happened, so there is nothing to re-query."""+ called = False++ class _V:+ async def verify(self, *a, **k): # noqa: ANN002, ANN003+ nonlocal called+ called = True+ raise AssertionError("must not verify a dry run")++ monkeypatch.setattr(dispatcher_mod, "PostActionVerifier", lambda: _V())+ result = await dispatch(_request(dry_run=True))+ assert not called+ assert "verification" not in result.details+++async def test_a_simulated_result_is_not_verified(monkeypatch: pytest.MonkeyPatch):+ """SIMULATED is the credential-less safe path — nothing to re-query."""++ class _Simulating:+ _legacy_action_type = ActionType.ISOLATE_HOST++ async def execute(self, request: LiveActionRequest) -> LiveActionResult:+ return LiveActionResult(+ request_id=request.request_id,+ status=LiveActionStatus.SIMULATED,+ capability=request.capability,+ vendor_id=request.vendor_id,+ summary="simulated",+ )++ monkeypatch.setattr(registry, "get_executor", lambda vendor, cap: _Simulating())++ class _V:+ async def verify(self, *a, **k): # noqa: ANN002, ANN003+ raise AssertionError("must not verify a simulation")++ monkeypatch.setattr(dispatcher_mod, "PostActionVerifier", lambda: _V())+ result = await dispatch(_request())+ assert result.status is LiveActionStatus.SIMULATED+ assert "verification" not in result.details+++async def test_verifier_crash_never_loses_the_succeeded_action(+ monkeypatch: pytest.MonkeyPatch,+):+ """The action already happened. Losing the record of it is worse."""++ class _V:+ async def verify(self, *a, **k): # noqa: ANN002, ANN003+ raise RuntimeError("probe exploded")++ monkeypatch.setattr(dispatcher_mod, "PostActionVerifier", lambda: _V())+ result = await dispatch(_request())+ assert result.status is LiveActionStatus.SUCCEEDED+ assert result.details["verification"] == "unverified"+ assert "RuntimeError" in result.details["verification_reason"]+++async def test_verification_is_skipped_when_policy_does_not_require_it(+ monkeypatch: pytest.MonkeyPatch,+):+ monkeypatch.setattr(+ dispatcher_mod,+ "_govern",+ _stub_govern(BlastRadius.LOW, requires_verification=False),+ )++ class _V:+ async def verify(self, *a, **k): # noqa: ANN002, ANN003+ raise AssertionError("must not verify when not required")++ monkeypatch.setattr(dispatcher_mod, "PostActionVerifier", lambda: _V())+ result = await dispatch(_request())+ assert "verification" not in result.detailsdiff --git a/services/actions/tests/test_live_action_router.py b/services/actions/tests/test_live_action_router.pyindex 04c2e4e20..99f958c17 100644--- a/services/actions/tests/test_live_action_router.py+++ b/services/actions/tests/test_live_action_router.py@@ -25,6 +25,7 @@ from __future__ import annotationsfrom collections.abc import Iteratorimport pytest+from app.core.config import get_settingsfrom app.live_actions import (LiveActionExecutor,LiveActionRequest,@@ -74,7 +75,7 @@ class _StubBlockIP(LiveActionExecutor):@pytest.fixture-def client() -> Iterator[TestClient]:+def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:"""Yield a TestClient with a clean registry seeded with two stubs.We deliberately *don't* let the startup hook run with the full set@@ -82,7 +83,15 @@ def client() -> Iterator[TestClient]:the universe shifts under them. The router code path is identicalregardless of how many executors are registered, so two stubs areenough to cover the contract.++ Dev mode is on because these tests pin the REST contract, not the auth+ boundary. The auth boundary has its own tests below, which is the point:+ this router dispatches real vendor containment and until now carried no+ authentication at all."""+ monkeypatch.delenv("AISOC_ACTIONS_SERVICE_TOKEN", raising=False)+ monkeypatch.setenv("AISOC_DEV_MODE", "true")+ get_settings.cache_clear()reset_for_tests()register_executor(_StubIsolateHost(), source="builtin")register_executor(_StubBlockIP(), source="builtin")diff --git a/services/actions/tests/test_live_action_router_auth.py b/services/actions/tests/test_live_action_router_auth.pynew file mode 100644index 000000000..0119fac48--- /dev/null+++ b/services/actions/tests/test_live_action_router_auth.py@@ -0,0 +1,109 @@+"""The route that executes real vendor containment must require a token.++`/api/v1/live-actions/dispatch` reaches `dispatch()`, which runs the registered+executor against a live vendor: isolating a host, disabling an account,+blocking an IP. It carried no authentication of any kind, while the legacy+`app.api.router` mounted beside it had required a service token on every+mutating route since it was written.++The dependency is applied to the router rather than to each route, so these+tests check the whole surface rather than one handler — a route added later+should be protected without anyone remembering to decorate it.+"""++from __future__ import annotations++from uuid import uuid4++import pytest+from app.api.live_actions_router import router+from app.core.config import get_settings+from fastapi import FastAPI+from fastapi.testclient import TestClient++_BODY = {+ "request_id": str(uuid4()),+ "capability": "isolate_host",+ "vendor_id": "crowdstrike",+ "target": "WIN-DC01",+ "dry_run": True,+}+++def _client(monkeypatch: pytest.MonkeyPatch, **env: str) -> TestClient:+ for key in ("AISOC_ACTIONS_SERVICE_TOKEN", "AISOC_DEV_MODE"):+ monkeypatch.delenv(key, raising=False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ get_settings.cache_clear()+ app = FastAPI()+ app.include_router(router)+ return TestClient(app)+++def test_dispatch_rejects_a_caller_with_no_token(monkeypatch: pytest.MonkeyPatch):+ client = _client(monkeypatch, AISOC_ACTIONS_SERVICE_TOKEN="s3cr3t", AISOC_DEV_MODE="false")+ assert client.post("/live-actions/dispatch", json=_BODY).status_code == 401+++def test_dispatch_rejects_a_wrong_token(monkeypatch: pytest.MonkeyPatch):+ client = _client(monkeypatch, AISOC_ACTIONS_SERVICE_TOKEN="s3cr3t", AISOC_DEV_MODE="false")+ resp = client.post(+ "/live-actions/dispatch",+ json=_BODY,+ headers={"Authorization": "Bearer wrong"},+ )+ assert resp.status_code == 401+++def test_dispatch_accepts_the_configured_token(monkeypatch: pytest.MonkeyPatch):+ """Auth passes, so the request reaches the dispatcher.++ The capability/vendor pair is unregistered in this bare app, and the+ router's documented contract is to report executor problems inside the+ result body rather than as an HTTP error — so a 200 here means the request+ got past auth and into `dispatch()`, which is what we are asserting.+ """+ client = _client(monkeypatch, AISOC_ACTIONS_SERVICE_TOKEN="s3cr3t", AISOC_DEV_MODE="false")+ resp = client.post(+ "/live-actions/dispatch",+ json=_BODY,+ headers={"Authorization": "Bearer s3cr3t"},+ )+ assert resp.status_code == 200+++def test_unconfigured_token_fails_closed_in_production(monkeypatch: pytest.MonkeyPatch):+ """No token and no dev mode must refuse service, not serve openly."""+ client = _client(monkeypatch, AISOC_DEV_MODE="false")+ assert client.post("/live-actions/dispatch", json=_BODY).status_code == 503+++def test_dev_mode_keeps_the_local_stack_usable(monkeypatch: pytest.MonkeyPatch):+ client = _client(monkeypatch, AISOC_DEV_MODE="true")+ assert client.post("/live-actions/dispatch", json=_BODY).status_code == 200+++def test_the_dry_run_route_is_protected_too(monkeypatch: pytest.MonkeyPatch):+ """A dry run leaks the vendor/capability inventory, so it is not public."""+ client = _client(monkeypatch, AISOC_ACTIONS_SERVICE_TOKEN="s3cr3t", AISOC_DEV_MODE="false")+ assert client.post("/live-actions/dry-run", json=_BODY).status_code == 401+++def test_every_route_on_the_router_is_protected(monkeypatch: pytest.MonkeyPatch):+ """Router-level, not per-route: a new endpoint inherits the dependency.++ Asserted by walking the router's real routes rather than a hand-kept list,+ so adding an unprotected one fails here.+ """+ client = _client(monkeypatch, AISOC_ACTIONS_SERVICE_TOKEN="s3cr3t", AISOC_DEV_MODE="false")+ checked = 0+ for route in router.routes:+ path = getattr(route, "path", "")+ if "{" in path: # skip templated paths; no safe placeholder to inject+ continue+ for method in sorted(getattr(route, "methods", set()) - {"HEAD", "OPTIONS"}):+ resp = client.request(method, path, json=_BODY if method == "POST" else None)+ assert resp.status_code == 401, f"{method} {path} is not protected"+ checked += 1+ assert checked >= 3diff --git a/services/actions/tests/test_tenant_policy.py b/services/actions/tests/test_tenant_policy.pynew file mode 100644index 000000000..e311e7164--- /dev/null+++ b/services/actions/tests/test_tenant_policy.py@@ -0,0 +1,201 @@+"""A tenant's chosen autonomy tier must actually govern their actions.++The console writes a per-tenant tier, per-action overrides and a HIGH-blast+whitelist to Postgres. The live-action dispatcher read none of it: autonomy came+from one deployment-wide `AISOC_MATURITY_TIER`, and the `whitelisted` flag that+gates L4 break-glass was left at its default so that path was unreachable.++These tests cover the resolution rules, with particular attention to the+fallback posture — an unreadable policy must not inherit a possibly more+permissive environment default.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from uuid import uuid4++import pytest+from app.services import tenant_policy+from app.services.maturity import MaturityTier+from app.services.tenant_policy import TenantPolicy, resolve_tenant_policy++pytestmark = pytest.mark.asyncio++[email protected](autouse=True)+def _clean(monkeypatch: pytest.MonkeyPatch):+ tenant_policy.clear_cache()+ monkeypatch.delenv("AISOC_MATURITY_TIER", raising=False)+ monkeypatch.delenv("DATABASE_DSN", raising=False)+ monkeypatch.delenv("DATABASE_URL", raising=False)+ yield+ tenant_policy.clear_cache()+++# ── resolution ────────────────────────────────────────────────────────────+++async def test_without_a_database_the_environment_default_applies(+ monkeypatch: pytest.MonkeyPatch,+):+ """Single-tenant installs keep their documented behaviour."""+ monkeypatch.setenv("AISOC_MATURITY_TIER", "L2")+ policy = await resolve_tenant_policy(uuid4())+ assert policy.tier is MaturityTier.L2_CONTAIN+ assert policy.from_store is False+ assert policy.source == "environment"+++async def test_stored_tier_wins_over_the_environment(monkeypatch: pytest.MonkeyPatch):+ """The regression: a tenant who picked L3 was governed by the env var."""+ monkeypatch.setenv("AISOC_MATURITY_TIER", "L1")+ monkeypatch.setenv("DATABASE_DSN", "postgresql://stub/db")++ async def _load(tenant_id, dsn): # noqa: ANN001, ANN202+ return TenantPolicy(tier=MaturityTier.L3_REMEDIATE, from_store=True, source="tenant_policy")++ monkeypatch.setattr(tenant_policy, "_load_from_store", _load)+ policy = await resolve_tenant_policy(uuid4())+ assert policy.tier is MaturityTier.L3_REMEDIATE+ assert policy.from_store is True+++async def test_two_tenants_get_their_own_tiers(monkeypatch: pytest.MonkeyPatch):+ """One global tier is the multi-tenant bug this exists to fix."""+ monkeypatch.setenv("DATABASE_DSN", "postgresql://stub/db")+ tiers = {}++ async def _load(tenant_id, dsn): # noqa: ANN001, ANN202+ return TenantPolicy(tier=tiers[tenant_id], from_store=True, source="tenant_policy")++ monkeypatch.setattr(tenant_policy, "_load_from_store", _load)+ cautious, bold = str(uuid4()), str(uuid4())+ tiers[cautious] = MaturityTier.L0_OBSERVE+ tiers[bold] = MaturityTier.L4_AUTOMATE++ assert (await resolve_tenant_policy(cautious)).tier is MaturityTier.L0_OBSERVE+ assert (await resolve_tenant_policy(bold)).tier is MaturityTier.L4_AUTOMATE+++async def test_a_tenant_with_no_stored_row_gets_the_deployment_default(+ monkeypatch: pytest.MonkeyPatch,+):+ """Never configured is not the same fact as could not be read."""+ monkeypatch.setenv("AISOC_MATURITY_TIER", "L2")+ monkeypatch.setenv("DATABASE_DSN", "postgresql://stub/db")++ async def _load(tenant_id, dsn): # noqa: ANN001, ANN202+ return None++ monkeypatch.setattr(tenant_policy, "_load_from_store", _load)+ policy = await resolve_tenant_policy(uuid4())+ assert policy.tier is MaturityTier.L2_CONTAIN+ assert policy.from_store is False+++async def test_unreadable_policy_falls_to_the_floor_not_the_env_default(+ monkeypatch: pytest.MonkeyPatch,+):+ """The safety-critical case.++ A database is configured, so a per-tenant policy exists, and the read+ failed. Inheriting `AISOC_MATURITY_TIER=L4` here would auto-execute+ high-blast actions the tenant may never have authorised.+ """+ monkeypatch.setenv("AISOC_MATURITY_TIER", "L4")+ monkeypatch.setenv("DATABASE_DSN", "postgresql://stub/db")++ async def _load(tenant_id, dsn): # noqa: ANN001, ANN202+ raise ConnectionError("postgres unreachable")++ monkeypatch.setattr(tenant_policy, "_load_from_store", _load)+ policy = await resolve_tenant_policy(uuid4())+ assert policy.tier is tenant_policy.UNREADABLE_POLICY_FLOOR+ assert policy.tier is MaturityTier.L1_NOTIFY+ assert policy.source == "unreadable_policy_floor"+++async def test_a_bad_tier_env_value_is_conservative(monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setenv("AISOC_MATURITY_TIER", "TOTALLY_AUTONOMOUS")+ policy = await resolve_tenant_policy(None)+ assert policy.tier is MaturityTier.L1_NOTIFY+++async def test_sqlalchemy_style_dsn_is_accepted(monkeypatch: pytest.MonkeyPatch):+ """Both DSN spellings are already present in deployed environments."""+ monkeypatch.setenv("DATABASE_URL", "postgresql+asyncpg://u:p@h/db")+ assert tenant_policy._dsn() == "postgresql://u:p@h/db"+++# ── whitelist ─────────────────────────────────────────────────────────────+++def _wl(**kw):+ entry = {"action_type": "isolate_host", "constraints": {}, "expires_at": None}+ entry.update(kw)+ return TenantPolicy(tier=MaturityTier.L4_AUTOMATE, whitelist=[entry])+++async def test_whitelist_matches_on_action_type():+ assert _wl().is_whitelisted("isolate_host", "WIN-DC01") is True+ assert _wl().is_whitelisted("disable_user", "alice") is False+++async def test_expired_whitelist_entry_does_not_authorise():+ past = datetime.now(UTC) - timedelta(hours=1)+ assert _wl(expires_at=past).is_whitelisted("isolate_host", "WIN-DC01") is False+++async def test_unexpired_whitelist_entry_authorises():+ future = datetime.now(UTC) + timedelta(hours=1)+ assert _wl(expires_at=future).is_whitelisted("isolate_host", "WIN-DC01") is True+++async def test_target_constraint_is_enforced():+ scoped = _wl(constraints={"targets": ["LAB-01", "LAB-02"]})+ assert scoped.is_whitelisted("isolate_host", "LAB-01") is True+ assert scoped.is_whitelisted("isolate_host", "PROD-DC01") is False+++async def test_no_whitelist_means_not_whitelisted():+ assert TenantPolicy(tier=MaturityTier.L4_AUTOMATE).is_whitelisted("isolate_host", "x") is False+++# ── caching ───────────────────────────────────────────────────────────────+++async def test_policy_is_cached_so_the_action_path_is_not_per_call_db_bound(+ monkeypatch: pytest.MonkeyPatch,+):+ monkeypatch.setenv("DATABASE_DSN", "postgresql://stub/db")+ calls = 0++ async def _load(tenant_id, dsn): # noqa: ANN001, ANN202+ nonlocal calls+ calls += 1+ return TenantPolicy(tier=MaturityTier.L2_CONTAIN, from_store=True)++ monkeypatch.setattr(tenant_policy, "_load_from_store", _load)+ tid = uuid4()+ await resolve_tenant_policy(tid)+ await resolve_tenant_policy(tid)+ await resolve_tenant_policy(tid)+ assert calls == 1+++async def test_a_read_failure_is_not_cached(monkeypatch: pytest.MonkeyPatch):+ """A transient outage must not pin a tenant to the floor for the TTL."""+ monkeypatch.setenv("DATABASE_DSN", "postgresql://stub/db")+ state = {"fail": True}++ async def _load(tenant_id, dsn): # noqa: ANN001, ANN202+ if state["fail"]:+ raise ConnectionError("down")+ return TenantPolicy(tier=MaturityTier.L3_REMEDIATE, from_store=True)++ monkeypatch.setattr(tenant_policy, "_load_from_store", _load)+ tid = uuid4()+ assert (await resolve_tenant_policy(tid)).tier is MaturityTier.L1_NOTIFY+ state["fail"] = False+ assert (await resolve_tenant_policy(tid)).tier is MaturityTier.L3_REMEDIATEdiff --git a/services/actions/tests/test_verification.py b/services/actions/tests/test_verification.pyindex 7b9625462..41a0e848c 100644--- a/services/actions/tests/test_verification.py+++ b/services/actions/tests/test_verification.py@@ -15,28 +15,60 @@ from app.services.verification import PostActionVerifier, VerificationOutcomepytestmark = pytest.mark.asyncio-async def test_verified_when_probe_confirms(monkeypatch):+CREDS = {"cs_client_id": "x", "cs_client_secret": "y"}+++def _stub_cs(monkeypatch, *, device_id="dev-1", status="contained"):+ """Stub the EDR client with a resolvable device and a containment state."""+class _CS:async def get_device_id(self, hostname): # noqa: ANN001- return "dev-1"+ return device_id++ async def get_containment_status(self, dev): # noqa: ANN001+ return statusmonkeypatch.setattr(verification, "_cs_client", lambda params: _CS())- v = PostActionVerifier()- res = await v.verify(ActionType.ISOLATE_HOST, "WIN-DC01", {"cs_client_id": "x", "cs_client_secret": "y"})+++async def test_verified_only_when_the_host_is_actually_contained(monkeypatch):+ _stub_cs(monkeypatch, status="contained")+ res = await PostActionVerifier().verify(ActionType.ISOLATE_HOST, "WIN-DC01", CREDS)assert res.outcome == VerificationOutcome.VERIFIED-async def test_failed_when_effect_absent(monkeypatch):- class _CS:- async def get_device_id(self, hostname): # noqa: ANN001- return None+async def test_resolvable_host_that_is_not_contained_is_a_failure(monkeypatch):+ """The regression this probe exists to catch.- monkeypatch.setattr(verification, "_cs_client", lambda params: _CS())- v = PostActionVerifier()- res = await v.verify(ActionType.ISOLATE_HOST, "ghost-host", {"cs_client_id": "x", "cs_client_secret": "y"})+ The probe used to return ``bool(device_id)`` — true for every host in the+ fleet, contained or not — so a host that was never contained was reported+ VERIFIED. Hostname resolution is not containment.+ """+ _stub_cs(monkeypatch, status="normal")+ res = await PostActionVerifier().verify(ActionType.ISOLATE_HOST, "WIN-DC01", CREDS)assert res.outcome == VerificationOutcome.FAILED+async def test_containment_pending_is_not_verified(monkeypatch):+ """Accepted-but-not-in-force is exactly what this must not certify."""+ _stub_cs(monkeypatch, status="containment_pending")+ res = await PostActionVerifier().verify(ActionType.ISOLATE_HOST, "WIN-DC01", CREDS)+ assert res.outcome == VerificationOutcome.FAILED+++async def test_missing_host_is_indeterminate_not_failed(monkeypatch):+ """A renamed or decommissioned host is not proof containment did not take."""+ _stub_cs(monkeypatch, device_id=None)+ res = await PostActionVerifier().verify(ActionType.ISOLATE_HOST, "ghost-host", CREDS)+ assert res.outcome == VerificationOutcome.UNVERIFIED+++async def test_unreadable_status_is_indeterminate(monkeypatch):+ _stub_cs(monkeypatch, status=None)+ res = await PostActionVerifier().verify(ActionType.ISOLATE_HOST, "WIN-DC01", CREDS)+ assert res.outcome == VerificationOutcome.UNVERIFIED++async def test_unverified_without_credentials(monkeypatch):monkeypatch.setattr(verification, "_cs_client", lambda params: None)v = PostActionVerifier()