Skip to content

Automate browser based workflows with AI

Fail-fast shadow observability (Phase 0) (#6685)

pedrohsdbdad762956ada
diff --git a/skyvern/config.py b/skyvern/config.py
index eb1aec365..24e394921 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -128,6 +128,9 @@ class Settings(BaseSettings):
# If the task has been running for more steps than this ratio of the max steps per run, then we'll log a warning.
LONG_RUNNING_TASK_WARNING_RATIO: float = 0.95
MAX_RETRIES_PER_STEP: int = 5
+ # Static kill-switch for fail-fast shadow observability. Per-org rollout is the
+ # PostHog flag FAIL_FAST_SHADOW; this only force-enables it everywhere (local/testing).
+ FAIL_FAST_SHADOW: bool = False
DEBUG_MODE: bool = False
DATABASE_STRING: str = Field(default_factory=_default_database_string)
DATABASE_REPLICA_STRING: str | None = None
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index a3754864c..c1742e5f9 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -100,6 +100,7 @@ from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
+from skyvern.forge.sdk.fail_fast.shadow import record_fail_fast_shadow
from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logs
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -867,6 +868,13 @@ class ForgeAgent:
)
await app.AGENT_FUNCTION.post_step_execution(task, step)
task = await self.update_task_errors_from_detailed_output(task, detailed_output) # type: ignore
+ # Shadow-only loop-stall observability; never raises, never terminates (see shadow.py).
+ await record_fail_fast_shadow(
+ task=task,
+ step=step,
+ organization=organization,
+ scraped_page=detailed_output.scraped_page if detailed_output else None,
+ )
retry = False
if task_block and task_block.complete_on_download and task.workflow_run_id:
diff --git a/skyvern/forge/sdk/fail_fast/__init__.py b/skyvern/forge/sdk/fail_fast/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/skyvern/forge/sdk/fail_fast/shadow.py b/skyvern/forge/sdk/fail_fast/shadow.py
new file mode 100644
index 000000000..0df495202
--- /dev/null
+++ b/skyvern/forge/sdk/fail_fast/shadow.py
@@ -0,0 +1,335 @@
+"""Shadow-mode tripwire observability for the agent loop's fail-fast feature (Phase 0).
+
+Detects when a fail-fast "tripwire" WOULD fire on a stuck/looping run and emits a
+structured log event the first time each (task, tripwire) trips while the task's ledger
+is resident — without ever terminating anything. Pure observability: a log-based metric
+joins these `would_fire` events to each task's final status to derive every tripwire's
+decision precision (the false-positive rate among its own would-be terminations) BEFORE
+any tripwire is allowed to act.
+
+The recorder runs for terminal steps where ``agent_step`` returns a status (completed or
+failed, including retries) — where the grinding-loop signals are observable. Steps that
+terminate by exception (scrape / browser failures, unsupported actions) carry no page or
+action state for these tripwires and already fail fast, so they are out of scope.
+
+Conventions mirror ``extraction_shadow.py``: best-effort (never raises into the loop),
+and only hashes / booleans / counts are logged — never raw page or model content.
+
+Gating is OFF by default: the per-org PostHog flag ``FAIL_FAST_SHADOW`` (or the static
+``settings.FAIL_FAST_SHADOW`` kill-switch) must opt a run in. State is a bounded,
+append-only per-task ledger written only from the authoritative step path, so parallel
+verification / speculative planning can never contaminate it.
+
+Limitations — ledger loss censors events NON-randomly, so it can bias the precision
+estimate in EITHER direction (not provably one-directional). Expected small because a
+task's step loop is in-process and eviction targets idle tasks, but Phase-0 analysis must
+quantify the ledger-loss rate (tasks that reached a terminal step with no resident ledger)
+before trusting the precision numbers:
+ - The ledger is in-memory and per-process. A task's step loop runs in one process, but
+ a task that resumes on a different worker starts with an empty ledger.
+ - Tripwires are evaluated at fixed default thresholds (see ``_K_*``/``_M_*``/``_N_*``),
+ which measures decision precision AT the threshold a first cut would ship. Measuring
+ precision across a range of thresholds (to tune them) is a separate follow-up.
+ - In-app per-(task, tripwire) dedup is best-effort: it lives in the resident ledger and
+ is dropped on LRU eviction, so a re-seen evicted task can re-emit. The offline metric
+ is the authoritative dedup (first event per task_id + tripwire_id).
+"""
+
+from __future__ import annotations
+
+import secrets
+from collections import Counter, OrderedDict, deque
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.exceptions import IllegitComplete
+from skyvern.forge import app
+from skyvern.forge.sdk.api.crypto import calculate_sha256
+from skyvern.forge.sdk.cache.extraction_cache import compute_cache_key
+from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.tasks import Task
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import Action
+from skyvern.webeye.scraper.scraped_page import ScrapedPage
+
+LOG = structlog.get_logger()
+
+_SHADOW_EVENT = "fail_fast.shadow_tripwire"
+_FLAG = "FAIL_FAST_SHADOW"
+
+# Tripwire ids (kept stable — they key the offline decision-precision metric).
+_T_NO_PROGRESS = "no_progress"
+_T_ACTION_REPETITION = "action_repetition"
+_T_ILLEGIT_STREAK = "illegit_complete_streak"
+_T_PLAN_STAGNATION = "plan_stagnation"
+
+# Shadow thresholds. Deliberately conservative; the real per-org values are tuned later
+# against the shadow data. These only decide when to LOG a would-fire event.
+_K_NO_PROGRESS = 3
+_M_ACTION_REPEAT = 3
+_N_ILLEGIT_STREAK = 3
+_K_PLAN_STAGNATION = 3
+
+_LEDGER_MAXLEN = 24 # per-task ring buffer of recent step fingerprints
+_MAX_TASKS = 2000 # bound module memory across concurrent tasks
+# Skip the content hash on pathologically large element trees so a huge DOM can't add
+# meaningful CPU to a terminal step (no-progress just under-counts there — conservative).
+_MAX_HTML_FOR_HASH = 500_000
+# Per-process random salt for the in-memory value signatures (never logged). Salting makes
+# them non-reversible against dictionary attacks on low-entropy inputs (OTPs, ids).
+_VALUE_SALT = secrets.token_hex(16)
+
+
+class _LoggerLike(Protocol):
+ def info(self, event: str, **kwargs: Any) -> None: ...
+
+ def warning(self, event: str, **kwargs: Any) -> None: ...
+
+
+@dataclass
+class _StepFingerprint:
+ step_order: int
+ retry_index: int
+ status: str
+ state_fp: str | None # canonical url+content hash; None when unavailable
+ act_fps: tuple[str, ...] # per-action stable signatures (element-targeting actions only)
+ plan_fp: str | None # hash of concatenated action reasoning; None when empty
+ illegit_complete: bool
+
+
+@dataclass
+class _TaskLedger:
+ steps: deque[_StepFingerprint] = field(default_factory=lambda: deque(maxlen=_LEDGER_MAXLEN))
+ fired: set[str] = field(default_factory=set) # tripwire ids already logged for this task
+
+
+_LEDGERS: OrderedDict[str, _TaskLedger] = OrderedDict()
+
+
+def _value_signature(action: Action) -> str:
+ """Salted, non-reversible signature of an action's input value.
+
+ Used ONLY as an in-memory ledger key (never logged) to distinguish re-filling a field
+ with the same value from a different value; the salt defeats dictionary attacks.
+ """
+ raw = action.text or ""
+ if action.option is not None:
+ raw = f"{raw}|{action.option}"
+ return calculate_sha256(f"{_VALUE_SALT}{raw}")[:12] if raw else ""
+
+
+def _act_fp(action: Action) -> str:
+ element = action.skyvern_element_hash or action.element_id or ""
+ return f"{action.action_type}:{element}:{_value_signature(action)}"
+
+
+def _state_fp(scraped_page: ScrapedPage | None) -> str | None:
+ """Stable content+url hash that ignores transient ids/nonces, or None if unavailable."""
+ if scraped_page is None:
+ return None
+ html = scraped_page.last_used_element_tree_html
+ if not html or len(html) > _MAX_HTML_FOR_HASH:
+ return None
+ try:
+ return compute_cache_key(call_path="fail_fast_shadow", element_tree=html, current_url=scraped_page.url)
+ except Exception: # noqa: BLE001 — a bad page must not lose the other signals
+ return None
+
+
+def _build_fingerprint(step: Step, scraped_page: ScrapedPage | None) -> _StepFingerprint:
+ actions: list[Action] = []
+ illegit = False
+ output = step.output
+ if output is not None and output.actions_and_results:
+ for action, results in output.actions_and_results:
+ actions.append(action)
+ if action.action_type == ActionType.COMPLETE:
+ for result in results:
+ if result.exception_type == IllegitComplete.__name__:
+ illegit = True
+ break
+
+ # Only true element interactions: a non-web action (WAIT/COMPLETE/TERMINATE) can carry a
+ # hallucinated skyvern_element_hash (parse_actions clears element_id but not the hash), which
+ # would otherwise let repeated waits/completes trip action_repetition. Mirror parse_actions.py.
+ act_fps = tuple(
+ _act_fp(a)
+ for a in actions
+ if (a.action_type.is_web_action() or a.action_type == ActionType.SCROLL)
+ and (a.skyvern_element_hash or a.element_id)
+ )
+ reasoning = " ".join(a.reasoning or "" for a in actions).strip()
+ plan_fp = calculate_sha256(reasoning)[:16] if reasoning else None
+
+ return _StepFingerprint(
+ step_order=step.order,
+ retry_index=step.retry_index,
+ status=str(step.status),
+ state_fp=_state_fp(scraped_page),
+ act_fps=act_fps,
+ plan_fp=plan_fp,
+ illegit_complete=illegit,
+ )
+
+
+def _trailing_run(values: list[str | None]) -> int:
+ """Length of the trailing run of equal, non-None values (from the end)."""
+ run = 0
+ anchor: str | None = None
+ for value in reversed(values):
+ if value is None:
+ break
+ if anchor is None:
+ anchor = value
+ run = 1
+ elif value == anchor:
+ run += 1
+ else:
+ break
+ return run
+
+
+def _evaluate(steps: deque[_StepFingerprint]) -> list[tuple[str, dict[str, Any]]]:
+ """Tripwires that WOULD fire given the ledger. Pure — no IO, no mutation."""
+ seq = list(steps)
+ fired: list[tuple[str, dict[str, Any]]] = []
+ if not seq:
+ return fired
+
+ # T0.1a no-progress: page content unchanged for K consecutive steps.
+ state_run = _trailing_run([fp.state_fp for fp in seq])
+ if state_run >= _K_NO_PROGRESS:
+ fired.append((_T_NO_PROGRESS, {"streak": state_run}))
+
+ # T0.1b action repetition: the same element interaction recurs across M+ steps.
+ step_counts: Counter[str] = Counter()
+ for fp in seq:
+ for act in set(fp.act_fps):
+ step_counts[act] += 1
+ if step_counts:
+ _, top_count = step_counts.most_common(1)[0]
+ if top_count >= _M_ACTION_REPEAT:
+ fired.append((_T_ACTION_REPETITION, {"repeats": top_count}))
+
+ # T0.2 illegit-complete streak in the same stuck state. Break on an unknown (None) state so
+ # we never count a streak we can't confirm is the same state — consistent with _trailing_run.
+ # `anchored` distinguishes "not yet anchored" from a real state_fp that happens to be None.
+ illegit_run = 0
+ anchor_state: str | None = None
+ anchored = False
+ for fp in reversed(seq):
+ if not fp.illegit_complete or fp.state_fp is None:
+ break
+ if not anchored:
+ anchor_state = fp.state_fp
+ anchored = True
+ illegit_run = 1
+ elif fp.state_fp == anchor_state:
+ illegit_run += 1
+ else:
+ break
+ if illegit_run >= _N_ILLEGIT_STREAK:
+ fired.append((_T_ILLEGIT_STREAK, {"streak": illegit_run}))
+
+ # T0.3 plan/reasoning stagnation (corroborating signal).
+ plan_run = _trailing_run([fp.plan_fp for fp in seq])
+ if plan_run >= _K_PLAN_STAGNATION:
+ fired.append((_T_PLAN_STAGNATION, {"streak": plan_run}))
+
+ return fired
+
+
+def _get_ledger(task_id: str) -> _TaskLedger:
+ ledger = _LEDGERS.get(task_id)
+ if ledger is None:
+ ledger = _TaskLedger()
+ _LEDGERS[task_id] = ledger
+ while len(_LEDGERS) > _MAX_TASKS:
+ _LEDGERS.popitem(last=False) # evict the oldest task
+ else:
+ _LEDGERS.move_to_end(task_id)
+ return ledger
+
+
+def _record_step(ledger: _TaskLedger, fingerprint: _StepFingerprint) -> list[tuple[str, dict[str, Any]]]:
+ """Append the fingerprint and return newly-fired (deduped) tripwires. Pure given the ledger."""
+ ledger.steps.append(fingerprint)
+ newly_fired: list[tuple[str, dict[str, Any]]] = []
+ for tripwire_id, signal in _evaluate(ledger.steps):
+ if tripwire_id in ledger.fired:
+ continue
+ ledger.fired.add(tripwire_id)
+ newly_fired.append((tripwire_id, signal))
+ return newly_fired
+
+
+async def _shadow_enabled(task: Task, organization: Organization) -> bool:
+ if settings.FAIL_FAST_SHADOW:
+ return True
+ try:
+ raw = await app.EXPERIMENTATION_PROVIDER.get_value_cached(
+ _FLAG,
+ task.workflow_run_id or task.task_id,
+ properties={"organization_id": organization.organization_id},
+ )
+ except Exception: # noqa: BLE001 — a flag-read failure must never affect the loop
+ return False
+ if raw is None:
+ return False
+ if isinstance(raw, bool):
+ # PostHog returns a bare bool for a disabled/enabled multivariate flag.
+ return raw
+ if not isinstance(raw, str):
+ return False
+ normalized = raw.strip().lower()
+ if normalized in ("1", "true", "on", "enabled"):
+ return True
+ try:
+ return float(normalized) > 0
+ except (TypeError, ValueError):
+ return False
+
+
+async def record_fail_fast_shadow(
+ *,
+ task: Task,
+ step: Step,
+ organization: Organization,
+ scraped_page: ScrapedPage | None,
+ logger: _LoggerLike | None = None,
+) -> None:
+ """Evaluate fail-fast tripwires in shadow and emit one ``would_fire`` event per (task, tripwire).
+
+ Never raises and never changes control flow. Call once per terminal step from the
+ authoritative step path (after ``agent_step`` returns), for both completed and failed steps.
+ """
+ log = logger or LOG
+ try:
+ if step.is_speculative:
+ # Speculative steps are discarded; never let them into the authoritative ledger.
+ return
+ if not await _shadow_enabled(task, organization):
+ return
+ ledger = _get_ledger(task.task_id)
+ fingerprint = _build_fingerprint(step, scraped_page)
+ for tripwire_id, signal in _record_step(ledger, fingerprint):
+ log.info(
+ _SHADOW_EVENT,
+ status="would_fire",
+ tripwire_id=tripwire_id,
+ would_action="terminate",
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ organization_id=organization.organization_id,
+ step_id=step.step_id,
+ step_order=step.order,
+ retry_index=step.retry_index,
+ step_status=str(step.status),
+ **{f"signal_{key}": value for key, value in signal.items()},
+ )
+ except Exception as exc: # noqa: BLE001 — shadow is best-effort; never affect the agent loop
+ log.warning(_SHADOW_EVENT, status="error", error_type=type(exc).__name__, task_id=task.task_id)
diff --git a/tests/unit/test_fail_fast_shadow.py b/tests/unit/test_fail_fast_shadow.py
new file mode 100644
index 000000000..2f83ecbbe
--- /dev/null
+++ b/tests/unit/test_fail_fast_shadow.py
@@ -0,0 +1,312 @@
+from collections import deque
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.exceptions import IllegitComplete
+from skyvern.forge.sdk.fail_fast import shadow
+from skyvern.forge.sdk.fail_fast.shadow import (
+ _act_fp,
+ _build_fingerprint,
+ _evaluate,
+ _record_step,
+ _StepFingerprint,
+ _TaskLedger,
+ _value_signature,
+)
+from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.schemas.steps import AgentStepOutput
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import Action
+from skyvern.webeye.actions.responses import ActionFailure
+
+
+def _fp(
+ *, state: str | None = "S", acts: tuple[str, ...] = (), plan: str | None = None, illegit: bool = False
+) -> _StepFingerprint:
+ return _StepFingerprint(
+ step_order=0,
+ retry_index=0,
+ status="completed",
+ state_fp=state,
+ act_fps=acts,
+ plan_fp=plan,
+ illegit_complete=illegit,
+ )
+
+
+def _ledger(fingerprints: list[_StepFingerprint]) -> deque[_StepFingerprint]:
+ return deque(fingerprints, maxlen=24)
+
+
+def _make_step(*, output: AgentStepOutput | None = None, order: int = 0, is_speculative: bool = False) -> Step:
+ now = datetime.now(timezone.utc)
+ return Step(
+ created_at=now,
+ modified_at=now,
+ task_id="tsk_test",
+ step_id=f"step_{order}",
+ status=StepStatus.completed,
+ order=order,
+ is_last=False,
+ organization_id="o_test",
+ output=output,
+ is_speculative=is_speculative,
+ )
+
+
[email protected](autouse=True)
+def _clear_ledgers():
+ shadow._LEDGERS.clear()
+ yield
+ shadow._LEDGERS.clear()
+
+
+def test_no_progress_fires_at_threshold():
+ fired = dict(_evaluate(_ledger([_fp(state="A"), _fp(state="A"), _fp(state="A")])))
+ assert fired["no_progress"]["streak"] == 3
+
+
+def test_no_progress_below_threshold_does_not_fire():
+ assert "no_progress" not in dict(_evaluate(_ledger([_fp(state="A"), _fp(state="A")])))
+
+
+def test_no_progress_resets_when_page_changes():
+ # Trailing run is broken by the change, so it must not fire.
+ assert "no_progress" not in dict(_evaluate(_ledger([_fp(state="A"), _fp(state="A"), _fp(state="B")])))
+
+
+def test_no_progress_ignores_unknown_state():
+ assert "no_progress" not in dict(_evaluate(_ledger([_fp(state=None), _fp(state=None), _fp(state=None)])))
+
+
+def test_action_repetition_fires_on_repeated_element():
+ same = "click:hash-1:"
+ fired = dict(_evaluate(_ledger([_fp(state=str(i), acts=(same,)) for i in range(3)])))
+ assert fired["action_repetition"]["repeats"] == 3
+
+
+def test_illegit_streak_requires_same_state():
+ same_state = [_fp(state="A", illegit=True) for _ in range(3)]
+ assert "illegit_complete_streak" in dict(_evaluate(_ledger(same_state)))
+
+ changing_state = [_fp(state="A", illegit=True), _fp(state="B", illegit=True), _fp(state="C", illegit=True)]
+ assert "illegit_complete_streak" not in dict(_evaluate(_ledger(changing_state)))
+
+
+def test_plan_stagnation_fires_on_repeated_plan():
+ fired = dict(_evaluate(_ledger([_fp(state=str(i), plan="same-plan") for i in range(3)])))
+ assert fired["plan_stagnation"]["streak"] == 3
+
+
+def test_record_step_dedups_per_tripwire():
+ ledger = _TaskLedger()
+ assert _record_step(ledger, _fp(state="A")) == []
+ assert _record_step(ledger, _fp(state="A")) == []
+ third = _record_step(ledger, _fp(state="A"))
+ assert any(tripwire == "no_progress" for tripwire, _ in third)
+ fourth = _record_step(ledger, _fp(state="A"))
+ assert all(tripwire != "no_progress" for tripwire, _ in fourth)
+
+
+def test_value_signature_never_leaks_raw_value():
+ secret = "123-45-6789"
+ action = Action(action_type=ActionType.INPUT_TEXT, element_id="e1", text=secret)
+ signature = _value_signature(action)
+ assert secret not in signature
+ assert len(signature) == 12
+ assert secret not in _act_fp(action)
+
+
+def test_act_fp_is_stable():
+ make = lambda: Action(action_type=ActionType.INPUT_TEXT, element_id="e1", text="value") # noqa: E731
+ assert _act_fp(make()) == _act_fp(make())
+
+
+def test_build_fingerprint_detects_illegit_complete():
+ complete = Action(action_type=ActionType.COMPLETE)
+ rejected = ActionFailure(exception=IllegitComplete(data={"error": "user goal not achieved"}))
+ step = _make_step(output=AgentStepOutput(actions_and_results=[(complete, [rejected])]))
+
+ fingerprint = _build_fingerprint(step, None)
+
+ assert fingerprint.illegit_complete is True
+ assert fingerprint.state_fp is None # no scraped page provided
+
+
+def test_build_fingerprint_no_illegit_for_other_failures():
+ complete = Action(action_type=ActionType.COMPLETE)
+ other = ActionFailure(exception=ValueError("transient"))
+ step = _make_step(output=AgentStepOutput(actions_and_results=[(complete, [other])]))
+ assert _build_fingerprint(step, None).illegit_complete is False
+
+
+async def test_recorder_skips_when_disabled(monkeypatch):
+ async def _disabled(_task, _org):
+ return False
+
+ monkeypatch.setattr(shadow, "_shadow_enabled", _disabled)
+ events: list[tuple[str, dict]] = []
+ logger = SimpleNamespace(info=lambda e, **k: events.append((e, k)), warning=lambda e, **k: events.append((e, k)))
+
+ await shadow.record_fail_fast_shadow(
+ task=SimpleNamespace(task_id="tsk_1", workflow_run_id="wr_1"),
+ step=_make_step(),
+ organization=SimpleNamespace(organization_id="o_1"),
+ scraped_page=None,
+ logger=logger,
+ )
+ assert events == []
+ assert not shadow._LEDGERS
+
+
+async def test_recorder_skips_speculative_steps(monkeypatch):
+ async def _enabled(_task, _org):
+ return True
+
+ monkeypatch.setattr(shadow, "_shadow_enabled", _enabled)
+ events: list[tuple[str, dict]] = []
+ logger = SimpleNamespace(info=lambda e, **k: events.append((e, k)), warning=lambda e, **k: events.append((e, k)))
+
+ await shadow.record_fail_fast_shadow(
+ task=SimpleNamespace(task_id="tsk_1", workflow_run_id="wr_1"),
+ step=_make_step(is_speculative=True),
+ organization=SimpleNamespace(organization_id="o_1"),
+ scraped_page=None,
+ logger=logger,
+ )
+ assert events == []
+ assert not shadow._LEDGERS
+
+
+async def test_recorder_emits_once_when_enabled(monkeypatch):
+ async def _enabled(_task, _org):
+ return True
+
+ monkeypatch.setattr(shadow, "_shadow_enabled", _enabled)
+ events: list[tuple[str, dict]] = []
+ logger = SimpleNamespace(info=lambda e, **k: events.append((e, k)), warning=lambda e, **k: events.append((e, k)))
+
+ # A stable fake page yields a stable state_fp, so no_progress trips after K identical steps.
+ page = SimpleNamespace(last_used_element_tree_html="<div>stable</div>", url="https://example.test/x")
+ task = SimpleNamespace(task_id="tsk_enabled", workflow_run_id="wr_1")
+ org = SimpleNamespace(organization_id="o_1")
+
+ for order in range(4):
+ await shadow.record_fail_fast_shadow(
+ task=task, step=_make_step(order=order), organization=org, scraped_page=page, logger=logger
+ )
+
+ no_progress = [payload for event, payload in events if payload.get("tripwire_id") == "no_progress"]
+ assert len(no_progress) == 1
+ assert no_progress[0]["status"] == "would_fire"
+ assert no_progress[0]["would_action"] == "terminate"
+ assert no_progress[0]["organization_id"] == "o_1"
+
+
+ "raw, expected",
+ [
+ (None, False),
+ (False, False), # PostHog returns a bare bool for a disabled multivariate flag
+ (True, True),
+ ("on", True),
+ ("1", True),
+ ("true", True),
+ ("ENABLED", True),
+ ("2.5", True),
+ ("0", False),
+ ("off", False),
+ ("", False),
+ ("nonsense", False),
+ ],
+)
+async def test_shadow_enabled_parses_flag_values(monkeypatch, raw, expected):
+ class _Provider:
+ async def get_value_cached(self, *args, **kwargs):
+ return raw
+
+ monkeypatch.setattr(shadow.app, "EXPERIMENTATION_PROVIDER", _Provider())
+ enabled = await shadow._shadow_enabled(
+ SimpleNamespace(task_id="t", workflow_run_id="wr"),
+ SimpleNamespace(organization_id="o"),
+ )
+ assert enabled is expected
+
+
+async def test_shadow_enabled_survives_provider_error(monkeypatch):
+ class _Provider:
+ async def get_value_cached(self, *args, **kwargs):
+ raise RuntimeError("posthog unavailable")
+
+ monkeypatch.setattr(shadow.app, "EXPERIMENTATION_PROVIDER", _Provider())
+ enabled = await shadow._shadow_enabled(
+ SimpleNamespace(task_id="t", workflow_run_id="wr"),
+ SimpleNamespace(organization_id="o"),
+ )
+ assert enabled is False
+
+
+def test_ledger_eviction_bounds_memory():
+ for i in range(shadow._MAX_TASKS + 25):
+ shadow._get_ledger(f"task_{i}")
+ assert len(shadow._LEDGERS) == shadow._MAX_TASKS
+ assert "task_0" not in shadow._LEDGERS # oldest evicted
+ assert f"task_{shadow._MAX_TASKS + 24}" in shadow._LEDGERS # newest retained
+
+
+def test_dedup_is_best_effort_after_eviction():
+ # In-app dedup lives in the resident ledger. A fresh ledger (what eviction + re-creation
+ # yields) re-emits the same tripwire. This is the accepted contract — the offline metric
+ # dedups by (task_id, tripwire_id).
+ resident = _TaskLedger()
+ for _ in range(3):
+ _record_step(resident, _fp(state="A"))
+ assert "no_progress" in resident.fired
+
+ refreshed = _TaskLedger()
+ re_emitted: set[str] = set()
+ for _ in range(3):
+ re_emitted.update(tripwire for tripwire, _ in _record_step(refreshed, _fp(state="A")))
+ assert "no_progress" in re_emitted
+
+
+def test_recorder_is_wired_into_execute_step():
+ # Guards against the seam call being silently removed by a future refactor.
+ import inspect
+
+ from skyvern.forge.agent import ForgeAgent
+
+ assert "record_fail_fast_shadow" in inspect.getsource(ForgeAgent.execute_step)
+
+
+def test_act_fps_exclude_non_web_actions_with_hallucinated_hash():
+ # WAIT/COMPLETE/TERMINATE can carry a hallucinated skyvern_element_hash; they must not
+ # count as element interactions for action_repetition.
+ waits = [Action(action_type=ActionType.WAIT, skyvern_element_hash="hallucinated") for _ in range(3)]
+ step = _make_step(output=AgentStepOutput(actions_and_results=[(a, []) for a in waits]))
+ assert _build_fingerprint(step, None).act_fps == ()
+
+
+def test_act_fps_include_web_actions():
+ clicks = [Action(action_type=ActionType.CLICK, skyvern_element_hash="h1") for _ in range(2)]
+ step = _make_step(output=AgentStepOutput(actions_and_results=[(a, []) for a in clicks]))
+ assert len(_build_fingerprint(step, None).act_fps) == 2
+
+
+def test_illegit_streak_breaks_on_unknown_latest_state():
+ # Three illegit-completes in state A, but the most-recent step has unknown (None) state:
+ # we can't confirm it's the same stuck state, so the streak must NOT fire.
+ seq = [
+ _fp(state="A", illegit=True),
+ _fp(state="A", illegit=True),
+ _fp(state="A", illegit=True),
+ _fp(state=None, illegit=True),
+ ]
+ assert "illegit_complete_streak" not in dict(_evaluate(_ledger(seq)))