Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fail-fast shadow observability (Phase 0) (#6685)
pedrohsdbdad762956ada
diff --git a/skyvern/config.py b/skyvern/config.pyindex eb1aec365..24e394921 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -128,6 +128,9 @@ class Settings(BaseSettings):# If the task has been running for more steps than this ratio of the max steps per run, then we'll log a warning.LONG_RUNNING_TASK_WARNING_RATIO: float = 0.95MAX_RETRIES_PER_STEP: int = 5+ # Static kill-switch for fail-fast shadow observability. Per-org rollout is the+ # PostHog flag FAIL_FAST_SHADOW; this only force-enables it everywhere (local/testing).+ FAIL_FAST_SHADOW: bool = FalseDEBUG_MODE: bool = FalseDATABASE_STRING: str = Field(default_factory=_default_database_string)DATABASE_REPLICA_STRING: str | None = Nonediff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex a3754864c..c1742e5f9 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -100,6 +100,7 @@ from skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handlerfrom skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value+from skyvern.forge.sdk.fail_fast.shadow import record_fail_fast_shadowfrom skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logsfrom skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -867,6 +868,13 @@ class ForgeAgent:)await app.AGENT_FUNCTION.post_step_execution(task, step)task = await self.update_task_errors_from_detailed_output(task, detailed_output) # type: ignore+ # Shadow-only loop-stall observability; never raises, never terminates (see shadow.py).+ await record_fail_fast_shadow(+ task=task,+ step=step,+ organization=organization,+ scraped_page=detailed_output.scraped_page if detailed_output else None,+ )retry = Falseif task_block and task_block.complete_on_download and task.workflow_run_id:diff --git a/skyvern/forge/sdk/fail_fast/__init__.py b/skyvern/forge/sdk/fail_fast/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/skyvern/forge/sdk/fail_fast/shadow.py b/skyvern/forge/sdk/fail_fast/shadow.pynew file mode 100644index 000000000..0df495202--- /dev/null+++ b/skyvern/forge/sdk/fail_fast/shadow.py@@ -0,0 +1,335 @@+"""Shadow-mode tripwire observability for the agent loop's fail-fast feature (Phase 0).++Detects when a fail-fast "tripwire" WOULD fire on a stuck/looping run and emits a+structured log event the first time each (task, tripwire) trips while the task's ledger+is resident — without ever terminating anything. Pure observability: a log-based metric+joins these `would_fire` events to each task's final status to derive every tripwire's+decision precision (the false-positive rate among its own would-be terminations) BEFORE+any tripwire is allowed to act.++The recorder runs for terminal steps where ``agent_step`` returns a status (completed or+failed, including retries) — where the grinding-loop signals are observable. Steps that+terminate by exception (scrape / browser failures, unsupported actions) carry no page or+action state for these tripwires and already fail fast, so they are out of scope.++Conventions mirror ``extraction_shadow.py``: best-effort (never raises into the loop),+and only hashes / booleans / counts are logged — never raw page or model content.++Gating is OFF by default: the per-org PostHog flag ``FAIL_FAST_SHADOW`` (or the static+``settings.FAIL_FAST_SHADOW`` kill-switch) must opt a run in. State is a bounded,+append-only per-task ledger written only from the authoritative step path, so parallel+verification / speculative planning can never contaminate it.++Limitations — ledger loss censors events NON-randomly, so it can bias the precision+estimate in EITHER direction (not provably one-directional). Expected small because a+task's step loop is in-process and eviction targets idle tasks, but Phase-0 analysis must+quantify the ledger-loss rate (tasks that reached a terminal step with no resident ledger)+before trusting the precision numbers:+ - The ledger is in-memory and per-process. A task's step loop runs in one process, but+ a task that resumes on a different worker starts with an empty ledger.+ - Tripwires are evaluated at fixed default thresholds (see ``_K_*``/``_M_*``/``_N_*``),+ which measures decision precision AT the threshold a first cut would ship. Measuring+ precision across a range of thresholds (to tune them) is a separate follow-up.+ - In-app per-(task, tripwire) dedup is best-effort: it lives in the resident ledger and+ is dropped on LRU eviction, so a re-seen evicted task can re-emit. The offline metric+ is the authoritative dedup (first event per task_id + tripwire_id).+"""++from __future__ import annotations++import secrets+from collections import Counter, OrderedDict, deque+from dataclasses import dataclass, field+from typing import Any, Protocol++import structlog++from skyvern.config import settings+from skyvern.exceptions import IllegitComplete+from skyvern.forge import app+from skyvern.forge.sdk.api.crypto import calculate_sha256+from skyvern.forge.sdk.cache.extraction_cache import compute_cache_key+from skyvern.forge.sdk.models import Step+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.tasks import Task+from skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.actions.actions import Action+from skyvern.webeye.scraper.scraped_page import ScrapedPage++LOG = structlog.get_logger()++_SHADOW_EVENT = "fail_fast.shadow_tripwire"+_FLAG = "FAIL_FAST_SHADOW"++# Tripwire ids (kept stable — they key the offline decision-precision metric).+_T_NO_PROGRESS = "no_progress"+_T_ACTION_REPETITION = "action_repetition"+_T_ILLEGIT_STREAK = "illegit_complete_streak"+_T_PLAN_STAGNATION = "plan_stagnation"++# Shadow thresholds. Deliberately conservative; the real per-org values are tuned later+# against the shadow data. These only decide when to LOG a would-fire event.+_K_NO_PROGRESS = 3+_M_ACTION_REPEAT = 3+_N_ILLEGIT_STREAK = 3+_K_PLAN_STAGNATION = 3++_LEDGER_MAXLEN = 24 # per-task ring buffer of recent step fingerprints+_MAX_TASKS = 2000 # bound module memory across concurrent tasks+# Skip the content hash on pathologically large element trees so a huge DOM can't add+# meaningful CPU to a terminal step (no-progress just under-counts there — conservative).+_MAX_HTML_FOR_HASH = 500_000+# Per-process random salt for the in-memory value signatures (never logged). Salting makes+# them non-reversible against dictionary attacks on low-entropy inputs (OTPs, ids).+_VALUE_SALT = secrets.token_hex(16)+++class _LoggerLike(Protocol):+ def info(self, event: str, **kwargs: Any) -> None: ...++ def warning(self, event: str, **kwargs: Any) -> None: ...+++@dataclass+class _StepFingerprint:+ step_order: int+ retry_index: int+ status: str+ state_fp: str | None # canonical url+content hash; None when unavailable+ act_fps: tuple[str, ...] # per-action stable signatures (element-targeting actions only)+ plan_fp: str | None # hash of concatenated action reasoning; None when empty+ illegit_complete: bool+++@dataclass+class _TaskLedger:+ steps: deque[_StepFingerprint] = field(default_factory=lambda: deque(maxlen=_LEDGER_MAXLEN))+ fired: set[str] = field(default_factory=set) # tripwire ids already logged for this task+++_LEDGERS: OrderedDict[str, _TaskLedger] = OrderedDict()+++def _value_signature(action: Action) -> str:+ """Salted, non-reversible signature of an action's input value.++ Used ONLY as an in-memory ledger key (never logged) to distinguish re-filling a field+ with the same value from a different value; the salt defeats dictionary attacks.+ """+ raw = action.text or ""+ if action.option is not None:+ raw = f"{raw}|{action.option}"+ return calculate_sha256(f"{_VALUE_SALT}{raw}")[:12] if raw else ""+++def _act_fp(action: Action) -> str:+ element = action.skyvern_element_hash or action.element_id or ""+ return f"{action.action_type}:{element}:{_value_signature(action)}"+++def _state_fp(scraped_page: ScrapedPage | None) -> str | None:+ """Stable content+url hash that ignores transient ids/nonces, or None if unavailable."""+ if scraped_page is None:+ return None+ html = scraped_page.last_used_element_tree_html+ if not html or len(html) > _MAX_HTML_FOR_HASH:+ return None+ try:+ return compute_cache_key(call_path="fail_fast_shadow", element_tree=html, current_url=scraped_page.url)+ except Exception: # noqa: BLE001 — a bad page must not lose the other signals+ return None+++def _build_fingerprint(step: Step, scraped_page: ScrapedPage | None) -> _StepFingerprint:+ actions: list[Action] = []+ illegit = False+ output = step.output+ if output is not None and output.actions_and_results:+ for action, results in output.actions_and_results:+ actions.append(action)+ if action.action_type == ActionType.COMPLETE:+ for result in results:+ if result.exception_type == IllegitComplete.__name__:+ illegit = True+ break++ # Only true element interactions: a non-web action (WAIT/COMPLETE/TERMINATE) can carry a+ # hallucinated skyvern_element_hash (parse_actions clears element_id but not the hash), which+ # would otherwise let repeated waits/completes trip action_repetition. Mirror parse_actions.py.+ act_fps = tuple(+ _act_fp(a)+ for a in actions+ if (a.action_type.is_web_action() or a.action_type == ActionType.SCROLL)+ and (a.skyvern_element_hash or a.element_id)+ )+ reasoning = " ".join(a.reasoning or "" for a in actions).strip()+ plan_fp = calculate_sha256(reasoning)[:16] if reasoning else None++ return _StepFingerprint(+ step_order=step.order,+ retry_index=step.retry_index,+ status=str(step.status),+ state_fp=_state_fp(scraped_page),+ act_fps=act_fps,+ plan_fp=plan_fp,+ illegit_complete=illegit,+ )+++def _trailing_run(values: list[str | None]) -> int:+ """Length of the trailing run of equal, non-None values (from the end)."""+ run = 0+ anchor: str | None = None+ for value in reversed(values):+ if value is None:+ break+ if anchor is None:+ anchor = value+ run = 1+ elif value == anchor:+ run += 1+ else:+ break+ return run+++def _evaluate(steps: deque[_StepFingerprint]) -> list[tuple[str, dict[str, Any]]]:+ """Tripwires that WOULD fire given the ledger. Pure — no IO, no mutation."""+ seq = list(steps)+ fired: list[tuple[str, dict[str, Any]]] = []+ if not seq:+ return fired++ # T0.1a no-progress: page content unchanged for K consecutive steps.+ state_run = _trailing_run([fp.state_fp for fp in seq])+ if state_run >= _K_NO_PROGRESS:+ fired.append((_T_NO_PROGRESS, {"streak": state_run}))++ # T0.1b action repetition: the same element interaction recurs across M+ steps.+ step_counts: Counter[str] = Counter()+ for fp in seq:+ for act in set(fp.act_fps):+ step_counts[act] += 1+ if step_counts:+ _, top_count = step_counts.most_common(1)[0]+ if top_count >= _M_ACTION_REPEAT:+ fired.append((_T_ACTION_REPETITION, {"repeats": top_count}))++ # T0.2 illegit-complete streak in the same stuck state. Break on an unknown (None) state so+ # we never count a streak we can't confirm is the same state — consistent with _trailing_run.+ # `anchored` distinguishes "not yet anchored" from a real state_fp that happens to be None.+ illegit_run = 0+ anchor_state: str | None = None+ anchored = False+ for fp in reversed(seq):+ if not fp.illegit_complete or fp.state_fp is None:+ break+ if not anchored:+ anchor_state = fp.state_fp+ anchored = True+ illegit_run = 1+ elif fp.state_fp == anchor_state:+ illegit_run += 1+ else:+ break+ if illegit_run >= _N_ILLEGIT_STREAK:+ fired.append((_T_ILLEGIT_STREAK, {"streak": illegit_run}))++ # T0.3 plan/reasoning stagnation (corroborating signal).+ plan_run = _trailing_run([fp.plan_fp for fp in seq])+ if plan_run >= _K_PLAN_STAGNATION:+ fired.append((_T_PLAN_STAGNATION, {"streak": plan_run}))++ return fired+++def _get_ledger(task_id: str) -> _TaskLedger:+ ledger = _LEDGERS.get(task_id)+ if ledger is None:+ ledger = _TaskLedger()+ _LEDGERS[task_id] = ledger+ while len(_LEDGERS) > _MAX_TASKS:+ _LEDGERS.popitem(last=False) # evict the oldest task+ else:+ _LEDGERS.move_to_end(task_id)+ return ledger+++def _record_step(ledger: _TaskLedger, fingerprint: _StepFingerprint) -> list[tuple[str, dict[str, Any]]]:+ """Append the fingerprint and return newly-fired (deduped) tripwires. Pure given the ledger."""+ ledger.steps.append(fingerprint)+ newly_fired: list[tuple[str, dict[str, Any]]] = []+ for tripwire_id, signal in _evaluate(ledger.steps):+ if tripwire_id in ledger.fired:+ continue+ ledger.fired.add(tripwire_id)+ newly_fired.append((tripwire_id, signal))+ return newly_fired+++async def _shadow_enabled(task: Task, organization: Organization) -> bool:+ if settings.FAIL_FAST_SHADOW:+ return True+ try:+ raw = await app.EXPERIMENTATION_PROVIDER.get_value_cached(+ _FLAG,+ task.workflow_run_id or task.task_id,+ properties={"organization_id": organization.organization_id},+ )+ except Exception: # noqa: BLE001 — a flag-read failure must never affect the loop+ return False+ if raw is None:+ return False+ if isinstance(raw, bool):+ # PostHog returns a bare bool for a disabled/enabled multivariate flag.+ return raw+ if not isinstance(raw, str):+ return False+ normalized = raw.strip().lower()+ if normalized in ("1", "true", "on", "enabled"):+ return True+ try:+ return float(normalized) > 0+ except (TypeError, ValueError):+ return False+++async def record_fail_fast_shadow(+ *,+ task: Task,+ step: Step,+ organization: Organization,+ scraped_page: ScrapedPage | None,+ logger: _LoggerLike | None = None,+) -> None:+ """Evaluate fail-fast tripwires in shadow and emit one ``would_fire`` event per (task, tripwire).++ Never raises and never changes control flow. Call once per terminal step from the+ authoritative step path (after ``agent_step`` returns), for both completed and failed steps.+ """+ log = logger or LOG+ try:+ if step.is_speculative:+ # Speculative steps are discarded; never let them into the authoritative ledger.+ return+ if not await _shadow_enabled(task, organization):+ return+ ledger = _get_ledger(task.task_id)+ fingerprint = _build_fingerprint(step, scraped_page)+ for tripwire_id, signal in _record_step(ledger, fingerprint):+ log.info(+ _SHADOW_EVENT,+ status="would_fire",+ tripwire_id=tripwire_id,+ would_action="terminate",+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ organization_id=organization.organization_id,+ step_id=step.step_id,+ step_order=step.order,+ retry_index=step.retry_index,+ step_status=str(step.status),+ **{f"signal_{key}": value for key, value in signal.items()},+ )+ except Exception as exc: # noqa: BLE001 — shadow is best-effort; never affect the agent loop+ log.warning(_SHADOW_EVENT, status="error", error_type=type(exc).__name__, task_id=task.task_id)diff --git a/tests/unit/test_fail_fast_shadow.py b/tests/unit/test_fail_fast_shadow.pynew file mode 100644index 000000000..2f83ecbbe--- /dev/null+++ b/tests/unit/test_fail_fast_shadow.py@@ -0,0 +1,312 @@+from collections import deque+from datetime import datetime, timezone+from types import SimpleNamespace++import pytest++from skyvern.exceptions import IllegitComplete+from skyvern.forge.sdk.fail_fast import shadow+from skyvern.forge.sdk.fail_fast.shadow import (+ _act_fp,+ _build_fingerprint,+ _evaluate,+ _record_step,+ _StepFingerprint,+ _TaskLedger,+ _value_signature,+)+from skyvern.forge.sdk.models import Step, StepStatus+from skyvern.schemas.steps import AgentStepOutput+from skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.actions.actions import Action+from skyvern.webeye.actions.responses import ActionFailure+++def _fp(+ *, state: str | None = "S", acts: tuple[str, ...] = (), plan: str | None = None, illegit: bool = False+) -> _StepFingerprint:+ return _StepFingerprint(+ step_order=0,+ retry_index=0,+ status="completed",+ state_fp=state,+ act_fps=acts,+ plan_fp=plan,+ illegit_complete=illegit,+ )+++def _ledger(fingerprints: list[_StepFingerprint]) -> deque[_StepFingerprint]:+ return deque(fingerprints, maxlen=24)+++def _make_step(*, output: AgentStepOutput | None = None, order: int = 0, is_speculative: bool = False) -> Step:+ now = datetime.now(timezone.utc)+ return Step(+ created_at=now,+ modified_at=now,+ task_id="tsk_test",+ step_id=f"step_{order}",+ status=StepStatus.completed,+ order=order,+ is_last=False,+ organization_id="o_test",+ output=output,+ is_speculative=is_speculative,+ )++[email protected](autouse=True)+def _clear_ledgers():+ shadow._LEDGERS.clear()+ yield+ shadow._LEDGERS.clear()+++def test_no_progress_fires_at_threshold():+ fired = dict(_evaluate(_ledger([_fp(state="A"), _fp(state="A"), _fp(state="A")])))+ assert fired["no_progress"]["streak"] == 3+++def test_no_progress_below_threshold_does_not_fire():+ assert "no_progress" not in dict(_evaluate(_ledger([_fp(state="A"), _fp(state="A")])))+++def test_no_progress_resets_when_page_changes():+ # Trailing run is broken by the change, so it must not fire.+ assert "no_progress" not in dict(_evaluate(_ledger([_fp(state="A"), _fp(state="A"), _fp(state="B")])))+++def test_no_progress_ignores_unknown_state():+ assert "no_progress" not in dict(_evaluate(_ledger([_fp(state=None), _fp(state=None), _fp(state=None)])))+++def test_action_repetition_fires_on_repeated_element():+ same = "click:hash-1:"+ fired = dict(_evaluate(_ledger([_fp(state=str(i), acts=(same,)) for i in range(3)])))+ assert fired["action_repetition"]["repeats"] == 3+++def test_illegit_streak_requires_same_state():+ same_state = [_fp(state="A", illegit=True) for _ in range(3)]+ assert "illegit_complete_streak" in dict(_evaluate(_ledger(same_state)))++ changing_state = [_fp(state="A", illegit=True), _fp(state="B", illegit=True), _fp(state="C", illegit=True)]+ assert "illegit_complete_streak" not in dict(_evaluate(_ledger(changing_state)))+++def test_plan_stagnation_fires_on_repeated_plan():+ fired = dict(_evaluate(_ledger([_fp(state=str(i), plan="same-plan") for i in range(3)])))+ assert fired["plan_stagnation"]["streak"] == 3+++def test_record_step_dedups_per_tripwire():+ ledger = _TaskLedger()+ assert _record_step(ledger, _fp(state="A")) == []+ assert _record_step(ledger, _fp(state="A")) == []+ third = _record_step(ledger, _fp(state="A"))+ assert any(tripwire == "no_progress" for tripwire, _ in third)+ fourth = _record_step(ledger, _fp(state="A"))+ assert all(tripwire != "no_progress" for tripwire, _ in fourth)+++def test_value_signature_never_leaks_raw_value():+ secret = "123-45-6789"+ action = Action(action_type=ActionType.INPUT_TEXT, element_id="e1", text=secret)+ signature = _value_signature(action)+ assert secret not in signature+ assert len(signature) == 12+ assert secret not in _act_fp(action)+++def test_act_fp_is_stable():+ make = lambda: Action(action_type=ActionType.INPUT_TEXT, element_id="e1", text="value") # noqa: E731+ assert _act_fp(make()) == _act_fp(make())+++def test_build_fingerprint_detects_illegit_complete():+ complete = Action(action_type=ActionType.COMPLETE)+ rejected = ActionFailure(exception=IllegitComplete(data={"error": "user goal not achieved"}))+ step = _make_step(output=AgentStepOutput(actions_and_results=[(complete, [rejected])]))++ fingerprint = _build_fingerprint(step, None)++ assert fingerprint.illegit_complete is True+ assert fingerprint.state_fp is None # no scraped page provided+++def test_build_fingerprint_no_illegit_for_other_failures():+ complete = Action(action_type=ActionType.COMPLETE)+ other = ActionFailure(exception=ValueError("transient"))+ step = _make_step(output=AgentStepOutput(actions_and_results=[(complete, [other])]))+ assert _build_fingerprint(step, None).illegit_complete is False+++async def test_recorder_skips_when_disabled(monkeypatch):+ async def _disabled(_task, _org):+ return False++ monkeypatch.setattr(shadow, "_shadow_enabled", _disabled)+ events: list[tuple[str, dict]] = []+ logger = SimpleNamespace(info=lambda e, **k: events.append((e, k)), warning=lambda e, **k: events.append((e, k)))++ await shadow.record_fail_fast_shadow(+ task=SimpleNamespace(task_id="tsk_1", workflow_run_id="wr_1"),+ step=_make_step(),+ organization=SimpleNamespace(organization_id="o_1"),+ scraped_page=None,+ logger=logger,+ )+ assert events == []+ assert not shadow._LEDGERS+++async def test_recorder_skips_speculative_steps(monkeypatch):+ async def _enabled(_task, _org):+ return True++ monkeypatch.setattr(shadow, "_shadow_enabled", _enabled)+ events: list[tuple[str, dict]] = []+ logger = SimpleNamespace(info=lambda e, **k: events.append((e, k)), warning=lambda e, **k: events.append((e, k)))++ await shadow.record_fail_fast_shadow(+ task=SimpleNamespace(task_id="tsk_1", workflow_run_id="wr_1"),+ step=_make_step(is_speculative=True),+ organization=SimpleNamespace(organization_id="o_1"),+ scraped_page=None,+ logger=logger,+ )+ assert events == []+ assert not shadow._LEDGERS+++async def test_recorder_emits_once_when_enabled(monkeypatch):+ async def _enabled(_task, _org):+ return True++ monkeypatch.setattr(shadow, "_shadow_enabled", _enabled)+ events: list[tuple[str, dict]] = []+ logger = SimpleNamespace(info=lambda e, **k: events.append((e, k)), warning=lambda e, **k: events.append((e, k)))++ # A stable fake page yields a stable state_fp, so no_progress trips after K identical steps.+ page = SimpleNamespace(last_used_element_tree_html="<div>stable</div>", url="https://example.test/x")+ task = SimpleNamespace(task_id="tsk_enabled", workflow_run_id="wr_1")+ org = SimpleNamespace(organization_id="o_1")++ for order in range(4):+ await shadow.record_fail_fast_shadow(+ task=task, step=_make_step(order=order), organization=org, scraped_page=page, logger=logger+ )++ no_progress = [payload for event, payload in events if payload.get("tripwire_id") == "no_progress"]+ assert len(no_progress) == 1+ assert no_progress[0]["status"] == "would_fire"+ assert no_progress[0]["would_action"] == "terminate"+ assert no_progress[0]["organization_id"] == "o_1"+++ "raw, expected",+ [+ (None, False),+ (False, False), # PostHog returns a bare bool for a disabled multivariate flag+ (True, True),+ ("on", True),+ ("1", True),+ ("true", True),+ ("ENABLED", True),+ ("2.5", True),+ ("0", False),+ ("off", False),+ ("", False),+ ("nonsense", False),+ ],+)+async def test_shadow_enabled_parses_flag_values(monkeypatch, raw, expected):+ class _Provider:+ async def get_value_cached(self, *args, **kwargs):+ return raw++ monkeypatch.setattr(shadow.app, "EXPERIMENTATION_PROVIDER", _Provider())+ enabled = await shadow._shadow_enabled(+ SimpleNamespace(task_id="t", workflow_run_id="wr"),+ SimpleNamespace(organization_id="o"),+ )+ assert enabled is expected+++async def test_shadow_enabled_survives_provider_error(monkeypatch):+ class _Provider:+ async def get_value_cached(self, *args, **kwargs):+ raise RuntimeError("posthog unavailable")++ monkeypatch.setattr(shadow.app, "EXPERIMENTATION_PROVIDER", _Provider())+ enabled = await shadow._shadow_enabled(+ SimpleNamespace(task_id="t", workflow_run_id="wr"),+ SimpleNamespace(organization_id="o"),+ )+ assert enabled is False+++def test_ledger_eviction_bounds_memory():+ for i in range(shadow._MAX_TASKS + 25):+ shadow._get_ledger(f"task_{i}")+ assert len(shadow._LEDGERS) == shadow._MAX_TASKS+ assert "task_0" not in shadow._LEDGERS # oldest evicted+ assert f"task_{shadow._MAX_TASKS + 24}" in shadow._LEDGERS # newest retained+++def test_dedup_is_best_effort_after_eviction():+ # In-app dedup lives in the resident ledger. A fresh ledger (what eviction + re-creation+ # yields) re-emits the same tripwire. This is the accepted contract — the offline metric+ # dedups by (task_id, tripwire_id).+ resident = _TaskLedger()+ for _ in range(3):+ _record_step(resident, _fp(state="A"))+ assert "no_progress" in resident.fired++ refreshed = _TaskLedger()+ re_emitted: set[str] = set()+ for _ in range(3):+ re_emitted.update(tripwire for tripwire, _ in _record_step(refreshed, _fp(state="A")))+ assert "no_progress" in re_emitted+++def test_recorder_is_wired_into_execute_step():+ # Guards against the seam call being silently removed by a future refactor.+ import inspect++ from skyvern.forge.agent import ForgeAgent++ assert "record_fail_fast_shadow" in inspect.getsource(ForgeAgent.execute_step)+++def test_act_fps_exclude_non_web_actions_with_hallucinated_hash():+ # WAIT/COMPLETE/TERMINATE can carry a hallucinated skyvern_element_hash; they must not+ # count as element interactions for action_repetition.+ waits = [Action(action_type=ActionType.WAIT, skyvern_element_hash="hallucinated") for _ in range(3)]+ step = _make_step(output=AgentStepOutput(actions_and_results=[(a, []) for a in waits]))+ assert _build_fingerprint(step, None).act_fps == ()+++def test_act_fps_include_web_actions():+ clicks = [Action(action_type=ActionType.CLICK, skyvern_element_hash="h1") for _ in range(2)]+ step = _make_step(output=AgentStepOutput(actions_and_results=[(a, []) for a in clicks]))+ assert len(_build_fingerprint(step, None).act_fps) == 2+++def test_illegit_streak_breaks_on_unknown_latest_state():+ # Three illegit-completes in state A, but the most-recent step has unknown (None) state:+ # we can't confirm it's the same stuck state, so the streak must NOT fire.+ seq = [+ _fp(state="A", illegit=True),+ _fp(state="A", illegit=True),+ _fp(state="A", illegit=True),+ _fp(state=None, illegit=True),+ ]+ assert "illegit_complete_streak" not in dict(_evaluate(_ledger(seq)))