Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add a page-state stall detector to task_v3: live re-plan nudge, shadow would-terminate verdict (SKY-15265) (#8340)
pedrohsdbd5bd873fdc8b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex e505105e9..bcba2056a 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1802,10 +1802,21 @@ class ForgeAgent:# the gate, not here. Built for both populations — the failure-evidence gate needs a sampler# to run at all — over the same page accessor each population's tools use, so the evidence# is sampled from the page the model actually acted on.+ # Field values are hashed separately: typing sets the value IDL property, which innerHTML+ # serialization does not reflect, so a form being filled would otherwise read as frozen.+ # Open shadow roots are traversed for the same reason — the tools pierce them, so work+ # inside a shadow widget must move the fingerprint too (closed roots stay invisible)._PAGE_FINGERPRINT_PROBE_JS = (- "() => { if (!document.body) return '0'; const s = document.body.innerHTML;"- " let h = 0; for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;"- " return h + ':' + s.length + ':' + document.querySelectorAll('*').length; }"+ "() => { if (!document.body) return '0'; let h = 0; let v = 0; let elems = 0;"+ " const mix = (str, seed) => { let x = seed;"+ " for (let i = 0; i < str.length; i++) x = (Math.imul(x, 31) + str.charCodeAt(i)) | 0; return x; };"+ " const walk = (root) => { h = mix(root.innerHTML || '', h);"+ " const all = root.querySelectorAll('*'); elems += all.length;"+ " for (const el of root.querySelectorAll('input, textarea, select'))"+ " v = mix(String(el.value || '') + '|' + (el.checked === true ? '1' : '0'), v);"+ " for (const el of all) { if (el.shadowRoot) walk(el.shadowRoot); } };"+ " walk(document.body);"+ " return h + ':' + elems + ':' + v; }")_DOCUMENT_NONCE_JS = (diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex d8985685a..047d0c0b9 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -205,6 +205,21 @@ ACTION_BUDGET_EXTENSION_EVIDENCE_WINDOW = 8ACTION_BUDGET_EXTENDED_EVENT = "taskv3 loop action budget extended"ACTION_BUDGET_EXTENSION_REFUSED_EVENT = "taskv3 loop action budget extension refused"+# Page-state stall policy (SKY-15265): rounds of billable batches that left the page fingerprint+# byte-identical, with no page-change flag, mean the run is cycling on a frozen document in shapes+# the per-tool guards cannot see (varied probes never streak; scroll/wait carry no digest at all).+# Completions of the worst-affected canary block finish in <=9 rounds; the nudge re-plans the model+# once before the verdict.+PAGE_STATE_STALL_NUDGE_AFTER = 8+PAGE_STATE_STALL_TERMINATE_AFTER = 12+# The verdict is SHADOW-ONLY for now: the fingerprint is blind to work inside iframes (main-frame+# innerHTML only), so a live termination could kill healthy embedded-widget runs. The nudge ships+# live (benign direction); the shadow event measures the would-terminate precision on the canary,+# and promotion to a live verdict is a separate release decision on that data.+PAGE_STATE_STALL_SHADOW_EVENT = "taskv3 loop page state stall would terminate"+# Facetable sibling of PERCEPTION_STALL_REASON_PREFIX; reserved for the future live verdict.+PAGE_STATE_STALL_REASON_PREFIX = "page_state_stall:"+# Hard "the resource does not exist / is gone" HTTP statuses. A navigation landing on one of these is# a genuine non-capability dead-end (a dead or removed posting), which v1 routes to `terminated`. Both# the in-loop `navigate` tool and the pre-loop initial-URL navigation classify against this set. NARROW@@ -817,6 +832,15 @@ def _unblocker_options(available_tools: set[str]) -> list[str]:return options+def _page_state_nudge_text(rounds: int) -> str:+ return (+ f"Your last {rounds} action rounds left the page's rendered content completely unchanged — "+ "whatever you are trying is not affecting this page. Stop repeating the current approach: "+ "re-plan from a fresh observe, try a genuinely different control or path, or finish honestly "+ "(status=failed or terminated) naming what is blocking you."+ )++def _stall_nudge_text(stalled: list[tuple[str, int]], available_tools: set[str]) -> str:"""One warning naming every stalled perception tool and the unblockers this run actually has —a model that cannot see the gate won't reach for solve_captcha unless the symptom names it."""@@ -1282,6 +1306,15 @@ async def run_agent_tool_loop(outcome: LoopOutcome | None = Nonepending_nav_dead_end: int | None = Nonestall_nudges_due: list[tuple[str, int]] = []+ # Page-state stall detector (SKY-15265): consecutive billable rounds on a byte-identical+ # fingerprint, whether the one re-plan nudge went out, and whether one is due this turn.+ page_state_stall_rounds = 0+ page_state_nudge_delivered = False+ page_state_nudge_due = False+ # The last fingerprint sample from the PREVIOUS batch: a delayed render can land between one+ # batch's after-sample and the next batch's before-sample, so movement is checked across+ # batches, not only within them.+ page_state_prev_fp: str | None = Nonerefresh_cycles = 0refresh_nudge_due = Falsereload_failed_nudge_due = False@@ -1340,6 +1373,7 @@ async def run_agent_tool_loop("""Clear the page-refresh signal and, unless dropped or past the cap, reload and void `remaining`."""nonlocal refresh_cycles, refresh_nudge_due, reload_failed_nudge_due, pending_screenshots, outcomenonlocal pending_nav_dead_end, stall_nudges_due, last_change_evidence_step+ nonlocal page_state_stall_rounds, page_state_nudge_delivered, page_state_prev_fpctx.refresh_working_page = Falserefresh_cycles += 1if drop:@@ -1378,6 +1412,9 @@ async def run_agent_tool_loop(# so the run must re-demonstrate progress before it can earn an extension._clear_action_state()last_change_evidence_step = None+ page_state_stall_rounds = 0+ page_state_nudge_delivered = False+ page_state_prev_fp = Noneperception.reset()auto_perception.reset()if activity is not None:@@ -1776,14 +1813,20 @@ async def run_agent_tool_loop()# A cancellation that already landed makes this batch's baselines dead work: the per-call# check below (before the first dispatch) ends the batch before anything they'd inform runs.- batch_will_sample_baseline = (- auto_observe and batch_has_billable_call and (page_probe is not None or page_fingerprint is not None)+ batch_will_sample_baseline = batch_has_billable_call and (+ page_fingerprint is not None or (auto_observe and page_probe is not None))batch_cancelled = batch_will_sample_baseline and should_cancel is not None and await should_cancel()if auto_observe and page_probe is not None and batch_has_billable_call and not batch_cancelled:batch_probe_before = await _sample_probe(page_probe, deadline_at=deadline_at)- if auto_observe and page_fingerprint is not None and batch_has_billable_call and not batch_cancelled:+ # Sampled on BOTH arms (not just auto-observe): the page-state stall detector reads the+ # before/after fingerprint pair for every billable batch.+ if page_fingerprint is not None and batch_has_billable_call and not batch_cancelled:batch_fp_before = await _sample_probe(page_fingerprint, deadline_at=deadline_at)+ batch_fp_after: str | None = None+ # The auto-observe path's FINAL page-changed verdict (resample included) when it ran; the+ # stall detector prefers this over re-comparing raw samples so the two can never disagree.+ batch_auto_page_changed: bool | None = None# The batch's carrier for an auto-observe digest: the LAST tool message appended for an# EXECUTED (dispatched) call whose spec is not compactable -- i.e. a real action result, never# a skip stub and never a compactable perception dump that would otherwise elide the digest's@@ -2041,20 +2084,29 @@ async def run_agent_tool_loop(batch_observed_ok = Falseif tool_name == "observe" and result.status == "ok":batch_observed_ok = True- # Keeps the FIRST qualifying signal this batch -- the reason field is diagnostic- # (telemetry), not the decision itself, so a later call's signal never overwrites it.- if batch_page_change_reason is None:- if result_data.get("page_transitioned") is True:- batch_page_change_reason = "page_transitioned"- elif result_data.get("page_state_changed"):- batch_page_change_reason = "page_state_changed"- elif tool_name == "hover" and result.status == "ok":- # A hover's only purpose is to reveal state (submenus, tooltips) that a- # CSS-only change leaves invisible to the innerHTML fingerprint and the- # document-identity probe alike -- always treat it as a change signal.- batch_page_change_reason = "hover"- elif tool_name == "navigate" and result.status == "ok":- batch_page_change_reason = "navigate"+ # Accumulated on BOTH arms (hoisted out of the auto-observe gate for SKY-15265): the+ # page-state stall detector re-baselines on these flags, and a hover or file-upload+ # signal only the auto arm could see would false-kill the manual arm. Keeps the FIRST+ # qualifying signal this batch -- the reason field is diagnostic (telemetry), not the+ # decision itself, so a later call's signal never overwrites it.+ if batch_page_change_reason is None or batch_page_change_reason == "page_transitioned":+ # page_transitioned checks LAST and can be superseded: it is a URL-only hint the+ # stall detector must not reset on, so a stronger same-batch signal outranks it.+ if result_data.get("page_state_changed"):+ batch_page_change_reason = "page_state_changed"+ elif result_data.get("download_new"):+ # A freshly detected download is progress even when the DOM never moves (a+ # download-next flow); a replayed notice deliberately does not qualify.+ batch_page_change_reason = "download_new"+ elif tool_name == "hover" and result.status == "ok":+ # A hover's only purpose is to reveal state (submenus, tooltips) that a+ # CSS-only change leaves invisible to the innerHTML fingerprint and the+ # document-identity probe alike -- always treat it as a change signal.+ batch_page_change_reason = "hover"+ elif tool_name == "navigate" and result.status == "ok":+ batch_page_change_reason = "navigate"+ elif batch_page_change_reason is None and result_data.get("page_transitioned") is True:+ batch_page_change_reason = "page_transitioned"if _absorb_result_data(tool_name, spec, result_data):# Every mark=N still queued in this batch was chosen before this look renumbered the# marks, so it now names an arbitrary element; a look refused before rebuilding its@@ -2264,6 +2316,7 @@ async def run_agent_tool_loop(# probe below reads as "unchanged". Only trusted when BOTH samples landed -- a missing# before or after reading is not evidence either way, so it falls back to the probe.fp_after = await _sample_probe(page_fingerprint, deadline_at=deadline_at)+ batch_fp_after = fp_afterif batch_fp_before is not None and fp_after is not None:page_changed = fp_after != batch_fp_beforereason = "fingerprint_mismatch" if page_changed else "unchanged"@@ -2301,6 +2354,7 @@ async def run_agent_tool_loop(page_changed = Truereason = "fingerprint_mismatch" if signal == "fingerprint" else "probe_mismatch"+ batch_auto_page_changed = page_changedif not page_changed:messages[batch_carrier_idx]["content"] = (str(messages[batch_carrier_idx]["content"]) + "\n\n[no markup change detected after this batch]"@@ -2474,6 +2528,44 @@ async def run_agent_tool_loop(wait_seconds=wait_seconds,)+ # Page-state stall detector (SKY-15265): tool-independent — any batch of billable work that+ # leaves the rendered document byte-identical ticks the counter, whatever tools produced it.+ # A missing sample is no evidence either way; any page-change flag or fingerprint movement+ # re-baselines. When the auto-observe path already resolved the batch's verdict (resample+ # included), that verdict is reused so the two can never disagree.+ if outcome is None and turn_did_action and page_fingerprint is not None and batch_fp_before is not None:+ if page_state_prev_fp is not None and batch_fp_before != page_state_prev_fp:+ # The page moved BETWEEN batches (a delayed render landing after the prior+ # after-sample): the streak the old samples described is stale.+ page_state_stall_rounds = 0+ page_state_nudge_delivered = False+ page_state_changed: bool | None+ if batch_page_change_reason is not None and batch_page_change_reason != "page_transitioned":+ page_state_changed = True+ elif batch_auto_page_changed is not None and batch_page_change_reason is None:+ # The auto verdict treats ANY flag as changed, so when the only signal is the+ # URL-only page_transitioned hint, fall through to the raw fingerprint instead.+ page_state_changed = batch_auto_page_changed+ else:+ if batch_fp_after is None and not (deadline_at is not None and deadline_at - time.monotonic() <= 0):+ batch_fp_after = await _sample_probe(page_fingerprint, deadline_at=deadline_at)+ page_state_changed = None if batch_fp_after is None else batch_fp_after != batch_fp_before+ page_state_prev_fp = batch_fp_after if batch_fp_after is not None else batch_fp_before+ if page_state_changed is True:+ page_state_stall_rounds = 0+ page_state_nudge_delivered = False+ elif page_state_changed is False:+ page_state_stall_rounds += 1+ if page_state_stall_rounds == PAGE_STATE_STALL_TERMINATE_AFTER and page_state_nudge_delivered:+ # Shadow-only verdict: measured, not enforced (see PAGE_STATE_STALL_SHADOW_EVENT).+ LOG.info(+ PAGE_STATE_STALL_SHADOW_EVENT,+ rounds=page_state_stall_rounds,+ turn=turns,+ )+ elif page_state_stall_rounds >= PAGE_STATE_STALL_NUDGE_AFTER and not page_state_nudge_delivered:+ page_state_nudge_due = True+# Warn only after the batch completes: a user message may not sit between an assistant# turn's tool results, and the model reads it with the snapshot that tripped it. Every note# due this turn (including auto-observe's own stall nudge, folded in above) shares ONE user@@ -2485,6 +2577,11 @@ async def run_agent_tool_loop(nudge_parts.append(_reload_failed_nudge_text())if outcome is None and stall_nudges_due:nudge_parts.append(_stall_nudge_text(stall_nudges_due, set(tool_by_name)))+ if outcome is None and page_state_nudge_due:+ page_state_nudge_due = False+ page_state_nudge_delivered = True+ LOG.info("taskv3 loop page state stall nudged", rounds=page_state_stall_rounds, turn=turns)+ nudge_parts.append(_page_state_nudge_text(page_state_stall_rounds))if outcome is None and ao_nudges_result:nudge_parts.append(_stall_nudge_text(ao_nudges_result, set(tool_by_name)))if outcome is None and action_nudges_due:diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex d30e0f976..17fe3c2a0 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -35,6 +35,7 @@ from skyvern.forge.taskv3.loop import (NAVIGATION_DEAD_END_STATUSES,NO_TOOL_CALL_NUDGE,PAGE_REFRESH_EXHAUSTED_REASON_PREFIX,+ PAGE_STATE_STALL_SHADOW_EVENT,PAGE_UNAVAILABLE_ERROR,PERCEPTION_STALL_NUDGE_AFTER,PERCEPTION_STALL_REASON_PREFIX,@@ -2052,6 +2053,203 @@ def test_budget_extension_gate_deadline_scales_with_observed_pace() -> None:)+async def test_page_state_stall_nudges_then_terminates_a_frozen_page_cycle() -> None:+ # SKY-15265: a tool cycle that leaves the page fingerprint byte-identical round after round is+ # a stall no per-tool guard can see (varied selectors never streak; scroll/wait carry no+ # digest). The detector re-plans the model once, then ends the run with a facetable verdict —+ # on EITHER arm (auto-observe off here).+ clicks: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", clicks, billable=True)+ observe = _perception_tool("observe", [f"panel variant {i}" for i in range(30)])++ async def frozen_fingerprint() -> str:+ return "FROZEN-DOM"++ script: list[list[tuple[str, dict[str, Any]]]] = []+ for i in range(14):+ script.append([("observe", {})])+ script.append([("click", {"selector": f"#try-{i}"})]) # varied: the action-loop guard is blind+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [observe, click, make_finish_tool()],+ page_fingerprint=frozen_fingerprint,+ max_action_steps=24,+ max_turns=60,+ max_tool_calls=200,+ )+ assert outcome.status == "completed" # the verdict is SHADOW-only: measured, never enforced yet+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]+ assert len(nudges) == 1 # exactly one re-plan nudge+ shadow = [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]+ assert len(shadow) == 1 and shadow[0]["rounds"] == 12+ assert len(clicks) == 14 # nothing was cut short+++async def test_page_state_stall_never_fires_while_the_fingerprint_moves() -> None:+ # A real form fill mutates innerHTML every round, so the fingerprint moves and the detector+ # stays silent for the life of the run.+ clicks: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", clicks, billable=True)+ fp_n = {"n": 0}++ async def moving_fingerprint() -> str:+ fp_n["n"] += 1+ return f"dom-{fp_n['n']}"++ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#field-{i}"})] for i in range(14)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ page_fingerprint=moving_fingerprint,+ max_action_steps=24,+ max_turns=60,+ max_tool_calls=200,+ )+ assert outcome.status == "completed"+ assert len(clicks) == 14+++async def test_page_state_stall_counter_resets_when_the_cycle_breaks_after_the_nudge() -> None:+ # The nudge is a real second chance: a run that changes the page after being warned survives.+ clicks: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", clicks, billable=True)+ fp_state = {"n": 0}++ async def thawing_fingerprint() -> str:+ fp_state["n"] += 1+ # Two samples per round: frozen through round 9's after-sample (18 calls), moving after.+ return "FROZEN" if fp_state["n"] <= 18 else f"dom-{fp_state['n']}"++ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#try-{i}"})] for i in range(12)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ page_fingerprint=thawing_fingerprint,+ max_action_steps=24,+ max_turns=60,+ max_tool_calls=200,+ )+ assert outcome.status == "completed"+ assert len(clicks) == 12+ assert not [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]+++async def test_page_state_stall_sees_movement_landing_between_batches() -> None:+ # A delayed render can land after one batch's after-sample and before the next batch's+ # before-sample: each batch reads internally frozen, but the page IS moving. The detector+ # compares across batches, so this healthy pattern never accumulates a stall streak.+ clicks: list[tuple[str, dict[str, Any]]] = []+ click = _recording_tool("click", clicks, billable=True)+ fp_calls = {"n": 0}++ async def between_batch_fingerprint() -> str:+ # Two samples per batch (before, after): identical within a batch, different across batches.+ fp_calls["n"] += 1+ return f"dom-{(fp_calls['n'] - 1) // 2}"++ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#step-{i}"})] for i in range(10)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ page_fingerprint=between_batch_fingerprint,+ max_action_steps=24,+ max_turns=60,+ max_tool_calls=200,+ )+ assert outcome.status == "completed"+ assert len(clicks) == 10+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]+ assert nudges == []+ assert not [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]+++async def test_page_state_stall_not_defeated_by_url_only_transitions() -> None:+ # history.pushState churn moves the URL without touching the document: a URL-only transition is+ # a hint, and it must not reset the stall counter while the fingerprint stays frozen.+ clicks: list[tuple[str, dict[str, Any]]] = []++ async def push_state_click(args: dict[str, Any]) -> ToolResult:+ clicks.append(("click", args))+ return ToolResult.ok("clicked", data={"page_transitioned": True})++ click = ToolSpec(+ name="click",+ description="c",+ parameters={"type": "object", "properties": {}},+ handler=push_state_click,+ billable=True,+ )++ async def frozen_fingerprint() -> str:+ return "FROZEN-DOM"++ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#tab-{i}"})] for i in range(13)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ page_fingerprint=frozen_fingerprint,+ max_action_steps=24,+ max_turns=60,+ max_tool_calls=200,+ )+ assert outcome.status == "completed"+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]+ assert len(nudges) == 1+ assert [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]+++async def test_page_state_stall_resets_when_new_downloads_land() -> None:+ # A download-next flow produces files without changing the DOM: a freshly detected download is+ # real progress for this detector too, so a healthy multi-download run is never nudged.+ clicks: list[tuple[str, dict[str, Any]]] = []++ async def download_click(args: dict[str, Any]) -> ToolResult:+ clicks.append(("click", args))+ return ToolResult.ok("clicked", data={"download_notice": True, "download_new": True})++ click = ToolSpec(+ name="click",+ description="c",+ parameters={"type": "object", "properties": {}},+ handler=download_click,+ billable=True,+ )++ async def frozen_fingerprint() -> str:+ return "FROZEN-DOM"++ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#next-file-{i}"})] for i in range(10)]+ script.append([("finish", {"status": "completed", "reason": "done"})])+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [click, make_finish_tool()],+ page_fingerprint=frozen_fingerprint,+ max_action_steps=24,+ max_turns=60,+ max_tool_calls=200,+ )+ assert outcome.status == "completed"+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]+ assert nudges == []+ assert not [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]++def test_budget_extension_gate_vetoes_fire_independently() -> None:ok, reason = _budget_extension_gate(action_steps=10,