Skip to content

Automate browser based workflows with AI

Add a page-state stall detector to task_v3: live re-plan nudge, shadow would-terminate verdict (SKY-15265) (#8340)

pedrohsdbd5bd873fdc8b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index e505105e9..bcba2056a 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -1802,10 +1802,21 @@ class ForgeAgent:
# the gate, not here. Built for both populations — the failure-evidence gate needs a sampler
# to run at all — over the same page accessor each population's tools use, so the evidence
# is sampled from the page the model actually acted on.
+ # Field values are hashed separately: typing sets the value IDL property, which innerHTML
+ # serialization does not reflect, so a form being filled would otherwise read as frozen.
+ # Open shadow roots are traversed for the same reason — the tools pierce them, so work
+ # inside a shadow widget must move the fingerprint too (closed roots stay invisible).
_PAGE_FINGERPRINT_PROBE_JS = (
- "() => { if (!document.body) return '0'; const s = document.body.innerHTML;"
- " let h = 0; for (let i = 0; i < s.length; i++) h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;"
- " return h + ':' + s.length + ':' + document.querySelectorAll('*').length; }"
+ "() => { if (!document.body) return '0'; let h = 0; let v = 0; let elems = 0;"
+ " const mix = (str, seed) => { let x = seed;"
+ " for (let i = 0; i < str.length; i++) x = (Math.imul(x, 31) + str.charCodeAt(i)) | 0; return x; };"
+ " const walk = (root) => { h = mix(root.innerHTML || '', h);"
+ " const all = root.querySelectorAll('*'); elems += all.length;"
+ " for (const el of root.querySelectorAll('input, textarea, select'))"
+ " v = mix(String(el.value || '') + '|' + (el.checked === true ? '1' : '0'), v);"
+ " for (const el of all) { if (el.shadowRoot) walk(el.shadowRoot); } };"
+ " walk(document.body);"
+ " return h + ':' + elems + ':' + v; }"
)
_DOCUMENT_NONCE_JS = (
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index d8985685a..047d0c0b9 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -205,6 +205,21 @@ ACTION_BUDGET_EXTENSION_EVIDENCE_WINDOW = 8
ACTION_BUDGET_EXTENDED_EVENT = "taskv3 loop action budget extended"
ACTION_BUDGET_EXTENSION_REFUSED_EVENT = "taskv3 loop action budget extension refused"
+# Page-state stall policy (SKY-15265): rounds of billable batches that left the page fingerprint
+# byte-identical, with no page-change flag, mean the run is cycling on a frozen document in shapes
+# the per-tool guards cannot see (varied probes never streak; scroll/wait carry no digest at all).
+# Completions of the worst-affected canary block finish in <=9 rounds; the nudge re-plans the model
+# once before the verdict.
+PAGE_STATE_STALL_NUDGE_AFTER = 8
+PAGE_STATE_STALL_TERMINATE_AFTER = 12
+# The verdict is SHADOW-ONLY for now: the fingerprint is blind to work inside iframes (main-frame
+# innerHTML only), so a live termination could kill healthy embedded-widget runs. The nudge ships
+# live (benign direction); the shadow event measures the would-terminate precision on the canary,
+# and promotion to a live verdict is a separate release decision on that data.
+PAGE_STATE_STALL_SHADOW_EVENT = "taskv3 loop page state stall would terminate"
+# Facetable sibling of PERCEPTION_STALL_REASON_PREFIX; reserved for the future live verdict.
+PAGE_STATE_STALL_REASON_PREFIX = "page_state_stall:"
+
# Hard "the resource does not exist / is gone" HTTP statuses. A navigation landing on one of these is
# a genuine non-capability dead-end (a dead or removed posting), which v1 routes to `terminated`. Both
# the in-loop `navigate` tool and the pre-loop initial-URL navigation classify against this set. NARROW
@@ -817,6 +832,15 @@ def _unblocker_options(available_tools: set[str]) -> list[str]:
return options
+def _page_state_nudge_text(rounds: int) -> str:
+ return (
+ f"Your last {rounds} action rounds left the page's rendered content completely unchanged — "
+ "whatever you are trying is not affecting this page. Stop repeating the current approach: "
+ "re-plan from a fresh observe, try a genuinely different control or path, or finish honestly "
+ "(status=failed or terminated) naming what is blocking you."
+ )
+
+
def _stall_nudge_text(stalled: list[tuple[str, int]], available_tools: set[str]) -> str:
"""One warning naming every stalled perception tool and the unblockers this run actually has —
a model that cannot see the gate won't reach for solve_captcha unless the symptom names it."""
@@ -1282,6 +1306,15 @@ async def run_agent_tool_loop(
outcome: LoopOutcome | None = None
pending_nav_dead_end: int | None = None
stall_nudges_due: list[tuple[str, int]] = []
+ # Page-state stall detector (SKY-15265): consecutive billable rounds on a byte-identical
+ # fingerprint, whether the one re-plan nudge went out, and whether one is due this turn.
+ page_state_stall_rounds = 0
+ page_state_nudge_delivered = False
+ page_state_nudge_due = False
+ # The last fingerprint sample from the PREVIOUS batch: a delayed render can land between one
+ # batch's after-sample and the next batch's before-sample, so movement is checked across
+ # batches, not only within them.
+ page_state_prev_fp: str | None = None
refresh_cycles = 0
refresh_nudge_due = False
reload_failed_nudge_due = False
@@ -1340,6 +1373,7 @@ async def run_agent_tool_loop(
"""Clear the page-refresh signal and, unless dropped or past the cap, reload and void `remaining`."""
nonlocal refresh_cycles, refresh_nudge_due, reload_failed_nudge_due, pending_screenshots, outcome
nonlocal pending_nav_dead_end, stall_nudges_due, last_change_evidence_step
+ nonlocal page_state_stall_rounds, page_state_nudge_delivered, page_state_prev_fp
ctx.refresh_working_page = False
refresh_cycles += 1
if drop:
@@ -1378,6 +1412,9 @@ async def run_agent_tool_loop(
# so the run must re-demonstrate progress before it can earn an extension.
_clear_action_state()
last_change_evidence_step = None
+ page_state_stall_rounds = 0
+ page_state_nudge_delivered = False
+ page_state_prev_fp = None
perception.reset()
auto_perception.reset()
if activity is not None:
@@ -1776,14 +1813,20 @@ async def run_agent_tool_loop(
)
# A cancellation that already landed makes this batch's baselines dead work: the per-call
# check below (before the first dispatch) ends the batch before anything they'd inform runs.
- batch_will_sample_baseline = (
- auto_observe and batch_has_billable_call and (page_probe is not None or page_fingerprint is not None)
+ batch_will_sample_baseline = batch_has_billable_call and (
+ page_fingerprint is not None or (auto_observe and page_probe is not None)
)
batch_cancelled = batch_will_sample_baseline and should_cancel is not None and await should_cancel()
if auto_observe and page_probe is not None and batch_has_billable_call and not batch_cancelled:
batch_probe_before = await _sample_probe(page_probe, deadline_at=deadline_at)
- if auto_observe and page_fingerprint is not None and batch_has_billable_call and not batch_cancelled:
+ # Sampled on BOTH arms (not just auto-observe): the page-state stall detector reads the
+ # before/after fingerprint pair for every billable batch.
+ if page_fingerprint is not None and batch_has_billable_call and not batch_cancelled:
batch_fp_before = await _sample_probe(page_fingerprint, deadline_at=deadline_at)
+ batch_fp_after: str | None = None
+ # The auto-observe path's FINAL page-changed verdict (resample included) when it ran; the
+ # stall detector prefers this over re-comparing raw samples so the two can never disagree.
+ batch_auto_page_changed: bool | None = None
# The batch's carrier for an auto-observe digest: the LAST tool message appended for an
# EXECUTED (dispatched) call whose spec is not compactable -- i.e. a real action result, never
# a skip stub and never a compactable perception dump that would otherwise elide the digest's
@@ -2041,20 +2084,29 @@ async def run_agent_tool_loop(
batch_observed_ok = False
if tool_name == "observe" and result.status == "ok":
batch_observed_ok = True
- # Keeps the FIRST qualifying signal this batch -- the reason field is diagnostic
- # (telemetry), not the decision itself, so a later call's signal never overwrites it.
- if batch_page_change_reason is None:
- if result_data.get("page_transitioned") is True:
- batch_page_change_reason = "page_transitioned"
- elif result_data.get("page_state_changed"):
- batch_page_change_reason = "page_state_changed"
- elif tool_name == "hover" and result.status == "ok":
- # A hover's only purpose is to reveal state (submenus, tooltips) that a
- # CSS-only change leaves invisible to the innerHTML fingerprint and the
- # document-identity probe alike -- always treat it as a change signal.
- batch_page_change_reason = "hover"
- elif tool_name == "navigate" and result.status == "ok":
- batch_page_change_reason = "navigate"
+ # Accumulated on BOTH arms (hoisted out of the auto-observe gate for SKY-15265): the
+ # page-state stall detector re-baselines on these flags, and a hover or file-upload
+ # signal only the auto arm could see would false-kill the manual arm. Keeps the FIRST
+ # qualifying signal this batch -- the reason field is diagnostic (telemetry), not the
+ # decision itself, so a later call's signal never overwrites it.
+ if batch_page_change_reason is None or batch_page_change_reason == "page_transitioned":
+ # page_transitioned checks LAST and can be superseded: it is a URL-only hint the
+ # stall detector must not reset on, so a stronger same-batch signal outranks it.
+ if result_data.get("page_state_changed"):
+ batch_page_change_reason = "page_state_changed"
+ elif result_data.get("download_new"):
+ # A freshly detected download is progress even when the DOM never moves (a
+ # download-next flow); a replayed notice deliberately does not qualify.
+ batch_page_change_reason = "download_new"
+ elif tool_name == "hover" and result.status == "ok":
+ # A hover's only purpose is to reveal state (submenus, tooltips) that a
+ # CSS-only change leaves invisible to the innerHTML fingerprint and the
+ # document-identity probe alike -- always treat it as a change signal.
+ batch_page_change_reason = "hover"
+ elif tool_name == "navigate" and result.status == "ok":
+ batch_page_change_reason = "navigate"
+ elif batch_page_change_reason is None and result_data.get("page_transitioned") is True:
+ batch_page_change_reason = "page_transitioned"
if _absorb_result_data(tool_name, spec, result_data):
# Every mark=N still queued in this batch was chosen before this look renumbered the
# marks, so it now names an arbitrary element; a look refused before rebuilding its
@@ -2264,6 +2316,7 @@ async def run_agent_tool_loop(
# probe below reads as "unchanged". Only trusted when BOTH samples landed -- a missing
# before or after reading is not evidence either way, so it falls back to the probe.
fp_after = await _sample_probe(page_fingerprint, deadline_at=deadline_at)
+ batch_fp_after = fp_after
if batch_fp_before is not None and fp_after is not None:
page_changed = fp_after != batch_fp_before
reason = "fingerprint_mismatch" if page_changed else "unchanged"
@@ -2301,6 +2354,7 @@ async def run_agent_tool_loop(
page_changed = True
reason = "fingerprint_mismatch" if signal == "fingerprint" else "probe_mismatch"
+ batch_auto_page_changed = page_changed
if not page_changed:
messages[batch_carrier_idx]["content"] = (
str(messages[batch_carrier_idx]["content"]) + "\n\n[no markup change detected after this batch]"
@@ -2474,6 +2528,44 @@ async def run_agent_tool_loop(
wait_seconds=wait_seconds,
)
+ # Page-state stall detector (SKY-15265): tool-independent — any batch of billable work that
+ # leaves the rendered document byte-identical ticks the counter, whatever tools produced it.
+ # A missing sample is no evidence either way; any page-change flag or fingerprint movement
+ # re-baselines. When the auto-observe path already resolved the batch's verdict (resample
+ # included), that verdict is reused so the two can never disagree.
+ if outcome is None and turn_did_action and page_fingerprint is not None and batch_fp_before is not None:
+ if page_state_prev_fp is not None and batch_fp_before != page_state_prev_fp:
+ # The page moved BETWEEN batches (a delayed render landing after the prior
+ # after-sample): the streak the old samples described is stale.
+ page_state_stall_rounds = 0
+ page_state_nudge_delivered = False
+ page_state_changed: bool | None
+ if batch_page_change_reason is not None and batch_page_change_reason != "page_transitioned":
+ page_state_changed = True
+ elif batch_auto_page_changed is not None and batch_page_change_reason is None:
+ # The auto verdict treats ANY flag as changed, so when the only signal is the
+ # URL-only page_transitioned hint, fall through to the raw fingerprint instead.
+ page_state_changed = batch_auto_page_changed
+ else:
+ if batch_fp_after is None and not (deadline_at is not None and deadline_at - time.monotonic() <= 0):
+ batch_fp_after = await _sample_probe(page_fingerprint, deadline_at=deadline_at)
+ page_state_changed = None if batch_fp_after is None else batch_fp_after != batch_fp_before
+ page_state_prev_fp = batch_fp_after if batch_fp_after is not None else batch_fp_before
+ if page_state_changed is True:
+ page_state_stall_rounds = 0
+ page_state_nudge_delivered = False
+ elif page_state_changed is False:
+ page_state_stall_rounds += 1
+ if page_state_stall_rounds == PAGE_STATE_STALL_TERMINATE_AFTER and page_state_nudge_delivered:
+ # Shadow-only verdict: measured, not enforced (see PAGE_STATE_STALL_SHADOW_EVENT).
+ LOG.info(
+ PAGE_STATE_STALL_SHADOW_EVENT,
+ rounds=page_state_stall_rounds,
+ turn=turns,
+ )
+ elif page_state_stall_rounds >= PAGE_STATE_STALL_NUDGE_AFTER and not page_state_nudge_delivered:
+ page_state_nudge_due = True
+
# Warn only after the batch completes: a user message may not sit between an assistant
# turn's tool results, and the model reads it with the snapshot that tripped it. Every note
# due this turn (including auto-observe's own stall nudge, folded in above) shares ONE user
@@ -2485,6 +2577,11 @@ async def run_agent_tool_loop(
nudge_parts.append(_reload_failed_nudge_text())
if outcome is None and stall_nudges_due:
nudge_parts.append(_stall_nudge_text(stall_nudges_due, set(tool_by_name)))
+ if outcome is None and page_state_nudge_due:
+ page_state_nudge_due = False
+ page_state_nudge_delivered = True
+ LOG.info("taskv3 loop page state stall nudged", rounds=page_state_stall_rounds, turn=turns)
+ nudge_parts.append(_page_state_nudge_text(page_state_stall_rounds))
if outcome is None and ao_nudges_result:
nudge_parts.append(_stall_nudge_text(ao_nudges_result, set(tool_by_name)))
if outcome is None and action_nudges_due:
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index d30e0f976..17fe3c2a0 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -35,6 +35,7 @@ from skyvern.forge.taskv3.loop import (
NAVIGATION_DEAD_END_STATUSES,
NO_TOOL_CALL_NUDGE,
PAGE_REFRESH_EXHAUSTED_REASON_PREFIX,
+ PAGE_STATE_STALL_SHADOW_EVENT,
PAGE_UNAVAILABLE_ERROR,
PERCEPTION_STALL_NUDGE_AFTER,
PERCEPTION_STALL_REASON_PREFIX,
@@ -2052,6 +2053,203 @@ def test_budget_extension_gate_deadline_scales_with_observed_pace() -> None:
)
+async def test_page_state_stall_nudges_then_terminates_a_frozen_page_cycle() -> None:
+ # SKY-15265: a tool cycle that leaves the page fingerprint byte-identical round after round is
+ # a stall no per-tool guard can see (varied selectors never streak; scroll/wait carry no
+ # digest). The detector re-plans the model once, then ends the run with a facetable verdict —
+ # on EITHER arm (auto-observe off here).
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", clicks, billable=True)
+ observe = _perception_tool("observe", [f"panel variant {i}" for i in range(30)])
+
+ async def frozen_fingerprint() -> str:
+ return "FROZEN-DOM"
+
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(14):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": f"#try-{i}"})]) # varied: the action-loop guard is blind
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [observe, click, make_finish_tool()],
+ page_fingerprint=frozen_fingerprint,
+ max_action_steps=24,
+ max_turns=60,
+ max_tool_calls=200,
+ )
+ assert outcome.status == "completed" # the verdict is SHADOW-only: measured, never enforced yet
+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]
+ assert len(nudges) == 1 # exactly one re-plan nudge
+ shadow = [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]
+ assert len(shadow) == 1 and shadow[0]["rounds"] == 12
+ assert len(clicks) == 14 # nothing was cut short
+
+
+async def test_page_state_stall_never_fires_while_the_fingerprint_moves() -> None:
+ # A real form fill mutates innerHTML every round, so the fingerprint moves and the detector
+ # stays silent for the life of the run.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", clicks, billable=True)
+ fp_n = {"n": 0}
+
+ async def moving_fingerprint() -> str:
+ fp_n["n"] += 1
+ return f"dom-{fp_n['n']}"
+
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#field-{i}"})] for i in range(14)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ page_fingerprint=moving_fingerprint,
+ max_action_steps=24,
+ max_turns=60,
+ max_tool_calls=200,
+ )
+ assert outcome.status == "completed"
+ assert len(clicks) == 14
+
+
+async def test_page_state_stall_counter_resets_when_the_cycle_breaks_after_the_nudge() -> None:
+ # The nudge is a real second chance: a run that changes the page after being warned survives.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", clicks, billable=True)
+ fp_state = {"n": 0}
+
+ async def thawing_fingerprint() -> str:
+ fp_state["n"] += 1
+ # Two samples per round: frozen through round 9's after-sample (18 calls), moving after.
+ return "FROZEN" if fp_state["n"] <= 18 else f"dom-{fp_state['n']}"
+
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#try-{i}"})] for i in range(12)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ page_fingerprint=thawing_fingerprint,
+ max_action_steps=24,
+ max_turns=60,
+ max_tool_calls=200,
+ )
+ assert outcome.status == "completed"
+ assert len(clicks) == 12
+ assert not [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]
+
+
+async def test_page_state_stall_sees_movement_landing_between_batches() -> None:
+ # A delayed render can land after one batch's after-sample and before the next batch's
+ # before-sample: each batch reads internally frozen, but the page IS moving. The detector
+ # compares across batches, so this healthy pattern never accumulates a stall streak.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", clicks, billable=True)
+ fp_calls = {"n": 0}
+
+ async def between_batch_fingerprint() -> str:
+ # Two samples per batch (before, after): identical within a batch, different across batches.
+ fp_calls["n"] += 1
+ return f"dom-{(fp_calls['n'] - 1) // 2}"
+
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#step-{i}"})] for i in range(10)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ page_fingerprint=between_batch_fingerprint,
+ max_action_steps=24,
+ max_turns=60,
+ max_tool_calls=200,
+ )
+ assert outcome.status == "completed"
+ assert len(clicks) == 10
+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]
+ assert nudges == []
+ assert not [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]
+
+
+async def test_page_state_stall_not_defeated_by_url_only_transitions() -> None:
+ # history.pushState churn moves the URL without touching the document: a URL-only transition is
+ # a hint, and it must not reset the stall counter while the fingerprint stays frozen.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+
+ async def push_state_click(args: dict[str, Any]) -> ToolResult:
+ clicks.append(("click", args))
+ return ToolResult.ok("clicked", data={"page_transitioned": True})
+
+ click = ToolSpec(
+ name="click",
+ description="c",
+ parameters={"type": "object", "properties": {}},
+ handler=push_state_click,
+ billable=True,
+ )
+
+ async def frozen_fingerprint() -> str:
+ return "FROZEN-DOM"
+
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#tab-{i}"})] for i in range(13)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ page_fingerprint=frozen_fingerprint,
+ max_action_steps=24,
+ max_turns=60,
+ max_tool_calls=200,
+ )
+ assert outcome.status == "completed"
+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]
+ assert len(nudges) == 1
+ assert [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]
+
+
+async def test_page_state_stall_resets_when_new_downloads_land() -> None:
+ # A download-next flow produces files without changing the DOM: a freshly detected download is
+ # real progress for this detector too, so a healthy multi-download run is never nudged.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+
+ async def download_click(args: dict[str, Any]) -> ToolResult:
+ clicks.append(("click", args))
+ return ToolResult.ok("clicked", data={"download_notice": True, "download_new": True})
+
+ click = ToolSpec(
+ name="click",
+ description="c",
+ parameters={"type": "object", "properties": {}},
+ handler=download_click,
+ billable=True,
+ )
+
+ async def frozen_fingerprint() -> str:
+ return "FROZEN-DOM"
+
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("click", {"selector": f"#next-file-{i}"})] for i in range(10)]
+ script.append([("finish", {"status": "completed", "reason": "done"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [click, make_finish_tool()],
+ page_fingerprint=frozen_fingerprint,
+ max_action_steps=24,
+ max_turns=60,
+ max_tool_calls=200,
+ )
+ assert outcome.status == "completed"
+ nudges = [m for m in outcome.messages if m.get("role") == "user" and "unchanged" in str(m.get("content"))]
+ assert nudges == []
+ assert not [entry for entry in logs if entry["event"] == PAGE_STATE_STALL_SHADOW_EVENT]
+
+
def test_budget_extension_gate_vetoes_fire_independently() -> None:
ok, reason = _budget_extension_gate(
action_steps=10,