Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Task V3: canonical progress tracker, Phase A log-only (SKY-15379) (#8385)
pedrohsdb84e8dfbd21b9
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex bbcdc63c6..601c17dfa 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -427,6 +427,21 @@ class _PerceptionLedger:PROGRESS_LEDGER_WINDOW = 8# Facetable event names; the offline precision/survival metrics key on these — change only with the# dashboards that read them.+# Canonical progress signal, Phase A (SKY-15379): ONE target-churn + compound-progress tracker,+# LOG-ONLY. The loop answers "is this run progressing?" five separate ways today; this tracker is+# the candidate replacement signal, emitting the parity/precision data any consolidation needs+# while changing no behavior. Its target key is the canonicalized SELECTOR, not tool+args — the+# incumbent action-loop key provably missed prod repeat-loops whose args varied on a fixed target.+CANONICAL_LOOP_EVENT = "taskv3 canonical progress loop detected"+CANONICAL_EXTEND_DELTA_EVENT = "taskv3 canonical progress extension delta"+# Log-only thresholds; the Phase-B intervention thresholds are tuned FROM this data, so the fire+# points are logged at every rung of the distribution rather than one cliff.+CANONICAL_LOOP_WINDOW = 10+CANONICAL_LOOP_FIRE_COUNTS = (3, 4, 6, 8)+# The rung that counts as "looping" for the extension-delta read — named so reordering the logging+# rungs above can never silently move the threshold Phase B tunes against.+CANONICAL_LOOP_TOUCHES = 4+PROGRESS_LEDGER_SHADOW_EVENT = "taskv3 loop progress ledger would fail-fast"PROGRESS_LEDGER_FINAL_EVENT = "taskv3 loop progress ledger final"@@ -917,6 +932,62 @@ _TOOL_CALL_RECORD_FIELDS = frozenset()+@dataclass+class _CanonicalProgressTracker:+ """Touches on canonicalized targets since the last compound-progress event (SKY-15379, Phase A).++ A touch is one billable tool dispatch, keyed by its selector (tool name when selector-less).+ Progress — any of the ledger's hard-progress events, an invalid-fields new-low, or a confirmed+ page change — clears the ring, so a repeat streak can only accumulate across a genuinely+ unchanged situation; a progressing run is untrippable by construction. Log-only: the counts+ feed telemetry, nothing reads them for control flow.+ """++ window: int = CANONICAL_LOOP_WINDOW+ touches: list[tuple[str, bool]] = field(default_factory=list)+ # Bumped on every clear: a pending loop event minted under an older generation was completed+ # by (or followed by) absorbed progress and must not be emitted.+ gen: int = 0+ # Rungs already fired this generation. Once the window saturates, same-target counts can PARK+ # on a fire rung while churn evicts other keys — a rung is one threshold crossing, not one+ # event per touch spent at it.+ fired: set[tuple[str, int]] = field(default_factory=set)++ def record_touch(self, target_key: str, is_error: bool) -> tuple[int, int]:+ """Record one billable touch; returns (same-target touches, same-target errors) in-window."""+ self.touches.append((target_key, is_error))+ if len(self.touches) > self.window:+ self.touches.pop(0)+ same = [err for key, err in self.touches if key == target_key]+ # A rung re-arms once the in-window count dips below it: a fresh streak after full eviction+ # is a genuine re-crossing, distinct from a saturated window PARKED on a fire count.+ self.fired = {entry for entry in self.fired if entry[0] != target_key or entry[1] <= len(same)}+ return len(same), sum(1 for err in same if err)++ def progress(self) -> None:+ self.touches.clear()+ self.fired.clear()+ self.gen += 1++ def invalidate_marks(self) -> None:+ """A look renumbered the manifest, so every mark=N key now names an arbitrary element: drop+ them rather than let four different controls alias into one false streak. A surviving mark+ streak is therefore always within a single manifest generation. Selector keys stay — their+ identity outlives the renumbering."""+ self.touches = [touch for touch in self.touches if not touch[0].startswith("mark=")]+ self.fired = {entry for entry in self.fired if not entry[0].startswith("mark=")}++ def looping_targets(self) -> int:+ counts: dict[str, list[bool]] = {}+ for key, err in self.touches:+ counts.setdefault(key, []).append(err)+ return sum(+ 1+ for errs in counts.values()+ if len(errs) >= CANONICAL_LOOP_TOUCHES and sum(errs) >= CANONICAL_LOOP_TOUCHES - 1+ )++def _observe_summary_fields(result: ToolResult) -> dict[str, int]:"""Counts only: the summary is built by the tool, but an indexed field is re-checked here."""summary = (result.data or {}).get("summary")@@ -1358,6 +1429,7 @@ async def run_agent_tool_loop(auto_perception = _PerceptionLedger()# Net-progress ledger (additive shadow); None disables it, mirroring the guard's *_after knobs.progress = _ProgressLedger(window=progress_window) if progress_window is not None else None+ canonical = _CanonicalProgressTracker()# The action-loop counter: (repeat count, first turn of the streak) per billable action# identity, cleared whenever evidence of page change arrives. action_warned holds the streaks# whose warning was actually DELIVERED — termination is gated on it, so the model always gets@@ -1434,19 +1506,31 @@ async def run_agent_tool_loop(stall_nudges_due = []if progress is not None:progress.hard_progress()+ canonical.progress()if submit_watch is not None:submit_watch.clear()_append_skipped_tool_results(messages, remaining, "the page was refreshed")refresh_nudge_due = Truereturn True- def _progress_observe_shadow(observe_summary: dict[str, int], tool_name: str, attribution: dict[str, Any]) -> None:+ def _progress_observe_shadow(+ observe_summary: dict[str, int], tool_name: str, attribution: dict[str, Any], baseline_before: int | None+ ) -> None:"""Shared by the model-dispatched and auto-observe paths so an auto-observe feeds the sameledger a model-issued no-arg observe would."""if progress is None or not observe_summary:returninvalid_fields = observe_summary.get("invalid_fields")- if invalid_fields is not None and progress.observe(invalid_fields):+ # The ledger's True return is the SHADOW STALL verdict, not progress — the canonical clear+ # keys on the ledger re-baselining (a new low, or a rise re-baseline: fresh context), which+ # is visible as its baseline moving under an already-set baseline. The baseline is captured+ # BEFORE _absorb_result_data runs: a replayed download notice hard-progresses the shadow+ # ledger (nulling the baseline) on the same result whose summary carries the new low, and+ # reading the post-absorb value would leave this clear dead for the rest of the page.+ stalled = invalid_fields is not None and progress.observe(invalid_fields)+ if baseline_before is not None and progress.invalid_baseline != baseline_before:+ canonical.progress()+ if stalled:LOG.info(PROGRESS_LEDGER_SHADOW_EVENT,actions=progress.actions_since_progress,@@ -1494,6 +1578,16 @@ async def run_agent_tool_loop(activity.perception_stall_imminent = Falseif progress is not None:progress.hard_progress()+ if (+ result_data.get("download_new")+ or result_data.get("page_state_changed")+ or result_data.get("same_url_reload")+ or result_data.get("nav_revisit")+ ):+ # A REPLAYED notice (download_notice without download_new) re-clears the retry+ # ledger but is not fresh progress: it must not keep wiping the canonical ring, or+ # a post-download loop could never accumulate enough touches to emit telemetry.+ canonical.progress()# A click that moved the URL is a real page transition (H1 hard progress) for the shadow# ledger, but URL equality does NOT prove same-page (a URL-stable SPA form advance) — so only# the positive direction is acted on, and kept OUT of the branch above so it never clears the@@ -1503,6 +1597,7 @@ async def run_agent_tool_loop(if result_data.get("page_transitioned") is True:if progress is not None:progress.hard_progress()+ canonical.progress()return tool_name == "look" and bool(result_data.get("marks_renumbered"))async def _completion_probe_outcome(@@ -1527,6 +1622,9 @@ async def run_agent_tool_loop(if not completion_reason:return NoneLOG.info("taskv3 loop completion probe fired", tool=tool_name, turn=turns)+ # No tracker-wide clear here: the probe firing is download progress for the COMPLETING+ # touch only, and its call site drops that one target's pending rungs — a whole-generation+ # bump would erase a sibling target's true-positive rung along with it.return LoopOutcome("completed", completion_reason)def _perception_stall_check(@@ -1558,6 +1656,9 @@ async def run_agent_tool_loop(# terminate verdict, since it fires far more often than the model calls observe on its own# and would trip those far too eagerly for a path the model never asked to take._clear_action_state()+ # Two landed digests that differ are positive evidence, exactly like a fingerprint+ # mismatch — and the only movement evidence there is when page_fingerprint is absent.+ canonical.progress()# Evidence requires NEW content: a URL-only flip (history.pushState) still clears the# repeat guards above but earns no budget, and neither does a return to a content state# in the probe's recent ring (a panel toggling open and shut).@@ -1814,6 +1915,9 @@ async def run_agent_tool_loop(failed_selectors: set[str] = set()batch_had_failure = Falsemarks_stale = False+ # Loop events minted this batch, emitted only after every progress signal the batch can+ # produce has been absorbed (see the end-of-batch emission below).+ pending_canonical_fires: list[dict[str, Any]] = []# Auto-observe signals, gathered across the whole batch so the end-of-batch check reads the# batch's net effect rather than any one call's.batch_observed_ok = False@@ -1839,10 +1943,19 @@ async def run_agent_tool_loop(# before/after fingerprint pair for every billable batch.if page_fingerprint is not None and batch_has_billable_call and not batch_cancelled:batch_fp_before = await _sample_probe(page_fingerprint, deadline_at=deadline_at)+ if page_state_prev_fp is not None and batch_fp_before is not None and batch_fp_before != page_state_prev_fp:+ # The page moved BETWEEN batches (a delayed render landing after the prior+ # after-sample): the touches the old samples described are stale, and this batch's+ # dispatch and extension decisions must not read them. Canonical-only — the+ # incumbent stall counters keep their end-of-batch turn_did_action gate.+ canonical.progress()batch_fp_after: str | None = None# The auto-observe path's FINAL page-changed verdict (resample included) when it ran; the# stall detector prefers this over re-comparing raw samples so the two can never disagree.batch_auto_page_changed: bool | None = None+ # Whether that verdict rested on POSITIVE evidence (a flag or two landed samples that+ # differ) rather than the missing-sample "assume changed" fallback.+ batch_auto_page_change_evidence: bool = False# The batch's carrier for an auto-observe digest: the LAST tool message appended for an# EXECUTED (dispatched) call whose spec is not compactable -- i.e. a real action result, never# a skip stub and never a compactable perception dump that would otherwise elide the digest's@@ -1955,6 +2068,18 @@ async def run_agent_tool_loop(extension,seconds_per_step=(time.monotonic() - started_at) / max(action_steps, 1),)+ # This read cannot see unabsorbed in-batch movement: action_steps is frozen during+ # a batch, so the cap check trips on the batch's FIRST billable call — before any+ # page action dispatches — and between-batch movement was absorbed when+ # batch_fp_before was sampled.+ looping_now = canonical.looping_targets()+ LOG.info(+ CANONICAL_EXTEND_DELTA_EVENT,+ current_decision=allowed,+ gate_reason=gate_reason,+ canonical_looping_targets=looping_now,+ would_block_extension=bool(allowed and looping_now),+ )if allowed:original_cap = max_action_stepsbudget_extension_granted = True@@ -2069,6 +2194,27 @@ async def run_agent_tool_loop(**observe_summary,**attribution,)+ if spec is not None and spec.billable:+ target_key = _call_selector(args) or f"tool:{tool_name}"+ same_count, same_errors = canonical.record_touch(target_key, result.status == "error")+ # Every rung of the distribution is logged (not one cliff): the Phase-B block+ # threshold is tuned from this data.+ if (+ same_count in CANONICAL_LOOP_FIRE_COUNTS+ and same_errors >= same_count - 1+ and (target_key, same_count) not in canonical.fired+ ):+ canonical.fired.add((target_key, same_count))+ pending_canonical_fires.append(+ {+ "gen": canonical.gen,+ "target_key_hash": telemetry_hash(telemetry_salt, target_key),+ "tool": tool_name,+ "repeat_count": same_count,+ "repeat_errors": same_errors,+ "turn": turns,+ }+ )if spec is not None and spec.compactable and result.status == "ok":snapshot_indices.add(len(messages)) # index this successful snapshot will occupy, pre-append@@ -2123,12 +2269,14 @@ async def run_agent_tool_loop(batch_page_change_reason = "navigate"elif batch_page_change_reason is None and result_data.get("page_transitioned") is True:batch_page_change_reason = "page_transitioned"+ shadow_baseline_before = progress.invalid_baseline if progress is not None else Noneif _absorb_result_data(tool_name, spec, result_data):# Every mark=N still queued in this batch was chosen before this look renumbered the# marks, so it now names an arbitrary element; a look refused before rebuilding its# manifest leaves the old marks (and their failed keys) live.marks_stale = True- _progress_observe_shadow(observe_summary, tool_name, attribution)+ canonical.invalidate_marks()+ _progress_observe_shadow(observe_summary, tool_name, attribution, shadow_baseline_before)if content_digest is not None:stall_outcome, nudges_due = _perception_stall_check(perception,@@ -2222,6 +2370,13 @@ async def run_agent_tool_loop(completion_outcome = await _completion_probe_outcome(tool_name, spec, result_data)if completion_outcome is not None:outcome = completion_outcome+ if spec is not None and spec.billable:+ # The download landed while this call's probe waited, so this call's touch is+ # the progressing one: drop ITS pending rungs only, never a sibling target's.+ completing_hash = telemetry_hash(telemetry_salt, _call_selector(args) or f"tool:{tool_name}")+ pending_canonical_fires[:] = [+ fire for fire in pending_canonical_fires if fire["target_key_hash"] != completing_hash+ ]_append_skipped_tool_results(messages, tool_calls[idx + 1 :], "completion probe fired")break@@ -2261,6 +2416,11 @@ async def run_agent_tool_loop(if not poisoned and spec is not None and page_probe is not None:probe_after = await _sample_probe(page_probe, deadline_at=deadline_at)poisoned = probe_before is None or probe_after is None or probe_after != probe_before+ if probe_before is not None and probe_after is not None and probe_after != probe_before:+ # Two landed identity samples that differ: the failed call still moved the+ # document, and the fingerprint may render identically on the new one (a+ # same-template step) — absorb it here or the rung survives that blindness.+ canonical.progress()if poisoned:_append_skipped_tool_results(messages,@@ -2324,8 +2484,13 @@ async def run_agent_tool_loop(page_changed: boolreason: strsignal: str+ # Distinguishes a POSITIVELY observed change (a flag, or two landed samples that+ # differ) from the missing-sample fallback below, which is a model-facing "assume+ # changed" but is not evidence and must not clear the canonical tracker.+ page_change_evidence: boolif batch_page_change_reason is not None:page_changed, reason, signal = True, batch_page_change_reason, "flag"+ page_change_evidence = Trueelif page_fingerprint is not None:# The fingerprint samples rendered content (innerHTML), so it catches an in-page# mutation (a dropdown, a validation error, a revealed section) the document-identity@@ -2335,26 +2500,33 @@ async def run_agent_tool_loop(batch_fp_after = fp_afterif batch_fp_before is not None and fp_after is not None:page_changed = fp_after != batch_fp_before+ page_change_evidence = page_changedreason = "fingerprint_mismatch" if page_changed else "unchanged"signal = "fingerprint"elif page_probe is not None:probe_after = await _sample_probe(page_probe, deadline_at=deadline_at)if batch_probe_before is None or probe_after is None or probe_after != batch_probe_before:page_changed, reason = True, "probe_mismatch"+ page_change_evidence = batch_probe_before is not None and probe_after is not Noneelse:page_changed, reason = False, "unchanged"+ page_change_evidence = Falsesignal = "probe"else:page_changed, reason, signal = False, "unchanged", "none"+ page_change_evidence = Falseelif page_probe is not None:probe_after = await _sample_probe(page_probe, deadline_at=deadline_at)if batch_probe_before is None or probe_after is None or probe_after != batch_probe_before:page_changed, reason = True, "probe_mismatch"+ page_change_evidence = batch_probe_before is not None and probe_after is not Noneelse:page_changed, reason = False, "unchanged"+ page_change_evidence = Falsesignal = "probe"else:page_changed, reason, signal = False, "unchanged", "none"+ page_change_evidence = Falseif not page_changed and signal in ("fingerprint", "probe"):# The immediate comparison above can be too early for an async render; give it one@@ -2368,9 +2540,13 @@ async def run_agent_tool_loop(wait_seconds += resample_waitif resampled_changed:page_changed = True+ page_change_evidence = Truereason = "fingerprint_mismatch" if signal == "fingerprint" else "probe_mismatch"batch_auto_page_changed = page_changed+ batch_auto_page_change_evidence = page_change_evidence+ if page_changed and page_change_evidence:+ canonical.progress()if not page_changed:messages[batch_carrier_idx]["content"] = (str(messages[batch_carrier_idx]["content"]) + "\n\n[no markup change detected after this batch]"@@ -2466,6 +2642,7 @@ async def run_agent_tool_loop(# progress, and completion) lands identically whichever path produced it —# independent of whether the observe also produced a digest below.ao_result_data = ao_result.data or {}+ ao_baseline_before = progress.invalid_baseline if progress is not None else None_absorb_result_data("observe", observe_spec, ao_result_data)completion_outcome = await _completion_probe_outcome("observe", observe_spec, ao_result_data@@ -2498,7 +2675,9 @@ async def run_agent_tool_loop("snapshot_digest": telemetry_hash(telemetry_salt, ao_digest),"probe_first_time": auto_perception.first_time(ao_action_key),}- _progress_observe_shadow(_observe_summary_fields(ao_result), "observe", ao_attribution)+ _progress_observe_shadow(+ _observe_summary_fields(ao_result), "observe", ao_attribution, ao_baseline_before+ )stall_outcome, ao_nudges_result = _perception_stall_check(auto_perception,ao_digest,@@ -2570,6 +2749,15 @@ async def run_agent_tool_loop(if page_state_changed is True:page_state_stall_rounds = 0page_state_nudge_delivered = False+ # The auto verdict's missing-sample fallback ("assume changed") re-baselines the+ # incumbent counters above but is not positive evidence: only a flagged or+ # sample-confirmed change may clear the canonical tracker.+ if not (+ batch_page_change_reason is None+ and batch_auto_page_changed is True+ and not batch_auto_page_change_evidence+ ):+ canonical.progress()elif page_state_changed is False:page_state_stall_rounds += 1if page_state_stall_rounds == PAGE_STATE_STALL_TERMINATE_AFTER and page_state_nudge_delivered:@@ -2582,6 +2770,31 @@ async def run_agent_tool_loop(elif page_state_stall_rounds >= PAGE_STATE_STALL_NUDGE_AFTER and not page_state_nudge_delivered:page_state_nudge_due = True+ if (+ outcome is not None+ # An acknowledged cancellation must not wait on a possibly-hung renderer just to+ # decide telemetry; and a batch whose pending fires are all generation-stale already+ # has nothing left to absorb for.+ and outcome.status != "canceled"+ and page_fingerprint is not None+ and batch_fp_before is not None+ and any(fire["gen"] == canonical.gen for fire in pending_canonical_fires)+ ):+ # A terminal outcome mid-batch (a finish, a fired completion probe) skips the detector+ # above, so the batch's own movement is unabsorbed here: take the after-sample now and+ # clear on a positive mismatch only, before deciding the pending events below.+ if batch_fp_after is None:+ batch_fp_after = await _sample_probe(page_fingerprint, deadline_at=deadline_at)+ if batch_fp_after is not None and batch_fp_after != batch_fp_before:+ canonical.progress()+ # A rung completed by — or followed in this batch by — absorbed progress describes a+ # progressing run, not a loop: only events whose generation survived every clear above+ # (same-call result data, invalid-fields baseline, batch fingerprint verdict) are emitted.+ for pending_fire in pending_canonical_fires:+ if pending_fire.pop("gen") == canonical.gen:+ LOG.info(CANONICAL_LOOP_EVENT, **pending_fire)+ pending_canonical_fires.clear()+# Warn only after the batch completes: a user message may not sit between an assistant# turn's tool results, and the model reads it with the snapshot that tripped it. Every note# due this turn (including auto-observe's own stall nudge, folded in above) shares ONE userdiff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 922d3b8f3..7b2ba0233 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -8717,3 +8717,606 @@ async def test_degrading_the_summary_dict_keeps_tool_choice() -> None:final_reasoning, final_tool_choice = caller.kwargs_per_call[-1]assert final_reasoning is Noneassert final_tool_choice == "required"+++def test_canonical_progress_tracker_counts_targets_and_clears_on_progress() -> None:+ from skyvern.forge.taskv3.loop import _CanonicalProgressTracker++ t = _CanonicalProgressTracker()+ assert t.record_touch("#code", True) == (1, 1)+ assert t.record_touch("#code", True) == (2, 2)+ assert t.record_touch("#other", False) == (1, 0)+ assert t.record_touch("#code", True) == (3, 3)+ assert t.looping_targets() == 0 # below the 4-touch rung+ assert t.record_touch("#code", False) == (4, 3)+ assert t.looping_targets() == 1+ t.progress()+ assert t.record_touch("#code", True) == (1, 1) # streak reset by progress+ assert t.looping_targets() == 0+++def _error_billable_tool(name: str, sink: list[tuple[str, dict[str, Any]]]) -> ToolSpec:+ async def handler(args: dict[str, Any]) -> ToolResult:+ sink.append((name, args))+ return ToolResult.error(f"{name} refused")++ return ToolSpec(+ name=name, description=name, parameters={"type": "object", "properties": {}}, handler=handler, billable=True+ )+++async def test_canonical_loop_event_fires_on_varying_args_same_target() -> None:+ # The class the incumbent action-loop key (tool+args) provably missed in prod: one selector+ # touched repeatedly with DIFFERENT args/tools, every touch refused, page unchanged. The+ # canonical tracker keys on the target and must emit its log-only event; the incumbent must NOT+ # have terminated (its exact-args streak never forms), which is the superset demonstration.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ script = [+ [("fill", {"selector": "#code", "value": "+44"})],+ [("fill", {"selector": "#code", "value": "United Kingdom"})],+ [("poke", {"selector": "#code"})],+ [("fill", {"selector": "#code", "value": "44"})],+ [("finish", {"status": "failed", "reason": "field kept refusing"})],+ ]+ tools = [_error_billable_tool("fill", touches), _error_billable_tool("poke", touches), make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ assert outcome.reason == "field kept refusing"+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 4]+ assert all(e["repeat_errors"] == e["repeat_count"] for e in fires)+++async def test_canonical_loop_event_silent_when_progress_intervenes() -> None:+ # The structural safety: a confirmed progress signal (here a page transition) clears the ring,+ # so the same four touches spread across real progress never read as a loop.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ clicks: list[tuple[str, dict[str, Any]]] = []+ script = [+ [("fill", {"selector": "#code", "value": "+44"})],+ [("fill", {"selector": "#code", "value": "United Kingdom"})],+ [("advance", {"selector": "#next"})],+ [("fill", {"selector": "#code", "value": "44"})],+ [("fill", {"selector": "#code", "value": "uk"})],+ [("finish", {"status": "completed"})],+ ]+ tools = [+ _error_billable_tool("fill", touches),+ _billable_tool("advance", clicks, data={"page_transitioned": True}),+ make_finish_tool(),+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "completed"+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_loop_clears_on_invalid_fields_new_low_not_on_stall_verdict() -> None:+ # The ledger's True return is the shadow STALL verdict; the canonical clear must key on the+ # ledger re-baselining (a new low) — real form progress between refused touches stays silent.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ seq = iter([5, 4, 3, 2, 1])++ async def observe_handler(args: dict[str, Any]) -> ToolResult:+ inv = next(seq)+ return ToolResult.ok(f"url=x inv={inv}", data={"summary": {"invalid_fields": inv}})++ observe_tool = ToolSpec(+ name="observe",+ description="observe",+ parameters={"type": "object", "properties": {}},+ handler=observe_handler,+ billable=False,+ compactable=True,+ )+ script = [+ [("observe", {})],+ [("fill", {"selector": "#code", "value": "a"})],+ [("observe", {})],+ [("fill", {"selector": "#code", "value": "b"})],+ [("observe", {})],+ [("fill", {"selector": "#code", "value": "c"})],+ [("observe", {})],+ [("fill", {"selector": "#code", "value": "d"})],+ [("finish", {"status": "completed"})],+ ]+ tools = [_error_billable_tool("fill", touches), observe_tool, make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "completed"+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_loop_event_keys_marks_like_selectors() -> None:+ # look-based actions carry mark=N, not selector — the same mark re-touched must accumulate as+ # one target, not collapse into a per-tool bucket with every other mark.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ script = [+ [("poke", {"mark": 7})],+ [("poke", {"mark": 7, "value": "x"})],+ [("poke", {"mark": 7, "value": "y"})],+ [("poke", {"mark": 7, "value": "z"})],+ [("finish", {"status": "failed", "reason": "mark kept refusing"})],+ ]+ tools = [_error_billable_tool("poke", touches), make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 4]+++async def test_canonical_loop_event_distinct_marks_are_distinct_targets() -> None:+ # The discriminating twin: four DIFFERENT marks are four targets — a per-tool bucket would+ # wrongly read them as one looping target.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ script = [+ [("poke", {"mark": 1})],+ [("poke", {"mark": 2})],+ [("poke", {"mark": 3})],+ [("poke", {"mark": 4})],+ [("finish", {"status": "failed", "reason": "distinct controls refused"})],+ ]+ tools = [_error_billable_tool("poke", touches), make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_loop_mark_keys_die_with_the_manifest_selector_keys_survive() -> None:+ # Each look renumbers marks from 1, so a mark=1 refused after every look is a DIFFERENT control+ # each time — no streak may form across manifest generations. The same-batch selector streak is+ # the discriminating pair: its identity outlives the renumbering and must still fire.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ looks: list[tuple[str, dict[str, Any]]] = []+ turn = [("fill", {"selector": "#code", "value": "x"}), ("poke", {"mark": 1}), ("look", {})]+ script = [list(turn) for _ in range(4)] + [[("finish", {"status": "failed", "reason": "kept refusing"})]]+ tools = [+ _error_billable_tool("fill", touches),+ _error_billable_tool("poke", touches),+ _look_tool(looks),+ make_finish_tool(),+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [(e["tool"], e["repeat_count"]) for e in fires] == [("fill", 3), ("fill", 4)]+++async def test_canonical_loop_event_suppressed_when_the_completing_touch_progresses() -> None:+ # Two refusals then a third touch that lands AND changes the page: the rung-3 predicate is+ # numerically satisfied at record time (2 errors >= 3-1), but the completing touch's own+ # progress must be absorbed before the verdict — a progressing run emits nothing.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ calls = {"n": 0}++ async def handler(args: dict[str, Any]) -> ToolResult:+ calls["n"] += 1+ if calls["n"] < 3:+ return ToolResult.error("fill refused")+ return ToolResult.ok("fill landed", data={"page_state_changed": True})++ fill = ToolSpec(+ name="fill", description="fill", parameters={"type": "object", "properties": {}}, handler=handler, billable=True+ )+ script = [+ [("fill", {"selector": "#code", "value": "a"})],+ [("fill", {"selector": "#code", "value": "b"})],+ [("fill", {"selector": "#code", "value": "ab"})],+ [("finish", {"status": "completed", "reason": "landed"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(script, [fill, make_finish_tool()], max_turns=50, max_tool_calls=100)+ assert outcome.status == "completed"+ assert calls["n"] == 3+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_loop_cleared_by_movement_landing_between_batches() -> None:+ # A delayed render lands after one batch's after-sample and before the next batch's+ # before-sample: the cross-batch fingerprint move is confirmed progress and must clear the+ # ring BEFORE the new batch's touches are read against the old ones.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ fp_calls = {"n": 0}++ async def between_batch_fingerprint() -> str:+ fp_calls["n"] += 1+ return f"dom-{(fp_calls['n'] - 1) // 2}"++ script = [[("fill", {"selector": "#code", "value": str(i)})] for i in range(4)]+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [_error_billable_tool("fill", touches), make_finish_tool()],+ page_fingerprint=between_batch_fingerprint,+ max_turns=50,+ max_tool_calls=100,+ )+ assert outcome.status == "failed"+ assert len(touches) == 4+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_loop_not_masked_by_replayed_download_notice() -> None:+ # A compactable tool replaying a retained download notice (download_notice without+ # download_new) is not fresh progress: it must not keep wiping the ring, or a post-download+ # loop could never accumulate enough touches to emit telemetry.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []++ async def replay_observe(args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("page\nDownloaded: report.pdf (1.0 MB)", data={"download_notice": True})++ observe = ToolSpec(+ name="observe",+ description="observe",+ parameters={"type": "object", "properties": {}},+ handler=replay_observe,+ billable=False,+ compactable=True,+ )+ script: list[list[tuple[str, dict[str, Any]]]] = []+ for i in range(4):+ script.append([("click", {"selector": "#dl", "note": str(i)})])+ script.append([("observe", {})])+ script.append([("finish", {"status": "failed", "reason": "kept refusing after the download"})])+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ assert len(touches) == 4+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 4]+++async def test_canonical_loop_suppressed_when_completion_probe_fires_on_the_rung_touch() -> None:+ # complete_on_download: the file can land while the probe waits, AFTER the tool result returned+ # without download_new. The probe's outcome skips the end-of-batch detector, but the pending+ # rung minted by that same touch must not be emitted — the run completed on real progress.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ calls = {"n": 0}++ async def handler(args: dict[str, Any]) -> ToolResult:+ calls["n"] += 1+ return ToolResult.error("click refused") if calls["n"] < 3 else ToolResult.ok("clicked")++ click = ToolSpec(+ name="click",+ description="click",+ parameters={"type": "object", "properties": {}},+ handler=handler,+ billable=True,+ )++ async def probe(_staged: frozenset[str]) -> str | None:+ return "a file finished downloading" if calls["n"] >= 3 else None++ script = [+ [("click", {"selector": "#dl", "note": "a"})],+ [("click", {"selector": "#dl", "note": "b"})],+ [("click", {"selector": "#dl", "note": "c"})],+ [("finish", {"status": "failed", "reason": "unreached"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(+ script, [click, make_finish_tool()], completion_probe=probe, max_turns=50, max_tool_calls=100+ )+ assert outcome.status == "completed"+ assert outcome.reason == "a file finished downloading"+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_loop_not_masked_by_missing_probe_samples() -> None:+ # Auto-observe deliberately treats a missing probe sample as "assume changed" for the model's+ # benefit, but that fallback is not evidence: intermittent probe timeouts must not keep+ # clearing the ring and permanently mask a genuine loop.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []++ async def dead_probe() -> str | None:+ return None++ async def observe_handler(_args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("url=x page digest (1 interactive elements)")++ observe = ToolSpec(+ name="observe",+ description="observe",+ parameters={"type": "object", "properties": {}},+ handler=observe_handler,+ compactable=True,+ )+ script = [[("click", {"selector": "#dl", "note": str(i)})] for i in range(4)]+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(+ script, tools, auto_observe=True, page_probe=dead_probe, max_turns=50, max_tool_calls=100+ )+ assert outcome.status == "failed"+ assert len(touches) == 4+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 4]+++async def test_canonical_loop_absorbs_movement_in_a_terminal_batch_before_emitting() -> None:+ # Two refusals, then a batch whose successful third touch changes the DOM (fingerprint-only)+ # and whose finish completes the run: the terminal outcome skips the detector, but the batch's+ # own movement must still be absorbed before the pending rung is decided.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ calls = {"n": 0}++ async def handler(args: dict[str, Any]) -> ToolResult:+ calls["n"] += 1+ return ToolResult.error("click refused") if calls["n"] < 3 else ToolResult.ok("clicked")++ click = ToolSpec(+ name="click",+ description="click",+ parameters={"type": "object", "properties": {}},+ handler=handler,+ billable=True,+ )+ fp_calls = {"n": 0}++ async def fingerprint() -> str:+ # Moves only at the terminal batch's after-sample (calls 1-5 are the three before-samples+ # and the first two batches' after-samples).+ fp_calls["n"] += 1+ return "dom-0" if fp_calls["n"] <= 5 else "dom-1"++ script = [+ [("click", {"selector": "#dl", "note": "a"})],+ [("click", {"selector": "#dl", "note": "b"})],+ [("click", {"selector": "#dl", "note": "c"}), ("finish", {"status": "completed", "reason": "done"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(+ script, [click, make_finish_tool()], page_fingerprint=fingerprint, max_turns=50, max_tool_calls=100+ )+ assert outcome.status == "completed"+ assert calls["n"] == 3+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canceled_run_emits_pending_fire_without_a_terminal_probe() -> None:+ # An acknowledged cancellation must not wait on the terminal-batch reconciliation sample (a+ # hung renderer can hold that probe for its full timeout); the pending rung is emitted as+ # minted, since no progress evidence contradicts it.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ fp_calls = {"n": 0}++ async def fingerprint() -> str:+ fp_calls["n"] += 1+ return "dom-0"++ async def should_cancel() -> bool:+ return len(touches) >= 3++ script = [+ [("click", {"selector": "#dl", "note": "a"})],+ [("click", {"selector": "#dl", "note": "b"})],+ [("click", {"selector": "#dl", "note": "c"}), ("click", {"selector": "#dl", "note": "d"})],+ ]+ with capture_logs() as logs:+ outcome, _ = await _run(+ script,+ [_error_billable_tool("click", touches), make_finish_tool()],+ page_fingerprint=fingerprint,+ should_cancel=should_cancel,+ max_turns=50,+ max_tool_calls=100,+ )+ assert outcome.status == "canceled"+ assert len(touches) == 3 # the fourth call was refused by the cancellation check+ # 2 samples per completed batch plus the final batch's before-sample; NO terminal probe.+ assert fp_calls["n"] == 5+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3]+++async def test_canonical_loop_cleared_by_changed_perception_digest_without_fingerprint() -> None:+ # With no page_fingerprint, a repeated observe whose digest changes is the only movement+ # evidence there is; two landed digests that differ must clear the ring like a fingerprint+ # mismatch would, so touches against superseded pages never alias into a rung.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ pages = {"n": 0}++ async def observe_handler(_args: dict[str, Any]) -> ToolResult:+ pages["n"] += 1+ return ToolResult.ok(f"url=x page {pages['n']} content (1 interactive elements)")++ observe = ToolSpec(+ name="observe",+ description="observe",+ parameters={"type": "object", "properties": {}},+ handler=observe_handler,+ compactable=True,+ )+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]+ for i in range(4):+ script.append([("click", {"selector": "#dl", "note": str(i)})])+ script.append([("observe", {})])+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ assert len(touches) == 4+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_canonical_rung_fires_once_even_when_the_window_parks_on_it() -> None:+ # Eight failed touches on A, two on B, then more on A: eviction keeps A's in-window count+ # parked at 8, which must not re-emit rung 8 on every subsequent touch — a rung is one+ # threshold crossing per generation.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ script = [[("click", {"selector": "#a", "note": str(i)})] for i in range(8)]+ script += [[("click", {"selector": "#b", "note": str(i)})] for i in range(2)]+ script += [[("click", {"selector": "#a", "note": f"again-{i}"})] for i in range(3)]+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])+ tools = [_error_billable_tool("click", touches), make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=60, max_tool_calls=200)+ assert outcome.status == "failed"+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 4, 6, 8]+++async def test_canonical_loop_clears_on_new_low_even_under_a_replayed_notice() -> None:+ # Blocker 1 (AronPerez round 6/10): a replayed download notice hard-progresses the shadow+ # ledger, nulling invalid_baseline BEFORE the shadow read on the same result — which left the+ # new-low clear dead for the rest of the page. Constant observe content keeps the perception+ # digest from masking the probe. A form ratcheting to a new low on every look is progressing;+ # the refusing selector must emit nothing.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ seq = iter([6, 5, 4, 3, 2])++ async def observe_handler(_args: dict[str, Any]) -> ToolResult:+ return ToolResult.ok("url=x form", data={"download_notice": True, "summary": {"invalid_fields": next(seq)}})++ observe = ToolSpec(+ name="observe",+ description="observe",+ parameters={"type": "object", "properties": {}},+ handler=observe_handler,+ compactable=True,+ )+ script: list[list[tuple[str, dict[str, Any]]]] = []+ for i in range(4):+ script.append([("observe", {})])+ script.append([("click", {"selector": "#stuck", "note": str(i)})])+ script.append([("observe", {})])+ script.append([("finish", {"status": "failed", "reason": "one field kept refusing"})])+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ assert len(touches) == 4+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++async def test_completion_probe_spares_a_sibling_targets_rung() -> None:+ # Blocker 2 (AronPerez round 10): the probe firing is progress for the COMPLETING touch only.+ # A five-error streak on one selector minted a genuine rung 6 the same batch a download on a+ # DIFFERENT selector completed the run — that sibling rung is real data and must survive.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ fetched = {"done": False}++ async def fetch_handler(_args: dict[str, Any]) -> ToolResult:+ fetched["done"] = True+ return ToolResult.ok("fetch dispatched")++ fetch = ToolSpec(+ name="fetch",+ description="fetch",+ parameters={"type": "object", "properties": {}},+ handler=fetch_handler,+ billable=True,+ )++ async def probe(_staged: frozenset[str]) -> str | None:+ return "a file finished downloading" if fetched["done"] else None++ script = [[("click", {"selector": "#stuck", "note": str(i)})] for i in range(5)]+ script.append([("click", {"selector": "#stuck", "note": "sixth"}), ("fetch", {"selector": "#other"})])+ tools = [_error_billable_tool("click", touches), fetch, make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, completion_probe=probe, max_turns=50, max_tool_calls=100)+ assert outcome.status == "completed"+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 4, 6]+++async def test_canonical_rung_rearms_after_full_eviction() -> None:+ # A rung re-arms once the target's in-window count dips below it: a fresh streak after full+ # eviction is a genuine re-crossing, unlike the parked-window case the once-per-generation+ # guard exists for.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ script = [[("click", {"selector": "#a", "note": str(i)})] for i in range(3)]+ script += [[("click", {"selector": "#b", "note": str(i)})] for i in range(10)]+ script += [[("click", {"selector": "#a", "note": f"again-{i}"})] for i in range(3)]+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])+ tools = [_error_billable_tool("click", touches), make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, max_turns=60, max_tool_calls=200)+ assert outcome.status == "failed"+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]+ assert [e["repeat_count"] for e in fires] == [3, 3, 4, 6, 8, 3]+++async def test_canonical_loop_cleared_by_positive_probe_mismatch_on_a_failed_call() -> None:+ # A failed call that moved the document (two landed identity samples that differ) is progress+ # the fingerprint can miss entirely (a same-template step renders identically): the poisoned+ # batch stop already knows the page moved, and the ring must learn it too.+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT++ touches: list[tuple[str, dict[str, Any]]] = []+ probe_calls = {"n": 0}++ async def moving_probe() -> str:+ probe_calls["n"] += 1+ return f"doc-{probe_calls['n']}"++ script = [[("click", {"selector": "#stuck", "note": str(i)})] for i in range(4)]+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])+ tools = [_error_billable_tool("click", touches), make_finish_tool()]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, page_probe=moving_probe, max_turns=50, max_tool_calls=100)+ assert outcome.status == "failed"+ assert len(touches) == 4+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []