Skip to content

Automate browser based workflows with AI

Task V3: canonical progress tracker, Phase A log-only (SKY-15379) (#8385)

pedrohsdb84e8dfbd21b9
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index bbcdc63c6..601c17dfa 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -427,6 +427,21 @@ class _PerceptionLedger:
PROGRESS_LEDGER_WINDOW = 8
# Facetable event names; the offline precision/survival metrics key on these — change only with the
# dashboards that read them.
+# Canonical progress signal, Phase A (SKY-15379): ONE target-churn + compound-progress tracker,
+# LOG-ONLY. The loop answers "is this run progressing?" five separate ways today; this tracker is
+# the candidate replacement signal, emitting the parity/precision data any consolidation needs
+# while changing no behavior. Its target key is the canonicalized SELECTOR, not tool+args — the
+# incumbent action-loop key provably missed prod repeat-loops whose args varied on a fixed target.
+CANONICAL_LOOP_EVENT = "taskv3 canonical progress loop detected"
+CANONICAL_EXTEND_DELTA_EVENT = "taskv3 canonical progress extension delta"
+# Log-only thresholds; the Phase-B intervention thresholds are tuned FROM this data, so the fire
+# points are logged at every rung of the distribution rather than one cliff.
+CANONICAL_LOOP_WINDOW = 10
+CANONICAL_LOOP_FIRE_COUNTS = (3, 4, 6, 8)
+# The rung that counts as "looping" for the extension-delta read — named so reordering the logging
+# rungs above can never silently move the threshold Phase B tunes against.
+CANONICAL_LOOP_TOUCHES = 4
+
PROGRESS_LEDGER_SHADOW_EVENT = "taskv3 loop progress ledger would fail-fast"
PROGRESS_LEDGER_FINAL_EVENT = "taskv3 loop progress ledger final"
@@ -917,6 +932,62 @@ _TOOL_CALL_RECORD_FIELDS = frozenset(
)
+@dataclass
+class _CanonicalProgressTracker:
+ """Touches on canonicalized targets since the last compound-progress event (SKY-15379, Phase A).
+
+ A touch is one billable tool dispatch, keyed by its selector (tool name when selector-less).
+ Progress — any of the ledger's hard-progress events, an invalid-fields new-low, or a confirmed
+ page change — clears the ring, so a repeat streak can only accumulate across a genuinely
+ unchanged situation; a progressing run is untrippable by construction. Log-only: the counts
+ feed telemetry, nothing reads them for control flow.
+ """
+
+ window: int = CANONICAL_LOOP_WINDOW
+ touches: list[tuple[str, bool]] = field(default_factory=list)
+ # Bumped on every clear: a pending loop event minted under an older generation was completed
+ # by (or followed by) absorbed progress and must not be emitted.
+ gen: int = 0
+ # Rungs already fired this generation. Once the window saturates, same-target counts can PARK
+ # on a fire rung while churn evicts other keys — a rung is one threshold crossing, not one
+ # event per touch spent at it.
+ fired: set[tuple[str, int]] = field(default_factory=set)
+
+ def record_touch(self, target_key: str, is_error: bool) -> tuple[int, int]:
+ """Record one billable touch; returns (same-target touches, same-target errors) in-window."""
+ self.touches.append((target_key, is_error))
+ if len(self.touches) > self.window:
+ self.touches.pop(0)
+ same = [err for key, err in self.touches if key == target_key]
+ # A rung re-arms once the in-window count dips below it: a fresh streak after full eviction
+ # is a genuine re-crossing, distinct from a saturated window PARKED on a fire count.
+ self.fired = {entry for entry in self.fired if entry[0] != target_key or entry[1] <= len(same)}
+ return len(same), sum(1 for err in same if err)
+
+ def progress(self) -> None:
+ self.touches.clear()
+ self.fired.clear()
+ self.gen += 1
+
+ def invalidate_marks(self) -> None:
+ """A look renumbered the manifest, so every mark=N key now names an arbitrary element: drop
+ them rather than let four different controls alias into one false streak. A surviving mark
+ streak is therefore always within a single manifest generation. Selector keys stay — their
+ identity outlives the renumbering."""
+ self.touches = [touch for touch in self.touches if not touch[0].startswith("mark=")]
+ self.fired = {entry for entry in self.fired if not entry[0].startswith("mark=")}
+
+ def looping_targets(self) -> int:
+ counts: dict[str, list[bool]] = {}
+ for key, err in self.touches:
+ counts.setdefault(key, []).append(err)
+ return sum(
+ 1
+ for errs in counts.values()
+ if len(errs) >= CANONICAL_LOOP_TOUCHES and sum(errs) >= CANONICAL_LOOP_TOUCHES - 1
+ )
+
+
def _observe_summary_fields(result: ToolResult) -> dict[str, int]:
"""Counts only: the summary is built by the tool, but an indexed field is re-checked here."""
summary = (result.data or {}).get("summary")
@@ -1358,6 +1429,7 @@ async def run_agent_tool_loop(
auto_perception = _PerceptionLedger()
# Net-progress ledger (additive shadow); None disables it, mirroring the guard's *_after knobs.
progress = _ProgressLedger(window=progress_window) if progress_window is not None else None
+ canonical = _CanonicalProgressTracker()
# The action-loop counter: (repeat count, first turn of the streak) per billable action
# identity, cleared whenever evidence of page change arrives. action_warned holds the streaks
# whose warning was actually DELIVERED — termination is gated on it, so the model always gets
@@ -1434,19 +1506,31 @@ async def run_agent_tool_loop(
stall_nudges_due = []
if progress is not None:
progress.hard_progress()
+ canonical.progress()
if submit_watch is not None:
submit_watch.clear()
_append_skipped_tool_results(messages, remaining, "the page was refreshed")
refresh_nudge_due = True
return True
- def _progress_observe_shadow(observe_summary: dict[str, int], tool_name: str, attribution: dict[str, Any]) -> None:
+ def _progress_observe_shadow(
+ observe_summary: dict[str, int], tool_name: str, attribution: dict[str, Any], baseline_before: int | None
+ ) -> None:
"""Shared by the model-dispatched and auto-observe paths so an auto-observe feeds the same
ledger a model-issued no-arg observe would."""
if progress is None or not observe_summary:
return
invalid_fields = observe_summary.get("invalid_fields")
- if invalid_fields is not None and progress.observe(invalid_fields):
+ # The ledger's True return is the SHADOW STALL verdict, not progress — the canonical clear
+ # keys on the ledger re-baselining (a new low, or a rise re-baseline: fresh context), which
+ # is visible as its baseline moving under an already-set baseline. The baseline is captured
+ # BEFORE _absorb_result_data runs: a replayed download notice hard-progresses the shadow
+ # ledger (nulling the baseline) on the same result whose summary carries the new low, and
+ # reading the post-absorb value would leave this clear dead for the rest of the page.
+ stalled = invalid_fields is not None and progress.observe(invalid_fields)
+ if baseline_before is not None and progress.invalid_baseline != baseline_before:
+ canonical.progress()
+ if stalled:
LOG.info(
PROGRESS_LEDGER_SHADOW_EVENT,
actions=progress.actions_since_progress,
@@ -1494,6 +1578,16 @@ async def run_agent_tool_loop(
activity.perception_stall_imminent = False
if progress is not None:
progress.hard_progress()
+ if (
+ result_data.get("download_new")
+ or result_data.get("page_state_changed")
+ or result_data.get("same_url_reload")
+ or result_data.get("nav_revisit")
+ ):
+ # A REPLAYED notice (download_notice without download_new) re-clears the retry
+ # ledger but is not fresh progress: it must not keep wiping the canonical ring, or
+ # a post-download loop could never accumulate enough touches to emit telemetry.
+ canonical.progress()
# A click that moved the URL is a real page transition (H1 hard progress) for the shadow
# ledger, but URL equality does NOT prove same-page (a URL-stable SPA form advance) — so only
# the positive direction is acted on, and kept OUT of the branch above so it never clears the
@@ -1503,6 +1597,7 @@ async def run_agent_tool_loop(
if result_data.get("page_transitioned") is True:
if progress is not None:
progress.hard_progress()
+ canonical.progress()
return tool_name == "look" and bool(result_data.get("marks_renumbered"))
async def _completion_probe_outcome(
@@ -1527,6 +1622,9 @@ async def run_agent_tool_loop(
if not completion_reason:
return None
LOG.info("taskv3 loop completion probe fired", tool=tool_name, turn=turns)
+ # No tracker-wide clear here: the probe firing is download progress for the COMPLETING
+ # touch only, and its call site drops that one target's pending rungs — a whole-generation
+ # bump would erase a sibling target's true-positive rung along with it.
return LoopOutcome("completed", completion_reason)
def _perception_stall_check(
@@ -1558,6 +1656,9 @@ async def run_agent_tool_loop(
# terminate verdict, since it fires far more often than the model calls observe on its own
# and would trip those far too eagerly for a path the model never asked to take.
_clear_action_state()
+ # Two landed digests that differ are positive evidence, exactly like a fingerprint
+ # mismatch — and the only movement evidence there is when page_fingerprint is absent.
+ canonical.progress()
# Evidence requires NEW content: a URL-only flip (history.pushState) still clears the
# repeat guards above but earns no budget, and neither does a return to a content state
# in the probe's recent ring (a panel toggling open and shut).
@@ -1814,6 +1915,9 @@ async def run_agent_tool_loop(
failed_selectors: set[str] = set()
batch_had_failure = False
marks_stale = False
+ # Loop events minted this batch, emitted only after every progress signal the batch can
+ # produce has been absorbed (see the end-of-batch emission below).
+ pending_canonical_fires: list[dict[str, Any]] = []
# Auto-observe signals, gathered across the whole batch so the end-of-batch check reads the
# batch's net effect rather than any one call's.
batch_observed_ok = False
@@ -1839,10 +1943,19 @@ async def run_agent_tool_loop(
# before/after fingerprint pair for every billable batch.
if page_fingerprint is not None and batch_has_billable_call and not batch_cancelled:
batch_fp_before = await _sample_probe(page_fingerprint, deadline_at=deadline_at)
+ if page_state_prev_fp is not None and batch_fp_before is not None and batch_fp_before != page_state_prev_fp:
+ # The page moved BETWEEN batches (a delayed render landing after the prior
+ # after-sample): the touches the old samples described are stale, and this batch's
+ # dispatch and extension decisions must not read them. Canonical-only — the
+ # incumbent stall counters keep their end-of-batch turn_did_action gate.
+ canonical.progress()
batch_fp_after: str | None = None
# The auto-observe path's FINAL page-changed verdict (resample included) when it ran; the
# stall detector prefers this over re-comparing raw samples so the two can never disagree.
batch_auto_page_changed: bool | None = None
+ # Whether that verdict rested on POSITIVE evidence (a flag or two landed samples that
+ # differ) rather than the missing-sample "assume changed" fallback.
+ batch_auto_page_change_evidence: bool = False
# The batch's carrier for an auto-observe digest: the LAST tool message appended for an
# EXECUTED (dispatched) call whose spec is not compactable -- i.e. a real action result, never
# a skip stub and never a compactable perception dump that would otherwise elide the digest's
@@ -1955,6 +2068,18 @@ async def run_agent_tool_loop(
extension,
seconds_per_step=(time.monotonic() - started_at) / max(action_steps, 1),
)
+ # This read cannot see unabsorbed in-batch movement: action_steps is frozen during
+ # a batch, so the cap check trips on the batch's FIRST billable call — before any
+ # page action dispatches — and between-batch movement was absorbed when
+ # batch_fp_before was sampled.
+ looping_now = canonical.looping_targets()
+ LOG.info(
+ CANONICAL_EXTEND_DELTA_EVENT,
+ current_decision=allowed,
+ gate_reason=gate_reason,
+ canonical_looping_targets=looping_now,
+ would_block_extension=bool(allowed and looping_now),
+ )
if allowed:
original_cap = max_action_steps
budget_extension_granted = True
@@ -2069,6 +2194,27 @@ async def run_agent_tool_loop(
**observe_summary,
**attribution,
)
+ if spec is not None and spec.billable:
+ target_key = _call_selector(args) or f"tool:{tool_name}"
+ same_count, same_errors = canonical.record_touch(target_key, result.status == "error")
+ # Every rung of the distribution is logged (not one cliff): the Phase-B block
+ # threshold is tuned from this data.
+ if (
+ same_count in CANONICAL_LOOP_FIRE_COUNTS
+ and same_errors >= same_count - 1
+ and (target_key, same_count) not in canonical.fired
+ ):
+ canonical.fired.add((target_key, same_count))
+ pending_canonical_fires.append(
+ {
+ "gen": canonical.gen,
+ "target_key_hash": telemetry_hash(telemetry_salt, target_key),
+ "tool": tool_name,
+ "repeat_count": same_count,
+ "repeat_errors": same_errors,
+ "turn": turns,
+ }
+ )
if spec is not None and spec.compactable and result.status == "ok":
snapshot_indices.add(len(messages)) # index this successful snapshot will occupy, pre-append
@@ -2123,12 +2269,14 @@ async def run_agent_tool_loop(
batch_page_change_reason = "navigate"
elif batch_page_change_reason is None and result_data.get("page_transitioned") is True:
batch_page_change_reason = "page_transitioned"
+ shadow_baseline_before = progress.invalid_baseline if progress is not None else None
if _absorb_result_data(tool_name, spec, result_data):
# Every mark=N still queued in this batch was chosen before this look renumbered the
# marks, so it now names an arbitrary element; a look refused before rebuilding its
# manifest leaves the old marks (and their failed keys) live.
marks_stale = True
- _progress_observe_shadow(observe_summary, tool_name, attribution)
+ canonical.invalidate_marks()
+ _progress_observe_shadow(observe_summary, tool_name, attribution, shadow_baseline_before)
if content_digest is not None:
stall_outcome, nudges_due = _perception_stall_check(
perception,
@@ -2222,6 +2370,13 @@ async def run_agent_tool_loop(
completion_outcome = await _completion_probe_outcome(tool_name, spec, result_data)
if completion_outcome is not None:
outcome = completion_outcome
+ if spec is not None and spec.billable:
+ # The download landed while this call's probe waited, so this call's touch is
+ # the progressing one: drop ITS pending rungs only, never a sibling target's.
+ completing_hash = telemetry_hash(telemetry_salt, _call_selector(args) or f"tool:{tool_name}")
+ pending_canonical_fires[:] = [
+ fire for fire in pending_canonical_fires if fire["target_key_hash"] != completing_hash
+ ]
_append_skipped_tool_results(messages, tool_calls[idx + 1 :], "completion probe fired")
break
@@ -2261,6 +2416,11 @@ async def run_agent_tool_loop(
if not poisoned and spec is not None and page_probe is not None:
probe_after = await _sample_probe(page_probe, deadline_at=deadline_at)
poisoned = probe_before is None or probe_after is None or probe_after != probe_before
+ if probe_before is not None and probe_after is not None and probe_after != probe_before:
+ # Two landed identity samples that differ: the failed call still moved the
+ # document, and the fingerprint may render identically on the new one (a
+ # same-template step) — absorb it here or the rung survives that blindness.
+ canonical.progress()
if poisoned:
_append_skipped_tool_results(
messages,
@@ -2324,8 +2484,13 @@ async def run_agent_tool_loop(
page_changed: bool
reason: str
signal: str
+ # Distinguishes a POSITIVELY observed change (a flag, or two landed samples that
+ # differ) from the missing-sample fallback below, which is a model-facing "assume
+ # changed" but is not evidence and must not clear the canonical tracker.
+ page_change_evidence: bool
if batch_page_change_reason is not None:
page_changed, reason, signal = True, batch_page_change_reason, "flag"
+ page_change_evidence = True
elif page_fingerprint is not None:
# The fingerprint samples rendered content (innerHTML), so it catches an in-page
# mutation (a dropdown, a validation error, a revealed section) the document-identity
@@ -2335,26 +2500,33 @@ async def run_agent_tool_loop(
batch_fp_after = fp_after
if batch_fp_before is not None and fp_after is not None:
page_changed = fp_after != batch_fp_before
+ page_change_evidence = page_changed
reason = "fingerprint_mismatch" if page_changed else "unchanged"
signal = "fingerprint"
elif page_probe is not None:
probe_after = await _sample_probe(page_probe, deadline_at=deadline_at)
if batch_probe_before is None or probe_after is None or probe_after != batch_probe_before:
page_changed, reason = True, "probe_mismatch"
+ page_change_evidence = batch_probe_before is not None and probe_after is not None
else:
page_changed, reason = False, "unchanged"
+ page_change_evidence = False
signal = "probe"
else:
page_changed, reason, signal = False, "unchanged", "none"
+ page_change_evidence = False
elif page_probe is not None:
probe_after = await _sample_probe(page_probe, deadline_at=deadline_at)
if batch_probe_before is None or probe_after is None or probe_after != batch_probe_before:
page_changed, reason = True, "probe_mismatch"
+ page_change_evidence = batch_probe_before is not None and probe_after is not None
else:
page_changed, reason = False, "unchanged"
+ page_change_evidence = False
signal = "probe"
else:
page_changed, reason, signal = False, "unchanged", "none"
+ page_change_evidence = False
if not page_changed and signal in ("fingerprint", "probe"):
# The immediate comparison above can be too early for an async render; give it one
@@ -2368,9 +2540,13 @@ async def run_agent_tool_loop(
wait_seconds += resample_wait
if resampled_changed:
page_changed = True
+ page_change_evidence = True
reason = "fingerprint_mismatch" if signal == "fingerprint" else "probe_mismatch"
batch_auto_page_changed = page_changed
+ batch_auto_page_change_evidence = page_change_evidence
+ if page_changed and page_change_evidence:
+ canonical.progress()
if not page_changed:
messages[batch_carrier_idx]["content"] = (
str(messages[batch_carrier_idx]["content"]) + "\n\n[no markup change detected after this batch]"
@@ -2466,6 +2642,7 @@ async def run_agent_tool_loop(
# progress, and completion) lands identically whichever path produced it —
# independent of whether the observe also produced a digest below.
ao_result_data = ao_result.data or {}
+ ao_baseline_before = progress.invalid_baseline if progress is not None else None
_absorb_result_data("observe", observe_spec, ao_result_data)
completion_outcome = await _completion_probe_outcome(
"observe", observe_spec, ao_result_data
@@ -2498,7 +2675,9 @@ async def run_agent_tool_loop(
"snapshot_digest": telemetry_hash(telemetry_salt, ao_digest),
"probe_first_time": auto_perception.first_time(ao_action_key),
}
- _progress_observe_shadow(_observe_summary_fields(ao_result), "observe", ao_attribution)
+ _progress_observe_shadow(
+ _observe_summary_fields(ao_result), "observe", ao_attribution, ao_baseline_before
+ )
stall_outcome, ao_nudges_result = _perception_stall_check(
auto_perception,
ao_digest,
@@ -2570,6 +2749,15 @@ async def run_agent_tool_loop(
if page_state_changed is True:
page_state_stall_rounds = 0
page_state_nudge_delivered = False
+ # The auto verdict's missing-sample fallback ("assume changed") re-baselines the
+ # incumbent counters above but is not positive evidence: only a flagged or
+ # sample-confirmed change may clear the canonical tracker.
+ if not (
+ batch_page_change_reason is None
+ and batch_auto_page_changed is True
+ and not batch_auto_page_change_evidence
+ ):
+ canonical.progress()
elif page_state_changed is False:
page_state_stall_rounds += 1
if page_state_stall_rounds == PAGE_STATE_STALL_TERMINATE_AFTER and page_state_nudge_delivered:
@@ -2582,6 +2770,31 @@ async def run_agent_tool_loop(
elif page_state_stall_rounds >= PAGE_STATE_STALL_NUDGE_AFTER and not page_state_nudge_delivered:
page_state_nudge_due = True
+ if (
+ outcome is not None
+ # An acknowledged cancellation must not wait on a possibly-hung renderer just to
+ # decide telemetry; and a batch whose pending fires are all generation-stale already
+ # has nothing left to absorb for.
+ and outcome.status != "canceled"
+ and page_fingerprint is not None
+ and batch_fp_before is not None
+ and any(fire["gen"] == canonical.gen for fire in pending_canonical_fires)
+ ):
+ # A terminal outcome mid-batch (a finish, a fired completion probe) skips the detector
+ # above, so the batch's own movement is unabsorbed here: take the after-sample now and
+ # clear on a positive mismatch only, before deciding the pending events below.
+ if batch_fp_after is None:
+ batch_fp_after = await _sample_probe(page_fingerprint, deadline_at=deadline_at)
+ if batch_fp_after is not None and batch_fp_after != batch_fp_before:
+ canonical.progress()
+ # A rung completed by — or followed in this batch by — absorbed progress describes a
+ # progressing run, not a loop: only events whose generation survived every clear above
+ # (same-call result data, invalid-fields baseline, batch fingerprint verdict) are emitted.
+ for pending_fire in pending_canonical_fires:
+ if pending_fire.pop("gen") == canonical.gen:
+ LOG.info(CANONICAL_LOOP_EVENT, **pending_fire)
+ pending_canonical_fires.clear()
+
# Warn only after the batch completes: a user message may not sit between an assistant
# turn's tool results, and the model reads it with the snapshot that tripped it. Every note
# due this turn (including auto-observe's own stall nudge, folded in above) shares ONE user
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 922d3b8f3..7b2ba0233 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -8717,3 +8717,606 @@ async def test_degrading_the_summary_dict_keeps_tool_choice() -> None:
final_reasoning, final_tool_choice = caller.kwargs_per_call[-1]
assert final_reasoning is None
assert final_tool_choice == "required"
+
+
+def test_canonical_progress_tracker_counts_targets_and_clears_on_progress() -> None:
+ from skyvern.forge.taskv3.loop import _CanonicalProgressTracker
+
+ t = _CanonicalProgressTracker()
+ assert t.record_touch("#code", True) == (1, 1)
+ assert t.record_touch("#code", True) == (2, 2)
+ assert t.record_touch("#other", False) == (1, 0)
+ assert t.record_touch("#code", True) == (3, 3)
+ assert t.looping_targets() == 0 # below the 4-touch rung
+ assert t.record_touch("#code", False) == (4, 3)
+ assert t.looping_targets() == 1
+ t.progress()
+ assert t.record_touch("#code", True) == (1, 1) # streak reset by progress
+ assert t.looping_targets() == 0
+
+
+def _error_billable_tool(name: str, sink: list[tuple[str, dict[str, Any]]]) -> ToolSpec:
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ sink.append((name, args))
+ return ToolResult.error(f"{name} refused")
+
+ return ToolSpec(
+ name=name, description=name, parameters={"type": "object", "properties": {}}, handler=handler, billable=True
+ )
+
+
+async def test_canonical_loop_event_fires_on_varying_args_same_target() -> None:
+ # The class the incumbent action-loop key (tool+args) provably missed in prod: one selector
+ # touched repeatedly with DIFFERENT args/tools, every touch refused, page unchanged. The
+ # canonical tracker keys on the target and must emit its log-only event; the incumbent must NOT
+ # have terminated (its exact-args streak never forms), which is the superset demonstration.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ script = [
+ [("fill", {"selector": "#code", "value": "+44"})],
+ [("fill", {"selector": "#code", "value": "United Kingdom"})],
+ [("poke", {"selector": "#code"})],
+ [("fill", {"selector": "#code", "value": "44"})],
+ [("finish", {"status": "failed", "reason": "field kept refusing"})],
+ ]
+ tools = [_error_billable_tool("fill", touches), _error_billable_tool("poke", touches), make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ assert outcome.reason == "field kept refusing"
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 4]
+ assert all(e["repeat_errors"] == e["repeat_count"] for e in fires)
+
+
+async def test_canonical_loop_event_silent_when_progress_intervenes() -> None:
+ # The structural safety: a confirmed progress signal (here a page transition) clears the ring,
+ # so the same four touches spread across real progress never read as a loop.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script = [
+ [("fill", {"selector": "#code", "value": "+44"})],
+ [("fill", {"selector": "#code", "value": "United Kingdom"})],
+ [("advance", {"selector": "#next"})],
+ [("fill", {"selector": "#code", "value": "44"})],
+ [("fill", {"selector": "#code", "value": "uk"})],
+ [("finish", {"status": "completed"})],
+ ]
+ tools = [
+ _error_billable_tool("fill", touches),
+ _billable_tool("advance", clicks, data={"page_transitioned": True}),
+ make_finish_tool(),
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "completed"
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_loop_clears_on_invalid_fields_new_low_not_on_stall_verdict() -> None:
+ # The ledger's True return is the shadow STALL verdict; the canonical clear must key on the
+ # ledger re-baselining (a new low) — real form progress between refused touches stays silent.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ seq = iter([5, 4, 3, 2, 1])
+
+ async def observe_handler(args: dict[str, Any]) -> ToolResult:
+ inv = next(seq)
+ return ToolResult.ok(f"url=x inv={inv}", data={"summary": {"invalid_fields": inv}})
+
+ observe_tool = ToolSpec(
+ name="observe",
+ description="observe",
+ parameters={"type": "object", "properties": {}},
+ handler=observe_handler,
+ billable=False,
+ compactable=True,
+ )
+ script = [
+ [("observe", {})],
+ [("fill", {"selector": "#code", "value": "a"})],
+ [("observe", {})],
+ [("fill", {"selector": "#code", "value": "b"})],
+ [("observe", {})],
+ [("fill", {"selector": "#code", "value": "c"})],
+ [("observe", {})],
+ [("fill", {"selector": "#code", "value": "d"})],
+ [("finish", {"status": "completed"})],
+ ]
+ tools = [_error_billable_tool("fill", touches), observe_tool, make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "completed"
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_loop_event_keys_marks_like_selectors() -> None:
+ # look-based actions carry mark=N, not selector — the same mark re-touched must accumulate as
+ # one target, not collapse into a per-tool bucket with every other mark.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ script = [
+ [("poke", {"mark": 7})],
+ [("poke", {"mark": 7, "value": "x"})],
+ [("poke", {"mark": 7, "value": "y"})],
+ [("poke", {"mark": 7, "value": "z"})],
+ [("finish", {"status": "failed", "reason": "mark kept refusing"})],
+ ]
+ tools = [_error_billable_tool("poke", touches), make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 4]
+
+
+async def test_canonical_loop_event_distinct_marks_are_distinct_targets() -> None:
+ # The discriminating twin: four DIFFERENT marks are four targets — a per-tool bucket would
+ # wrongly read them as one looping target.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ script = [
+ [("poke", {"mark": 1})],
+ [("poke", {"mark": 2})],
+ [("poke", {"mark": 3})],
+ [("poke", {"mark": 4})],
+ [("finish", {"status": "failed", "reason": "distinct controls refused"})],
+ ]
+ tools = [_error_billable_tool("poke", touches), make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_loop_mark_keys_die_with_the_manifest_selector_keys_survive() -> None:
+ # Each look renumbers marks from 1, so a mark=1 refused after every look is a DIFFERENT control
+ # each time — no streak may form across manifest generations. The same-batch selector streak is
+ # the discriminating pair: its identity outlives the renumbering and must still fire.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ looks: list[tuple[str, dict[str, Any]]] = []
+ turn = [("fill", {"selector": "#code", "value": "x"}), ("poke", {"mark": 1}), ("look", {})]
+ script = [list(turn) for _ in range(4)] + [[("finish", {"status": "failed", "reason": "kept refusing"})]]
+ tools = [
+ _error_billable_tool("fill", touches),
+ _error_billable_tool("poke", touches),
+ _look_tool(looks),
+ make_finish_tool(),
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [(e["tool"], e["repeat_count"]) for e in fires] == [("fill", 3), ("fill", 4)]
+
+
+async def test_canonical_loop_event_suppressed_when_the_completing_touch_progresses() -> None:
+ # Two refusals then a third touch that lands AND changes the page: the rung-3 predicate is
+ # numerically satisfied at record time (2 errors >= 3-1), but the completing touch's own
+ # progress must be absorbed before the verdict — a progressing run emits nothing.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ calls = {"n": 0}
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ calls["n"] += 1
+ if calls["n"] < 3:
+ return ToolResult.error("fill refused")
+ return ToolResult.ok("fill landed", data={"page_state_changed": True})
+
+ fill = ToolSpec(
+ name="fill", description="fill", parameters={"type": "object", "properties": {}}, handler=handler, billable=True
+ )
+ script = [
+ [("fill", {"selector": "#code", "value": "a"})],
+ [("fill", {"selector": "#code", "value": "b"})],
+ [("fill", {"selector": "#code", "value": "ab"})],
+ [("finish", {"status": "completed", "reason": "landed"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, [fill, make_finish_tool()], max_turns=50, max_tool_calls=100)
+ assert outcome.status == "completed"
+ assert calls["n"] == 3
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_loop_cleared_by_movement_landing_between_batches() -> None:
+ # A delayed render lands after one batch's after-sample and before the next batch's
+ # before-sample: the cross-batch fingerprint move is confirmed progress and must clear the
+ # ring BEFORE the new batch's touches are read against the old ones.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ fp_calls = {"n": 0}
+
+ async def between_batch_fingerprint() -> str:
+ fp_calls["n"] += 1
+ return f"dom-{(fp_calls['n'] - 1) // 2}"
+
+ script = [[("fill", {"selector": "#code", "value": str(i)})] for i in range(4)]
+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [_error_billable_tool("fill", touches), make_finish_tool()],
+ page_fingerprint=between_batch_fingerprint,
+ max_turns=50,
+ max_tool_calls=100,
+ )
+ assert outcome.status == "failed"
+ assert len(touches) == 4
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_loop_not_masked_by_replayed_download_notice() -> None:
+ # A compactable tool replaying a retained download notice (download_notice without
+ # download_new) is not fresh progress: it must not keep wiping the ring, or a post-download
+ # loop could never accumulate enough touches to emit telemetry.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+
+ async def replay_observe(args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("page\nDownloaded: report.pdf (1.0 MB)", data={"download_notice": True})
+
+ observe = ToolSpec(
+ name="observe",
+ description="observe",
+ parameters={"type": "object", "properties": {}},
+ handler=replay_observe,
+ billable=False,
+ compactable=True,
+ )
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(4):
+ script.append([("click", {"selector": "#dl", "note": str(i)})])
+ script.append([("observe", {})])
+ script.append([("finish", {"status": "failed", "reason": "kept refusing after the download"})])
+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ assert len(touches) == 4
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 4]
+
+
+async def test_canonical_loop_suppressed_when_completion_probe_fires_on_the_rung_touch() -> None:
+ # complete_on_download: the file can land while the probe waits, AFTER the tool result returned
+ # without download_new. The probe's outcome skips the end-of-batch detector, but the pending
+ # rung minted by that same touch must not be emitted — the run completed on real progress.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ calls = {"n": 0}
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ calls["n"] += 1
+ return ToolResult.error("click refused") if calls["n"] < 3 else ToolResult.ok("clicked")
+
+ click = ToolSpec(
+ name="click",
+ description="click",
+ parameters={"type": "object", "properties": {}},
+ handler=handler,
+ billable=True,
+ )
+
+ async def probe(_staged: frozenset[str]) -> str | None:
+ return "a file finished downloading" if calls["n"] >= 3 else None
+
+ script = [
+ [("click", {"selector": "#dl", "note": "a"})],
+ [("click", {"selector": "#dl", "note": "b"})],
+ [("click", {"selector": "#dl", "note": "c"})],
+ [("finish", {"status": "failed", "reason": "unreached"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script, [click, make_finish_tool()], completion_probe=probe, max_turns=50, max_tool_calls=100
+ )
+ assert outcome.status == "completed"
+ assert outcome.reason == "a file finished downloading"
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_loop_not_masked_by_missing_probe_samples() -> None:
+ # Auto-observe deliberately treats a missing probe sample as "assume changed" for the model's
+ # benefit, but that fallback is not evidence: intermittent probe timeouts must not keep
+ # clearing the ring and permanently mask a genuine loop.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+
+ async def dead_probe() -> str | None:
+ return None
+
+ async def observe_handler(_args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("url=x page digest (1 interactive elements)")
+
+ observe = ToolSpec(
+ name="observe",
+ description="observe",
+ parameters={"type": "object", "properties": {}},
+ handler=observe_handler,
+ compactable=True,
+ )
+ script = [[("click", {"selector": "#dl", "note": str(i)})] for i in range(4)]
+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])
+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script, tools, auto_observe=True, page_probe=dead_probe, max_turns=50, max_tool_calls=100
+ )
+ assert outcome.status == "failed"
+ assert len(touches) == 4
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 4]
+
+
+async def test_canonical_loop_absorbs_movement_in_a_terminal_batch_before_emitting() -> None:
+ # Two refusals, then a batch whose successful third touch changes the DOM (fingerprint-only)
+ # and whose finish completes the run: the terminal outcome skips the detector, but the batch's
+ # own movement must still be absorbed before the pending rung is decided.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ calls = {"n": 0}
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ calls["n"] += 1
+ return ToolResult.error("click refused") if calls["n"] < 3 else ToolResult.ok("clicked")
+
+ click = ToolSpec(
+ name="click",
+ description="click",
+ parameters={"type": "object", "properties": {}},
+ handler=handler,
+ billable=True,
+ )
+ fp_calls = {"n": 0}
+
+ async def fingerprint() -> str:
+ # Moves only at the terminal batch's after-sample (calls 1-5 are the three before-samples
+ # and the first two batches' after-samples).
+ fp_calls["n"] += 1
+ return "dom-0" if fp_calls["n"] <= 5 else "dom-1"
+
+ script = [
+ [("click", {"selector": "#dl", "note": "a"})],
+ [("click", {"selector": "#dl", "note": "b"})],
+ [("click", {"selector": "#dl", "note": "c"}), ("finish", {"status": "completed", "reason": "done"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script, [click, make_finish_tool()], page_fingerprint=fingerprint, max_turns=50, max_tool_calls=100
+ )
+ assert outcome.status == "completed"
+ assert calls["n"] == 3
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canceled_run_emits_pending_fire_without_a_terminal_probe() -> None:
+ # An acknowledged cancellation must not wait on the terminal-batch reconciliation sample (a
+ # hung renderer can hold that probe for its full timeout); the pending rung is emitted as
+ # minted, since no progress evidence contradicts it.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ fp_calls = {"n": 0}
+
+ async def fingerprint() -> str:
+ fp_calls["n"] += 1
+ return "dom-0"
+
+ async def should_cancel() -> bool:
+ return len(touches) >= 3
+
+ script = [
+ [("click", {"selector": "#dl", "note": "a"})],
+ [("click", {"selector": "#dl", "note": "b"})],
+ [("click", {"selector": "#dl", "note": "c"}), ("click", {"selector": "#dl", "note": "d"})],
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [_error_billable_tool("click", touches), make_finish_tool()],
+ page_fingerprint=fingerprint,
+ should_cancel=should_cancel,
+ max_turns=50,
+ max_tool_calls=100,
+ )
+ assert outcome.status == "canceled"
+ assert len(touches) == 3 # the fourth call was refused by the cancellation check
+ # 2 samples per completed batch plus the final batch's before-sample; NO terminal probe.
+ assert fp_calls["n"] == 5
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3]
+
+
+async def test_canonical_loop_cleared_by_changed_perception_digest_without_fingerprint() -> None:
+ # With no page_fingerprint, a repeated observe whose digest changes is the only movement
+ # evidence there is; two landed digests that differ must clear the ring like a fingerprint
+ # mismatch would, so touches against superseded pages never alias into a rung.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ pages = {"n": 0}
+
+ async def observe_handler(_args: dict[str, Any]) -> ToolResult:
+ pages["n"] += 1
+ return ToolResult.ok(f"url=x page {pages['n']} content (1 interactive elements)")
+
+ observe = ToolSpec(
+ name="observe",
+ description="observe",
+ parameters={"type": "object", "properties": {}},
+ handler=observe_handler,
+ compactable=True,
+ )
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]
+ for i in range(4):
+ script.append([("click", {"selector": "#dl", "note": str(i)})])
+ script.append([("observe", {})])
+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])
+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ assert len(touches) == 4
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_canonical_rung_fires_once_even_when_the_window_parks_on_it() -> None:
+ # Eight failed touches on A, two on B, then more on A: eviction keeps A's in-window count
+ # parked at 8, which must not re-emit rung 8 on every subsequent touch — a rung is one
+ # threshold crossing per generation.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ script = [[("click", {"selector": "#a", "note": str(i)})] for i in range(8)]
+ script += [[("click", {"selector": "#b", "note": str(i)})] for i in range(2)]
+ script += [[("click", {"selector": "#a", "note": f"again-{i}"})] for i in range(3)]
+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])
+ tools = [_error_billable_tool("click", touches), make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=60, max_tool_calls=200)
+ assert outcome.status == "failed"
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 4, 6, 8]
+
+
+async def test_canonical_loop_clears_on_new_low_even_under_a_replayed_notice() -> None:
+ # Blocker 1 (AronPerez round 6/10): a replayed download notice hard-progresses the shadow
+ # ledger, nulling invalid_baseline BEFORE the shadow read on the same result — which left the
+ # new-low clear dead for the rest of the page. Constant observe content keeps the perception
+ # digest from masking the probe. A form ratcheting to a new low on every look is progressing;
+ # the refusing selector must emit nothing.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ seq = iter([6, 5, 4, 3, 2])
+
+ async def observe_handler(_args: dict[str, Any]) -> ToolResult:
+ return ToolResult.ok("url=x form", data={"download_notice": True, "summary": {"invalid_fields": next(seq)}})
+
+ observe = ToolSpec(
+ name="observe",
+ description="observe",
+ parameters={"type": "object", "properties": {}},
+ handler=observe_handler,
+ compactable=True,
+ )
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(4):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": "#stuck", "note": str(i)})])
+ script.append([("observe", {})])
+ script.append([("finish", {"status": "failed", "reason": "one field kept refusing"})])
+ tools = [_error_billable_tool("click", touches), observe, make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ assert len(touches) == 4
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+async def test_completion_probe_spares_a_sibling_targets_rung() -> None:
+ # Blocker 2 (AronPerez round 10): the probe firing is progress for the COMPLETING touch only.
+ # A five-error streak on one selector minted a genuine rung 6 the same batch a download on a
+ # DIFFERENT selector completed the run — that sibling rung is real data and must survive.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ fetched = {"done": False}
+
+ async def fetch_handler(_args: dict[str, Any]) -> ToolResult:
+ fetched["done"] = True
+ return ToolResult.ok("fetch dispatched")
+
+ fetch = ToolSpec(
+ name="fetch",
+ description="fetch",
+ parameters={"type": "object", "properties": {}},
+ handler=fetch_handler,
+ billable=True,
+ )
+
+ async def probe(_staged: frozenset[str]) -> str | None:
+ return "a file finished downloading" if fetched["done"] else None
+
+ script = [[("click", {"selector": "#stuck", "note": str(i)})] for i in range(5)]
+ script.append([("click", {"selector": "#stuck", "note": "sixth"}), ("fetch", {"selector": "#other"})])
+ tools = [_error_billable_tool("click", touches), fetch, make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, completion_probe=probe, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "completed"
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 4, 6]
+
+
+async def test_canonical_rung_rearms_after_full_eviction() -> None:
+ # A rung re-arms once the target's in-window count dips below it: a fresh streak after full
+ # eviction is a genuine re-crossing, unlike the parked-window case the once-per-generation
+ # guard exists for.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ script = [[("click", {"selector": "#a", "note": str(i)})] for i in range(3)]
+ script += [[("click", {"selector": "#b", "note": str(i)})] for i in range(10)]
+ script += [[("click", {"selector": "#a", "note": f"again-{i}"})] for i in range(3)]
+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])
+ tools = [_error_billable_tool("click", touches), make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, max_turns=60, max_tool_calls=200)
+ assert outcome.status == "failed"
+ fires = [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT]
+ assert [e["repeat_count"] for e in fires] == [3, 3, 4, 6, 8, 3]
+
+
+async def test_canonical_loop_cleared_by_positive_probe_mismatch_on_a_failed_call() -> None:
+ # A failed call that moved the document (two landed identity samples that differ) is progress
+ # the fingerprint can miss entirely (a same-template step renders identically): the poisoned
+ # batch stop already knows the page moved, and the ring must learn it too.
+ from skyvern.forge.taskv3.loop import CANONICAL_LOOP_EVENT
+
+ touches: list[tuple[str, dict[str, Any]]] = []
+ probe_calls = {"n": 0}
+
+ async def moving_probe() -> str:
+ probe_calls["n"] += 1
+ return f"doc-{probe_calls['n']}"
+
+ script = [[("click", {"selector": "#stuck", "note": str(i)})] for i in range(4)]
+ script.append([("finish", {"status": "failed", "reason": "kept refusing"})])
+ tools = [_error_billable_tool("click", touches), make_finish_tool()]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, page_probe=moving_probe, max_turns=50, max_tool_calls=100)
+ assert outcome.status == "failed"
+ assert len(touches) == 4
+ assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []