Skip to content

Automate browser based workflows with AI

Hold a give-up while verification-code polling budget is still unspent (SKY-15918) (#8494)

pedrohsdbd0615cbe5647
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 979796f36..44afecfe1 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -2200,7 +2200,7 @@ class ForgeAgent:
completion_blocker=completion_blocker,
staged_downloads=staged_downloads,
deadline_seconds=loop_deadline_seconds,
- verification_blocker=verification_state.block_completion,
+ verification_blocker=verification_state.block_finish,
# Only for a bare task, where setup navigated this browser_state to task.url and the
# status unambiguously belongs to the starting posting. A workflow block reuses the
# browser_state across blocks, so its last status may be a prior block's — skip it there
diff --git a/skyvern/forge/taskv3/auth_tools.py b/skyvern/forge/taskv3/auth_tools.py
index 9669950db..28ff89dce 100644
--- a/skyvern/forge/taskv3/auth_tools.py
+++ b/skyvern/forge/taskv3/auth_tools.py
@@ -43,6 +43,9 @@ from skyvern.webeye.navigation import revalidate_redirect_chain
LOG = structlog.get_logger()
+_CODE_TOOL = "get_verification_code"
+_LINK_TOOL = "open_verification_link"
+
# One tool call polls at most this long, so a call made before the page has sent the code (or while
# the source is still empty) returns and lets the model act instead of blocking the loop for the
# whole budget.
@@ -50,6 +53,14 @@ _PER_CALL_WAIT_SECONDS = 120.0
# The poll loop fetches once per 10s sleep, so a slice shorter than that would never fetch; a tail
# that small counts as spent.
_MIN_SLICE_SECONDS = 10.0
+# How long a "not yet" answer keeps the give-up gate armed. Past it the run has moved on and a
+# non-complete verdict is about something else; same idiom (and duration order) as the loop's
+# FAILURE_EVIDENCE_WINDOW_TURNS recency, expressed in seconds because the budget is.
+_NOT_YET_RECENCY_SECONDS = 180.0
+# Termination is guaranteed by the budget shrinking under every productive deferral, but a slice may
+# advance spend by as little as _MIN_SLICE_SECONDS, which would admit ~90 holds. 8 >= ceil(900/120),
+# so this never binds on the healthy path.
+_MAX_GIVEUP_DEFERRALS = 8
# Shorter values are codes/flags (lang=en, v=2), not link secrets, and a real link secret is at
# least this long; redacting the short ones would blank harmless text across the run's artifacts.
_MIN_REDACTED_QUERY_VALUE_CHARS = 16
@@ -145,7 +156,7 @@ _LINK_GUIDANCE = (
def _tool_name(expected_otp_type: OTPType) -> str:
- return "open_verification_link" if expected_otp_type == OTPType.MAGIC_LINK else "get_verification_code"
+ return _LINK_TOOL if expected_otp_type == OTPType.MAGIC_LINK else _CODE_TOOL
def _is_token_shaped(value: str) -> bool:
@@ -263,9 +274,10 @@ async def _cookie_jar(page: Any) -> list[tuple[str, str, str, str]] | None:
class VerificationFailure(StrEnum):
"""Every condition that arms the completion refusal. This closes the VOCABULARY, which is what the
- earlier hand-kept inventory could not do — it was missing six sites. Each member is written at
- exactly one place today, so the covered set and the site set coincide; nothing here would stop a
- future site from reusing a member, so that coincidence is a fact to re-check, not a guarantee."""
+ earlier hand-kept inventory could not do — it was missing six sites. Members are not one-per-site:
+ BUDGET_EXHAUSTED is written both where a slice spends the last of the budget and where a later
+ call finds it already spent. Closing the vocabulary is what this buys; a member-to-site bijection
+ is not, and nothing here depends on one."""
BUDGET_EXHAUSTED = "budget_exhausted"
NO_CODE_TWICE = "no_code_twice"
@@ -285,7 +297,8 @@ class VerificationFailure(StrEnum):
@dataclass
class VerificationState:
- """Lets the finish tool refuse a completed verdict once the source terminally failed to deliver.
+ """Lets the finish tool refuse a completed verdict once the source terminally failed to deliver, and
+ hold a non-complete one while the run is still awaiting a code it has unspent budget for.
The latch is armed by every terminal non-delivery answer (budget exhaustion, repeated lookup
empty answers, refused or unopenable link), never by a retryable "not yet" or by a link failure
@@ -298,16 +311,39 @@ class VerificationState:
same fields (`skyvern/forge/agent.py`, "TOTP polling timed out — terminating task"); without this
the v3 side latched silently and the failure could not be found in production at all."""
- # Excluded from the repr: the bound `block_completion` handed to the loop would otherwise carry
+ # Excluded from the repr: the bound `block_finish` handed to the loop would otherwise carry
# the task's navigation payload into any message that formats the callback.
task: Task = field(repr=False)
values_delivered: int = field(default=0, init=False)
+ # The shared polling budget the tools draw down, lifted onto the state so the finish gate can
+ # read it. `build_auth_tools` advances `polling_spent_seconds` from its poll accounting.
+ budget_seconds: float = field(
+ default_factory=lambda: settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS * 60.0, init=False
+ )
+ polling_spent_seconds: float = field(default=0.0, init=False)
+ # Monotonic timestamp of the last answer that invited another call, cleared where the source
+ # produces a value. Written by the two retryable answers -- "not yet", and a first empty or
+ # errored lookup -- so "the run is awaiting a code right now" is a tool-call fact rather than a
+ # reading of the page. An arming does NOT clear it: `source_failed` is the conjunct that stops a
+ # hold once the source is judged, and it has to keep working when a later poll re-arms this latch.
+ awaiting_code_since: float | None = field(default=None, init=False)
+ # Which tool the latch belongs to. Both tools drain one source and share the budget, so a
+ # magic-link run must not be sent back to `get_verification_code`.
+ awaiting_code_tool: str = field(default=_CODE_TOOL, init=False)
+ giveup_deferrals: int = field(default=0, init=False)
+ # Spend at the previous honored deferral. A finish that arrives with spend unchanged means the
+ # model did not re-poll, so the next give-up stands rather than ping-ponging against the gate.
+ spend_at_last_giveup_deferral: float | None = field(default=None, init=False, repr=False)
_armings: list[VerificationFailure] = field(default_factory=list, init=False, repr=False)
@property
def source_failed(self) -> bool:
return bool(self._armings)
+ @property
+ def remaining_budget_seconds(self) -> float:
+ return self.budget_seconds - self.polling_spent_seconds
+
def arm(self, reason: VerificationFailure, tool: str) -> None:
self._armings.append(reason)
try:
@@ -347,6 +383,79 @@ class VerificationState:
return _COMPLETION_BLOCKED
return None
+ async def block_giveup(self, status: str) -> str | None:
+ """Defer a non-complete verdict exactly while the run is actively awaiting a verification
+ code it has unspent budget for. The first four conditions operationalize that one invariant;
+ the last two bound the hold, so a model that ignores the deferral still ends within two
+ finishes and one that polls minimally within `_MAX_GIVEUP_DEFERRALS`."""
+ remaining = self.remaining_budget_seconds
+ reason: str | None = None
+ if self.awaiting_code_since is None:
+ reason = "not_awaiting"
+ elif self.source_failed:
+ reason = "source_failed"
+ elif time.monotonic() - self.awaiting_code_since > _NOT_YET_RECENCY_SECONDS:
+ reason = "stale"
+ elif remaining < _MIN_SLICE_SECONDS:
+ reason = "budget_spent"
+ elif self.giveup_deferrals >= _MAX_GIVEUP_DEFERRALS:
+ reason = "deferral_cap"
+ elif (
+ self.spend_at_last_giveup_deferral is not None
+ and self.polling_spent_seconds <= self.spend_at_last_giveup_deferral
+ ):
+ reason = "unproductive"
+ if reason is not None:
+ self._record_giveup(held=False, reason=reason, remaining=remaining, status=status)
+ return None
+ self.giveup_deferrals += 1
+ self.spend_at_last_giveup_deferral = self.polling_spent_seconds
+ self._record_giveup(held=True, reason="held", remaining=remaining, status=status)
+ # Both tools drain one source, so the artifact has to come off the same latch the tool does:
+ # telling a run that polled for a sign-in link that "the code has not arrived" contradicts the
+ # page and the answer the tool just gave it.
+ subject = "sign-in link" if self.awaiting_code_tool == _LINK_TOOL else "verification code"
+ return (
+ f"about {remaining / 60.0:.1f} minutes of {subject} polling budget remain and the "
+ f"{subject} has not arrived yet. Call {self.awaiting_code_tool} again to keep waiting for "
+ "it. If the page never sent one you may re-request it; otherwise do not act on the page "
+ "for this. Once the budget is spent the tool will say so; finish then and the verdict "
+ "will stand."
+ )
+
+ async def block_finish(self, status: str) -> str | None:
+ """The one verification gate the finish tool consults, for every verdict: a completed claim is
+ refused once the source terminally failed, a non-complete one is deferred while budget the
+ run is still waiting on remains."""
+ if status == "completed":
+ return await self.block_completion()
+ return await self.block_giveup(status)
+
+ def _record_giveup(self, *, held: bool, reason: str, remaining: float, status: str) -> None:
+ # The pre-registered probe: today the unspent residual at a give-up is only reconstructible by
+ # differencing poll timestamps, so it cannot be counted directly. `finish_status` is carried
+ # because failed and terminated are separate populations in that read and the record is the
+ # only place they can be told apart.
+ if not held and self.polling_spent_seconds <= 0:
+ return
+ try:
+ LOG.info(
+ "task_v3 verification give-up gate",
+ task_id=self.task.task_id,
+ organization_id=self.task.organization_id,
+ workflow_run_id=self.task.workflow_run_id,
+ held=held,
+ reason=reason,
+ finish_status=status,
+ polling_spent_seconds=round(self.polling_spent_seconds, 1),
+ remaining_seconds=round(remaining, 1),
+ budget_seconds=self.budget_seconds,
+ giveup_deferrals=self.giveup_deferrals,
+ )
+ except Exception:
+ # Narration must never cost the gate its verdict, for the same reason `arm` swallows.
+ pass
+
def build_auth_tools(
task: Task,
@@ -357,8 +466,8 @@ def build_auth_tools(
"""Return (tools, system-prompt guidance) for verification handling, or ([], "") when the task has
no verification source configured (so the tools aren't offered needlessly). The link tool also needs
a page to navigate, so a page-free run never gets it. `state`, if given, is mutated as the tools
- poll and deliver values; pass its `block_completion` to `make_finish_tool` to gate a completed
- verdict on it. A caller with no use for that gate can omit `state` entirely."""
+ poll and deliver values; pass its `block_finish` to `make_finish_tool` to gate every finish verdict
+ on it. A caller with no use for that gate can omit `state` entirely."""
if state is None:
state = VerificationState(task=task)
offer_code_tool = has_otp_source(
@@ -372,9 +481,11 @@ def build_auth_tools(
# The model re-calls after every empty answer, so the cumulative polling across this task's calls
# is capped at VERIFICATION_CODE_POLLING_TIMEOUT_MINS (the step engine's single poll window) and
- # then the tools refuse with stop guidance. One budget for both tools: they drain one source.
- budget_seconds = settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS * 60.0
- polling_spent_seconds = 0.0
+ # then the tools refuse with stop guidance. One budget for both tools: they drain one source; it
+ # lives on `state` so the finish gate can see what is left of it. Re-derived here rather than
+ # taken from the state's default because a caller may have built the state before the setting was
+ # read, and the budget must be the one in force where it is actually drawn down.
+ state.budget_seconds = settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS * 60.0
# A value one tool resolved that the other tool owns (the webhook source does not filter by type).
cached_otp_value: OTPValue | None = None
first_poll_started_at: datetime | None = None
@@ -396,15 +507,46 @@ def build_auth_tools(
state.arm(reason, tool)
return ToolResult.error(message, data=data)
- def _not_yet(expected_otp_type: OTPType, detail: str) -> ToolResult:
- if expected_otp_type == OTPType.MAGIC_LINK:
- return ToolResult.error(
- f"no sign-in link available yet ({detail}). If the page has not sent one, trigger it "
- "first, then call open_verification_link again."
+ def _not_yet(expected_otp_type: OTPType, detail: str, elapsed_seconds: float) -> ToolResult:
+ """A healthy source that has not delivered yet: the run is still awaiting a code, and the
+ message says how much budget is left to wait with — the old one gave no indication any
+ remained. Stays a ToolResult.error: the status is never serialized to the model (the loop
+ sends `content` only), and it is what routes a call that blocked for up to 120s through the
+ batch-poisoning check, so flipping it would let a stale queued click dispatch against a page
+ that moved during the wait. `elapsed_seconds` is this slice's wait, which the caller's
+ `finally` has not yet added to the running spend."""
+ state.awaiting_code_since = time.monotonic()
+ state.awaiting_code_tool = _tool_name(expected_otp_type)
+ spent = state.polling_spent_seconds + elapsed_seconds
+ remaining = max(0.0, state.budget_seconds - spent)
+ LOG.warning(
+ "task_v3 verification poll slice timed out",
+ task_id=task.task_id,
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ tool=_tool_name(expected_otp_type),
+ polling_spent_seconds=round(spent, 1),
+ remaining_seconds=round(remaining, 1),
+ budget_seconds=state.budget_seconds,
+ )
+ tool = _tool_name(expected_otp_type)
+ subject = "sign-in link" if expected_otp_type == OTPType.MAGIC_LINK else "verification code"
+ if remaining < _MIN_SLICE_SECONDS:
+ # Nothing left to buy: the next call refuses without polling, so inviting one would spend
+ # a tool call and a turn to be told that -- and near the loop's own limits that trades a
+ # truthful verification failure for a generic budget-exhausted exit. Terminal, so it arms:
+ # the budget is exhausted here whether or not the model makes that pointless call, and
+ # leaving the latch to be set by the call itself is what let a model that obeys the
+ # instruction go on to claim the verification step completed.
+ return _failed(
+ f"no {subject} available yet ({detail}), and the polling budget is now spent. Do not "
+ f"call {tool} again; finish the task as failed and say the {subject} never arrived.",
+ reason=VerificationFailure.BUDGET_EXHAUSTED,
+ tool=tool,
)
return ToolResult.error(
- f"no verification code available yet ({detail}). If the page has not sent one, trigger it "
- "first, then call get_verification_code again."
+ f"no {subject} available yet ({detail}); about {remaining / 60.0:.1f} minutes of polling "
+ f"budget remain. If the page has not sent one, trigger it first, then call {tool} again."
)
def _empty_answer(once: str, terminal: str, *, reason: VerificationFailure, tool: str) -> ToolResult:
@@ -412,12 +554,18 @@ def build_auth_tools(
empty_answer_streak += 1
if empty_answer_streak > 1:
return _failed(terminal, reason=reason, tool=tool)
+ # Retryable, so it arms the give-up gate for the same reason `_not_yet` does: this answer
+ # invites another call, and a run that gives up right after it has exactly as much unspent
+ # budget as one that gave up after a "not yet". Arming only the "not yet" path would leave the
+ # gate holding on one of the four answers that ask the model to wait.
+ state.awaiting_code_since = time.monotonic()
+ state.awaiting_code_tool = tool
return ToolResult.error(once)
async def _poll(expected_otp_type: OTPType) -> ToolResult | OTPValue:
"""One budget-accounted polling slice. A ToolResult is the model-facing answer to return as-is."""
- nonlocal polling_spent_seconds, first_poll_started_at, empty_answer_streak
- remaining = budget_seconds - polling_spent_seconds
+ nonlocal first_poll_started_at, empty_answer_streak
+ remaining = state.remaining_budget_seconds
if remaining < _MIN_SLICE_SECONDS:
return _budget_exhausted(expected_otp_type)
if first_poll_started_at is None:
@@ -446,9 +594,14 @@ def build_auth_tools(
tool=_tool_name(expected_otp_type),
)
empty_answer_streak = 0
+ # Cleared here, not at delivery: several paths reach a value that is never handed to the
+ # model -- a link when a code was asked for, a code when a link was, a link the browser
+ # could not open -- and on all of them the run has stopped waiting on the SOURCE. Holding
+ # a give-up there would contradict the very answer those paths return.
+ state.awaiting_code_since = None
return otp_value
except (NoTOTPVerificationCodeFound, FailedToGetTOTPVerificationCode) as exc:
- if polling_spent_seconds + (time.monotonic() - started) >= budget_seconds:
+ if state.polling_spent_seconds + (time.monotonic() - started) >= state.budget_seconds:
return _budget_exhausted(expected_otp_type)
detail = type(exc).__name__
if isinstance(exc, FailedToGetTOTPVerificationCode):
@@ -467,7 +620,7 @@ def build_auth_tools(
empty_answer_streak = 0
if exc.webhook_diagnostics:
detail = f"{detail}: {exc.webhook_diagnostics}"
- return _not_yet(expected_otp_type, detail)
+ return _not_yet(expected_otp_type, detail, time.monotonic() - started)
except Exception as exc:
LOG.warning(
"task_v3 verification tool lookup failed",
@@ -488,7 +641,7 @@ def build_auth_tools(
tool=_tool_name(expected_otp_type),
)
finally:
- polling_spent_seconds += time.monotonic() - started
+ state.polling_spent_seconds += time.monotonic() - started
async def _get_verification_code(args: dict[str, Any]) -> ToolResult:
nonlocal cached_otp_value
@@ -541,9 +694,22 @@ def build_auth_tools(
if context is not None:
# Redact the code from this task's artifacts/logs (task-scoped, so bare tasks are covered).
context.register_secret_value(code)
- state.values_delivered += 1
+ _record_delivery("get_verification_code")
return ToolResult.ok(f"verification_code: {code}")
+ def _record_delivery(tool: str) -> None:
+ state.values_delivered += 1
+ LOG.info(
+ "task_v3 verification value delivered",
+ task_id=task.task_id,
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ tool=tool,
+ values_delivered=state.values_delivered,
+ polling_spent_seconds=round(state.polling_spent_seconds, 1),
+ budget_seconds=state.budget_seconds,
+ )
+
async def _open_verification_link(args: dict[str, Any]) -> ToolResult:
nonlocal cached_otp_value, first_poll_started_at
otp_value: OTPValue | None = cached_otp_value
@@ -683,7 +849,7 @@ def build_auth_tools(
tool="open_verification_link",
data={"page_state_changed": True},
)
- state.values_delivered += 1
+ _record_delivery("open_verification_link")
return ToolResult.ok(_LINK_OPENED, data={"page_state_changed": True})
tools: list[ToolSpec] = []
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 76db984f3..2d00b31fb 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -64,9 +64,12 @@ ToolHandler = Callable[[dict[str, Any]], Awaitable[ToolResult]]
# receive the basenames tools staged into the downloads dir this run, to exclude from detection.
CompletionProbe = Callable[[frozenset[str]], Awaitable[str | None]]
CompletionBlocker = Callable[[frozenset[str]], Awaitable[str | None]]
-# Consulted from finish(completed) like CompletionBlocker, but takes no arguments -- it gates on
-# state the caller already tracks (e.g. a verification-code budget), not on staged downloads.
-VerificationBlocker = Callable[[], Awaitable[str | None]]
+# Consulted from finish for EVERY verdict, unlike CompletionBlocker: it takes the finish status and
+# gates on state the caller already tracks (e.g. a verification-code budget), not on staged
+# downloads. One callback, because a completed claim made on a blank verification step and a
+# non-complete verdict given up with polling budget still unspent are the same concern seen from two
+# sides; splitting them into two hooks would scatter it.
+VerificationBlocker = Callable[[str], Awaitable[str | None]]
@dataclass
@@ -677,6 +680,10 @@ FAILURE_EVIDENCE_SETTLE_MAX_SECONDS = 8.0
FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS = 60.0
FAILURE_EVIDENCE_MIN_TOOL_CALLS = 3
FAILURE_EVIDENCE_MIN_TURNS = 3
+# A verification give-up deferral asks for a BLOCKING poll slice (auth_tools caps one at 120s) on top
+# of that cycle, so the 60s above would let the gate convert an honest failure into the
+# budget-exhausted end it exists to prevent. Not imported from auth_tools: that module imports this one.
+VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS = 180.0
def _is_enter_submit(tool_name: str, args: dict[str, Any]) -> bool:
@@ -791,7 +798,11 @@ def _names_submit_control(tool_name: str, args: dict[str, Any], ok: bool) -> str
return selector if isinstance(selector, str) and selector else None
-def _has_hold_headroom(activity: ActivityRecency | None, deadline_at: float | None) -> bool:
+def _has_hold_headroom(
+ activity: ActivityRecency | None,
+ deadline_at: float | None,
+ min_deadline_headroom_seconds: float = FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS,
+) -> bool:
"""Whether a deferral has the budget to buy the re-verification turn it asks for.
Without it the run ends budget_exhausted, which is unmapped and lands on failed -- turning an
@@ -814,7 +825,7 @@ def _has_hold_headroom(activity: ActivityRecency | None, deadline_at: float | No
return False
if activity.perception_stall_imminent:
return False
- if deadline_at is not None and deadline_at - time.monotonic() < FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS:
+ if deadline_at is not None and deadline_at - time.monotonic() < min_deadline_headroom_seconds:
return False
return True
@@ -1312,7 +1323,14 @@ def make_finish_tool(
completed-side cap, not per verdict attempt) — a quiescence wait
bounded by `failure_settle_max_seconds`, then a deferral asking the model to re-observe —
because async submissions and captcha protocols otherwise produce false-negative verdicts.
- terminated is never gated on either side.
+
+ `verification_blocker` is the one gate consulted for EVERY verdict: it refuses a completed claim
+ once the verification source terminally failed, and holds a failed OR terminated one while the
+ run is still awaiting a code it has unspent polling budget for -- a give-up at one 120s slice of
+ a 15-minute budget throws away minutes of waiting the run already owns. The hold is bounded by
+ the callee (the budget shrinks under every productive hold) and refused here without the deadline
+ headroom to fund the blocking poll slice it asks for. Apart from that gate, terminated is
+ ungated on both sides.
`pending_marker` reports the text the page still shows the control in `submit_watch` as in
flight with, or None. A settled page is not a submitted one -- a submit frozen mid-flight is
@@ -1426,17 +1444,36 @@ def make_finish_tool(
)
if blocker_message:
return ToolResult.error(blocker_message)
- if status == "completed" and verification_blocker is not None:
+ if verification_blocker is not None and (
+ status == "completed"
+ # A non-complete verdict is only ever HELD, never refused, so unlike the completed side it
+ # must fund the retry it asks for -- and that retry is a blocking poll slice, not just a
+ # re-observe. Absent the accounting to check that (no `activity`), the hold is not
+ # justifiable and the verdict stands.
+ or (
+ activity is not None
+ and _has_hold_headroom(
+ activity,
+ deadline_at,
+ min_deadline_headroom_seconds=VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS,
+ )
+ )
+ ):
try:
- verification_message = await verification_blocker()
+ verification_message = await verification_blocker(status)
except Exception:
- # Fail closed: an exception here must not let a blank verification step read as done.
- LOG.warning("taskv3 verification_blocker failed; failing closed", exc_info=True)
- return ToolResult.error(
- "Could not verify that the verification-code step completed cleanly; retry "
- "finish(status=completed) once verified, or finish with status=failed or "
- "status=terminated."
- )
+ if status == "completed":
+ # Fail closed: an exception here must not let a blank verification step read as done.
+ LOG.warning("taskv3 verification_blocker failed; failing closed", exc_info=True)
+ return ToolResult.error(
+ "Could not verify that the verification-code step completed cleanly; retry "
+ "finish(status=completed) once verified, or finish with status=failed or "
+ "status=terminated."
+ )
+ # Fail open on the give-up side: the opposite verdict. A broken gate must not trap a
+ # run that wants to end.
+ LOG.warning("taskv3 verification_blocker failed; honoring the verdict", exc_info=True)
+ verification_message = None
if verification_message:
return ToolResult.error(verification_message)
if (
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index c599fe1b9..86063026d 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -3157,9 +3157,9 @@ async def test_execute_task_v3_hands_the_loop_the_live_verification_gate(monkeyp
blocker = loop_mock.await_args.kwargs["verification_blocker"]
assert blocker is not None
assert len(seen_states) == 1 and blocker.__self__ is seen_states[0]
- assert await blocker() is None
+ assert await blocker("completed") is None
seen_states[0].arm(VerificationFailure.NO_CODE_TWICE, "get_verification_code")
- assert await blocker() is not None
+ assert await blocker("completed") is not None
@pytest.mark.asyncio
diff --git a/tests/unit/test_taskv3_auth_tools.py b/tests/unit/test_taskv3_auth_tools.py
index 4a4cd2109..49b11f500 100644
--- a/tests/unit/test_taskv3_auth_tools.py
+++ b/tests/unit/test_taskv3_auth_tools.py
@@ -4,6 +4,8 @@ from __future__ import annotations
import asyncio
import logging
+import re
+import time
from datetime import datetime, timezone
from types import SimpleNamespace
from typing import Any
@@ -1560,3 +1562,392 @@ def test_block_credential_parameter_keys_script_mode_returns_none(monkeypatch: p
assert agent_module.block_credential_parameter_keys(block, "wr_test") is None
with skyvern_context.scoped(SkyvernContext()):
assert agent_module.block_credential_parameter_keys(NS(parameters=[]), "wr_test") == []
+
+
[email protected]("awaiting", [True, False], ids=["awaiting", "not_awaiting"])
[email protected]("armed", [True, False], ids=["source_failed", "source_healthy"])
[email protected]("stale", [True, False], ids=["stale", "recent"])
[email protected]("spent", [True, False], ids=["budget_spent", "budget_left"])
+async def test_giveup_gate_holds_exactly_while_awaiting_a_code_it_has_budget_for(
+ awaiting: bool, armed: bool, stale: bool, spent: bool
+) -> None:
+ # The invariant in one sentence: a non-complete finish is held exactly while the run is actively
+ # awaiting a verification code it has purchased-but-unspent budget for. The four conjuncts are
+ # that sentence's operationalization, so this asserts the BICONDITIONAL over the whole
+ # (awaiting, armed, recency, spend) space rather than four example negative controls -- an
+ # enumerated field list cannot close a class, and a conjunct silently dropped from the gate would
+ # leave every example that does not exercise it green.
+ state = auth_tools.VerificationState(task=_task())
+ state.budget_seconds = 900.0
+ state.polling_spent_seconds = 900.0 - (auth_tools._MIN_SLICE_SECONDS / 2 if spent else 600.0)
+ if awaiting:
+ state.awaiting_code_since = time.monotonic() - (auth_tools._NOT_YET_RECENCY_SECONDS + 1 if stale else 0.0)
+ if armed:
+ # Through `arm`, not by poking the latch: it is the only way the latch is reachable in
+ # production, so a state built any other way would not be one this gate can actually see.
+ state.arm(auth_tools.VerificationFailure.NO_CODE_TWICE, "get_verification_code")
+
+ should_hold = awaiting and not armed and not stale and not spent
+ message = await state.block_giveup("failed")
+
+ assert (message is not None) is should_hold
+ if should_hold:
+ assert "get_verification_code" in message
+ # It must send the model back to the TOOL, not onto the page: holding a `terminated` verdict
+ # keeps a run alive on a page the model declared blocked, and the hold must not widen that.
+ assert "do not act on the page" in message
+ # A completed verdict is a separate question the same primitive answers, and unspent budget is
+ # never a reason to refuse one.
+ assert await state.block_finish("completed") == await state.block_completion()
+ assert await state.block_finish("failed") == await state.block_giveup("failed")
+
+
+async def test_a_not_yet_answer_arms_the_giveup_gate_and_says_what_budget_is_left(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Untestable through the scripted loop test, which finishes regardless of what the tool returned.
+ # The old message gave no indication that any budget remained, so "call it again" was prose the
+ # task prompt's own "do not retry" instruction could simply outweigh. The STATUS deliberately
+ # stays `error`: it is never serialized to the model, and it is what routes a call that blocked
+ # for up to 120s through the loop's batch-poisoning check.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))
+ # Two slices far enough apart that the recorded spends differ at the record's own resolution;
+ # a shorter wait would let a constant "remaining" pass the tracking assertion below.
+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.2)
+
+ async def _not_yet(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:
+ await asyncio.sleep(max_wait_seconds)
+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")
+
+ monkeypatch.setattr(auth_tools, "resolve_otp_value", _not_yet)
+ state = auth_tools.VerificationState(task=_task(totp_verification_url="https://totp.example"))
+ tools, _ = auth_tools.build_auth_tools(_task(totp_verification_url="https://totp.example"), state=state)
+
+ with capture_logs() as logs:
+ first = await tools[0].handler({})
+ second = await tools[0].handler({})
+
+ assert first.status == "error" and second.status == "error"
+ # A retryable answer must not arm the terminal latch, or the completed-side gate would fire on it.
+ assert state.source_failed is False
+ assert await state.block_completion() is None
+ # It DOES arm the give-up gate: this is the state the run is in when it throws its budget away.
+ assert state.awaiting_code_since is not None
+ assert await state.block_giveup("failed") is not None
+
+ # The spend curve is only reconstructible today by differencing poll timestamps; the probe that
+ # sizes this fix in production needs it emitted directly, and v3-only (otp_service is v1's path).
+ slices = [e for e in logs if e.get("event") == "task_v3 verification poll slice timed out"]
+ assert len(slices) == 2
+ assert [e["tool"] for e in slices] == ["get_verification_code", "get_verification_code"]
+ assert slices[0]["polling_spent_seconds"] < slices[1]["polling_spent_seconds"]
+ for record in slices:
+ assert record["budget_seconds"] == 600.0
+ assert record["polling_spent_seconds"] + record["remaining_seconds"] == pytest.approx(600.0, abs=0.1)
+
+ # The message carries the remaining budget as a NUMBER, and the number is the budget MINUS THE
+ # SPEND AT THAT CALL -- checked against two different spends, so a constant (or a stale one that
+ # ignores the slice it just paid for) fails. An unquantified "you may retry" is prose competing
+ # with the "do not retry" prose in the task prompt; a number is not.
+ def _minutes(content: str) -> float:
+ return float(re.search(r"about ([\d.]+) minutes of polling budget remain", content).group(1))
+
+ for result, record in zip((first, second), slices):
+ assert _minutes(result.content) == pytest.approx((600.0 - record["polling_spent_seconds"]) / 60.0, abs=0.05)
+
+
+async def test_the_giveup_gate_records_both_the_hold_and_the_budget_it_let_go(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Gate 7's prod probe reads these records, so they have to exist and carry the residual. Today a
+ # give-up with budget left is invisible: nothing logs at a non-complete finish at all.
+ state = auth_tools.VerificationState(task=_task())
+ state.budget_seconds = 900.0
+ state.polling_spent_seconds = 120.0
+ state.awaiting_code_since = time.monotonic()
+
+ with capture_logs() as logs:
+ held = await state.block_giveup("failed")
+ # Same finish again with no polling in between: the model ignored the hold, so the next
+ # verdict stands and the residual it walks away from is recorded.
+ released = await state.block_giveup("failed")
+
+ assert held is not None and released is None
+ records = [e for e in logs if e.get("event") == "task_v3 verification give-up gate"]
+ assert [e["held"] for e in records] == [True, False]
+ assert [e["reason"] for e in records] == ["held", "unproductive"]
+ assert all(e["polling_spent_seconds"] == 120.0 and e["remaining_seconds"] == 780.0 for e in records)
+ assert [e["giveup_deferrals"] for e in records] == [1, 1]
+ # The probe splits held-vs-honored by finish verdict, and this record is the only place the two
+ # populations can be told apart -- without it both verdicts produce identical records.
+ assert all(e["finish_status"] == "failed" for e in records)
+ state.spend_at_last_giveup_deferral = None
+ with capture_logs() as terminated_logs:
+ assert await state.block_giveup("terminated") is not None
+ terminated = [e for e in terminated_logs if e.get("event") == "task_v3 verification give-up gate"]
+ assert [e["finish_status"] for e in terminated] == ["terminated"]
+
+
+async def test_a_delivered_code_disarms_the_giveup_gate(monkeypatch: pytest.MonkeyPatch) -> None:
+ # (b) in the negative controls: a run that got its code and later fails for an unrelated reason
+ # must end on its first verdict. The disarm has to happen at the delivery site, not be inferred.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))
+ state = auth_tools.VerificationState(task=_task(totp_verification_url="https://totp.example"))
+ state.awaiting_code_since = time.monotonic()
+ monkeypatch.setattr(
+ auth_tools, "resolve_otp_value", AsyncMock(return_value=OTPValue(value="123456", type=OTPType.TOTP))
+ )
+ tools, _ = auth_tools.build_auth_tools(_task(totp_verification_url="https://totp.example"), state=state)
+
+ with capture_logs() as logs:
+ result = await tools[0].handler({})
+
+ assert result.status == "ok" and "123456" in result.content
+ assert state.awaiting_code_since is None
+ assert await state.block_giveup("failed") is None
+ delivered = [e for e in logs if e.get("event") == "task_v3 verification value delivered"]
+ assert len(delivered) == 1 and delivered[0]["values_delivered"] == 1
+
+
+async def test_a_not_yet_after_a_terminal_failure_does_not_rearm_the_giveup_gate() -> None:
+ # The ordering that makes `source_failed` a real conjunct rather than a restatement of
+ # `awaiting_code_since is None`: the source can terminally fail (two empty answers, say) and a
+ # LATER call still land on the healthy-source "not yet" path, which sets the awaiting latch
+ # again. Clearing the latch inside `arm` would look equivalent and would hold this run, which is
+ # exactly wrong -- the source has already been judged.
+ state = auth_tools.VerificationState(task=_task())
+ state.budget_seconds = 900.0
+ state.polling_spent_seconds = 120.0
+ state.arm(auth_tools.VerificationFailure.NO_CODE_TWICE, "get_verification_code")
+ state.awaiting_code_since = time.monotonic()
+
+ assert state.source_failed is True
+ assert state.awaiting_code_since is not None
+ assert await state.block_giveup("failed") is None
+
+
+async def test_the_hold_names_the_tool_the_run_is_actually_waiting_on(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Both tools drain one source and share one budget, so the latch has to remember which one armed
+ # it. A magic-link run told to "call get_verification_code again" is being sent to the wrong tool.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))
+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.05)
+
+ async def _not_yet(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:
+ await asyncio.sleep(max_wait_seconds)
+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")
+
+ monkeypatch.setattr(auth_tools, "resolve_otp_value", _not_yet)
+ monkeypatch.setattr(otp_service, "has_credential_totp_candidate", lambda *_a, **_k: False)
+ task = _task(totp_verification_url="https://totp.example")
+ state = auth_tools.VerificationState(task=task)
+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock(), state=state)
+
+ link_tool = next(t for t in tools if t.name == "open_verification_link")
+ await link_tool.handler({})
+
+ assert state.awaiting_code_tool == "open_verification_link"
+ message = await state.block_giveup("failed")
+ assert message is not None and "open_verification_link" in message
+ assert "get_verification_code" not in message
+
+
+async def test_a_value_that_arrives_but_is_not_delivered_still_disarms_the_giveup_gate(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The latch means "the source has not produced anything yet", so it has to clear when the source
+ # produces something -- not when the model is handed something. Several paths reach a value without
+ # a delivery (a link when a code was asked for, a code when a link was, a link the browser could
+ # not open), and on all of them the run is done waiting on the SOURCE. Holding a give-up there
+ # would contradict the very answer the tool just returned, and would send the model back to
+ # re-poll for a value it already has.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))
+ task = _task(totp_verification_url="https://totp.example")
+ state = auth_tools.VerificationState(task=task)
+ state.awaiting_code_since = time.monotonic()
+ state.polling_spent_seconds = 120.0
+ monkeypatch.setattr(
+ auth_tools,
+ "resolve_otp_value",
+ AsyncMock(return_value=OTPValue(value="https://example.test/magic?token=abc", type=OTPType.MAGIC_LINK)),
+ )
+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock(), state=state)
+
+ result = await next(t for t in tools if t.name == "get_verification_code").handler({})
+
+ assert result.status == "error" and result.content == auth_tools._MAGIC_LINK_REDIRECT
+ assert state.values_delivered == 0 # nothing was handed to the model...
+ assert state.awaiting_code_since is None # ...but the source is no longer being waited on
+ assert await state.block_giveup("failed") is None
+
+
+async def test_a_not_yet_with_less_than_one_slice_left_says_so_instead_of_inviting_a_refused_call(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A slice can end with budget left but not enough to buy another one, because the tail is checked
+ # before the poll and consumed by it. Reporting "about 0.0 minutes remain ... call it again" would
+ # send the model into a call that answers _BUDGET_EXHAUSTED -- and the whole argument for putting a
+ # number in this message is that a number beats prose the task prompt can outweigh.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=0.55 / 60))
+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.2)
+ monkeypatch.setattr(auth_tools, "_MIN_SLICE_SECONDS", 0.5)
+
+ async def _not_yet(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:
+ await asyncio.sleep(max_wait_seconds)
+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")
+
+ monkeypatch.setattr(auth_tools, "resolve_otp_value", _not_yet)
+ task = _task(totp_verification_url="https://totp.example")
+ state = auth_tools.VerificationState(task=task)
+ tools, _ = auth_tools.build_auth_tools(task, state=state)
+
+ first = await tools[0].handler({})
+ second = await tools[0].handler({})
+
+ assert 0 < state.remaining_budget_seconds < auth_tools._MIN_SLICE_SECONDS
+ assert first.status == "error"
+ assert "the polling budget is now spent" in first.content
+ assert "minutes of polling budget remain" not in first.content
+ # ...and it must not invite the call it just said is pointless. The next call really is refused
+ # without polling, so an invitation costs a tool call and a turn to be told so -- and near the
+ # loop's limits that trades this truthful failure for a generic budget-exhausted exit.
+ assert "Do not call get_verification_code again" in first.content
+ assert "trigger it first" not in first.content
+ assert second.content == auth_tools._BUDGET_EXHAUSTED
+ # ...and it is a TERMINAL answer, so it arms. The budget is exhausted at this point whether or
+ # not the model makes the call that used to be the only thing that armed the latch -- and a model
+ # that obeys "do not call it again" must not thereby be free to claim the step completed.
+ assert state.source_failed is True
+ assert await state.block_completion() == auth_tools._COMPLETION_BLOCKED
+ # Giving up IS correct now, so the give-up gate releases rather than holding.
+ assert await state.block_giveup("failed") is None
+
+
+def test_the_giveup_headroom_reservation_still_covers_a_whole_poll_slice() -> None:
+ # The 180s is not a round number: it is one blocking poll slice plus the re-finish cycle the
+ # pre-existing holds already reserve. loop.py cannot import auth_tools to say so (auth_tools
+ # imports loop), and asserting only that it exceeds the 60s reservation still passes at 61 --
+ # which would under-fund the hold and convert an honest failure into budget_exhausted, the exact
+ # conversion the constant exists to prevent. This file imports both, so it can pin the relation.
+ from skyvern.forge.taskv3 import loop as loop_module
+
+ assert loop_module.VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS == (
+ auth_tools._PER_CALL_WAIT_SECONDS + loop_module.FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS
+ )
+
+
+async def test_the_spent_budget_answer_names_the_link_tool_on_a_link_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The point of building this message from `_tool_name` is that a magic-link run polls the OTHER
+ # tool, so telling it to stop calling `get_verification_code` names something it never called.
+ # Every other assertion on this message runs the code tool, so without this one the whole
+ # parameterisation can be replaced by the literal and the file stays green.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=20 / 60))
+ clock = [0.0]
+ monkeypatch.setattr(auth_tools, "time", SimpleNamespace(monotonic=lambda: clock[0]))
+ monkeypatch.setattr(otp_service, "has_credential_totp_candidate", lambda *_a, **_k: False)
+
+ async def _spends_12s(*_a: Any, **_k: Any) -> OTPValue | None:
+ clock[0] += 12.0
+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")
+
+ monkeypatch.setattr(auth_tools, "resolve_otp_value", _spends_12s)
+ task = _task(totp_verification_url="https://totp.example")
+ state = auth_tools.VerificationState(task=task)
+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock(), state=state)
+
+ result = await next(t for t in tools if t.name == "open_verification_link").handler({})
+
+ assert "no sign-in link available yet" in result.content
+ assert "Do not call open_verification_link again" in result.content
+ assert "get_verification_code" not in result.content
+ assert state.source_failed is True
+
+
+async def test_a_terminally_failed_source_is_recorded_as_such_not_as_a_spent_budget() -> None:
+ # The give-up record's `reason` feeds a production probe, and `block_giveup` checks `source_failed`
+ # ahead of both the recency window and the budget -- so once the source is armed, a give-up is
+ # labelled `source_failed` whether the budget ran out or the latch went stale. Nothing pinned that
+ # label, so the shift was invisible to the tests that do pin the held-vs-honored split.
+ state = auth_tools.VerificationState(task=_task())
+ state.budget_seconds = 900.0
+ state.polling_spent_seconds = 895.0
+ state.awaiting_code_since = time.monotonic()
+ state.arm(auth_tools.VerificationFailure.BUDGET_EXHAUSTED, "get_verification_code")
+
+ with capture_logs() as logs:
+ assert await state.block_giveup("failed") is None
+ record = next(e for e in logs if e.get("event") == "task_v3 verification give-up gate")
+ assert record["reason"] == "source_failed"
+ # Still an honored give-up, which is what the probe's held-vs-honored split actually counts.
+ assert record["held"] is False
+
+
[email protected]("link_run", [False, True], ids=["code_tool", "link_tool"])
+ "resolver_result",
+ ["returns_nothing", "source_errored", "lookup_raised"],
+)
+async def test_every_retryable_answer_arms_the_giveup_gate_not_just_a_not_yet(
+ monkeypatch: pytest.MonkeyPatch, resolver_result: str, link_run: bool
+) -> None:
+ # Four answers ask the model to call again: "not yet", a bare None, a source that errored, and a
+ # lookup that raised. Arming on only the first left the gate holding on one of them, so a run that
+ # gave up right after any of the other three discarded its budget exactly as it did before this
+ # change -- the same defect, on sibling paths.
+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))
+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.05)
+
+ async def _answer(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:
+ await asyncio.sleep(max_wait_seconds)
+ if resolver_result == "returns_nothing":
+ return None
+ if resolver_result == "source_errored":
+ raise FailedToGetTOTPVerificationCode(task_id="tsk_1", reason="http_status=500")
+ raise RuntimeError("resolver exploded")
+
+ monkeypatch.setattr(auth_tools, "resolve_otp_value", _answer)
+ if link_run:
+ monkeypatch.setattr(otp_service, "has_credential_totp_candidate", lambda *_a, **_k: False)
+ task = _task(totp_verification_url="https://totp.example")
+ state = auth_tools.VerificationState(task=task)
+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock() if link_run else None, state=state)
+ # Both tools drain one source, so the tool the latch records is load-bearing here and cannot be
+ # checked on the code run alone -- "get_verification_code" is also the field's default, so an
+ # assertion that only ever sees that value passes whether or not the tool is recorded at all.
+ expected_tool = "open_verification_link" if link_run else "get_verification_code"
+ handler = {t.name: t.handler for t in tools}[expected_tool]
+
+ first = await handler({})
+
+ # Retryable, not terminal: it invites another call and must not arm the completion refusal.
+ assert first.status == "error"
+ assert state.source_failed is False
+ assert await state.block_completion() is None
+ # ...and it holds a give-up, which is the whole point.
+ assert state.awaiting_code_since is not None
+ assert state.awaiting_code_tool == expected_tool
+ held = await state.block_giveup("failed")
+ assert held is not None and expected_tool in held
+ # The hold must name the right ARTIFACT as well as the right tool: telling a run that polled for
+ # a sign-in link that "the verification code has not arrived" contradicts both the page and the
+ # answer the tool just gave it. Asserted on the link run too, because "verification code" is the
+ # wording a hardcoded default would produce and the code run alone cannot tell them apart.
+ if link_run:
+ assert "sign-in link" in held and "verification code" not in held
+ else:
+ assert "verification code" in held and "sign-in link" not in held
+
+ # The SECOND occurrence is terminal, and then giving up is correct again.
+ await handler({})
+ assert state.source_failed is True
+ assert await state.block_giveup("failed") is None
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index da8100f91..8c2cbeb1c 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -1059,7 +1059,7 @@ async def test_engine_drops_download_hooks_in_page_free_mode(monkeypatch: pytest
async def blocker(_staged: frozenset[str]) -> str | None:
return "no download yet"
- async def verification_blocker() -> str | None:
+ async def verification_blocker(_status: str) -> str | None:
return "no code arrived"
await run_task_v3_agent_loop(
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 3d4214d6e..8776b1fa6 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -529,7 +529,7 @@ async def test_initial_navigation_dead_end_yields_canceled_when_cancelling() ->
@pytest.mark.asyncio
async def test_verification_blocker_refuses_completed_but_not_failed() -> None:
- async def _blocked() -> str | None:
+ async def _blocked(status: str) -> str | None:
return "verification never delivered a code"
tools = [make_finish_tool(verification_blocker=_blocked)]
@@ -8923,3 +8923,178 @@ async def test_a_renumbered_mark_does_not_alias_with_its_pre_renumber_history()
assert outcome.status == "failed"
assert len(touches) == 3
assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []
+
+
+def _awaiting_verification_state(*, spent: float = 120.0, budget: float = 900.0) -> Any:
+ """A VerificationState in the exact shape the bug produces: a healthy source that answered "not
+ yet" one 120s slice ago, with most of the 900s budget still unspent."""
+ from skyvern.forge.taskv3.auth_tools import VerificationState
+
+ state = VerificationState(
+ task=SimpleNamespace(task_id="tsk_1", organization_id="o_1", workflow_run_id=None) # type: ignore[arg-type]
+ )
+ state.budget_seconds = budget
+ state.polling_spent_seconds = spent
+ state.awaiting_code_since = time.monotonic()
+ return state
+
+
+def _spend_tool(state: Any, per_call_seconds: float) -> ToolSpec:
+ """Stands in for get_verification_code: the only thing the loop can see it do is draw the budget
+ down, which is exactly what the gate's monotone bound is keyed on."""
+
+ async def _handler(args: dict[str, Any]) -> ToolResult:
+ state.polling_spent_seconds += per_call_seconds
+ state.awaiting_code_since = time.monotonic()
+ return ToolResult.ok("no verification code available yet")
+
+ return ToolSpec(name="get_verification_code", description="poll", parameters={"type": "object"}, handler=_handler)
+
+
[email protected]("status", ["failed", "terminated"])
+async def test_giveup_with_unspent_verification_budget_is_deferred(status: str) -> None:
+ # The defect: a run that has spent one 120s slice of a 900s budget on a source that has not
+ # answered yet ends on the model's first non-complete verdict, throwing away ~13 minutes of
+ # polling it already owns. Both non-complete verdicts, not just `failed` -- `terminated` was
+ # gated by nothing at all.
+ state = _awaiting_verification_state()
+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]
+ script = [
+ [("finish", {"status": status, "reason": "the code never arrived"})],
+ [("finish", {"status": status, "reason": "the code never arrived"})],
+ ]
+ outcome, _ = await _run(script, tools)
+
+ finish_messages = [m["content"] for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "finish"]
+ assert any("polling budget remain" in m for m in finish_messages)
+ # The number is the point: an unquantified "keep waiting" is the prose the customer's own "do not
+ # retry" instruction competes with. 900 - 120 = 780s = 13.0 minutes.
+ assert any("13.0 minutes" in m for m in finish_messages)
+ assert state.giveup_deferrals == 1
+ # ...and the hold is a hold, not a trap: the verdict still lands.
+ assert outcome.status == status
+
+
+async def test_a_model_that_ignores_the_giveup_deferral_ends_on_its_second_verdict() -> None:
+ # Bounded termination for the model that refuses to re-poll. Asserted on the run's TERMINAL
+ # STATUS, not on a deferral counter: without the unproductive-deferral bound this run ping-pongs
+ # against the gate until the budget is exhausted, and the assertion that catches that is the run
+ # never reaching `failed`.
+ state = _awaiting_verification_state()
+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]
+ script = [[("finish", {"status": "failed", "reason": "no code"})] for _ in range(6)]
+ outcome, caller = await _run(script, tools)
+
+ assert outcome.status == "failed"
+ assert caller.calls == 2
+ assert state.giveup_deferrals == 1
+
+
+async def test_a_model_that_polls_minimally_ends_within_the_absolute_deferral_cap() -> None:
+ # The monotone-budget bound alone does not terminate: a slice may advance spend by as little as
+ # _MIN_SLICE_SECONDS, so 10s polls would buy ~90 deferrals out of a 900s budget. The absolute cap
+ # is what ends this run, and the assertion is again the terminal status -- a run that never
+ # reaches `failed` here is one the gate trapped.
+ from skyvern.forge.taskv3.auth_tools import _MAX_GIVEUP_DEFERRALS
+
+ state = _awaiting_verification_state(spent=0.0)
+ tools = [
+ _spend_tool(state, 10.0),
+ make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency()),
+ ]
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for _ in range(15):
+ script.append([("get_verification_code", {})])
+ script.append([("finish", {"status": "failed", "reason": "no code"})])
+ outcome, _ = await _run(script, tools, max_turns=60, max_tool_calls=200)
+
+ assert outcome.status == "failed"
+ assert state.giveup_deferrals == _MAX_GIVEUP_DEFERRALS
+ # The cap, not the budget, is what bound this run: 8 x 10s of polling leaves the budget nearly full.
+ assert state.remaining_budget_seconds > 800.0
+
+
+async def test_giveup_deferral_is_refused_without_the_deadline_headroom_to_fund_a_poll_slice() -> None:
+ # A deferral asks for a 120s BLOCKING poll on top of the re-finish cycle. Deferring with less
+ # than that leaves the run to die on the wall clock as budget_exhausted -- converting the
+ # model's honest verdict into an unmapped one, the exact conversion the headroom gates exist to
+ # prevent. The failure-evidence gate's 60s reservation is not enough to cover a poll slice.
+ from skyvern.forge.taskv3.loop import (
+ FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS,
+ VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS,
+ )
+
+ assert VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS > FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS
+ state = _awaiting_verification_state()
+ tools = [
+ make_finish_tool(
+ verification_blocker=state.block_finish,
+ activity=ActivityRecency(),
+ deadline_at=time.monotonic() + (VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS - 5.0),
+ )
+ ]
+ script = [[("finish", {"status": "failed", "reason": "no code"})]]
+ with capture_logs() as logs:
+ outcome, caller = await _run(script, tools)
+
+ assert outcome.status == "failed"
+ assert caller.calls == 1
+ assert state.giveup_deferrals == 0
+ # The gate is not consulted at all here, so it writes nothing. That is a KNOWN LIMIT of the
+ # probe, asserted rather than papered over: a give-up refused for want of hold budget is not
+ # directly counted, because only the gate knows whether the run was awaiting a code and this is
+ # the one case where the gate is not asked.
+ assert [e for e in logs if e.get("event") == "task_v3 verification give-up gate"] == []
+
+
+async def test_a_completed_verdict_is_never_held_for_unspent_verification_budget() -> None:
+ # The widened primitive answers all three verdicts from one branch, so the completed side has to
+ # be shown unchanged: unspent budget is a reason to keep WAITING, never a reason to refuse a
+ # success the model can see on the page.
+ state = _awaiting_verification_state()
+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]
+ script = [[("finish", {"status": "completed", "reason": "signed in"})]]
+ outcome, caller = await _run(script, tools)
+
+ assert outcome.status == "completed"
+ assert caller.calls == 1
+ assert state.giveup_deferrals == 0
+
+
+async def test_a_completion_blocked_run_still_escapes_via_failed_at_production_wiring() -> None:
+ # The escape hatch `test_verification_blocker_refuses_completed_but_not_failed` protects, pinned
+ # where it actually has to hold. That test passes a bare mock and no `activity`, so the widened
+ # primitive is consulted for `completed` only and the give-up side is never exercised -- but
+ # engine.py always passes an ActivityRecency, so the production configuration DOES reach the
+ # give-up side. What makes the escape safe there is not the missing accounting, it is that a
+ # terminally-failed source returns None from `block_giveup` by construction: giving up IS the
+ # correct answer once the source has been judged. Asserted against the real VerificationState,
+ # not a mock that cannot disagree with the gate it is standing in for.
+ from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
+
+ state = VerificationState(
+ task=SimpleNamespace(task_id="tsk_1", organization_id="o_1", workflow_run_id=None) # type: ignore[arg-type]
+ )
+ state.polling_spent_seconds = 120.0
+ state.awaiting_code_since = time.monotonic()
+ state.arm(VerificationFailure.NO_CODE_TWICE, "get_verification_code")
+
+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]
+ script = [
+ [("finish", {"status": "completed", "reason": "signed in"})],
+ [("finish", {"status": "failed", "reason": "the code never arrived"})],
+ ]
+ outcome, caller = await _run(script, tools)
+
+ finish_messages = [m["content"] for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "finish"]
+ assert any("the verification step never completed" in m for m in finish_messages)
+ # On its FIRST failed verdict, with no hold in between -- two LLM turns total.
+ assert outcome.status == "failed"
+ assert caller.calls == 2
+ assert state.giveup_deferrals == 0