Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Hold a give-up while verification-code polling budget is still unspent (SKY-15918) (#8494)
pedrohsdbd0615cbe5647
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 979796f36..44afecfe1 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2200,7 +2200,7 @@ class ForgeAgent:completion_blocker=completion_blocker,staged_downloads=staged_downloads,deadline_seconds=loop_deadline_seconds,- verification_blocker=verification_state.block_completion,+ verification_blocker=verification_state.block_finish,# Only for a bare task, where setup navigated this browser_state to task.url and the# status unambiguously belongs to the starting posting. A workflow block reuses the# browser_state across blocks, so its last status may be a prior block's — skip it therediff --git a/skyvern/forge/taskv3/auth_tools.py b/skyvern/forge/taskv3/auth_tools.pyindex 9669950db..28ff89dce 100644--- a/skyvern/forge/taskv3/auth_tools.py+++ b/skyvern/forge/taskv3/auth_tools.py@@ -43,6 +43,9 @@ from skyvern.webeye.navigation import revalidate_redirect_chainLOG = structlog.get_logger()+_CODE_TOOL = "get_verification_code"+_LINK_TOOL = "open_verification_link"+# One tool call polls at most this long, so a call made before the page has sent the code (or while# the source is still empty) returns and lets the model act instead of blocking the loop for the# whole budget.@@ -50,6 +53,14 @@ _PER_CALL_WAIT_SECONDS = 120.0# The poll loop fetches once per 10s sleep, so a slice shorter than that would never fetch; a tail# that small counts as spent._MIN_SLICE_SECONDS = 10.0+# How long a "not yet" answer keeps the give-up gate armed. Past it the run has moved on and a+# non-complete verdict is about something else; same idiom (and duration order) as the loop's+# FAILURE_EVIDENCE_WINDOW_TURNS recency, expressed in seconds because the budget is.+_NOT_YET_RECENCY_SECONDS = 180.0+# Termination is guaranteed by the budget shrinking under every productive deferral, but a slice may+# advance spend by as little as _MIN_SLICE_SECONDS, which would admit ~90 holds. 8 >= ceil(900/120),+# so this never binds on the healthy path.+_MAX_GIVEUP_DEFERRALS = 8# Shorter values are codes/flags (lang=en, v=2), not link secrets, and a real link secret is at# least this long; redacting the short ones would blank harmless text across the run's artifacts._MIN_REDACTED_QUERY_VALUE_CHARS = 16@@ -145,7 +156,7 @@ _LINK_GUIDANCE = (def _tool_name(expected_otp_type: OTPType) -> str:- return "open_verification_link" if expected_otp_type == OTPType.MAGIC_LINK else "get_verification_code"+ return _LINK_TOOL if expected_otp_type == OTPType.MAGIC_LINK else _CODE_TOOLdef _is_token_shaped(value: str) -> bool:@@ -263,9 +274,10 @@ async def _cookie_jar(page: Any) -> list[tuple[str, str, str, str]] | None:class VerificationFailure(StrEnum):"""Every condition that arms the completion refusal. This closes the VOCABULARY, which is what the- earlier hand-kept inventory could not do — it was missing six sites. Each member is written at- exactly one place today, so the covered set and the site set coincide; nothing here would stop a- future site from reusing a member, so that coincidence is a fact to re-check, not a guarantee."""+ earlier hand-kept inventory could not do — it was missing six sites. Members are not one-per-site:+ BUDGET_EXHAUSTED is written both where a slice spends the last of the budget and where a later+ call finds it already spent. Closing the vocabulary is what this buys; a member-to-site bijection+ is not, and nothing here depends on one."""BUDGET_EXHAUSTED = "budget_exhausted"NO_CODE_TWICE = "no_code_twice"@@ -285,7 +297,8 @@ class VerificationFailure(StrEnum):@dataclassclass VerificationState:- """Lets the finish tool refuse a completed verdict once the source terminally failed to deliver.+ """Lets the finish tool refuse a completed verdict once the source terminally failed to deliver, and+ hold a non-complete one while the run is still awaiting a code it has unspent budget for.The latch is armed by every terminal non-delivery answer (budget exhaustion, repeated lookupempty answers, refused or unopenable link), never by a retryable "not yet" or by a link failure@@ -298,16 +311,39 @@ class VerificationState:same fields (`skyvern/forge/agent.py`, "TOTP polling timed out — terminating task"); without thisthe v3 side latched silently and the failure could not be found in production at all."""- # Excluded from the repr: the bound `block_completion` handed to the loop would otherwise carry+ # Excluded from the repr: the bound `block_finish` handed to the loop would otherwise carry# the task's navigation payload into any message that formats the callback.task: Task = field(repr=False)values_delivered: int = field(default=0, init=False)+ # The shared polling budget the tools draw down, lifted onto the state so the finish gate can+ # read it. `build_auth_tools` advances `polling_spent_seconds` from its poll accounting.+ budget_seconds: float = field(+ default_factory=lambda: settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS * 60.0, init=False+ )+ polling_spent_seconds: float = field(default=0.0, init=False)+ # Monotonic timestamp of the last answer that invited another call, cleared where the source+ # produces a value. Written by the two retryable answers -- "not yet", and a first empty or+ # errored lookup -- so "the run is awaiting a code right now" is a tool-call fact rather than a+ # reading of the page. An arming does NOT clear it: `source_failed` is the conjunct that stops a+ # hold once the source is judged, and it has to keep working when a later poll re-arms this latch.+ awaiting_code_since: float | None = field(default=None, init=False)+ # Which tool the latch belongs to. Both tools drain one source and share the budget, so a+ # magic-link run must not be sent back to `get_verification_code`.+ awaiting_code_tool: str = field(default=_CODE_TOOL, init=False)+ giveup_deferrals: int = field(default=0, init=False)+ # Spend at the previous honored deferral. A finish that arrives with spend unchanged means the+ # model did not re-poll, so the next give-up stands rather than ping-ponging against the gate.+ spend_at_last_giveup_deferral: float | None = field(default=None, init=False, repr=False)_armings: list[VerificationFailure] = field(default_factory=list, init=False, repr=False)@propertydef source_failed(self) -> bool:return bool(self._armings)+ @property+ def remaining_budget_seconds(self) -> float:+ return self.budget_seconds - self.polling_spent_seconds+def arm(self, reason: VerificationFailure, tool: str) -> None:self._armings.append(reason)try:@@ -347,6 +383,79 @@ class VerificationState:return _COMPLETION_BLOCKEDreturn None+ async def block_giveup(self, status: str) -> str | None:+ """Defer a non-complete verdict exactly while the run is actively awaiting a verification+ code it has unspent budget for. The first four conditions operationalize that one invariant;+ the last two bound the hold, so a model that ignores the deferral still ends within two+ finishes and one that polls minimally within `_MAX_GIVEUP_DEFERRALS`."""+ remaining = self.remaining_budget_seconds+ reason: str | None = None+ if self.awaiting_code_since is None:+ reason = "not_awaiting"+ elif self.source_failed:+ reason = "source_failed"+ elif time.monotonic() - self.awaiting_code_since > _NOT_YET_RECENCY_SECONDS:+ reason = "stale"+ elif remaining < _MIN_SLICE_SECONDS:+ reason = "budget_spent"+ elif self.giveup_deferrals >= _MAX_GIVEUP_DEFERRALS:+ reason = "deferral_cap"+ elif (+ self.spend_at_last_giveup_deferral is not None+ and self.polling_spent_seconds <= self.spend_at_last_giveup_deferral+ ):+ reason = "unproductive"+ if reason is not None:+ self._record_giveup(held=False, reason=reason, remaining=remaining, status=status)+ return None+ self.giveup_deferrals += 1+ self.spend_at_last_giveup_deferral = self.polling_spent_seconds+ self._record_giveup(held=True, reason="held", remaining=remaining, status=status)+ # Both tools drain one source, so the artifact has to come off the same latch the tool does:+ # telling a run that polled for a sign-in link that "the code has not arrived" contradicts the+ # page and the answer the tool just gave it.+ subject = "sign-in link" if self.awaiting_code_tool == _LINK_TOOL else "verification code"+ return (+ f"about {remaining / 60.0:.1f} minutes of {subject} polling budget remain and the "+ f"{subject} has not arrived yet. Call {self.awaiting_code_tool} again to keep waiting for "+ "it. If the page never sent one you may re-request it; otherwise do not act on the page "+ "for this. Once the budget is spent the tool will say so; finish then and the verdict "+ "will stand."+ )++ async def block_finish(self, status: str) -> str | None:+ """The one verification gate the finish tool consults, for every verdict: a completed claim is+ refused once the source terminally failed, a non-complete one is deferred while budget the+ run is still waiting on remains."""+ if status == "completed":+ return await self.block_completion()+ return await self.block_giveup(status)++ def _record_giveup(self, *, held: bool, reason: str, remaining: float, status: str) -> None:+ # The pre-registered probe: today the unspent residual at a give-up is only reconstructible by+ # differencing poll timestamps, so it cannot be counted directly. `finish_status` is carried+ # because failed and terminated are separate populations in that read and the record is the+ # only place they can be told apart.+ if not held and self.polling_spent_seconds <= 0:+ return+ try:+ LOG.info(+ "task_v3 verification give-up gate",+ task_id=self.task.task_id,+ organization_id=self.task.organization_id,+ workflow_run_id=self.task.workflow_run_id,+ held=held,+ reason=reason,+ finish_status=status,+ polling_spent_seconds=round(self.polling_spent_seconds, 1),+ remaining_seconds=round(remaining, 1),+ budget_seconds=self.budget_seconds,+ giveup_deferrals=self.giveup_deferrals,+ )+ except Exception:+ # Narration must never cost the gate its verdict, for the same reason `arm` swallows.+ pass+def build_auth_tools(task: Task,@@ -357,8 +466,8 @@ def build_auth_tools("""Return (tools, system-prompt guidance) for verification handling, or ([], "") when the task hasno verification source configured (so the tools aren't offered needlessly). The link tool also needsa page to navigate, so a page-free run never gets it. `state`, if given, is mutated as the tools- poll and deliver values; pass its `block_completion` to `make_finish_tool` to gate a completed- verdict on it. A caller with no use for that gate can omit `state` entirely."""+ poll and deliver values; pass its `block_finish` to `make_finish_tool` to gate every finish verdict+ on it. A caller with no use for that gate can omit `state` entirely."""if state is None:state = VerificationState(task=task)offer_code_tool = has_otp_source(@@ -372,9 +481,11 @@ def build_auth_tools(# The model re-calls after every empty answer, so the cumulative polling across this task's calls# is capped at VERIFICATION_CODE_POLLING_TIMEOUT_MINS (the step engine's single poll window) and- # then the tools refuse with stop guidance. One budget for both tools: they drain one source.- budget_seconds = settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS * 60.0- polling_spent_seconds = 0.0+ # then the tools refuse with stop guidance. One budget for both tools: they drain one source; it+ # lives on `state` so the finish gate can see what is left of it. Re-derived here rather than+ # taken from the state's default because a caller may have built the state before the setting was+ # read, and the budget must be the one in force where it is actually drawn down.+ state.budget_seconds = settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS * 60.0# A value one tool resolved that the other tool owns (the webhook source does not filter by type).cached_otp_value: OTPValue | None = Nonefirst_poll_started_at: datetime | None = None@@ -396,15 +507,46 @@ def build_auth_tools(state.arm(reason, tool)return ToolResult.error(message, data=data)- def _not_yet(expected_otp_type: OTPType, detail: str) -> ToolResult:- if expected_otp_type == OTPType.MAGIC_LINK:- return ToolResult.error(- f"no sign-in link available yet ({detail}). If the page has not sent one, trigger it "- "first, then call open_verification_link again."+ def _not_yet(expected_otp_type: OTPType, detail: str, elapsed_seconds: float) -> ToolResult:+ """A healthy source that has not delivered yet: the run is still awaiting a code, and the+ message says how much budget is left to wait with — the old one gave no indication any+ remained. Stays a ToolResult.error: the status is never serialized to the model (the loop+ sends `content` only), and it is what routes a call that blocked for up to 120s through the+ batch-poisoning check, so flipping it would let a stale queued click dispatch against a page+ that moved during the wait. `elapsed_seconds` is this slice's wait, which the caller's+ `finally` has not yet added to the running spend."""+ state.awaiting_code_since = time.monotonic()+ state.awaiting_code_tool = _tool_name(expected_otp_type)+ spent = state.polling_spent_seconds + elapsed_seconds+ remaining = max(0.0, state.budget_seconds - spent)+ LOG.warning(+ "task_v3 verification poll slice timed out",+ task_id=task.task_id,+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ tool=_tool_name(expected_otp_type),+ polling_spent_seconds=round(spent, 1),+ remaining_seconds=round(remaining, 1),+ budget_seconds=state.budget_seconds,+ )+ tool = _tool_name(expected_otp_type)+ subject = "sign-in link" if expected_otp_type == OTPType.MAGIC_LINK else "verification code"+ if remaining < _MIN_SLICE_SECONDS:+ # Nothing left to buy: the next call refuses without polling, so inviting one would spend+ # a tool call and a turn to be told that -- and near the loop's own limits that trades a+ # truthful verification failure for a generic budget-exhausted exit. Terminal, so it arms:+ # the budget is exhausted here whether or not the model makes that pointless call, and+ # leaving the latch to be set by the call itself is what let a model that obeys the+ # instruction go on to claim the verification step completed.+ return _failed(+ f"no {subject} available yet ({detail}), and the polling budget is now spent. Do not "+ f"call {tool} again; finish the task as failed and say the {subject} never arrived.",+ reason=VerificationFailure.BUDGET_EXHAUSTED,+ tool=tool,)return ToolResult.error(- f"no verification code available yet ({detail}). If the page has not sent one, trigger it "- "first, then call get_verification_code again."+ f"no {subject} available yet ({detail}); about {remaining / 60.0:.1f} minutes of polling "+ f"budget remain. If the page has not sent one, trigger it first, then call {tool} again.")def _empty_answer(once: str, terminal: str, *, reason: VerificationFailure, tool: str) -> ToolResult:@@ -412,12 +554,18 @@ def build_auth_tools(empty_answer_streak += 1if empty_answer_streak > 1:return _failed(terminal, reason=reason, tool=tool)+ # Retryable, so it arms the give-up gate for the same reason `_not_yet` does: this answer+ # invites another call, and a run that gives up right after it has exactly as much unspent+ # budget as one that gave up after a "not yet". Arming only the "not yet" path would leave the+ # gate holding on one of the four answers that ask the model to wait.+ state.awaiting_code_since = time.monotonic()+ state.awaiting_code_tool = toolreturn ToolResult.error(once)async def _poll(expected_otp_type: OTPType) -> ToolResult | OTPValue:"""One budget-accounted polling slice. A ToolResult is the model-facing answer to return as-is."""- nonlocal polling_spent_seconds, first_poll_started_at, empty_answer_streak- remaining = budget_seconds - polling_spent_seconds+ nonlocal first_poll_started_at, empty_answer_streak+ remaining = state.remaining_budget_secondsif remaining < _MIN_SLICE_SECONDS:return _budget_exhausted(expected_otp_type)if first_poll_started_at is None:@@ -446,9 +594,14 @@ def build_auth_tools(tool=_tool_name(expected_otp_type),)empty_answer_streak = 0+ # Cleared here, not at delivery: several paths reach a value that is never handed to the+ # model -- a link when a code was asked for, a code when a link was, a link the browser+ # could not open -- and on all of them the run has stopped waiting on the SOURCE. Holding+ # a give-up there would contradict the very answer those paths return.+ state.awaiting_code_since = Nonereturn otp_valueexcept (NoTOTPVerificationCodeFound, FailedToGetTOTPVerificationCode) as exc:- if polling_spent_seconds + (time.monotonic() - started) >= budget_seconds:+ if state.polling_spent_seconds + (time.monotonic() - started) >= state.budget_seconds:return _budget_exhausted(expected_otp_type)detail = type(exc).__name__if isinstance(exc, FailedToGetTOTPVerificationCode):@@ -467,7 +620,7 @@ def build_auth_tools(empty_answer_streak = 0if exc.webhook_diagnostics:detail = f"{detail}: {exc.webhook_diagnostics}"- return _not_yet(expected_otp_type, detail)+ return _not_yet(expected_otp_type, detail, time.monotonic() - started)except Exception as exc:LOG.warning("task_v3 verification tool lookup failed",@@ -488,7 +641,7 @@ def build_auth_tools(tool=_tool_name(expected_otp_type),)finally:- polling_spent_seconds += time.monotonic() - started+ state.polling_spent_seconds += time.monotonic() - startedasync def _get_verification_code(args: dict[str, Any]) -> ToolResult:nonlocal cached_otp_value@@ -541,9 +694,22 @@ def build_auth_tools(if context is not None:# Redact the code from this task's artifacts/logs (task-scoped, so bare tasks are covered).context.register_secret_value(code)- state.values_delivered += 1+ _record_delivery("get_verification_code")return ToolResult.ok(f"verification_code: {code}")+ def _record_delivery(tool: str) -> None:+ state.values_delivered += 1+ LOG.info(+ "task_v3 verification value delivered",+ task_id=task.task_id,+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ tool=tool,+ values_delivered=state.values_delivered,+ polling_spent_seconds=round(state.polling_spent_seconds, 1),+ budget_seconds=state.budget_seconds,+ )+async def _open_verification_link(args: dict[str, Any]) -> ToolResult:nonlocal cached_otp_value, first_poll_started_atotp_value: OTPValue | None = cached_otp_value@@ -683,7 +849,7 @@ def build_auth_tools(tool="open_verification_link",data={"page_state_changed": True},)- state.values_delivered += 1+ _record_delivery("open_verification_link")return ToolResult.ok(_LINK_OPENED, data={"page_state_changed": True})tools: list[ToolSpec] = []diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 76db984f3..2d00b31fb 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -64,9 +64,12 @@ ToolHandler = Callable[[dict[str, Any]], Awaitable[ToolResult]]# receive the basenames tools staged into the downloads dir this run, to exclude from detection.CompletionProbe = Callable[[frozenset[str]], Awaitable[str | None]]CompletionBlocker = Callable[[frozenset[str]], Awaitable[str | None]]-# Consulted from finish(completed) like CompletionBlocker, but takes no arguments -- it gates on-# state the caller already tracks (e.g. a verification-code budget), not on staged downloads.-VerificationBlocker = Callable[[], Awaitable[str | None]]+# Consulted from finish for EVERY verdict, unlike CompletionBlocker: it takes the finish status and+# gates on state the caller already tracks (e.g. a verification-code budget), not on staged+# downloads. One callback, because a completed claim made on a blank verification step and a+# non-complete verdict given up with polling budget still unspent are the same concern seen from two+# sides; splitting them into two hooks would scatter it.+VerificationBlocker = Callable[[str], Awaitable[str | None]]@dataclass@@ -677,6 +680,10 @@ FAILURE_EVIDENCE_SETTLE_MAX_SECONDS = 8.0FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS = 60.0FAILURE_EVIDENCE_MIN_TOOL_CALLS = 3FAILURE_EVIDENCE_MIN_TURNS = 3+# A verification give-up deferral asks for a BLOCKING poll slice (auth_tools caps one at 120s) on top+# of that cycle, so the 60s above would let the gate convert an honest failure into the+# budget-exhausted end it exists to prevent. Not imported from auth_tools: that module imports this one.+VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS = 180.0def _is_enter_submit(tool_name: str, args: dict[str, Any]) -> bool:@@ -791,7 +798,11 @@ def _names_submit_control(tool_name: str, args: dict[str, Any], ok: bool) -> strreturn selector if isinstance(selector, str) and selector else None-def _has_hold_headroom(activity: ActivityRecency | None, deadline_at: float | None) -> bool:+def _has_hold_headroom(+ activity: ActivityRecency | None,+ deadline_at: float | None,+ min_deadline_headroom_seconds: float = FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS,+) -> bool:"""Whether a deferral has the budget to buy the re-verification turn it asks for.Without it the run ends budget_exhausted, which is unmapped and lands on failed -- turning an@@ -814,7 +825,7 @@ def _has_hold_headroom(activity: ActivityRecency | None, deadline_at: float | Noreturn Falseif activity.perception_stall_imminent:return False- if deadline_at is not None and deadline_at - time.monotonic() < FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS:+ if deadline_at is not None and deadline_at - time.monotonic() < min_deadline_headroom_seconds:return Falsereturn True@@ -1312,7 +1323,14 @@ def make_finish_tool(completed-side cap, not per verdict attempt) — a quiescence waitbounded by `failure_settle_max_seconds`, then a deferral asking the model to re-observe —because async submissions and captcha protocols otherwise produce false-negative verdicts.- terminated is never gated on either side.++ `verification_blocker` is the one gate consulted for EVERY verdict: it refuses a completed claim+ once the verification source terminally failed, and holds a failed OR terminated one while the+ run is still awaiting a code it has unspent polling budget for -- a give-up at one 120s slice of+ a 15-minute budget throws away minutes of waiting the run already owns. The hold is bounded by+ the callee (the budget shrinks under every productive hold) and refused here without the deadline+ headroom to fund the blocking poll slice it asks for. Apart from that gate, terminated is+ ungated on both sides.`pending_marker` reports the text the page still shows the control in `submit_watch` as inflight with, or None. A settled page is not a submitted one -- a submit frozen mid-flight is@@ -1426,17 +1444,36 @@ def make_finish_tool()if blocker_message:return ToolResult.error(blocker_message)- if status == "completed" and verification_blocker is not None:+ if verification_blocker is not None and (+ status == "completed"+ # A non-complete verdict is only ever HELD, never refused, so unlike the completed side it+ # must fund the retry it asks for -- and that retry is a blocking poll slice, not just a+ # re-observe. Absent the accounting to check that (no `activity`), the hold is not+ # justifiable and the verdict stands.+ or (+ activity is not None+ and _has_hold_headroom(+ activity,+ deadline_at,+ min_deadline_headroom_seconds=VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS,+ )+ )+ ):try:- verification_message = await verification_blocker()+ verification_message = await verification_blocker(status)except Exception:- # Fail closed: an exception here must not let a blank verification step read as done.- LOG.warning("taskv3 verification_blocker failed; failing closed", exc_info=True)- return ToolResult.error(- "Could not verify that the verification-code step completed cleanly; retry "- "finish(status=completed) once verified, or finish with status=failed or "- "status=terminated."- )+ if status == "completed":+ # Fail closed: an exception here must not let a blank verification step read as done.+ LOG.warning("taskv3 verification_blocker failed; failing closed", exc_info=True)+ return ToolResult.error(+ "Could not verify that the verification-code step completed cleanly; retry "+ "finish(status=completed) once verified, or finish with status=failed or "+ "status=terminated."+ )+ # Fail open on the give-up side: the opposite verdict. A broken gate must not trap a+ # run that wants to end.+ LOG.warning("taskv3 verification_blocker failed; honoring the verdict", exc_info=True)+ verification_message = Noneif verification_message:return ToolResult.error(verification_message)if (diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex c599fe1b9..86063026d 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -3157,9 +3157,9 @@ async def test_execute_task_v3_hands_the_loop_the_live_verification_gate(monkeypblocker = loop_mock.await_args.kwargs["verification_blocker"]assert blocker is not Noneassert len(seen_states) == 1 and blocker.__self__ is seen_states[0]- assert await blocker() is None+ assert await blocker("completed") is Noneseen_states[0].arm(VerificationFailure.NO_CODE_TWICE, "get_verification_code")- assert await blocker() is not None+ assert await blocker("completed") is not None@pytest.mark.asynciodiff --git a/tests/unit/test_taskv3_auth_tools.py b/tests/unit/test_taskv3_auth_tools.pyindex 4a4cd2109..49b11f500 100644--- a/tests/unit/test_taskv3_auth_tools.py+++ b/tests/unit/test_taskv3_auth_tools.py@@ -4,6 +4,8 @@ from __future__ import annotationsimport asyncioimport logging+import re+import timefrom datetime import datetime, timezonefrom types import SimpleNamespacefrom typing import Any@@ -1560,3 +1562,392 @@ def test_block_credential_parameter_keys_script_mode_returns_none(monkeypatch: passert agent_module.block_credential_parameter_keys(block, "wr_test") is Nonewith skyvern_context.scoped(SkyvernContext()):assert agent_module.block_credential_parameter_keys(NS(parameters=[]), "wr_test") == []++[email protected]("awaiting", [True, False], ids=["awaiting", "not_awaiting"])[email protected]("armed", [True, False], ids=["source_failed", "source_healthy"])[email protected]("stale", [True, False], ids=["stale", "recent"])[email protected]("spent", [True, False], ids=["budget_spent", "budget_left"])+async def test_giveup_gate_holds_exactly_while_awaiting_a_code_it_has_budget_for(+ awaiting: bool, armed: bool, stale: bool, spent: bool+) -> None:+ # The invariant in one sentence: a non-complete finish is held exactly while the run is actively+ # awaiting a verification code it has purchased-but-unspent budget for. The four conjuncts are+ # that sentence's operationalization, so this asserts the BICONDITIONAL over the whole+ # (awaiting, armed, recency, spend) space rather than four example negative controls -- an+ # enumerated field list cannot close a class, and a conjunct silently dropped from the gate would+ # leave every example that does not exercise it green.+ state = auth_tools.VerificationState(task=_task())+ state.budget_seconds = 900.0+ state.polling_spent_seconds = 900.0 - (auth_tools._MIN_SLICE_SECONDS / 2 if spent else 600.0)+ if awaiting:+ state.awaiting_code_since = time.monotonic() - (auth_tools._NOT_YET_RECENCY_SECONDS + 1 if stale else 0.0)+ if armed:+ # Through `arm`, not by poking the latch: it is the only way the latch is reachable in+ # production, so a state built any other way would not be one this gate can actually see.+ state.arm(auth_tools.VerificationFailure.NO_CODE_TWICE, "get_verification_code")++ should_hold = awaiting and not armed and not stale and not spent+ message = await state.block_giveup("failed")++ assert (message is not None) is should_hold+ if should_hold:+ assert "get_verification_code" in message+ # It must send the model back to the TOOL, not onto the page: holding a `terminated` verdict+ # keeps a run alive on a page the model declared blocked, and the hold must not widen that.+ assert "do not act on the page" in message+ # A completed verdict is a separate question the same primitive answers, and unspent budget is+ # never a reason to refuse one.+ assert await state.block_finish("completed") == await state.block_completion()+ assert await state.block_finish("failed") == await state.block_giveup("failed")+++async def test_a_not_yet_answer_arms_the_giveup_gate_and_says_what_budget_is_left(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Untestable through the scripted loop test, which finishes regardless of what the tool returned.+ # The old message gave no indication that any budget remained, so "call it again" was prose the+ # task prompt's own "do not retry" instruction could simply outweigh. The STATUS deliberately+ # stays `error`: it is never serialized to the model, and it is what routes a call that blocked+ # for up to 120s through the loop's batch-poisoning check.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))+ # Two slices far enough apart that the recorded spends differ at the record's own resolution;+ # a shorter wait would let a constant "remaining" pass the tracking assertion below.+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.2)++ async def _not_yet(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:+ await asyncio.sleep(max_wait_seconds)+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")++ monkeypatch.setattr(auth_tools, "resolve_otp_value", _not_yet)+ state = auth_tools.VerificationState(task=_task(totp_verification_url="https://totp.example"))+ tools, _ = auth_tools.build_auth_tools(_task(totp_verification_url="https://totp.example"), state=state)++ with capture_logs() as logs:+ first = await tools[0].handler({})+ second = await tools[0].handler({})++ assert first.status == "error" and second.status == "error"+ # A retryable answer must not arm the terminal latch, or the completed-side gate would fire on it.+ assert state.source_failed is False+ assert await state.block_completion() is None+ # It DOES arm the give-up gate: this is the state the run is in when it throws its budget away.+ assert state.awaiting_code_since is not None+ assert await state.block_giveup("failed") is not None++ # The spend curve is only reconstructible today by differencing poll timestamps; the probe that+ # sizes this fix in production needs it emitted directly, and v3-only (otp_service is v1's path).+ slices = [e for e in logs if e.get("event") == "task_v3 verification poll slice timed out"]+ assert len(slices) == 2+ assert [e["tool"] for e in slices] == ["get_verification_code", "get_verification_code"]+ assert slices[0]["polling_spent_seconds"] < slices[1]["polling_spent_seconds"]+ for record in slices:+ assert record["budget_seconds"] == 600.0+ assert record["polling_spent_seconds"] + record["remaining_seconds"] == pytest.approx(600.0, abs=0.1)++ # The message carries the remaining budget as a NUMBER, and the number is the budget MINUS THE+ # SPEND AT THAT CALL -- checked against two different spends, so a constant (or a stale one that+ # ignores the slice it just paid for) fails. An unquantified "you may retry" is prose competing+ # with the "do not retry" prose in the task prompt; a number is not.+ def _minutes(content: str) -> float:+ return float(re.search(r"about ([\d.]+) minutes of polling budget remain", content).group(1))++ for result, record in zip((first, second), slices):+ assert _minutes(result.content) == pytest.approx((600.0 - record["polling_spent_seconds"]) / 60.0, abs=0.05)+++async def test_the_giveup_gate_records_both_the_hold_and_the_budget_it_let_go(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Gate 7's prod probe reads these records, so they have to exist and carry the residual. Today a+ # give-up with budget left is invisible: nothing logs at a non-complete finish at all.+ state = auth_tools.VerificationState(task=_task())+ state.budget_seconds = 900.0+ state.polling_spent_seconds = 120.0+ state.awaiting_code_since = time.monotonic()++ with capture_logs() as logs:+ held = await state.block_giveup("failed")+ # Same finish again with no polling in between: the model ignored the hold, so the next+ # verdict stands and the residual it walks away from is recorded.+ released = await state.block_giveup("failed")++ assert held is not None and released is None+ records = [e for e in logs if e.get("event") == "task_v3 verification give-up gate"]+ assert [e["held"] for e in records] == [True, False]+ assert [e["reason"] for e in records] == ["held", "unproductive"]+ assert all(e["polling_spent_seconds"] == 120.0 and e["remaining_seconds"] == 780.0 for e in records)+ assert [e["giveup_deferrals"] for e in records] == [1, 1]+ # The probe splits held-vs-honored by finish verdict, and this record is the only place the two+ # populations can be told apart -- without it both verdicts produce identical records.+ assert all(e["finish_status"] == "failed" for e in records)+ state.spend_at_last_giveup_deferral = None+ with capture_logs() as terminated_logs:+ assert await state.block_giveup("terminated") is not None+ terminated = [e for e in terminated_logs if e.get("event") == "task_v3 verification give-up gate"]+ assert [e["finish_status"] for e in terminated] == ["terminated"]+++async def test_a_delivered_code_disarms_the_giveup_gate(monkeypatch: pytest.MonkeyPatch) -> None:+ # (b) in the negative controls: a run that got its code and later fails for an unrelated reason+ # must end on its first verdict. The disarm has to happen at the delivery site, not be inferred.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))+ state = auth_tools.VerificationState(task=_task(totp_verification_url="https://totp.example"))+ state.awaiting_code_since = time.monotonic()+ monkeypatch.setattr(+ auth_tools, "resolve_otp_value", AsyncMock(return_value=OTPValue(value="123456", type=OTPType.TOTP))+ )+ tools, _ = auth_tools.build_auth_tools(_task(totp_verification_url="https://totp.example"), state=state)++ with capture_logs() as logs:+ result = await tools[0].handler({})++ assert result.status == "ok" and "123456" in result.content+ assert state.awaiting_code_since is None+ assert await state.block_giveup("failed") is None+ delivered = [e for e in logs if e.get("event") == "task_v3 verification value delivered"]+ assert len(delivered) == 1 and delivered[0]["values_delivered"] == 1+++async def test_a_not_yet_after_a_terminal_failure_does_not_rearm_the_giveup_gate() -> None:+ # The ordering that makes `source_failed` a real conjunct rather than a restatement of+ # `awaiting_code_since is None`: the source can terminally fail (two empty answers, say) and a+ # LATER call still land on the healthy-source "not yet" path, which sets the awaiting latch+ # again. Clearing the latch inside `arm` would look equivalent and would hold this run, which is+ # exactly wrong -- the source has already been judged.+ state = auth_tools.VerificationState(task=_task())+ state.budget_seconds = 900.0+ state.polling_spent_seconds = 120.0+ state.arm(auth_tools.VerificationFailure.NO_CODE_TWICE, "get_verification_code")+ state.awaiting_code_since = time.monotonic()++ assert state.source_failed is True+ assert state.awaiting_code_since is not None+ assert await state.block_giveup("failed") is None+++async def test_the_hold_names_the_tool_the_run_is_actually_waiting_on(monkeypatch: pytest.MonkeyPatch) -> None:+ # Both tools drain one source and share one budget, so the latch has to remember which one armed+ # it. A magic-link run told to "call get_verification_code again" is being sent to the wrong tool.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.05)++ async def _not_yet(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:+ await asyncio.sleep(max_wait_seconds)+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")++ monkeypatch.setattr(auth_tools, "resolve_otp_value", _not_yet)+ monkeypatch.setattr(otp_service, "has_credential_totp_candidate", lambda *_a, **_k: False)+ task = _task(totp_verification_url="https://totp.example")+ state = auth_tools.VerificationState(task=task)+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock(), state=state)++ link_tool = next(t for t in tools if t.name == "open_verification_link")+ await link_tool.handler({})++ assert state.awaiting_code_tool == "open_verification_link"+ message = await state.block_giveup("failed")+ assert message is not None and "open_verification_link" in message+ assert "get_verification_code" not in message+++async def test_a_value_that_arrives_but_is_not_delivered_still_disarms_the_giveup_gate(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # The latch means "the source has not produced anything yet", so it has to clear when the source+ # produces something -- not when the model is handed something. Several paths reach a value without+ # a delivery (a link when a code was asked for, a code when a link was, a link the browser could+ # not open), and on all of them the run is done waiting on the SOURCE. Holding a give-up there+ # would contradict the very answer the tool just returned, and would send the model back to+ # re-poll for a value it already has.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))+ task = _task(totp_verification_url="https://totp.example")+ state = auth_tools.VerificationState(task=task)+ state.awaiting_code_since = time.monotonic()+ state.polling_spent_seconds = 120.0+ monkeypatch.setattr(+ auth_tools,+ "resolve_otp_value",+ AsyncMock(return_value=OTPValue(value="https://example.test/magic?token=abc", type=OTPType.MAGIC_LINK)),+ )+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock(), state=state)++ result = await next(t for t in tools if t.name == "get_verification_code").handler({})++ assert result.status == "error" and result.content == auth_tools._MAGIC_LINK_REDIRECT+ assert state.values_delivered == 0 # nothing was handed to the model...+ assert state.awaiting_code_since is None # ...but the source is no longer being waited on+ assert await state.block_giveup("failed") is None+++async def test_a_not_yet_with_less_than_one_slice_left_says_so_instead_of_inviting_a_refused_call(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # A slice can end with budget left but not enough to buy another one, because the tail is checked+ # before the poll and consumed by it. Reporting "about 0.0 minutes remain ... call it again" would+ # send the model into a call that answers _BUDGET_EXHAUSTED -- and the whole argument for putting a+ # number in this message is that a number beats prose the task prompt can outweigh.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=0.55 / 60))+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.2)+ monkeypatch.setattr(auth_tools, "_MIN_SLICE_SECONDS", 0.5)++ async def _not_yet(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:+ await asyncio.sleep(max_wait_seconds)+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")++ monkeypatch.setattr(auth_tools, "resolve_otp_value", _not_yet)+ task = _task(totp_verification_url="https://totp.example")+ state = auth_tools.VerificationState(task=task)+ tools, _ = auth_tools.build_auth_tools(task, state=state)++ first = await tools[0].handler({})+ second = await tools[0].handler({})++ assert 0 < state.remaining_budget_seconds < auth_tools._MIN_SLICE_SECONDS+ assert first.status == "error"+ assert "the polling budget is now spent" in first.content+ assert "minutes of polling budget remain" not in first.content+ # ...and it must not invite the call it just said is pointless. The next call really is refused+ # without polling, so an invitation costs a tool call and a turn to be told so -- and near the+ # loop's limits that trades this truthful failure for a generic budget-exhausted exit.+ assert "Do not call get_verification_code again" in first.content+ assert "trigger it first" not in first.content+ assert second.content == auth_tools._BUDGET_EXHAUSTED+ # ...and it is a TERMINAL answer, so it arms. The budget is exhausted at this point whether or+ # not the model makes the call that used to be the only thing that armed the latch -- and a model+ # that obeys "do not call it again" must not thereby be free to claim the step completed.+ assert state.source_failed is True+ assert await state.block_completion() == auth_tools._COMPLETION_BLOCKED+ # Giving up IS correct now, so the give-up gate releases rather than holding.+ assert await state.block_giveup("failed") is None+++def test_the_giveup_headroom_reservation_still_covers_a_whole_poll_slice() -> None:+ # The 180s is not a round number: it is one blocking poll slice plus the re-finish cycle the+ # pre-existing holds already reserve. loop.py cannot import auth_tools to say so (auth_tools+ # imports loop), and asserting only that it exceeds the 60s reservation still passes at 61 --+ # which would under-fund the hold and convert an honest failure into budget_exhausted, the exact+ # conversion the constant exists to prevent. This file imports both, so it can pin the relation.+ from skyvern.forge.taskv3 import loop as loop_module++ assert loop_module.VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS == (+ auth_tools._PER_CALL_WAIT_SECONDS + loop_module.FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS+ )+++async def test_the_spent_budget_answer_names_the_link_tool_on_a_link_run(monkeypatch: pytest.MonkeyPatch) -> None:+ # The point of building this message from `_tool_name` is that a magic-link run polls the OTHER+ # tool, so telling it to stop calling `get_verification_code` names something it never called.+ # Every other assertion on this message runs the code tool, so without this one the whole+ # parameterisation can be replaced by the literal and the file stays green.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=20 / 60))+ clock = [0.0]+ monkeypatch.setattr(auth_tools, "time", SimpleNamespace(monotonic=lambda: clock[0]))+ monkeypatch.setattr(otp_service, "has_credential_totp_candidate", lambda *_a, **_k: False)++ async def _spends_12s(*_a: Any, **_k: Any) -> OTPValue | None:+ clock[0] += 12.0+ raise NoTOTPVerificationCodeFound(task_id="tsk_1")++ monkeypatch.setattr(auth_tools, "resolve_otp_value", _spends_12s)+ task = _task(totp_verification_url="https://totp.example")+ state = auth_tools.VerificationState(task=task)+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock(), state=state)++ result = await next(t for t in tools if t.name == "open_verification_link").handler({})++ assert "no sign-in link available yet" in result.content+ assert "Do not call open_verification_link again" in result.content+ assert "get_verification_code" not in result.content+ assert state.source_failed is True+++async def test_a_terminally_failed_source_is_recorded_as_such_not_as_a_spent_budget() -> None:+ # The give-up record's `reason` feeds a production probe, and `block_giveup` checks `source_failed`+ # ahead of both the recency window and the budget -- so once the source is armed, a give-up is+ # labelled `source_failed` whether the budget ran out or the latch went stale. Nothing pinned that+ # label, so the shift was invisible to the tests that do pin the held-vs-honored split.+ state = auth_tools.VerificationState(task=_task())+ state.budget_seconds = 900.0+ state.polling_spent_seconds = 895.0+ state.awaiting_code_since = time.monotonic()+ state.arm(auth_tools.VerificationFailure.BUDGET_EXHAUSTED, "get_verification_code")++ with capture_logs() as logs:+ assert await state.block_giveup("failed") is None+ record = next(e for e in logs if e.get("event") == "task_v3 verification give-up gate")+ assert record["reason"] == "source_failed"+ # Still an honored give-up, which is what the probe's held-vs-honored split actually counts.+ assert record["held"] is False++[email protected]("link_run", [False, True], ids=["code_tool", "link_tool"])+ "resolver_result",+ ["returns_nothing", "source_errored", "lookup_raised"],+)+async def test_every_retryable_answer_arms_the_giveup_gate_not_just_a_not_yet(+ monkeypatch: pytest.MonkeyPatch, resolver_result: str, link_run: bool+) -> None:+ # Four answers ask the model to call again: "not yet", a bare None, a source that errored, and a+ # lookup that raised. Arming on only the first left the gate holding on one of them, so a run that+ # gave up right after any of the other three discarded its budget exactly as it did before this+ # change -- the same defect, on sibling paths.+ monkeypatch.setattr(auth_tools, "settings", SimpleNamespace(VERIFICATION_CODE_POLLING_TIMEOUT_MINS=10))+ monkeypatch.setattr(auth_tools, "_PER_CALL_WAIT_SECONDS", 0.05)++ async def _answer(*_a: Any, max_wait_seconds: float, **_k: Any) -> OTPValue | None:+ await asyncio.sleep(max_wait_seconds)+ if resolver_result == "returns_nothing":+ return None+ if resolver_result == "source_errored":+ raise FailedToGetTOTPVerificationCode(task_id="tsk_1", reason="http_status=500")+ raise RuntimeError("resolver exploded")++ monkeypatch.setattr(auth_tools, "resolve_otp_value", _answer)+ if link_run:+ monkeypatch.setattr(otp_service, "has_credential_totp_candidate", lambda *_a, **_k: False)+ task = _task(totp_verification_url="https://totp.example")+ state = auth_tools.VerificationState(task=task)+ tools, _ = auth_tools.build_auth_tools(task, page_provider=AsyncMock() if link_run else None, state=state)+ # Both tools drain one source, so the tool the latch records is load-bearing here and cannot be+ # checked on the code run alone -- "get_verification_code" is also the field's default, so an+ # assertion that only ever sees that value passes whether or not the tool is recorded at all.+ expected_tool = "open_verification_link" if link_run else "get_verification_code"+ handler = {t.name: t.handler for t in tools}[expected_tool]++ first = await handler({})++ # Retryable, not terminal: it invites another call and must not arm the completion refusal.+ assert first.status == "error"+ assert state.source_failed is False+ assert await state.block_completion() is None+ # ...and it holds a give-up, which is the whole point.+ assert state.awaiting_code_since is not None+ assert state.awaiting_code_tool == expected_tool+ held = await state.block_giveup("failed")+ assert held is not None and expected_tool in held+ # The hold must name the right ARTIFACT as well as the right tool: telling a run that polled for+ # a sign-in link that "the verification code has not arrived" contradicts both the page and the+ # answer the tool just gave it. Asserted on the link run too, because "verification code" is the+ # wording a hardcoded default would produce and the code run alone cannot tell them apart.+ if link_run:+ assert "sign-in link" in held and "verification code" not in held+ else:+ assert "verification code" in held and "sign-in link" not in held++ # The SECOND occurrence is terminal, and then giving up is correct again.+ await handler({})+ assert state.source_failed is True+ assert await state.block_giveup("failed") is Nonediff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex da8100f91..8c2cbeb1c 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -1059,7 +1059,7 @@ async def test_engine_drops_download_hooks_in_page_free_mode(monkeypatch: pytestasync def blocker(_staged: frozenset[str]) -> str | None:return "no download yet"- async def verification_blocker() -> str | None:+ async def verification_blocker(_status: str) -> str | None:return "no code arrived"await run_task_v3_agent_loop(diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 3d4214d6e..8776b1fa6 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -529,7 +529,7 @@ async def test_initial_navigation_dead_end_yields_canceled_when_cancelling() ->@pytest.mark.asyncioasync def test_verification_blocker_refuses_completed_but_not_failed() -> None:- async def _blocked() -> str | None:+ async def _blocked(status: str) -> str | None:return "verification never delivered a code"tools = [make_finish_tool(verification_blocker=_blocked)]@@ -8923,3 +8923,178 @@ async def test_a_renumbered_mark_does_not_alias_with_its_pre_renumber_history()assert outcome.status == "failed"assert len(touches) == 3assert [e for e in logs if e.get("event") == CANONICAL_LOOP_EVENT] == []+++def _awaiting_verification_state(*, spent: float = 120.0, budget: float = 900.0) -> Any:+ """A VerificationState in the exact shape the bug produces: a healthy source that answered "not+ yet" one 120s slice ago, with most of the 900s budget still unspent."""+ from skyvern.forge.taskv3.auth_tools import VerificationState++ state = VerificationState(+ task=SimpleNamespace(task_id="tsk_1", organization_id="o_1", workflow_run_id=None) # type: ignore[arg-type]+ )+ state.budget_seconds = budget+ state.polling_spent_seconds = spent+ state.awaiting_code_since = time.monotonic()+ return state+++def _spend_tool(state: Any, per_call_seconds: float) -> ToolSpec:+ """Stands in for get_verification_code: the only thing the loop can see it do is draw the budget+ down, which is exactly what the gate's monotone bound is keyed on."""++ async def _handler(args: dict[str, Any]) -> ToolResult:+ state.polling_spent_seconds += per_call_seconds+ state.awaiting_code_since = time.monotonic()+ return ToolResult.ok("no verification code available yet")++ return ToolSpec(name="get_verification_code", description="poll", parameters={"type": "object"}, handler=_handler)++[email protected]("status", ["failed", "terminated"])+async def test_giveup_with_unspent_verification_budget_is_deferred(status: str) -> None:+ # The defect: a run that has spent one 120s slice of a 900s budget on a source that has not+ # answered yet ends on the model's first non-complete verdict, throwing away ~13 minutes of+ # polling it already owns. Both non-complete verdicts, not just `failed` -- `terminated` was+ # gated by nothing at all.+ state = _awaiting_verification_state()+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]+ script = [+ [("finish", {"status": status, "reason": "the code never arrived"})],+ [("finish", {"status": status, "reason": "the code never arrived"})],+ ]+ outcome, _ = await _run(script, tools)++ finish_messages = [m["content"] for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "finish"]+ assert any("polling budget remain" in m for m in finish_messages)+ # The number is the point: an unquantified "keep waiting" is the prose the customer's own "do not+ # retry" instruction competes with. 900 - 120 = 780s = 13.0 minutes.+ assert any("13.0 minutes" in m for m in finish_messages)+ assert state.giveup_deferrals == 1+ # ...and the hold is a hold, not a trap: the verdict still lands.+ assert outcome.status == status+++async def test_a_model_that_ignores_the_giveup_deferral_ends_on_its_second_verdict() -> None:+ # Bounded termination for the model that refuses to re-poll. Asserted on the run's TERMINAL+ # STATUS, not on a deferral counter: without the unproductive-deferral bound this run ping-pongs+ # against the gate until the budget is exhausted, and the assertion that catches that is the run+ # never reaching `failed`.+ state = _awaiting_verification_state()+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]+ script = [[("finish", {"status": "failed", "reason": "no code"})] for _ in range(6)]+ outcome, caller = await _run(script, tools)++ assert outcome.status == "failed"+ assert caller.calls == 2+ assert state.giveup_deferrals == 1+++async def test_a_model_that_polls_minimally_ends_within_the_absolute_deferral_cap() -> None:+ # The monotone-budget bound alone does not terminate: a slice may advance spend by as little as+ # _MIN_SLICE_SECONDS, so 10s polls would buy ~90 deferrals out of a 900s budget. The absolute cap+ # is what ends this run, and the assertion is again the terminal status -- a run that never+ # reaches `failed` here is one the gate trapped.+ from skyvern.forge.taskv3.auth_tools import _MAX_GIVEUP_DEFERRALS++ state = _awaiting_verification_state(spent=0.0)+ tools = [+ _spend_tool(state, 10.0),+ make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency()),+ ]+ script: list[list[tuple[str, dict[str, Any]]]] = []+ for _ in range(15):+ script.append([("get_verification_code", {})])+ script.append([("finish", {"status": "failed", "reason": "no code"})])+ outcome, _ = await _run(script, tools, max_turns=60, max_tool_calls=200)++ assert outcome.status == "failed"+ assert state.giveup_deferrals == _MAX_GIVEUP_DEFERRALS+ # The cap, not the budget, is what bound this run: 8 x 10s of polling leaves the budget nearly full.+ assert state.remaining_budget_seconds > 800.0+++async def test_giveup_deferral_is_refused_without_the_deadline_headroom_to_fund_a_poll_slice() -> None:+ # A deferral asks for a 120s BLOCKING poll on top of the re-finish cycle. Deferring with less+ # than that leaves the run to die on the wall clock as budget_exhausted -- converting the+ # model's honest verdict into an unmapped one, the exact conversion the headroom gates exist to+ # prevent. The failure-evidence gate's 60s reservation is not enough to cover a poll slice.+ from skyvern.forge.taskv3.loop import (+ FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS,+ VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS,+ )++ assert VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS > FAILURE_EVIDENCE_MIN_DEADLINE_HEADROOM_SECONDS+ state = _awaiting_verification_state()+ tools = [+ make_finish_tool(+ verification_blocker=state.block_finish,+ activity=ActivityRecency(),+ deadline_at=time.monotonic() + (VERIFICATION_GIVEUP_MIN_DEADLINE_HEADROOM_SECONDS - 5.0),+ )+ ]+ script = [[("finish", {"status": "failed", "reason": "no code"})]]+ with capture_logs() as logs:+ outcome, caller = await _run(script, tools)++ assert outcome.status == "failed"+ assert caller.calls == 1+ assert state.giveup_deferrals == 0+ # The gate is not consulted at all here, so it writes nothing. That is a KNOWN LIMIT of the+ # probe, asserted rather than papered over: a give-up refused for want of hold budget is not+ # directly counted, because only the gate knows whether the run was awaiting a code and this is+ # the one case where the gate is not asked.+ assert [e for e in logs if e.get("event") == "task_v3 verification give-up gate"] == []+++async def test_a_completed_verdict_is_never_held_for_unspent_verification_budget() -> None:+ # The widened primitive answers all three verdicts from one branch, so the completed side has to+ # be shown unchanged: unspent budget is a reason to keep WAITING, never a reason to refuse a+ # success the model can see on the page.+ state = _awaiting_verification_state()+ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]+ script = [[("finish", {"status": "completed", "reason": "signed in"})]]+ outcome, caller = await _run(script, tools)++ assert outcome.status == "completed"+ assert caller.calls == 1+ assert state.giveup_deferrals == 0+++async def test_a_completion_blocked_run_still_escapes_via_failed_at_production_wiring() -> None:+ # The escape hatch `test_verification_blocker_refuses_completed_but_not_failed` protects, pinned+ # where it actually has to hold. That test passes a bare mock and no `activity`, so the widened+ # primitive is consulted for `completed` only and the give-up side is never exercised -- but+ # engine.py always passes an ActivityRecency, so the production configuration DOES reach the+ # give-up side. What makes the escape safe there is not the missing accounting, it is that a+ # terminally-failed source returns None from `block_giveup` by construction: giving up IS the+ # correct answer once the source has been judged. Asserted against the real VerificationState,+ # not a mock that cannot disagree with the gate it is standing in for.+ from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState++ state = VerificationState(+ task=SimpleNamespace(task_id="tsk_1", organization_id="o_1", workflow_run_id=None) # type: ignore[arg-type]+ )+ state.polling_spent_seconds = 120.0+ state.awaiting_code_since = time.monotonic()+ state.arm(VerificationFailure.NO_CODE_TWICE, "get_verification_code")++ tools = [make_finish_tool(verification_blocker=state.block_finish, activity=ActivityRecency())]+ script = [+ [("finish", {"status": "completed", "reason": "signed in"})],+ [("finish", {"status": "failed", "reason": "the code never arrived"})],+ ]+ outcome, caller = await _run(script, tools)++ finish_messages = [m["content"] for m in outcome.messages if m.get("role") == "tool" and m.get("name") == "finish"]+ assert any("the verification step never completed" in m for m in finish_messages)+ # On its FIRST failed verdict, with no hold in between -- two LLM turns total.+ assert outcome.status == "failed"+ assert caller.calls == 2+ assert state.giveup_deferrals == 0