Skip to content

Automate browser based workflows with AI

Task V3: grant one final observed turn at every budget cap trip (SKY-15420) (#8421)

pedrohsdb1ce1fd44d21f
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index e10d6533a..da1b6740e 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -2171,7 +2171,15 @@ class ForgeAgent:
# step even when the task fails. Canceled runs stay unbilled, matching the step engine.
if step_status == StepStatus.failed and outcome.billable_actions:
step_status = StepStatus.completed
- extracted_information = outcome.extracted_output if completed else None
+ # A cap-tripped run that still produced extracted_output (a granted final turn that finished,
+ # or a partial the model had staged before the honest budget_exhausted exit) carries it even
+ # when the terminal status isn't `completed` -- AC-2 is that the cap doesn't discard work the
+ # model already had in hand.
+ extracted_information = (
+ outcome.extracted_output
+ if (completed or (outcome.cap_trip is not None and outcome.extracted_output is not None))
+ else None
+ )
if completed and task.extracted_information_schema is not None and extracted_information is not None:
# Repair the model's output against the schema (fill missing required fields, drop
# hallucinated array items), matching the step engine — but only when the output's shape
@@ -2234,10 +2242,39 @@ class ForgeAgent:
failure_reason = "task_v3 reported completion but returned no extracted_output for the data-extraction goal"
else:
failure_reason = outcome.reason or outcome.status
- if task_status == TaskStatus.failed:
+ if outcome.cap_trip is not None:
+ # The typed fact a budget cap tripped this run, independent of whether the model went on
+ # to finish on the granted final turn (a completed/failed/terminated verdict) or the loop
+ # gave up honestly without one (budget_exhausted).
+ LOG.info(
+ "task_v3 budget cap tripped",
+ task_id=task.task_id,
+ cap_trip=outcome.cap_trip,
+ final_turn_finished=outcome.status != "budget_exhausted",
+ )
+ # failure_category must agree with failure_reason: a vetoed or extraction-less completion,
+ # a loop_error, and any finish verdict the model delivered on the granted turn all failed
+ # for THEIR reason, not the budget's. BUDGET_EXHAUSTED is reserved for the exits where the
+ # cap genuinely is the cause: the loop gave up without a verdict, or the granted-turn
+ # verdict's reason classifies to nothing (usually the model narrating the truncation).
+ cap_is_the_cause = (
+ outcome.cap_trip is not None
+ and not completion_vetoed
+ and not missing_extraction
+ and outcome.status != "loop_error"
+ )
+ if task_status == TaskStatus.failed and outcome.status == "budget_exhausted" and cap_is_the_cause:
+ failure_category = [
+ {"category": "BUDGET_EXHAUSTED", "confidence_float": 1.0, "reasoning": outcome.cap_trip}
+ ]
+ elif task_status == TaskStatus.failed:
# Same code-level failure classification fail_task records, so v3 failures carry a
# failure_category like step-engine failures.
- failure_category = classify_from_failure_reason(failure_reason, fallback_to_unknown=True)
+ failure_category = classify_from_failure_reason(failure_reason, fallback_to_unknown=not cap_is_the_cause)
+ if failure_category is None and cap_is_the_cause:
+ failure_category = [
+ {"category": "BUDGET_EXHAUSTED", "confidence_float": 1.0, "reasoning": outcome.cap_trip}
+ ]
if task.error_code_mapping and task_status in (TaskStatus.failed, TaskStatus.terminated):
# Same user-defined error detection the step engine runs on failure, so workflows that
# consume configured error codes keep that contract on v3.
diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.py
index bb356a642..81765da77 100644
--- a/skyvern/forge/sdk/schemas/tasks.py
+++ b/skyvern/forge/sdk/schemas/tasks.py
@@ -292,12 +292,12 @@ class TaskStatus(StrEnum):
return self in status_requires_extracted_information
def cant_have_extracted_info(self) -> bool:
+ # failed/terminated may carry a PARTIAL extraction — a budget-capped run's staged output
+ # survives the failure rather than being discarded with it. Pre-run statuses cannot.
status_cant_have_extracted_information = {
TaskStatus.created,
TaskStatus.queued,
TaskStatus.running,
- TaskStatus.failed,
- TaskStatus.terminated,
}
return self in status_cant_have_extracted_information
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 0162727c4..8d86a64ac 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -204,6 +204,7 @@ from skyvern.forge.sdk.workflow.status_mapping import (
)
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
+from skyvern.schemas.browser_session_timeouts import REUSE_MIN_REMAINING_LIFETIME_SECONDS
from skyvern.schemas.proxy_pinning import (
derive_proxy_session_id,
redact_proxy_session_id,
@@ -1201,6 +1202,13 @@ class ReusedSessionOwnerProof:
)
+class ReusedSessionBelowLifetimeFloor(Exception):
+ def __init__(self, *, browser_session: PersistentBrowserSession, shortfall: dict[str, object]) -> None:
+ super().__init__(browser_session.persistent_browser_session_id)
+ self.browser_session = browser_session
+ self.shortfall = shortfall
+
+
def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition) -> dict[str, Any]:
"""
This function dumps the workflow definition and removes the irrelevant data to the definition, like created_at and modified_at fields inside:
@@ -3864,11 +3872,14 @@ class WorkflowService:
return bool(profile and profile.is_managed)
@staticmethod
- async def _close_reused_session_best_effort(*, organization_id: str, session_id: str) -> None:
+ async def _close_reused_session_best_effort(
+ *,
+ organization_id: str,
+ session_id: str,
+ reason: BrowserSessionCloseReason = BrowserSessionCloseReason.aborted,
+ ) -> None:
try:
- await app.PERSISTENT_SESSIONS_MANAGER.close_session(
- organization_id, session_id, reason=BrowserSessionCloseReason.aborted
- )
+ await app.PERSISTENT_SESSIONS_MANAGER.close_session(organization_id, session_id, reason=reason)
except Exception:
LOG.warning(
"Failed to close unusable reused browser session",
@@ -4060,6 +4071,41 @@ class WorkflowService:
current_context.browser_session_runnable_generation_id = lease_generation_id
return session_id
+ async def _reused_session_lifetime_shortfall(
+ self,
+ *,
+ organization_id: str,
+ workflow_run_id: str,
+ workflow_permanent_id: str,
+ browser_session: PersistentBrowserSession,
+ ) -> dict[str, object] | None:
+ try:
+ remaining_lifetime_seconds = await app.PERSISTENT_SESSIONS_MANAGER.remaining_lifetime_seconds(
+ browser_session.persistent_browser_session_id,
+ organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Could not evaluate reusable browser session remaining lifetime; leaving reuse enabled",
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ browser_session_id=browser_session.persistent_browser_session_id,
+ exc_info=True,
+ )
+ return None
+ if remaining_lifetime_seconds is None or remaining_lifetime_seconds >= REUSE_MIN_REMAINING_LIFETIME_SECONDS:
+ return None
+ return {
+ "organization_id": organization_id,
+ "workflow_run_id": workflow_run_id,
+ "workflow_permanent_id": workflow_permanent_id,
+ "bound_key": browser_session.bound_key,
+ "browser_session_id": browser_session.persistent_browser_session_id,
+ "remaining_lifetime_seconds": remaining_lifetime_seconds,
+ "min_remaining_lifetime_seconds": REUSE_MIN_REMAINING_LIFETIME_SECONDS,
+ }
+
async def _adopt_reused_session(
self,
*,
@@ -4068,6 +4114,7 @@ class WorkflowService:
expected_workflow_permanent_id: str,
expected_bound_key: str | None,
browser_session: PersistentBrowserSession,
+ lifetime_floor_session_id: str | None = None,
) -> str | None:
session_id = browser_session.persistent_browser_session_id
workflow_identity_matches = browser_session.bound_workflow_permanent_id == expected_workflow_permanent_id or (
@@ -4094,7 +4141,7 @@ class WorkflowService:
if latest is None or is_final_status(latest.status):
return None
raise BrowserSessionAlreadyOccupiedError(session_id, latest.runnable_id or "unknown")
-
+ stale_owner_released = False
stale_owner_id = browser_session.runnable_id
if stale_owner_id and stale_owner_id != workflow_run_id:
owner_proof = await self._read_reused_session_owner_proof(
@@ -4135,6 +4182,25 @@ class WorkflowService:
stale_runnable_id=stale_owner_id,
browser_session_id=session_id,
)
+ # A trusted stale-owner release clears runnable_id in the stored row; mirror that state for retirement.
+ stale_owner_released = True
+
+ if lifetime_floor_session_id == session_id and (browser_session.runnable_id is None or stale_owner_released):
+ shortfall = await self._reused_session_lifetime_shortfall(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=expected_workflow_permanent_id,
+ browser_session=browser_session,
+ )
+ if shortfall is not None:
+ raise ReusedSessionBelowLifetimeFloor(
+ browser_session=(
+ browser_session.model_copy(update={"runnable_id": None})
+ if stale_owner_released
+ else browser_session
+ ),
+ shortfall=shortfall,
+ )
try:
return await self._claim_reused_session(
@@ -4164,6 +4230,7 @@ class WorkflowService:
workflow_permanent_id: str,
bound_key: str | None,
browser_session: PersistentBrowserSession,
+ lifetime_floor_session_id: str | None = None,
) -> tuple[str | None, PersistentBrowserSession]:
candidate = browser_session
for attempt in range(2):
@@ -4175,6 +4242,7 @@ class WorkflowService:
expected_workflow_permanent_id=workflow_permanent_id,
expected_bound_key=bound_key,
browser_session=candidate,
+ lifetime_floor_session_id=lifetime_floor_session_id,
),
candidate,
)
@@ -4205,6 +4273,8 @@ class WorkflowService:
workflow_permanent_id: str,
bound_key: str | None,
browser_session: PersistentBrowserSession,
+ reason: BrowserSessionCloseReason = BrowserSessionCloseReason.aborted,
+ lifetime_floor_session_id: str | None = None,
) -> str | None:
session_id = browser_session.persistent_browser_session_id
observed_workflow_permanent_id = browser_session.bound_workflow_permanent_id
@@ -4212,6 +4282,7 @@ class WorkflowService:
await self._close_reused_session_best_effort(
organization_id=organization_id,
session_id=session_id,
+ reason=reason,
)
return None
occupied_owner_id = browser_session.runnable_id
@@ -4235,13 +4306,21 @@ class WorkflowService:
)
if latest is None:
return None
- adopted_session_id, latest = await self._adopt_reused_session_with_occupancy_retry(
- organization_id=organization_id,
- workflow_run_id=workflow_run_id,
- workflow_permanent_id=workflow_permanent_id,
- bound_key=bound_key,
- browser_session=latest,
- )
+ try:
+ adopted_session_id, latest = await self._adopt_reused_session_with_occupancy_retry(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ bound_key=bound_key,
+ browser_session=latest,
+ lifetime_floor_session_id=lifetime_floor_session_id,
+ )
+ latest_reason = (
+ reason if latest.persistent_browser_session_id == session_id else BrowserSessionCloseReason.aborted
+ )
+ except ReusedSessionBelowLifetimeFloor as below_floor:
+ adopted_session_id, latest = None, below_floor.browser_session
+ latest_reason = BrowserSessionCloseReason.expired
if adopted_session_id is not None:
return adopted_session_id
latest_workflow_permanent_id = latest.bound_workflow_permanent_id
@@ -4265,6 +4344,7 @@ class WorkflowService:
await self._close_reused_session_best_effort(
organization_id=organization_id,
session_id=latest.persistent_browser_session_id,
+ reason=latest_reason,
)
return None
if occupied_owner_id is not None:
@@ -4279,6 +4359,7 @@ class WorkflowService:
await self._close_reused_session_best_effort(
organization_id=organization_id,
session_id=session_id,
+ reason=reason,
)
return None
@@ -4327,13 +4408,21 @@ class WorkflowService:
bound_key=bound_key,
)
if browser_session is not None:
- adopted_session_id, browser_session = await self._adopt_reused_session_with_occupancy_retry(
- organization_id=organization_id,
- workflow_run_id=workflow_run_id,
- workflow_permanent_id=workflow_permanent_id,
- bound_key=bound_key,
- browser_session=browser_session,
- )
+ lifetime_floor_session_id = browser_session.persistent_browser_session_id
+ close_reason = BrowserSessionCloseReason.aborted
+ try:
+ adopted_session_id, browser_session = await self._adopt_reused_session_with_occupancy_retry(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ bound_key=bound_key,
+ browser_session=browser_session,
+ lifetime_floor_session_id=lifetime_floor_session_id,
+ )
+ except ReusedSessionBelowLifetimeFloor as below_floor:
+ LOG.info("Retiring reusable browser session below lifetime floor", **below_floor.shortfall)
+ adopted_session_id, browser_session = None, below_floor.browser_session
+ close_reason = BrowserSessionCloseReason.expired
if adopted_session_id is not None:
return adopted_session_id
LOG.info(
@@ -4351,6 +4440,8 @@ class WorkflowService:
workflow_permanent_id=workflow_permanent_id,
bound_key=bound_key,
browser_session=browser_session,
+ reason=close_reason,
+ lifetime_floor_session_id=lifetime_floor_session_id,
)
if adopted_after_unbind_race is not None:
return adopted_after_unbind_race
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index a15b59287..4c11d92ff 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -369,6 +369,7 @@ async def run_task_v3_agent_loop(
page_probe=None if page_free else page_probe,
page_fingerprint=None if page_free else page_fingerprint,
reload_page=None if page_free else reload_page,
+ final_turn_token_reserve=MAX_TOKENS_PER_ACTION_STEP,
)
finally:
# The context outlives this run; a signal raised as the loop was cancelled must not fire
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 43890c34b..c4f7f518c 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -96,6 +96,10 @@ class LoopOutcome:
status: Literal["completed", "failed", "terminated", "budget_exhausted", "loop_error", "canceled"]
reason: str
extracted_output: Any = None
+ # The raw budget-cap string (e.g. "max_turns (40) reached") that granted this run its one final
+ # observed turn, carried on whatever outcome the final turn produces — a finish verdict or, if
+ # the model didn't finish, the honest budget_exhausted exit. None for a run that never tripped a cap.
+ cap_trip: str | None = None
turns: int = 0
tool_calls: int = 0
action_steps: int = 0
@@ -216,6 +220,12 @@ ACTION_BUDGET_EXTENSION_EVIDENCE_WINDOW = 8
ACTION_BUDGET_EXTENDED_EVENT = "taskv3 loop action budget extended"
ACTION_BUDGET_EXTENSION_REFUSED_EVENT = "taskv3 loop action budget extension refused"
+# Single-shot final-turn grant (budget-exhaustion final turn): the first budget cap trip in a run
+# buys exactly one more unconstrained model turn instead of ending the run immediately, so a model
+# mid-extraction can still call finish with what it has. Facetable — change only with the dashboards
+# that read it.
+FINAL_TURN_GRANTED_EVENT = "taskv3 loop final turn granted"
+
# Page-state stall policy (SKY-15265): rounds of billable batches that left the page fingerprint
# byte-identical, with no page-change flag, mean the run is cycling on a frozen document in shapes
# the per-tool guards cannot see (varied probes never streak; scroll/wait carry no digest at all).
@@ -638,6 +648,10 @@ class ActivityRecency:
# run that reaches the edge and then stops reading that tool altogether leaves this true for the
# rest of the run, disabling the deferral (same as the argument-blind counter it shipped with).
perception_stall_imminent: bool = False
+ # True from the moment the loop's granted final turn starts running. A hold asks for a retry
+ # turn that no longer exists — honoring it would silently convert the model's verdict into
+ # budget_exhausted, the exact conversion the hold headroom gates exist to prevent.
+ final_turn_active: bool = False
def armed(self, window: int = FAILURE_EVIDENCE_WINDOW_TURNS) -> bool:
return self.last_trigger_turn is not None and (self.turn - self.last_trigger_turn) <= window
@@ -664,6 +678,8 @@ def _has_hold_headroom(activity: ActivityRecency | None, deadline_at: float | No
reserves, for the same reason: a token exhaustion and a stall streak one short of its terminator
each convert the deferral into a verdict the gate did not choose."""
if activity is not None:
+ if activity.final_turn_active:
+ return False
if activity.turns_remaining is not None and activity.turns_remaining < FAILURE_EVIDENCE_MIN_TURNS:
return False
if (
@@ -839,6 +855,40 @@ def _refresh_nudge_text() -> str:
)
+def _budget_exhausted_observation(cap: str, recency: ActivityRecency | None) -> str:
+ """A typed, model-facing fact (not an instruction) that this is the run's last granted turn.
+ Deliberately silent on what to do next -- finishing with a partial result, retrying, or reporting
+ failure are all legitimate depending on what the model has."""
+ payload = {
+ "budget_exhausted": True,
+ "cap": cap,
+ "headroom": {
+ "tool_calls": recency.tool_calls_remaining if recency is not None else None,
+ "tokens": recency.tokens_remaining if recency is not None else None,
+ },
+ }
+ json_line = json.dumps(payload, separators=(",", ":"))
+ return f"{json_line}\nThe run's budget cap has tripped; this is the final turn before the run ends."
+
+
+def _budget_exhausted_reason(cap_trip: str) -> str:
+ """Human sentence for a no-finish budget exit. Never includes the raw cap literal (that lives
+ only in `cap_trip` and the logs) so a status/reason readout doesn't leak an internal counter."""
+ if "deadline" in cap_trip:
+ axis = "time"
+ elif "max_tokens" in cap_trip:
+ axis = "token"
+ elif "max_turns" in cap_trip:
+ axis = "turn"
+ elif "max_tool_calls" in cap_trip:
+ axis = "tool-call"
+ elif "maximum steps" in cap_trip:
+ axis = "step"
+ else:
+ axis = "run"
+ return f"The run reached its {axis} budget before the model finished; the recorded output may be partial."
+
+
_TOOL_CALL_RECORD_FIELDS = frozenset(
{
"tool",
@@ -1117,7 +1167,14 @@ def make_finish_tool(
)
if verification_message:
return ToolResult.error(verification_message)
- if status == "completed" and page_fingerprint is not None and deferrals < max_settle_deferrals:
+ if (
+ status == "completed"
+ and page_fingerprint is not None
+ and deferrals < max_settle_deferrals
+ # On the granted final turn the re-verification turn a deferral asks for no longer
+ # exists: holding would convert the verdict into budget_exhausted instead.
+ and not (activity is not None and activity.final_turn_active)
+ ):
try:
settled = await _settled()
except Exception:
@@ -1137,6 +1194,8 @@ def make_finish_tool(
and activity is not None
and page_fingerprint is not None
and failure_deferrals < max_failure_deferrals
+ # Same rule as the completed-side holds: no retry turn exists on the granted final turn.
+ and not activity.final_turn_active
and activity.armed()
# The corrected cycle needs budget for its worst case (wait + observe + re-finish);
# without headroom on every budget axis a deferral would convert an honest failure
@@ -1287,6 +1346,10 @@ async def run_agent_tool_loop(
reload_page: Callable[[], Awaitable[None]] | None = None,
max_refresh_cycles: int = 3,
page_fingerprint: Callable[[], Awaitable[str | None]] | None = None,
+ # One model call's worth of token headroom, reserved so the single-shot final turn (below) is
+ # actually fundable rather than being immediately re-tripped by the same max_tokens check it
+ # was granted under. 0 (the default) reproduces today's unreserved check.
+ final_turn_token_reserve: int = 0,
) -> LoopOutcome:
tool_by_name = {tool.name: tool for tool in tools}
outcome: LoopOutcome | None = None
@@ -1336,6 +1399,21 @@ async def run_agent_tool_loop(
# page-change evidence, and whether the single extension has been spent.
last_change_evidence_step: int | None = None
budget_extension_granted = False
+ # Single-shot final-turn grant (budget-exhaustion final turn): mirrors budget_extension_granted's
+ # shape. The first budget-cap trip anywhere in the loop sets this and buys exactly one more
+ # unconstrained model turn; cap_trip_pending remembers which cap granted it, so a finish() outcome
+ # produced on that turn can carry the fact forward even though finish is a different code path.
+ # final_turn_started distinguishes "granted, turn not run yet" from "granted turn already ran": a
+ # mid-batch grant (max_tool_calls, the action-step gate) leaves the SAME counter tripped for the
+ # very next top-of-turn check, one code site removed from the grant with no turn boundary between
+ # them -- without this flag that check would end the run before the granted turn ever started.
+ final_turn_granted = False
+ final_turn_started = False
+ cap_trip_pending: str | None = None
+ # Extraction carried by a finish the granted turn staged but never got honored (skipped behind a
+ # failed or refused call, or refused by a fail-closed blocker). The verdict itself stays
+ # unhonored — its premise didn't hold — but the data rides the spent-grant exit.
+ final_turn_staged_output: Any = None
def _clear_action_state() -> None:
action_counts.clear()
@@ -1666,18 +1744,51 @@ async def run_agent_tool_loop(
if should_cancel is not None and await should_cancel():
outcome = LoopOutcome("canceled", "run canceled")
break
- if deadline_seconds is not None and time.monotonic() - started_at > deadline_seconds:
- outcome = LoopOutcome("budget_exhausted", f"deadline ({deadline_seconds:.0f}s) reached")
- break
- if max_tokens is not None and total_tokens >= max_tokens:
- outcome = LoopOutcome("budget_exhausted", f"max_tokens ({max_tokens}) reached")
- break
- if turns >= max_turns:
- outcome = LoopOutcome("budget_exhausted", f"max_turns ({max_turns}) reached")
- break
- if total_tool_calls >= max_tool_calls:
- outcome = LoopOutcome("budget_exhausted", f"max_tool_calls ({max_tool_calls}) reached")
+ # The single-shot final-turn grant (see final_turn_granted above): the first budget trip
+ # here buys exactly one more turn instead of ending the run, so the checks below are elif'd
+ # (only the first tripped cap matters this iteration). A spent grant ends the run BEFORE any
+ # counter is consulted: the granted turn may have been granted at a gate no top-of-turn
+ # check re-reads (the action-step gate), so waiting for a counter to re-trip would let a
+ # finish-less granted turn keep looping on other budgets — and the cap that granted the
+ # turn, not whichever counter happens to re-trip first, is the honest fact to report.
+ # final_turn_started (set below, right before the turn actually runs) distinguishes a spent
+ # grant from a mid-batch grant whose turn has not run yet. Cancellation above is exempt: it
+ # is not a budget cap.
+ if final_turn_granted and final_turn_started:
+ outcome = LoopOutcome(
+ "budget_exhausted",
+ _budget_exhausted_reason(cap_trip_pending or "budget"),
+ cap_trip=cap_trip_pending,
+ extracted_output=final_turn_staged_output,
+ )
break
+ # Once the grant fired no top-of-turn cap can trip again, so the checks only run pre-grant.
+ top_of_turn_trip: str | None = None
+ if not final_turn_granted:
+ if deadline_seconds is not None and time.monotonic() - started_at > deadline_seconds:
+ top_of_turn_trip = f"deadline ({deadline_seconds:.0f}s) reached"
+ elif max_tokens is not None and total_tokens >= max(0, max_tokens - final_turn_token_reserve):
+ top_of_turn_trip = f"max_tokens ({max_tokens}) reached"
+ elif turns >= max_turns:
+ top_of_turn_trip = f"max_turns ({max_turns}) reached"
+ elif total_tool_calls >= max_tool_calls:
+ top_of_turn_trip = f"max_tool_calls ({max_tool_calls}) reached"
+ if top_of_turn_trip is not None:
+ final_turn_granted = True
+ cap_trip_pending = top_of_turn_trip
+ LOG.info(
+ FINAL_TURN_GRANTED_EVENT,
+ cap=top_of_turn_trip,
+ turn=turns,
+ tool_calls_remaining=None if activity is None else activity.tool_calls_remaining,
+ tokens_remaining=None if activity is None else activity.tokens_remaining,
+ )
+ messages.append({"role": "user", "content": _budget_exhausted_observation(top_of_turn_trip, activity)})
+ if final_turn_granted:
+ final_turn_started = True
+ if activity is not None:
+ # Read by the finish tool's hold gates: a hold's retry turn no longer exists.
+ activity.final_turn_active = True
turns += 1
if activity is not None:
activity.turn = turns
@@ -1813,9 +1924,23 @@ async def run_agent_tool_loop(
# Enforce the cap per tool call so one batched turn cannot overrun it, and honor a
# cancellation that arrives mid-batch before the next click/type/submit runs. Neither
# this call nor the rest of the batch executes, so answer them as skipped.
- if total_tool_calls >= max_tool_calls:
- outcome = LoopOutcome("budget_exhausted", f"max_tool_calls ({max_tool_calls}) reached")
+ # Gated on `not final_turn_granted`: once the final turn is granted this same counter is
+ # already at (or past) the cap by construction, so re-enforcing it here would block the
+ # granted turn's very first call (including finish) before it ever ran. The top-of-turn
+ # check is what actually ends the run if this granted turn doesn't finish either.
+ if not final_turn_granted and total_tool_calls >= max_tool_calls:
+ mid_batch_trip = f"max_tool_calls ({max_tool_calls}) reached"
+ final_turn_granted = True
+ cap_trip_pending = mid_batch_trip
+ LOG.info(
+ FINAL_TURN_GRANTED_EVENT,
+ cap=mid_batch_trip,
+ turn=turns,
+ tool_calls_remaining=None if activity is None else activity.tool_calls_remaining,
+ tokens_remaining=None if activity is None else activity.tokens_remaining,
+ )
_append_skipped_tool_results(messages, tool_calls[idx:], "tool-call budget reached")
+ messages.append({"role": "user", "content": _budget_exhausted_observation(mid_batch_trip, activity)})
break
if should_cancel is not None and await should_cancel():
outcome = LoopOutcome("canceled", "run canceled")
@@ -1949,8 +2074,45 @@ async def run_agent_tool_loop(
turn=turns,
tool=tool_name,
)
- outcome = LoopOutcome("budget_exhausted", f"Reached the maximum steps ({max_action_steps})")
+ step_cap_trip = f"Reached the maximum steps ({max_action_steps})"
_append_skipped_tool_results(messages, tool_calls[idx:], "action-step budget reached")
+ # Unlike the mid-batch max_tool_calls check above, the step gate is NOT special-cased
+ # away once the final turn is granted: a billable dispatch on the granted turn still
+ # hits it, which is the honest exit the grant exists to produce.
+ if final_turn_granted:
+ # A finish staged later in this skipped batch: its VERDICT is not honored
+ # (it presumed the refused action would run — accepting a completed there
+ # could claim a success that never happened), but the extraction it was
+ # already carrying is salvaged onto the honest exit.
+ staged_finish_output = next(
+ (
+ t_args.get("extracted_output")
+ for _t_id, t_name, t_args in tool_calls[idx:]
+ if t_name == "finish" and isinstance(t_args, dict)
+ ),
+ None,
+ )
+ # The cap that granted the final turn is the honest fact to report — the
+ # step gate merely happens to be where the spent grant gets caught.
+ outcome = LoopOutcome(
+ "budget_exhausted",
+ _budget_exhausted_reason(cap_trip_pending or step_cap_trip),
+ cap_trip=cap_trip_pending or step_cap_trip,
+ extracted_output=staged_finish_output,
+ )
+ else:
+ final_turn_granted = True
+ cap_trip_pending = step_cap_trip
+ LOG.info(
+ FINAL_TURN_GRANTED_EVENT,
+ cap=step_cap_trip,
+ turn=turns,
+ tool_calls_remaining=None if activity is None else activity.tool_calls_remaining,
+ tokens_remaining=None if activity is None else activity.tokens_remaining,
+ )
+ messages.append(
+ {"role": "user", "content": _budget_exhausted_observation(step_cap_trip, activity)}
+ )
break
# Submit-shaped actions (the failure-evidence predicate, minus captcha) are reported BEFORE
# dispatch, since after it the page may be the confirmation page. A failure here never fails
@@ -2230,6 +2392,9 @@ async def run_agent_tool_loop(
status=data.get("status", "completed"),
reason=data.get("reason", ""),
extracted_output=data.get("extracted_output"),
+ # The model's own verdict wins whether or not it landed on the granted final turn;
+ # cap_trip just records the fact that a cap forced this to be the last turn.
+ cap_trip=cap_trip_pending if final_turn_granted else None,
)
break
@@ -2268,6 +2433,23 @@ async def run_agent_tool_loop(
if call_selector is not None:
failed_selectors.add(call_selector)
+ # Whatever skipped or refused a finish once a cap tripped — behind the very call that
+ # tripped it in the GRANTING batch, behind a failed call or fail-closed blocker on the
+ # granted turn, or under a guard that terminated the batch — the extraction it was already
+ # carrying is remembered; the post-loop stamp rides it out on whatever terminal the run
+ # ends with. Last write wins, so a granted-turn restatement supersedes the older capture.
+ if final_turn_granted:
+ staged = next(
+ (
+ t_args.get("extracted_output")
+ for _t_id, t_name, t_args in tool_calls
+ if t_name == "finish" and isinstance(t_args, dict) and t_args.get("extracted_output") is not None
+ ),
+ None,
+ )
+ if staged is not None:
+ final_turn_staged_output = staged
+
# The batch settled on a dead page (an in-loop navigate hit a hard 404/410 and no later navigate
# recovered): end the run as terminated deterministically, matching v1, rather than leaving the
# failed/terminated choice to the model's finish tool (which does not converge on this class).
@@ -2393,6 +2575,18 @@ async def run_agent_tool_loop(
if outcome is None:
outcome = LoopOutcome("loop_error", "loop exited without an outcome")
+ # Every model- or loop-produced terminal once the cap tripped — a finish verdict, a guard
+ # termination (even in the granting batch itself), a stall exit, the spent-grant exit —
+ # happened UNDER the cap: it carries the cap fact, and a missing extraction is filled from
+ # what a skipped finish had staged (the model's own earlier data), so no exit path can
+ # re-discard the partial output. A completed verdict that didn't restate its output would
+ # otherwise be demoted for missing extraction.
+ if final_turn_granted and outcome.status in ("completed", "failed", "terminated", "budget_exhausted", "loop_error"):
+ if outcome.cap_trip is None:
+ outcome.cap_trip = cap_trip_pending
+ if outcome.extracted_output is None:
+ outcome.extracted_output = final_turn_staged_output
+
if progress is not None and progress.ever_armed:
# Per-run survival record, emitted only for runs that ever saw a form (the population the
# ledger applies to — this bounds the added log volume to formful runs, not every v3 run):
diff --git a/skyvern/schemas/browser_session_timeouts.py b/skyvern/schemas/browser_session_timeouts.py
index 44dc4d4db..a08ada5cf 100644
--- a/skyvern/schemas/browser_session_timeouts.py
+++ b/skyvern/schemas/browser_session_timeouts.py
@@ -10,6 +10,8 @@ from __future__ import annotations
MIN_TIMEOUT = 5
MAX_TIMEOUT = 60 * 4 # 4 hours
MAX_LIFETIME_SECONDS = MAX_TIMEOUT * 60
+# A found reusable session with less lifetime left than this is retired and replaced rather than handed to a run.
+REUSE_MIN_REMAINING_LIFETIME_SECONDS = 30 * 60
DEFAULT_TIMEOUT = 60
MAX_TIMEOUT_EXCEEDED_MESSAGE = (
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index ed9372c99..c4a67e1e6 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -749,6 +749,10 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
"""These sessions run on browsers this process owns, which can always be given longer."""
return None
+ async def remaining_lifetime_seconds(self, session_id: str, organization_id: str) -> float | None:
+ """This manager enforces no lifetime cap of its own, so there is no deadline the run cannot renew past."""
+ return None
+
async def update_status(
self,
session_id: str,
diff --git a/skyvern/webeye/persistent_sessions_manager.py b/skyvern/webeye/persistent_sessions_manager.py
index 81d28f032..f481659ed 100644
--- a/skyvern/webeye/persistent_sessions_manager.py
+++ b/skyvern/webeye/persistent_sessions_manager.py
@@ -218,6 +218,12 @@ class PersistentSessionsManager(Protocol):
"""
...
+ async def remaining_lifetime_seconds(self, session_id: str, organization_id: str) -> float | None:
+ """Seconds until this session ends regardless of activity or renewal, measured on the clock its own
+ retirement uses and bounded by a pinned-infrastructure deadline. None when no such deadline exists or
+ it cannot be read."""
+ ...
+
async def update_status(
self, session_id: str, organization_id: str, status: str
) -> PersistentBrowserSession | None:
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 29514183a..85c04afa0 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -2693,6 +2693,164 @@ async def test_execute_task_v3_failed_run_carries_failure_category(
assert update_kwargs.get("failure_category")
+async def test_execute_task_v3_budget_exit_carries_typed_category_and_partial_output(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A cap-tripped run that never finished: the human sentence (not the raw cap literal) is the
+ # failure_reason, the raw literal rides the BUDGET_EXHAUSTED category's reasoning, and a partial
+ # extraction the model had staged is not discarded with the failure.
+ outcome = LoopOutcome(
+ status="budget_exhausted",
+ reason="The run reached its turn budget before the model finished; the recorded output may be partial.",
+ cap_trip="max_turns (40) reached",
+ billable_actions=["click"],
+ extracted_output={"partial": True},
+ )
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None
+ )
+ assert task.status == TaskStatus.failed
+ update_kwargs = loop_mock.update_task_kwargs
+ assert "max_turns (" not in (update_kwargs.get("failure_reason") or "")
+ assert update_kwargs.get("failure_category") == [
+ {"category": "BUDGET_EXHAUSTED", "confidence_float": 1.0, "reasoning": "max_turns (40) reached"}
+ ]
+ assert update_kwargs.get("extracted_information") == {"partial": True}
+
+
+async def test_execute_task_v3_cap_tripped_finish_keeps_output_and_decision_row(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A finish delivered on the granted final turn: the model's verdict and reason stand, the
+ # extracted_output persists despite the non-completed status, the category classifies from the
+ # model's reason (a captcha block is anti-bot, not budget — the cap merely coincided), and the
+ # terminal decision row is written like any other finish.
+ from skyvern.forge import agent as agent_mod
+
+ outcome = LoopOutcome(
+ status="failed",
+ reason="blocked by a captcha",
+ cap_trip="Reached the maximum steps (25)",
+ billable_actions=[],
+ extracted_output={"rows": [1]},
+ )
+ _step, task, loop_mock, _post = await _run_execute_task_v3(monkeypatch, outcome, action_rounds=None)
+ assert task.status == TaskStatus.failed
+ update_kwargs = loop_mock.update_task_kwargs
+ assert update_kwargs.get("failure_reason") == "blocked by a captcha"
+ category = update_kwargs.get("failure_category")
+ assert category and category[0]["category"] == "ANTI_BOT_DETECTION", category
+ assert update_kwargs.get("extracted_information") == {"rows": [1]}
+ persisted = agent_mod.app.DATABASE.workflow_params.create_action.await_args.kwargs["action"]
+ assert persisted.action_type == ActionType.TERMINATE
+ assert persisted.reasoning == "blocked by a captcha"
+
+
+async def test_execute_task_v3_cap_tripped_finish_with_unclassifiable_reason_falls_back_to_budget(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A granted-turn failed verdict whose reason carries no classifiable signal is usually the
+ # model narrating the truncation: the typed cap fact beats an UNKNOWN keyword fallback.
+ outcome = LoopOutcome(
+ status="failed",
+ reason="could not finish filling in the remaining sections",
+ cap_trip="max_turns (40) reached",
+ billable_actions=["click"],
+ extracted_output={"partial": True},
+ )
+ _step, task, loop_mock, _post = await _run_execute_task_v3(monkeypatch, outcome, action_rounds=None)
+ assert task.status == TaskStatus.failed
+ category = loop_mock.update_task_kwargs.get("failure_category")
+ assert category == [
+ {"category": "BUDGET_EXHAUSTED", "confidence_float": 1.0, "reasoning": "max_turns (40) reached"}
+ ]
+
+
+async def test_execute_task_v3_cap_tripped_guard_termination_is_not_budget_categorized(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A repeat-guard termination on the granted turn terminated for the guard's reason (its typed
+ # prefix rides failure_reason); stamping BUDGET_EXHAUSTED over it would mix the label axes.
+ outcome = LoopOutcome(
+ status="terminated",
+ reason="action_loop: repeated identical click on the same target",
+ cap_trip="max_turns (40) reached",
+ billable_actions=["click"],
+ extracted_output=None,
+ )
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None
+ )
+ assert task.status == TaskStatus.terminated
+ assert loop_mock.update_task_kwargs.get("failure_category") is None
+
+
+async def test_execute_task_v3_cap_tripped_missing_extraction_keeps_its_own_category(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A completion demoted for missing extraction failed for THAT reason even when it landed on a
+ # granted final turn: failure_category must agree with failure_reason, not read BUDGET_EXHAUSTED.
+ outcome = LoopOutcome(
+ status="completed",
+ reason="done",
+ cap_trip="max_turns (40) reached",
+ billable_actions=["type"],
+ extracted_output=None,
+ )
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch, outcome, data_extraction_goal="Extract the rows", extracted_information_schema=None
+ )
+ assert task.status == TaskStatus.failed
+ category = loop_mock.update_task_kwargs.get("failure_category")
+ assert category, category
+ assert category[0]["category"] != "BUDGET_EXHAUSTED", category
+
+
+async def test_execute_task_v3_cap_tripped_loop_error_keeps_provider_category_and_partial_output(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A provider failure on the granted call is an infra failure that merely happened after the
+ # cap tripped: the category must reflect the error (not BUDGET_EXHAUSTED), while the staged
+ # extraction the loop salvaged still persists through the relaxed gate.
+ outcome = LoopOutcome(
+ status="loop_error",
+ reason="llm_call_failed: RuntimeError: provider unavailable",
+ cap_trip="max_tool_calls (100) reached",
+ billable_actions=[],
+ extracted_output={"rows": [8]},
+ )
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch, outcome, data_extraction_goal=None, extracted_information_schema=None
+ )
+ assert task.status == TaskStatus.failed
+ category = loop_mock.update_task_kwargs.get("failure_category")
+ assert category, category
+ assert category[0]["category"] != "BUDGET_EXHAUSTED", category
+ assert loop_mock.update_task_kwargs.get("extracted_information") == {"rows": [8]}
+
+
+def test_task_validate_update_allows_partial_extraction_on_failed_and_terminated() -> None:
+ # The budget-cap final turn persists PARTIAL extraction with a failed/terminated verdict. The
+ # execute harness stubs update_task, so the real contract is pinned here: failed/terminated
+ # accept data, pre-run statuses keep rejecting it.
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ make_task(now, organization, status=TaskStatus.running).validate_update(
+ TaskStatus.failed, {"partial": 1}, "budget capped"
+ )
+ make_task(now, organization, status=TaskStatus.running).validate_update(
+ TaskStatus.terminated, {"partial": 1}, "budget capped"
+ )
+ with pytest.raises(ValueError):
+ make_task(now, organization, status=TaskStatus.created).validate_update(TaskStatus.running, {"partial": 1})
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_settle_completion_fenced_to_block_tasks(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 67ee9d1fa..749a1ddd8 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -327,8 +327,12 @@ async def test_max_turns_budget_exhausted() -> None:
outcome, _ = await _run(script, tools, max_turns=3)
assert outcome.status == "budget_exhausted"
- assert "max_turns" in outcome.reason
- assert outcome.turns == 3
+ # Deliberate contract change: the first trip grants one final observed turn instead of ending
+ # the run immediately, so `reason` is now a human sentence (never the raw cap literal) and the
+ # raw fact lives on `cap_trip`; the granted turn (turn 4) still had nothing but click to call.
+ assert "max_turns" not in outcome.reason
+ assert outcome.cap_trip == "max_turns (3) reached"
+ assert outcome.turns == 4
@pytest.mark.asyncio
@@ -339,8 +343,11 @@ async def test_max_tool_calls_budget_exhausted() -> None:
outcome, _ = await _run(script, tools, max_tool_calls=2)
assert outcome.status == "budget_exhausted"
- assert "max_tool_calls" in outcome.reason
- assert outcome.tool_calls == 2
+ # Deliberate contract change: the top-of-turn trip grants one final observed turn (an entire
+ # batch here, since the script always calls two clicks per turn) before ending the run.
+ assert "max_tool_calls" not in outcome.reason
+ assert outcome.cap_trip == "max_tool_calls (2) reached"
+ assert outcome.tool_calls == 4
@pytest.mark.asyncio
@@ -679,8 +686,11 @@ async def test_deadline_seconds_exhausts_budget(monkeypatch: pytest.MonkeyPatch)
deadline_seconds=60.0,
)
assert outcome.status == "budget_exhausted"
- assert "deadline" in outcome.reason
- assert caller.calls == 0 # tripped before any LLM call
+ # Deliberate contract change: the deadline trip grants one final observed turn before the run
+ # actually ends, so one LLM call now runs (the raw literal moved to cap_trip).
+ assert "deadline" not in outcome.reason
+ assert outcome.cap_trip == "deadline (60s) reached"
+ assert caller.calls == 1
@pytest.mark.asyncio
@@ -697,8 +707,11 @@ async def test_max_tokens_exhausts_budget() -> None:
max_tokens=10,
)
assert outcome.status == "budget_exhausted"
- assert "max_tokens" in outcome.reason
- assert caller.calls == 1 # one turn ran, then the token budget stopped it
+ # Deliberate contract change: the trip after turn 1 grants one more observed turn (turn 2) before
+ # the run actually ends, so two turns now run instead of one.
+ assert "max_tokens" not in outcome.reason
+ assert outcome.cap_trip == "max_tokens (10) reached"
+ assert caller.calls == 2 # the granted turn ran before the run actually ended
@pytest.mark.asyncio
@@ -733,7 +746,14 @@ async def test_tool_call_cap_stops_mid_batch() -> None:
observe_calls: list[tuple[str, dict[str, Any]]] = []
tools = [_recording_tool("observe", observe_calls), make_finish_tool()]
outcome = await run_agent_tool_loop(
- llm_caller=_ScriptedCaller([[("observe", {}), ("observe", {})]]), # two tool calls in one turn
+ llm_caller=_ScriptedCaller(
+ [
+ [("observe", {}), ("observe", {})], # two tool calls in one turn
+ # Deliberate contract change: the mid-batch trip grants one final observed turn
+ # (unconstrained -- the cap that stopped the batch above is not re-enforced here).
+ [("observe", {})],
+ ]
+ ),
system_prompt="sys",
user_prompt="goal",
tools=tools,
@@ -741,7 +761,312 @@ async def test_tool_call_cap_stops_mid_batch() -> None:
max_tool_calls=1, # only one dispatch allowed
)
assert outcome.status == "budget_exhausted"
+ assert outcome.cap_trip == "max_tool_calls (1) reached"
+ assert len(observe_calls) == 2 # 1 from the capped batch + 1 from the granted final turn
+
+
+async def test_budget_trip_grants_one_final_observed_turn() -> None:
+ # The core of the grant: a run that trips its cap mid-extraction gets exactly one more turn, the
+ # trip is announced as a typed observation, and a finish on that turn carries its output out.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("click", click_calls), make_finish_tool()]
+ script = [
+ [("click", {})],
+ [("click", {})],
+ [("finish", {"status": "completed", "reason": "got what was needed", "extracted_output": {"partial": 1}})],
+ ]
+ outcome, caller = await _run(script, tools, max_turns=2)
+
+ assert outcome.status == "completed"
+ assert outcome.reason == "got what was needed"
+ assert outcome.extracted_output == {"partial": 1}
+ assert outcome.cap_trip == "max_turns (2) reached"
+ assert caller.calls == 3 # the 2 budgeted turns plus exactly one granted final turn
+ budget_msgs = [
+ m
+ for m in caller.message_history
+ if m.get("role") == "user" and '"budget_exhausted":true' in str(m.get("content"))
+ ]
+ assert len(budget_msgs) == 1
+ assert '"cap":"max_turns (2) reached"' in budget_msgs[0]["content"]
+ assert '"tool_calls"' in budget_msgs[0]["content"] and '"tokens"' in budget_msgs[0]["content"]
+
+
+async def test_final_turn_without_finish_exits_honestly() -> None:
+ # Single-shot: a granted turn that observes instead of finishing ends the run — no second
+ # observation, no second grant, and the reason is a sentence while the raw cap rides cap_trip.
+ observe_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("observe", observe_calls), make_finish_tool()]
+ script = [[("observe", {})], [("observe", {})], [("observe", {})]]
+ outcome, caller = await _run(script, tools, max_turns=2)
+
+ assert outcome.status == "budget_exhausted"
+ assert outcome.cap_trip == "max_turns (2) reached"
+ assert "max_turns (" not in outcome.reason
+ assert caller.calls == 3 # exactly one bonus turn
+ budget_msgs = [
+ m
+ for m in caller.message_history
+ if m.get("role") == "user" and '"budget_exhausted":true' in str(m.get("content"))
+ ]
+ assert len(budget_msgs) == 1
+
+
+async def test_token_reserve_trips_early_and_funds_the_final_turn() -> None:
+ # The scripted caller reports 15 tokens/turn. Raw max_tokens=25 would allow a second unremarked
+ # turn; the 15-token reserve trips the adjusted check after one turn (15 >= 25-15), so the second
+ # turn is the granted final turn — proven by cap_trip being set at all.
+ tools = [_recording_tool("observe", []), make_finish_tool()]
+ script = [
+ [("observe", {})],
+ [("finish", {"status": "completed", "reason": "ok", "extracted_output": {"k": 2}})],
+ ]
+ outcome, caller = await _run(script, tools, max_tokens=25, final_turn_token_reserve=15)
+
+ assert outcome.status == "completed"
+ assert outcome.cap_trip == "max_tokens (25) reached"
+ assert outcome.extracted_output == {"k": 2}
+ assert caller.calls == 2
+
+
+async def test_midbatch_tool_call_trip_answers_skips_then_grants_final_turn() -> None:
+ # A mid-batch trip must leave a valid transcript: the undispatched calls get skipped answers
+ # BEFORE the typed observation, and the granted turn's finish still carries its output out.
+ observe_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("observe", observe_calls), make_finish_tool()]
+ script = [
+ [("observe", {}), ("observe", {}), ("observe", {})],
+ [("finish", {"status": "completed", "reason": "ok", "extracted_output": {"n": 3}})],
+ ]
+ outcome, caller = await _run(script, tools, max_tool_calls=1)
+
+ assert outcome.status == "completed"
+ assert outcome.cap_trip == "max_tool_calls (1) reached"
+ assert outcome.extracted_output == {"n": 3}
assert len(observe_calls) == 1 # the cap stopped the batch after the first dispatch
+ history = caller.message_history
+ skipped_idx = [i for i, m in enumerate(history) if m.get("role") == "tool" and "skipped:" in str(m.get("content"))]
+ budget_idx = [
+ i
+ for i, m in enumerate(history)
+ if m.get("role") == "user" and '"budget_exhausted":true' in str(m.get("content"))
+ ]
+ assert len(skipped_idx) == 2 and len(budget_idx) == 1
+ assert max(skipped_idx) < budget_idx[0]
+
+
+async def test_step_cap_trip_after_refused_extension_grants_final_turn() -> None:
+ # The step gate (post-extension-refusal) is a grant site like every other cap, and a finish on
+ # the granted turn keeps the model's own verdict — a failed status with its reason verbatim.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls)
+ click.billable = True
+ script = [
+ [("click", {})],
+ [("click", {})],
+ [("finish", {"status": "failed", "reason": "blocked by a captcha", "extracted_output": {"rows": [1]}})],
+ ]
+ outcome, _ = await _run(script, [click, make_finish_tool()], max_action_steps=1, max_turns=20)
+
+ assert outcome.status == "failed"
+ assert outcome.reason == "blocked by a captcha"
+ assert outcome.extracted_output == {"rows": [1]}
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+ assert len(click_calls) == 1 # round 2 was blocked at the gate; the granted turn chose to finish
+
+
+async def test_spent_grant_caught_at_the_step_gate_reports_the_granting_cap() -> None:
+ # Cross-axis: max_turns granted the final turn, whose billable dispatch then hits the step
+ # gate. The cap that granted the turn is the honest fact — not the gate that caught it.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls)
+ click.billable = True
+ script = [[("click", {})], [("click", {})]]
+ outcome, _ = await _run(script, [click, make_finish_tool()], max_turns=1, max_action_steps=1)
+
+ assert outcome.status == "budget_exhausted"
+ assert outcome.cap_trip == "max_turns (1) reached"
+ assert "turn budget" in outcome.reason
+
+
+async def test_step_gate_on_the_granted_turn_salvages_a_staged_finish_output() -> None:
+ # The granted turn batches an over-cap action AND a finish: the refused action voids the
+ # verdict (its premise never ran, so a completed claim there could be a success that never
+ # happened) but the extraction the finish already carried must not be re-discarded.
+ click_calls: list[tuple[str, dict[str, Any]]] = []
+ click = _recording_tool("click", click_calls)
+ click.billable = True
+ script = [
+ [("click", {})],
+ [("click", {})],
+ [("click", {}), ("finish", {"status": "completed", "reason": "done", "extracted_output": {"rows": [7]}})],
+ ]
+ outcome, _ = await _run(script, [click, make_finish_tool()], max_action_steps=1, max_turns=20)
+
+ assert outcome.status == "budget_exhausted"
+ assert outcome.extracted_output == {"rows": [7]}
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
+ assert len(click_calls) == 1
+
+
+async def test_failure_skipped_finish_on_the_granted_turn_still_carries_its_extraction() -> None:
+ # The granted turn batches an erroring action with a finish queued behind it: the failure skip
+ # voids the verdict (written before the model saw the error), but the extraction it staged
+ # rides the spent-grant exit instead of being re-discarded.
+ async def err_handler(_args: dict[str, Any]) -> ToolResult:
+ return ToolResult.error("boom")
+
+ err_click = ToolSpec(
+ name="click",
+ description="click",
+ parameters={"type": "object", "properties": {}},
+ handler=err_handler,
+ billable=True,
+ )
+ script = [
+ [("observe", {})],
+ [("click", {}), ("finish", {"status": "completed", "reason": "done", "extracted_output": {"rows": [9]}})],
+ ]
+ outcome, _ = await _run(script, [err_click, _recording_tool("observe", []), make_finish_tool()], max_turns=1)
+
+ assert outcome.status == "budget_exhausted"
+ assert outcome.extracted_output == {"rows": [9]}
+ assert outcome.cap_trip == "max_turns (1) reached"
+
+
+async def test_guard_terminal_on_the_granted_turn_carries_the_cap_and_staged_extraction() -> None:
+ # A terminal the loop itself creates on the granted turn (the action-loop terminator here,
+ # skipping the finish batched behind the repeated click) still happened under the cap grant:
+ # it must carry cap_trip and the staged extraction, or the consumer never sees the data.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ tools = [_billable_tool("click", clicks), make_finish_tool()]
+ script = [
+ [("click", {"selector": "#submit"})],
+ [
+ ("click", {"selector": "#submit"}),
+ ("finish", {"status": "completed", "reason": "done", "extracted_output": {"rows": [3]}}),
+ ],
+ ]
+ outcome, _ = await _run(script, tools, max_turns=1, action_nudge_after=None, action_terminate_after=2)
+
+ assert outcome.status == "terminated"
+ assert outcome.reason.startswith(ACTION_LOOP_REASON_PREFIX)
+ assert outcome.cap_trip == "max_turns (1) reached"
+ assert outcome.extracted_output == {"rows": [3]}
+
+
+async def test_finish_staged_in_the_cap_granting_batch_is_salvaged_if_not_restated() -> None:
+ # The batch that TRIPS the cap can itself stage a finish behind the over-cap call; when the
+ # granted turn doesn't restate it, that extraction still rides the spent-grant exit.
+ observe_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("observe", observe_calls), make_finish_tool()]
+ script = [
+ [
+ ("observe", {}),
+ ("observe", {}),
+ ("finish", {"status": "completed", "reason": "done", "extracted_output": {"rows": [5]}}),
+ ],
+ [("observe", {})],
+ ]
+ outcome, _ = await _run(script, tools, max_tool_calls=1)
+
+ assert outcome.status == "budget_exhausted"
+ assert outcome.cap_trip == "max_tool_calls (1) reached"
+ assert outcome.extracted_output == {"rows": [5]}
+
+
+async def test_completed_final_turn_without_restated_output_is_filled_from_the_staged_finish() -> None:
+ # The granted turn's finish(completed) needn't re-type the extraction its skipped attempt
+ # already staged: the missing output is filled from it, or an otherwise successful extraction
+ # run gets demoted for missing extraction downstream.
+ observe_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("observe", observe_calls), make_finish_tool()]
+ script = [
+ [
+ ("observe", {}),
+ ("observe", {}),
+ ("finish", {"status": "completed", "reason": "done", "extracted_output": {"rows": [5]}}),
+ ],
+ [("finish", {"status": "completed", "reason": "done"})],
+ ]
+ outcome, _ = await _run(script, tools, max_tool_calls=1)
+
+ assert outcome.status == "completed"
+ assert outcome.cap_trip == "max_tool_calls (1) reached"
+ assert outcome.extracted_output == {"rows": [5]}
+
+
+async def test_loop_error_on_the_granted_call_still_carries_the_staged_extraction() -> None:
+ # The granted LLM call itself failing (provider error) must not re-discard what the granting
+ # batch staged: the loop_error keeps its own reason, but cap_trip and the extraction ride out.
+ class _FailsSecondCallCaller(_ScriptedCaller):
+ async def call(self, **kwargs: Any) -> dict[str, Any]:
+ if self.calls >= 1:
+ self.calls += 1
+ raise RuntimeError("provider unavailable")
+ return await super().call(**kwargs)
+
+ observe_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("observe", observe_calls), make_finish_tool()]
+ caller = _FailsSecondCallCaller(
+ [
+ [
+ ("observe", {}),
+ ("observe", {}),
+ ("finish", {"status": "completed", "reason": "done", "extracted_output": {"rows": [8]}}),
+ ]
+ ]
+ )
+ outcome = await run_agent_tool_loop(
+ llm_caller=caller,
+ system_prompt="sys",
+ user_prompt="goal",
+ tools=tools,
+ max_turns=20,
+ max_tool_calls=1,
+ )
+
+ assert outcome.status == "loop_error"
+ assert "llm_call_failed" in outcome.reason
+ assert outcome.cap_trip == "max_tool_calls (1) reached"
+ assert outcome.extracted_output == {"rows": [8]}
+
+
+async def test_finish_on_the_granted_turn_is_not_held_for_settling() -> None:
+ # The settle hold's retry turn no longer exists on the granted final turn: holding there would
+ # silently convert the model's verdict into budget_exhausted — the verdict must stand instead.
+ fp = {"n": 0}
+
+ async def page_fingerprint() -> str:
+ fp["n"] += 1
+ return f"fp-{fp['n']}" # never settles: every sample differs
+
+ activity = ActivityRecency()
+ finish = make_finish_tool(page_fingerprint=page_fingerprint, activity=activity, settle_wait_seconds=0.0)
+ observe_calls: list[tuple[str, dict[str, Any]]] = []
+ tools = [_recording_tool("observe", observe_calls), finish]
+ script = [
+ [("observe", {})],
+ [("finish", {"status": "completed", "reason": "done", "extracted_output": {"k": 1}})],
+ ]
+ outcome, _ = await _run(script, tools, max_turns=1, activity=activity)
+
+ assert outcome.status == "completed"
+ assert outcome.extracted_output == {"k": 1}
+ assert outcome.cap_trip == "max_turns (1) reached"
@pytest.mark.asyncio
@@ -1435,12 +1760,16 @@ async def test_action_step_budget_counts_rounds_not_individual_actions() -> None
[("click", {}), ("type", {"t": "a"})], # action round 1 (2 actions)
[("click", {}), ("type", {"t": "b"})], # action round 2 (2 actions)
[("click", {}), ("type", {"t": "c"})], # round 3 -> blocked by the 2-step budget
+ # Deliberate contract change: the block above grants one final observed turn; retrying the
+ # same over-cap round on it hits the gate again and ends the run for real.
+ [("click", {}), ("type", {"t": "d"})],
]
outcome, _ = await _run(script, [click, type_, make_finish_tool()], max_action_steps=2, max_turns=20)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert outcome.action_steps == 2 # two action rounds counted, exposed on the outcome
- # 2 rounds (4 actions) ran; the 3rd was blocked at the top -> per-round, not per-action, counting.
+ # 2 rounds (4 actions) ran; rounds 3 and 4 were both blocked at the top -> per-round counting.
assert len(click_calls) == 2 and len(type_calls) == 2
@@ -1471,12 +1800,15 @@ async def test_action_step_budget_terminates_only_on_action_beyond_budget() -> N
click_calls: list[tuple[str, dict[str, Any]]] = []
click = _recording_tool("click", click_calls)
click.billable = True
- script = [[("click", {})], [("click", {})]] # 2nd click is the (cap+1)th action round
+ # 2nd and 3rd clicks are both beyond-cap attempts; the 3rd is the deliberate contract change:
+ # the 2nd blocked attempt grants one final observed turn, and retrying on it ends the run for real.
+ script = [[("click", {})], [("click", {})], [("click", {})]]
outcome, _ = await _run(script, [click, make_finish_tool()], max_action_steps=1, max_turns=20)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (1)" in outcome.reason
+ assert "maximum steps (1)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (1)"
assert outcome.action_steps == 1
- assert len(click_calls) == 1 # the over-budget action was refused, not executed
+ assert len(click_calls) == 1 # both over-budget actions were refused, not executed
@pytest.mark.asyncio
@@ -1524,14 +1856,22 @@ async def test_action_step_budget_no_extension_without_page_change_evidence() ->
# refused at the original cap exactly as before, and the refusal is a queryable event.
clicks: list[tuple[str, dict[str, Any]]] = []
click = _recording_tool("click", clicks, billable=True)
- script = [[("click", {"selector": "#a"})], [("click", {"selector": "#b"})], [("click", {"selector": "#c"})]]
+ script = [
+ [("click", {"selector": "#a"})],
+ [("click", {"selector": "#b"})],
+ [("click", {"selector": "#c"})],
+ # Deliberate contract change: the #c block above grants one final observed turn; retrying
+ # the same over-cap click on it hits the gate again (still no evidence) and ends the run.
+ [("click", {"selector": "#c"})],
+ ]
with capture_logs() as logs:
outcome, _ = await _run(script, [click, make_finish_tool()], max_action_steps=2, max_turns=20)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
- assert len(refused) == 1 and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
+ assert len(refused) == 2 and all(r["gate_reason"] == "no_recent_page_change_evidence" for r in refused)
@pytest.mark.asyncio
@@ -1550,10 +1890,14 @@ async def test_action_step_budget_extension_is_granted_at_most_once() -> None:
[("click", {"selector": "#c"})], # extension: cap 2 -> 3
[("observe", {})],
[("click", {"selector": "#d"})], # beyond the extended cap: refused for good
+ # Deliberate contract change: the #d block above grants one final observed turn; retrying
+ # the same beyond-cap click on it hits the gate again ("already_extended") and ends the run.
+ [("click", {"selector": "#d"})],
]
outcome, _ = await _run(script, [observe, click, make_finish_tool()], max_action_steps=2, max_turns=30)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (3)" in outcome.reason
+ assert "maximum steps (3)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (3)"
assert len(clicks) == 3
@@ -1571,6 +1915,7 @@ async def test_action_step_budget_extension_refused_without_turn_headroom() -> N
[("click", {"selector": "#b"})],
[("observe", {})],
[("click", {"selector": "#c"})],
+ [("click", {"selector": "#c"})],
]
outcome, _ = await _run(
script,
@@ -1580,7 +1925,7 @@ async def test_action_step_budget_extension_refused_without_turn_headroom() -> N
activity=ActivityRecency(),
)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
@@ -1692,7 +2037,10 @@ async def test_action_step_budget_extension_not_granted_on_pre_reload_evidence()
skyvern_context.reset()
assert len(reload_calls) == 1
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -1721,7 +2069,10 @@ async def test_action_step_budget_extension_respects_workflow_run_ceiling() -> N
max_turns=20,
)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "hard_step_ceiling"
@@ -1784,7 +2135,10 @@ async def test_action_step_budget_extension_not_laundered_by_same_url_reload() -
with capture_logs() as logs:
outcome, _ = await _run(script, [click, navigate, make_finish_tool()], max_action_steps=2, max_turns=20)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -1827,7 +2181,10 @@ async def test_action_step_budget_extension_deferred_while_a_refresh_is_pending(
finally:
skyvern_context.reset()
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "refresh_pending"
@@ -1871,7 +2228,10 @@ async def test_action_step_budget_extension_not_stamped_by_nav_revisit() -> None
)
assert len(fresh_nav_calls) == 1
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -1905,7 +2265,10 @@ async def test_action_step_budget_extension_not_laundered_by_post_reload_observe
script, [click, navigate, observe, make_finish_tool()], max_action_steps=2, max_turns=20
)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -1937,7 +2300,10 @@ async def test_action_step_budget_extension_not_stamped_by_url_only_transitions(
with capture_logs() as logs:
outcome, _ = await _run(script, [click, make_finish_tool()], max_action_steps=2, max_turns=20)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
assert len(clicks) == 2
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -2012,7 +2378,10 @@ async def test_action_step_budget_extension_dries_up_on_content_oscillation() ->
with capture_logs() as logs:
outcome, _ = await _run(script, [observe, click, make_finish_tool()], max_action_steps=10, max_turns=60)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (10)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (10)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (10)"
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -2045,7 +2414,10 @@ async def test_action_step_budget_extension_not_stamped_by_replayed_download_not
with capture_logs() as logs:
outcome, _ = await _run(script, [click, observe, make_finish_tool()], max_action_steps=2, max_turns=20)
assert outcome.status == "budget_exhausted"
- assert "maximum steps (2)" in outcome.reason
+ # Deliberate contract change: the refused-extension step-cap exit grants one final observed
+ # turn first, and the raw cap literal lives on cap_trip while reason is a human sentence.
+ assert "maximum steps (2)" not in outcome.reason
+ assert outcome.cap_trip == "Reached the maximum steps (2)"
refused = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENSION_REFUSED_EVENT]
assert refused and refused[0]["gate_reason"] == "no_recent_page_change_evidence"
@@ -4634,7 +5006,11 @@ async def test_pre_batch_fingerprint_sample_is_bounded_by_an_already_elapsed_dea
)
elapsed = time.monotonic() - started
- assert outcome.status == "budget_exhausted"
+ # Deliberate contract change: the elapsed deadline grants one final observed turn instead of
+ # ending the run, and the scripted finish on that turn wins; cap_trip carries the deadline fact.
+ # The hang guard is unchanged: the sampler must stay un-awaited on the granted turn too.
+ assert outcome.status == "completed"
+ assert outcome.cap_trip is not None and "deadline" in outcome.cap_trip
assert fp_calls["n"] == 0 # deadline already gone -- the hanging sampler was never awaited
assert elapsed <= 0.3