Skip to content

Automate browser based workflows with AI

Persist Task V3's own URL navigations as goto_url actions (SKY-16374) (#8593)

Shuchang Zhengc288f170706b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index af8751b30..6d402422d 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -140,6 +140,7 @@ from skyvern.forge.sdk.core.skyvern_context import (
MultiFieldTotpAttempt,
SkyvernContext,
action_for_multi_field_totp_persistence,
+ canonical_url,
redact_multi_field_totp_element_data,
)
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
@@ -389,9 +390,18 @@ _TASKV3_TOOL_ACTION_TYPES = {
}
-def _taskv3_row_intention(task: Task, round_action: RoundAction, secret_values: set[str]) -> str | None:
+def _taskv3_row_intention(
+ task: Task, round_action: RoundAction, secret_values: set[str], redacted_args: dict[str, Any]
+) -> str | None:
"""The persisted label for one v3 action. This is the only frame that holds the unfloored drop-check
secrets, and a failure here is logged without exc_info, so no traceback renderer can print them."""
+ if round_action.tool == "navigate":
+ # A navigation names no element, so the element-label composer has nothing to say about it:
+ # the URL is the whole subject. Read from the already-redacted args, never the raw ones.
+ url = redacted_args.get("url")
+ if not isinstance(url, str) or not url:
+ return None
+ return f"Navigated to {url}" if round_action.succeeded else f"Tried to navigate to {url}"
try:
# Drop-only: this set decides whether to drop a page-supplied name and reaches no redaction path.
label_secret_values = (
@@ -435,6 +445,36 @@ def _redact_tool_arg(value: Any, secret_values: set[str]) -> Any:
# Cap on the persisted turn text: actions is a high-traffic table and readers clamp far lower for
# display; a runaway multi-paragraph turn must not become a per-row payload.
_TASKV3_REASONING_MAX_CHARS = 1000
+# Same reason, for the outcome line: it can carry two page-supplied URLs (or, when the call reached no
+# page, the tool's own error text), and neither a URL nor a tool error has a length bound.
+_TASKV3_RESPONSE_MAX_CHARS = 500
+
+
+def _taskv3_row_response(outcome: dict[str, Any]) -> str | None:
+ """The persisted outcome of one v3 action: where it asked to go, where it landed, and what the page
+ answered. Reads only the keys it knows (see ACTION_OUTCOME_DATA_KEY), so a tool reporting something
+ else leaves the row's response empty rather than printing a shape nobody can read."""
+ requested = outcome.get("requested_url") if isinstance(outcome.get("requested_url"), str) else None
+ landed = outcome.get("url") if isinstance(outcome.get("url"), str) else None
+ where = requested or landed or ""
+ # Canonical comparison, the same identity the handler classified `page_transitioned` on, so a
+ # landing that only re-spells the requested URL (a trailing slash, an escape case) is not
+ # rendered as a move the row then calls "same page".
+ if requested and landed and canonical_url(landed) != canonical_url(requested):
+ where = f"{requested} -> {landed}"
+ notes: list[str] = []
+ status = outcome.get("http_status")
+ if isinstance(status, int):
+ notes.append(f"HTTP {status}")
+ if outcome.get("navigation_dead_end") is not None:
+ notes.append("dead end")
+ transitioned = outcome.get("page_transitioned")
+ if isinstance(transitioned, bool):
+ notes.append("page changed" if transitioned else "same page")
+ if not where:
+ return ", ".join(notes) or None
+ return f"{where} ({', '.join(notes)})" if notes else where
+
# Block-engine to run-type labels for the "Task duration metrics" discriminator (SKY-15499):
# workflow-block tasks have no task_runs row, so the block's resolved engine stands in.
@@ -2170,27 +2210,54 @@ class ForgeAgent:
# can be glued to adjacent alphanumerics — boundary anchoring would let it through.
turn_reasoning = redact_secrets_from_text(turn_reasoning, secret_values)
turn_reasoning = turn_reasoning[:_TASKV3_REASONING_MAX_CHARS]
+ # A round that bills nothing claims no budget unit, so its rows ride the LAST consumed
+ # index (or the single Step's own order 0) the way the decision row below does: a fresh
+ # index nothing later claims would be a distinct (task_id, step_order) pair, and the
+ # workflow-run step budget counts those pairs.
+ billable_round = any(entry.billable for entry in round_actions)
+ row_step_order = v3_round_index if billable_round else max(v3_round_index - 1, 0)
for round_action in round_actions:
name, args, succeeded = round_action.tool, round_action.args, round_action.succeeded
try:
tool_args = _redact_tool_args(args if isinstance(args, dict) else {}, secret_values)
selector = tool_args.get("selector", "")
+ row_response = _taskv3_row_response(round_action.outcome) if round_action.outcome else None
+ if row_response is not None:
+ # The outcome carries page-supplied URLs the arg redaction above never saw, and a
+ # landing URL's query routinely holds the credential itself (a sign-in link's token).
+ if secret_values:
+ row_response = redact_secrets_from_text(
+ row_response, secret_values, boundary_all_lengths=True
+ )
+ elif name == "navigate" and round_action.error:
+ # A navigation the engine refused on purpose (the destructive same-URL reload
+ # guard), or one that never loaded, reached no page and reports no outcome: the
+ # tool's own error is all the row can say, and without it the customer reads a
+ # failed navigation with no reason. Free-form prose, so it is matched the way the
+ # turn text is rather than the way the URL line above is.
+ row_response = round_action.error
+ if secret_values:
+ row_response = redact_secrets_from_text(row_response, secret_values)
+ if row_response is not None:
+ row_response = row_response[:_TASKV3_RESPONSE_MAX_CHARS]
action = _taskv3_action_for_tool_call(
name,
tool_args,
status=ActionStatus.completed if succeeded else ActionStatus.failed,
+ response=row_response,
organization_id=task.organization_id,
workflow_run_id=task.workflow_run_id,
task_id=task.task_id,
step_id=step.step_id,
- # Round index, not the single Step's order: makes each v3 action ROUND count
- # as one unit of the workflow-run step budget (distinct (task, order) pairs).
- step_order=v3_round_index,
+ # Round index, not the single Step's order: makes each BILLABLE v3 action
+ # ROUND count as one unit of the workflow-run step budget (distinct (task,
+ # order) pairs).
+ step_order=row_step_order,
action_order=len(v3_persisted_actions),
description=f"{TASK_V3_ACTION_DESCRIPTION_PREFIX}{name} {selector}".strip(),
screenshot_artifact_id=screenshot_artifact_id,
reasoning=turn_reasoning,
- intention=_taskv3_row_intention(task, round_action, secret_values),
+ intention=_taskv3_row_intention(task, round_action, secret_values, tool_args),
)
v3_persisted_actions.append(action)
await app.DATABASE.workflow_params.create_action(
@@ -2198,9 +2265,7 @@ class ForgeAgent:
)
except Exception:
LOG.warning("task_v3 failed to persist action row", task_id=task.task_id, exc_info=True)
- if any(entry.billable for entry in round_actions):
- # Only billable rounds consume the budget unit; recordable-only rounds (navigate/
- # scroll/wait) keep the current index so they never inflate the workflow-run count.
+ if billable_round:
v3_round_index += 1
pre_submit_ring: PreSubmitCaptureRing | None = None
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index c2cce32ae..3b556040e 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -154,6 +154,14 @@ ToolHandler = Callable[[dict[str, Any]], Awaitable[ToolResult]]
# lives in target_label.py; this module only carries the two raw values from probe to `RoundAction`.
TARGET_LABEL_DATA_KEY = "target_label"
TARGET_KIND_DATA_KEY = "target_kind"
+# The tool-result `data` key a recorded action's outcome rides on: the machine facts about what the
+# call achieved (`requested_url`, `url`, `http_status`, `page_transitioned`, `navigation_dead_end`),
+# carried verbatim to `RoundAction.outcome` for the caller to persist on the action row. Internal to
+# the loop like the two keys above -- never shown to the model, which reads the tool's own content.
+ACTION_OUTCOME_DATA_KEY = "action_outcome"
+# An HTTP status at or above this reached no usable page, so the action row reads as failed even
+# though the tool honestly returned ok (the model still gets the status and decides what to do).
+ACTION_OUTCOME_FAILED_HTTP_STATUS = 400
# How long a call spent turning an address into a target, before the act. A context variable rather
# than a field on the result, because the cohort this exists to price is the one where the handler
# RAISES -- a driver timeout on a resolved target -- and a result the handler never returned cannot
@@ -312,6 +320,14 @@ class RoundAction(NamedTuple):
# downstream: the loop already read it off the ToolSpec, and a second name list is a second place
# for a new tool to be missing from.
billable: bool = False
+ # What the action achieved, as the tool itself reported it (see ACTION_OUTCOME_DATA_KEY): machine
+ # facts only, for the caller to persist alongside the verb. None when the tool reported none --
+ # a call that errored before it reached a page has only its `error` below to offer.
+ outcome: dict[str, Any] | None = None
+ # The tool's own error text when the call failed, which is all a call that never reached a page
+ # can say about itself -- a refusal the engine issued on purpose reads as an unexplained failed
+ # row without it. Model-facing prose, so a caller that persists it must redact and cap it.
+ error: str | None = None
# A probe consulted after a billable/download-signaling tool result; a truthy return ends the run as
@@ -1074,6 +1090,16 @@ def _is_finish(tool_name: str) -> bool:
return tool_name == "finish"
+def _outcome_reports_failure(outcome: dict[str, Any] | None) -> bool:
+ """Whether a tool that returned ok nonetheless reported reaching no usable page. Read off the
+ machine facts the tool exposed (an HTTP status), so the tool never has to adjudicate its own
+ success -- and only the persisted row moves: the model still reads the tool's own ok result."""
+ if not outcome:
+ return False
+ status = outcome.get("http_status")
+ return isinstance(status, int) and status >= ACTION_OUTCOME_FAILED_HTTP_STATUS
+
+
def _arms_failure_evidence(tool_name: str, args: dict[str, Any], ok: bool) -> bool:
"""solve_captcha arms on ANY dispatch — its "not solved" error is exactly the verdict the async
protocol can contradict. Other actions arm only when they reached the page AND in their
@@ -3393,21 +3419,25 @@ async def run_agent_tool_loop(
):
action_nudges_due.append((tool_name, args, repeat_count))
if submit_watch is not None and tool_name == "navigate" and result.status == "ok":
- # Outside the billable/recordable branch on purpose: navigate is neither, so a clear
- # placed in there never runs. The run left the page; the control it clicked went too.
+ # Outside the billable/recordable branch on purpose: the run left the page and the
+ # control it clicked went too, whatever navigate's spec flags happen to say.
submit_watch.clear()
if spec is not None and (spec.billable or spec.recordable):
# Dispatched page actions enter the round with their outcome: a failed billable round
# still consumed budget and must persist (else later blocks undercount the run
# budget); recordable tools persist for artifact parity without billing/budget.
+ round_outcome = result_data.get(ACTION_OUTCOME_DATA_KEY)
+ round_outcome = round_outcome if isinstance(round_outcome, dict) else None
round_actions.append(
RoundAction(
tool_name,
args,
- result.status == "ok",
+ result.status == "ok" and not _outcome_reports_failure(round_outcome),
result_data.get(TARGET_LABEL_DATA_KEY) or None,
result_data.get(TARGET_KIND_DATA_KEY) or None,
spec.billable,
+ round_outcome,
+ result.content if result.status == "error" else None,
)
)
if spec.billable and result.status == "ok":
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index c6a5b50f2..fb4d8f8c0 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -46,6 +46,7 @@ from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import URL_IN_TEXT, canonical_url, opaque_url_echo_window
from skyvern.forge.taskv3.frame_perception import frame_perception_enabled
from skyvern.forge.taskv3.loop import (
+ ACTION_OUTCOME_DATA_KEY,
FILL_TOOLS,
NAVIGATION_DEAD_END_STATUSES,
PAGE_UNAVAILABLE_ERROR,
@@ -11809,6 +11810,16 @@ def build_browser_tools(
# terminated (v1's behavior) rather than defaulting the outcome to failed.
if response is not None and response.status in NAVIGATION_DEAD_END_STATUSES:
data["navigation_dead_end"] = response.status
+ # What the persisted action row says happened. The requested URL is the model's own argument
+ # (a placeholder or payload ref must not be unwrapped into a row), and the dead-end status is
+ # repeated from the loop's signal above because the row never sees that one.
+ outcome: dict[str, Any] = {"requested_url": requested, "url": landed}
+ if response is not None:
+ outcome["http_status"] = response.status
+ outcome["page_transitioned"] = landed_canonical != pre_nav_canonical
+ if "navigation_dead_end" in data:
+ outcome["navigation_dead_end"] = data["navigation_dead_end"]
+ data[ACTION_OUTCOME_DATA_KEY] = outcome
return ToolResult.ok(f"navigated to {landed}{status}", data=data)
async def file_upload(args: dict[str, Any]) -> ToolResult:
@@ -12880,6 +12891,11 @@ def build_browser_tools(
"file_upload",
):
_tool_spec.billable = True
+ if _tool_spec.name == "navigate":
+ # Recordable, not billable: a URL the model typed itself is an action the customer needs
+ # to see (one action row + the round's screenshot), but it mutates no page, so it must not
+ # consume the action-step budget or meter like one that does.
+ _tool_spec.recordable = True
if _tool_spec.name in ("observe", "get_html", "look"):
# Large perception dumps: only the latest snapshot is relevant, so let the loop elide older
# ones from the re-sent transcript (bounds context on perception-heavy pages). look's legend
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index b32113305..7f4ceffd0 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -59,6 +59,7 @@ from skyvern.webeye.actions.actions import (
ActionStatus,
ActionType,
ClickAction,
+ GotoUrlAction,
HoverAction,
InputTextAction,
KeypressAction,
@@ -895,6 +896,229 @@ async def test_execute_task_v3_persists_per_action_screenshots_and_rows(monkeypa
assert all(a.response is None for a in persisted)
+_GOTO_OUTCOME = {
+ "requested_url": "https://forms.example.test/contact-us",
+ "url": "https://forms.example.test/contact",
+ "http_status": 200,
+ "page_transitioned": True,
+}
+
+
+async def test_execute_task_v3_persists_a_navigation_as_a_goto_url_row(monkeypatch: pytest.MonkeyPatch) -> None:
+ # SKY-16374: a URL the model typed itself is an action on a par with a click, so it persists as a
+ # `goto_url` row carrying the outcome -- where it asked to go, where it landed, what the page
+ # answered. It must not meter: nothing billable, and the round does not advance the workflow-run
+ # step index (so a navigation cannot inflate a workflow's step budget).
+ from skyvern.forge import agent as agent_mod
+
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+ rounds = [
+ [RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, True, None, None, False, _GOTO_OUTCOME)],
+ [RoundAction("click", {"selector": "#send"}, True, billable=True)],
+ ]
+ step, _task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=rounds,
+ action_round_texts=["the contact link 404s, typing the contact URL instead", "sending the form"],
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]
+ navigation, click = persisted[0], persisted[1]
+ assert navigation.action_type == ActionType.GOTO_URL
+ assert isinstance(navigation, GotoUrlAction) and navigation.url == _GOTO_OUTCOME["requested_url"]
+ assert navigation.status == ActionStatus.completed
+ assert navigation.intention == f"Navigated to {_GOTO_OUTCOME['requested_url']}"
+ assert navigation.reasoning == "the contact link 404s, typing the contact URL instead"
+ assert navigation.screenshot_artifact_id == "artifact-1" # the round's own screenshot
+ assert navigation.response is not None
+ assert _GOTO_OUTCOME["url"] in navigation.response and "HTTP 200" in navigation.response
+ # Recorded, never metered: the navigation round leaves the step index where it was, so the click
+ # that follows it is still round 0, and only the click bills.
+ assert (navigation.step_order, click.step_order) == (0, 0)
+ assert len(step.output.actions_and_results) == 1
+
+
+def test_a_navigation_row_calls_a_move_a_move_by_the_same_identity_the_page_was_judged_by() -> None:
+ # The row renders "requested -> landed" and "page changed / same page" out of the SAME outcome, so
+ # the two have to agree. The handler judged the transition on CANONICAL URLs, so comparing the raw
+ # strings here made a browser re-spelling (the model types a bare host, the page reports it with
+ # the root slash) read as a move the very next clause calls "same page".
+ same_page = {
+ "requested_url": "https://forms.example.test",
+ "url": "https://forms.example.test/",
+ "http_status": 200,
+ "page_transitioned": False,
+ }
+ assert agent_module._taskv3_row_response(same_page) == "https://forms.example.test (HTTP 200, same page)"
+ moved = dict(same_page, url="https://forms.example.test/thanks", page_transitioned=True)
+ assert agent_module._taskv3_row_response(moved) == (
+ "https://forms.example.test -> https://forms.example.test/thanks (HTTP 200, page changed)"
+ )
+
+
+async def test_execute_task_v3_keeps_a_trailing_navigation_off_the_step_budget(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The workflow-run step budget counts DISTINCT (task_id, step_order) pairs across steps and
+ # round-stamped action rows (get_total_unique_progress_round_count_by_task_ids), so a round that
+ # bills nothing must never open a pair of its own. A navigation BEFORE a billable round shares
+ # the index that round goes on to claim; one AFTER the last billable round has nobody left to
+ # share with, so it has to step back onto the index already spent -- the way the decision row does.
+ from skyvern.forge import agent as agent_mod
+
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+
+ async def _persisted(rounds: list[list[RoundAction]]) -> set[int | None]:
+ await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=rounds,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ calls = agent_mod.app.DATABASE.workflow_params.create_action.await_args_list
+ return {call.kwargs["action"].step_order for call in calls}
+
+ click_round: list[RoundAction] = [RoundAction("click", {"selector": "#send"}, True, billable=True)]
+ nav_round: list[RoundAction] = [
+ RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, True, None, None, False, _GOTO_OUTCOME)
+ ]
+ assert await _persisted([click_round]) == {0}
+ # Same run, one navigation appended: the budget the run spends must be identical.
+ assert await _persisted([click_round, nav_round]) == {0}
+ # And a run whose only action is a navigation spends the one unit the Step row already occupies.
+ assert await _persisted([nav_round]) == {0}
+
+
+async def test_execute_task_v3_persists_a_dead_end_navigation_as_a_failed_row(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The live shape this exists for: two navigations the model typed, the second landing on a hard 404,
+ # then the loop's dead-end terminate. The 404 must read as a FAILED goto_url row carrying the status,
+ # with the terminate row still following it -- not a terminate out of nowhere.
+ from skyvern.forge import agent as agent_mod
+
+ dead_end = {
+ "requested_url": "https://forms.example.test/get-in-touch",
+ "url": "https://forms.example.test/get-in-touch",
+ "http_status": 404,
+ "page_transitioned": True,
+ "navigation_dead_end": 404,
+ }
+ outcome = LoopOutcome(
+ status="terminated",
+ reason="navigation_dead_end: navigate landed on a dead page (HTTP 404)",
+ billable_actions=[],
+ )
+ rounds = [
+ [
+ RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, True, None, None, False, _GOTO_OUTCOME),
+ RoundAction("navigate", {"url": dead_end["requested_url"]}, False, None, None, False, dead_end),
+ ]
+ ]
+ _step, _task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=rounds,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]
+ assert [a.action_type for a in persisted] == [ActionType.GOTO_URL, ActionType.GOTO_URL, ActionType.TERMINATE]
+ live, dead, terminate = persisted
+ assert (live.status, dead.status) == (ActionStatus.completed, ActionStatus.failed)
+ assert dead.response is not None and "HTTP 404" in dead.response and "dead end" in dead.response
+ assert live.intention == f"Navigated to {_GOTO_OUTCOME['requested_url']}"
+ assert dead.intention == f"Tried to navigate to {dead_end['requested_url']}"
+ assert terminate.reasoning == outcome.reason
+
+
+async def test_execute_task_v3_persists_a_failed_navigation_with_the_reason_it_failed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A navigate whose tool call errored (a load timeout, the destructive-reload refusal the engine
+ # issues on purpose) reached no page, so it reports no outcome -- but a failed row with an empty
+ # response reads as a navigation that broke for no reason, which is worse than not showing it. The
+ # tool's own error is what the row has to say, and it is scrubbed like every other persisted text.
+ from skyvern.forge import agent as agent_mod
+
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: True
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run",
+ lambda *_a, **_k: {"sk4829137765"},
+ )
+ refusal = (
+ "already on this page and it has filled fields (including any attached file); reloading it "
+ "would discard them. Act on the current page instead. token=sk4829137765"
+ )
+ outcome = LoopOutcome(status="failed", reason="could not load the page", billable_actions=[])
+ rounds = [
+ [RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, False, None, None, False, None, refusal)],
+ # ...and a failure the tool said nothing about still leaves the response empty rather than
+ # inventing one.
+ [RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, False)],
+ ]
+ _step, _task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=rounds,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]
+ refused, silent = persisted[0], persisted[1]
+ assert refused.action_type == ActionType.GOTO_URL
+ assert refused.status == ActionStatus.failed
+ assert refused.intention == f"Tried to navigate to {_GOTO_OUTCOME['requested_url']}"
+ assert refused.response is not None and refused.response.startswith("already on this page")
+ assert "sk4829137765" not in refused.response and REDACTED_SECRET_PLACEHOLDER in refused.response
+ assert silent.response is None
+
+
+async def test_execute_task_v3_scrubs_a_secret_out_of_a_navigation_row(monkeypatch: pytest.MonkeyPatch) -> None:
+ # A landed URL is page-supplied and its query routinely carries the credential itself (a sign-in
+ # link's token). It reaches a persisted, displayed row for the first time here, so it goes through
+ # the same scrub the row's args already get -- in the intention AND in the outcome line.
+ from skyvern.forge import agent as agent_mod
+
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: True
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run",
+ lambda *_a, **_k: {"sk4829137765"},
+ )
+ landed = {
+ "requested_url": "https://forms.example.test/login?token=sk4829137765",
+ "url": "https://forms.example.test/account?session=sk4829137765",
+ "http_status": 200,
+ "page_transitioned": True,
+ }
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])
+ rounds = [[RoundAction("navigate", {"url": landed["requested_url"]}, True, None, None, False, landed)]]
+ _step, _task, _loop, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=rounds,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ navigation = agent_mod.app.DATABASE.workflow_params.create_action.await_args_list[0].kwargs["action"]
+ assert "sk4829137765" not in (navigation.response or "")
+ assert "sk4829137765" not in (navigation.intention or "")
+ assert "sk4829137765" not in navigation.url
+ assert REDACTED_SECRET_PLACEHOLDER in (navigation.response or "")
+
+
def test_every_tool_that_pays_for_a_name_has_something_to_say_about_it() -> None:
# The set that triggers the capture lives with the tools; the verbs that render it live in
# target_label.py. A tool added to one and not the other pays the probe on every call and throws
@@ -3616,7 +3840,8 @@ async def test_execute_task_v3_recordable_round_persists_without_budget_unit(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# navigate/scroll/wait persist as action rows with screenshots (artifact parity) but never
- # consume a workflow-run budget unit: their rows keep the current round index.
+ # consume a workflow-run budget unit: their rows share the step_order of the billable round
+ # nearest them, ahead OR behind, so they open no (task_id, step_order) pair of their own.
outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
rounds = [
[RoundAction("navigate", {"url": "https://a.test"}, True)],
@@ -3633,7 +3858,9 @@ async def test_execute_task_v3_recordable_round_persists_without_budget_unit(
assert stamped == [
(ActionType.GOTO_URL, 0),
(ActionType.CLICK, 0),
- (ActionType.SCROLL, 1),
+ # The trailing scroll rides the index the click already spent: a fresh one would be a second
+ # distinct pair for the budget to count, charging the run a step nothing billable claimed.
+ (ActionType.SCROLL, 0),
]
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 65e782fb7..aae18eedc 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -37,6 +37,7 @@ from skyvern.forge.taskv3.loop import (
ACTION_LOOP_NUDGE_AFTER,
ACTION_LOOP_REASON_PREFIX,
ACTION_LOOP_TERMINATE_AFTER,
+ ACTION_OUTCOME_DATA_KEY,
CODE_TOOL_NAME,
FAILURE_EVIDENCE_MIN_TOOL_CALLS,
FAILURE_EVIDENCE_MIN_TURNS,
@@ -3095,7 +3096,7 @@ async def test_on_action_round_fires_for_all_failed_round_with_failure_flag() ->
script = [[("click", {})], [("finish", {"status": "completed", "reason": "ok"})]]
outcome, _ = await _run(script, [click, make_finish_tool()], on_action_round=_on_round)
assert outcome.status == "completed"
- assert rounds == [[RoundAction("click", {}, False, billable=True)]]
+ assert rounds == [[RoundAction("click", {}, False, billable=True, error="tool_error: RuntimeError: boom")]]
assert outcome.billable_actions == [] # billing still counts successes only
@@ -4440,6 +4441,154 @@ async def test_navigate_resets_action_counters() -> None:
assert len(clicks) == 10
+_NAV_URL = "https://forms.example.test/contact-us"
+
+
+def _recordable_navigate(sink: list[tuple[str, dict[str, Any]]], *, outcome: dict[str, Any] | None = None) -> ToolSpec:
+ """The production navigate shape (SKY-16374): recordable so the navigation persists as an action
+ row with the round's screenshot, never billable, and carrying the outcome the caller writes onto
+ that row."""
+
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ sink.append(("navigate", args))
+ data: dict[str, Any] = {"page_state_changed": True}
+ if outcome is not None:
+ data[ACTION_OUTCOME_DATA_KEY] = outcome
+ if outcome.get("navigation_dead_end") is not None:
+ # As the real handler does: the top-level flag is what the loop's dead-end verdict
+ # reads, the outcome's copy is what the persisted row carries.
+ data["navigation_dead_end"] = outcome["navigation_dead_end"]
+ return ToolResult.ok("navigated", data=data)
+
+ return ToolSpec(
+ name="navigate",
+ description="n",
+ parameters={"type": "object", "properties": {}},
+ handler=handler,
+ recordable=True,
+ )
+
+
+async def test_a_navigation_reaches_the_round_with_its_outcome_and_costs_no_budget() -> None:
+ # The customer must see a URL the model typed itself, so navigate enters the action round like a
+ # click — carrying what the navigation achieved, not just the verb. It stays unbilled and unbudgeted:
+ # a navigation is not a page-mutating step, and a recorded row must never start metering.
+ rounds: list[list[RoundAction]] = []
+
+ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:
+ rounds.append(actions)
+
+ navs: list[tuple[str, dict[str, Any]]] = []
+ nav_outcome = {"requested_url": _NAV_URL, "url": _NAV_URL, "http_status": 200, "page_transitioned": True}
+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "completed", "reason": "read it"})]]
+ outcome, _ = await _run(
+ script, [_recordable_navigate(navs, outcome=nav_outcome), make_finish_tool()], on_action_round=_on_round
+ )
+ assert outcome.status == "completed"
+ assert rounds == [[RoundAction("navigate", {"url": _NAV_URL}, True, None, None, False, nav_outcome)]]
+ assert outcome.billable_actions == []
+ assert outcome.action_steps == 0
+
+
+async def test_a_navigation_that_answered_an_http_error_is_recorded_as_a_failed_action() -> None:
+ # A 404 landing stays a ToolResult.ok — the model reads the status and decides what to do, and that
+ # transcript must not change. The ROW is the customer's view, and a navigation onto a dead page has
+ # to read as a failed one there, or a run that died on a 404 shows a terminate out of nowhere.
+ rounds: list[list[RoundAction]] = []
+
+ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:
+ rounds.append(actions)
+
+ navs: list[tuple[str, dict[str, Any]]] = []
+ nav_outcome = {"requested_url": _NAV_URL, "url": _NAV_URL, "http_status": 404, "navigation_dead_end": 404}
+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "completed", "reason": "should not win"})]]
+ outcome, _ = await _run(
+ script, [_recordable_navigate(navs, outcome=nav_outcome), make_finish_tool()], on_action_round=_on_round
+ )
+ assert [action.succeeded for round_actions in rounds for action in round_actions] == [False]
+ assert [action.outcome for round_actions in rounds for action in round_actions] == [nav_outcome]
+ # The dead-end verdict still ends the run (the row explains it; it does not replace it), and the
+ # model still read the tool's own ok result.
+ assert outcome.status == "terminated" and outcome.reason.startswith(NAV_DEAD_END_REASON_PREFIX)
+ assert [m["content"] for m in outcome.messages if m.get("name") == "navigate"] == ["navigated"]
+
+
+async def test_a_recorded_call_that_failed_carries_its_error_text_to_the_round() -> None:
+ # A navigation that reached no page -- a load timeout, or the destructive-reload guard's deliberate
+ # refusal -- has no outcome to report, so the tool's own error is the only thing its persisted row
+ # can say about it. The caller has no other view of the call, and a failed row that explains
+ # nothing reads as a navigation that broke for no reason.
+ rounds: list[list[RoundAction]] = []
+
+ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:
+ rounds.append(actions)
+
+ navs: list[tuple[str, dict[str, Any]]] = []
+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "failed", "reason": "never loaded"})]]
+ await _run(
+ script,
+ [_erroring_tool("navigate", navs, recordable=True), make_finish_tool()],
+ on_action_round=_on_round,
+ )
+ assert [(a.succeeded, a.outcome, a.error) for round_actions in rounds for a in round_actions] == [
+ (False, None, "navigate failed")
+ ]
+
+
+async def test_repeated_navigations_to_one_url_are_recorded_without_arming_the_action_loop_guard() -> None:
+ # CONTRACT (deliberate exclusion): the action-loop terminator keys on BILLABLE calls, and a
+ # navigation is recorded but not billable, so repeating one is not treated like repeating a click.
+ # The guard exists to stop a run burning its action-step budget on an action that cannot progress,
+ # and a navigation spends none of it; and a successful navigate is itself page-change evidence,
+ # which clears the streak ledger, so counting navigations could never reach the verdict anyway.
+ # Recording a row must not change what ends a run.
+ rounds: list[list[RoundAction]] = []
+
+ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:
+ rounds.append(actions)
+
+ navs: list[tuple[str, dict[str, Any]]] = []
+ repeats = ACTION_LOOP_TERMINATE_AFTER + 2
+ script = [[("navigate", {"url": _NAV_URL})] for _ in range(repeats)]
+ script.append([("finish", {"status": "completed", "reason": "found the page in the end"})])
+ outcome, _ = await _run(
+ script,
+ [_recordable_navigate(navs), make_finish_tool()],
+ on_action_round=_on_round,
+ max_turns=200,
+ max_tool_calls=500,
+ )
+ assert outcome.status == "completed"
+ assert len(navs) == repeats # never cut short by the action-loop verdict
+ assert len(rounds) == repeats # and every one of them persisted as its own round
+
+
+async def test_a_navigation_does_not_arm_the_finish_gate_failure_evidence() -> None:
+ # navigate now enters the same round branch that arms the failure-evidence trigger, which exists
+ # for actions that may have SUBMITTED something. A navigation has not, so arming it would make the
+ # finish gate hold verdicts on runs whose last action merely changed page.
+ nav_activity = ActivityRecency(turn=0, turns_remaining=5, tool_calls_remaining=10)
+ navs: list[tuple[str, dict[str, Any]]] = []
+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "failed", "reason": "no contact form"})]]
+ await _run(script, [_recordable_navigate(navs), make_finish_tool()], activity=nav_activity)
+ assert nav_activity.last_trigger_turn is None
+
+ # Control: a click on the same path does arm it, so the None above is a live assertion.
+ click_activity = ActivityRecency(turn=0, turns_remaining=5, tool_calls_remaining=10)
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ click_script = [
+ [("click", {"selector": "#submit"})],
+ [("finish", {"status": "failed", "reason": "no contact form"})],
+ ]
+ await _run(click_script, [_billable_tool("click", clicks), make_finish_tool()], activity=click_activity)
+ assert click_activity.last_trigger_turn is not None
+
+
def _captcha_tool(results: list[str]) -> ToolSpec:
"""solve_captcha fake: recordable, non-billable, returns each result as a tool ERROR (the
tri-state's not-solved arm) — the arm the false-negative verdicts followed in production."""
@@ -5538,19 +5687,22 @@ async def test_actions_that_name_no_control_never_arm_the_pending_gate() -> None
assert asked == [], (tool_name, asked)
[email protected]("recordable", [False, True])
@pytest.mark.asyncio
-async def test_navigating_away_clears_the_recorded_control() -> None:
+async def test_navigating_away_clears_the_recorded_control(recordable: bool) -> None:
# The run left the page deliberately; the control it clicked went with it, so a marker found at
- # that selector on the new page belongs to something the run never submitted. `navigate` is
- # neither billable nor recordable in the production tool set, so the clear has to be reachable
- # from a plain tool.
+ # that selector on the new page belongs to something the run never submitted. The clear is keyed on
+ # the tool NAME and sits outside the billable/recordable branch, so it must fire whatever navigate's
+ # spec flags say: `recordable=True` is the production shape (SKY-16374), `False` the shape before it.
probe, asked = _pending_probe(_PENDING)
watch = SubmitWatch()
clicks: list[tuple[str, dict[str, Any]]] = []
navigations: list[tuple[str, dict[str, Any]]] = []
+ navigate = _recording_tool("navigate", navigations)
+ navigate.recordable = recordable
tools = [
_billable_tool("click", clicks),
- _recording_tool("navigate", navigations),
+ navigate,
make_finish_tool(pending_marker=probe, submit_watch=watch),
]
script = [
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index ce425920f..5081d6098 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -39,7 +39,7 @@ from skyvern.forge.taskv3.code_surface import (
apply_surface,
configured_surface,
)
-from skyvern.forge.taskv3.loop import CODE_TOOL_NAME, SemanticCommitStats, ToolSpec
+from skyvern.forge.taskv3.loop import ACTION_OUTCOME_DATA_KEY, CODE_TOOL_NAME, SemanticCommitStats, ToolSpec
from skyvern.forge.taskv3.tools import (
_OPAQUE_ID_RUN_RE,
_SEMANTIC_COMMIT_STATE_JS,
@@ -3207,6 +3207,50 @@ async def test_navigate_reports_http_status(monkeypatch: pytest.MonkeyPatch) ->
assert "HTTP 400" in r.content
+async def test_navigate_is_recordable_and_reports_the_outcome_of_the_navigation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # SKY-16374: a URL the model typed itself is an action the customer must see, so navigate is
+ # recordable (one action row + that round's screenshot) while staying out of billing and the
+ # action-step budget. The row is only worth having if it says what HAPPENED, so the handler
+ # reports where it asked to go, where it landed, what the page answered, and whether the page
+ # moved -- the facts the caller persists on the row.
+ import skyvern.utils.url_validators as urlv
+
+ monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)
+
+ class _RedirectTo404(_FakePage):
+ async def goto(self, url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:
+ self.calls.append(("goto", {"url": url}))
+ self.url = "https://example.test/not-found"
+ return SimpleNamespace(status=404)
+
+ tools = build_browser_tools(_fixed_page_provider(_RedirectTo404()))
+ navigate = _tool(tools, "navigate")
+ assert (navigate.recordable, navigate.billable) == (True, False)
+ r = await navigate.handler({"url": "https://example.test/contact-us"})
+ assert r.status == "ok", r.content
+ assert (r.data or {}).get(ACTION_OUTCOME_DATA_KEY) == {
+ "requested_url": "https://example.test/contact-us",
+ "url": "https://example.test/not-found",
+ "http_status": 404,
+ "page_transitioned": True,
+ "navigation_dead_end": 404,
+ }
+
+ # A navigation that landed back where it started reports no transition, and a page that answered
+ # no response carries no status at all rather than a fabricated one.
+ page, same_page_tools = _reload_guard_tools(monkeypatch, filled=0)
+ r2 = await _tool(same_page_tools, "navigate").handler({"url": page.url})
+ assert r2.status == "ok", r2.content
+ assert (r2.data or {}).get(ACTION_OUTCOME_DATA_KEY) == {
+ "requested_url": page.url,
+ "url": page.url,
+ "page_transitioned": False,
+ }
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_observe_digest_containment_dedupe_keeps_one_banner_copy() -> None:
@@ -4904,6 +4948,11 @@ async def test_navigate_dead_end_terminates_run_through_real_handler(monkeypatch
[("navigate", {"url": "https://jobs.example.test/acme/closed"})],
[("finish", {"status": "completed", "reason": "should not win"})],
]
+ rounds: list[list[Any]] = []
+
+ async def _on_round(round_actions: list[Any], _turn_text: str | None) -> None:
+ rounds.append(round_actions)
+
outcome = await run_agent_tool_loop(
llm_caller=_ScriptedCaller(script),
system_prompt="sys",
@@ -4911,9 +4960,23 @@ async def test_navigate_dead_end_terminates_run_through_real_handler(monkeypatch
tools=all_tools,
max_turns=10,
max_tool_calls=20,
+ on_action_round=_on_round,
)
assert outcome.status == "terminated"
+ # ...and the whole production chain behind the row the customer reads: the real spec is recordable,
+ # so the navigation reaches the caller as its own round; the real handler's outcome rides it; and a
+ # 404 landing is flagged as a FAILED action even though the tool honestly returned ok (SKY-16374).
+ assert [(a.tool, a.succeeded, a.billable) for round_actions in rounds for a in round_actions] == [
+ ("navigate", False, False)
+ ]
+ assert rounds[0][0].outcome == {
+ "requested_url": "https://jobs.example.test/acme/closed",
+ "url": "https://jobs.example.test/acme/closed",
+ "http_status": 404,
+ "page_transitioned": True,
+ "navigation_dead_end": 404,
+ }
# --- Commit-verified click-open dropdown selection. The staging specimen: a click-open