Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Persist Task V3's own URL navigations as goto_url actions (SKY-16374) (#8593)
Shuchang Zhengc288f170706b
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex af8751b30..6d402422d 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -140,6 +140,7 @@ from skyvern.forge.sdk.core.skyvern_context import (MultiFieldTotpAttempt,SkyvernContext,action_for_multi_field_totp_persistence,+ canonical_url,redact_multi_field_totp_element_data,)from skyvern.forge.sdk.db.datetime_utils import naive_utc_now@@ -389,9 +390,18 @@ _TASKV3_TOOL_ACTION_TYPES = {}-def _taskv3_row_intention(task: Task, round_action: RoundAction, secret_values: set[str]) -> str | None:+def _taskv3_row_intention(+ task: Task, round_action: RoundAction, secret_values: set[str], redacted_args: dict[str, Any]+) -> str | None:"""The persisted label for one v3 action. This is the only frame that holds the unfloored drop-checksecrets, and a failure here is logged without exc_info, so no traceback renderer can print them."""+ if round_action.tool == "navigate":+ # A navigation names no element, so the element-label composer has nothing to say about it:+ # the URL is the whole subject. Read from the already-redacted args, never the raw ones.+ url = redacted_args.get("url")+ if not isinstance(url, str) or not url:+ return None+ return f"Navigated to {url}" if round_action.succeeded else f"Tried to navigate to {url}"try:# Drop-only: this set decides whether to drop a page-supplied name and reaches no redaction path.label_secret_values = (@@ -435,6 +445,36 @@ def _redact_tool_arg(value: Any, secret_values: set[str]) -> Any:# Cap on the persisted turn text: actions is a high-traffic table and readers clamp far lower for# display; a runaway multi-paragraph turn must not become a per-row payload._TASKV3_REASONING_MAX_CHARS = 1000+# Same reason, for the outcome line: it can carry two page-supplied URLs (or, when the call reached no+# page, the tool's own error text), and neither a URL nor a tool error has a length bound.+_TASKV3_RESPONSE_MAX_CHARS = 500+++def _taskv3_row_response(outcome: dict[str, Any]) -> str | None:+ """The persisted outcome of one v3 action: where it asked to go, where it landed, and what the page+ answered. Reads only the keys it knows (see ACTION_OUTCOME_DATA_KEY), so a tool reporting something+ else leaves the row's response empty rather than printing a shape nobody can read."""+ requested = outcome.get("requested_url") if isinstance(outcome.get("requested_url"), str) else None+ landed = outcome.get("url") if isinstance(outcome.get("url"), str) else None+ where = requested or landed or ""+ # Canonical comparison, the same identity the handler classified `page_transitioned` on, so a+ # landing that only re-spells the requested URL (a trailing slash, an escape case) is not+ # rendered as a move the row then calls "same page".+ if requested and landed and canonical_url(landed) != canonical_url(requested):+ where = f"{requested} -> {landed}"+ notes: list[str] = []+ status = outcome.get("http_status")+ if isinstance(status, int):+ notes.append(f"HTTP {status}")+ if outcome.get("navigation_dead_end") is not None:+ notes.append("dead end")+ transitioned = outcome.get("page_transitioned")+ if isinstance(transitioned, bool):+ notes.append("page changed" if transitioned else "same page")+ if not where:+ return ", ".join(notes) or None+ return f"{where} ({', '.join(notes)})" if notes else where+# Block-engine to run-type labels for the "Task duration metrics" discriminator (SKY-15499):# workflow-block tasks have no task_runs row, so the block's resolved engine stands in.@@ -2170,27 +2210,54 @@ class ForgeAgent:# can be glued to adjacent alphanumerics — boundary anchoring would let it through.turn_reasoning = redact_secrets_from_text(turn_reasoning, secret_values)turn_reasoning = turn_reasoning[:_TASKV3_REASONING_MAX_CHARS]+ # A round that bills nothing claims no budget unit, so its rows ride the LAST consumed+ # index (or the single Step's own order 0) the way the decision row below does: a fresh+ # index nothing later claims would be a distinct (task_id, step_order) pair, and the+ # workflow-run step budget counts those pairs.+ billable_round = any(entry.billable for entry in round_actions)+ row_step_order = v3_round_index if billable_round else max(v3_round_index - 1, 0)for round_action in round_actions:name, args, succeeded = round_action.tool, round_action.args, round_action.succeededtry:tool_args = _redact_tool_args(args if isinstance(args, dict) else {}, secret_values)selector = tool_args.get("selector", "")+ row_response = _taskv3_row_response(round_action.outcome) if round_action.outcome else None+ if row_response is not None:+ # The outcome carries page-supplied URLs the arg redaction above never saw, and a+ # landing URL's query routinely holds the credential itself (a sign-in link's token).+ if secret_values:+ row_response = redact_secrets_from_text(+ row_response, secret_values, boundary_all_lengths=True+ )+ elif name == "navigate" and round_action.error:+ # A navigation the engine refused on purpose (the destructive same-URL reload+ # guard), or one that never loaded, reached no page and reports no outcome: the+ # tool's own error is all the row can say, and without it the customer reads a+ # failed navigation with no reason. Free-form prose, so it is matched the way the+ # turn text is rather than the way the URL line above is.+ row_response = round_action.error+ if secret_values:+ row_response = redact_secrets_from_text(row_response, secret_values)+ if row_response is not None:+ row_response = row_response[:_TASKV3_RESPONSE_MAX_CHARS]action = _taskv3_action_for_tool_call(name,tool_args,status=ActionStatus.completed if succeeded else ActionStatus.failed,+ response=row_response,organization_id=task.organization_id,workflow_run_id=task.workflow_run_id,task_id=task.task_id,step_id=step.step_id,- # Round index, not the single Step's order: makes each v3 action ROUND count- # as one unit of the workflow-run step budget (distinct (task, order) pairs).- step_order=v3_round_index,+ # Round index, not the single Step's order: makes each BILLABLE v3 action+ # ROUND count as one unit of the workflow-run step budget (distinct (task,+ # order) pairs).+ step_order=row_step_order,action_order=len(v3_persisted_actions),description=f"{TASK_V3_ACTION_DESCRIPTION_PREFIX}{name} {selector}".strip(),screenshot_artifact_id=screenshot_artifact_id,reasoning=turn_reasoning,- intention=_taskv3_row_intention(task, round_action, secret_values),+ intention=_taskv3_row_intention(task, round_action, secret_values, tool_args),)v3_persisted_actions.append(action)await app.DATABASE.workflow_params.create_action(@@ -2198,9 +2265,7 @@ class ForgeAgent:)except Exception:LOG.warning("task_v3 failed to persist action row", task_id=task.task_id, exc_info=True)- if any(entry.billable for entry in round_actions):- # Only billable rounds consume the budget unit; recordable-only rounds (navigate/- # scroll/wait) keep the current index so they never inflate the workflow-run count.+ if billable_round:v3_round_index += 1pre_submit_ring: PreSubmitCaptureRing | None = Nonediff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex c2cce32ae..3b556040e 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -154,6 +154,14 @@ ToolHandler = Callable[[dict[str, Any]], Awaitable[ToolResult]]# lives in target_label.py; this module only carries the two raw values from probe to `RoundAction`.TARGET_LABEL_DATA_KEY = "target_label"TARGET_KIND_DATA_KEY = "target_kind"+# The tool-result `data` key a recorded action's outcome rides on: the machine facts about what the+# call achieved (`requested_url`, `url`, `http_status`, `page_transitioned`, `navigation_dead_end`),+# carried verbatim to `RoundAction.outcome` for the caller to persist on the action row. Internal to+# the loop like the two keys above -- never shown to the model, which reads the tool's own content.+ACTION_OUTCOME_DATA_KEY = "action_outcome"+# An HTTP status at or above this reached no usable page, so the action row reads as failed even+# though the tool honestly returned ok (the model still gets the status and decides what to do).+ACTION_OUTCOME_FAILED_HTTP_STATUS = 400# How long a call spent turning an address into a target, before the act. A context variable rather# than a field on the result, because the cohort this exists to price is the one where the handler# RAISES -- a driver timeout on a resolved target -- and a result the handler never returned cannot@@ -312,6 +320,14 @@ class RoundAction(NamedTuple):# downstream: the loop already read it off the ToolSpec, and a second name list is a second place# for a new tool to be missing from.billable: bool = False+ # What the action achieved, as the tool itself reported it (see ACTION_OUTCOME_DATA_KEY): machine+ # facts only, for the caller to persist alongside the verb. None when the tool reported none --+ # a call that errored before it reached a page has only its `error` below to offer.+ outcome: dict[str, Any] | None = None+ # The tool's own error text when the call failed, which is all a call that never reached a page+ # can say about itself -- a refusal the engine issued on purpose reads as an unexplained failed+ # row without it. Model-facing prose, so a caller that persists it must redact and cap it.+ error: str | None = None# A probe consulted after a billable/download-signaling tool result; a truthy return ends the run as@@ -1074,6 +1090,16 @@ def _is_finish(tool_name: str) -> bool:return tool_name == "finish"+def _outcome_reports_failure(outcome: dict[str, Any] | None) -> bool:+ """Whether a tool that returned ok nonetheless reported reaching no usable page. Read off the+ machine facts the tool exposed (an HTTP status), so the tool never has to adjudicate its own+ success -- and only the persisted row moves: the model still reads the tool's own ok result."""+ if not outcome:+ return False+ status = outcome.get("http_status")+ return isinstance(status, int) and status >= ACTION_OUTCOME_FAILED_HTTP_STATUS++def _arms_failure_evidence(tool_name: str, args: dict[str, Any], ok: bool) -> bool:"""solve_captcha arms on ANY dispatch — its "not solved" error is exactly the verdict the asyncprotocol can contradict. Other actions arm only when they reached the page AND in their@@ -3393,21 +3419,25 @@ async def run_agent_tool_loop():action_nudges_due.append((tool_name, args, repeat_count))if submit_watch is not None and tool_name == "navigate" and result.status == "ok":- # Outside the billable/recordable branch on purpose: navigate is neither, so a clear- # placed in there never runs. The run left the page; the control it clicked went too.+ # Outside the billable/recordable branch on purpose: the run left the page and the+ # control it clicked went too, whatever navigate's spec flags happen to say.submit_watch.clear()if spec is not None and (spec.billable or spec.recordable):# Dispatched page actions enter the round with their outcome: a failed billable round# still consumed budget and must persist (else later blocks undercount the run# budget); recordable tools persist for artifact parity without billing/budget.+ round_outcome = result_data.get(ACTION_OUTCOME_DATA_KEY)+ round_outcome = round_outcome if isinstance(round_outcome, dict) else Noneround_actions.append(RoundAction(tool_name,args,- result.status == "ok",+ result.status == "ok" and not _outcome_reports_failure(round_outcome),result_data.get(TARGET_LABEL_DATA_KEY) or None,result_data.get(TARGET_KIND_DATA_KEY) or None,spec.billable,+ round_outcome,+ result.content if result.status == "error" else None,))if spec.billable and result.status == "ok":diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex c6a5b50f2..fb4d8f8c0 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -46,6 +46,7 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import URL_IN_TEXT, canonical_url, opaque_url_echo_windowfrom skyvern.forge.taskv3.frame_perception import frame_perception_enabledfrom skyvern.forge.taskv3.loop import (+ ACTION_OUTCOME_DATA_KEY,FILL_TOOLS,NAVIGATION_DEAD_END_STATUSES,PAGE_UNAVAILABLE_ERROR,@@ -11809,6 +11810,16 @@ def build_browser_tools(# terminated (v1's behavior) rather than defaulting the outcome to failed.if response is not None and response.status in NAVIGATION_DEAD_END_STATUSES:data["navigation_dead_end"] = response.status+ # What the persisted action row says happened. The requested URL is the model's own argument+ # (a placeholder or payload ref must not be unwrapped into a row), and the dead-end status is+ # repeated from the loop's signal above because the row never sees that one.+ outcome: dict[str, Any] = {"requested_url": requested, "url": landed}+ if response is not None:+ outcome["http_status"] = response.status+ outcome["page_transitioned"] = landed_canonical != pre_nav_canonical+ if "navigation_dead_end" in data:+ outcome["navigation_dead_end"] = data["navigation_dead_end"]+ data[ACTION_OUTCOME_DATA_KEY] = outcomereturn ToolResult.ok(f"navigated to {landed}{status}", data=data)async def file_upload(args: dict[str, Any]) -> ToolResult:@@ -12880,6 +12891,11 @@ def build_browser_tools("file_upload",):_tool_spec.billable = True+ if _tool_spec.name == "navigate":+ # Recordable, not billable: a URL the model typed itself is an action the customer needs+ # to see (one action row + the round's screenshot), but it mutates no page, so it must not+ # consume the action-step budget or meter like one that does.+ _tool_spec.recordable = Trueif _tool_spec.name in ("observe", "get_html", "look"):# Large perception dumps: only the latest snapshot is relevant, so let the loop elide older# ones from the re-sent transcript (bounds context on perception-heavy pages). look's legenddiff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex b32113305..7f4ceffd0 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -59,6 +59,7 @@ from skyvern.webeye.actions.actions import (ActionStatus,ActionType,ClickAction,+ GotoUrlAction,HoverAction,InputTextAction,KeypressAction,@@ -895,6 +896,229 @@ async def test_execute_task_v3_persists_per_action_screenshots_and_rows(monkeypaassert all(a.response is None for a in persisted)+_GOTO_OUTCOME = {+ "requested_url": "https://forms.example.test/contact-us",+ "url": "https://forms.example.test/contact",+ "http_status": 200,+ "page_transitioned": True,+}+++async def test_execute_task_v3_persists_a_navigation_as_a_goto_url_row(monkeypatch: pytest.MonkeyPatch) -> None:+ # SKY-16374: a URL the model typed itself is an action on a par with a click, so it persists as a+ # `goto_url` row carrying the outcome -- where it asked to go, where it landed, what the page+ # answered. It must not meter: nothing billable, and the round does not advance the workflow-run+ # step index (so a navigation cannot inflate a workflow's step budget).+ from skyvern.forge import agent as agent_mod++ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])+ rounds = [+ [RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, True, None, None, False, _GOTO_OUTCOME)],+ [RoundAction("click", {"selector": "#send"}, True, billable=True)],+ ]+ step, _task, _loop, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ action_rounds=rounds,+ action_round_texts=["the contact link 404s, typing the contact URL instead", "sending the form"],+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]+ navigation, click = persisted[0], persisted[1]+ assert navigation.action_type == ActionType.GOTO_URL+ assert isinstance(navigation, GotoUrlAction) and navigation.url == _GOTO_OUTCOME["requested_url"]+ assert navigation.status == ActionStatus.completed+ assert navigation.intention == f"Navigated to {_GOTO_OUTCOME['requested_url']}"+ assert navigation.reasoning == "the contact link 404s, typing the contact URL instead"+ assert navigation.screenshot_artifact_id == "artifact-1" # the round's own screenshot+ assert navigation.response is not None+ assert _GOTO_OUTCOME["url"] in navigation.response and "HTTP 200" in navigation.response+ # Recorded, never metered: the navigation round leaves the step index where it was, so the click+ # that follows it is still round 0, and only the click bills.+ assert (navigation.step_order, click.step_order) == (0, 0)+ assert len(step.output.actions_and_results) == 1+++def test_a_navigation_row_calls_a_move_a_move_by_the_same_identity_the_page_was_judged_by() -> None:+ # The row renders "requested -> landed" and "page changed / same page" out of the SAME outcome, so+ # the two have to agree. The handler judged the transition on CANONICAL URLs, so comparing the raw+ # strings here made a browser re-spelling (the model types a bare host, the page reports it with+ # the root slash) read as a move the very next clause calls "same page".+ same_page = {+ "requested_url": "https://forms.example.test",+ "url": "https://forms.example.test/",+ "http_status": 200,+ "page_transitioned": False,+ }+ assert agent_module._taskv3_row_response(same_page) == "https://forms.example.test (HTTP 200, same page)"+ moved = dict(same_page, url="https://forms.example.test/thanks", page_transitioned=True)+ assert agent_module._taskv3_row_response(moved) == (+ "https://forms.example.test -> https://forms.example.test/thanks (HTTP 200, page changed)"+ )+++async def test_execute_task_v3_keeps_a_trailing_navigation_off_the_step_budget(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # The workflow-run step budget counts DISTINCT (task_id, step_order) pairs across steps and+ # round-stamped action rows (get_total_unique_progress_round_count_by_task_ids), so a round that+ # bills nothing must never open a pair of its own. A navigation BEFORE a billable round shares+ # the index that round goes on to claim; one AFTER the last billable round has nobody left to+ # share with, so it has to step back onto the index already spent -- the way the decision row does.+ from skyvern.forge import agent as agent_mod++ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])++ async def _persisted(rounds: list[list[RoundAction]]) -> set[int | None]:+ await _run_execute_task_v3(+ monkeypatch,+ outcome,+ action_rounds=rounds,+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ calls = agent_mod.app.DATABASE.workflow_params.create_action.await_args_list+ return {call.kwargs["action"].step_order for call in calls}++ click_round: list[RoundAction] = [RoundAction("click", {"selector": "#send"}, True, billable=True)]+ nav_round: list[RoundAction] = [+ RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, True, None, None, False, _GOTO_OUTCOME)+ ]+ assert await _persisted([click_round]) == {0}+ # Same run, one navigation appended: the budget the run spends must be identical.+ assert await _persisted([click_round, nav_round]) == {0}+ # And a run whose only action is a navigation spends the one unit the Step row already occupies.+ assert await _persisted([nav_round]) == {0}+++async def test_execute_task_v3_persists_a_dead_end_navigation_as_a_failed_row(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # The live shape this exists for: two navigations the model typed, the second landing on a hard 404,+ # then the loop's dead-end terminate. The 404 must read as a FAILED goto_url row carrying the status,+ # with the terminate row still following it -- not a terminate out of nowhere.+ from skyvern.forge import agent as agent_mod++ dead_end = {+ "requested_url": "https://forms.example.test/get-in-touch",+ "url": "https://forms.example.test/get-in-touch",+ "http_status": 404,+ "page_transitioned": True,+ "navigation_dead_end": 404,+ }+ outcome = LoopOutcome(+ status="terminated",+ reason="navigation_dead_end: navigate landed on a dead page (HTTP 404)",+ billable_actions=[],+ )+ rounds = [+ [+ RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, True, None, None, False, _GOTO_OUTCOME),+ RoundAction("navigate", {"url": dead_end["requested_url"]}, False, None, None, False, dead_end),+ ]+ ]+ _step, _task, _loop, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ action_rounds=rounds,+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]+ assert [a.action_type for a in persisted] == [ActionType.GOTO_URL, ActionType.GOTO_URL, ActionType.TERMINATE]+ live, dead, terminate = persisted+ assert (live.status, dead.status) == (ActionStatus.completed, ActionStatus.failed)+ assert dead.response is not None and "HTTP 404" in dead.response and "dead end" in dead.response+ assert live.intention == f"Navigated to {_GOTO_OUTCOME['requested_url']}"+ assert dead.intention == f"Tried to navigate to {dead_end['requested_url']}"+ assert terminate.reasoning == outcome.reason+++async def test_execute_task_v3_persists_a_failed_navigation_with_the_reason_it_failed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # A navigate whose tool call errored (a load timeout, the destructive-reload refusal the engine+ # issues on purpose) reached no page, so it reports no outcome -- but a failed row with an empty+ # response reads as a navigation that broke for no reason, which is worse than not showing it. The+ # tool's own error is what the row has to say, and it is scrubbed like every other persisted text.+ from skyvern.forge import agent as agent_mod++ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: True+ )+ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run",+ lambda *_a, **_k: {"sk4829137765"},+ )+ refusal = (+ "already on this page and it has filled fields (including any attached file); reloading it "+ "would discard them. Act on the current page instead. token=sk4829137765"+ )+ outcome = LoopOutcome(status="failed", reason="could not load the page", billable_actions=[])+ rounds = [+ [RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, False, None, None, False, None, refusal)],+ # ...and a failure the tool said nothing about still leaves the response empty rather than+ # inventing one.+ [RoundAction("navigate", {"url": _GOTO_OUTCOME["requested_url"]}, False)],+ ]+ _step, _task, _loop, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ action_rounds=rounds,+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]+ refused, silent = persisted[0], persisted[1]+ assert refused.action_type == ActionType.GOTO_URL+ assert refused.status == ActionStatus.failed+ assert refused.intention == f"Tried to navigate to {_GOTO_OUTCOME['requested_url']}"+ assert refused.response is not None and refused.response.startswith("already on this page")+ assert "sk4829137765" not in refused.response and REDACTED_SECRET_PLACEHOLDER in refused.response+ assert silent.response is None+++async def test_execute_task_v3_scrubs_a_secret_out_of_a_navigation_row(monkeypatch: pytest.MonkeyPatch) -> None:+ # A landed URL is page-supplied and its query routinely carries the credential itself (a sign-in+ # link's token). It reaches a persisted, displayed row for the first time here, so it goes through+ # the same scrub the row's args already get -- in the intention AND in the outcome line.+ from skyvern.forge import agent as agent_mod++ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: True+ )+ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run",+ lambda *_a, **_k: {"sk4829137765"},+ )+ landed = {+ "requested_url": "https://forms.example.test/login?token=sk4829137765",+ "url": "https://forms.example.test/account?session=sk4829137765",+ "http_status": 200,+ "page_transitioned": True,+ }+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])+ rounds = [[RoundAction("navigate", {"url": landed["requested_url"]}, True, None, None, False, landed)]]+ _step, _task, _loop, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ action_rounds=rounds,+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ navigation = agent_mod.app.DATABASE.workflow_params.create_action.await_args_list[0].kwargs["action"]+ assert "sk4829137765" not in (navigation.response or "")+ assert "sk4829137765" not in (navigation.intention or "")+ assert "sk4829137765" not in navigation.url+ assert REDACTED_SECRET_PLACEHOLDER in (navigation.response or "")++def test_every_tool_that_pays_for_a_name_has_something_to_say_about_it() -> None:# The set that triggers the capture lives with the tools; the verbs that render it live in# target_label.py. A tool added to one and not the other pays the probe on every call and throws@@ -3616,7 +3840,8 @@ async def test_execute_task_v3_recordable_round_persists_without_budget_unit(monkeypatch: pytest.MonkeyPatch,) -> None:# navigate/scroll/wait persist as action rows with screenshots (artifact parity) but never- # consume a workflow-run budget unit: their rows keep the current round index.+ # consume a workflow-run budget unit: their rows share the step_order of the billable round+ # nearest them, ahead OR behind, so they open no (task_id, step_order) pair of their own.outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])rounds = [[RoundAction("navigate", {"url": "https://a.test"}, True)],@@ -3633,7 +3858,9 @@ async def test_execute_task_v3_recordable_round_persists_without_budget_unit(assert stamped == [(ActionType.GOTO_URL, 0),(ActionType.CLICK, 0),- (ActionType.SCROLL, 1),+ # The trailing scroll rides the index the click already spent: a fresh one would be a second+ # distinct pair for the budget to count, charging the run a step nothing billable claimed.+ (ActionType.SCROLL, 0),]diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex 65e782fb7..aae18eedc 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -37,6 +37,7 @@ from skyvern.forge.taskv3.loop import (ACTION_LOOP_NUDGE_AFTER,ACTION_LOOP_REASON_PREFIX,ACTION_LOOP_TERMINATE_AFTER,+ ACTION_OUTCOME_DATA_KEY,CODE_TOOL_NAME,FAILURE_EVIDENCE_MIN_TOOL_CALLS,FAILURE_EVIDENCE_MIN_TURNS,@@ -3095,7 +3096,7 @@ async def test_on_action_round_fires_for_all_failed_round_with_failure_flag() ->script = [[("click", {})], [("finish", {"status": "completed", "reason": "ok"})]]outcome, _ = await _run(script, [click, make_finish_tool()], on_action_round=_on_round)assert outcome.status == "completed"- assert rounds == [[RoundAction("click", {}, False, billable=True)]]+ assert rounds == [[RoundAction("click", {}, False, billable=True, error="tool_error: RuntimeError: boom")]]assert outcome.billable_actions == [] # billing still counts successes only@@ -4440,6 +4441,154 @@ async def test_navigate_resets_action_counters() -> None:assert len(clicks) == 10+_NAV_URL = "https://forms.example.test/contact-us"+++def _recordable_navigate(sink: list[tuple[str, dict[str, Any]]], *, outcome: dict[str, Any] | None = None) -> ToolSpec:+ """The production navigate shape (SKY-16374): recordable so the navigation persists as an action+ row with the round's screenshot, never billable, and carrying the outcome the caller writes onto+ that row."""++ async def handler(args: dict[str, Any]) -> ToolResult:+ sink.append(("navigate", args))+ data: dict[str, Any] = {"page_state_changed": True}+ if outcome is not None:+ data[ACTION_OUTCOME_DATA_KEY] = outcome+ if outcome.get("navigation_dead_end") is not None:+ # As the real handler does: the top-level flag is what the loop's dead-end verdict+ # reads, the outcome's copy is what the persisted row carries.+ data["navigation_dead_end"] = outcome["navigation_dead_end"]+ return ToolResult.ok("navigated", data=data)++ return ToolSpec(+ name="navigate",+ description="n",+ parameters={"type": "object", "properties": {}},+ handler=handler,+ recordable=True,+ )+++async def test_a_navigation_reaches_the_round_with_its_outcome_and_costs_no_budget() -> None:+ # The customer must see a URL the model typed itself, so navigate enters the action round like a+ # click — carrying what the navigation achieved, not just the verb. It stays unbilled and unbudgeted:+ # a navigation is not a page-mutating step, and a recorded row must never start metering.+ rounds: list[list[RoundAction]] = []++ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:+ rounds.append(actions)++ navs: list[tuple[str, dict[str, Any]]] = []+ nav_outcome = {"requested_url": _NAV_URL, "url": _NAV_URL, "http_status": 200, "page_transitioned": True}+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "completed", "reason": "read it"})]]+ outcome, _ = await _run(+ script, [_recordable_navigate(navs, outcome=nav_outcome), make_finish_tool()], on_action_round=_on_round+ )+ assert outcome.status == "completed"+ assert rounds == [[RoundAction("navigate", {"url": _NAV_URL}, True, None, None, False, nav_outcome)]]+ assert outcome.billable_actions == []+ assert outcome.action_steps == 0+++async def test_a_navigation_that_answered_an_http_error_is_recorded_as_a_failed_action() -> None:+ # A 404 landing stays a ToolResult.ok — the model reads the status and decides what to do, and that+ # transcript must not change. The ROW is the customer's view, and a navigation onto a dead page has+ # to read as a failed one there, or a run that died on a 404 shows a terminate out of nowhere.+ rounds: list[list[RoundAction]] = []++ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:+ rounds.append(actions)++ navs: list[tuple[str, dict[str, Any]]] = []+ nav_outcome = {"requested_url": _NAV_URL, "url": _NAV_URL, "http_status": 404, "navigation_dead_end": 404}+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "completed", "reason": "should not win"})]]+ outcome, _ = await _run(+ script, [_recordable_navigate(navs, outcome=nav_outcome), make_finish_tool()], on_action_round=_on_round+ )+ assert [action.succeeded for round_actions in rounds for action in round_actions] == [False]+ assert [action.outcome for round_actions in rounds for action in round_actions] == [nav_outcome]+ # The dead-end verdict still ends the run (the row explains it; it does not replace it), and the+ # model still read the tool's own ok result.+ assert outcome.status == "terminated" and outcome.reason.startswith(NAV_DEAD_END_REASON_PREFIX)+ assert [m["content"] for m in outcome.messages if m.get("name") == "navigate"] == ["navigated"]+++async def test_a_recorded_call_that_failed_carries_its_error_text_to_the_round() -> None:+ # A navigation that reached no page -- a load timeout, or the destructive-reload guard's deliberate+ # refusal -- has no outcome to report, so the tool's own error is the only thing its persisted row+ # can say about it. The caller has no other view of the call, and a failed row that explains+ # nothing reads as a navigation that broke for no reason.+ rounds: list[list[RoundAction]] = []++ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:+ rounds.append(actions)++ navs: list[tuple[str, dict[str, Any]]] = []+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "failed", "reason": "never loaded"})]]+ await _run(+ script,+ [_erroring_tool("navigate", navs, recordable=True), make_finish_tool()],+ on_action_round=_on_round,+ )+ assert [(a.succeeded, a.outcome, a.error) for round_actions in rounds for a in round_actions] == [+ (False, None, "navigate failed")+ ]+++async def test_repeated_navigations_to_one_url_are_recorded_without_arming_the_action_loop_guard() -> None:+ # CONTRACT (deliberate exclusion): the action-loop terminator keys on BILLABLE calls, and a+ # navigation is recorded but not billable, so repeating one is not treated like repeating a click.+ # The guard exists to stop a run burning its action-step budget on an action that cannot progress,+ # and a navigation spends none of it; and a successful navigate is itself page-change evidence,+ # which clears the streak ledger, so counting navigations could never reach the verdict anyway.+ # Recording a row must not change what ends a run.+ rounds: list[list[RoundAction]] = []++ async def _on_round(actions: list[RoundAction], _turn_text: str | None) -> None:+ rounds.append(actions)++ navs: list[tuple[str, dict[str, Any]]] = []+ repeats = ACTION_LOOP_TERMINATE_AFTER + 2+ script = [[("navigate", {"url": _NAV_URL})] for _ in range(repeats)]+ script.append([("finish", {"status": "completed", "reason": "found the page in the end"})])+ outcome, _ = await _run(+ script,+ [_recordable_navigate(navs), make_finish_tool()],+ on_action_round=_on_round,+ max_turns=200,+ max_tool_calls=500,+ )+ assert outcome.status == "completed"+ assert len(navs) == repeats # never cut short by the action-loop verdict+ assert len(rounds) == repeats # and every one of them persisted as its own round+++async def test_a_navigation_does_not_arm_the_finish_gate_failure_evidence() -> None:+ # navigate now enters the same round branch that arms the failure-evidence trigger, which exists+ # for actions that may have SUBMITTED something. A navigation has not, so arming it would make the+ # finish gate hold verdicts on runs whose last action merely changed page.+ nav_activity = ActivityRecency(turn=0, turns_remaining=5, tool_calls_remaining=10)+ navs: list[tuple[str, dict[str, Any]]] = []+ script = [[("navigate", {"url": _NAV_URL})], [("finish", {"status": "failed", "reason": "no contact form"})]]+ await _run(script, [_recordable_navigate(navs), make_finish_tool()], activity=nav_activity)+ assert nav_activity.last_trigger_turn is None++ # Control: a click on the same path does arm it, so the None above is a live assertion.+ click_activity = ActivityRecency(turn=0, turns_remaining=5, tool_calls_remaining=10)+ clicks: list[tuple[str, dict[str, Any]]] = []+ click_script = [+ [("click", {"selector": "#submit"})],+ [("finish", {"status": "failed", "reason": "no contact form"})],+ ]+ await _run(click_script, [_billable_tool("click", clicks), make_finish_tool()], activity=click_activity)+ assert click_activity.last_trigger_turn is not None++def _captcha_tool(results: list[str]) -> ToolSpec:"""solve_captcha fake: recordable, non-billable, returns each result as a tool ERROR (thetri-state's not-solved arm) — the arm the false-negative verdicts followed in production."""@@ -5538,19 +5687,22 @@ async def test_actions_that_name_no_control_never_arm_the_pending_gate() -> Noneassert asked == [], (tool_name, asked)[email protected]("recordable", [False, True])@pytest.mark.asyncio-async def test_navigating_away_clears_the_recorded_control() -> None:+async def test_navigating_away_clears_the_recorded_control(recordable: bool) -> None:# The run left the page deliberately; the control it clicked went with it, so a marker found at- # that selector on the new page belongs to something the run never submitted. `navigate` is- # neither billable nor recordable in the production tool set, so the clear has to be reachable- # from a plain tool.+ # that selector on the new page belongs to something the run never submitted. The clear is keyed on+ # the tool NAME and sits outside the billable/recordable branch, so it must fire whatever navigate's+ # spec flags say: `recordable=True` is the production shape (SKY-16374), `False` the shape before it.probe, asked = _pending_probe(_PENDING)watch = SubmitWatch()clicks: list[tuple[str, dict[str, Any]]] = []navigations: list[tuple[str, dict[str, Any]]] = []+ navigate = _recording_tool("navigate", navigations)+ navigate.recordable = recordabletools = [_billable_tool("click", clicks),- _recording_tool("navigate", navigations),+ navigate,make_finish_tool(pending_marker=probe, submit_watch=watch),]script = [diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex ce425920f..5081d6098 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -39,7 +39,7 @@ from skyvern.forge.taskv3.code_surface import (apply_surface,configured_surface,)-from skyvern.forge.taskv3.loop import CODE_TOOL_NAME, SemanticCommitStats, ToolSpec+from skyvern.forge.taskv3.loop import ACTION_OUTCOME_DATA_KEY, CODE_TOOL_NAME, SemanticCommitStats, ToolSpecfrom skyvern.forge.taskv3.tools import (_OPAQUE_ID_RUN_RE,_SEMANTIC_COMMIT_STATE_JS,@@ -3207,6 +3207,50 @@ async def test_navigate_reports_http_status(monkeypatch: pytest.MonkeyPatch) ->assert "HTTP 400" in r.content+async def test_navigate_is_recordable_and_reports_the_outcome_of_the_navigation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # SKY-16374: a URL the model typed itself is an action the customer must see, so navigate is+ # recordable (one action row + that round's screenshot) while staying out of billing and the+ # action-step budget. The row is only worth having if it says what HAPPENED, so the handler+ # reports where it asked to go, where it landed, what the page answered, and whether the page+ # moved -- the facts the caller persists on the row.+ import skyvern.utils.url_validators as urlv++ monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)++ class _RedirectTo404(_FakePage):+ async def goto(self, url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:+ self.calls.append(("goto", {"url": url}))+ self.url = "https://example.test/not-found"+ return SimpleNamespace(status=404)++ tools = build_browser_tools(_fixed_page_provider(_RedirectTo404()))+ navigate = _tool(tools, "navigate")+ assert (navigate.recordable, navigate.billable) == (True, False)+ r = await navigate.handler({"url": "https://example.test/contact-us"})+ assert r.status == "ok", r.content+ assert (r.data or {}).get(ACTION_OUTCOME_DATA_KEY) == {+ "requested_url": "https://example.test/contact-us",+ "url": "https://example.test/not-found",+ "http_status": 404,+ "page_transitioned": True,+ "navigation_dead_end": 404,+ }++ # A navigation that landed back where it started reports no transition, and a page that answered+ # no response carries no status at all rather than a fabricated one.+ page, same_page_tools = _reload_guard_tools(monkeypatch, filled=0)+ r2 = await _tool(same_page_tools, "navigate").handler({"url": page.url})+ assert r2.status == "ok", r2.content+ assert (r2.data or {}).get(ACTION_OUTCOME_DATA_KEY) == {+ "requested_url": page.url,+ "url": page.url,+ "page_transitioned": False,+ }++@_skip_no_browser@pytest.mark.asyncioasync def test_observe_digest_containment_dedupe_keeps_one_banner_copy() -> None:@@ -4904,6 +4948,11 @@ async def test_navigate_dead_end_terminates_run_through_real_handler(monkeypatch[("navigate", {"url": "https://jobs.example.test/acme/closed"})],[("finish", {"status": "completed", "reason": "should not win"})],]+ rounds: list[list[Any]] = []++ async def _on_round(round_actions: list[Any], _turn_text: str | None) -> None:+ rounds.append(round_actions)+outcome = await run_agent_tool_loop(llm_caller=_ScriptedCaller(script),system_prompt="sys",@@ -4911,9 +4960,23 @@ async def test_navigate_dead_end_terminates_run_through_real_handler(monkeypatchtools=all_tools,max_turns=10,max_tool_calls=20,+ on_action_round=_on_round,)assert outcome.status == "terminated"+ # ...and the whole production chain behind the row the customer reads: the real spec is recordable,+ # so the navigation reaches the caller as its own round; the real handler's outcome rides it; and a+ # 404 landing is flagged as a FAILED action even though the tool honestly returned ok (SKY-16374).+ assert [(a.tool, a.succeeded, a.billable) for round_actions in rounds for a in round_actions] == [+ ("navigate", False, False)+ ]+ assert rounds[0][0].outcome == {+ "requested_url": "https://jobs.example.test/acme/closed",+ "url": "https://jobs.example.test/acme/closed",+ "http_status": 404,+ "page_transitioned": True,+ "navigation_dead_end": 404,+ }# --- Commit-verified click-open dropdown selection. The staging specimen: a click-open