Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Task V3: log where a navigation dead end's URL came from (SKY-16206) (#8867)
pedrohsdbf045e06eef67
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 38317aac8..adbad8019 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -2216,6 +2216,7 @@ class ForgeAgent:caller_known_published_urls,mask_signed_urls_in_text,sanitize_handoff_url,+ string_leaves,)from skyvern.forge.taskv3.loop import DEFAULT_MAX_SETTLE_DEFERRALS, CompletionBlocker, CompletionProbefrom skyvern.forge.taskv3.opaque_refs import mask_opaque_urls@@ -2401,6 +2402,10 @@ class ForgeAgent:block_label=task_block.label if task_block is not None else None,))+ payload_strings = string_leaves(coerce_v3_parameters(task.navigation_payload))+ task_target_urls = caller_known_published_urls(+ task.url, task.navigation_goal, *payload_strings, keep_query=True+ )def _label_secret_values() -> Collection[str]:# Read per verdict, not once here: a run resolves credentials and codes as it goes, and a@@ -3079,7 +3084,9 @@ class ForgeAgent:browser_state.last_navigation_status if task_block is None and not workflow_owned_recovery else None),initial_navigation_url=initial_navigation_url,+ initial_navigation_requested_url=task.url,caller_known_urls=verdict_known_urls,+ task_target_urls=task_target_urls,label_secret_values=_label_secret_values,login_identifier_tokens=_login_identifier_tokens,single_action_block=single_action_block,diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex 529bf9ba1..4a0a4d424 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -283,7 +283,9 @@ async def run_task_v3_agent_loop(verification_blocker: VerificationBlocker | None = None,initial_navigation_status: int | None = None,initial_navigation_url: str | None = None,+ initial_navigation_requested_url: str | None = None,caller_known_urls: frozenset[str] = frozenset(),+ task_target_urls: frozenset[str] = frozenset(),label_secret_values: Callable[[], Collection[str]] | None = None,login_identifier_tokens: Callable[[], Collection[str]] | None = None,page_probe: Callable[[], Awaitable[str | None]] | None = None,@@ -629,7 +631,9 @@ async def run_task_v3_agent_loop(staged_downloads=staged_downloads,initial_navigation_status=initial_navigation_status,initial_navigation_url=initial_navigation_url,+ initial_navigation_requested_url=initial_navigation_requested_url,caller_known_urls=caller_known_urls,+ task_target_urls=task_target_urls,label_secret_values=label_secret_values,login_identifier_tokens=login_identifier_tokens,page_probe=None if page_free else page_probe,diff --git a/skyvern/forge/taskv3/handoff_redaction.py b/skyvern/forge/taskv3/handoff_redaction.pyindex ce2387dd8..b80ba5468 100644--- a/skyvern/forge/taskv3/handoff_redaction.py+++ b/skyvern/forge/taskv3/handoff_redaction.py@@ -9,7 +9,7 @@ finish_reason is masked as it is written, because that row is what the next blocfrom __future__ import annotationsimport re-from typing import Collection+from typing import Any, Collection, Iteratorfrom urllib.parse import unquote, urlsplitfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext@@ -93,6 +93,14 @@ def sanitize_handoff_url(url: str | None) -> str | None:ELIDED_URL_PATH = "/…"+def normalized_url(url: object) -> str | None:+ """Provenance-only form: the published form plus the query, which on many sites names a different page."""+ if not isinstance(url, str) or (base := _normalized_published_url(url)) is None:+ return None+ query = urlsplit(url.strip()).query+ return f"{base}?{query}" if query else base++def _normalized_published_url(url: str | None) -> str | None:"""The one form both sides of the caller-known comparison are reduced to, so the comparison isbetween the same text and the same text. None when there is no host to name."""@@ -100,16 +108,19 @@ def _normalized_published_url(url: str | None) -> str | None:return None if bare_parts is None else _joined(*bare_parts)-def caller_known_published_urls(*sources: str | None) -> frozenset[str]:+def caller_known_published_urls(*sources: str | None, keep_query: bool = False) -> frozenset[str]:"""The normalized URLs a guard verdict may publish a PATH for: each source is either a URL thecaller configured or caller-authored text whose URL literals are read out of it.- Sources are what the AUTHOR TYPED, never what a render produced. A composed goal carries a prior+ For publishing, sources are what the AUTHOR TYPED, never what a render produced. A composed goal carries a priorblock's handoff prose; a workflow block's own `url`/`navigation_goal` are Jinja templates renderedfrom a run context that holds prior blocks' outputs. Either way the URL came off a page rather thanfrom the caller, so a block's sources are read before `format_potential_template_parameters`(`pin_caller_authored_block_urls`). The disclosed limit: a URL that reaches a block only throughtemplating is not caller-known, and a verdict names its host alone.++ `keep_query=True` builds the log-only dead-end provenance set instead: its sources ARE the rendered+ task fields and payload strings, and the query is kept. That set is never published."""candidates: list[str | None] = []for source in sources:@@ -117,7 +128,17 @@ def caller_known_published_urls(*sources: str | None) -> frozenset[str]:continuecandidates.append(source)candidates.extend(urls_in_text(source))- return frozenset(normalized for normalized in map(_normalized_published_url, candidates) if normalized)+ normalize = normalized_url if keep_query else _normalized_published_url+ return frozenset(normalized for normalized in map(normalize, candidates) if normalized)+++def string_leaves(value: Any) -> Iterator[str]:+ """Each string in a parsed JSON value, read as itself: scanning a dump would glue escapes onto URLs."""+ if isinstance(value, str):+ yield value+ elif isinstance(value, (dict, list)):+ for item in value.values() if isinstance(value, dict) else value:+ yield from string_leaves(item)def sanitize_published_url(url: str | None, caller_known_urls: Collection[str] = frozenset()) -> str | None:diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex 2c21810ba..828b746ac 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -40,7 +40,7 @@ from skyvern.forge.taskv3.goal_check import (TrailEntry,UnlistedReask,)-from skyvern.forge.taskv3.handoff_redaction import MAX_HANDOFF_URL_CHARS, sanitize_published_url+from skyvern.forge.taskv3.handoff_redaction import MAX_HANDOFF_URL_CHARS, normalized_url, sanitize_published_urlfrom skyvern.forge.taskv3.target_label import describe_targetfrom skyvern.webeye.navigation import clear_task_nav_error_code, redact_url_secrets@@ -876,6 +876,14 @@ class _NavDeadEnd(NamedTuple):status: inturl: str | None+ requested_url: str | None = None+ revisit: bool = False+++def _dead_end_provenance(urls: tuple[object, ...], task_target_urls: Collection[str], revisit: bool) -> dict[str, Any]:+ task_supplied = any(normalized_url(url) in task_target_urls for url in urls)+ provenance = "task_supplied" if task_supplied else "previously_visited" if revisit else "undecided"+ return {"task_supplied": task_supplied, "previously_visited": revisit, "provenance": provenance}# Defined here (not tools.py) so the batch-dispatch poisoning check below can compare against it@@ -3307,10 +3315,13 @@ async def run_agent_tool_loop(# Only ever named in the dead-end verdict's text, so a caller that has the status but not the URL# (or whose URL is unfit to print) still gets the same verdict, minus the place.initial_navigation_url: str | None = None,+ initial_navigation_requested_url: str | None = None,# Every URL the CALLER gave this run, normalized by `caller_known_published_urls`. A guard verdict# publishes a landed URL's path only when it is one of these; every other path is elided to its# host. Computed once by the caller (it owns the task config) and never read by a tool.caller_known_urls: frozenset[str] = frozenset(),+ # Log-only: the URLs the task's RENDERED inputs name, for a dead end's provenance fields. Never published.+ task_target_urls: frozenset[str] = frozenset(),# Resolves the run's drop-check secret values when a verdict is about to name a page-supplied# element. Read at verdict time, not loop start: the registry grows as a run resolves credentials.label_secret_values: Callable[[], Collection[str]] | None = None,@@ -3863,6 +3874,9 @@ async def run_agent_tool_loop(LOG.info("taskv3 loop initial navigation dead end",http_status=initial_navigation_status,+ **_dead_end_provenance(+ (initial_navigation_url, initial_navigation_requested_url), task_target_urls, False+ ),guard=NAV_DEAD_END_GUARD,)st.outcome = _guard_verdict(@@ -5072,7 +5086,12 @@ async def run_agent_tool_loop(# A hard 404/410 landing is a non-capability dead-end (a dead/removed posting). Remember# it but do NOT break the batch: a later navigate in the same turn can land the run on a# live page and clear it below. Applied once the batch settles (after this for-loop).- st.pending_nav_dead_end = _NavDeadEnd(dead_end_status, result_data.get("navigation_dead_end_url"))+ st.pending_nav_dead_end = _NavDeadEnd(+ dead_end_status,+ result_data.get("navigation_dead_end_url"),+ args.get("url"),+ result_data.get("nav_revisit") is True,+ )elif result_data.get("page_state_changed"):# A successful navigate moved the run off any dead page seen earlier this batch.st.pending_nav_dead_end = None@@ -5206,11 +5225,13 @@ async def run_agent_tool_loop(# recovered): end the run as terminated deterministically, matching v1, rather than leaving the# failed/terminated choice to the model's finish tool (which does not converge on this class).if st.outcome is None and st.pending_nav_dead_end is not None:+ dead = st.pending_nav_dead_endLOG.info("taskv3 loop navigation dead end",- http_status=st.pending_nav_dead_end.status,+ http_status=dead.status,turn=st.turns,guard=NAV_DEAD_END_GUARD,+ **_dead_end_provenance((dead.url, dead.requested_url), task_target_urls, dead.revisit),)st.outcome = _guard_verdict(NAV_DEAD_END_GUARD,diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 5d02aed16..695db9cc8 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -6293,6 +6293,10 @@ async def test_execute_task_v3_reads_a_blocks_caller_urls_from_its_unrendered_de"https://portal.example.test/start","Open {{ mail_output }}, then check https://portal.example.test/faq.",),+ navigation_payload={+ "posting": {"note": "Apply at https://jobs.example.test/acme/42/apply\nthanks"},+ "links": ["see\thttps://jobs.example.test/b/7\tnow", 'Open "https://jobs.example.test/c/9" now'],+ },data_extraction_goal=None,extracted_information_schema=None,)@@ -6303,9 +6307,66 @@ async def test_execute_task_v3_reads_a_blocks_caller_urls_from_its_unrendered_deassert kwargs["caller_known_urls"] == frozenset({"https://portal.example.test/start", "https://portal.example.test/faq"})+ # The log-only provenance set counts the RENDERED fields and every payload string, each read as+ # itself so a newline, tab or quote next to a URL is not glued onto it.+ assert kwargs["task_target_urls"] >= {+ "https://mail.example.test/reset/9f2c8a1b4d6e",+ "https://jobs.example.test/acme/42/apply",+ "https://jobs.example.test/b/7",+ "https://jobs.example.test/c/9",+ }assert off_the_page in kwargs["goal"] # nosemgrep: incomplete-url-substring-sanitization[email protected]("initial", [False, True])+async def test_execute_task_v3_logs_a_dead_task_url_with_a_query_as_task_supplied(+ monkeypatch: pytest.MonkeyPatch, initial: bool+) -> None:+ # agent.py's task-target set and requested start URL, through the real engine and navigate tool: a+ # task URL whose query names the page is the task's own target, whether the in-loop navigate or the+ # setup navigation (redirected first) is the one that 404s.+ import skyvern.utils.url_validators as urlv++ monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)+ task_url = "https://jobs.example.test/portal.action?page=apply&id=42"+ page = _FakePage()++ async def _goto(url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:+ page.url = url+ return SimpleNamespace(status=404, url=url)++ page.goto = _goto # type: ignore[method-assign]+ script = [[("navigate", {"url": task_url})], [("finish", {"status": "completed", "reason": "never"})]]++ async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:+ return await run_task_v3_agent_loop(+ **{+ **kwargs,+ "page_provider": _fixed_page_provider(page),+ "llm_caller": _ScriptedCaller(script),+ "step": None,+ }+ )++ with capture_logs() as logs:+ await _run_execute_task_v3(+ monkeypatch,+ LoopOutcome(status="failed", reason="the canned outcome must not be used"),+ url=task_url,+ navigation_goal="Apply.",+ navigation_status=404 if initial else None,+ landed_url="https://jobs.example.test/expired" if initial else None,+ data_extraction_goal=None,+ extracted_information_schema=None,+ loop_body=_real_loop,+ working_page=page,+ )++ event = "taskv3 loop initial navigation dead end" if initial else "taskv3 loop navigation dead end"+ assert [entry["provenance"] for entry in logs if entry["event"] == event] == ["task_supplied"]++@pytest.mark.asyncioasync def test_execute_task_v3_publishes_no_path_for_a_block_that_never_reached_the_pin(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.pyindex 6e73d58b7..79493bbae 100644--- a/tests/unit/test_taskv3_engine.py+++ b/tests/unit/test_taskv3_engine.py@@ -49,6 +49,7 @@ from skyvern.forge.taskv3.engine import ()from skyvern.forge.taskv3.goal_check import INSTRUCTIONS_MAX_CHARS, UNLISTED_REASK_PROMPT_NAMEfrom skyvern.forge.taskv3.goal_composition import CodeTypedValue+from skyvern.forge.taskv3.handoff_redaction import caller_known_published_urlsfrom skyvern.forge.taskv3.llm_call_params import reasoning_effort_with_summaryfrom skyvern.forge.taskv3.loop import (CODE_TOOL_NAME,@@ -2781,3 +2782,42 @@ async def test_typed_values_are_withheld_only_past_the_smallest_model_input_limiassert all(row.endswith(f'into "#f{n}"') and json.dumps(chunk) in row for n, row in enumerate(rows, start=1))assert f"- {len(values) - len(rows)} more typed values not listed" in user_promptassert sent <= limit++[email protected]("via_click", [False, True])+async def test_engine_logs_a_dead_end_provenance_through_the_real_navigate_tool(+ monkeypatch: pytest.MonkeyPatch, via_click: bool+) -> None:+ # The real navigate tool behind the engine: a task URL redirected to a missing page is the task's own+ # target; a page a click reached earlier, navigated back to and found missing, is a revisit.+ import skyvern.utils.url_validators as urlv++ monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)+ task_url = "https://jobs.example.test/acme/42"+ page = _FakePage()++ async def _goto(url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:+ dead = url == task_url+ page.url = "https://jobs.example.test/acme/expired" if dead and not via_click else url+ return SimpleNamespace(status=404 if dead else 200, url=page.url)++ async def _click(selector: str, timeout: int | None = None) -> None:+ page.url = task_url++ page.goto = _goto # type: ignore[method-assign]+ page.click = _click # type: ignore[method-assign]+ script = [[("navigate", {"url": task_url})], [("finish", {"status": "completed", "reason": "never"})]]+ if via_click:+ script[:0] = [[("click", {"selector": "#go"})], [("navigate", {"url": "https://example.test/other"})]]+ with capture_logs() as logs:+ outcome = await run_task_v3_agent_loop(+ page_provider=_fixed_page_provider(page),+ llm_caller=_ScriptedCaller(script),+ goal="Apply.",+ task_target_urls=frozenset() if via_click else caller_known_published_urls(task_url, keep_query=True),+ )++ assert (outcome.status, outcome.guard) == ("terminated", NAV_DEAD_END_GUARD)+ (line,) = (entry for entry in logs if entry["event"] == "taskv3 loop navigation dead end")+ assert line["provenance"] == ("previously_visited" if via_click else "task_supplied")diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex a48b214ce..5bbfda4e8 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -641,6 +641,74 @@ async def test_initial_navigation_dead_end_yields_canceled_when_cancelling() ->assert caller.calls == 0+_PORTAL = "https://jobs.example.test/portal.action"+++ ("targets", "requested", "landed", "revisit", "expected"),+ [+ ({_PORTAL + "?page=apply&id=42"}, _PORTAL + "?page=apply&id=42", None, False, (True, False)),+ # A redirect: the task named the URL asked for, not the one the 404 came back on.+ ({_PORTAL + "?page=apply&id=42"}, _PORTAL + "?page=apply&id=42", _PORTAL + "?gone=1", False, (True, False)),+ (set(), _PORTAL, None, True, (False, True)),+ # Both hold: the logged provenance prefers the task's own URL, the flags stay independent.+ ({_PORTAL}, _PORTAL, None, True, (True, True)),+ # Same page, different query: on query-routed sites that is a different page, not the task's.+ ({_PORTAL + "?page=intro&id=42"}, _PORTAL + "?page=apply&id=42", None, False, (False, False)),+ ],+)+async def test_navigate_dead_end_logs_its_provenance_without_a_url(+ targets: set[str], requested: str, landed: str | None, revisit: bool, expected: tuple[bool, bool]+) -> None:+ async def handler(args: dict[str, Any]) -> ToolResult:+ data = {+ "page_state_changed": True,+ "navigation_dead_end": 404,+ "navigation_dead_end_url": landed or args["url"],+ }+ return ToolResult.ok("navigated", data={**data, "nav_revisit": True} if revisit else data)++ tools = [ToolSpec(name="navigate", description="n", parameters={}, handler=handler), make_finish_tool()]+ script = [[("navigate", {"url": requested})], [("finish", {"status": "completed", "reason": "never"})]]+ with capture_logs() as logs:+ outcome, _ = await _run(script, tools, task_target_urls=caller_known_published_urls(*targets, keep_query=True))++ assert (outcome.status, outcome.guard) == ("terminated", NAV_DEAD_END_GUARD)+ (line,) = (entry for entry in logs if entry["event"] == "taskv3 loop navigation dead end")+ assert (line["task_supplied"], line["previously_visited"]) == expected+ assert line["provenance"] == (+ "task_supplied" if expected[0] else "previously_visited" if expected[1] else "undecided"+ )+ assert not any("http" in str(value) for value in line.values())+++ ("requested", "landed"),+ [+ ("http://jobs.example.test/acme/42", "https://jobs.example.test/acme/42"),+ ("https://jobs.example.test/acme/42", "https://jobs.example.test/acme/expired"),+ ],+)+async def test_initial_navigation_dead_end_after_a_redirect_is_logged_task_supplied(+ requested: str, landed: str+) -> None:+ with capture_logs() as logs:+ outcome, _ = await _run(+ [[("finish", {"status": "completed", "reason": "never"})]],+ [make_finish_tool()],+ initial_navigation_status=404,+ initial_navigation_url=landed,+ initial_navigation_requested_url=requested,+ task_target_urls=caller_known_published_urls(requested, keep_query=True),+ )++ assert (outcome.status, outcome.guard) == ("terminated", NAV_DEAD_END_GUARD)+ (line,) = (entry for entry in logs if entry["event"] == "taskv3 loop initial navigation dead end")+ assert (line["task_supplied"], line["previously_visited"], line["provenance"]) == (True, False, "task_supplied")++@pytest.mark.asyncioasync def test_verification_blocker_refuses_completed_but_not_failed() -> None:async def _blocked(status: str) -> str | None: