Skip to content

Automate browser based workflows with AI

Task V3: log where a navigation dead end's URL came from (SKY-16206) (#8867)

pedrohsdbf045e06eef67
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 38317aac8..adbad8019 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -2216,6 +2216,7 @@ class ForgeAgent:
caller_known_published_urls,
mask_signed_urls_in_text,
sanitize_handoff_url,
+ string_leaves,
)
from skyvern.forge.taskv3.loop import DEFAULT_MAX_SETTLE_DEFERRALS, CompletionBlocker, CompletionProbe
from skyvern.forge.taskv3.opaque_refs import mask_opaque_urls
@@ -2401,6 +2402,10 @@ class ForgeAgent:
block_label=task_block.label if task_block is not None else None,
)
)
+ payload_strings = string_leaves(coerce_v3_parameters(task.navigation_payload))
+ task_target_urls = caller_known_published_urls(
+ task.url, task.navigation_goal, *payload_strings, keep_query=True
+ )
def _label_secret_values() -> Collection[str]:
# Read per verdict, not once here: a run resolves credentials and codes as it goes, and a
@@ -3079,7 +3084,9 @@ class ForgeAgent:
browser_state.last_navigation_status if task_block is None and not workflow_owned_recovery else None
),
initial_navigation_url=initial_navigation_url,
+ initial_navigation_requested_url=task.url,
caller_known_urls=verdict_known_urls,
+ task_target_urls=task_target_urls,
label_secret_values=_label_secret_values,
login_identifier_tokens=_login_identifier_tokens,
single_action_block=single_action_block,
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 529bf9ba1..4a0a4d424 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -283,7 +283,9 @@ async def run_task_v3_agent_loop(
verification_blocker: VerificationBlocker | None = None,
initial_navigation_status: int | None = None,
initial_navigation_url: str | None = None,
+ initial_navigation_requested_url: str | None = None,
caller_known_urls: frozenset[str] = frozenset(),
+ task_target_urls: frozenset[str] = frozenset(),
label_secret_values: Callable[[], Collection[str]] | None = None,
login_identifier_tokens: Callable[[], Collection[str]] | None = None,
page_probe: Callable[[], Awaitable[str | None]] | None = None,
@@ -629,7 +631,9 @@ async def run_task_v3_agent_loop(
staged_downloads=staged_downloads,
initial_navigation_status=initial_navigation_status,
initial_navigation_url=initial_navigation_url,
+ initial_navigation_requested_url=initial_navigation_requested_url,
caller_known_urls=caller_known_urls,
+ task_target_urls=task_target_urls,
label_secret_values=label_secret_values,
login_identifier_tokens=login_identifier_tokens,
page_probe=None if page_free else page_probe,
diff --git a/skyvern/forge/taskv3/handoff_redaction.py b/skyvern/forge/taskv3/handoff_redaction.py
index ce2387dd8..b80ba5468 100644
--- a/skyvern/forge/taskv3/handoff_redaction.py
+++ b/skyvern/forge/taskv3/handoff_redaction.py
@@ -9,7 +9,7 @@ finish_reason is masked as it is written, because that row is what the next bloc
from __future__ import annotations
import re
-from typing import Collection
+from typing import Any, Collection, Iterator
from urllib.parse import unquote, urlsplit
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
@@ -93,6 +93,14 @@ def sanitize_handoff_url(url: str | None) -> str | None:
ELIDED_URL_PATH = "/…"
+def normalized_url(url: object) -> str | None:
+ """Provenance-only form: the published form plus the query, which on many sites names a different page."""
+ if not isinstance(url, str) or (base := _normalized_published_url(url)) is None:
+ return None
+ query = urlsplit(url.strip()).query
+ return f"{base}?{query}" if query else base
+
+
def _normalized_published_url(url: str | None) -> str | None:
"""The one form both sides of the caller-known comparison are reduced to, so the comparison is
between the same text and the same text. None when there is no host to name."""
@@ -100,16 +108,19 @@ def _normalized_published_url(url: str | None) -> str | None:
return None if bare_parts is None else _joined(*bare_parts)
-def caller_known_published_urls(*sources: str | None) -> frozenset[str]:
+def caller_known_published_urls(*sources: str | None, keep_query: bool = False) -> frozenset[str]:
"""The normalized URLs a guard verdict may publish a PATH for: each source is either a URL the
caller configured or caller-authored text whose URL literals are read out of it.
- Sources are what the AUTHOR TYPED, never what a render produced. A composed goal carries a prior
+ For publishing, sources are what the AUTHOR TYPED, never what a render produced. A composed goal carries a prior
block's handoff prose; a workflow block's own `url`/`navigation_goal` are Jinja templates rendered
from a run context that holds prior blocks' outputs. Either way the URL came off a page rather than
from the caller, so a block's sources are read before `format_potential_template_parameters`
(`pin_caller_authored_block_urls`). The disclosed limit: a URL that reaches a block only through
templating is not caller-known, and a verdict names its host alone.
+
+ `keep_query=True` builds the log-only dead-end provenance set instead: its sources ARE the rendered
+ task fields and payload strings, and the query is kept. That set is never published.
"""
candidates: list[str | None] = []
for source in sources:
@@ -117,7 +128,17 @@ def caller_known_published_urls(*sources: str | None) -> frozenset[str]:
continue
candidates.append(source)
candidates.extend(urls_in_text(source))
- return frozenset(normalized for normalized in map(_normalized_published_url, candidates) if normalized)
+ normalize = normalized_url if keep_query else _normalized_published_url
+ return frozenset(normalized for normalized in map(normalize, candidates) if normalized)
+
+
+def string_leaves(value: Any) -> Iterator[str]:
+ """Each string in a parsed JSON value, read as itself: scanning a dump would glue escapes onto URLs."""
+ if isinstance(value, str):
+ yield value
+ elif isinstance(value, (dict, list)):
+ for item in value.values() if isinstance(value, dict) else value:
+ yield from string_leaves(item)
def sanitize_published_url(url: str | None, caller_known_urls: Collection[str] = frozenset()) -> str | None:
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 2c21810ba..828b746ac 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -40,7 +40,7 @@ from skyvern.forge.taskv3.goal_check import (
TrailEntry,
UnlistedReask,
)
-from skyvern.forge.taskv3.handoff_redaction import MAX_HANDOFF_URL_CHARS, sanitize_published_url
+from skyvern.forge.taskv3.handoff_redaction import MAX_HANDOFF_URL_CHARS, normalized_url, sanitize_published_url
from skyvern.forge.taskv3.target_label import describe_target
from skyvern.webeye.navigation import clear_task_nav_error_code, redact_url_secrets
@@ -876,6 +876,14 @@ class _NavDeadEnd(NamedTuple):
status: int
url: str | None
+ requested_url: str | None = None
+ revisit: bool = False
+
+
+def _dead_end_provenance(urls: tuple[object, ...], task_target_urls: Collection[str], revisit: bool) -> dict[str, Any]:
+ task_supplied = any(normalized_url(url) in task_target_urls for url in urls)
+ provenance = "task_supplied" if task_supplied else "previously_visited" if revisit else "undecided"
+ return {"task_supplied": task_supplied, "previously_visited": revisit, "provenance": provenance}
# Defined here (not tools.py) so the batch-dispatch poisoning check below can compare against it
@@ -3307,10 +3315,13 @@ async def run_agent_tool_loop(
# Only ever named in the dead-end verdict's text, so a caller that has the status but not the URL
# (or whose URL is unfit to print) still gets the same verdict, minus the place.
initial_navigation_url: str | None = None,
+ initial_navigation_requested_url: str | None = None,
# Every URL the CALLER gave this run, normalized by `caller_known_published_urls`. A guard verdict
# publishes a landed URL's path only when it is one of these; every other path is elided to its
# host. Computed once by the caller (it owns the task config) and never read by a tool.
caller_known_urls: frozenset[str] = frozenset(),
+ # Log-only: the URLs the task's RENDERED inputs name, for a dead end's provenance fields. Never published.
+ task_target_urls: frozenset[str] = frozenset(),
# Resolves the run's drop-check secret values when a verdict is about to name a page-supplied
# element. Read at verdict time, not loop start: the registry grows as a run resolves credentials.
label_secret_values: Callable[[], Collection[str]] | None = None,
@@ -3863,6 +3874,9 @@ async def run_agent_tool_loop(
LOG.info(
"taskv3 loop initial navigation dead end",
http_status=initial_navigation_status,
+ **_dead_end_provenance(
+ (initial_navigation_url, initial_navigation_requested_url), task_target_urls, False
+ ),
guard=NAV_DEAD_END_GUARD,
)
st.outcome = _guard_verdict(
@@ -5072,7 +5086,12 @@ async def run_agent_tool_loop(
# A hard 404/410 landing is a non-capability dead-end (a dead/removed posting). Remember
# it but do NOT break the batch: a later navigate in the same turn can land the run on a
# live page and clear it below. Applied once the batch settles (after this for-loop).
- st.pending_nav_dead_end = _NavDeadEnd(dead_end_status, result_data.get("navigation_dead_end_url"))
+ st.pending_nav_dead_end = _NavDeadEnd(
+ dead_end_status,
+ result_data.get("navigation_dead_end_url"),
+ args.get("url"),
+ result_data.get("nav_revisit") is True,
+ )
elif result_data.get("page_state_changed"):
# A successful navigate moved the run off any dead page seen earlier this batch.
st.pending_nav_dead_end = None
@@ -5206,11 +5225,13 @@ async def run_agent_tool_loop(
# recovered): end the run as terminated deterministically, matching v1, rather than leaving the
# failed/terminated choice to the model's finish tool (which does not converge on this class).
if st.outcome is None and st.pending_nav_dead_end is not None:
+ dead = st.pending_nav_dead_end
LOG.info(
"taskv3 loop navigation dead end",
- http_status=st.pending_nav_dead_end.status,
+ http_status=dead.status,
turn=st.turns,
guard=NAV_DEAD_END_GUARD,
+ **_dead_end_provenance((dead.url, dead.requested_url), task_target_urls, dead.revisit),
)
st.outcome = _guard_verdict(
NAV_DEAD_END_GUARD,
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 5d02aed16..695db9cc8 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -6293,6 +6293,10 @@ async def test_execute_task_v3_reads_a_blocks_caller_urls_from_its_unrendered_de
"https://portal.example.test/start",
"Open {{ mail_output }}, then check https://portal.example.test/faq.",
),
+ navigation_payload={
+ "posting": {"note": "Apply at https://jobs.example.test/acme/42/apply\nthanks"},
+ "links": ["see\thttps://jobs.example.test/b/7\tnow", 'Open "https://jobs.example.test/c/9" now'],
+ },
data_extraction_goal=None,
extracted_information_schema=None,
)
@@ -6303,9 +6307,66 @@ async def test_execute_task_v3_reads_a_blocks_caller_urls_from_its_unrendered_de
assert kwargs["caller_known_urls"] == frozenset(
{"https://portal.example.test/start", "https://portal.example.test/faq"}
)
+ # The log-only provenance set counts the RENDERED fields and every payload string, each read as
+ # itself so a newline, tab or quote next to a URL is not glued onto it.
+ assert kwargs["task_target_urls"] >= {
+ "https://mail.example.test/reset/9f2c8a1b4d6e",
+ "https://jobs.example.test/acme/42/apply",
+ "https://jobs.example.test/b/7",
+ "https://jobs.example.test/c/9",
+ }
assert off_the_page in kwargs["goal"] # nosemgrep: incomplete-url-substring-sanitization
[email protected]("initial", [False, True])
+async def test_execute_task_v3_logs_a_dead_task_url_with_a_query_as_task_supplied(
+ monkeypatch: pytest.MonkeyPatch, initial: bool
+) -> None:
+ # agent.py's task-target set and requested start URL, through the real engine and navigate tool: a
+ # task URL whose query names the page is the task's own target, whether the in-loop navigate or the
+ # setup navigation (redirected first) is the one that 404s.
+ import skyvern.utils.url_validators as urlv
+
+ monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)
+ task_url = "https://jobs.example.test/portal.action?page=apply&id=42"
+ page = _FakePage()
+
+ async def _goto(url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:
+ page.url = url
+ return SimpleNamespace(status=404, url=url)
+
+ page.goto = _goto # type: ignore[method-assign]
+ script = [[("navigate", {"url": task_url})], [("finish", {"status": "completed", "reason": "never"})]]
+
+ async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:
+ return await run_task_v3_agent_loop(
+ **{
+ **kwargs,
+ "page_provider": _fixed_page_provider(page),
+ "llm_caller": _ScriptedCaller(script),
+ "step": None,
+ }
+ )
+
+ with capture_logs() as logs:
+ await _run_execute_task_v3(
+ monkeypatch,
+ LoopOutcome(status="failed", reason="the canned outcome must not be used"),
+ url=task_url,
+ navigation_goal="Apply.",
+ navigation_status=404 if initial else None,
+ landed_url="https://jobs.example.test/expired" if initial else None,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ loop_body=_real_loop,
+ working_page=page,
+ )
+
+ event = "taskv3 loop initial navigation dead end" if initial else "taskv3 loop navigation dead end"
+ assert [entry["provenance"] for entry in logs if entry["event"] == event] == ["task_supplied"]
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_publishes_no_path_for_a_block_that_never_reached_the_pin(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_taskv3_engine.py b/tests/unit/test_taskv3_engine.py
index 6e73d58b7..79493bbae 100644
--- a/tests/unit/test_taskv3_engine.py
+++ b/tests/unit/test_taskv3_engine.py
@@ -49,6 +49,7 @@ from skyvern.forge.taskv3.engine import (
)
from skyvern.forge.taskv3.goal_check import INSTRUCTIONS_MAX_CHARS, UNLISTED_REASK_PROMPT_NAME
from skyvern.forge.taskv3.goal_composition import CodeTypedValue
+from skyvern.forge.taskv3.handoff_redaction import caller_known_published_urls
from skyvern.forge.taskv3.llm_call_params import reasoning_effort_with_summary
from skyvern.forge.taskv3.loop import (
CODE_TOOL_NAME,
@@ -2781,3 +2782,42 @@ async def test_typed_values_are_withheld_only_past_the_smallest_model_input_limi
assert all(row.endswith(f'into "#f{n}"') and json.dumps(chunk) in row for n, row in enumerate(rows, start=1))
assert f"- {len(values) - len(rows)} more typed values not listed" in user_prompt
assert sent <= limit
+
+
[email protected]("via_click", [False, True])
+async def test_engine_logs_a_dead_end_provenance_through_the_real_navigate_tool(
+ monkeypatch: pytest.MonkeyPatch, via_click: bool
+) -> None:
+ # The real navigate tool behind the engine: a task URL redirected to a missing page is the task's own
+ # target; a page a click reached earlier, navigated back to and found missing, is a revisit.
+ import skyvern.utils.url_validators as urlv
+
+ monkeypatch.setattr(urlv, "validate_fetch_url", lambda url: url)
+ task_url = "https://jobs.example.test/acme/42"
+ page = _FakePage()
+
+ async def _goto(url: str, timeout: int | None = None, wait_until: str | None = None) -> Any:
+ dead = url == task_url
+ page.url = "https://jobs.example.test/acme/expired" if dead and not via_click else url
+ return SimpleNamespace(status=404 if dead else 200, url=page.url)
+
+ async def _click(selector: str, timeout: int | None = None) -> None:
+ page.url = task_url
+
+ page.goto = _goto # type: ignore[method-assign]
+ page.click = _click # type: ignore[method-assign]
+ script = [[("navigate", {"url": task_url})], [("finish", {"status": "completed", "reason": "never"})]]
+ if via_click:
+ script[:0] = [[("click", {"selector": "#go"})], [("navigate", {"url": "https://example.test/other"})]]
+ with capture_logs() as logs:
+ outcome = await run_task_v3_agent_loop(
+ page_provider=_fixed_page_provider(page),
+ llm_caller=_ScriptedCaller(script),
+ goal="Apply.",
+ task_target_urls=frozenset() if via_click else caller_known_published_urls(task_url, keep_query=True),
+ )
+
+ assert (outcome.status, outcome.guard) == ("terminated", NAV_DEAD_END_GUARD)
+ (line,) = (entry for entry in logs if entry["event"] == "taskv3 loop navigation dead end")
+ assert line["provenance"] == ("previously_visited" if via_click else "task_supplied")
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index a48b214ce..5bbfda4e8 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -641,6 +641,74 @@ async def test_initial_navigation_dead_end_yields_canceled_when_cancelling() ->
assert caller.calls == 0
+_PORTAL = "https://jobs.example.test/portal.action"
+
+
+ ("targets", "requested", "landed", "revisit", "expected"),
+ [
+ ({_PORTAL + "?page=apply&id=42"}, _PORTAL + "?page=apply&id=42", None, False, (True, False)),
+ # A redirect: the task named the URL asked for, not the one the 404 came back on.
+ ({_PORTAL + "?page=apply&id=42"}, _PORTAL + "?page=apply&id=42", _PORTAL + "?gone=1", False, (True, False)),
+ (set(), _PORTAL, None, True, (False, True)),
+ # Both hold: the logged provenance prefers the task's own URL, the flags stay independent.
+ ({_PORTAL}, _PORTAL, None, True, (True, True)),
+ # Same page, different query: on query-routed sites that is a different page, not the task's.
+ ({_PORTAL + "?page=intro&id=42"}, _PORTAL + "?page=apply&id=42", None, False, (False, False)),
+ ],
+)
+async def test_navigate_dead_end_logs_its_provenance_without_a_url(
+ targets: set[str], requested: str, landed: str | None, revisit: bool, expected: tuple[bool, bool]
+) -> None:
+ async def handler(args: dict[str, Any]) -> ToolResult:
+ data = {
+ "page_state_changed": True,
+ "navigation_dead_end": 404,
+ "navigation_dead_end_url": landed or args["url"],
+ }
+ return ToolResult.ok("navigated", data={**data, "nav_revisit": True} if revisit else data)
+
+ tools = [ToolSpec(name="navigate", description="n", parameters={}, handler=handler), make_finish_tool()]
+ script = [[("navigate", {"url": requested})], [("finish", {"status": "completed", "reason": "never"})]]
+ with capture_logs() as logs:
+ outcome, _ = await _run(script, tools, task_target_urls=caller_known_published_urls(*targets, keep_query=True))
+
+ assert (outcome.status, outcome.guard) == ("terminated", NAV_DEAD_END_GUARD)
+ (line,) = (entry for entry in logs if entry["event"] == "taskv3 loop navigation dead end")
+ assert (line["task_supplied"], line["previously_visited"]) == expected
+ assert line["provenance"] == (
+ "task_supplied" if expected[0] else "previously_visited" if expected[1] else "undecided"
+ )
+ assert not any("http" in str(value) for value in line.values())
+
+
+ ("requested", "landed"),
+ [
+ ("http://jobs.example.test/acme/42", "https://jobs.example.test/acme/42"),
+ ("https://jobs.example.test/acme/42", "https://jobs.example.test/acme/expired"),
+ ],
+)
+async def test_initial_navigation_dead_end_after_a_redirect_is_logged_task_supplied(
+ requested: str, landed: str
+) -> None:
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ [[("finish", {"status": "completed", "reason": "never"})]],
+ [make_finish_tool()],
+ initial_navigation_status=404,
+ initial_navigation_url=landed,
+ initial_navigation_requested_url=requested,
+ task_target_urls=caller_known_published_urls(requested, keep_query=True),
+ )
+
+ assert (outcome.status, outcome.guard) == ("terminated", NAV_DEAD_END_GUARD)
+ (line,) = (entry for entry in logs if entry["event"] == "taskv3 loop initial navigation dead end")
+ assert (line["task_supplied"], line["previously_visited"], line["provenance"]) == (True, False, "task_supplied")
+
+
@pytest.mark.asyncio
async def test_verification_blocker_refuses_completed_but_not_failed() -> None:
async def _blocked(status: str) -> str | None: