Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: add page URL to agent action summaries for script reviewer (#5384)
pedrohsdbe5bbc6b0e05e
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2index a819671bd..e710aa14a 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2@@ -106,14 +106,18 @@ The cached code failed and the AI agent took over. Here is what happened:{% endif %}- **Agent Actions After Fallback** (use these to write the NEW branch actions):{% if episode.agent_actions is mapping and episode.agent_actions.actions is defined %}+{% set ns = namespace(prev_url='') %}{% for a in episode.agent_actions.actions %}{{ loop.index }}. {{ a.action_type }}: {{ a.intention or a.reasoning or "N/A" }} ({{ a.status }})+{%- if a.page_url and loop.first %} [url: {{ a.page_url }}]{% endif %}+{%- if a.page_url and not loop.first and a.page_url != ns.prev_url %} [url changed: {{ a.page_url }}]{% endif %}{%- if a.field %} [field: {{ a.field }}]{% endif %}{%- if a.element_tag %} [tag: {{ a.element_tag }}]{% endif %}{%- if a.element_text %} [text: "{{ a.element_text }}"]{% endif %}{%- if a.css_suggestion %} **[USE THIS SELECTOR: `{{ a.css_suggestion }}`]**{% endif %}{%- if a.all_attributes %} [attrs: {{ a.all_attributes | tojson }}]{% endif %}{%- if not a.all_attributes and a.element_attributes %} [attrs: {{ a.element_attributes | tojson }}]{% endif %}+{%- if a.page_url %}{% set ns.prev_url = a.page_url %}{% endif %}{% endfor %}{% elif episode.agent_actions is mapping and episode.agent_actions.failed_selector is defined %}**Element-level fallback** — the selector failed but AI found the element:diff --git a/skyvern/utils/css_selector.py b/skyvern/utils/css_selector.pyindex a6ce7cda5..44b4b1329 100644--- a/skyvern/utils/css_selector.py+++ b/skyvern/utils/css_selector.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport refrom typing import TYPE_CHECKING+from urllib.parse import urlparseif TYPE_CHECKING:from skyvern.webeye.actions.actions import Action@@ -155,6 +156,13 @@ def build_action_summary(action: Action) -> dict:"intention": action.intention,"reasoning": action.reasoning,"status": action.status,+ # Strip query params from URL — they can contain OAuth tokens, email+ # addresses, session IDs. The reviewer only needs host+path for redirect detection.+ "page_url": (+ f"{parsed.scheme}://{parsed.netloc}{parsed.path}"+ if (raw_url := elem.get("page_url")) and (parsed := urlparse(raw_url)).netloc+ else None+ ),"field": (action.input_or_select_context.field if action.input_or_select_context else None),# Legacy: 6 core attributes (kept for backward compat with older templates)"element_attributes": (diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 0136af818..4ef1bbb5f 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -76,6 +76,13 @@ def parse_action(intention = action["user_detail_query"] if "user_detail_query" in action else Noneresponse = action["user_detail_answer"] if "user_detail_answer" in action else None+ # Inject the page URL into element data so the script reviewer can detect+ # URL changes between consecutive actions (e.g. SSO redirects).+ if skyvern_element_data is not None:+ skyvern_element_data = {**skyvern_element_data, "page_url": scraped_page.url}+ else:+ skyvern_element_data = {"page_url": scraped_page.url}+base_action_dict = {"element_id": element_id,"skyvern_element_hash": skyvern_element_hash,diff --git a/tests/unit/test_action_url.py b/tests/unit/test_action_url.pynew file mode 100644index 000000000..179919562--- /dev/null+++ b/tests/unit/test_action_url.py@@ -0,0 +1,126 @@+"""Tests for page URL propagation in action summaries and reviewer templates."""++from skyvern.utils.css_selector import build_action_summary+from skyvern.webeye.actions.actions import Action, ActionType+++def _make_action(page_url: str | None = None, **kwargs) -> Action:+ """Create a minimal Action with optional page_url in skyvern_element_data."""+ element_data = {"page_url": page_url} if page_url else {}+ defaults = {+ "action_type": ActionType.CLICK,+ "status": "completed",+ "intention": "Click button",+ "skyvern_element_data": element_data or None,+ }+ defaults.update(kwargs)+ return Action(**defaults)+++def test_build_action_summary_includes_page_url():+ """page_url from skyvern_element_data is included in the summary."""+ action = _make_action(page_url="https://example.com/login")+ summary = build_action_summary(action)+ assert summary["page_url"] == "https://example.com/login"+++def test_build_action_summary_strips_query_params():+ """Query params are stripped to avoid leaking OAuth tokens, emails, session IDs."""+ action = _make_action(page_url="https://sso.example.com/auth?code=abc123&state=xyz&[email protected]")+ summary = build_action_summary(action)+ assert summary["page_url"] == "https://sso.example.com/auth"+ assert "abc123" not in summary["page_url"]+ assert "email" not in summary["page_url"]+++def test_build_action_summary_page_url_none_when_missing():+ """page_url is None when skyvern_element_data has no page_url."""+ action = _make_action()+ summary = build_action_summary(action)+ assert summary["page_url"] is None+++def test_build_action_summary_page_url_with_element_data():+ """page_url coexists with element attributes without interference."""+ action = Action(+ action_type=ActionType.INPUT_TEXT,+ status="completed",+ intention="Fill email",+ skyvern_element_data={+ "page_url": "https://sso.example.com/auth",+ "tagName": "input",+ "attributes": {"id": "email", "type": "text"},+ },+ )+ summary = build_action_summary(action)+ assert summary["page_url"] == "https://sso.example.com/auth"+ assert summary["element_tag"] == "input"+ assert summary["all_attributes"]["id"] == "email"+++def test_template_url_rendering():+ """The reviewer template renders [url: ...] and [url changed: ...] correctly."""+ from skyvern.forge.prompts import prompt_engine++ episodes = [+ {+ "block_label": "login",+ "fallback_type": "full_block",+ "error_message": "Selector failed",+ "classify_result": None,+ "agent_actions": {+ "actions": [+ {+ "action_type": "click",+ "intention": "Click Continue",+ "status": "completed",+ "page_url": "https://portal.example.com/login",+ },+ {+ "action_type": "input_text",+ "intention": "Fill password",+ "status": "completed",+ "page_url": "https://portal.example.com/login",+ },+ {+ "action_type": "click",+ "intention": "Click Submit",+ "status": "completed",+ "page_url": "https://sso.example.com/auth",+ },+ {+ "action_type": "input_text",+ "intention": "Fill SSO password",+ "status": "completed",+ "page_url": "https://sso.example.com/auth",+ },+ ],+ },+ "page_url": "https://portal.example.com/login",+ "page_text_snapshot": "Login page",+ }+ ]++ prompt = prompt_engine.load_prompt(+ template="script-reviewer",+ navigation_goal="Log in to the portal",+ existing_code="async def login(page, context): pass",+ episodes=episodes,+ function_signature="async def login(page: SkyvernPage, context: RunContext)",+ stale_branches=[],+ parameter_keys=["username", "password"],+ historical_episodes=[],+ run_parameter_values={},+ user_instructions=None,+ )++ # First action should show [url: portal.example.com/login]+ assert "[url: https://portal.example.com/login]" in prompt+ # Third action should show [url changed: sso.example.com/auth]+ assert "[url changed: https://sso.example.com/auth]" in prompt+ # Second action (same URL as first) should NOT show url+ lines = prompt.split("\n")+ action_2_lines = [line for line in lines if "2. input_text:" in line]+ assert action_2_lines, "Action 2 should exist"+ assert "[url:" not in action_2_lines[0]+ assert "[url changed:" not in action_2_lines[0]