Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10452): select proxy from first block URL before script SDK creates browser (#6306)
Shuchang Zheng5dfda2242ae3
diff --git a/skyvern/core/script_generations/run_initializer.py b/skyvern/core/script_generations/run_initializer.pyindex e533aee51..1a8d03a94 100644--- a/skyvern/core/script_generations/run_initializer.py+++ b/skyvern/core/script_generations/run_initializer.py@@ -121,6 +121,7 @@ async def setup(generated_parameter_cls: type[BaseModel] | None = None,browser_session_id: str | None = None,adaptive_caching: bool = False,+ url: str | None = None,) -> tuple[SkyvernPage, RunContext]:# transform any secrets/credential parameters. For example, if there's only one credential in the parameters: {"cred_12345": "cred_12345"},# it should be transformed to {"cred_12345": {"username": "secret_5fBoa_username", "password": "secret_5fBoa_password"}}@@ -143,7 +144,7 @@ async def setup(else:extracted_params = {}- skyvern_page = await ScriptSkyvernPage.create(browser_session_id=browser_session_id)+ skyvern_page = await ScriptSkyvernPage.create(browser_session_id=browser_session_id, url=url)run_context = RunContext(parameters=parameters,page=skyvern_page,diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex 18c4d8202..148717881 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -82,17 +82,23 @@ class ScriptSkyvernPage(SkyvernPage):self._record = recorder or (lambda ac: None)@classmethod- async def _get_or_create_browser_state(cls, browser_session_id: str | None = None) -> BrowserState:+ async def _get_or_create_browser_state(+ cls, browser_session_id: str | None = None, url: str | None = None+ ) -> BrowserState:context = skyvern_context.current()if context and context.workflow_run_id and context.organization_id:workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id=context.workflow_run_id, organization_id=context.organization_id)if workflow_run:+ # url selects the proxy at browser-context creation; navigate=False because the+ # generated script issues its own goto, so navigating here would double-load.browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(workflow_run=workflow_run,+ url=url,browser_session_id=browser_session_id,browser_profile_id=workflow_run.browser_profile_id,+ navigate=False,)else:raise WorkflowRunNotFound(workflow_run_id=context.workflow_run_id)@@ -119,8 +125,9 @@ class ScriptSkyvernPage(SkyvernPage):async def create(cls,browser_session_id: str | None = None,+ url: str | None = None,) -> ScriptSkyvernPage:- scraped_page = await cls.create_scraped_page(browser_session_id=browser_session_id)+ scraped_page = await cls.create_scraped_page(browser_session_id=browser_session_id, url=url)page = await scraped_page._browser_state.must_get_working_page()ai = RealSkyvernPageAi(scraped_page, page)return cls(scraped_page=scraped_page, page=page, ai=ai)@@ -129,10 +136,11 @@ class ScriptSkyvernPage(SkyvernPage):async def create_scraped_page(cls,browser_session_id: str | None = None,+ url: str | None = None,) -> ScrapedPage:# initialize browser state# TODO: add workflow_run_id or eventually script_id/script_run_id- browser_state = await cls._get_or_create_browser_state(browser_session_id=browser_session_id)+ browser_state = await cls._get_or_create_browser_state(browser_session_id=browser_session_id, url=url)return await browser_state.scrape_website(url="",cleanup_element_tree=app.AGENT_FUNCTION.cleanup_element_tree_factory(),diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex d491085fc..b4de86353 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -378,28 +378,29 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)return workflow_dict-async def _precreate_script_browser(- block: BlockTypeVar,+def _resolve_first_block_url(+ blocks: list[BlockTypeVar],workflow_run: "WorkflowRun",- browser_session_id: str | None,-) -> None:- url = getattr(block, "url", None)- if not url:- return+) -> str | None:+ """Resolve the URL of the first block that has one, for proxy selection.++ In script mode ``skyvern.setup()`` creates the browser before any block runs;+ threading this URL lets the ``SKYVERN_PROXY`` flag (which matches on ``task_url``)+ resolve instead of falling back to the org default. Best-effort: any failure+ returns None so proxy selection degrades to the previous (url=None) behavior.+ """try:wrc = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run.workflow_run_id)+ for block in blocks:+ url = getattr(block, "url", None)+ if not url:+ continue+ resolved_url = block.format_block_parameter_template_from_workflow_run_context(url, wrc)+ if resolved_url and isinstance(resolved_url, str):+ return resolved_urlexcept Exception:- LOG.warning("Workflow run context unavailable for browser pre-creation", exc_info=True)- return- resolved_url = block.format_block_parameter_template_from_workflow_run_context(url, wrc)- if not resolved_url or not isinstance(resolved_url, str):- return- await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,- url=resolved_url,- browser_session_id=browser_session_id,- browser_profile_id=workflow_run.browser_profile_id,- )+ LOG.warning("Failed to resolve first block URL for proxy selection", exc_info=True)+ return Noneclass WorkflowService:@@ -1950,6 +1951,10 @@ class WorkflowService:is_script_run = await self.should_run_script(workflow, workflow_run)+ # skyvern.setup() below creates the browser before any block runs, so resolve the+ # first block's URL here to drive proxy selection from task_url (see SKYVERN_PROXY).+ first_block_url = _resolve_first_block_url(all_blocks, workflow_run) if is_script_run else None+if script:LOG.info("Loading script blocks for workflow execution",@@ -2009,6 +2014,7 @@ class WorkflowService:await skyvern.setup(script_parameters,generated_parameter_cls=param_cls,+ url=first_block_url,)if loaded_script_module:# Mark static (pinned) scripts so complete() skips LLM verification@@ -2066,6 +2072,7 @@ class WorkflowService:await skyvern.setup(script_parameters,generated_parameter_cls=param_cls,+ url=first_block_url,)# Mark context so static scripts skip LLM completion verificationstatic_ctx = skyvern_context.current()@@ -2724,18 +2731,6 @@ class WorkflowService:block_label=block.label,exc_info=True,)- try:- await _precreate_script_browser(- block=block,- workflow_run=workflow_run,- browser_session_id=browser_session_id,- )- except Exception:- LOG.warning(- "Failed to pre-create browser with block URL; script will create lazily",- block_label=block.label,- exc_info=True,- )block_exec_start = time.monotonic()try:diff --git a/skyvern/webeye/browser_manager.py b/skyvern/webeye/browser_manager.pyindex b6cd40805..e0582bc8b 100644--- a/skyvern/webeye/browser_manager.py+++ b/skyvern/webeye/browser_manager.py@@ -23,6 +23,7 @@ class BrowserManager(Protocol):url: str | None = None,browser_session_id: str | None = None,browser_profile_id: str | None = None,+ navigate: bool = True,) -> BrowserState: ...async def cleanup_for_task(diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex cf758a04d..a2b89fb20 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -164,6 +164,7 @@ class RealBrowserManager(BrowserManager):url: str | None = None,browser_session_id: str | None = None,browser_profile_id: str | None = None,+ navigate: bool = True,) -> BrowserState:parent_workflow_run_id = workflow_run.parent_workflow_run_idworkflow_run_id = workflow_run.workflow_run_id@@ -210,7 +211,7 @@ class RealBrowserManager(BrowserManager):LOG.info("Used to occupy browser session here", browser_session_id=browser_session_id)page = await browser_state.get_working_page()if page:- if url:+ if url and navigate:await browser_state.navigate_to_url(page=page, url=url)else:LOG.warning("Browser state has no page", workflow_run_id=workflow_run.workflow_run_id)@@ -255,8 +256,11 @@ class RealBrowserManager(BrowserManager):# The URL here is only used when creating a new page, and not when using an existing page.# This will make sure browser_state.page is not None.+ # When navigate is False, the URL has already been used for proxy selection in+ # _create_browser_state above; we skip navigation so the caller (e.g. a generated+ # script) performs the first goto itself, avoiding a redundant page load.await browser_state.get_or_create_page(- url=url,+ url=url if navigate else None,proxy_location=workflow_run.proxy_location,workflow_run_id=workflow_run.workflow_run_id,workflow_permanent_id=workflow_run.workflow_permanent_id,diff --git a/tests/unit/test_script_mode_browser_url.py b/tests/unit/test_script_mode_browser_url.pyindex d8b6f6e28..094578d1b 100644--- a/tests/unit/test_script_mode_browser_url.py+++ b/tests/unit/test_script_mode_browser_url.py@@ -2,7 +2,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from skyvern.forge.sdk.workflow.service import _precreate_script_browser+from skyvern.forge.sdk.workflow.service import _resolve_first_block_urldef _make_block(url: str | None = None) -> MagicMock:@@ -25,37 +25,23 @@ def _make_workflow_run(return wfr-async def test_precreate_with_static_url() -> None:- """Block with a static URL: browser pre-created with that URL."""+def test_resolve_first_block_url_static() -> None:+ """First block with a static URL resolves to that URL."""block = _make_block(url="https://www.example.com/target")wfr = _make_workflow_run()mock_wrc = MagicMock()with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = mock_wrc- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run = AsyncMock()-- await _precreate_script_browser(- block=block,- workflow_run=wfr,- browser_session_id="bs_123",- )- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run.assert_called_once_with(- workflow_run=wfr,- url="https://www.example.com/target",- browser_session_id="bs_123",- browser_profile_id=wfr.browser_profile_id,- )+ assert _resolve_first_block_url([block], wfr) == "https://www.example.com/target"block.format_block_parameter_template_from_workflow_run_context.assert_called_once_with("https://www.example.com/target", mock_wrc)-async def test_precreate_with_templated_url() -> None:- """Block with a templated URL: resolved URL is passed to browser creation."""+def test_resolve_first_block_url_templated() -> None:+ """A templated URL is resolved against the workflow run context."""block = _make_block(url="{{ parameters.target_url }}")block.format_block_parameter_template_from_workflow_run_context = MagicMock(return_value="https://www.resolved-site.com/page"@@ -64,82 +50,95 @@ async def test_precreate_with_templated_url() -> None:with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = MagicMock()- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run = AsyncMock()- await _precreate_script_browser(- block=block,- workflow_run=wfr,- browser_session_id=None,- )+ assert _resolve_first_block_url([block], wfr) == "https://www.resolved-site.com/page"- call_kwargs = mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run.call_args- assert call_kwargs.kwargs["url"] == "https://www.resolved-site.com/page"+def test_resolve_first_block_url_skips_leading_blocks_without_url() -> None:+ """A leading block without a URL (e.g. a code block) is skipped."""+ no_url = _make_block(url=None)+ nav = _make_block(url="https://www.navigates.com")+ wfr = _make_workflow_run()-async def test_no_precreation_when_url_missing() -> None:- """Block without URL: no browser pre-creation, no app calls."""- block = _make_block(url=None)+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:+ mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = MagicMock()++ assert _resolve_first_block_url([no_url, nav], wfr) == "https://www.navigates.com"+++def test_resolve_first_block_url_uses_first_of_many() -> None:+ """When several blocks have URLs, the first one wins."""+ first = _make_block(url="https://www.first.com")+ second = _make_block(url="https://www.second.com")wfr = _make_workflow_run()with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:- await _precreate_script_browser(- block=block,- workflow_run=wfr,- browser_session_id=None,- )+ mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = MagicMock()- mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.assert_not_called()- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run.assert_not_called()+ assert _resolve_first_block_url([first, second], wfr) == "https://www.first.com"-async def test_no_precreation_when_url_empty_string() -> None:- """Block with empty-string URL: treated as falsy, no pre-creation."""- block = _make_block(url="")+def test_resolve_first_block_url_none_when_no_url() -> None:+ """No block has a URL → None."""+ blocks = [_make_block(url=None), _make_block(url="")]wfr = _make_workflow_run()with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:- await _precreate_script_browser(- block=block,- workflow_run=wfr,- browser_session_id=None,- )+ mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = MagicMock()- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run.assert_not_called()+ assert _resolve_first_block_url(blocks, wfr) is None-async def test_no_precreation_when_block_lacks_url_attr() -> None:- """Block type without a url attribute: skipped via getattr guard."""- block = MagicMock(spec=[]) # empty spec = no attributes+def test_resolve_first_block_url_continues_past_empty_resolution() -> None:+ """A block whose URL resolves to empty falls through to the next block."""+ empties_out = _make_block(url="{{ parameters.missing }}")+ empties_out.format_block_parameter_template_from_workflow_run_context = MagicMock(return_value="")+ nav = _make_block(url="https://www.real.com")wfr = _make_workflow_run()with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:- await _precreate_script_browser(- block=block,- workflow_run=wfr,- browser_session_id=None,- )+ mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = MagicMock()- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run.assert_not_called()+ assert _resolve_first_block_url([empties_out, nav], wfr) == "https://www.real.com"-async def test_browser_creation_failure_propagates() -> None:- """When browser creation raises, the exception propagates to the caller."""+def test_resolve_first_block_url_none_when_context_unavailable() -> None:+ """If the workflow run context can't be fetched, return None instead of raising."""block = _make_block(url="https://www.example.com")wfr = _make_workflow_run()with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:- mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.return_value = MagicMock()- mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run = AsyncMock(- side_effect=RuntimeError("browser creation failed")- )+ mock_app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context.side_effect = RuntimeError("no context")- with pytest.raises(RuntimeError, match="browser creation failed"):- await _precreate_script_browser(- block=block,- workflow_run=wfr,- browser_session_id=None,- )+ assert _resolve_first_block_url([block], wfr) is None+++async def test_script_page_selects_proxy_url_without_navigating() -> None:+ """The script SDK creates the browser with the URL (for proxy selection) but+ navigate=False, so the generated script performs the first goto itself."""+ from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage++ workflow_run = _make_workflow_run(browser_profile_id="bp_1")++ ctx = MagicMock()+ ctx.workflow_run_id = "wfr_test"+ ctx.organization_id = "o_1"++ with (+ patch("skyvern.core.script_generations.script_skyvern_page.app") as mock_app,+ patch("skyvern.core.script_generations.script_skyvern_page.skyvern_context") as mock_context,+ ):+ mock_context.current.return_value = ctx+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=workflow_run)+ mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run = AsyncMock()++ await ScriptSkyvernPage._get_or_create_browser_state(url="https://www.example.com/target")++ mock_app.BROWSER_MANAGER.get_or_create_for_workflow_run.assert_called_once_with(+ workflow_run=workflow_run,+ url="https://www.example.com/target",+ browser_session_id=None,+ browser_profile_id="bp_1",+ navigate=False,+ )