Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10585: seed copilot frontier starter URL (#6292)
Andrew Neilsonbdcf0d151453
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex e8f86f8a0..62d8f49b8 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -3254,6 +3254,21 @@ def _valid_runtime_anchor_url(value: object) -> str | None:return url+def _blank_runtime_page_url(value: object) -> bool:+ if not isinstance(value, str):+ return True+ # Chrome/CDP can expose ":" while the page is still in early blank-page initialization.+ return value.strip() in {"about:blank", "", ":"}+++def _missing_runtime_frontier_block_url(value: object) -> bool:+ if value is None:+ return True+ if isinstance(value, str):+ return not value.strip()+ return False++def _same_runtime_page(left: str | None, right: str | None) -> bool:if not left or not right:return False@@ -3397,6 +3412,70 @@ def _workflow_with_runtime_frontier_anchor(return workflow, anchor_url+async def _workflow_with_runtime_frontier_starter_url_seed(+ workflow: Workflow,+ ctx: CopilotContext,+ *,+ labels_to_execute: list[str],+ runtime_frontier_anchor_url: str | None,+) -> Workflow:+ if not labels_to_execute or runtime_frontier_anchor_url is None or not ctx.browser_session_id:+ return workflow++ first_label = labels_to_execute[0]+ first_block = _workflow_model_block_by_label(workflow.workflow_definition, first_label)+ if (+ first_block is None+ or not hasattr(first_block, "url")+ or _block_type_name(first_block) != BlockType.NAVIGATION.value+ or not _missing_runtime_frontier_block_url(first_block.url)+ ):+ return workflow++ current_page_url: str | None = None+ try:+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(+ session_id=ctx.browser_session_id,+ organization_id=ctx.organization_id,+ )+ if browser_state is not None:+ page = await browser_state.get_working_page()+ # Playwright Page.url is exposed as a dynamic property at this boundary.+ current_page_url = page.url if page is not None else None+ except Exception:+ LOG.debug(+ "Failed to inspect runtime frontier browser page before starter URL seed",+ browser_session_id=ctx.browser_session_id,+ frontier_start_label=first_label,+ exc_info=True,+ )++ if not _blank_runtime_page_url(current_page_url):+ LOG.info(+ "Preserved attached runtime frontier browser page",+ browser_session_id=ctx.browser_session_id,+ frontier_start_label=first_label,+ current_url=current_page_url,+ continuation_url=runtime_frontier_anchor_url,+ )+ return workflow++ seeded = workflow.model_copy(deep=True)+ seeded_block = _workflow_model_block_by_label(seeded.workflow_definition, first_label)+ # Defensive: the copied workflow definition should preserve labels and URL fields.+ if seeded_block is None or not hasattr(seeded_block, "url"):+ return workflow+ seeded_block.url = runtime_frontier_anchor_url+ LOG.info(+ "Seeded runtime frontier starter URL because attached browser page was blank",+ browser_session_id=ctx.browser_session_id,+ frontier_start_label=first_label,+ current_url=current_page_url,+ continuation_url=runtime_frontier_anchor_url,+ )+ return seeded++async def _run_blocks_and_collect_debug(params: dict[str, Any],ctx: CopilotContext,@@ -3466,6 +3545,7 @@ async def _run_blocks_and_collect_debug(frontier_start_label=frontier_start_label,block_outputs_to_seed=block_outputs_to_seed,)+ runtime_frontier_starter_url_seeded = Falseuser_params: dict[str, Any] = params.get("parameters") or {}all_workflow_params, all_output_params = await asyncio.gather(@@ -3523,6 +3603,15 @@ async def _run_blocks_and_collect_debug(if session_err is not None:return session_err+ seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(+ runtime_workflow,+ ctx,+ labels_to_execute=labels_to_execute,+ runtime_frontier_anchor_url=runtime_frontier_anchor_url,+ )+ runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow+ runtime_workflow = seeded_runtime_workflow+workflow_request = WorkflowRequestBody(data=data if data else None,browser_session_id=ctx.browser_session_id,@@ -3836,6 +3925,8 @@ async def _run_blocks_and_collect_debug(}if runtime_frontier_anchor_url is not None:result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_url+ if runtime_frontier_starter_url_seeded:+ result_data["runtime_frontier_starter_url_seeded"] = Trueif screenshot_b64 is not None:result_data["screenshot_base64"] = screenshot_b64if not run_ok and run and getattr(run, "failure_reason", None):diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex 787fb06fc..4dbf8ec31 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -84,6 +84,32 @@ class _FakeStream:return None+class _FakePage:+ def __init__(self, url: str) -> None:+ self.url = url+++class _FakeBrowserState:+ def __init__(self, page: _FakePage | None) -> None:+ self._page = page++ async def get_working_page(self) -> _FakePage | None:+ return self._page+++class _FakePersistentSessionsManager:+ def __init__(self, browser_state: _FakeBrowserState | None) -> None:+ self._browser_state = browser_state++ async def get_browser_state(self, session_id: str, organization_id: str) -> _FakeBrowserState | None:+ return self._browser_state+++class _FakeFailingPersistentSessionsManager:+ async def get_browser_state(self, session_id: str, organization_id: str) -> _FakeBrowserState | None:+ raise RuntimeError("browser state unavailable")++def _make_ctx(**kwargs: object) -> CopilotContext:defaults: dict[str, Any] = dict(organization_id="org",@@ -268,6 +294,129 @@ def test_runtime_frontier_anchor_keeps_url_empty_to_preserve_live_state() -> Nonassert workflow.workflow_definition.blocks[1].url is None+async def test_runtime_frontier_starter_url_seed_fills_blank_browser_state(monkeypatch: pytest.MonkeyPatch) -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": None}),+ _FakeBlock("extract", "extraction"),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx(browser_session_id="pbs_123")++ monkeypatch.setattr(+ tools.app,+ "PERSISTENT_SESSIONS_MANAGER",+ _FakePersistentSessionsManager(_FakeBrowserState(_FakePage("about:blank"))),+ )++ seeded = await tools._workflow_with_runtime_frontier_starter_url_seed(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search", "extract"],+ runtime_frontier_anchor_url="https://example.com/search",+ )++ assert seeded is not workflow+ assert seeded.workflow_definition.blocks[1].url == "https://example.com/search"+ assert workflow.workflow_definition.blocks[1].url is None+++async def test_runtime_frontier_starter_url_seed_fills_when_browser_state_lookup_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": None}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx(browser_session_id="pbs_123")++ monkeypatch.setattr(+ tools.app,+ "PERSISTENT_SESSIONS_MANAGER",+ _FakeFailingPersistentSessionsManager(),+ )++ seeded = await tools._workflow_with_runtime_frontier_starter_url_seed(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search"],+ runtime_frontier_anchor_url="https://example.com/search",+ )++ assert seeded is not workflow+ assert seeded.workflow_definition.blocks[1].url == "https://example.com/search"+ assert workflow.workflow_definition.blocks[1].url is None+++async def test_runtime_frontier_starter_url_seed_preserves_attached_live_page(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": None}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx(browser_session_id="pbs_123")++ monkeypatch.setattr(+ tools.app,+ "PERSISTENT_SESSIONS_MANAGER",+ _FakePersistentSessionsManager(_FakeBrowserState(_FakePage("https://example.com/search/results"))),+ )++ seeded = await tools._workflow_with_runtime_frontier_starter_url_seed(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search"],+ runtime_frontier_anchor_url="https://example.com/search",+ )++ assert seeded is workflow+ assert workflow.workflow_definition.blocks[1].url is None++[email protected]("explicit_url", ["start_url", "{{ start_url }}", "example.com"])+async def test_runtime_frontier_starter_url_seed_preserves_runtime_resolved_url(+ monkeypatch: pytest.MonkeyPatch,+ explicit_url: str,+) -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": explicit_url}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx(browser_session_id="pbs_123")++ monkeypatch.setattr(+ tools.app,+ "PERSISTENT_SESSIONS_MANAGER",+ _FakePersistentSessionsManager(_FakeBrowserState(_FakePage("about:blank"))),+ )++ seeded = await tools._workflow_with_runtime_frontier_starter_url_seed(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search"],+ runtime_frontier_anchor_url="https://example.com/search",+ )++ assert seeded is workflow+ assert workflow.workflow_definition.blocks[1].url == explicit_url++def test_runtime_frontier_anchor_requires_verified_prefix() -> None:definition = _FakeDefinition([