Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10346: Preserve runtime frontier browser state (PR 10/12) (#6262)
Andrew Neilson88387a208314
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 134a3435f..2b646aadd 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -106,6 +106,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yamlfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException+from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, get_all_blocksfrom skyvern.forge.sdk.workflow.models.parameter import (RESERVED_PARAMETER_KEYS,OutputParameter,@@ -2864,6 +2865,149 @@ def _valid_runtime_anchor_url(value: object) -> str | None:return url+def _same_runtime_page(left: str | None, right: str | None) -> bool:+ if not left or not right:+ return False+ try:+ left_parsed = urlparse(left)+ right_parsed = urlparse(right)+ except ValueError:+ return False+ return (+ left_parsed.scheme.lower(),+ left_parsed.netloc.lower(),+ left_parsed.path,+ left_parsed.query,+ ) == (+ right_parsed.scheme.lower(),+ right_parsed.netloc.lower(),+ right_parsed.path,+ right_parsed.query,+ )+++def _frontier_anchor_url_from_value(value: object, *, depth: int = 0) -> str | None:+ if depth > 3:+ return None+ direct = _valid_runtime_anchor_url(value)+ if direct is not None:+ return direct+ if isinstance(value, dict):+ for key in ("current_url", "url", "page_url"):+ candidate = _valid_runtime_anchor_url(value.get(key))+ if candidate is not None:+ return candidate+ for nested in value.values():+ candidate = _frontier_anchor_url_from_value(nested, depth=depth + 1)+ if candidate is not None:+ return candidate+ if isinstance(value, list):+ for nested in value:+ candidate = _frontier_anchor_url_from_value(nested, depth=depth + 1)+ if candidate is not None:+ return candidate+ return None+++def _frontier_runtime_anchor_url(ctx: CopilotContext, block_outputs_to_seed: dict[str, Any]) -> str | None:+ for value in (+ ctx.verified_prefix_current_url,+ getattr(ctx, "composition_page_evidence", None),+ ):+ candidate = _frontier_anchor_url_from_value(value)+ if candidate is not None:+ return candidate+ for value in reversed(list((block_outputs_to_seed or {}).values())):+ candidate = _frontier_anchor_url_from_value(value)+ if candidate is not None:+ return candidate+ return None+++def _iter_workflow_model_blocks(blocks: list[BlockTypeVar] | None) -> list[BlockTypeVar]:+ if not isinstance(blocks, list):+ return []+ return get_all_blocks(blocks)+++def _workflow_model_block_by_label(workflow_definition: object | None, label: str | None) -> BlockTypeVar | None:+ if workflow_definition is None or not label:+ return None+ blocks = workflow_definition.blocks if hasattr(workflow_definition, "blocks") else None+ for block in _iter_workflow_model_blocks(blocks):+ if block.label == label:+ return block+ return None+++def _has_verified_prefix_before_frontier(+ ctx: CopilotContext, workflow_definition: object | None, frontier_label: str | None+) -> bool:+ if not frontier_label:+ return False+ workflow_labels = _workflow_definition_block_labels(workflow_definition)+ if frontier_label not in workflow_labels:+ return False+ prefix_labels = workflow_labels[: workflow_labels.index(frontier_label)]+ if not prefix_labels:+ return False+ verified = set(ctx.verified_prefix_labels or [])+ return all(label in verified for label in prefix_labels)+++def _workflow_with_runtime_frontier_anchor(+ workflow: Workflow,+ ctx: CopilotContext,+ *,+ labels_to_execute: list[str],+ frontier_start_label: str | None,+ block_outputs_to_seed: dict[str, Any],+) -> tuple[Workflow, str | None]:+ if not labels_to_execute:+ return workflow, None+ workflow_definition = workflow.workflow_definition+ if not _has_verified_prefix_before_frontier(ctx, workflow_definition, frontier_start_label):+ return workflow, None++ first_label = labels_to_execute[0]+ first_block = _workflow_model_block_by_label(workflow_definition, first_label)+ if first_block is None or not hasattr(first_block, "url"):+ return workflow, None++ anchor_url = _frontier_runtime_anchor_url(ctx, block_outputs_to_seed)+ if anchor_url is None:+ return workflow, None++ existing_url = _valid_runtime_anchor_url(first_block.url if hasattr(first_block, "url") else None)+ if existing_url is not None and not _same_runtime_page(existing_url, anchor_url):+ return workflow, None++ if existing_url is not None:+ if _block_type_name(first_block) != BlockType.NAVIGATION.value:+ return workflow, None+ anchored = workflow.model_copy(deep=True)+ anchored_block = _workflow_model_block_by_label(anchored.workflow_definition, first_label)+ if anchored_block is None or not hasattr(anchored_block, "url"):+ return workflow, None+ anchored_block.url = None+ LOG.info(+ "Cleared runtime frontier URL to preserve browser state",+ frontier_start_label=frontier_start_label,+ first_block_label=first_label,+ existing_url=existing_url,+ continuation_url=anchor_url,+ )+ return anchored, anchor_url++ LOG.info(+ "Preserved runtime frontier browser state without URL reload",+ frontier_start_label=frontier_start_label,+ first_block_label=first_label,+ continuation_url=anchor_url,+ )+ return workflow, anchor_url++async def _run_blocks_and_collect_debug(params: dict[str, Any],ctx: CopilotContext,@@ -2926,6 +3070,13 @@ async def _run_blocks_and_collect_debug(organization = Organization.model_validate(org)runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)+ runtime_workflow, runtime_frontier_anchor_url = _workflow_with_runtime_frontier_anchor(+ runtime_workflow,+ ctx,+ labels_to_execute=labels_to_execute,+ frontier_start_label=frontier_start_label,+ block_outputs_to_seed=block_outputs_to_seed,+ )user_params: dict[str, Any] = params.get("parameters") or {}all_workflow_params, all_output_params = await asyncio.gather(@@ -3294,6 +3445,8 @@ async def _run_blocks_and_collect_debug("page_title": page_title,"action_trace_summary": action_trace_summary,}+ if runtime_frontier_anchor_url is not None:+ result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_urlif screenshot_b64 is not None:result_data["screenshot_base64"] = screenshot_b64if not run_ok and run and getattr(run, "failure_reason", None):diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex ac842a461..787fb06fc 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -2,6 +2,7 @@from __future__ import annotations+import copyfrom typing import Anyimport pytest@@ -71,6 +72,9 @@ class _FakeWorkflow:def __init__(self, definition: _FakeDefinition) -> None:self.workflow_definition = definition+ def model_copy(self, *, deep: bool = False) -> _FakeWorkflow:+ return copy.deepcopy(self) if deep else _FakeWorkflow(self.workflow_definition)+class _FakeStream:async def is_disconnected(self) -> bool:@@ -238,6 +242,133 @@ def test_plan_frontier_suffix_only_request_seeds_prior_browser_state_outputs() -assert frontier == "expand"+def test_runtime_frontier_anchor_keeps_url_empty_to_preserve_live_state() -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": None}),+ _FakeBlock("extract", "extraction"),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open"]+ ctx.verified_prefix_current_url = "https://example.com/search"++ anchored, anchor_url = tools._workflow_with_runtime_frontier_anchor(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search", "extract"],+ frontier_start_label="search",+ block_outputs_to_seed={},+ )++ assert anchor_url == "https://example.com/search"+ assert anchored is workflow+ assert workflow.workflow_definition.blocks[1].url is None+++def test_runtime_frontier_anchor_requires_verified_prefix() -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": None}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx()+ ctx.verified_prefix_current_url = "https://example.com/search"++ anchored, anchor_url = tools._workflow_with_runtime_frontier_anchor(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search"],+ frontier_start_label="search",+ block_outputs_to_seed={},+ )++ assert anchor_url is None+ assert anchored is workflow+ assert workflow.workflow_definition.blocks[1].url is None+++def test_runtime_frontier_anchor_does_not_override_explicit_block_url() -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("search", "navigation", {"url": "https://example.com/explicit"}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open"]+ ctx.verified_prefix_current_url = "https://example.com/search"++ anchored, anchor_url = tools._workflow_with_runtime_frontier_anchor(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["search"],+ frontier_start_label="search",+ block_outputs_to_seed={},+ )++ assert anchor_url is None+ assert anchored is workflow+ assert workflow.workflow_definition.blocks[1].url == "https://example.com/explicit"+++def test_runtime_frontier_anchor_clears_same_page_url_to_preserve_state() -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("set_search", "navigation", {"url": None}),+ _FakeBlock("submit_search", "navigation", {"url": "https://example.com/search"}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open", "set_search"]+ ctx.verified_prefix_current_url = "https://example.com/search"++ anchored, anchor_url = tools._workflow_with_runtime_frontier_anchor(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["submit_search"],+ frontier_start_label="submit_search",+ block_outputs_to_seed={},+ )++ assert anchor_url == "https://example.com/search"+ assert anchored is not workflow+ assert anchored.workflow_definition.blocks[2].url is None+ assert workflow.workflow_definition.blocks[2].url == "https://example.com/search"+++def test_runtime_frontier_anchor_does_not_clear_same_page_goto_url() -> None:+ definition = _FakeDefinition(+ [+ _FakeBlock("open", "goto_url", {"url": "https://example.com/search"}),+ _FakeBlock("refresh", "goto_url", {"url": "https://example.com/search"}),+ ]+ )+ workflow = _FakeWorkflow(definition)+ ctx = _make_ctx()+ ctx.verified_prefix_labels = ["open"]+ ctx.verified_prefix_current_url = "https://example.com/search"++ anchored, anchor_url = tools._workflow_with_runtime_frontier_anchor(+ workflow, # type: ignore[arg-type]+ ctx,+ labels_to_execute=["refresh"],+ frontier_start_label="refresh",+ block_outputs_to_seed={},+ )++ assert anchor_url is None+ assert anchored is workflow+ assert workflow.workflow_definition.blocks[1].url == "https://example.com/search"++def test_frontier_run_size_error_limits_long_page_changing_frontier() -> None:definition = _FakeDefinition([