Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11025: harden extraction guardrail AST traversal (review follow-up to #12165) (#6585)
Andrew Neilson8cf43bffad44
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 3ae092218..db6a1031f 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -54,6 +54,7 @@ CodeArtifactMetadataValue: TypeAlias = (str | int | float | bool | None | list["CodeArtifactMetadataValue"] | dict[str, "CodeArtifactMetadataValue"])CodeArtifactMetadataPayload: TypeAlias = dict[str, CodeArtifactMetadataValue]+SdkActionWorkflowRunCacheKey: TypeAlias = tuple[str, str]def _playwright_private_impl(browser_context: object) -> object | None:@@ -138,6 +139,10 @@ class AgentContext:browser_session_id: str | Nonestream: EventSourceStreamapi_key: str | None = None+ # Ephemeral carrier for SDK-action run reuse, bounded by browser sessions touched in one Copilot run.+ sdk_action_workflow_run_ids_by_browser_session: dict[SdkActionWorkflowRunCacheKey, str] = field(+ default_factory=dict+ )supports_vision: bool = Truepending_screenshots: list[ScreenshotEntry] = field(default_factory=list)tool_activity: list[dict[str, Any]] = field(default_factory=list)@@ -313,6 +318,8 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:"""Push copilot browser state into the MCP session ContextVar for tool calls."""if not ctx.browser_session_id:raise RuntimeError("No browser_session_id set on agent context")+ browser_session_id = ctx.browser_session_id+ sdk_action_workflow_run_cache_key: SdkActionWorkflowRunCacheKey = (ctx.organization_id, browser_session_id)# Validate api_key at the boundary, before touching any backend.## The copilot FastAPI route runs outside MCPAPIKeyMiddleware, so the CLI@@ -325,13 +332,13 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:if not ctx.api_key:LOG.warning("mcp_browser_context invoked without api_key",- session_id=ctx.browser_session_id,+ session_id=browser_session_id,organization_id=ctx.organization_id,)raise RuntimeError("Copilot agent context missing api_key")browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(- session_id=ctx.browser_session_id,+ session_id=browser_session_id,organization_id=ctx.organization_id,)if not browser_state or not _browser_context_is_attachable(browser_state.browser_context):@@ -339,7 +346,7 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:# to LLM- or user-visible output -- but log it for operators.LOG.warning("No browser context for copilot session",- session_id=ctx.browser_session_id,+ session_id=browser_session_id,organization_id=ctx.organization_id,)raise RuntimeError("No browser context for copilot session")@@ -350,11 +357,14 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:skyvern_browser = SkyvernBrowser(skyvern_client,browser_state.browser_context,- browser_session_id=ctx.browser_session_id,+ browser_session_id=browser_session_id,+ )+ skyvern_browser.workflow_run_id = ctx.sdk_action_workflow_run_ids_by_browser_session.get(+ sdk_action_workflow_run_cache_key)mcp_ctx = MCPBrowserContext(mode="cloud_session",- session_id=ctx.browser_session_id,+ session_id=browser_session_id,can_access_localhost=_copilot_session_can_access_localhost(),)active_key = get_active_api_key()@@ -363,12 +373,18 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:context=mcp_ctx,api_key_hash=hash_api_key_for_cache(active_key) if active_key else None,)- register_copilot_session(ctx.browser_session_id, state)+ register_copilot_session(browser_session_id, state)try:async with scoped_session(state):yieldfinally:- unregister_copilot_session(ctx.browser_session_id)+ if skyvern_browser.workflow_run_id:+ ctx.sdk_action_workflow_run_ids_by_browser_session[sdk_action_workflow_run_cache_key] = (+ skyvern_browser.workflow_run_id+ )+ else:+ ctx.sdk_action_workflow_run_ids_by_browser_session.pop(sdk_action_workflow_run_cache_key, None)+ unregister_copilot_session(browser_session_id)finally:reset_api_key_override(override_token)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex d5543af67..fac8d2aee 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -1155,6 +1155,8 @@ def _iter_top_level_scope(statements: list[ast.stmt]) -> Iterator[ast.stmt]:for child in ast.iter_child_nodes(statement):if isinstance(child, ast.stmt):yield from _iter_top_level_scope([child])+ elif isinstance(child, (ast.ExceptHandler, ast.match_case)):+ yield from _iter_top_level_scope(child.body)def _code_block_returns_flat_string(code: str) -> bool:@@ -1170,8 +1172,11 @@ def _code_block_returns_flat_string(code: str) -> bool:string_locals: set[str] = set()for node in scope_statements:if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):+ name = node.targets[0].idif _expr_is_flat_string(node.value, string_locals):- string_locals.add(node.targets[0].id)+ string_locals.add(name)+ else:+ string_locals.discard(name)returns = [node for node in scope_statements if isinstance(node, ast.Return) and node.value is not None]if not returns:@@ -1182,12 +1187,16 @@ def _code_block_returns_flat_string(code: str) -> bool:def _function_body_has_structured_return(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:+ own_scope = list(_iter_top_level_scope(node.body))structured_locals: set[str] = set()- for inner in ast.walk(node):+ for inner in own_scope:if isinstance(inner, ast.Assign) and len(inner.targets) == 1 and isinstance(inner.targets[0], ast.Name):+ name = inner.targets[0].idif _expr_is_structured(inner.value):- structured_locals.add(inner.targets[0].id)- for inner in ast.walk(node):+ structured_locals.add(name)+ else:+ structured_locals.discard(name)+ for inner in own_scope:if not isinstance(inner, ast.Return) or inner.value is None:continueif _expr_is_structured(inner.value):@@ -1214,6 +1223,8 @@ def _code_block_returns_uninvoked_structured_function(code: str) -> bool:the function object instead of its data. Top-level structured returns or structuredlocal bindings (legit implicit capture) are not flagged, and anything indeterminatereturns False."""+ # CodeBlock wraps the snippet and appends `return __capture_locals()`, so a nested+ # function defined-but-never-called is captured as a function object, not its data.try:tree = ast.parse(textwrap.dedent(code).strip() or "pass")except SyntaxError:diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.pyindex c7a3e7f11..ff35e05a4 100644--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py@@ -698,6 +698,49 @@ class TestExtractionUninvokedNestedReturn:assert error is Noneassert list(normalized.keys()) == ["my_block"]+ def test_flat_return_inside_except_block_is_rejected(self) -> None:+ code = """+ try:+ data = page.locator("#results")+ except Exception:+ return page.inner_text("body")+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert normalized == {}+ assert error is not None+ assert "flat text blob" in error++ def test_structured_return_inside_except_block_passes(self) -> None:+ code = """+ try:+ rows = await page.locator(".row").all()+ except Exception:+ rows = []+ return {"records": [{"number": "1-25-80030"}]}+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_capture_then_wrap_rebind_passes(self) -> None:+ code = """+ text = await page.locator("#results").inner_text()+ text = {"records": [{"number": "1-25-80030"}]}+ return text+ """+ normalized, error = _normalize_code_artifact_metadata(+ [_extraction_metadata("my_block", ["records[].number"])],+ _extraction_code_block_yaml("my_block", code),+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]+class TestUninvokedStructuredFunctionClassifier:def test_uninvoked_structured_function_with_literal_return_is_flagged(self) -> None:@@ -746,6 +789,47 @@ class TestUninvokedStructuredFunctionClassifier:"""assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False+ def test_outer_with_only_deeper_nested_structured_return_is_not_flagged(self) -> None:+ # The structured return lives in a doubly-nested helper, not in the+ # uninvoked outer's own scope, so the outer must not look structured.+ code = """+ async def run(page):+ def helper():+ return {"records": []}+ await page.goto("https://example.com/")+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False++ def test_uninvoked_outer_with_own_structured_return_is_still_flagged(self) -> None:+ code = """+ async def run(page):+ def helper():+ return {"x": 1}+ result = {"records": []}+ return result+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is True++ def test_top_level_structured_return_in_except_is_not_flagged(self) -> None:+ code = """+ try:+ rows = page.locator(".row")+ except Exception:+ return {"records": []}+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False++ def test_nested_function_structured_local_rebound_to_flat_is_not_flagged(self) -> None:+ # The nested function rebinds its structured local to a flat read before+ # returning it, so it does not actually return structure and must not be flagged.+ code = """+ def run(page):+ result = {"records": []}+ result = page.inner_text("body")+ return result+ """+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False+class TestFlatStringClassifier:def test_string_literal_return_is_flat(self) -> None:@@ -777,6 +861,50 @@ class TestFlatStringClassifier:"""assert _code_block_returns_flat_string(textwrap.dedent(code)) is False+ def test_capture_then_wrap_rebind_under_same_name_is_not_flat(self) -> None:+ # Re-binding a flat local to a structured value must clear it from+ # string_locals so the final structured return is not falsely rejected.+ code = """+ text = await page.inner_text("body")+ text = {"records": [{"number": "1-25-80030"}]}+ return text+ """+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is False++ def test_rebound_still_flat_local_is_flat(self) -> None:+ code = """+ text = "seed"+ text = await page.inner_text("body")+ return text+ """+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is True++ def test_flat_return_inside_except_block_is_flat(self) -> None:+ code = """+ try:+ data = page.locator("#x")+ except Exception:+ return page.inner_text("body")+ """+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is True++ def test_flat_return_inside_match_case_is_flat(self) -> None:+ code = """+ match mode:+ case "x":+ return page.inner_text("body")+ """+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is True++ def test_structured_return_inside_except_block_is_not_flat(self) -> None:+ code = """+ try:+ rows = page.locator(".row")+ except Exception:+ return {"records": []}+ """+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is False+def _two_code_block_yaml(first: str, second: str) -> str:return textwrap.dedent(diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.pyindex d3599b8ad..4eeaedcf1 100644--- a/tests/unit/test_copilot_session_injection.py+++ b/tests/unit/test_copilot_session_injection.py@@ -201,6 +201,67 @@ class TestMcpBrowserContextBridge:# Override installed then reset.assert [c[0] for c in override_calls] == ["set", "reset"]+ @pytest.mark.asyncio+ async def test_sdk_action_workflow_run_id_is_reused_across_tool_contexts(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import skyvern.forge.sdk.copilot.runtime as runtime+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context++ _, register_mock, unregister_mock, _, _ = self._install_happy_path_mocks(monkeypatch)++ first_browser = MagicMock()+ first_browser.workflow_run_id = None+ second_browser = MagicMock()+ second_browser.workflow_run_id = None+ third_browser = MagicMock()+ third_browser.workflow_run_id = None+ fourth_browser = MagicMock()+ fourth_browser.workflow_run_id = None+ fifth_browser = MagicMock()+ fifth_browser.workflow_run_id = None+ browsers = iter([first_browser, second_browser, third_browser, fourth_browser, fifth_browser])+ monkeypatch.setattr(runtime, "SkyvernBrowser", lambda *a, **kw: next(browsers))++ ctx = _make_ctx()+ assert ctx.browser_session_id is not None+ original_cache_key = (ctx.organization_id, ctx.browser_session_id)++ async with mcp_browser_context(ctx):+ first_state = register_mock.call_args.args[1]+ assert first_state.browser.workflow_run_id is None+ first_state.browser.workflow_run_id = "wr_sdk_action_1"++ assert ctx.sdk_action_workflow_run_ids_by_browser_session[original_cache_key] == "wr_sdk_action_1"++ async with mcp_browser_context(ctx):+ second_state = register_mock.call_args.args[1]+ assert second_state.browser.workflow_run_id == "wr_sdk_action_1"+ second_state.browser.workflow_run_id = None++ assert original_cache_key not in ctx.sdk_action_workflow_run_ids_by_browser_session++ async with mcp_browser_context(ctx):+ third_state = register_mock.call_args.args[1]+ assert third_state.browser.workflow_run_id is None++ ctx.browser_session_id = "pbs_test_456"+ async with mcp_browser_context(ctx):+ fourth_state = register_mock.call_args.args[1]+ assert fourth_state.browser.workflow_run_id is None++ assert (ctx.organization_id, "pbs_test_456") not in ctx.sdk_action_workflow_run_ids_by_browser_session++ ctx.organization_id = "org-2"+ ctx.browser_session_id = "pbs_test_123"+ async with mcp_browser_context(ctx):+ fifth_state = register_mock.call_args.args[1]+ assert fifth_state.browser.workflow_run_id is None++ assert ("org-2", "pbs_test_123") not in ctx.sdk_action_workflow_run_ids_by_browser_session+ assert register_mock.call_count == 5+ assert unregister_mock.call_count == 5+@pytest.mark.asyncioasync def test_missing_browser_context_raises_without_leaking_session_id(self, monkeypatch: pytest.MonkeyPatch