Skip to content

Automate browser based workflows with AI

SKY-11025: harden extraction guardrail AST traversal (review follow-up to #12165) (#6585)

Andrew Neilson8cf43bffad44
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 3ae092218..db6a1031f 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -54,6 +54,7 @@ CodeArtifactMetadataValue: TypeAlias = (
str | int | float | bool | None | list["CodeArtifactMetadataValue"] | dict[str, "CodeArtifactMetadataValue"]
)
CodeArtifactMetadataPayload: TypeAlias = dict[str, CodeArtifactMetadataValue]
+SdkActionWorkflowRunCacheKey: TypeAlias = tuple[str, str]
def _playwright_private_impl(browser_context: object) -> object | None:
@@ -138,6 +139,10 @@ class AgentContext:
browser_session_id: str | None
stream: EventSourceStream
api_key: str | None = None
+ # Ephemeral carrier for SDK-action run reuse, bounded by browser sessions touched in one Copilot run.
+ sdk_action_workflow_run_ids_by_browser_session: dict[SdkActionWorkflowRunCacheKey, str] = field(
+ default_factory=dict
+ )
supports_vision: bool = True
pending_screenshots: list[ScreenshotEntry] = field(default_factory=list)
tool_activity: list[dict[str, Any]] = field(default_factory=list)
@@ -313,6 +318,8 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
"""Push copilot browser state into the MCP session ContextVar for tool calls."""
if not ctx.browser_session_id:
raise RuntimeError("No browser_session_id set on agent context")
+ browser_session_id = ctx.browser_session_id
+ sdk_action_workflow_run_cache_key: SdkActionWorkflowRunCacheKey = (ctx.organization_id, browser_session_id)
# Validate api_key at the boundary, before touching any backend.
#
# The copilot FastAPI route runs outside MCPAPIKeyMiddleware, so the CLI
@@ -325,13 +332,13 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
if not ctx.api_key:
LOG.warning(
"mcp_browser_context invoked without api_key",
- session_id=ctx.browser_session_id,
+ session_id=browser_session_id,
organization_id=ctx.organization_id,
)
raise RuntimeError("Copilot agent context missing api_key")
browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
- session_id=ctx.browser_session_id,
+ session_id=browser_session_id,
organization_id=ctx.organization_id,
)
if not browser_state or not _browser_context_is_attachable(browser_state.browser_context):
@@ -339,7 +346,7 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
# to LLM- or user-visible output -- but log it for operators.
LOG.warning(
"No browser context for copilot session",
- session_id=ctx.browser_session_id,
+ session_id=browser_session_id,
organization_id=ctx.organization_id,
)
raise RuntimeError("No browser context for copilot session")
@@ -350,11 +357,14 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
skyvern_browser = SkyvernBrowser(
skyvern_client,
browser_state.browser_context,
- browser_session_id=ctx.browser_session_id,
+ browser_session_id=browser_session_id,
+ )
+ skyvern_browser.workflow_run_id = ctx.sdk_action_workflow_run_ids_by_browser_session.get(
+ sdk_action_workflow_run_cache_key
)
mcp_ctx = MCPBrowserContext(
mode="cloud_session",
- session_id=ctx.browser_session_id,
+ session_id=browser_session_id,
can_access_localhost=_copilot_session_can_access_localhost(),
)
active_key = get_active_api_key()
@@ -363,12 +373,18 @@ async def mcp_browser_context(ctx: AgentContext) -> AsyncIterator[None]:
context=mcp_ctx,
api_key_hash=hash_api_key_for_cache(active_key) if active_key else None,
)
- register_copilot_session(ctx.browser_session_id, state)
+ register_copilot_session(browser_session_id, state)
try:
async with scoped_session(state):
yield
finally:
- unregister_copilot_session(ctx.browser_session_id)
+ if skyvern_browser.workflow_run_id:
+ ctx.sdk_action_workflow_run_ids_by_browser_session[sdk_action_workflow_run_cache_key] = (
+ skyvern_browser.workflow_run_id
+ )
+ else:
+ ctx.sdk_action_workflow_run_ids_by_browser_session.pop(sdk_action_workflow_run_cache_key, None)
+ unregister_copilot_session(browser_session_id)
finally:
reset_api_key_override(override_token)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index d5543af67..fac8d2aee 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1155,6 +1155,8 @@ def _iter_top_level_scope(statements: list[ast.stmt]) -> Iterator[ast.stmt]:
for child in ast.iter_child_nodes(statement):
if isinstance(child, ast.stmt):
yield from _iter_top_level_scope([child])
+ elif isinstance(child, (ast.ExceptHandler, ast.match_case)):
+ yield from _iter_top_level_scope(child.body)
def _code_block_returns_flat_string(code: str) -> bool:
@@ -1170,8 +1172,11 @@ def _code_block_returns_flat_string(code: str) -> bool:
string_locals: set[str] = set()
for node in scope_statements:
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ name = node.targets[0].id
if _expr_is_flat_string(node.value, string_locals):
- string_locals.add(node.targets[0].id)
+ string_locals.add(name)
+ else:
+ string_locals.discard(name)
returns = [node for node in scope_statements if isinstance(node, ast.Return) and node.value is not None]
if not returns:
@@ -1182,12 +1187,16 @@ def _code_block_returns_flat_string(code: str) -> bool:
def _function_body_has_structured_return(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
+ own_scope = list(_iter_top_level_scope(node.body))
structured_locals: set[str] = set()
- for inner in ast.walk(node):
+ for inner in own_scope:
if isinstance(inner, ast.Assign) and len(inner.targets) == 1 and isinstance(inner.targets[0], ast.Name):
+ name = inner.targets[0].id
if _expr_is_structured(inner.value):
- structured_locals.add(inner.targets[0].id)
- for inner in ast.walk(node):
+ structured_locals.add(name)
+ else:
+ structured_locals.discard(name)
+ for inner in own_scope:
if not isinstance(inner, ast.Return) or inner.value is None:
continue
if _expr_is_structured(inner.value):
@@ -1214,6 +1223,8 @@ def _code_block_returns_uninvoked_structured_function(code: str) -> bool:
the function object instead of its data. Top-level structured returns or structured
local bindings (legit implicit capture) are not flagged, and anything indeterminate
returns False."""
+ # CodeBlock wraps the snippet and appends `return __capture_locals()`, so a nested
+ # function defined-but-never-called is captured as a function object, not its data.
try:
tree = ast.parse(textwrap.dedent(code).strip() or "pass")
except SyntaxError:
diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.py
index c7a3e7f11..ff35e05a4 100644
--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py
+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py
@@ -698,6 +698,49 @@ class TestExtractionUninvokedNestedReturn:
assert error is None
assert list(normalized.keys()) == ["my_block"]
+ def test_flat_return_inside_except_block_is_rejected(self) -> None:
+ code = """
+ try:
+ data = page.locator("#results")
+ except Exception:
+ return page.inner_text("body")
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "flat text blob" in error
+
+ def test_structured_return_inside_except_block_passes(self) -> None:
+ code = """
+ try:
+ rows = await page.locator(".row").all()
+ except Exception:
+ rows = []
+ return {"records": [{"number": "1-25-80030"}]}
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_capture_then_wrap_rebind_passes(self) -> None:
+ code = """
+ text = await page.locator("#results").inner_text()
+ text = {"records": [{"number": "1-25-80030"}]}
+ return text
+ """
+ normalized, error = _normalize_code_artifact_metadata(
+ [_extraction_metadata("my_block", ["records[].number"])],
+ _extraction_code_block_yaml("my_block", code),
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
class TestUninvokedStructuredFunctionClassifier:
def test_uninvoked_structured_function_with_literal_return_is_flagged(self) -> None:
@@ -746,6 +789,47 @@ class TestUninvokedStructuredFunctionClassifier:
"""
assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+ def test_outer_with_only_deeper_nested_structured_return_is_not_flagged(self) -> None:
+ # The structured return lives in a doubly-nested helper, not in the
+ # uninvoked outer's own scope, so the outer must not look structured.
+ code = """
+ async def run(page):
+ def helper():
+ return {"records": []}
+ await page.goto("https://example.com/")
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
+ def test_uninvoked_outer_with_own_structured_return_is_still_flagged(self) -> None:
+ code = """
+ async def run(page):
+ def helper():
+ return {"x": 1}
+ result = {"records": []}
+ return result
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is True
+
+ def test_top_level_structured_return_in_except_is_not_flagged(self) -> None:
+ code = """
+ try:
+ rows = page.locator(".row")
+ except Exception:
+ return {"records": []}
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
+ def test_nested_function_structured_local_rebound_to_flat_is_not_flagged(self) -> None:
+ # The nested function rebinds its structured local to a flat read before
+ # returning it, so it does not actually return structure and must not be flagged.
+ code = """
+ def run(page):
+ result = {"records": []}
+ result = page.inner_text("body")
+ return result
+ """
+ assert _code_block_returns_uninvoked_structured_function(textwrap.dedent(code)) is False
+
class TestFlatStringClassifier:
def test_string_literal_return_is_flat(self) -> None:
@@ -777,6 +861,50 @@ class TestFlatStringClassifier:
"""
assert _code_block_returns_flat_string(textwrap.dedent(code)) is False
+ def test_capture_then_wrap_rebind_under_same_name_is_not_flat(self) -> None:
+ # Re-binding a flat local to a structured value must clear it from
+ # string_locals so the final structured return is not falsely rejected.
+ code = """
+ text = await page.inner_text("body")
+ text = {"records": [{"number": "1-25-80030"}]}
+ return text
+ """
+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is False
+
+ def test_rebound_still_flat_local_is_flat(self) -> None:
+ code = """
+ text = "seed"
+ text = await page.inner_text("body")
+ return text
+ """
+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is True
+
+ def test_flat_return_inside_except_block_is_flat(self) -> None:
+ code = """
+ try:
+ data = page.locator("#x")
+ except Exception:
+ return page.inner_text("body")
+ """
+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is True
+
+ def test_flat_return_inside_match_case_is_flat(self) -> None:
+ code = """
+ match mode:
+ case "x":
+ return page.inner_text("body")
+ """
+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is True
+
+ def test_structured_return_inside_except_block_is_not_flat(self) -> None:
+ code = """
+ try:
+ rows = page.locator(".row")
+ except Exception:
+ return {"records": []}
+ """
+ assert _code_block_returns_flat_string(textwrap.dedent(code)) is False
+
def _two_code_block_yaml(first: str, second: str) -> str:
return textwrap.dedent(
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
index d3599b8ad..4eeaedcf1 100644
--- a/tests/unit/test_copilot_session_injection.py
+++ b/tests/unit/test_copilot_session_injection.py
@@ -201,6 +201,67 @@ class TestMcpBrowserContextBridge:
# Override installed then reset.
assert [c[0] for c in override_calls] == ["set", "reset"]
+ @pytest.mark.asyncio
+ async def test_sdk_action_workflow_run_id_is_reused_across_tool_contexts(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import skyvern.forge.sdk.copilot.runtime as runtime
+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
+
+ _, register_mock, unregister_mock, _, _ = self._install_happy_path_mocks(monkeypatch)
+
+ first_browser = MagicMock()
+ first_browser.workflow_run_id = None
+ second_browser = MagicMock()
+ second_browser.workflow_run_id = None
+ third_browser = MagicMock()
+ third_browser.workflow_run_id = None
+ fourth_browser = MagicMock()
+ fourth_browser.workflow_run_id = None
+ fifth_browser = MagicMock()
+ fifth_browser.workflow_run_id = None
+ browsers = iter([first_browser, second_browser, third_browser, fourth_browser, fifth_browser])
+ monkeypatch.setattr(runtime, "SkyvernBrowser", lambda *a, **kw: next(browsers))
+
+ ctx = _make_ctx()
+ assert ctx.browser_session_id is not None
+ original_cache_key = (ctx.organization_id, ctx.browser_session_id)
+
+ async with mcp_browser_context(ctx):
+ first_state = register_mock.call_args.args[1]
+ assert first_state.browser.workflow_run_id is None
+ first_state.browser.workflow_run_id = "wr_sdk_action_1"
+
+ assert ctx.sdk_action_workflow_run_ids_by_browser_session[original_cache_key] == "wr_sdk_action_1"
+
+ async with mcp_browser_context(ctx):
+ second_state = register_mock.call_args.args[1]
+ assert second_state.browser.workflow_run_id == "wr_sdk_action_1"
+ second_state.browser.workflow_run_id = None
+
+ assert original_cache_key not in ctx.sdk_action_workflow_run_ids_by_browser_session
+
+ async with mcp_browser_context(ctx):
+ third_state = register_mock.call_args.args[1]
+ assert third_state.browser.workflow_run_id is None
+
+ ctx.browser_session_id = "pbs_test_456"
+ async with mcp_browser_context(ctx):
+ fourth_state = register_mock.call_args.args[1]
+ assert fourth_state.browser.workflow_run_id is None
+
+ assert (ctx.organization_id, "pbs_test_456") not in ctx.sdk_action_workflow_run_ids_by_browser_session
+
+ ctx.organization_id = "org-2"
+ ctx.browser_session_id = "pbs_test_123"
+ async with mcp_browser_context(ctx):
+ fifth_state = register_mock.call_args.args[1]
+ assert fifth_state.browser.workflow_run_id is None
+
+ assert ("org-2", "pbs_test_123") not in ctx.sdk_action_workflow_run_ids_by_browser_session
+ assert register_mock.call_count == 5
+ assert unregister_mock.call_count == 5
+
@pytest.mark.asyncio
async def test_missing_browser_context_raises_without_leaking_session_id(
self, monkeypatch: pytest.MonkeyPatch