Skip to content

Automate browser based workflows with AI

SKY-11139: Harden code block runtime boundary (#6686)

Andrew Neilsonef62e8aafcfe
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.py
index 1123a26fb..bdd11032f 100644
--- a/skyvern/forge/sdk/copilot/code_block_steps.py
+++ b/skyvern/forge/sdk/copilot/code_block_steps.py
@@ -12,27 +12,23 @@ LOG = structlog.get_logger()
# Playwright/SkyvernPage method name -> step action_type (values are ActionType members;
# kept as string literals so this module does not import skyvern.webeye, matching code_block_synthesis.py).
-# Mirror the runtime recorder's maps (code_block_recorder._PAGE_ACTION_MAP / _LOCATOR_ACTION_MAP) so the
-# static editor preview surfaces the same calls the timeline records — otherwise a recorded call (e.g.
-# page.evaluate) renders in the timeline but is silently dropped from the editor step list.
+# Mirror the restricted runtime recorder's action maps so the static editor preview only
+# surfaces calls that code blocks can execute through the capability proxy.
_METHOD_ACTION_TYPES: dict[str, str] = {
"goto": "goto_url",
"click": "click",
"dblclick": "click",
"check": "click",
"uncheck": "click",
- "tap": "click",
"fill": "input_text",
"type": "input_text",
"press": "keypress",
- "press_sequentially": "input_text",
"select_option": "select_option",
"set_input_files": "upload_file",
"hover": "hover",
"go_back": "go_back",
"go_forward": "go_forward",
"reload": "reload_page",
- "evaluate": "execute_js",
"wait_for_timeout": "wait",
}
# Awaited calls that are sync/no-op helpers — never surfaced as their own step.
@@ -146,8 +142,6 @@ def _describe(span: CodeActionSpan) -> str:
return "Go forward"
if span.action_type == "reload_page":
return "Reload the page"
- if span.action_type == "execute_js":
- return "Run a script"
return "Run a step"
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index fcaa8d3bd..41aeeba34 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -125,6 +125,7 @@ from skyvern.forge.sdk.workflow.models._jinja import (
)
from skyvern.forge.sdk.workflow.models.code_block_recorder import (
CODE_BLOCK_FILENAME,
+ CodeBlockSecret,
RecordingPage,
user_code_line_from_exception,
)
@@ -3675,8 +3676,10 @@ class CodeBlock(Block):
def is_safe_code(code: str) -> None:
tree = ast.parse(code)
for node in ast.walk(tree):
- # Block dunder attribute access (obj.__foo__)
- if hasattr(node, "attr") and str(node.attr).startswith("__"):
+ # Block all private attribute access (obj._foo, obj.__foo__, obj._Class__foo).
+ # This is intentionally wider than the old dunder-only guard so opaque handles
+ # and proxy internals cannot be reached through single-underscore helpers.
+ if hasattr(node, "attr") and str(node.attr).startswith("_"):
raise InsecureCodeDetected("Not allowed to access private methods or attributes")
# Block bare dunder identifiers (__capture_locals, __builtins__, etc.)
if isinstance(node, ast.Name) and node.id.startswith("__"):
@@ -4048,14 +4051,20 @@ async def wrapper({default_args}):
)
real_secret_value = secret_value if secret_value is not None else credential_place_holder
- parameter_values[credential_field] = real_secret_value
- real_secret_values[credential_field] = real_secret_value
+ runtime_secret_value = (
+ CodeBlockSecret(real_secret_value) if isinstance(real_secret_value, str) else real_secret_value
+ )
+ parameter_values[credential_field] = runtime_secret_value
+ real_secret_values[credential_field] = runtime_secret_value
credential_namespace = Credential(**real_secret_values)
credential_namespace.otp = _bind_code_block_otp(parameter.key, organization_id, workflow_run_id)
parameter_values[parameter.key] = credential_namespace
else:
secret_value = workflow_run_context.get_original_secret_value_or_none(value)
- parameter_values[parameter.key] = secret_value if secret_value is not None else value
+ real_secret_value = secret_value if secret_value is not None else value
+ parameter_values[parameter.key] = (
+ CodeBlockSecret(real_secret_value) if isinstance(real_secret_value, str) else real_secret_value
+ )
workflow_run_block = None
@@ -4080,9 +4089,8 @@ async def wrapper({default_args}):
if task is None:
return
# Every action that reaches this sink is one the recorder chose to surface on the timeline
- # (goto, click, input, page.evaluate, select, hover, ...), so each one earns a screenshot.
- # Re-listing eligible types here only drifts from the recorder's maps — which is exactly how
- # page.evaluate (EXECUTE_JS) ended up with no screenshot.
+ # (goto, click, input, select, hover, wait, ...), so each one earns a screenshot.
+ # Re-listing eligible types here only drifts from the recorder's maps.
# page.screenshot() shares the CDP channel with the user's page calls, so it must run synchronously
# in the user-await chain (a backgrounded capture races the next action and clips a mid-nav frame);
# only the page-free S3 upload is deferred off the critical path.
@@ -4187,7 +4195,7 @@ async def wrapper({default_args}):
settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS,
)
except InsecureCodeDetected as e:
- await _drain_screenshots()
+ await _persist_recorded_actions(recording_page._recorded_actions())
await _finalize_code_block_task(success=False)
return await self.build_block_result(
success=False,
@@ -4198,7 +4206,7 @@ async def wrapper({default_args}):
organization_id=organization_id,
)
except asyncio.TimeoutError:
- await _persist_recorded_actions(recording_page.recorded_actions())
+ await _persist_recorded_actions(recording_page._recorded_actions())
await _finalize_code_block_task(success=False)
return await self.build_block_result(
success=False,
@@ -4217,8 +4225,8 @@ async def wrapper({default_args}):
# User code can raise an exception carrying a resolved secret (e.g.
# `raise Exception(await cred.otp())`); mask before it reaches the persisted reason.
failure_reason = workflow_run_context.mask_secrets_in_data(exc.message)
- recorded = recording_page.recorded_actions()
- if recording_page.last_recorded_exception() is not e:
+ recorded = recording_page._recorded_actions()
+ if recording_page._last_recorded_exception() is not e:
# The exception did not come from a recorded page call; add a synthetic failure row.
recorded.append(
Action(
@@ -4242,7 +4250,7 @@ async def wrapper({default_args}):
)
else:
- await _persist_recorded_actions(recording_page.recorded_actions())
+ await _persist_recorded_actions(recording_page._recorded_actions())
await _finalize_code_block_task(success=True)
finally:
# Safety net for paths the except arms miss (CancelledError, update_workflow_run_block failure).
diff --git a/skyvern/forge/sdk/workflow/models/code_block_recorder.py b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
index a25db2e7f..922b1ddad 100644
--- a/skyvern/forge/sdk/workflow/models/code_block_recorder.py
+++ b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
@@ -1,12 +1,18 @@
from __future__ import annotations
+import inspect
+import ipaddress
import sys
import time
from types import FrameType
from typing import Any, Awaitable, Callable
+from urllib.parse import urlparse
import structlog
+from skyvern.config import settings
+from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetected
+from skyvern.utils.url_validators import is_blocked_host
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, ActionStatus
@@ -21,8 +27,11 @@ _PAGE_ACTION_MAP: dict[str, ActionType] = {
"go_back": ActionType.GO_BACK,
"go_forward": ActionType.GO_FORWARD,
"reload": ActionType.RELOAD_PAGE,
- "evaluate": ActionType.EXECUTE_JS,
+ "wait_for_timeout": ActionType.WAIT,
}
+# Playwright navigation APIs return raw Response objects with frame/page escape hatches.
+# Code blocks only need the action side effect, so suppress these returns after recording.
+_PAGE_ACTIONS_WITH_RAW_BROWSER_RESULTS = frozenset({"goto", "go_back", "go_forward", "reload"})
_LOCATOR_ACTION_MAP: dict[str, ActionType] = {
"click": ActionType.CLICK,
"dblclick": ActionType.CLICK,
@@ -48,9 +57,233 @@ _LOCATOR_FACTORY_METHODS = frozenset(
"filter",
}
)
+_SAFE_PAGE_METHODS = frozenset({"content", "title", "wait_for_load_state", "wait_for_url"})
+_SAFE_LOCATOR_METHODS = frozenset(
+ {
+ "all_inner_texts",
+ "all_text_contents",
+ "bounding_box",
+ "count",
+ "get_attribute",
+ "inner_html",
+ "inner_text",
+ "input_value",
+ "is_checked",
+ "is_disabled",
+ "is_editable",
+ "is_enabled",
+ "is_visible",
+ "text_content",
+ "wait_for",
+ }
+)
+_SAFE_DOWNLOAD_METHODS = frozenset({"failure", "path"})
+_SAFE_DOWNLOAD_PROPERTIES = frozenset({"suggested_filename", "url"})
+_DENIED_DOWNLOAD_ATTRIBUTES = frozenset({"page"})
OnAction = Callable[[Action], Awaitable[None]]
+_DENIED_PAGE_ATTRIBUTES = frozenset(
+ {
+ "add_init_script",
+ "context",
+ "evaluate",
+ "evaluate_handle",
+ "expose_binding",
+ "expose_function",
+ "frame",
+ "frame_locator",
+ "frames",
+ "main_frame",
+ "request",
+ "route",
+ }
+)
+
+
+class CodeBlockSecret:
+ __slots__ = ("__value",)
+ __hash__ = None # type: ignore[assignment]
+
+ def __init__(self, value: str) -> None:
+ self.__value = value
+
+ def __str__(self) -> str:
+ return "*****"
+
+ def __repr__(self) -> str:
+ return "*****"
+
+ def __bool__(self) -> bool:
+ return bool(self.__value)
+
+ def __eq__(self, other: object) -> bool:
+ # Always false: no comparison-based exfiltration path.
+ return False
+
+ def _reveal_for_input_action(self) -> str:
+ return self.__value
+
+
+def _reveal_code_block_secret(value: Any) -> Any:
+ if isinstance(value, CodeBlockSecret):
+ return value._reveal_for_input_action()
+ return value
+
+
+def _resolve_input_secrets(value: Any) -> Any:
+ if isinstance(value, tuple):
+ return tuple(_resolve_input_secrets(item) for item in value)
+ if isinstance(value, list):
+ return [_resolve_input_secrets(item) for item in value]
+ if isinstance(value, dict):
+ return {key: _resolve_input_secrets(item) for key, item in value.items()}
+ return _reveal_code_block_secret(value)
+
+
+def _contains_code_block_secret(value: Any) -> bool:
+ if isinstance(value, CodeBlockSecret):
+ return True
+ if isinstance(value, (tuple, list, set)):
+ return any(_contains_code_block_secret(item) for item in value)
+ if isinstance(value, dict):
+ return any(_contains_code_block_secret(item) for pair in value.items() for item in pair)
+ return False
+
+
+def _action_call_args(
+ action_type: ActionType, args: tuple[Any, ...], kwargs: dict[str, Any]
+) -> tuple[tuple[Any, ...], dict[str, Any]]:
+ if action_type != ActionType.INPUT_TEXT:
+ if _contains_code_block_secret(args) or _contains_code_block_secret(kwargs):
+ raise InsecureCodeDetected("CodeBlockSecret can only be used in fill/type (INPUT_TEXT) actions")
+ return args, kwargs
+ return _resolve_input_secrets(args), _resolve_input_secrets(kwargs)
+
+
+def _origin(url: str | None) -> tuple[str, str, int | None] | None:
+ if not url:
+ return None
+ parsed = urlparse(url)
+ if not parsed.scheme or not parsed.hostname:
+ return None
+ return (parsed.scheme.lower(), parsed.hostname.lower(), parsed.port)
+
+
+def _is_loopback_fixture_host(host: str) -> bool:
+ normalized = host[1:-1].lower() if host.startswith("[") and host.endswith("]") else host.lower()
+ if normalized == "localhost":
+ return True
+ try:
+ ip = ipaddress.ip_address(normalized)
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+ ip = ip.ipv4_mapped
+ except ValueError:
+ return False
+ return ip.is_loopback
+
+
+def _host_allowed_for_loopback_fixture(host: str) -> bool:
+ bare = host[1:-1] if host.startswith("[") and host.endswith("]") else host
+ candidate_forms = {host.lower(), bare.lower()}
+ try:
+ ip = ipaddress.ip_address(bare)
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+ ip = ip.ipv4_mapped
+ candidate_forms.add(str(ip).lower())
+ except ValueError:
+ pass
+ allowed_hosts = {allowed.lower() for allowed in settings.ALLOWED_HOSTS}
+ return bool(candidate_forms & allowed_hosts)
+
+
+def _parse_browser_numeric_host(host: str) -> ipaddress.IPv4Address | None:
+ """Parse legacy numeric IPv4 forms that browsers normalize but ipaddress rejects."""
+ if not host or ":" in host:
+ return None
+ parts = host.split(".")
+ if len(parts) > 4:
+ return None
+
+ parsed_parts: list[int] = []
+ for part in parts:
+ if not part:
+ return None
+ lowered = part.lower()
+ try:
+ if lowered.startswith("0x"):
+ value = int(lowered[2:], 16)
+ elif len(lowered) > 1 and lowered.startswith("0"):
+ if any(ch not in "01234567" for ch in lowered):
+ return None
+ value = int(lowered, 8)
+ elif lowered.isdigit():
+ value = int(lowered, 10)
+ else:
+ return None
+ except ValueError:
+ return None
+ parsed_parts.append(value)
+
+ if any(part > 255 for part in parsed_parts[:-1]):
+ return None
+ tail_max = (256 ** (5 - len(parsed_parts))) - 1
+ if parsed_parts[-1] > tail_max:
+ return None
+
+ address = 0
+ for parsed_part in parsed_parts[:-1]:
+ address = (address * 256) + parsed_part
+ address = (address * (256 ** (5 - len(parsed_parts)))) + parsed_parts[-1]
+ if address > 0xFFFFFFFF:
+ return None
+ return ipaddress.IPv4Address(address)
+
+
+def _parse_host_ip(host: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
+ bare = host[1:-1] if host.startswith("[") and host.endswith("]") else host
+ try:
+ ip = ipaddress.ip_address(bare)
+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
+ return ip.ipv4_mapped
+ return ip
+ except ValueError:
+ return _parse_browser_numeric_host(bare.lower())
+
+
+def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+ return ip.is_private or ip.is_link_local or ip.is_loopback or ip.is_multicast or ip.is_reserved
+
+
+def _validate_goto_target(target_url: str, current_url: str | None) -> None:
+ if not target_url.strip():
+ raise InsecureCodeDetected("Code block page.goto blocked empty target")
+ target = urlparse(target_url)
+ if not target.scheme and not target.netloc:
+ return
+ if target.scheme not in {"http", "https"}:
+ raise InsecureCodeDetected(f"Code block page.goto blocked non-http target: {target_url}")
+ if not target.hostname:
+ raise InsecureCodeDetected(f"Code block page.goto blocked target: {target_url}")
+
+ target_ip = _parse_host_ip(target.hostname)
+ if target_ip is not None and _is_blocked_ip(target_ip):
+ # ALLOWED_HOSTS is a loopback-only fixture escape hatch here; private/link-local
+ # non-loopback addresses stay blocked even when globally allowlisted.
+ if _is_loopback_fixture_host(target.hostname) and (
+ _origin(target_url) == _origin(current_url) or _host_allowed_for_loopback_fixture(target.hostname)
+ ):
+ return
+ raise InsecureCodeDetected(f"Code block page.goto blocked target: {target_url}")
+
+ if not is_blocked_host(target.hostname):
+ return
+ if _is_loopback_fixture_host(target.hostname) and (
+ _origin(target_url) == _origin(current_url) or _host_allowed_for_loopback_fixture(target.hostname)
+ ):
+ return
+ raise InsecureCodeDetected(f"Code block page.goto blocked target: {target_url}")
+
def _frame_user_line() -> int | None:
# Walk f_back instead of inspect.stack(), which reads source for every frame; this runs on
@@ -145,10 +378,23 @@ class RecordingLocator:
def first(self) -> RecordingLocator:
return RecordingLocator(self.__locator.first, self.__recorder, self.__selector)
+ @property
+ def last(self) -> RecordingLocator:
+ return RecordingLocator(self.__locator.last, self.__recorder, self.__selector)
+
def nth(self, index: int) -> RecordingLocator:
return RecordingLocator(self.__locator.nth(index), self.__recorder, self.__selector)
+ async def all(self) -> list[RecordingLocator]:
+ locators = self.__locator.all()
+ if inspect.isawaitable(locators):
+ locators = await locators
+ return [RecordingLocator(locator, self.__recorder, self.__selector) for locator in locators]
+
def __getattr__(self, name: str) -> Any:
+ action_type = _LOCATOR_ACTION_MAP.get(name)
+ if name not in _LOCATOR_FACTORY_METHODS and name not in _SAFE_LOCATOR_METHODS and action_type is None:
+ raise InsecureCodeDetected(f"Code block locator.{name} is not available in the restricted page API")
attr = getattr(self.__locator, name)
if name in _LOCATOR_FACTORY_METHODS and callable(attr):
@@ -156,27 +402,72 @@ class RecordingLocator:
return RecordingLocator(attr(*args, **kwargs), self.__recorder, _factory_selector(name, args))
return factory
- action_type = _LOCATOR_ACTION_MAP.get(name)
- if action_type is None or not callable(attr):
- return attr
+ if action_type is None:
+ if name in _SAFE_LOCATOR_METHODS and callable(attr):
+ return attr
+ raise InsecureCodeDetected(f"Code block locator.{name} is not available in the restricted page API")
+ if not callable(attr):
+ raise InsecureCodeDetected(f"Code block locator.{name} is not available in the restricted page API")
async def recorded(*args: Any, **kwargs: Any) -> Any:
+ call_args, call_kwargs = _action_call_args(action_type, args, kwargs)
return await self.__recorder.record(
- action_type, f"locator.{name}", self.__selector, lambda: attr(*args, **kwargs), args
+ action_type, f"locator.{name}", self.__selector, lambda: attr(*call_args, **call_kwargs), args
)
return recorded
+class RecordingDownload:
+ def __init__(self, download: Any) -> None:
+ self.__download = download
+
+ def __getattr__(self, name: str) -> Any:
+ if name in _DENIED_DOWNLOAD_ATTRIBUTES or name.startswith("_"):
+ raise InsecureCodeDetected(f"Code block download.{name} is not available in the restricted download API")
+ if name in _SAFE_DOWNLOAD_PROPERTIES:
+ return getattr(self.__download, name)
+ if name in _SAFE_DOWNLOAD_METHODS:
+ attr = getattr(self.__download, name)
+ if callable(attr):
+ return attr
+ raise InsecureCodeDetected(f"Code block download.{name} is not available in the restricted download API")
+
+
+class RecordingDownloadContext:
+ def __init__(self, context_manager: Any) -> None:
+ self.__context_manager = context_manager
+ self.__entered_context: Any = context_manager
+
+ async def __aenter__(self) -> RecordingDownloadContext:
+ entered = await self.__context_manager.__aenter__()
+ self.__entered_context = entered if entered is not None else self.__context_manager
+ return self
+
+ async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> Any:
+ return await self.__context_manager.__aexit__(exc_type, exc, traceback)
+
+ @property
+ def value(self) -> Awaitable[RecordingDownload]:
+ async def wrapped_value() -> RecordingDownload:
+ value = self.__entered_context.value
+ download = await value if inspect.isawaitable(value) else value
+ return RecordingDownload(download)
+
+ return wrapped_value()
+
+
class RecordingKeyboard:
def __init__(self, keyboard: Any, recorder: _Recorder) -> None:
self.__keyboard = keyboard
self.__recorder = recorder
def __getattr__(self, name: str) -> Any:
+ if name != "press":
+ raise InsecureCodeDetected(f"Code block keyboard.{name} is not available in the restricted page API")
attr = getattr(self.__keyboard, name)
- if name != "press" or not callable(attr):
- return attr
+ if not callable(attr):
+ raise InsecureCodeDetected(f"Code block keyboard.{name} is not available in the restricted page API")
async def recorded(*args: Any, **kwargs: Any) -> Any:
return await self.__recorder.record(
@@ -189,29 +480,40 @@ class RecordingKeyboard:
class RecordingPage:
"""Proxy around a playwright page that records mapped calls as Actions.
- Name-mangled private state keeps casual user code away from recorder
- internals, but the mangled `_RecordingPage__*` form is still reachable;
- treat recordings as telemetry, not a tamper-proof audit trail.
+ User code receives this restricted surface, while block execution keeps the
+ raw Playwright page for internal screenshot capture and persistence work.
"""
def __init__(self, page: Any, on_action: OnAction | None = None) -> None:
self.__page = page
self.__recorder = _Recorder(on_action)
- def recorded_actions(self) -> list[Action]:
+ def _recorded_actions(self) -> list[Action]:
return list(self.__recorder.actions)
- def last_recorded_exception(self) -> BaseException | None:
+ def _last_recorded_exception(self) -> BaseException | None:
return self.__recorder.last_exception
def locator(self, selector: str, **kwargs: Any) -> RecordingLocator:
return RecordingLocator(self.__page.locator(selector, **kwargs), self.__recorder, selector)
+ def expect_download(self, *args: Any, **kwargs: Any) -> RecordingDownloadContext:
+ return RecordingDownloadContext(self.__page.expect_download(*args, **kwargs))
+
@property
def keyboard(self) -> RecordingKeyboard:
return RecordingKeyboard(self.__page.keyboard, self.__recorder)
+ @property
+ def url(self) -> str:
+ return self.__page.url
+
def __getattr__(self, name: str) -> Any:
+ if name in _DENIED_PAGE_ATTRIBUTES:
+ raise InsecureCodeDetected(f"Code block page.{name} is not available in the restricted page API")
+ action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name)
+ if name not in _LOCATOR_FACTORY_METHODS and name not in _SAFE_PAGE_METHODS and action_type is None:
+ raise InsecureCodeDetected(f"Code block page.{name} is not available in the restricted page API")
attr = getattr(self.__page, name)
if name in _LOCATOR_FACTORY_METHODS and callable(attr):
@@ -221,11 +523,25 @@ class RecordingPage:
return factory
# Record direct page-level interactions (page.click/fill/press/...) with the same
# redaction as the locator path, alongside navigation calls.
- action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name)
- if action_type is None or not callable(attr):
- return attr
+ if action_type is None:
+ if name in _SAFE_PAGE_METHODS and callable(attr):
+ return attr
+ raise InsecureCodeDetected(f"Code block page.{name} is not available in the restricted page API")
+ if not callable(attr):
+ raise InsecureCodeDetected(f"Code block page.{name} is not available in the restricted page API")
async def recorded(*args: Any, **kwargs: Any) -> Any:
- return await self.__recorder.record(action_type, f"page.{name}", None, lambda: attr(*args, **kwargs), args)
+ if name == "goto":
+ target_url = args[0] if args else kwargs.get("url")
+ if not isinstance(target_url, str):
+ raise InsecureCodeDetected("Code block page.goto requires a string URL")
+ _validate_goto_target(target_url, self.__page.url)
+ call_args, call_kwargs = _action_call_args(action_type, args, kwargs)
+ result = await self.__recorder.record(
+ action_type, f"page.{name}", None, lambda: attr(*call_args, **call_kwargs), args
+ )
+ if name in _PAGE_ACTIONS_WITH_RAW_BROWSER_RESULTS:
+ return None
+ return result
return recorded
diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.py
index d12465cf0..eab1c4c40 100644
--- a/tests/unit/test_code_block_sandbox.py
+++ b/tests/unit/test_code_block_sandbox.py
@@ -883,6 +883,7 @@ async def wrapper({default_args}):
# RFC 6238 published test seed — safe to ship in OSS-synced fixtures.
_RFC_TOTP_SEED = "JBSWY3DPEHPK3PXP"
_CREDENTIAL_KEY = "login_credential"
+_SCALAR_SECRET_KEY = "api_secret"
_WORKFLOW_RUN_ID = "wr_otp_test"
_ORG_ID = "o_otp_test"
@@ -932,6 +933,48 @@ def _build_wrc_with_identifier(identifier: str = "[email protected]"):
return wrc
+def _build_wrc_with_password_credential():
+ from unittest.mock import MagicMock
+
+ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+
+ wrc = WorkflowRunContext(
+ workflow_title="t",
+ workflow_id="w",
+ workflow_permanent_id="wpid",
+ workflow_run_id=_WORKFLOW_RUN_ID,
+ aws_client=MagicMock(),
+ )
+ username_secret_id = wrc.generate_random_secret_id()
+ password_secret_id = wrc.generate_random_secret_id()
+ wrc.secrets[username_secret_id] = "demo_business_user"
+ wrc.secrets[password_secret_id] = "Demo!Pass123"
+ wrc.values[_CREDENTIAL_KEY] = {
+ "context": "placeholder note",
+ "username": username_secret_id,
+ "password": password_secret_id,
+ }
+ return wrc
+
+
+def _build_wrc_with_scalar_secret():
+ from unittest.mock import MagicMock
+
+ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+
+ wrc = WorkflowRunContext(
+ workflow_title="t",
+ workflow_id="w",
+ workflow_permanent_id="wpid",
+ workflow_run_id=_WORKFLOW_RUN_ID,
+ aws_client=MagicMock(),
+ )
+ secret_id = wrc.generate_random_secret_id()
+ wrc.secrets[secret_id] = "scalar-secret-token"
+ wrc.values[_SCALAR_SECRET_KEY] = secret_id
+ return wrc
+
+
class _FakeWorkflowRun:
def __init__(self) -> None:
self.workflow_id = "w"
@@ -1017,6 +1060,68 @@ async def _run_credential_code_block(monkeypatch: "pytest.MonkeyPatch", wrc, cod
return result, persisted.get("value")
+async def _run_scalar_secret_code_block(monkeypatch: "pytest.MonkeyPatch", wrc, code: str, label: str):
+ from unittest.mock import MagicMock
+
+ from skyvern.forge.sdk.workflow.models.parameter import AWSSecretParameter, OutputParameter, ParameterType
+
+ class FakeBrowserState:
+ def __init__(self) -> None:
+ self.browser_artifacts = BrowserArtifacts()
+
+ async def get_working_page(self) -> object:
+ return MagicMock()
+
+ async def validate_code_block(*args: object, **kwargs: object) -> None:
+ return None
+
+ async def get_browser_state(*args: object, **kwargs: object) -> FakeBrowserState:
+ return FakeBrowserState()
+
+ persisted: dict[str, object] = {}
+
+ async def record_output(self: object, ctx: object, run_id: object, value: object) -> None:
+ persisted["value"] = value
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.validate_code_block",
+ validate_code_block,
+ )
+ monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", get_browser_state)
+ monkeypatch.setattr(CodeBlock, "get_workflow_run_context", lambda self, run_id: wrc)
+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output)
+ _patch_context_resolution(monkeypatch, wrc)
+
+ now = datetime.now(timezone.utc)
+ output_parameter = OutputParameter(
+ parameter_type=ParameterType.OUTPUT,
+ key=f"{label}_output",
+ description="test output",
+ output_parameter_id=f"op_{label}",
+ workflow_id="w",
+ created_at=now,
+ modified_at=now,
+ )
+ secret_parameter = AWSSecretParameter(
+ parameter_type=ParameterType.AWS_SECRET,
+ key=_SCALAR_SECRET_KEY,
+ description="secret",
+ workflow_id="w",
+ aws_secret_parameter_id=f"sp_{label}",
+ aws_key="test/secret",
+ created_at=now,
+ modified_at=now,
+ )
+ block = CodeBlock(
+ label=label,
+ code=code,
+ output_parameter=output_parameter,
+ parameters=[secret_parameter],
+ )
+ result = await block.execute(workflow_run_id=_WORKFLOW_RUN_ID, workflow_run_block_id="")
+ return result, persisted.get("value")
+
+
class TestCodeBlockOtpMintAtCallTime:
"""AC1/AC4: the primitive re-mints the TOTP at call time, not at block start."""
@@ -1469,3 +1574,141 @@ class TestCodeBlockLegacyTotpRegression:
assert result.success is True
assert output_value == {"visible": "ok"}
assert _CREDENTIAL_KEY not in output_value
+
+
+class TestCodeBlockOpaqueCredentialFields:
+ @pytest.mark.asyncio
+ async def test_bound_credential_fields_are_not_plaintext_strings(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ wrc = _build_wrc_with_password_credential()
+
+ result, output_value = await _run_credential_code_block(
+ monkeypatch,
+ wrc,
+ code=(
+ f"password_is_plaintext = {_CREDENTIAL_KEY}.password == 'Demo!Pass123'\n"
+ f"username_is_plaintext = {_CREDENTIAL_KEY}.username == 'demo_business_user'"
+ ),
+ label="opaque_password",
+ )
+
+ assert result.success is True
+ assert output_value == {
+ "password_is_plaintext": False,
+ "username_is_plaintext": False,
+ }
+
+ @pytest.mark.asyncio
+ async def test_bound_credential_fields_stringify_to_mask(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ wrc = _build_wrc_with_password_credential()
+
+ result, output_value = await _run_credential_code_block(
+ monkeypatch,
+ wrc,
+ code=(
+ f"password_text = str({_CREDENTIAL_KEY}.password)\n"
+ f"username_text = str({_CREDENTIAL_KEY}.username)\n"
+ f"bare_password_text = str(password)"
+ ),
+ label="opaque_str",
+ )
+
+ assert result.success is True
+ assert output_value == {
+ "password_text": "*****",
+ "username_text": "*****",
+ "bare_password_text": "*****",
+ }
+ assert "Demo!Pass123" not in json.dumps(output_value)
+ assert "demo_business_user" not in json.dumps(output_value)
+
+ @pytest.mark.asyncio
+ async def test_bound_credential_fields_cannot_be_concatenated_to_exfiltrate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ wrc = _build_wrc_with_password_credential()
+
+ result, _ = await _run_credential_code_block(
+ monkeypatch,
+ wrc,
+ code=f"leak = {_CREDENTIAL_KEY}.password + '-suffix'",
+ label="opaque_concat",
+ )
+
+ assert result.success is False
+ assert result.failure_reason is not None
+ assert "Demo!Pass123" not in result.failure_reason
+
+ def test_user_code_cannot_reach_secret_handle_private_slot(self) -> None:
+ with pytest.raises(InsecureCodeDetected, match="private"):
+ CodeBlock.is_safe_code("credential.password._CodeBlockSecret__value")
+
+ def test_user_code_cannot_reach_secret_handle_protected_reveal(self) -> None:
+ with pytest.raises(InsecureCodeDetected, match="private"):
+ CodeBlock.is_safe_code("credential.password._reveal_for_input_action()")
+
+ def test_user_code_cannot_reach_recording_page_protected_hooks(self) -> None:
+ with pytest.raises(InsecureCodeDetected, match="private"):
+ CodeBlock.is_safe_code("page._recorded_actions()")
+ with pytest.raises(InsecureCodeDetected, match="private"):
+ CodeBlock.is_safe_code("page._last_recorded_exception()")
+
+ def test_public_attribute_with_double_underscore_is_not_treated_as_private(self) -> None:
+ CodeBlock.is_safe_code("value = record.public__field")
+
+ @pytest.mark.asyncio
+ async def test_bound_credential_fields_have_no_public_plaintext_reveal_method(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ wrc = _build_wrc_with_password_credential()
+
+ result, _ = await _run_credential_code_block(
+ monkeypatch,
+ wrc,
+ code=f"leak = {_CREDENTIAL_KEY}.password.reveal_for_input_action()",
+ label="opaque_public_reveal",
+ )
+
+ assert result.success is False
+ assert result.failure_reason is not None
+ assert "reveal_for_input_action" in result.failure_reason
+ assert "Demo!Pass123" not in result.failure_reason
+
+
+class TestCodeBlockOpaqueScalarSecrets:
+ @pytest.mark.asyncio
+ async def test_scalar_secret_parameter_is_not_plaintext_string(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ wrc = _build_wrc_with_scalar_secret()
+
+ result, output_value = await _run_scalar_secret_code_block(
+ monkeypatch,
+ wrc,
+ code=(
+ f"secret_is_plaintext = {_SCALAR_SECRET_KEY} == 'scalar-secret-token'\n"
+ f"secret_text = str({_SCALAR_SECRET_KEY})"
+ ),
+ label="opaque_scalar",
+ )
+
+ assert result.success is True
+ assert output_value == {
+ "secret_is_plaintext": False,
+ "secret_text": "*****",
+ }
+ assert "scalar-secret-token" not in json.dumps(output_value)
+
+ @pytest.mark.asyncio
+ async def test_scalar_secret_parameter_cannot_be_concatenated_to_exfiltrate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ wrc = _build_wrc_with_scalar_secret()
+
+ result, _ = await _run_scalar_secret_code_block(
+ monkeypatch,
+ wrc,
+ code=f"leak = {_SCALAR_SECRET_KEY} + '-suffix'",
+ label="opaque_scalar_concat",
+ )
+
+ assert result.success is False
+ assert result.failure_reason is not None
+ assert "scalar-secret-token" not in result.failure_reason
diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.py
index 30c925710..2b865389f 100644
--- a/tests/unit/test_code_block_steps.py
+++ b/tests/unit/test_code_block_steps.py
@@ -31,10 +31,9 @@ def test_analyze_maps_playwright_calls_to_action_types_with_line_ranges():
]
-def test_analyze_maps_page_evaluate_and_other_recorder_calls_to_action_types():
- # The static editor preview must surface the same calls the runtime recorder
- # records (code_block_recorder._PAGE_ACTION_MAP / _LOCATOR_ACTION_MAP), so the
- # editor step count matches the timeline. page.evaluate was previously dropped.
+def test_analyze_skips_page_evaluate_and_maps_supported_recorder_calls_to_action_types():
+ # The runtime capability proxy rejects page.evaluate, so the static editor preview
+ # should not present it as an executable generated step.
code = (
"async def run(page):\n"
" await page.goto('https://example.com/')\n"
@@ -45,22 +44,30 @@ def test_analyze_maps_page_evaluate_and_other_recorder_calls_to_action_types():
spans = analyze_code_actions(code)
assert [(s.action_type, s.line_start) for s in spans] == [
("goto_url", 2),
- ("execute_js", 3),
("hover", 4),
("go_forward", 5),
]
-def test_derive_steps_surfaces_page_evaluate_with_a_label():
+def test_derive_steps_skips_page_evaluate():
code = (
"async def run(page):\n"
" await page.goto('https://example.com/')\n"
" await page.evaluate('() => document.title')\n"
)
steps = derive_code_block_steps(code)
- # Step count must match the number of actions actually in the script (2, not 1).
- assert [s["action_type"] for s in steps] == ["goto_url", "execute_js"]
- assert steps[1]["description"] # surfaced with a non-empty, human label, not dropped
+ assert [s["action_type"] for s in steps] == ["goto_url"]
+
+
+def test_analyze_skips_methods_denied_by_runtime_proxy():
+ code = (
+ "async def run(page):\n"
+ " await page.locator('#x').tap()\n"
+ " await page.locator('#x').press_sequentially('abc')\n"
+ " await page.locator('#x').click()\n"
+ )
+ spans = analyze_code_actions(code)
+ assert [(s.method, s.action_type, s.line_start) for s in spans] == [("click", "click", 4)]
def test_analyze_skips_noise_and_returns_empty_on_syntax_error():
diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.py
index 61ddf2809..1b0386bfb 100644
--- a/tests/unit/workflow/test_code_block_recorder.py
+++ b/tests/unit/workflow/test_code_block_recorder.py
@@ -5,19 +5,24 @@ from __future__ import annotations
import asyncio
import json
from datetime import datetime, timezone
+from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.models import StepStatus
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetected
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.code_block_recorder import (
CODE_BLOCK_FILENAME,
CODE_LINE_OFFSET,
+ CodeBlockSecret,
+ RecordingLocator,
RecordingPage,
user_code_line_from_exception,
)
@@ -31,6 +36,7 @@ from skyvern.webeye.browser_artifacts import BrowserArtifacts
class FakeLocator:
def __init__(self) -> None:
self.calls: list[str] = []
+ self.page = SimpleNamespace(url="https://raw-playwright-page.invalid")
def locator(self, selector): # noqa: ANN001, ANN201
return self
@@ -42,6 +48,13 @@ class FakeLocator:
def first(self): # noqa: ANN201
return self
+ @property
+ def last(self): # noqa: ANN201
+ return self
+
+ def nth(self, index): # noqa: ANN001, ANN201
+ return self
+
async def click(self, **kwargs): # noqa: ANN003, ANN201
self.calls.append("click")
@@ -60,24 +73,100 @@ class FakeLocator:
def filter(self, **kwargs): # noqa: ANN003, ANN201
return self
+ async def inner_text(self, **kwargs): # noqa: ANN003, ANN201
+ return "ready"
+
+ async def inner_html(self, **kwargs): # noqa: ANN003, ANN201
+ return "<span>ready</span>"
+
+ async def count(self, **kwargs): # noqa: ANN003, ANN201
+ return 1
+
+ async def all(self): # noqa: ANN201
+ return [self, FakeLocator()]
+
+ async def evaluate(self, expression, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN201
+ return "raw-js"
+
+ async def evaluate_handle(self, expression, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN201
+ return "raw-handle"
+
class FakeKeyboard:
async def press(self, key, **kwargs): # noqa: ANN001, ANN003, ANN201
return None
+ async def type(self, text, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+
+class FakeDownload:
+ def __init__(self, page) -> None: # noqa: ANN001
+ self.page = page
+ self.suggested_filename = "report.pdf"
+ self.url = "https://example.com/report.pdf"
+
+ async def path(self): # noqa: ANN201
+ return "/tmp/downloads/report.pdf"
+
+ async def failure(self): # noqa: ANN201
+ return None
+
+ async def save_as(self, path): # noqa: ANN001, ANN201
+ return None
+
+
+class FakeDownloadContext:
+ def __init__(self, download: FakeDownload) -> None:
+ self._download = download
+
+ async def __aenter__(self) -> FakeDownloadContext:
+ return self
+
+ async def __aexit__(self, exc_type, exc, traceback): # noqa: ANN001, ANN201
+ return None
+
+ @property
+ def value(self): # noqa: ANN201
+ async def _value():
+ return self._download
+
+ return _value()
+
class FakePage:
def __init__(self) -> None:
self.inner = FakeLocator()
self.keyboard = FakeKeyboard()
+ self.download = FakeDownload(self)
+ self.request = SimpleNamespace(post=AsyncMock(return_value=None))
+ self.context = SimpleNamespace(storage_state=AsyncMock(return_value={}))
+ self.main_frame = SimpleNamespace(evaluate=AsyncMock(return_value=None))
+ self.frames = [self.main_frame]
self.url = "about:blank"
async def goto(self, url, **kwargs): # noqa: ANN001, ANN003, ANN201
- return None
+ self.url = url
+ return SimpleNamespace(frame=SimpleNamespace(page=self))
+
+ async def go_back(self, **kwargs): # noqa: ANN003, ANN201
+ return SimpleNamespace(frame=SimpleNamespace(page=self))
+
+ async def go_forward(self, **kwargs): # noqa: ANN003, ANN201
+ return SimpleNamespace(frame=SimpleNamespace(page=self))
+
+ async def reload(self, **kwargs): # noqa: ANN003, ANN201
+ return SimpleNamespace(frame=SimpleNamespace(page=self))
async def wait_for_load_state(self, state="load", **kwargs): # noqa: ANN001, ANN003, ANN201
return None
+ async def title(self): # noqa: ANN201
+ return "Example"
+
+ async def content(self): # noqa: ANN201
+ return "<html><body>Example</body></html>"
+
def locator(self, selector): # noqa: ANN001, ANN201
return self.inner
@@ -96,6 +185,21 @@ class FakePage:
async def evaluate(self, expression, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN201
return None
+ async def evaluate_handle(self, expression, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN201
+ return None
+
+ async def route(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ return None
+
+ async def add_init_script(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ return None
+
+ async def wait_for_timeout(self, timeout, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+ def expect_download(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ return FakeDownloadContext(self.download)
+
@pytest.mark.asyncio
async def test_records_goto_click_fill_with_types_and_order() -> None:
@@ -103,7 +207,7 @@ async def test_records_goto_click_fill_with_types_and_order() -> None:
await page.goto("https://example.com")
await page.locator("#q").fill("hello")
await page.locator("#go").click()
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert [a.action_type for a in recorded] == [
ActionType.GOTO_URL,
ActionType.INPUT_TEXT,
@@ -115,13 +219,300 @@ async def test_records_goto_click_fill_with_types_and_order() -> None:
@pytest.mark.asyncio
-async def test_page_evaluate_records_execute_js_action() -> None:
+async def test_page_evaluate_is_denied_by_capability_proxy() -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="page.evaluate"):
+ await page.evaluate("() => document.title")
+ assert page._recorded_actions() == []
+
+
+ ("name", "action_type"),
+ [
+ ("goto", ActionType.GOTO_URL),
+ ("go_back", ActionType.GO_BACK),
+ ("go_forward", ActionType.GO_FORWARD),
+ ("reload", ActionType.RELOAD_PAGE),
+ ],
+)
+async def test_navigation_actions_do_not_return_raw_playwright_response(name: str, action_type: ActionType) -> None:
+ fake = FakePage()
+ page = RecordingPage(fake)
+ method = getattr(page, name)
+ result = await method("https://example.com") if name == "goto" else await method()
+
+ assert result is None
+ assert [a.action_type for a in page._recorded_actions()] == [action_type]
+
+
+ "name",
+ [
+ "request",
+ "context",
+ "route",
+ "add_init_script",
+ "evaluate_handle",
+ "main_frame",
+ "frames",
+ "frame",
+ "frame_locator",
+ ],
+)
+def test_unsafe_page_capabilities_are_denied(name: str) -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match=f"page.{name}"):
+ getattr(page, name)
+
+
+ "name",
+ [
+ "wait_for_selector",
+ "wait_for_function",
+ ],
+)
+def test_raw_page_handle_reads_are_denied(name: str) -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match=f"page.{name}"):
+ getattr(page, name)
+
+
+async def test_safe_page_read_methods_pass_through_without_recording() -> None:
+ page = RecordingPage(FakePage())
+
+ assert await page.title() == "Example"
+ assert await page.content() == "<html><body>Example</body></html>"
+ assert page._recorded_actions() == []
+
+
+async def test_locator_evaluate_capabilities_are_denied() -> None:
+ page = RecordingPage(FakePage())
+ locator = page.locator("body")
+
+ with pytest.raises(InsecureCodeDetected, match="locator.evaluate"):
+ await locator.evaluate("node => node.textContent")
+ with pytest.raises(InsecureCodeDetected, match="locator.evaluate_handle"):
+ await locator.evaluate_handle("node => node")
+ with pytest.raises(InsecureCodeDetected, match="locator.page"):
+ locator.page
+
+ assert page._recorded_actions() == []
+
+
+async def test_safe_locator_read_methods_pass_through_without_recording() -> None:
+ page = RecordingPage(FakePage())
+
+ assert await page.locator("#status").inner_text() == "ready"
+ assert await page.locator("#status").inner_html() == "<span>ready</span>"
+ assert await page.locator("#status").count() == 1
+ assert page._recorded_actions() == []
+
+
+async def test_locator_all_returns_wrapped_locators() -> None:
+ page = RecordingPage(FakePage())
+
+ locators = await page.locator(".row").all()
+
+ assert len(locators) == 2
+ assert all(isinstance(locator, RecordingLocator) for locator in locators)
+ with pytest.raises(InsecureCodeDetected, match="locator.page"):
+ locators[0].page
+ await locators[0].click()
+ assert [a.action_type for a in page._recorded_actions()] == [ActionType.CLICK]
+
+
+async def test_expect_download_returns_wrapped_download_handle() -> None:
+ page = RecordingPage(FakePage())
+
+ async with page.expect_download() as download_info:
+ await page.locator("#download").click()
+ download = await download_info.value
+
+ assert download.suggested_filename == "report.pdf"
+ assert download.url == "https://example.com/report.pdf"
+ assert await download.path() == "/tmp/downloads/report.pdf"
+ assert await download.failure() is None
+ with pytest.raises(InsecureCodeDetected, match="download.page"):
+ download.page
+ with pytest.raises(InsecureCodeDetected, match="download.save_as"):
+ download.save_as
+ with pytest.raises(InsecureCodeDetected, match=r"download\._impl_obj"):
+ download._impl_obj
+ assert [a.action_type for a in page._recorded_actions()] == [ActionType.CLICK]
+
+
+async def test_generated_download_idiom_executes_with_restricted_page() -> None:
+ code = """
+async with page.expect_download() as download_info:
+ await page.locator("#download").click()
+downloaded_file = await download_info.value
+download_path = await downloaded_file.path()
+"""
+ user_function = _make_code_block(code).generate_async_user_function(code, RecordingPage(FakePage()))
+
+ result = await user_function()
+
+ assert result["download_path"] == "/tmp/downloads/report.pdf"
+
+
+async def test_unknown_page_locator_and_keyboard_apis_are_denied() -> None:
+ page = RecordingPage(FakePage())
+
+ with pytest.raises(InsecureCodeDetected, match="page.screenshot"):
+ await page.screenshot()
+ with pytest.raises(InsecureCodeDetected, match="locator.screenshot"):
+ await page.locator("#status").screenshot()
+ with pytest.raises(InsecureCodeDetected, match="keyboard.type"):
+ await page.keyboard.type("secret")
+
+
+async def test_page_goto_blocks_link_local_metadata_target() -> None:
page = RecordingPage(FakePage())
- await page.evaluate("() => document.title")
- recorded = page.recorded_actions()
- assert [a.action_type for a in recorded] == [ActionType.EXECUTE_JS]
- assert recorded[0].description == "page.evaluate () => document.title"
- assert recorded[0].status == ActionStatus.completed
+ with pytest.raises(InsecureCodeDetected, match="169.254.169.254"):
+ await page.goto("http://169.254.169.254/latest/meta-data/")
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_blocks_same_origin_link_local_target() -> None:
+ fake = FakePage()
+ fake.url = "http://169.254.169.254/latest/"
+ page = RecordingPage(fake)
+ with pytest.raises(InsecureCodeDetected, match="169.254.169.254"):
+ await page.goto("http://169.254.169.254/latest/meta-data/")
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_blocks_same_origin_private_target() -> None:
+ fake = FakePage()
+ fake.url = "http://10.0.0.1/admin/"
+ page = RecordingPage(fake)
+ with pytest.raises(InsecureCodeDetected, match="10.0.0.1"):
+ await page.goto("http://10.0.0.1/admin/status")
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_blocks_multicast_target() -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="224.0.0.1"):
+ await page.goto("http://224.0.0.1/status")
+ assert page._recorded_actions() == []
+
+
+ "url",
+ [
+ "http://2130706433/",
+ "http://0x7f000001/",
+ "http://0177.0.0.1/",
+ "http://2852039166/latest/meta-data/",
+ ],
+)
+async def test_page_goto_blocks_browser_normalized_internal_ip_forms(url: str) -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="blocked target"):
+ await page.goto(url)
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_blocks_private_target_even_when_globally_allowlisted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", ["10.0.0.1"])
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="10.0.0.1"):
+ await page.goto("http://10.0.0.1/admin/status")
+ assert page._recorded_actions() == []
+
+
[email protected]("url", ["http:///etc/passwd", "https:///169.254.169.254/latest/meta-data/"])
+async def test_page_goto_blocks_malformed_absolute_targets_without_hostname(url: str) -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="blocked target"):
+ await page.goto(url)
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_blocks_scheme_relative_targets() -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="blocked non-http target"):
+ await page.goto("//evil.internal/path")
+ assert page._recorded_actions() == []
+
+
[email protected]("target", ["", " "])
+async def test_page_goto_blocks_empty_targets(target: str) -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="blocked empty target"):
+ await page.goto(target)
+ assert page._recorded_actions() == []
+
+
[email protected]("target", [2130706433, CodeBlockSecret("https://example.com")])
+async def test_page_goto_blocks_non_string_targets(target: object) -> None:
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="page.goto requires a string URL"):
+ await page.goto(target)
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_blocks_first_hop_localhost_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", [])
+ page = RecordingPage(FakePage())
+ with pytest.raises(InsecureCodeDetected, match="localhost:8900"):
+ await page.goto("http://localhost:8900/demo/app/")
+ assert page._recorded_actions() == []
+
+
+async def test_page_goto_allows_first_hop_localhost_when_explicitly_allowed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(settings, "ALLOWED_HOSTS", ["localhost"])
+ fake = FakePage()
+ page = RecordingPage(fake)
+ await page.goto("http://localhost:8900/demo/app/")
+ assert fake.url == "http://localhost:8900/demo/app/"
+ assert [a.action_type for a in page._recorded_actions()] == [ActionType.GOTO_URL]
+
+
+async def test_page_goto_allows_same_origin_localhost_fixture() -> None:
+ fake = FakePage()
+ fake.url = "http://localhost:8900/demo/app/"
+ page = RecordingPage(fake)
+ await page.goto("http://localhost:8900/demo/app/settings")
+ assert fake.url == "http://localhost:8900/demo/app/settings"
+ assert [a.action_type for a in page._recorded_actions()] == [ActionType.GOTO_URL]
+
+
+async def test_page_goto_allows_same_origin_loopback_fixture() -> None:
+ fake = FakePage()
+ fake.url = "http://127.0.0.1:8900/demo/app/"
+ page = RecordingPage(fake)
+ await page.goto("http://127.0.0.1:8900/demo/app/settings")
+ assert fake.url == "http://127.0.0.1:8900/demo/app/settings"
+ assert [a.action_type for a in page._recorded_actions()] == [ActionType.GOTO_URL]
@pytest.mark.asyncio
@@ -130,14 +521,23 @@ async def test_unmapped_calls_and_attributes_pass_through_unrecorded() -> None:
page = RecordingPage(fake)
await page.wait_for_load_state("networkidle")
assert page.url == "about:blank"
- assert page.recorded_actions() == []
+ assert page._recorded_actions() == []
+
+
+async def test_wait_for_timeout_records_wait_action() -> None:
+ page = RecordingPage(FakePage())
+ await page.wait_for_timeout(500)
+ recorded = page._recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.WAIT]
+ assert "500" in (recorded[0].description or "")
@pytest.mark.asyncio
async def test_keyboard_press_records_keypress_action() -> None:
page = RecordingPage(FakePage())
await page.keyboard.press("Enter")
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert [a.action_type for a in recorded] == [ActionType.KEYPRESS]
assert recorded[0].description and "Enter" in recorded[0].description
@@ -146,7 +546,7 @@ async def test_keyboard_press_records_keypress_action() -> None:
async def test_get_by_role_click_is_recorded() -> None:
page = RecordingPage(FakePage())
await page.get_by_role("button", name="Go").click()
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert [a.action_type for a in recorded] == [ActionType.CLICK]
assert recorded[0].description == "locator.click get_by_role(button)"
@@ -155,7 +555,7 @@ async def test_get_by_role_click_is_recorded() -> None:
async def test_locator_get_by_chain_is_recorded() -> None:
page = RecordingPage(FakePage())
await page.locator("#form").get_by_text("Submit").click()
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert [a.action_type for a in recorded] == [ActionType.CLICK]
assert recorded[0].description == "locator.click get_by_text(Submit)"
@@ -165,7 +565,7 @@ async def test_direct_page_actions_are_recorded_with_redaction() -> None:
page = RecordingPage(FakePage())
await page.click("#submit")
await page.fill("#email", "[email protected]")
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert [a.action_type for a in recorded] == [ActionType.CLICK, ActionType.INPUT_TEXT]
# Input values may be credentials; the fill value must never reach the description.
assert all("[email protected]" not in (a.description or "") for a in recorded)
@@ -175,7 +575,7 @@ async def test_direct_page_actions_are_recorded_with_redaction() -> None:
async def test_filter_locator_chain_click_is_recorded() -> None:
page = RecordingPage(FakePage())
await page.get_by_role("button", name="Go").filter(has_text="Submit").click()
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert [a.action_type for a in recorded] == [ActionType.CLICK]
@@ -190,7 +590,7 @@ async def test_failed_call_records_failed_action_and_reraises() -> None:
page = RecordingPage(fake)
with pytest.raises(RuntimeError):
await page.locator("#x").click()
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
assert recorded[-1].action_type == ActionType.CLICK
assert recorded[-1].status == ActionStatus.failed
assert "element detached" in (recorded[-1].response or "")
@@ -208,7 +608,7 @@ async def test_on_action_sink_receives_each_action_and_errors_are_swallowed() ->
await page.goto("https://example.com")
await page.locator("#go").click()
assert seen == [ActionType.GOTO_URL, ActionType.CLICK]
- assert len(page.recorded_actions()) == 2
+ assert len(page._recorded_actions()) == 2
def test_user_code_line_from_exception_unwraps_wrapper_offset() -> None:
@@ -236,7 +636,7 @@ async def test_input_values_are_elided_from_descriptions() -> None:
page = RecordingPage(FakePage())
await page.locator("#pw").fill("hunter2-credential")
await page.locator("#user").type("alice-credential")
- recorded = page.recorded_actions()
+ recorded = page._recorded_actions()
dumped = json.dumps([a.model_dump(mode="json") for a in recorded])
assert "hunter2-credential" not in dumped
assert "alice-credential" not in dumped
@@ -244,6 +644,31 @@ async def test_input_values_are_elided_from_descriptions() -> None:
assert recorded[1].description == "locator.type #user"
+async def test_secret_handle_resolves_only_inside_input_actions() -> None:
+ fake = FakePage()
+ page = RecordingPage(fake)
+ secret = CodeBlockSecret("hunter2-credential")
+
+ await page.locator("#pw").fill(secret)
+
+ assert fake.inner.calls == ["fill:hunter2-credential"]
+ dumped = json.dumps([a.model_dump(mode="json") for a in page._recorded_actions()])
+ assert "hunter2-credential" not in dumped
+ assert str(secret) == "*****"
+
+
+async def test_secret_handle_in_unsupported_action_fails_clearly() -> None:
+ page = RecordingPage(FakePage())
+ secret = CodeBlockSecret("gold-plan")
+
+ with pytest.raises(InsecureCodeDetected, match="CodeBlockSecret can only be used"):
+ await page.locator("#plan").select_option(secret)
+
+ assert page._recorded_actions() == []
+
+
def _make_code_block(code: str, goal: str | None = None) -> CodeBlock:
now = datetime.now(timezone.utc)
output_parameter = OutputParameter(
@@ -543,9 +968,7 @@ async def test_backgrounded_screenshots_are_drained_and_linked_before_persist(
@pytest.mark.asyncio
-async def test_page_evaluate_action_captures_and_links_screenshot(monkeypatch: pytest.MonkeyPatch) -> None:
- """page.evaluate (EXECUTE_JS) is a recorded, timeline-visible action and must get a screenshot
- like clicks and navigations do, so the run detail panel can render it instead of "No screenshot"."""
+async def test_page_evaluate_is_denied_before_action_persistence(monkeypatch: pytest.MonkeyPatch) -> None:
page = FakePage()
context = FakeWorkflowRunContext()
mocks = _patch_execute_environment(monkeypatch, page, context)
@@ -553,11 +976,33 @@ async def test_page_evaluate_action_captures_and_links_screenshot(monkeypatch: p
block = _make_code_block("await page.evaluate('() => document.title')", goal="go")
result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
- assert result.success is True
+ assert result.success is False
+ assert result.failure_reason is not None
+ assert "page" in result.failure_reason and "evaluate" in result.failure_reason
+ assert _created_actions(mocks) == []
+ assert mocks["create_artifact"].await_count == 0
+
+
+async def test_runtime_denial_after_recorded_action_persists_prior_actions(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block(
+ "await page.goto('https://example.com')\nawait page.evaluate('() => document.title')", goal="go"
+ )
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is False
+ assert result.failure_reason is not None
+ assert "page" in result.failure_reason and "evaluate" in result.failure_reason
actions = _created_actions(mocks)
- assert [a.action_type for a in actions] == [ActionType.EXECUTE_JS]
- assert mocks["create_artifact"].await_count == 1
+ assert [a.action_type for a in actions] == [ActionType.GOTO_URL]
assert actions[0].screenshot_artifact_id == "artifact_1"
+ assert mocks["create_artifact"].await_count == 1
@pytest.mark.asyncio