Skip to content

Automate browser based workflows with AI

fix(SKY-11463): surface extraction/AI steps in code-block run timeline (#6857)

Aaron Perezdc7f200a6f8f
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index 469549246..4c37dcb53 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -145,6 +145,8 @@ class SkyvernPage(Page):
) -> Any:
return await skyvern_page._decorate_call(fn, action, *args, **kwargs)
+ # Tag the wrapper so the recorder/deriver mirrors can be regression-tested against this set.
+ wrapper.__skyvern_action_type__ = action # type: ignore[attr-defined]
return wrapper
return decorator
diff --git a/skyvern/forge/sdk/workflow/models/code_block_recorder.py b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
index a25db2e7f..3ff448655 100644
--- a/skyvern/forge/sdk/workflow/models/code_block_recorder.py
+++ b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
@@ -48,6 +48,32 @@ _LOCATOR_FACTORY_METHODS = frozenset(
"filter",
}
)
+# SkyvernPage high-level API (page.extract / page.complete / ...). These are not raw
+# Playwright calls, so they fall through the maps above and used to execute unrecorded —
+# a navigate+extract block then rendered as only repeated "Goto URL" on the timeline.
+# Mirror skyvern_page.py's @action_wrap table and the editor deriver
+# (code_block_steps._METHOD_ACTION_TYPES) so extraction and the rest of the surface
+# record as distinct, reader-facing steps.
+_HIGH_LEVEL_ACTION_MAP: dict[str, ActionType] = {
+ "extract": ActionType.EXTRACT,
+ "complete": ActionType.COMPLETE,
+ "terminate": ActionType.TERMINATE,
+ "wait": ActionType.WAIT,
+ "reload_page": ActionType.RELOAD_PAGE,
+ "scroll": ActionType.SCROLL,
+ "keypress": ActionType.KEYPRESS,
+ "move": ActionType.MOVE,
+ "drag": ActionType.DRAG,
+ "left_mouse": ActionType.LEFT_MOUSE,
+ "download_file": ActionType.DOWNLOAD_FILE,
+ "solve_captcha": ActionType.SOLVE_CAPTCHA,
+ "verification_code": ActionType.VERIFICATION_CODE,
+ "upload_file": ActionType.UPLOAD_FILE,
+ "fill_autocomplete": ActionType.INPUT_TEXT,
+}
+# High-level methods whose natural-language `prompt` (positional or keyword) is the
+# reader-facing description; mirrors code_block_steps._PROMPT_POSITIONAL_METHODS.
+_PROMPT_METHODS: frozenset[str] = frozenset({"extract", "complete", "solve_captcha", "verification_code"})
OnAction = Callable[[Action], Awaitable[None]]
@@ -101,6 +127,7 @@ class _Recorder:
target: str | None,
call: Callable[[], Awaitable[Any]],
args: tuple[Any, ...],
+ description: str | None = None,
) -> Any:
started = time.monotonic()
# Input values may be credentials (incl. derived TOTP codes); never describe them.
@@ -109,7 +136,10 @@ class _Recorder:
action_type=action_type,
status=ActionStatus.completed,
action_order=len(self.actions),
- description=_describe(name, target, describe_args),
+ # A reader-facing prompt (page.extract/complete) is the action's own copy; prefer it over
+ # the "page.method arg" form so the timeline reads as plain language even when the editor's
+ # derived step is missing or stale and the UI falls back to this description.
+ description=description if description is not None else _describe(name, target, describe_args),
output={"code_line": _frame_user_line()},
)
try:
@@ -219,13 +249,21 @@ class RecordingPage:
return RecordingLocator(attr(*args, **kwargs), self.__recorder, _factory_selector(name, args))
return factory
- # Record direct page-level interactions (page.click/fill/press/...) with the same
- # redaction as the locator path, alongside navigation calls.
- action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name)
+ # Record direct page-level interactions (page.click/fill/press/...) and the high-level
+ # SkyvernPage API (page.extract/complete/...) with the same redaction as the locator path.
+ action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name) or _HIGH_LEVEL_ACTION_MAP.get(name)
if action_type is None or not callable(attr):
return attr
+ record_prompt = name in _PROMPT_METHODS
async def recorded(*args: Any, **kwargs: Any) -> Any:
- return await self.__recorder.record(action_type, f"page.{name}", None, lambda: attr(*args, **kwargs), args)
+ description: str | None = None
+ if record_prompt:
+ prompt = kwargs.get("prompt", args[0] if args else None)
+ if isinstance(prompt, str) and prompt.strip():
+ description = " ".join(prompt.split())[:200]
+ return await self.__recorder.record(
+ action_type, f"page.{name}", None, lambda: attr(*args, **kwargs), args, description=description
+ )
return recorded
diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.py
index 61ddf2809..387a2c028 100644
--- a/tests/unit/workflow/test_code_block_recorder.py
+++ b/tests/unit/workflow/test_code_block_recorder.py
@@ -9,13 +9,18 @@ from unittest.mock import AsyncMock
import pytest
+from skyvern.core.script_generations.skyvern_page import SkyvernPage
from skyvern.forge import app
from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.copilot.code_block_steps import _METHOD_ACTION_TYPES
from skyvern.forge.sdk.models import StepStatus
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.code_block_recorder import (
+ _HIGH_LEVEL_ACTION_MAP,
+ _LOCATOR_ACTION_MAP,
+ _PAGE_ACTION_MAP,
CODE_BLOCK_FILENAME,
CODE_LINE_OFFSET,
RecordingPage,
@@ -96,6 +101,15 @@ class FakePage:
async def evaluate(self, expression, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN201
return None
+ async def extract(self, prompt, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return {"data": "extracted"}
+
+ async def complete(self, prompt=None, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+ async def scroll(self, **kwargs): # noqa: ANN003, ANN201
+ return None
+
@pytest.mark.asyncio
async def test_records_goto_click_fill_with_types_and_order() -> None:
@@ -124,6 +138,69 @@ async def test_page_evaluate_records_execute_js_action() -> None:
assert recorded[0].status == ActionStatus.completed
+async def test_extract_high_level_call_records_extract_action_with_prompt() -> None:
+ """page.extract() must surface on the run timeline as an EXTRACT action carrying its
+ prompt, so a navigate+extract code block doesn't render as only repeated 'Goto URL'."""
+ page = RecordingPage(FakePage())
+ await page.goto("https://news.ycombinator.com/")
+ await page.extract(prompt="Extract the URLs of the top 20 posts")
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.GOTO_URL, ActionType.EXTRACT]
+ # The reader-facing prompt is the description verbatim (no "page.extract" prefix), so a
+ # timeline row reads as plain language with or without a matching editor step.
+ assert recorded[1].description == "Extract the URLs of the top 20 posts"
+
+
+async def test_extract_positional_prompt_is_recorded_in_description() -> None:
+ """The prompt is positional-or-keyword on extract; a positional prompt is still surfaced."""
+ page = RecordingPage(FakePage())
+ await page.extract("Extract the top visible comment")
+ recorded = page.recorded_actions()
+ assert recorded[0].action_type == ActionType.EXTRACT
+ assert recorded[0].description == "Extract the top visible comment"
+
+
+async def test_other_high_level_skyvern_page_calls_are_recorded() -> None:
+ """High-level SkyvernPage methods without a prompt still record their action type."""
+ page = RecordingPage(FakePage())
+ await page.scroll()
+ await page.complete()
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.SCROLL, ActionType.COMPLETE]
+
+
+def test_recorder_maps_cover_every_action_wrapped_skyvern_page_method() -> None:
+ """The recorder and editor-deriver maps are hand-maintained mirrors of SkyvernPage's
+ @action_wrap set. A high-level method added there but absent here would execute
+ unrecorded -- the exact SKY-11463 regression -- so assert every @action_wrap method
+ is mapped (to the same action_type) on both surfaces, or is an explicit no-op exclusion."""
+ live = {}
+ for name in dir(SkyvernPage):
+ action_type = getattr(getattr(SkyvernPage, name), "__skyvern_action_type__", None)
+ if action_type is not None:
+ live[name] = action_type
+ # Guard against a vacuous pass if introspection ever stops finding the decorated surface.
+ assert {"extract", "click", "complete", "scroll"} <= live.keys()
+
+ recorder = {**_PAGE_ACTION_MAP, **_LOCATOR_ACTION_MAP, **_HIGH_LEVEL_ACTION_MAP}
+ excluded = {"null_action"} # NULL_ACTION is a no-op probe, never a timeline step
+
+ for name, action_type in live.items():
+ if name in excluded:
+ continue
+ assert recorder.get(name) == action_type, (
+ f"SkyvernPage.{name} is @action_wrap({action_type}) but RecordingPage maps it to "
+ f"{recorder.get(name)!r}; add it to code_block_recorder or it executes unrecorded"
+ )
+ assert _METHOD_ACTION_TYPES.get(name) == action_type.value, (
+ f"SkyvernPage.{name} ({action_type}) is missing/mismatched in "
+ f"code_block_steps._METHOD_ACTION_TYPES; the editor step preview will drift from the timeline"
+ )
+
+
@pytest.mark.asyncio
async def test_unmapped_calls_and_attributes_pass_through_unrecorded() -> None:
fake = FakePage()