Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11463): surface extraction/AI steps in code-block run timeline (#6857)
Aaron Perezdc7f200a6f8f
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 469549246..4c37dcb53 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -145,6 +145,8 @@ class SkyvernPage(Page):) -> Any:return await skyvern_page._decorate_call(fn, action, *args, **kwargs)+ # Tag the wrapper so the recorder/deriver mirrors can be regression-tested against this set.+ wrapper.__skyvern_action_type__ = action # type: ignore[attr-defined]return wrapperreturn decoratordiff --git a/skyvern/forge/sdk/workflow/models/code_block_recorder.py b/skyvern/forge/sdk/workflow/models/code_block_recorder.pyindex a25db2e7f..3ff448655 100644--- a/skyvern/forge/sdk/workflow/models/code_block_recorder.py+++ b/skyvern/forge/sdk/workflow/models/code_block_recorder.py@@ -48,6 +48,32 @@ _LOCATOR_FACTORY_METHODS = frozenset("filter",})+# SkyvernPage high-level API (page.extract / page.complete / ...). These are not raw+# Playwright calls, so they fall through the maps above and used to execute unrecorded —+# a navigate+extract block then rendered as only repeated "Goto URL" on the timeline.+# Mirror skyvern_page.py's @action_wrap table and the editor deriver+# (code_block_steps._METHOD_ACTION_TYPES) so extraction and the rest of the surface+# record as distinct, reader-facing steps.+_HIGH_LEVEL_ACTION_MAP: dict[str, ActionType] = {+ "extract": ActionType.EXTRACT,+ "complete": ActionType.COMPLETE,+ "terminate": ActionType.TERMINATE,+ "wait": ActionType.WAIT,+ "reload_page": ActionType.RELOAD_PAGE,+ "scroll": ActionType.SCROLL,+ "keypress": ActionType.KEYPRESS,+ "move": ActionType.MOVE,+ "drag": ActionType.DRAG,+ "left_mouse": ActionType.LEFT_MOUSE,+ "download_file": ActionType.DOWNLOAD_FILE,+ "solve_captcha": ActionType.SOLVE_CAPTCHA,+ "verification_code": ActionType.VERIFICATION_CODE,+ "upload_file": ActionType.UPLOAD_FILE,+ "fill_autocomplete": ActionType.INPUT_TEXT,+}+# High-level methods whose natural-language `prompt` (positional or keyword) is the+# reader-facing description; mirrors code_block_steps._PROMPT_POSITIONAL_METHODS.+_PROMPT_METHODS: frozenset[str] = frozenset({"extract", "complete", "solve_captcha", "verification_code"})OnAction = Callable[[Action], Awaitable[None]]@@ -101,6 +127,7 @@ class _Recorder:target: str | None,call: Callable[[], Awaitable[Any]],args: tuple[Any, ...],+ description: str | None = None,) -> Any:started = time.monotonic()# Input values may be credentials (incl. derived TOTP codes); never describe them.@@ -109,7 +136,10 @@ class _Recorder:action_type=action_type,status=ActionStatus.completed,action_order=len(self.actions),- description=_describe(name, target, describe_args),+ # A reader-facing prompt (page.extract/complete) is the action's own copy; prefer it over+ # the "page.method arg" form so the timeline reads as plain language even when the editor's+ # derived step is missing or stale and the UI falls back to this description.+ description=description if description is not None else _describe(name, target, describe_args),output={"code_line": _frame_user_line()},)try:@@ -219,13 +249,21 @@ class RecordingPage:return RecordingLocator(attr(*args, **kwargs), self.__recorder, _factory_selector(name, args))return factory- # Record direct page-level interactions (page.click/fill/press/...) with the same- # redaction as the locator path, alongside navigation calls.- action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name)+ # Record direct page-level interactions (page.click/fill/press/...) and the high-level+ # SkyvernPage API (page.extract/complete/...) with the same redaction as the locator path.+ action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name) or _HIGH_LEVEL_ACTION_MAP.get(name)if action_type is None or not callable(attr):return attr+ record_prompt = name in _PROMPT_METHODSasync def recorded(*args: Any, **kwargs: Any) -> Any:- return await self.__recorder.record(action_type, f"page.{name}", None, lambda: attr(*args, **kwargs), args)+ description: str | None = None+ if record_prompt:+ prompt = kwargs.get("prompt", args[0] if args else None)+ if isinstance(prompt, str) and prompt.strip():+ description = " ".join(prompt.split())[:200]+ return await self.__recorder.record(+ action_type, f"page.{name}", None, lambda: attr(*args, **kwargs), args, description=description+ )return recordeddiff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.pyindex 61ddf2809..387a2c028 100644--- a/tests/unit/workflow/test_code_block_recorder.py+++ b/tests/unit/workflow/test_code_block_recorder.py@@ -9,13 +9,18 @@ from unittest.mock import AsyncMockimport pytest+from skyvern.core.script_generations.skyvern_page import SkyvernPagefrom skyvern.forge import appfrom skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.copilot.code_block_steps import _METHOD_ACTION_TYPESfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.forge.sdk.workflow.models.code_block_recorder import (+ _HIGH_LEVEL_ACTION_MAP,+ _LOCATOR_ACTION_MAP,+ _PAGE_ACTION_MAP,CODE_BLOCK_FILENAME,CODE_LINE_OFFSET,RecordingPage,@@ -96,6 +101,15 @@ class FakePage:async def evaluate(self, expression, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN201return None+ async def extract(self, prompt, **kwargs): # noqa: ANN001, ANN003, ANN201+ return {"data": "extracted"}++ async def complete(self, prompt=None, **kwargs): # noqa: ANN001, ANN003, ANN201+ return None++ async def scroll(self, **kwargs): # noqa: ANN003, ANN201+ return None+@pytest.mark.asyncioasync def test_records_goto_click_fill_with_types_and_order() -> None:@@ -124,6 +138,69 @@ async def test_page_evaluate_records_execute_js_action() -> None:assert recorded[0].status == ActionStatus.completed+async def test_extract_high_level_call_records_extract_action_with_prompt() -> None:+ """page.extract() must surface on the run timeline as an EXTRACT action carrying its+ prompt, so a navigate+extract code block doesn't render as only repeated 'Goto URL'."""+ page = RecordingPage(FakePage())+ await page.goto("https://news.ycombinator.com/")+ await page.extract(prompt="Extract the URLs of the top 20 posts")+ recorded = page.recorded_actions()+ assert [a.action_type for a in recorded] == [ActionType.GOTO_URL, ActionType.EXTRACT]+ # The reader-facing prompt is the description verbatim (no "page.extract" prefix), so a+ # timeline row reads as plain language with or without a matching editor step.+ assert recorded[1].description == "Extract the URLs of the top 20 posts"+++async def test_extract_positional_prompt_is_recorded_in_description() -> None:+ """The prompt is positional-or-keyword on extract; a positional prompt is still surfaced."""+ page = RecordingPage(FakePage())+ await page.extract("Extract the top visible comment")+ recorded = page.recorded_actions()+ assert recorded[0].action_type == ActionType.EXTRACT+ assert recorded[0].description == "Extract the top visible comment"+++async def test_other_high_level_skyvern_page_calls_are_recorded() -> None:+ """High-level SkyvernPage methods without a prompt still record their action type."""+ page = RecordingPage(FakePage())+ await page.scroll()+ await page.complete()+ recorded = page.recorded_actions()+ assert [a.action_type for a in recorded] == [ActionType.SCROLL, ActionType.COMPLETE]+++def test_recorder_maps_cover_every_action_wrapped_skyvern_page_method() -> None:+ """The recorder and editor-deriver maps are hand-maintained mirrors of SkyvernPage's+ @action_wrap set. A high-level method added there but absent here would execute+ unrecorded -- the exact SKY-11463 regression -- so assert every @action_wrap method+ is mapped (to the same action_type) on both surfaces, or is an explicit no-op exclusion."""+ live = {}+ for name in dir(SkyvernPage):+ action_type = getattr(getattr(SkyvernPage, name), "__skyvern_action_type__", None)+ if action_type is not None:+ live[name] = action_type+ # Guard against a vacuous pass if introspection ever stops finding the decorated surface.+ assert {"extract", "click", "complete", "scroll"} <= live.keys()++ recorder = {**_PAGE_ACTION_MAP, **_LOCATOR_ACTION_MAP, **_HIGH_LEVEL_ACTION_MAP}+ excluded = {"null_action"} # NULL_ACTION is a no-op probe, never a timeline step++ for name, action_type in live.items():+ if name in excluded:+ continue+ assert recorder.get(name) == action_type, (+ f"SkyvernPage.{name} is @action_wrap({action_type}) but RecordingPage maps it to "+ f"{recorder.get(name)!r}; add it to code_block_recorder or it executes unrecorded"+ )+ assert _METHOD_ACTION_TYPES.get(name) == action_type.value, (+ f"SkyvernPage.{name} ({action_type}) is missing/mismatched in "+ f"code_block_steps._METHOD_ACTION_TYPES; the editor step preview will drift from the timeline"+ )++@pytest.mark.asyncioasync def test_unmapped_calls_and_attributes_pass_through_unrecorded() -> None:fake = FakePage()