Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SDK: support actions skeleton (#3817)
Stanislav Novosad33ad4cfcd1d3
diff --git a/fern/docs.yml b/fern/docs.ymlindex 3e608bb87..e18637045 100644--- a/fern/docs.yml+++ b/fern/docs.yml@@ -204,6 +204,10 @@ navigation:- POST /v1/scripts/{script_id}/deploy- GET /v1/scripts/{script_id}- GET /v1/scripts+ - section: SDK+ hidden: true+ contents:+ - POST /v1/sdk/run_action- tab: blog- tab: community- tab: githubdiff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex fee372730..b02ea9077 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -2248,6 +2248,101 @@}}}+ },+ "/v1/sdk/run_action": {+ "post": {+ "tags": [+ "SDK"+ ],+ "summary": "Run an SDK action",+ "description": "Execute a single SDK action with the specified parameters",+ "operationId": "run_sdk_action_v1_sdk_run_action_post",+ "parameters": [+ {+ "name": "x-api-key",+ "in": "header",+ "required": false,+ "schema": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings.",+ "title": "X-Api-Key"+ },+ "description": "Skyvern API key for authentication. API key can be found at https://app.skyvern.com/settings."+ },+ {+ "name": "x-user-agent",+ "in": "header",+ "required": false,+ "schema": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "X-User-Agent"+ }+ }+ ],+ "requestBody": {+ "required": true,+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunSdkActionRequest"+ }+ }+ }+ },+ "responses": {+ "200": {+ "description": "Successfully executed SDK action",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/RunSdkActionResponse"+ }+ }+ }+ },+ "403": {+ "description": "Unauthorized - Invalid or missing authentication"+ },+ "400": {+ "description": "Invalid operation"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/HTTPValidationError"+ }+ }+ }+ }+ },+ "x-fern-sdk-method-name": "run_sdk_action",+ "x-fern-examples": [+ {+ "code-samples": [+ {+ "sdk": "python",+ "code": "from skyvern import Skyvern\nfrom skyvern.forge.sdk.schemas.sdk_actions import ClickAction\n\nskyvern = Skyvern(api_key=\"YOUR_API_KEY\")\nresponse = await skyvern.run_sdk_action(\n url=\"https://example.com\",\n browser_session_id=\"pbs_123\",\n action=ClickAction(\n selector=\"button\",\n intention=\"Click the submit button\"\n )\n)\nprint(response.workflow_run_id)\n"+ }+ ]+ }+ ]+ }}},"components": {@@ -9298,6 +9393,353 @@],"title": "RunStatus"},+ "ClickAction": {+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "ai_click",+ "title": "Type",+ "default": "ai_click"+ },+ "selector": {+ "type": "string",+ "title": "Selector",+ "description": "CSS selector for the element",+ "default": ""+ },+ "intention": {+ "type": "string",+ "title": "Intention",+ "description": "The intention or goal of the click",+ "default": ""+ },+ "data": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data",+ "description": "Additional context data"+ },+ "timeout": {+ "type": "number",+ "title": "Timeout",+ "description": "Timeout in milliseconds",+ "default": 30000+ }+ },+ "required": ["type"],+ "title": "ClickAction"+ },+ "InputTextAction": {+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "ai_input_text",+ "title": "Type",+ "default": "ai_input_text"+ },+ "selector": {+ "type": "string",+ "title": "Selector",+ "description": "CSS selector for the element",+ "default": ""+ },+ "value": {+ "type": "string",+ "title": "Value",+ "description": "Value to input",+ "default": ""+ },+ "intention": {+ "type": "string",+ "title": "Intention",+ "description": "The intention or goal of the input",+ "default": ""+ },+ "data": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data",+ "description": "Additional context data"+ },+ "totp_identifier": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Totp Identifier",+ "description": "TOTP identifier for input_text actions"+ },+ "totp_url": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Totp Url",+ "description": "TOTP URL for input_text actions"+ },+ "timeout": {+ "type": "number",+ "title": "Timeout",+ "description": "Timeout in milliseconds",+ "default": 30000+ }+ },+ "required": ["type"],+ "title": "InputTextAction"+ },+ "SelectOptionAction": {+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "ai_select_option",+ "title": "Type",+ "default": "ai_select_option"+ },+ "selector": {+ "type": "string",+ "title": "Selector",+ "description": "CSS selector for the element",+ "default": ""+ },+ "value": {+ "type": "string",+ "title": "Value",+ "description": "Value to select",+ "default": ""+ },+ "intention": {+ "type": "string",+ "title": "Intention",+ "description": "The intention or goal of the selection",+ "default": ""+ },+ "data": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data",+ "description": "Additional context data"+ },+ "timeout": {+ "type": "number",+ "title": "Timeout",+ "description": "Timeout in milliseconds",+ "default": 30000+ }+ },+ "required": ["type"],+ "title": "SelectOptionAction"+ },+ "ExtractAction": {+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "extract",+ "title": "Type",+ "default": "extract"+ },+ "prompt": {+ "type": "string",+ "title": "Prompt",+ "description": "Extraction prompt",+ "default": ""+ },+ "extract_schema": {+ "anyOf": [+ {+ "type": "object"+ },+ {+ "type": "array"+ },+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Extract Schema",+ "description": "Schema for extraction"+ },+ "error_code_mapping": {+ "anyOf": [+ {+ "type": "object",+ "additionalProperties": {+ "type": "string"+ }+ },+ {+ "type": "null"+ }+ ],+ "title": "Error Code Mapping",+ "description": "Error code mapping for extraction"+ },+ "intention": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Intention",+ "description": "The intention or goal of the extraction"+ },+ "data": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "object"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data",+ "description": "Additional context data"+ }+ },+ "required": ["type"],+ "title": "ExtractAction"+ },+ "SdkAction": {+ "oneOf": [+ {+ "$ref": "#/components/schemas/ClickAction"+ },+ {+ "$ref": "#/components/schemas/InputTextAction"+ },+ {+ "$ref": "#/components/schemas/SelectOptionAction"+ },+ {+ "$ref": "#/components/schemas/ExtractAction"+ }+ ],+ "discriminator": {+ "propertyName": "type",+ "mapping": {+ "ai_click": "#/components/schemas/ClickAction",+ "ai_input_text": "#/components/schemas/InputTextAction",+ "ai_select_option": "#/components/schemas/SelectOptionAction",+ "extract": "#/components/schemas/ExtractAction"+ }+ },+ "title": "SdkAction"+ },+ "RunSdkActionRequest": {+ "type": "object",+ "properties": {+ "url": {+ "type": "string",+ "title": "Url",+ "description": "The URL where the action should be executed"+ },+ "browser_session_id": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Browser Session Id",+ "description": "The browser session ID"+ },+ "browser_address": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Browser Address",+ "description": "The browser address"+ },+ "workflow_run_id": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Workflow Run Id",+ "description": "Optional workflow run ID to continue an existing workflow run"+ },+ "action": {+ "$ref": "#/components/schemas/SdkAction",+ "description": "The action to execute with its specific parameters"+ }+ },+ "required": ["url", "action"],+ "title": "RunSdkActionRequest"+ },+ "RunSdkActionResponse": {+ "type": "object",+ "properties": {+ "workflow_run_id": {+ "type": "string",+ "title": "Workflow Run Id",+ "description": "The workflow run ID used for this action"+ },+ "result": {+ "title": "Result",+ "description": "The result from the action (e.g., selector, value, extracted data)"+ }+ },+ "required": ["workflow_run_id"],+ "title": "RunSdkActionResponse"+ },"Script": {"properties": {"script_revision_id": {diff --git a/skyvern/client/__init__.py b/skyvern/client/__init__.pyindex d94ef537e..8048b0950 100644--- a/skyvern/client/__init__.py+++ b/skyvern/client/__init__.py@@ -40,6 +40,8 @@ if typing.TYPE_CHECKING:BitwardenSensitiveInformationParameterYaml,BlockType,BrowserSessionResponse,+ ClickAction,+ ClickActionData,CodeBlock,CodeBlockParametersItem,CodeBlockParametersItem_AwsSecret,@@ -79,6 +81,9 @@ if typing.TYPE_CHECKING:CreditCardCredentialResponse,DownloadToS3Block,DownloadToS3BlockYaml,+ ExtractAction,+ ExtractActionData,+ ExtractActionExtractSchema,ExtractionBlock,ExtractionBlockDataSchema,ExtractionBlockParametersItem,@@ -215,6 +220,8 @@ if typing.TYPE_CHECKING:HumanInteractionBlockParametersItem_Workflow,HumanInteractionBlockYaml,InputOrSelectContext,+ InputTextAction,+ InputTextActionData,LoginBlock,LoginBlockDataSchema,LoginBlockParametersItem,@@ -257,11 +264,19 @@ if typing.TYPE_CHECKING:PdfParserBlockYaml,ProxyLocation,RunEngine,+ RunSdkActionResponse,RunStatus,Script,ScriptFileCreate,ScriptRunResponse,+ SdkAction,+ SdkAction_AiClick,+ SdkAction_AiInputText,+ SdkAction_AiSelectOption,+ SdkAction_Extract,SelectOption,+ SelectOptionAction,+ SelectOptionActionData,SendEmailBlock,SendEmailBlockYaml,SkyvernForgeSdkSchemasCredentialsCredentialType,@@ -485,6 +500,8 @@ _dynamic_imports: typing.Dict[str, str] = {"BitwardenSensitiveInformationParameterYaml": ".types","BlockType": ".types","BrowserSessionResponse": ".types",+ "ClickAction": ".types",+ "ClickActionData": ".types","CodeBlock": ".types","CodeBlockParametersItem": ".types","CodeBlockParametersItem_AwsSecret": ".types",@@ -524,6 +541,9 @@ _dynamic_imports: typing.Dict[str, str] = {"CreditCardCredentialResponse": ".types","DownloadToS3Block": ".types","DownloadToS3BlockYaml": ".types",+ "ExtractAction": ".types",+ "ExtractActionData": ".types",+ "ExtractActionExtractSchema": ".types","ExtractionBlock": ".types","ExtractionBlockDataSchema": ".types","ExtractionBlockParametersItem": ".types",@@ -661,6 +681,8 @@ _dynamic_imports: typing.Dict[str, str] = {"HumanInteractionBlockParametersItem_Workflow": ".types","HumanInteractionBlockYaml": ".types","InputOrSelectContext": ".types",+ "InputTextAction": ".types",+ "InputTextActionData": ".types","LoginBlock": ".types","LoginBlockDataSchema": ".types","LoginBlockParametersItem": ".types",@@ -704,11 +726,19 @@ _dynamic_imports: typing.Dict[str, str] = {"PdfParserBlockYaml": ".types","ProxyLocation": ".types","RunEngine": ".types",+ "RunSdkActionResponse": ".types","RunStatus": ".types","Script": ".types","ScriptFileCreate": ".types","ScriptRunResponse": ".types",+ "SdkAction": ".types",+ "SdkAction_AiClick": ".types",+ "SdkAction_AiInputText": ".types",+ "SdkAction_AiSelectOption": ".types",+ "SdkAction_Extract": ".types","SelectOption": ".types",+ "SelectOptionAction": ".types",+ "SelectOptionActionData": ".types","SendEmailBlock": ".types","SendEmailBlockYaml": ".types","Skyvern": ".client",@@ -955,6 +985,8 @@ __all__ = ["BitwardenSensitiveInformationParameterYaml","BlockType","BrowserSessionResponse",+ "ClickAction",+ "ClickActionData","CodeBlock","CodeBlockParametersItem","CodeBlockParametersItem_AwsSecret",@@ -994,6 +1026,9 @@ __all__ = ["CreditCardCredentialResponse","DownloadToS3Block","DownloadToS3BlockYaml",+ "ExtractAction",+ "ExtractActionData",+ "ExtractActionExtractSchema","ExtractionBlock","ExtractionBlockDataSchema","ExtractionBlockParametersItem",@@ -1131,6 +1166,8 @@ __all__ = ["HumanInteractionBlockParametersItem_Workflow","HumanInteractionBlockYaml","InputOrSelectContext",+ "InputTextAction",+ "InputTextActionData","LoginBlock","LoginBlockDataSchema","LoginBlockParametersItem",@@ -1174,11 +1211,19 @@ __all__ = ["PdfParserBlockYaml","ProxyLocation","RunEngine",+ "RunSdkActionResponse","RunStatus","Script","ScriptFileCreate","ScriptRunResponse",+ "SdkAction",+ "SdkAction_AiClick",+ "SdkAction_AiInputText",+ "SdkAction_AiSelectOption",+ "SdkAction_Extract","SelectOption",+ "SelectOptionAction",+ "SelectOptionActionData","SendEmailBlock","SendEmailBlockYaml","Skyvern",diff --git a/skyvern/client/client.py b/skyvern/client/client.pyindex cfc95d853..a8307f4b2 100644--- a/skyvern/client/client.py+++ b/skyvern/client/client.py@@ -19,8 +19,10 @@ from .types.credential_response import CredentialResponsefrom .types.get_run_response import GetRunResponsefrom .types.proxy_location import ProxyLocationfrom .types.run_engine import RunEngine+from .types.run_sdk_action_response import RunSdkActionResponsefrom .types.script import Scriptfrom .types.script_file_create import ScriptFileCreate+from .types.sdk_action import SdkActionfrom .types.skyvern_forge_sdk_schemas_credentials_credential_type import SkyvernForgeSdkSchemasCredentialsCredentialTypefrom .types.skyvern_schemas_run_blocks_credential_type import SkyvernSchemasRunBlocksCredentialTypefrom .types.task_run_request_data_extraction_schema import TaskRunRequestDataExtractionSchema@@ -1494,6 +1496,71 @@ class Skyvern:_response = self._raw_client.deploy_script(script_id, files=files, request_options=request_options)return _response.data+ def run_sdk_action(+ self,+ *,+ url: str,+ action: SdkAction,+ user_agent: typing.Optional[str] = None,+ browser_session_id: typing.Optional[str] = OMIT,+ browser_address: typing.Optional[str] = OMIT,+ workflow_run_id: typing.Optional[str] = OMIT,+ request_options: typing.Optional[RequestOptions] = None,+ ) -> RunSdkActionResponse:+ """+ Execute a single SDK action with the specified parameters++ Parameters+ ----------+ url : str+ The URL where the action should be executed++ action : SdkAction+ The action to execute with its specific parameters++ user_agent : typing.Optional[str]++ browser_session_id : typing.Optional[str]+ The browser session ID++ browser_address : typing.Optional[str]+ The browser address++ workflow_run_id : typing.Optional[str]+ Optional workflow run ID to continue an existing workflow run++ request_options : typing.Optional[RequestOptions]+ Request-specific configuration.++ Returns+ -------+ RunSdkActionResponse+ Successfully executed SDK action++ Examples+ --------+ from skyvern import SdkAction_AiClick, Skyvern++ client = Skyvern(+ api_key="YOUR_API_KEY",+ )+ client.run_sdk_action(+ user_agent="x-user-agent",+ url="url",+ action=SdkAction_AiClick(),+ )+ """+ _response = self._raw_client.run_sdk_action(+ url=url,+ action=action,+ user_agent=user_agent,+ browser_session_id=browser_session_id,+ browser_address=browser_address,+ workflow_run_id=workflow_run_id,+ request_options=request_options,+ )+ return _response.data+@propertydef scripts(self):if self._scripts is None:@@ -3174,6 +3241,79 @@ class AsyncSkyvern:_response = await self._raw_client.deploy_script(script_id, files=files, request_options=request_options)return _response.data+ async def run_sdk_action(+ self,+ *,+ url: str,+ action: SdkAction,+ user_agent: typing.Optional[str] = None,+ browser_session_id: typing.Optional[str] = OMIT,+ browser_address: typing.Optional[str] = OMIT,+ workflow_run_id: typing.Optional[str] = OMIT,+ request_options: typing.Optional[RequestOptions] = None,+ ) -> RunSdkActionResponse:+ """+ Execute a single SDK action with the specified parameters++ Parameters+ ----------+ url : str+ The URL where the action should be executed++ action : SdkAction+ The action to execute with its specific parameters++ user_agent : typing.Optional[str]++ browser_session_id : typing.Optional[str]+ The browser session ID++ browser_address : typing.Optional[str]+ The browser address++ workflow_run_id : typing.Optional[str]+ Optional workflow run ID to continue an existing workflow run++ request_options : typing.Optional[RequestOptions]+ Request-specific configuration.++ Returns+ -------+ RunSdkActionResponse+ Successfully executed SDK action++ Examples+ --------+ import asyncio++ from skyvern import AsyncSkyvern, SdkAction_AiClick++ client = AsyncSkyvern(+ api_key="YOUR_API_KEY",+ )+++ async def main() -> None:+ await client.run_sdk_action(+ user_agent="x-user-agent",+ url="url",+ action=SdkAction_AiClick(),+ )+++ asyncio.run(main())+ """+ _response = await self._raw_client.run_sdk_action(+ url=url,+ action=action,+ user_agent=user_agent,+ browser_session_id=browser_session_id,+ browser_address=browser_address,+ workflow_run_id=workflow_run_id,+ request_options=request_options,+ )+ return _response.data+@propertydef scripts(self):if self._scripts is None:diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.pyindex ea71b6214..feb8b1ea0 100644--- a/skyvern/client/raw_client.py+++ b/skyvern/client/raw_client.py@@ -24,8 +24,10 @@ from .types.credential_response import CredentialResponsefrom .types.get_run_response import GetRunResponsefrom .types.proxy_location import ProxyLocationfrom .types.run_engine import RunEngine+from .types.run_sdk_action_response import RunSdkActionResponsefrom .types.script import Scriptfrom .types.script_file_create import ScriptFileCreate+from .types.sdk_action import SdkActionfrom .types.skyvern_forge_sdk_schemas_credentials_credential_type import SkyvernForgeSdkSchemasCredentialsCredentialTypefrom .types.skyvern_schemas_run_blocks_credential_type import SkyvernSchemasRunBlocksCredentialTypefrom .types.task_run_request_data_extraction_schema import TaskRunRequestDataExtractionSchema@@ -2052,6 +2054,114 @@ class RawSkyvern:raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)+ def run_sdk_action(+ self,+ *,+ url: str,+ action: SdkAction,+ user_agent: typing.Optional[str] = None,+ browser_session_id: typing.Optional[str] = OMIT,+ browser_address: typing.Optional[str] = OMIT,+ workflow_run_id: typing.Optional[str] = OMIT,+ request_options: typing.Optional[RequestOptions] = None,+ ) -> HttpResponse[RunSdkActionResponse]:+ """+ Execute a single SDK action with the specified parameters++ Parameters+ ----------+ url : str+ The URL where the action should be executed++ action : SdkAction+ The action to execute with its specific parameters++ user_agent : typing.Optional[str]++ browser_session_id : typing.Optional[str]+ The browser session ID++ browser_address : typing.Optional[str]+ The browser address++ workflow_run_id : typing.Optional[str]+ Optional workflow run ID to continue an existing workflow run++ request_options : typing.Optional[RequestOptions]+ Request-specific configuration.++ Returns+ -------+ HttpResponse[RunSdkActionResponse]+ Successfully executed SDK action+ """+ _response = self._client_wrapper.httpx_client.request(+ "v1/sdk/run_action",+ method="POST",+ json={+ "url": url,+ "browser_session_id": browser_session_id,+ "browser_address": browser_address,+ "workflow_run_id": workflow_run_id,+ "action": convert_and_respect_annotation_metadata(+ object_=action, annotation=SdkAction, direction="write"+ ),+ },+ headers={+ "content-type": "application/json",+ "x-user-agent": str(user_agent) if user_agent is not None else None,+ },+ request_options=request_options,+ omit=OMIT,+ )+ try:+ if 200 <= _response.status_code < 300:+ _data = typing.cast(+ RunSdkActionResponse,+ parse_obj_as(+ type_=RunSdkActionResponse, # type: ignore+ object_=_response.json(),+ ),+ )+ return HttpResponse(response=_response, data=_data)+ if _response.status_code == 400:+ raise BadRequestError(+ headers=dict(_response.headers),+ body=typing.cast(+ typing.Optional[typing.Any],+ parse_obj_as(+ type_=typing.Optional[typing.Any], # type: ignore+ object_=_response.json(),+ ),+ ),+ )+ if _response.status_code == 403:+ raise ForbiddenError(+ headers=dict(_response.headers),+ body=typing.cast(+ typing.Optional[typing.Any],+ parse_obj_as(+ type_=typing.Optional[typing.Any], # type: ignore+ object_=_response.json(),+ ),+ ),+ )+ if _response.status_code == 422:+ raise UnprocessableEntityError(+ headers=dict(_response.headers),+ body=typing.cast(+ typing.Optional[typing.Any],+ parse_obj_as(+ type_=typing.Optional[typing.Any], # type: ignore+ object_=_response.json(),+ ),+ ),+ )+ _response_json = _response.json()+ except JSONDecodeError:+ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)+ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)+class AsyncRawSkyvern:def __init__(self, *, client_wrapper: AsyncClientWrapper):@@ -4064,3 +4174,111 @@ class AsyncRawSkyvern:except JSONDecodeError:raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)++ async def run_sdk_action(+ self,+ *,+ url: str,+ action: SdkAction,+ user_agent: typing.Optional[str] = None,+ browser_session_id: typing.Optional[str] = OMIT,+ browser_address: typing.Optional[str] = OMIT,+ workflow_run_id: typing.Optional[str] = OMIT,+ request_options: typing.Optional[RequestOptions] = None,+ ) -> AsyncHttpResponse[RunSdkActionResponse]:+ """+ Execute a single SDK action with the specified parameters++ Parameters+ ----------+ url : str+ The URL where the action should be executed++ action : SdkAction+ The action to execute with its specific parameters++ user_agent : typing.Optional[str]++ browser_session_id : typing.Optional[str]+ The browser session ID++ browser_address : typing.Optional[str]+ The browser address++ workflow_run_id : typing.Optional[str]+ Optional workflow run ID to continue an existing workflow run++ request_options : typing.Optional[RequestOptions]+ Request-specific configuration.++ Returns+ -------+ AsyncHttpResponse[RunSdkActionResponse]+ Successfully executed SDK action+ """+ _response = await self._client_wrapper.httpx_client.request(+ "v1/sdk/run_action",+ method="POST",+ json={+ "url": url,+ "browser_session_id": browser_session_id,+ "browser_address": browser_address,+ "workflow_run_id": workflow_run_id,+ "action": convert_and_respect_annotation_metadata(+ object_=action, annotation=SdkAction, direction="write"+ ),+ },+ headers={+ "content-type": "application/json",+ "x-user-agent": str(user_agent) if user_agent is not None else None,+ },+ request_options=request_options,+ omit=OMIT,+ )+ try:+ if 200 <= _response.status_code < 300:+ _data = typing.cast(+ RunSdkActionResponse,+ parse_obj_as(+ type_=RunSdkActionResponse, # type: ignore+ object_=_response.json(),+ ),+ )+ return AsyncHttpResponse(response=_response, data=_data)+ if _response.status_code == 400:+ raise BadRequestError(+ headers=dict(_response.headers),+ body=typing.cast(+ typing.Optional[typing.Any],+ parse_obj_as(+ type_=typing.Optional[typing.Any], # type: ignore+ object_=_response.json(),+ ),+ ),+ )+ if _response.status_code == 403:+ raise ForbiddenError(+ headers=dict(_response.headers),+ body=typing.cast(+ typing.Optional[typing.Any],+ parse_obj_as(+ type_=typing.Optional[typing.Any], # type: ignore+ object_=_response.json(),+ ),+ ),+ )+ if _response.status_code == 422:+ raise UnprocessableEntityError(+ headers=dict(_response.headers),+ body=typing.cast(+ typing.Optional[typing.Any],+ parse_obj_as(+ type_=typing.Optional[typing.Any], # type: ignore+ object_=_response.json(),+ ),+ ),+ )+ _response_json = _response.json()+ except JSONDecodeError:+ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)+ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)diff --git a/skyvern/client/types/__init__.py b/skyvern/client/types/__init__.pyindex 363815d61..44001b535 100644--- a/skyvern/client/types/__init__.py+++ b/skyvern/client/types/__init__.py@@ -41,6 +41,8 @@ if typing.TYPE_CHECKING:from .bitwarden_sensitive_information_parameter_yaml import BitwardenSensitiveInformationParameterYamlfrom .block_type import BlockTypefrom .browser_session_response import BrowserSessionResponse+ from .click_action import ClickAction+ from .click_action_data import ClickActionDatafrom .code_block import CodeBlockfrom .code_block_parameters_item import (CodeBlockParametersItem,@@ -84,6 +86,9 @@ if typing.TYPE_CHECKING:from .credit_card_credential_response import CreditCardCredentialResponsefrom .download_to_s3block import DownloadToS3Blockfrom .download_to_s3block_yaml import DownloadToS3BlockYaml+ from .extract_action import ExtractAction+ from .extract_action_data import ExtractActionData+ from .extract_action_extract_schema import ExtractActionExtractSchemafrom .extraction_block import ExtractionBlockfrom .extraction_block_data_schema import ExtractionBlockDataSchemafrom .extraction_block_parameters_item import (@@ -236,6 +241,8 @@ if typing.TYPE_CHECKING:)from .human_interaction_block_yaml import HumanInteractionBlockYamlfrom .input_or_select_context import InputOrSelectContext+ from .input_text_action import InputTextAction+ from .input_text_action_data import InputTextActionDatafrom .login_block import LoginBlockfrom .login_block_data_schema import LoginBlockDataSchemafrom .login_block_parameters_item import (@@ -282,11 +289,21 @@ if typing.TYPE_CHECKING:from .pdf_parser_block_yaml import PdfParserBlockYamlfrom .proxy_location import ProxyLocationfrom .run_engine import RunEngine+ from .run_sdk_action_response import RunSdkActionResponsefrom .run_status import RunStatusfrom .script import Scriptfrom .script_file_create import ScriptFileCreatefrom .script_run_response import ScriptRunResponse+ from .sdk_action import (+ SdkAction,+ SdkAction_AiClick,+ SdkAction_AiInputText,+ SdkAction_AiSelectOption,+ SdkAction_Extract,+ )from .select_option import SelectOption+ from .select_option_action import SelectOptionAction+ from .select_option_action_data import SelectOptionActionDatafrom .send_email_block import SendEmailBlockfrom .send_email_block_yaml import SendEmailBlockYamlfrom .skyvern_forge_sdk_schemas_credentials_credential_type import SkyvernForgeSdkSchemasCredentialsCredentialType@@ -520,6 +537,8 @@ _dynamic_imports: typing.Dict[str, str] = {"BitwardenSensitiveInformationParameterYaml": ".bitwarden_sensitive_information_parameter_yaml","BlockType": ".block_type","BrowserSessionResponse": ".browser_session_response",+ "ClickAction": ".click_action",+ "ClickActionData": ".click_action_data","CodeBlock": ".code_block","CodeBlockParametersItem": ".code_block_parameters_item","CodeBlockParametersItem_AwsSecret": ".code_block_parameters_item",@@ -559,6 +578,9 @@ _dynamic_imports: typing.Dict[str, str] = {"CreditCardCredentialResponse": ".credit_card_credential_response","DownloadToS3Block": ".download_to_s3block","DownloadToS3BlockYaml": ".download_to_s3block_yaml",+ "ExtractAction": ".extract_action",+ "ExtractActionData": ".extract_action_data",+ "ExtractActionExtractSchema": ".extract_action_extract_schema","ExtractionBlock": ".extraction_block","ExtractionBlockDataSchema": ".extraction_block_data_schema","ExtractionBlockParametersItem": ".extraction_block_parameters_item",@@ -695,6 +717,8 @@ _dynamic_imports: typing.Dict[str, str] = {"HumanInteractionBlockParametersItem_Workflow": ".human_interaction_block_parameters_item","HumanInteractionBlockYaml": ".human_interaction_block_yaml","InputOrSelectContext": ".input_or_select_context",+ "InputTextAction": ".input_text_action",+ "InputTextActionData": ".input_text_action_data","LoginBlock": ".login_block","LoginBlockDataSchema": ".login_block_data_schema","LoginBlockParametersItem": ".login_block_parameters_item",@@ -737,11 +761,19 @@ _dynamic_imports: typing.Dict[str, str] = {"PdfParserBlockYaml": ".pdf_parser_block_yaml","ProxyLocation": ".proxy_location","RunEngine": ".run_engine",+ "RunSdkActionResponse": ".run_sdk_action_response","RunStatus": ".run_status","Script": ".script","ScriptFileCreate": ".script_file_create","ScriptRunResponse": ".script_run_response",+ "SdkAction": ".sdk_action",+ "SdkAction_AiClick": ".sdk_action",+ "SdkAction_AiInputText": ".sdk_action",+ "SdkAction_AiSelectOption": ".sdk_action",+ "SdkAction_Extract": ".sdk_action","SelectOption": ".select_option",+ "SelectOptionAction": ".select_option_action",+ "SelectOptionActionData": ".select_option_action_data","SendEmailBlock": ".send_email_block","SendEmailBlockYaml": ".send_email_block_yaml","SkyvernForgeSdkSchemasCredentialsCredentialType": ".skyvern_forge_sdk_schemas_credentials_credential_type",@@ -981,6 +1013,8 @@ __all__ = ["BitwardenSensitiveInformationParameterYaml","BlockType","BrowserSessionResponse",+ "ClickAction",+ "ClickActionData","CodeBlock","CodeBlockParametersItem","CodeBlockParametersItem_AwsSecret",@@ -1020,6 +1054,9 @@ __all__ = ["CreditCardCredentialResponse","DownloadToS3Block","DownloadToS3BlockYaml",+ "ExtractAction",+ "ExtractActionData",+ "ExtractActionExtractSchema","ExtractionBlock","ExtractionBlockDataSchema","ExtractionBlockParametersItem",@@ -1156,6 +1193,8 @@ __all__ = ["HumanInteractionBlockParametersItem_Workflow","HumanInteractionBlockYaml","InputOrSelectContext",+ "InputTextAction",+ "InputTextActionData","LoginBlock","LoginBlockDataSchema","LoginBlockParametersItem",@@ -1198,11 +1237,19 @@ __all__ = ["PdfParserBlockYaml","ProxyLocation","RunEngine",+ "RunSdkActionResponse","RunStatus","Script","ScriptFileCreate","ScriptRunResponse",+ "SdkAction",+ "SdkAction_AiClick",+ "SdkAction_AiInputText",+ "SdkAction_AiSelectOption",+ "SdkAction_Extract","SelectOption",+ "SelectOptionAction",+ "SelectOptionActionData","SendEmailBlock","SendEmailBlockYaml","SkyvernForgeSdkSchemasCredentialsCredentialType",diff --git a/skyvern/client/types/click_action.py b/skyvern/client/types/click_action.pynew file mode 100644index 000000000..5cbcde5b3--- /dev/null+++ b/skyvern/client/types/click_action.py@@ -0,0 +1,38 @@+# This file was auto-generated by Fern from our API Definition.++import typing++import pydantic+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+from .click_action_data import ClickActionData+++class ClickAction(UniversalBaseModel):+ selector: typing.Optional[str] = pydantic.Field(default=None)+ """+ CSS selector for the element+ """++ intention: typing.Optional[str] = pydantic.Field(default=None)+ """+ The intention or goal of the click+ """++ data: typing.Optional[ClickActionData] = pydantic.Field(default=None)+ """+ Additional context data+ """++ timeout: typing.Optional[float] = pydantic.Field(default=None)+ """+ Timeout in milliseconds+ """++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allowdiff --git a/skyvern/client/types/click_action_data.py b/skyvern/client/types/click_action_data.pynew file mode 100644index 000000000..acab32a42--- /dev/null+++ b/skyvern/client/types/click_action_data.py@@ -0,0 +1,5 @@+# This file was auto-generated by Fern from our API Definition.++import typing++ClickActionData = typing.Union[str, typing.Dict[str, typing.Optional[typing.Any]]]diff --git a/skyvern/client/types/extract_action.py b/skyvern/client/types/extract_action.pynew file mode 100644index 000000000..7a2730c66--- /dev/null+++ b/skyvern/client/types/extract_action.py@@ -0,0 +1,44 @@+# This file was auto-generated by Fern from our API Definition.++import typing++import pydantic+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+from .extract_action_data import ExtractActionData+from .extract_action_extract_schema import ExtractActionExtractSchema+++class ExtractAction(UniversalBaseModel):+ prompt: typing.Optional[str] = pydantic.Field(default=None)+ """+ Extraction prompt+ """++ extract_schema: typing.Optional[ExtractActionExtractSchema] = pydantic.Field(default=None)+ """+ Schema for extraction+ """++ error_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = pydantic.Field(default=None)+ """+ Error code mapping for extraction+ """++ intention: typing.Optional[str] = pydantic.Field(default=None)+ """+ The intention or goal of the extraction+ """++ data: typing.Optional[ExtractActionData] = pydantic.Field(default=None)+ """+ Additional context data+ """++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allowdiff --git a/skyvern/client/types/extract_action_data.py b/skyvern/client/types/extract_action_data.pynew file mode 100644index 000000000..65ed5301e--- /dev/null+++ b/skyvern/client/types/extract_action_data.py@@ -0,0 +1,5 @@+# This file was auto-generated by Fern from our API Definition.++import typing++ExtractActionData = typing.Union[str, typing.Dict[str, typing.Optional[typing.Any]]]diff --git a/skyvern/client/types/extract_action_extract_schema.py b/skyvern/client/types/extract_action_extract_schema.pynew file mode 100644index 000000000..96fdd56e0--- /dev/null+++ b/skyvern/client/types/extract_action_extract_schema.py@@ -0,0 +1,7 @@+# This file was auto-generated by Fern from our API Definition.++import typing++ExtractActionExtractSchema = typing.Union[+ typing.Dict[str, typing.Optional[typing.Any]], typing.List[typing.Optional[typing.Any]], str+]diff --git a/skyvern/client/types/input_text_action.py b/skyvern/client/types/input_text_action.pynew file mode 100644index 000000000..2effef635--- /dev/null+++ b/skyvern/client/types/input_text_action.py@@ -0,0 +1,53 @@+# This file was auto-generated by Fern from our API Definition.++import typing++import pydantic+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+from .input_text_action_data import InputTextActionData+++class InputTextAction(UniversalBaseModel):+ selector: typing.Optional[str] = pydantic.Field(default=None)+ """+ CSS selector for the element+ """++ value: typing.Optional[str] = pydantic.Field(default=None)+ """+ Value to input+ """++ intention: typing.Optional[str] = pydantic.Field(default=None)+ """+ The intention or goal of the input+ """++ data: typing.Optional[InputTextActionData] = pydantic.Field(default=None)+ """+ Additional context data+ """++ totp_identifier: typing.Optional[str] = pydantic.Field(default=None)+ """+ TOTP identifier for input_text actions+ """++ totp_url: typing.Optional[str] = pydantic.Field(default=None)+ """+ TOTP URL for input_text actions+ """++ timeout: typing.Optional[float] = pydantic.Field(default=None)+ """+ Timeout in milliseconds+ """++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allowdiff --git a/skyvern/client/types/input_text_action_data.py b/skyvern/client/types/input_text_action_data.pynew file mode 100644index 000000000..5cd89b254--- /dev/null+++ b/skyvern/client/types/input_text_action_data.py@@ -0,0 +1,5 @@+# This file was auto-generated by Fern from our API Definition.++import typing++InputTextActionData = typing.Union[str, typing.Dict[str, typing.Optional[typing.Any]]]diff --git a/skyvern/client/types/run_sdk_action_response.py b/skyvern/client/types/run_sdk_action_response.pynew file mode 100644index 000000000..065f19a80--- /dev/null+++ b/skyvern/client/types/run_sdk_action_response.py@@ -0,0 +1,27 @@+# This file was auto-generated by Fern from our API Definition.++import typing++import pydantic+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+++class RunSdkActionResponse(UniversalBaseModel):+ workflow_run_id: str = pydantic.Field()+ """+ The workflow run ID used for this action+ """++ result: typing.Optional[typing.Optional[typing.Any]] = pydantic.Field(default=None)+ """+ The result from the action (e.g., selector, value, extracted data)+ """++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allowdiff --git a/skyvern/client/types/sdk_action.py b/skyvern/client/types/sdk_action.pynew file mode 100644index 000000000..ec0bd5ee9--- /dev/null+++ b/skyvern/client/types/sdk_action.py@@ -0,0 +1,89 @@+# This file was auto-generated by Fern from our API Definition.++from __future__ import annotations++import typing++import pydantic+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+from .click_action_data import ClickActionData+from .extract_action_data import ExtractActionData+from .extract_action_extract_schema import ExtractActionExtractSchema+from .input_text_action_data import InputTextActionData+from .select_option_action_data import SelectOptionActionData+++class SdkAction_AiClick(UniversalBaseModel):+ type: typing.Literal["ai_click"] = "ai_click"+ selector: typing.Optional[str] = None+ intention: typing.Optional[str] = None+ data: typing.Optional[ClickActionData] = None+ timeout: typing.Optional[float] = None++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allow+++class SdkAction_AiInputText(UniversalBaseModel):+ type: typing.Literal["ai_input_text"] = "ai_input_text"+ selector: typing.Optional[str] = None+ value: typing.Optional[str] = None+ intention: typing.Optional[str] = None+ data: typing.Optional[InputTextActionData] = None+ totp_identifier: typing.Optional[str] = None+ totp_url: typing.Optional[str] = None+ timeout: typing.Optional[float] = None++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allow+++class SdkAction_AiSelectOption(UniversalBaseModel):+ type: typing.Literal["ai_select_option"] = "ai_select_option"+ selector: typing.Optional[str] = None+ value: typing.Optional[str] = None+ intention: typing.Optional[str] = None+ data: typing.Optional[SelectOptionActionData] = None+ timeout: typing.Optional[float] = None++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allow+++class SdkAction_Extract(UniversalBaseModel):+ type: typing.Literal["extract"] = "extract"+ prompt: typing.Optional[str] = None+ extract_schema: typing.Optional[ExtractActionExtractSchema] = None+ error_code_mapping: typing.Optional[typing.Dict[str, typing.Optional[str]]] = None+ intention: typing.Optional[str] = None+ data: typing.Optional[ExtractActionData] = None++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allow+++SdkAction = typing.Union[SdkAction_AiClick, SdkAction_AiInputText, SdkAction_AiSelectOption, SdkAction_Extract]diff --git a/skyvern/client/types/select_option_action.py b/skyvern/client/types/select_option_action.pynew file mode 100644index 000000000..35ca2485d--- /dev/null+++ b/skyvern/client/types/select_option_action.py@@ -0,0 +1,43 @@+# This file was auto-generated by Fern from our API Definition.++import typing++import pydantic+from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel+from .select_option_action_data import SelectOptionActionData+++class SelectOptionAction(UniversalBaseModel):+ selector: typing.Optional[str] = pydantic.Field(default=None)+ """+ CSS selector for the element+ """++ value: typing.Optional[str] = pydantic.Field(default=None)+ """+ Value to select+ """++ intention: typing.Optional[str] = pydantic.Field(default=None)+ """+ The intention or goal of the selection+ """++ data: typing.Optional[SelectOptionActionData] = pydantic.Field(default=None)+ """+ Additional context data+ """++ timeout: typing.Optional[float] = pydantic.Field(default=None)+ """+ Timeout in milliseconds+ """++ if IS_PYDANTIC_V2:+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2+ else:++ class Config:+ frozen = True+ smart_union = True+ extra = pydantic.Extra.allowdiff --git a/skyvern/client/types/select_option_action_data.py b/skyvern/client/types/select_option_action_data.pynew file mode 100644index 000000000..1b2c35e98--- /dev/null+++ b/skyvern/client/types/select_option_action_data.py@@ -0,0 +1,5 @@+# This file was auto-generated by Fern from our API Definition.++import typing++SelectOptionActionData = typing.Union[str, typing.Dict[str, typing.Optional[typing.Any]]]diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pynew file mode 100644index 000000000..44577ace2--- /dev/null+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -0,0 +1,441 @@+from __future__ import annotations++import json+from datetime import datetime, timezone+from typing import Any++import structlog+from jinja2.sandbox import SandboxedEnvironment+from playwright.async_api import Page++from skyvern.config import settings+from skyvern.constants import SPECIAL_FIELD_VERIFICATION_CODE+from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi+from skyvern.forge import app+from skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.files import download_file+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.schemas.totp_codes import OTPType+from skyvern.services.otp_service import poll_otp_value+from skyvern.utils.prompt_engine import load_prompt_with_elements+from skyvern.webeye.actions import handler_utils+from skyvern.webeye.actions.actions import (+ ActionStatus,+ InputTextAction,+)+from skyvern.webeye.actions.handler import (+ handle_click_action,+ handle_input_text_action,+ handle_select_option_action,+)+from skyvern.webeye.actions.parse_actions import parse_actions+from skyvern.webeye.scraper.scraper import ScrapedPage++jinja_sandbox_env = SandboxedEnvironment()++LOG = structlog.get_logger()++SELECT_OPTION_GOAL = """- The intention to select an option: {intention}.+- The overall goal that the user wants to achieve: {prompt}."""+++async def _get_element_id_by_selector(selector: str, page: Page) -> str | None:+ locator = page.locator(selector)+ element_id = await locator.get_attribute("unique_id")+ return element_id+++def _get_context_data(data: str | dict[str, Any] | None = None) -> dict[str, Any] | str | None:+ context = skyvern_context.current()+ global_context_data = context.script_run_parameters if context else None+ if not data:+ return global_context_data+ result: dict[str, Any] | str | None+ if isinstance(data, dict):+ result = {k: v for k, v in data.items() if v}+ if global_context_data:+ result.update(global_context_data)+ else:+ global_context_data_str = json.dumps(global_context_data) if global_context_data else ""+ result = f"{data}\n{global_context_data_str}"+ return result+++def _render_template_with_label(template: str, label: str | None = None) -> str:+ template_data = {}+ context = skyvern_context.current()+ if context and context.workflow_run_id:+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(context.workflow_run_id)+ block_reference_data: dict[str, Any] = workflow_run_context.get_block_metadata(label)+ template_data = workflow_run_context.values.copy()+ if label in template_data:+ current_value = template_data[label]+ if isinstance(current_value, dict):+ block_reference_data.update(current_value)+ else:+ LOG.warning(+ f"Script service: Parameter {label} has a registered reference value, going to overwrite it by block metadata"+ )++ if label:+ template_data[label] = block_reference_data++ # inject the forloop metadata as global variables+ if "current_index" in block_reference_data:+ template_data["current_index"] = block_reference_data["current_index"]+ if "current_item" in block_reference_data:+ template_data["current_item"] = block_reference_data["current_item"]+ if "current_value" in block_reference_data:+ template_data["current_value"] = block_reference_data["current_value"]+ try:+ return render_template(template, data=template_data)+ except Exception:+ LOG.exception("Failed to render template", template=template, data=template_data)+ return template+++def render_template(template: str, data: dict[str, Any] | None = None) -> str:+ """+ Refer to Block.format_block_parameter_template_from_workflow_run_context++ TODO: complete this function so that block code shares the same template rendering logic+ """+ template_data = data.copy() if data else {}+ jinja_template = jinja_sandbox_env.from_string(template)+ context = skyvern_context.current()+ if context and context.workflow_run_id:+ workflow_run_id = context.workflow_run_id+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)+ template_data.update(workflow_run_context.values)+ if template in template_data:+ return template_data[template]++ return jinja_template.render(template_data)+++class RealSkyvernPageAi(SkyvernPageAi):+ def __init__(+ self,+ scraped_page: ScrapedPage,+ page: Page,+ ):+ self.scraped_page = scraped_page+ self.page = page+ self.current_label: str | None = None++ async def ai_click(+ self,+ selector: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Click an element using AI to locate it based on intention."""+ try:+ # Build the element tree of the current page for the prompt+ context = skyvern_context.ensure_context()+ payload_str = _get_context_data(data)+ refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()+ element_tree = refreshed_page.build_element_tree()+ single_click_prompt = prompt_engine.load_prompt(+ template="single-click-action",+ navigation_goal=intention,+ navigation_payload_str=payload_str,+ current_url=self.page.url,+ elements=element_tree,+ local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),+ # user_context=getattr(context, "prompt", None),+ )+ json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(+ prompt=single_click_prompt,+ prompt_name="single-click-action",+ organization_id=context.organization_id,+ )+ actions_json = json_response.get("actions", [])+ if actions_json:+ organization_id = context.organization_id if context else None+ task_id = context.task_id if context else None+ step_id = context.step_id if context else None+ task = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else None+ step = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None+ if organization_id and task and step:+ actions = parse_actions(+ task, step.step_id, step.order, self.scraped_page, json_response.get("actions", [])+ )+ action = actions[0]+ result = await handle_click_action(action, self.page, self.scraped_page, task, step)+ if result and result[-1].success is False:+ raise Exception(result[-1].exception_message)+ xpath = action.get_xpath()+ selector = f"xpath={xpath}" if xpath else selector+ return selector+ except Exception:+ LOG.exception(+ f"Failed to do ai click. Falling back to original selector={selector}, intention={intention}, data={data}"+ )++ if selector:+ locator = self.page.locator(selector)+ await locator.click(timeout=timeout)+ return selector++ async def ai_input_text(+ self,+ selector: str,+ value: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Input text into an element using AI to determine the value."""++ context = skyvern_context.current()+ value = value or ""+ transformed_value = value+ element_id: str | None = None+ organization_id = context.organization_id if context else None+ task_id = context.task_id if context else None+ step_id = context.step_id if context else None+ workflow_run_id = context.workflow_run_id if context else None+ task = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else None+ step = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None+ if intention:+ try:+ prompt = context.prompt if context else None+ data = data or {}+ if (totp_identifier or totp_url) and context and organization_id and task_id:+ if totp_identifier:+ totp_identifier = _render_template_with_label(totp_identifier, label=self.current_label)+ if totp_url:+ totp_url = _render_template_with_label(totp_url, label=self.current_label)+ otp_value = await poll_otp_value(+ organization_id=organization_id,+ task_id=task_id,+ workflow_run_id=workflow_run_id,+ totp_identifier=totp_identifier,+ totp_verification_url=totp_url,+ )+ if otp_value and otp_value.get_otp_type() == OTPType.TOTP:+ verification_code = otp_value.value+ if isinstance(data, dict) and SPECIAL_FIELD_VERIFICATION_CODE not in data:+ data[SPECIAL_FIELD_VERIFICATION_CODE] = verification_code+ elif isinstance(data, str) and SPECIAL_FIELD_VERIFICATION_CODE not in data:+ data = f"{data}\n" + str({SPECIAL_FIELD_VERIFICATION_CODE: verification_code})+ elif isinstance(data, list):+ data.append({SPECIAL_FIELD_VERIFICATION_CODE: verification_code})+ else:+ data = {SPECIAL_FIELD_VERIFICATION_CODE: verification_code}++ refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()+ self.scraped_page = refreshed_page+ # get the element_id by the selector+ element_id = await _get_element_id_by_selector(selector, self.page)+ script_generation_input_text_prompt = prompt_engine.load_prompt(+ template="script-generation-input-text-generatiion",+ intention=intention,+ goal=prompt,+ data=data,+ )+ json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ prompt=script_generation_input_text_prompt,+ prompt_name="script-generation-input-text-generatiion",+ organization_id=organization_id,+ )+ value = json_response.get("answer", value)+ except Exception:+ LOG.exception(f"Failed to adapt value for input text action on selector={selector}, value={value}")++ if context and context.workflow_run_id:+ transformed_value = await _get_actual_value_of_parameter_if_secret(context.workflow_run_id, str(value))++ if element_id and organization_id and task and step:+ action = InputTextAction(+ element_id=element_id,+ text=value,+ status=ActionStatus.pending,+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ task_id=task_id,+ step_id=context.step_id if context else None,+ reasoning=intention,+ intention=intention,+ response=value,+ )+ result = await handle_input_text_action(action, self.page, self.scraped_page, task, step)+ if result and result[-1].success is False:+ raise Exception(result[-1].exception_message)+ else:+ locator = self.page.locator(selector)+ await handler_utils.input_sequentially(locator, transformed_value, timeout=timeout)+ return value++ async def ai_upload_file(+ self,+ selector: str,+ files: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Upload a file using AI to process the file URL."""++ if intention:+ try:+ context = skyvern_context.current()+ prompt = context.prompt if context else None+ data = _get_context_data(data)+ script_generation_file_url_prompt = prompt_engine.load_prompt(+ template="script-generation-file-url-generation",+ intention=intention,+ data=data,+ goal=prompt,+ )+ json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ prompt=script_generation_file_url_prompt,+ prompt_name="script-generation-file-url-generation",+ organization_id=context.organization_id if context else None,+ )+ files = json_response.get("answer", files)+ except Exception:+ LOG.exception(f"Failed to adapt value for input text action on selector={selector}, file={files}")+ if not files:+ raise ValueError("file url must be provided")+ file_path = await download_file(files)+ locator = self.page.locator(selector)+ await locator.set_input_files(file_path, timeout=timeout)+ return files++ async def ai_select_option(+ self,+ selector: str,+ value: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Select an option from a dropdown using AI."""++ option_value = value or ""+ context = skyvern_context.current()+ if context and context.task_id and context.step_id and context.organization_id:+ task = await app.DATABASE.get_task(context.task_id, organization_id=context.organization_id)+ step = await app.DATABASE.get_step(context.step_id, organization_id=context.organization_id)+ if intention and task and step:+ try:+ prompt = context.prompt if context else None+ # data = _get_context_data(data)+ data = data or {}+ refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()+ self.scraped_page = refreshed_page+ element_tree = refreshed_page.build_element_tree()+ merged_goal = SELECT_OPTION_GOAL.format(intention=intention, prompt=prompt)+ single_select_prompt = prompt_engine.load_prompt(+ template="single-select-action",+ navigation_payload_str=data,+ navigation_goal=merged_goal,+ current_url=self.page.url,+ elements=element_tree,+ local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),+ )+ json_response = await app.SELECT_AGENT_LLM_API_HANDLER(+ prompt=single_select_prompt,+ prompt_name="single-select-action",+ organization_id=context.organization_id if context else None,+ )+ actions = parse_actions(+ task, step.step_id, step.order, self.scraped_page, json_response.get("actions", [])+ )+ if actions:+ action = actions[0]+ if not action.option:+ raise ValueError("SelectOptionAction requires an 'option' field")+ option_value = action.option.value or action.option.label or ""+ await handle_select_option_action(+ action=action,+ page=self.page,+ scraped_page=self.scraped_page,+ task=task,+ step=step,+ )+ else:+ LOG.exception(+ f"Failed to parse actions for select option action on selector={selector}, value={value}"+ )+ except Exception:+ LOG.exception(+ f"Failed to adapt value for select option action on selector={selector}, value={value}"+ )+ else:+ locator = self.page.locator(selector)+ await locator.select_option(option_value, timeout=timeout)+ return option_value++ async def ai_extract(+ self,+ prompt: str,+ schema: dict[str, Any] | list | str | None = None,+ error_code_mapping: dict[str, str] | None = None,+ intention: str | None = None,+ data: str | dict[str, Any] | None = None,+ ) -> dict[str, Any] | list | str | None:+ """Extract information from the page using AI."""++ scraped_page_refreshed = await self.scraped_page.refresh()+ context = skyvern_context.current()+ tz_info = datetime.now(tz=timezone.utc).tzinfo+ if context and context.tz_info:+ tz_info = context.tz_info+ prompt = _render_template_with_label(prompt, label=self.current_label)+ extract_information_prompt = load_prompt_with_elements(+ element_tree_builder=scraped_page_refreshed,+ prompt_engine=prompt_engine,+ template_name="extract-information",+ html_need_skyvern_attrs=False,+ data_extraction_goal=prompt,+ extracted_information_schema=schema,+ current_url=scraped_page_refreshed.url,+ extracted_text=scraped_page_refreshed.extracted_text,+ error_code_mapping_str=(json.dumps(error_code_mapping) if error_code_mapping else None),+ local_datetime=datetime.now(tz_info).isoformat(),+ )+ step = None+ if context and context.organization_id and context.task_id and context.step_id:+ step = await app.DATABASE.get_step(+ step_id=context.step_id,+ organization_id=context.organization_id,+ )++ result = await app.EXTRACTION_LLM_API_HANDLER(+ prompt=extract_information_prompt,+ step=step,+ screenshots=scraped_page_refreshed.screenshots,+ prompt_name="extract-information",+ )+ if context and context.script_mode:+ print(f"\n✨ 📊 Extracted Information:\n{'-' * 50}")++ try:+ # Pretty print JSON if result is a dict/list+ if isinstance(result, (dict, list)):+ print(json.dumps(result, indent=2, ensure_ascii=False))+ else:+ print(result)+ except Exception:+ print(result)+ print(f"{'-' * 50}\n")+ return result+++async def _get_actual_value_of_parameter_if_secret(workflow_run_id: str, parameter: str) -> Any:+ """+ Get the actual value of a parameter if it's a secret. If it's not a secret, return the parameter value as is.++ Just return the parameter value if the task isn't a workflow's task.++ This is only used for InputTextAction, UploadFileAction, and ClickAction (if it has a file_url).+ """+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)+ secret_value = workflow_run_context.get_original_secret_value_or_none(parameter)+ return secret_value if secret_value is not None else parameterdiff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 30cffc169..b3512571a 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -10,7 +10,8 @@ import structlogfrom playwright.async_api import Pagefrom skyvern.config import settings-from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi, render_template+from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi, render_template+from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAifrom skyvern.exceptions import ScriptTerminationException, WorkflowRunNotFoundfrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine@@ -123,7 +124,7 @@ class SkyvernPage:) -> SkyvernPage:scraped_page = await cls.create_scraped_page(browser_session_id=browser_session_id)page = await scraped_page._browser_state.must_get_working_page()- ai = SkyvernPageAi(scraped_page, page)+ ai = RealSkyvernPageAi(scraped_page, page)return cls(scraped_page=scraped_page, page=page, ai=ai)@classmethoddiff --git a/skyvern/core/script_generations/skyvern_page_ai.py b/skyvern/core/script_generations/skyvern_page_ai.pyindex 51949a9d6..1b8c7c61a 100644--- a/skyvern/core/script_generations/skyvern_page_ai.py+++ b/skyvern/core/script_generations/skyvern_page_ai.py@@ -1,126 +1,12 @@from __future__ import annotations-import json-from datetime import datetime, timezone-from typing import Any--import structlog-from jinja2.sandbox import SandboxedEnvironment-from playwright.async_api import Page+from typing import Any, Protocolfrom skyvern.config import settings-from skyvern.constants import SPECIAL_FIELD_VERIFICATION_CODE-from skyvern.forge import app-from skyvern.forge.prompts import prompt_engine-from skyvern.forge.sdk.api.files import download_file-from skyvern.forge.sdk.core import skyvern_context-from skyvern.forge.sdk.schemas.totp_codes import OTPType-from skyvern.services.otp_service import poll_otp_value-from skyvern.utils.prompt_engine import load_prompt_with_elements-from skyvern.webeye.actions import handler_utils-from skyvern.webeye.actions.actions import (- ActionStatus,- InputTextAction,-)-from skyvern.webeye.actions.handler import (- handle_click_action,- handle_input_text_action,- handle_select_option_action,-)-from skyvern.webeye.actions.parse_actions import parse_actions-from skyvern.webeye.scraper.scraper import ScrapedPage--jinja_sandbox_env = SandboxedEnvironment()--LOG = structlog.get_logger()--SELECT_OPTION_GOAL = """- The intention to select an option: {intention}.-- The overall goal that the user wants to achieve: {prompt}."""---async def _get_element_id_by_selector(selector: str, page: Page) -> str | None:- locator = page.locator(selector)- element_id = await locator.get_attribute("unique_id")- return element_id---def _get_context_data(data: str | dict[str, Any] | None = None) -> dict[str, Any] | str | None:- context = skyvern_context.current()- global_context_data = context.script_run_parameters if context else None- if not data:- return global_context_data- result: dict[str, Any] | str | None- if isinstance(data, dict):- result = {k: v for k, v in data.items() if v}- if global_context_data:- result.update(global_context_data)- else:- global_context_data_str = json.dumps(global_context_data) if global_context_data else ""- result = f"{data}\n{global_context_data_str}"- return result-def _render_template_with_label(template: str, label: str | None = None) -> str:- template_data = {}- context = skyvern_context.current()- if context and context.workflow_run_id:- workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(context.workflow_run_id)- block_reference_data: dict[str, Any] = workflow_run_context.get_block_metadata(label)- template_data = workflow_run_context.values.copy()- if label in template_data:- current_value = template_data[label]- if isinstance(current_value, dict):- block_reference_data.update(current_value)- else:- LOG.warning(- f"Script service: Parameter {label} has a registered reference value, going to overwrite it by block metadata"- )-- if label:- template_data[label] = block_reference_data-- # inject the forloop metadata as global variables- if "current_index" in block_reference_data:- template_data["current_index"] = block_reference_data["current_index"]- if "current_item" in block_reference_data:- template_data["current_item"] = block_reference_data["current_item"]- if "current_value" in block_reference_data:- template_data["current_value"] = block_reference_data["current_value"]- try:- return render_template(template, data=template_data)- except Exception:- LOG.exception("Failed to render template", template=template, data=template_data)- return template---def render_template(template: str, data: dict[str, Any] | None = None) -> str:- """- Refer to Block.format_block_parameter_template_from_workflow_run_context-- TODO: complete this function so that block code shares the same template rendering logic- """- template_data = data.copy() if data else {}- jinja_template = jinja_sandbox_env.from_string(template)- context = skyvern_context.current()- if context and context.workflow_run_id:- workflow_run_id = context.workflow_run_id- workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)- template_data.update(workflow_run_context.values)- if template in template_data:- return template_data[template]-- return jinja_template.render(template_data)---class SkyvernPageAi:- def __init__(- self,- scraped_page: ScrapedPage,- page: Page,- ):- self.scraped_page = scraped_page- self.page = page- self.current_label: str | None = None+class SkyvernPageAi(Protocol):+ """Protocol defining the interface for AI-powered page interactions."""async def ai_click(self,@@ -130,52 +16,7 @@ class SkyvernPageAi:timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:"""Click an element using AI to locate it based on intention."""- try:- # Build the element tree of the current page for the prompt- context = skyvern_context.ensure_context()- payload_str = _get_context_data(data)- refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()- element_tree = refreshed_page.build_element_tree()- single_click_prompt = prompt_engine.load_prompt(- template="single-click-action",- navigation_goal=intention,- navigation_payload_str=payload_str,- current_url=self.page.url,- elements=element_tree,- local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),- # user_context=getattr(context, "prompt", None),- )- json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(- prompt=single_click_prompt,- prompt_name="single-click-action",- organization_id=context.organization_id,- )- actions_json = json_response.get("actions", [])- if actions_json:- organization_id = context.organization_id if context else None- task_id = context.task_id if context else None- step_id = context.step_id if context else None- task = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else None- step = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None- if organization_id and task and step:- actions = parse_actions(- task, step.step_id, step.order, self.scraped_page, json_response.get("actions", [])- )- action = actions[0]- result = await handle_click_action(action, self.page, self.scraped_page, task, step)- if result and result[-1].success is False:- raise Exception(result[-1].exception_message)- xpath = action.get_xpath()- selector = f"xpath={xpath}" if xpath else selector- return selector- except Exception:- LOG.exception(- f"Failed to do ai click. Falling back to original selector={selector}, intention={intention}, data={data}"- )-- locator = self.page.locator(selector)- await locator.click(timeout=timeout)- return selector+ ...async def ai_input_text(self,@@ -188,86 +29,7 @@ class SkyvernPageAi:timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:"""Input text into an element using AI to determine the value."""-- context = skyvern_context.current()- value = value or ""- transformed_value = value- element_id: str | None = None- organization_id = context.organization_id if context else None- task_id = context.task_id if context else None- step_id = context.step_id if context else None- workflow_run_id = context.workflow_run_id if context else None- task = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else None- step = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None- if intention:- try:- prompt = context.prompt if context else None- data = data or {}- if (totp_identifier or totp_url) and context and organization_id and task_id:- if totp_identifier:- totp_identifier = _render_template_with_label(totp_identifier, label=self.current_label)- if totp_url:- totp_url = _render_template_with_label(totp_url, label=self.current_label)- otp_value = await poll_otp_value(- organization_id=organization_id,- task_id=task_id,- workflow_run_id=workflow_run_id,- totp_identifier=totp_identifier,- totp_verification_url=totp_url,- )- if otp_value and otp_value.get_otp_type() == OTPType.TOTP:- verification_code = otp_value.value- if isinstance(data, dict) and SPECIAL_FIELD_VERIFICATION_CODE not in data:- data[SPECIAL_FIELD_VERIFICATION_CODE] = verification_code- elif isinstance(data, str) and SPECIAL_FIELD_VERIFICATION_CODE not in data:- data = f"{data}\n" + str({SPECIAL_FIELD_VERIFICATION_CODE: verification_code})- elif isinstance(data, list):- data.append({SPECIAL_FIELD_VERIFICATION_CODE: verification_code})- else:- data = {SPECIAL_FIELD_VERIFICATION_CODE: verification_code}-- refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()- self.scraped_page = refreshed_page- # get the element_id by the selector- element_id = await _get_element_id_by_selector(selector, self.page)- script_generation_input_text_prompt = prompt_engine.load_prompt(- template="script-generation-input-text-generatiion",- intention=intention,- goal=prompt,- data=data,- )- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(- prompt=script_generation_input_text_prompt,- prompt_name="script-generation-input-text-generatiion",- organization_id=organization_id,- )- value = json_response.get("answer", value)- except Exception:- LOG.exception(f"Failed to adapt value for input text action on selector={selector}, value={value}")-- if context and context.workflow_run_id:- transformed_value = await _get_actual_value_of_parameter_if_secret(context.workflow_run_id, str(value))-- if element_id and organization_id and task and step:- action = InputTextAction(- element_id=element_id,- text=value,- status=ActionStatus.pending,- organization_id=organization_id,- workflow_run_id=workflow_run_id,- task_id=task_id,- step_id=context.step_id if context else None,- reasoning=intention,- intention=intention,- response=value,- )- result = await handle_input_text_action(action, self.page, self.scraped_page, task, step)- if result and result[-1].success is False:- raise Exception(result[-1].exception_message)- else:- locator = self.page.locator(selector)- await handler_utils.input_sequentially(locator, transformed_value, timeout=timeout)- return value+ ...async def ai_upload_file(self,@@ -278,32 +40,7 @@ class SkyvernPageAi:timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:"""Upload a file using AI to process the file URL."""-- if intention:- try:- context = skyvern_context.current()- prompt = context.prompt if context else None- data = _get_context_data(data)- script_generation_file_url_prompt = prompt_engine.load_prompt(- template="script-generation-file-url-generation",- intention=intention,- data=data,- goal=prompt,- )- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(- prompt=script_generation_file_url_prompt,- prompt_name="script-generation-file-url-generation",- organization_id=context.organization_id if context else None,- )- files = json_response.get("answer", files)- except Exception:- LOG.exception(f"Failed to adapt value for input text action on selector={selector}, file={files}")- if not files:- raise ValueError("file url must be provided")- file_path = await download_file(files)- locator = self.page.locator(selector)- await locator.set_input_files(file_path, timeout=timeout)- return files+ ...async def ai_select_option(self,@@ -314,61 +51,7 @@ class SkyvernPageAi:timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> str:"""Select an option from a dropdown using AI."""-- option_value = value or ""- context = skyvern_context.current()- if context and context.task_id and context.step_id and context.organization_id:- task = await app.DATABASE.get_task(context.task_id, organization_id=context.organization_id)- step = await app.DATABASE.get_step(context.step_id, organization_id=context.organization_id)- if intention and task and step:- try:- prompt = context.prompt if context else None- # data = _get_context_data(data)- data = data or {}- refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()- self.scraped_page = refreshed_page- element_tree = refreshed_page.build_element_tree()- merged_goal = SELECT_OPTION_GOAL.format(intention=intention, prompt=prompt)- single_select_prompt = prompt_engine.load_prompt(- template="single-select-action",- navigation_payload_str=data,- navigation_goal=merged_goal,- current_url=self.page.url,- elements=element_tree,- local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),- )- json_response = await app.SELECT_AGENT_LLM_API_HANDLER(- prompt=single_select_prompt,- prompt_name="single-select-action",- organization_id=context.organization_id if context else None,- )- actions = parse_actions(- task, step.step_id, step.order, self.scraped_page, json_response.get("actions", [])- )- if actions:- action = actions[0]- if not action.option:- raise ValueError("SelectOptionAction requires an 'option' field")- option_value = action.option.value or action.option.label or ""- await handle_select_option_action(- action=action,- page=self.page,- scraped_page=self.scraped_page,- task=task,- step=step,- )- else:- LOG.exception(- f"Failed to parse actions for select option action on selector={selector}, value={value}"- )- except Exception:- LOG.exception(- f"Failed to adapt value for select option action on selector={selector}, value={value}"- )- else:- locator = self.page.locator(selector)- await locator.select_option(option_value, timeout=timeout)- return option_value+ ...async def ai_extract(self,@@ -379,61 +62,4 @@ class SkyvernPageAi:data: str | dict[str, Any] | None = None,) -> dict[str, Any] | list | str | None:"""Extract information from the page using AI."""-- scraped_page_refreshed = await self.scraped_page.refresh()- context = skyvern_context.current()- tz_info = datetime.now(tz=timezone.utc).tzinfo- if context and context.tz_info:- tz_info = context.tz_info- prompt = _render_template_with_label(prompt, label=self.current_label)- extract_information_prompt = load_prompt_with_elements(- element_tree_builder=scraped_page_refreshed,- prompt_engine=prompt_engine,- template_name="extract-information",- html_need_skyvern_attrs=False,- data_extraction_goal=prompt,- extracted_information_schema=schema,- current_url=scraped_page_refreshed.url,- extracted_text=scraped_page_refreshed.extracted_text,- error_code_mapping_str=(json.dumps(error_code_mapping) if error_code_mapping else None),- local_datetime=datetime.now(tz_info).isoformat(),- )- step = None- if context and context.organization_id and context.task_id and context.step_id:- step = await app.DATABASE.get_step(- step_id=context.step_id,- organization_id=context.organization_id,- )-- result = await app.EXTRACTION_LLM_API_HANDLER(- prompt=extract_information_prompt,- step=step,- screenshots=scraped_page_refreshed.screenshots,- prompt_name="extract-information",- )- if context and context.script_mode:- print(f"\n✨ 📊 Extracted Information:\n{'-' * 50}")-- try:- # Pretty print JSON if result is a dict/list- if isinstance(result, (dict, list)):- print(json.dumps(result, indent=2, ensure_ascii=False))- else:- print(result)- except Exception:- print(result)- print(f"{'-' * 50}\n")- return result---async def _get_actual_value_of_parameter_if_secret(workflow_run_id: str, parameter: str) -> Any:- """- Get the actual value of a parameter if it's a secret. If it's not a secret, return the parameter value as is.-- Just return the parameter value if the task isn't a workflow's task.-- This is only used for InputTextAction, UploadFileAction, and ClickAction (if it has a file_url).- """- workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(workflow_run_id)- secret_value = workflow_run_context.get_original_secret_value_or_none(parameter)- return secret_value if secret_value is not None else parameter+ ...diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.pyindex 53188c8b8..662f0e61d 100644--- a/skyvern/forge/sdk/routes/__init__.py+++ b/skyvern/forge/sdk/routes/__init__.py@@ -5,6 +5,7 @@ from skyvern.forge.sdk.routes import debug_sessions # noqa: F401from skyvern.forge.sdk.routes import pylon # noqa: F401from skyvern.forge.sdk.routes import run_blocks # noqa: F401from skyvern.forge.sdk.routes import scripts # noqa: F401+from skyvern.forge.sdk.routes import sdk # noqa: F401from skyvern.forge.sdk.routes import streaming # noqa: F401from skyvern.forge.sdk.routes import streaming_messages # noqa: F401from skyvern.forge.sdk.routes import streaming_vnc # noqa: F401diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.pynew file mode 100644index 000000000..bfb9ecbbb--- /dev/null+++ b/skyvern/forge/sdk/routes/sdk.py@@ -0,0 +1,180 @@+import json++import structlog+from fastapi import Depends, HTTPException, status++from skyvern import SkyvernPage+from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi+from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.routes.routers import base_router+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.sdk_actions import (+ RunSdkActionRequest,+ RunSdkActionResponse,+)+from skyvern.forge.sdk.services import org_auth_service+from skyvern.forge.sdk.workflow.models.workflow import (+ WorkflowRequestBody,+ WorkflowRunStatus,+)+from skyvern.schemas.workflows import BlockType, WorkflowStatus++LOG = structlog.get_logger()+++@base_router.post(+ "/sdk/run_action",+ response_model=RunSdkActionResponse,+ summary="Run an SDK action",+ description="Execute a single SDK action with the specified parameters",+ tags=["SDK"],+ openapi_extra={+ "x-fern-sdk-method-name": "run_sdk_action",+ },+)+@base_router.post("/sdk/run_action/", include_in_schema=False)+async def run_sdk_action(+ action_request: RunSdkActionRequest,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> RunSdkActionResponse:+ """Execute a single SDK action with the specified parameters."""+ LOG.info(+ "Running SDK action",+ organization_id=organization.organization_id,+ action_type=action_request.action.type,+ )++ organization_id = organization.organization_id+ browser_session_id = action_request.browser_session_id+ browser_address = action_request.browser_address+ action = action_request.action++ # Use existing workflow_run_id if provided, otherwise create a new one+ if action_request.workflow_run_id:+ workflow_run = await app.DATABASE.get_workflow_run(+ workflow_run_id=action_request.workflow_run_id,+ organization_id=organization_id,+ )+ if not workflow_run:+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail=f"Workflow run {action_request.workflow_run_id} not found",+ )+ workflow = await app.DATABASE.get_workflow(+ workflow_id=workflow_run.workflow_id,+ organization_id=organization_id,+ )+ if not workflow:+ raise HTTPException(+ status_code=status.HTTP_404_NOT_FOUND,+ detail=f"Workflow {workflow_run.workflow_id} not found",+ )+ else:+ workflow = await app.WORKFLOW_SERVICE.create_empty_workflow(+ organization,+ title="SDK Workflow",+ status=WorkflowStatus.auto_generated,+ )+ workflow_run = await app.WORKFLOW_SERVICE.setup_workflow_run(+ request_id=None,+ workflow_request=WorkflowRequestBody(+ browser_session_id=browser_session_id,+ browser_address=browser_address,+ ),+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization=organization,+ version=None,+ )+ workflow_run = await app.DATABASE.update_workflow_run(+ workflow_run_id=workflow_run.workflow_run_id,+ status=WorkflowRunStatus.completed,+ )++ task = await app.DATABASE.create_task(+ organization_id=organization_id,+ url=action_request.url,+ navigation_goal=None,+ navigation_payload=None,+ data_extraction_goal=None,+ title=f"SDK Action Task: {action_request.action.type}",+ workflow_run_id=workflow_run.workflow_run_id,+ browser_session_id=browser_session_id,+ browser_address=browser_address,+ )++ step = await app.DATABASE.create_step(+ task.task_id,+ order=0,+ retry_index=0,+ organization_id=organization.organization_id,+ )++ await app.DATABASE.create_workflow_run_block(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ block_type=BlockType.ACTION,+ task_id=task.task_id,+ )++ context = skyvern_context.ensure_context()+ skyvern_context.set(+ SkyvernContext(+ request_id=context.request_id,+ organization_id=task.organization_id,+ task_id=task.task_id,+ step_id=step.step_id,+ browser_session_id=browser_session_id,+ max_screenshot_scrolls=task.max_screenshot_scrolls,+ workflow_id=workflow.workflow_id,+ workflow_run_id=workflow_run.workflow_run_id,+ )+ )+ result = None+ try:+ scraped_page = await SkyvernPage.create_scraped_page(browser_session_id=browser_session_id)+ page = await scraped_page._browser_state.must_get_working_page()+ page_ai = RealSkyvernPageAi(scraped_page, page)++ if action.type == "ai_click":+ result = await page_ai.ai_click(+ selector=action.selector,+ intention=action.intention,+ data=action.data,+ timeout=action.timeout,+ )+ elif action.type == "ai_input_text":+ result = await page_ai.ai_input_text(+ selector=action.selector,+ value=action.value,+ intention=action.intention,+ data=action.data,+ totp_identifier=action.totp_identifier,+ totp_url=action.totp_url,+ timeout=action.timeout,+ )+ elif action.type == "ai_select_option":+ result = await page_ai.ai_select_option(+ selector=action.selector,+ value=action.value,+ intention=action.intention,+ data=action.data,+ timeout=action.timeout,+ )+ elif action.type == "extract":+ extract_result = await page_ai.ai_extract(+ prompt=action.prompt,+ schema=action.extract_schema,+ error_code_mapping=action.error_code_mapping,+ intention=action.intention,+ data=action.data,+ )+ result = json.dumps(extract_result)+ finally:+ skyvern_context.reset()++ return RunSdkActionResponse(+ workflow_run_id=workflow_run.workflow_run_id,+ result=result,+ )diff --git a/skyvern/forge/sdk/schemas/sdk_actions.py b/skyvern/forge/sdk/schemas/sdk_actions.pynew file mode 100644index 000000000..cdf3b3020--- /dev/null+++ b/skyvern/forge/sdk/schemas/sdk_actions.py@@ -0,0 +1,100 @@+from enum import Enum+from typing import Annotated, Any, Literal, Union++from pydantic import BaseModel, Field++from skyvern.config import settings+++class SdkActionType(str, Enum):+ """Enum for SDK action types that can be executed."""++ AI_CLICK = "ai_click"+ AI_INPUT_TEXT = "ai_input_text"+ AI_SELECT_OPTION = "ai_select_option"+ EXTRACT = "extract"+++# Base action class+class SdkActionBase(BaseModel):+ """Base class for SDK actions."""++ type: str = Field(..., description="The type of action")+++# Specific action types+class ClickAction(SdkActionBase):+ """Click action parameters."""++ type: Literal["ai_click"] = "ai_click"+ selector: str = Field(default="", description="CSS selector for the element")+ intention: str = Field(default="", description="The intention or goal of the click")+ data: str | dict[str, Any] | None = Field(None, description="Additional context data")+ timeout: float = Field(default=settings.BROWSER_ACTION_TIMEOUT_MS, description="Timeout in milliseconds")+++class InputTextAction(SdkActionBase):+ """Input text action parameters."""++ type: Literal["ai_input_text"] = "ai_input_text"+ selector: str = Field(default="", description="CSS selector for the element")+ value: str = Field(default="", description="Value to input")+ intention: str = Field(default="", description="The intention or goal of the input")+ data: str | dict[str, Any] | None = Field(None, description="Additional context data")+ totp_identifier: str | None = Field(None, description="TOTP identifier for input_text actions")+ totp_url: str | None = Field(None, description="TOTP URL for input_text actions")+ timeout: float = Field(default=settings.BROWSER_ACTION_TIMEOUT_MS, description="Timeout in milliseconds")+++class SelectOptionAction(SdkActionBase):+ """Select option action parameters."""++ type: Literal["ai_select_option"] = "ai_select_option"+ selector: str = Field(default="", description="CSS selector for the element")+ value: str = Field(default="", description="Value to select")+ intention: str = Field(default="", description="The intention or goal of the selection")+ data: str | dict[str, Any] | None = Field(None, description="Additional context data")+ timeout: float = Field(default=settings.BROWSER_ACTION_TIMEOUT_MS, description="Timeout in milliseconds")+++class ExtractAction(SdkActionBase):+ """Extract data action parameters."""++ type: Literal["extract"] = "extract"+ prompt: str = Field(default="", description="Extraction prompt")+ extract_schema: dict[str, Any] | list | str | None = Field(None, description="Schema for extraction")+ error_code_mapping: dict[str, str] | None = Field(None, description="Error code mapping for extraction")+ intention: str | None = Field(None, description="The intention or goal of the extraction")+ data: str | dict[str, Any] | None = Field(None, description="Additional context data")+++# Discriminated union of all action types+SdkAction = Annotated[+ Union[ClickAction, InputTextAction, SelectOptionAction, ExtractAction],+ Field(discriminator="type"),+]+++class RunActionResponse(BaseModel):+ """Response from running an action."""++ workflow_run_id: str = Field(..., description="The workflow run ID used for this action")+++class RunSdkActionRequest(BaseModel):+ """Request to run a single SDK action."""++ url: str = Field(..., description="The URL where the action should be executed")+ browser_session_id: str | None = Field(None, description="The browser session ID")+ browser_address: str | None = Field(None, description="The browser address")+ workflow_run_id: str | None = Field(+ None, description="Optional workflow run ID to continue an existing workflow run"+ )+ action: SdkAction = Field(..., description="The action to execute with its specific parameters")+++class RunSdkActionResponse(BaseModel):+ """Response from running an SDK action."""++ workflow_run_id: str = Field(..., description="The workflow run ID used for this action")+ result: Any | None = Field(None, description="The result from the action (e.g., selector, value, extracted data)")diff --git a/skyvern/library/SdkSkyvernPageAi.py b/skyvern/library/SdkSkyvernPageAi.pynew file mode 100644index 000000000..c3a412987--- /dev/null+++ b/skyvern/library/SdkSkyvernPageAi.py@@ -0,0 +1,144 @@+from typing import TYPE_CHECKING, Any++from playwright.async_api import Page++from skyvern.config import settings+from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi+from skyvern.forge.sdk.schemas.sdk_actions import (+ ClickAction,+ ExtractAction,+ InputTextAction,+ SelectOptionAction,+)++if TYPE_CHECKING:+ from skyvern.library.skyvern_browser import SkyvernBrowser+++class SdkSkyvernPageAi(SkyvernPageAi):+ """Implementation of SkyvernPageAi that makes API calls to the server."""++ def __init__(+ self,+ browser: "SkyvernBrowser",+ page: Page,+ ):+ self._browser = browser+ self._page = page++ async def ai_click(+ self,+ selector: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Click an element using AI via API call."""++ action = ClickAction(+ selector=selector,+ intention=intention,+ data=data,+ timeout=timeout,+ )+ response = await self._browser.client.run_sdk_action(+ url=self._page.url,+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,+ workflow_run_id=self._browser.workflow_run_id,+ action=action,+ )+ self._browser.workflow_run_id = response.workflow_run_id+ return response.result if response.result else selector++ async def ai_input_text(+ self,+ selector: str,+ value: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Input text into an element using AI via API call."""++ response = await self._browser.client.run_sdk_action(+ url=self._page.url,+ action=InputTextAction(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ timeout=timeout,+ ),+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,+ workflow_run_id=self._browser.workflow_run_id,+ )+ self._browser.workflow_run_id = response.workflow_run_id+ return response.result if response.result else value++ async def ai_select_option(+ self,+ selector: str,+ value: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Select an option from a dropdown using AI via API call."""++ response = await self._browser.client.run_sdk_action(+ url=self._page.url,+ action=SelectOptionAction(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ timeout=timeout,+ ),+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,+ workflow_run_id=self._browser.workflow_run_id,+ )+ self._browser.workflow_run_id = response.workflow_run_id+ return response.result if response.result else value++ async def ai_upload_file(+ self,+ selector: str,+ files: str,+ intention: str,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ raise NotImplementedError("Upload is not supported yet")++ async def ai_extract(+ self,+ prompt: str,+ schema: dict[str, Any] | list | str | None = None,+ error_code_mapping: dict[str, str] | None = None,+ intention: str | None = None,+ data: str | dict[str, Any] | None = None,+ ) -> dict[str, Any] | list | str | None:+ """Extract information from the page using AI via API call."""++ response = await self._browser.client.run_sdk_action(+ url=self._page.url,+ action=ExtractAction(+ prompt=prompt,+ extract_schema=schema,+ error_code_mapping=error_code_mapping,+ intention=intention,+ data=data,+ ),+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,+ workflow_run_id=self._browser.workflow_run_id,+ )+ self._browser.workflow_run_id = response.workflow_run_id+ return response.result if response.result else Nonediff --git a/skyvern/library/skyvern_browser.py b/skyvern/library/skyvern_browser.pyindex 8c763b5c1..f78686dd3 100644--- a/skyvern/library/skyvern_browser.py+++ b/skyvern/library/skyvern_browser.py@@ -1,7 +1,7 @@from playwright.async_api import BrowserContext, Pagefrom skyvern.client import AsyncSkyvern-from skyvern.library.skyvern_browser_page import SkyvernBrowserPage, SkyvernPageRun+from skyvern.library.skyvern_browser_page import SkyvernBrowserPageclass SkyvernBrowser:@@ -44,6 +44,20 @@ class SkyvernBrowser:self._browser_address = browser_addressself._client = client+ self.workflow_run_id: None | str = None++ @property+ def browser_session_id(self) -> str | None:+ return self._browser_session_id++ @property+ def browser_address(self) -> str | None:+ return self._browser_address++ @property+ def client(self) -> AsyncSkyvern:+ return self._client+async def get_working_page(self) -> SkyvernBrowserPage:"""Get the most recent page or create a new one if none exists.@@ -73,5 +87,4 @@ class SkyvernBrowser:return await self._create_skyvern_page(page)async def _create_skyvern_page(self, page: Page) -> SkyvernBrowserPage:- page_ai = SkyvernPageRun(page, self._browser_session_id, self._browser_address, self._client)- return SkyvernBrowserPage(page, page_ai)+ return SkyvernBrowserPage(self, page)diff --git a/skyvern/library/skyvern_browser_page.py b/skyvern/library/skyvern_browser_page.pyindex 8179e942f..447857426 100644--- a/skyvern/library/skyvern_browser_page.py+++ b/skyvern/library/skyvern_browser_page.py@@ -1,11 +1,18 @@import asyncio-from typing import Any+from typing import TYPE_CHECKING, Anyfrom playwright.async_api import Page-from skyvern.client import AsyncSkyvern, GetRunResponse+from skyvern.client import GetRunResponsefrom skyvern.client.types.workflow_run_response import WorkflowRunResponse+from skyvern.config import settingsfrom skyvern.library.constants import DEFAULT_AGENT_HEARTBEAT_INTERVAL, DEFAULT_AGENT_TIMEOUT+from skyvern.library.SdkSkyvernPageAi import SdkSkyvernPageAi+from skyvern.webeye.actions import handler_utils++if TYPE_CHECKING:+ from skyvern.library.skyvern_browser import SkyvernBrowser+from skyvern.schemas.run_blocks import CredentialTypefrom skyvern.schemas.runs import RunEngine, RunStatus, TaskRunResponse@@ -18,13 +25,9 @@ class SkyvernPageRun:and pre-defined workflows with automatic waiting for completion."""- def __init__(- self, page: Page, browser_session_id: str | None, browser_address: str | None, client: AsyncSkyvern- ) -> None:+ def __init__(self, browser: "SkyvernBrowser", page: Page) -> None:+ self._browser = browserself._page = page- self._browser_session_id = browser_session_id- self._browser_address = browser_address- self._client = clientasync def run_task(self,@@ -63,7 +66,7 @@ class SkyvernPageRun:TaskRunResponse containing the task execution results."""- task_run = await self._client.run_task(+ task_run = await self._browser.client.run_task(prompt=prompt,engine=engine,model=model,@@ -75,8 +78,8 @@ class SkyvernPageRun:error_code_mapping=error_code_mapping,data_extraction_schema=data_extraction_schema,max_steps=max_steps,- browser_session_id=self._browser_session_id,- browser_address=self._browser_address,+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,user_agent=user_agent,)@@ -121,7 +124,7 @@ class SkyvernPageRun:WorkflowRunResponse containing the login workflow execution results."""- workflow_run = await self._client.login(+ workflow_run = await self._browser.client.login(credential_type=credential_type,url=url or self._get_page_url(),credential_id=credential_id,@@ -133,8 +136,8 @@ class SkyvernPageRun:webhook_url=webhook_url,totp_identifier=totp_identifier,totp_url=totp_url,- browser_session_id=self._browser_session_id,- browser_address=self._browser_address,+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,extra_http_headers=extra_http_headers,)@@ -167,7 +170,7 @@ class SkyvernPageRun:Returns:WorkflowRunResponse containing the workflow execution results."""- workflow_run = await self._client.run_workflow(+ workflow_run = await self._browser.client.run_workflow(workflow_id=workflow_id,parameters=parameters,template=template,@@ -175,8 +178,8 @@ class SkyvernPageRun:webhook_url=webhook_url,totp_url=totp_url,totp_identifier=totp_identifier,- browser_session_id=self._browser_session_id,- browser_address=self._browser_address,+ browser_session_id=self._browser.browser_session_id,+ browser_address=self._browser.browser_address,)workflow_run = await self._wait_for_run_completion(workflow_run.run_id, timeout)@@ -185,7 +188,7 @@ class SkyvernPageRun:async def _wait_for_run_completion(self, run_id: str, timeout: float) -> GetRunResponse:async with asyncio.timeout(timeout):while True:- task_run = await self._client.get_run(run_id)+ task_run = await self._browser.client.get_run(run_id)if RunStatus(task_run.status).is_final():breakawait asyncio.sleep(DEFAULT_AGENT_HEARTBEAT_INTERVAL)@@ -221,28 +224,150 @@ class SkyvernBrowserPage:run: SkyvernPageRun instance for executing AI-powered tasks and workflows."""- def __init__(self, page: Page, run: SkyvernPageRun):- self.run = run- self._playwright_page = page-- async def click(self, selector: str, **kwargs: Any) -> None:- """Click an element matching the selector.+ def __init__(self, browser: "SkyvernBrowser", page: Page):+ self._browser = browser+ self._page = page+ self._ai = SdkSkyvernPageAi(browser, page)+ self.run = SkyvernPageRun(browser, page)- Args:- selector: A selector to search for an element to click.- **kwargs: Additional options like timeout, force, position, etc.+ async def click(+ self,+ *,+ selector: str | None = None,+ intention: str | None = None,+ ai: str | None = "fallback",+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str | None:+ """Click an element identified by ``selector``.++ When ``intention`` and ``data`` are provided a new click action is+ generated via the ``single-click-action`` prompt. The model returns a+ fresh "xpath=..." selector based on the current DOM and the updated data for this run.+ The browser then clicks the element using this newly generated xpath selector.++ If the prompt generation or parsing fails for any reason we fall back to+ clicking the originally supplied ``selector``."""- await self._playwright_page.click(selector, **kwargs)- async def fill(self, selector: str, value: str, **kwargs: Any) -> None:- """Fill an input field with the given value.+ if ai == "fallback":+ # try to click the element with the original selector first+ error_to_raise = None+ if selector:+ try:+ locator = self._page.locator(selector)+ await locator.click(timeout=timeout)+ return selector+ except Exception as e:+ error_to_raise = e++ # if the original selector doesn't work, try to click the element with the ai generated selector+ if intention:+ return await self._ai.ai_click(+ selector=selector or "",+ intention=intention,+ data=data,+ timeout=timeout,+ )+ if error_to_raise:+ raise error_to_raise+ else:+ return selector+ elif ai == "proactive":+ if intention:+ return await self._ai.ai_click(+ selector=selector or "",+ intention=intention,+ data=data,+ timeout=timeout,+ )++ if selector:+ locator = self._page.locator(selector)+ await locator.click(timeout=timeout)+ return selector++ async def _input_text(+ self,+ selector: str,+ value: str,+ ai: str | None = "fallback",+ intention: str | None = None,+ data: str | dict[str, Any] | None = None,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> str:+ """Input text into an element identified by ``selector``.++ When ``intention`` and ``data`` are provided a new input text action is+ generated via the `script-generation-input-text-generation` prompt. The model returns a+ fresh text based on the current DOM and the updated data for this run.+ The browser then inputs the text using this newly generated text.- Args:- selector: A selector to search for an element to fill.- value: Value to fill for the input field.- **kwargs: Additional options like timeout, force, no_wait_after, etc.+ If the prompt generation or parsing fails for any reason we fall back to+ inputting the originally supplied ``value``."""- await self._playwright_page.fill(selector, value, **kwargs)++ # format the text with the actual value of the parameter if it's a secret when running a workflow+ if ai == "fallback":+ error_to_raise = None+ try:+ locator = self._page.locator(selector)+ await handler_utils.input_sequentially(locator, value, timeout=timeout)+ return value+ except Exception as e:+ error_to_raise = e++ if intention:+ return await self._ai.ai_input_text(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ timeout=timeout,+ )+ if error_to_raise:+ raise error_to_raise+ else:+ return value+ elif ai == "proactive" and intention:+ return await self._ai.ai_input_text(+ selector=selector,+ value=value,+ intention=intention,+ data=data,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ timeout=timeout,+ )+ locator = self._page.locator(selector)+ await handler_utils.input_sequentially(locator, value, timeout=timeout)+ return value++ async def fill(+ self,+ selector: str,+ value: str,+ ai: str | None = "fallback",+ intention: str | None = None,+ data: str | dict[str, Any] | None = None,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ totp_identifier: str | None = None,+ totp_url: str | None = None,+ ) -> str:+ return await self._input_text(+ selector=selector,+ value=value,+ ai=ai,+ intention=intention,+ data=data,+ timeout=timeout,+ totp_identifier=totp_identifier,+ totp_url=totp_url,+ )async def goto(self, url: str, **kwargs: Any) -> None:"""Navigate to the given URL.@@ -251,7 +376,7 @@ class SkyvernBrowserPage:url: URL to navigate page to.**kwargs: Additional options like timeout, wait_until, referer, etc."""- await self._playwright_page.goto(url, **kwargs)+ await self._page.goto(url, **kwargs)async def type(self, selector: str, text: str, **kwargs: Any) -> None:"""Type text into an element character by character.@@ -261,7 +386,7 @@ class SkyvernBrowserPage:text: Text to type into the element.**kwargs: Additional options like delay, timeout, no_wait_after, etc."""- await self._playwright_page.type(selector, text, **kwargs)+ await self._page.type(selector, text, **kwargs)async def select_option(self, selector: str, value: Any = None, **kwargs: Any) -> list[str]:"""Select option(s) in a <select> element.@@ -274,7 +399,7 @@ class SkyvernBrowserPage:Returns:List of option values that have been successfully selected."""- return await self._playwright_page.select_option(selector, value, **kwargs)+ return await self._page.select_option(selector, value, **kwargs)async def reload(self, **kwargs: Any) -> None:"""Reload the current page.@@ -282,7 +407,7 @@ class SkyvernBrowserPage:Args:**kwargs: Additional options like timeout, wait_until, etc."""- await self._playwright_page.reload(**kwargs)+ await self._page.reload(**kwargs)async def screenshot(self, **kwargs: Any) -> bytes:"""Take a screenshot of the page.@@ -293,4 +418,4 @@ class SkyvernBrowserPage:Returns:bytes: The screenshot as bytes (unless path is specified, then saves to file)."""- return await self._playwright_page.screenshot(**kwargs)+ return await self._page.screenshot(**kwargs)