Skip to content

Automate browser based workflows with AI

SDK: support select_option and extract (#3850)

Stanislav Novosadaf9a5f31e4c4
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.json
index b02ea9077..fb25bde35 100644
--- a/fern/openapi/skyvern_openapi.json
+++ b/fern/openapi/skyvern_openapi.json
@@ -2317,6 +2317,9 @@
"403": {
"description": "Unauthorized - Invalid or missing authentication"
},
+ "404": {
+ "description": "Workflow run or workflow not found"
+ },
"400": {
"description": "Invalid operation"
},
@@ -9733,6 +9736,26 @@
"description": "The workflow run ID used for this action"
},
"result": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "object"
+ },
+ {
+ "type": "array"
+ },
+ {
+ "type": "number"
+ },
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
"title": "Result",
"description": "The result from the action (e.g., selector, value, extracted data)"
}
diff --git a/skyvern-ts/client/reference.md b/skyvern-ts/client/reference.md
index 99422970d..64f98f806 100644
--- a/skyvern-ts/client/reference.md
+++ b/skyvern-ts/client/reference.md
@@ -1,5 +1,5 @@
# Reference
-<details><summary><code>client.<a href="/src/Client.ts">deployScript</a>(scriptId, { ...params }) -> Skyvern.CreateScriptResponse</code></summary>
+<details><summary><code>client.<a href="/src/Client.ts">runSdkAction</a>({ ...params }) -> Skyvern.RunSdkActionResponse</code></summary>
<dl>
<dd>
@@ -11,7 +11,7 @@
<dl>
<dd>
-Deploy a script with updated files, creating a new version
+Execute a single SDK action with the specified parameters
</dd>
</dl>
</dd>
@@ -26,11 +26,12 @@ Deploy a script with updated files, creating a new version
<dd>
```typescript
-await client.deployScript("s_abc123", {
- files: [{
- path: "src/main.py",
- content: "content"
- }]
+await client.runSdkAction({
+ "x-user-agent": "x-user-agent",
+ url: "url",
+ action: {
+ type: "ai_click"
+ }
});
```
@@ -47,15 +48,7 @@ await client.deployScript("s_abc123", {
<dl>
<dd>
-**scriptId:** `string` — The unique identifier of the script
-
-</dd>
-</dl>
-
-<dl>
-<dd>
-
-**request:** `Skyvern.DeployScriptRequest`
+**request:** `Skyvern.RunSdkActionRequest`
</dd>
</dl>
diff --git a/skyvern-ts/client/src/Client.ts b/skyvern-ts/client/src/Client.ts
index 908763565..35735e939 100644
--- a/skyvern-ts/client/src/Client.ts
+++ b/skyvern-ts/client/src/Client.ts
@@ -2202,4 +2202,101 @@ export class SkyvernClient {
});
}
}
+
+ /**
+ * Execute a single SDK action with the specified parameters
+ *
+ * @param {Skyvern.RunSdkActionRequest} request
+ * @param {SkyvernClient.RequestOptions} requestOptions - Request-specific configuration.
+ *
+ * @throws {@link Skyvern.BadRequestError}
+ * @throws {@link Skyvern.ForbiddenError}
+ * @throws {@link Skyvern.NotFoundError}
+ * @throws {@link Skyvern.UnprocessableEntityError}
+ *
+ * @example
+ * await client.runSdkAction({
+ * "x-user-agent": "x-user-agent",
+ * url: "url",
+ * action: {
+ * type: "ai_click"
+ * }
+ * })
+ */
+ public runSdkAction(
+ request: Skyvern.RunSdkActionRequest,
+ requestOptions?: SkyvernClient.RequestOptions,
+ ): core.HttpResponsePromise<Skyvern.RunSdkActionResponse> {
+ return core.HttpResponsePromise.fromPromise(this.__runSdkAction(request, requestOptions));
+ }
+
+ private async __runSdkAction(
+ request: Skyvern.RunSdkActionRequest,
+ requestOptions?: SkyvernClient.RequestOptions,
+ ): Promise<core.WithRawResponse<Skyvern.RunSdkActionResponse>> {
+ const { "x-user-agent": userAgent, ..._body } = request;
+ const _headers: core.Fetcher.Args["headers"] = mergeHeaders(
+ this._options?.headers,
+ mergeOnlyDefinedHeaders({
+ "x-user-agent": userAgent != null ? userAgent : undefined,
+ "x-api-key": requestOptions?.apiKey ?? this._options?.apiKey,
+ }),
+ requestOptions?.headers,
+ );
+ const _response = await core.fetcher({
+ url: core.url.join(
+ (await core.Supplier.get(this._options.baseUrl)) ??
+ (await core.Supplier.get(this._options.environment)) ??
+ environments.SkyvernEnvironment.Production,
+ "v1/sdk/run_action",
+ ),
+ method: "POST",
+ headers: _headers,
+ contentType: "application/json",
+ queryParameters: requestOptions?.queryParams,
+ requestType: "json",
+ body: _body,
+ timeoutMs: (requestOptions?.timeoutInSeconds ?? this._options?.timeoutInSeconds ?? 60) * 1000,
+ maxRetries: requestOptions?.maxRetries ?? this._options?.maxRetries,
+ abortSignal: requestOptions?.abortSignal,
+ });
+ if (_response.ok) {
+ return { data: _response.body as Skyvern.RunSdkActionResponse, rawResponse: _response.rawResponse };
+ }
+
+ if (_response.error.reason === "status-code") {
+ switch (_response.error.statusCode) {
+ case 400:
+ throw new Skyvern.BadRequestError(_response.error.body as unknown, _response.rawResponse);
+ case 403:
+ throw new Skyvern.ForbiddenError(_response.error.body as unknown, _response.rawResponse);
+ case 404:
+ throw new Skyvern.NotFoundError(_response.error.body as unknown, _response.rawResponse);
+ case 422:
+ throw new Skyvern.UnprocessableEntityError(_response.error.body as unknown, _response.rawResponse);
+ default:
+ throw new errors.SkyvernError({
+ statusCode: _response.error.statusCode,
+ body: _response.error.body,
+ rawResponse: _response.rawResponse,
+ });
+ }
+ }
+
+ switch (_response.error.reason) {
+ case "non-json":
+ throw new errors.SkyvernError({
+ statusCode: _response.error.statusCode,
+ body: _response.error.rawBody,
+ rawResponse: _response.rawResponse,
+ });
+ case "timeout":
+ throw new errors.SkyvernTimeoutError("Timeout exceeded when calling POST /v1/sdk/run_action.");
+ case "unknown":
+ throw new errors.SkyvernError({
+ message: _response.error.errorMessage,
+ rawResponse: _response.rawResponse,
+ });
+ }
+ }
}
diff --git a/skyvern-ts/client/src/api/client/requests/RunSdkActionRequest.ts b/skyvern-ts/client/src/api/client/requests/RunSdkActionRequest.ts
new file mode 100644
index 000000000..3ffc9f70b
--- /dev/null
+++ b/skyvern-ts/client/src/api/client/requests/RunSdkActionRequest.ts
@@ -0,0 +1,27 @@
+// This file was auto-generated by Fern from our API Definition.
+
+import type * as Skyvern from "../../index.js";
+
+/**
+ * @example
+ * {
+ * "x-user-agent": "x-user-agent",
+ * url: "url",
+ * action: {
+ * type: "ai_click"
+ * }
+ * }
+ */
+export interface RunSdkActionRequest {
+ "x-user-agent"?: string;
+ /** The URL where the action should be executed */
+ url: string;
+ /** The browser session ID */
+ browser_session_id?: string;
+ /** The browser address */
+ browser_address?: string;
+ /** Optional workflow run ID to continue an existing workflow run */
+ workflow_run_id?: string;
+ /** The action to execute with its specific parameters */
+ action: Skyvern.SdkAction;
+}
diff --git a/skyvern-ts/client/src/api/client/requests/index.ts b/skyvern-ts/client/src/api/client/requests/index.ts
index aca2642c3..42868e3eb 100644
--- a/skyvern-ts/client/src/api/client/requests/index.ts
+++ b/skyvern-ts/client/src/api/client/requests/index.ts
@@ -7,6 +7,7 @@ export type { GetRunArtifactsRequest } from "./GetRunArtifactsRequest.js";
export type { GetScriptsRequest } from "./GetScriptsRequest.js";
export type { GetWorkflowsRequest } from "./GetWorkflowsRequest.js";
export type { LoginRequest } from "./LoginRequest.js";
+export type { RunSdkActionRequest } from "./RunSdkActionRequest.js";
export type { RunTaskRequest } from "./RunTaskRequest.js";
export type { RunWorkflowRequest } from "./RunWorkflowRequest.js";
export type { TotpCodeCreate } from "./TotpCodeCreate.js";
diff --git a/skyvern-ts/client/src/api/types/ClickAction.ts b/skyvern-ts/client/src/api/types/ClickAction.ts
new file mode 100644
index 000000000..4983ff0ff
--- /dev/null
+++ b/skyvern-ts/client/src/api/types/ClickAction.ts
@@ -0,0 +1,19 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface ClickAction {
+ /** CSS selector for the element */
+ selector?: string;
+ /** The intention or goal of the click */
+ intention?: string;
+ /** Additional context data */
+ data?: ClickAction.Data;
+ /** Timeout in milliseconds */
+ timeout?: number;
+}
+
+export namespace ClickAction {
+ /**
+ * Additional context data
+ */
+ export type Data = string | Record<string, unknown>;
+}
diff --git a/skyvern-ts/client/src/api/types/ExtractAction.ts b/skyvern-ts/client/src/api/types/ExtractAction.ts
new file mode 100644
index 000000000..6382cbbc3
--- /dev/null
+++ b/skyvern-ts/client/src/api/types/ExtractAction.ts
@@ -0,0 +1,25 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface ExtractAction {
+ /** Extraction prompt */
+ prompt?: string;
+ /** Schema for extraction */
+ extract_schema?: ExtractAction.ExtractSchema;
+ /** Error code mapping for extraction */
+ error_code_mapping?: Record<string, string | undefined>;
+ /** The intention or goal of the extraction */
+ intention?: string;
+ /** Additional context data */
+ data?: ExtractAction.Data;
+}
+
+export namespace ExtractAction {
+ /**
+ * Schema for extraction
+ */
+ export type ExtractSchema = Record<string, unknown> | unknown[] | string;
+ /**
+ * Additional context data
+ */
+ export type Data = string | Record<string, unknown>;
+}
diff --git a/skyvern-ts/client/src/api/types/InputTextAction.ts b/skyvern-ts/client/src/api/types/InputTextAction.ts
new file mode 100644
index 000000000..695615eca
--- /dev/null
+++ b/skyvern-ts/client/src/api/types/InputTextAction.ts
@@ -0,0 +1,25 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface InputTextAction {
+ /** CSS selector for the element */
+ selector?: string;
+ /** Value to input */
+ value?: string;
+ /** The intention or goal of the input */
+ intention?: string;
+ /** Additional context data */
+ data?: InputTextAction.Data;
+ /** TOTP identifier for input_text actions */
+ totp_identifier?: string;
+ /** TOTP URL for input_text actions */
+ totp_url?: string;
+ /** Timeout in milliseconds */
+ timeout?: number;
+}
+
+export namespace InputTextAction {
+ /**
+ * Additional context data
+ */
+ export type Data = string | Record<string, unknown>;
+}
diff --git a/skyvern-ts/client/src/api/types/RunSdkActionResponse.ts b/skyvern-ts/client/src/api/types/RunSdkActionResponse.ts
new file mode 100644
index 000000000..c3c4352e2
--- /dev/null
+++ b/skyvern-ts/client/src/api/types/RunSdkActionResponse.ts
@@ -0,0 +1,15 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface RunSdkActionResponse {
+ /** The workflow run ID used for this action */
+ workflow_run_id: string;
+ /** The result from the action (e.g., selector, value, extracted data) */
+ result?: RunSdkActionResponse.Result;
+}
+
+export namespace RunSdkActionResponse {
+ /**
+ * The result from the action (e.g., selector, value, extracted data)
+ */
+ export type Result = string | Record<string, unknown> | unknown[] | number | boolean;
+}
diff --git a/skyvern-ts/client/src/api/types/SdkAction.ts b/skyvern-ts/client/src/api/types/SdkAction.ts
new file mode 100644
index 000000000..2bb07cba1
--- /dev/null
+++ b/skyvern-ts/client/src/api/types/SdkAction.ts
@@ -0,0 +1,27 @@
+// This file was auto-generated by Fern from our API Definition.
+
+import type * as Skyvern from "../index.js";
+
+export type SdkAction =
+ | Skyvern.SdkAction.AiClick
+ | Skyvern.SdkAction.AiInputText
+ | Skyvern.SdkAction.AiSelectOption
+ | Skyvern.SdkAction.Extract;
+
+export namespace SdkAction {
+ export interface AiClick extends Skyvern.ClickAction {
+ type: "ai_click";
+ }
+
+ export interface AiInputText extends Skyvern.InputTextAction {
+ type: "ai_input_text";
+ }
+
+ export interface AiSelectOption extends Skyvern.SelectOptionAction {
+ type: "ai_select_option";
+ }
+
+ export interface Extract extends Skyvern.ExtractAction {
+ type: "extract";
+ }
+}
diff --git a/skyvern-ts/client/src/api/types/SelectOptionAction.ts b/skyvern-ts/client/src/api/types/SelectOptionAction.ts
new file mode 100644
index 000000000..966ab0944
--- /dev/null
+++ b/skyvern-ts/client/src/api/types/SelectOptionAction.ts
@@ -0,0 +1,21 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface SelectOptionAction {
+ /** CSS selector for the element */
+ selector?: string;
+ /** Value to select */
+ value?: string;
+ /** The intention or goal of the selection */
+ intention?: string;
+ /** Additional context data */
+ data?: SelectOptionAction.Data;
+ /** Timeout in milliseconds */
+ timeout?: number;
+}
+
+export namespace SelectOptionAction {
+ /**
+ * Additional context data
+ */
+ export type Data = string | Record<string, unknown>;
+}
diff --git a/skyvern-ts/client/src/api/types/index.ts b/skyvern-ts/client/src/api/types/index.ts
index 26eca5852..07bbca998 100644
--- a/skyvern-ts/client/src/api/types/index.ts
+++ b/skyvern-ts/client/src/api/types/index.ts
@@ -19,6 +19,7 @@ export * from "./BitwardenSensitiveInformationParameter.js";
export * from "./BitwardenSensitiveInformationParameterYaml.js";
export * from "./BlockType.js";
export * from "./BrowserSessionResponse.js";
+export * from "./ClickAction.js";
export * from "./CodeBlock.js";
export * from "./CodeBlockParametersItem.js";
export * from "./CodeBlockYaml.js";
@@ -33,6 +34,7 @@ export * from "./CredentialTypeOutput.js";
export * from "./CreditCardCredentialResponse.js";
export * from "./DownloadToS3Block.js";
export * from "./DownloadToS3BlockYaml.js";
+export * from "./ExtractAction.js";
export * from "./ExtractionBlock.js";
export * from "./ExtractionBlockParametersItem.js";
export * from "./ExtractionBlockYaml.js";
@@ -62,6 +64,7 @@ export * from "./HumanInteractionBlock.js";
export * from "./HumanInteractionBlockParametersItem.js";
export * from "./HumanInteractionBlockYaml.js";
export * from "./InputOrSelectContext.js";
+export * from "./InputTextAction.js";
export * from "./LoginBlock.js";
export * from "./LoginBlockParametersItem.js";
export * from "./LoginBlockYaml.js";
@@ -80,11 +83,14 @@ export * from "./PdfParserBlock.js";
export * from "./PdfParserBlockYaml.js";
export * from "./ProxyLocation.js";
export * from "./RunEngine.js";
+export * from "./RunSdkActionResponse.js";
export * from "./RunStatus.js";
export * from "./Script.js";
export * from "./ScriptFileCreate.js";
export * from "./ScriptRunResponse.js";
+export * from "./SdkAction.js";
export * from "./SelectOption.js";
+export * from "./SelectOptionAction.js";
export * from "./SendEmailBlock.js";
export * from "./SendEmailBlockYaml.js";
export * from "./SkyvernForgeSdkSchemasCredentialsCredentialType.js";
diff --git a/skyvern-ts/client/tests/wire/main.test.ts b/skyvern-ts/client/tests/wire/main.test.ts
index 6a5e178bd..7fc0a08aa 100644
--- a/skyvern-ts/client/tests/wire/main.test.ts
+++ b/skyvern-ts/client/tests/wire/main.test.ts
@@ -2429,4 +2429,128 @@ describe("SkyvernClient", () => {
});
}).rejects.toThrow(Skyvern.UnprocessableEntityError);
});
+
+ test("run_sdk_action (1)", async () => {
+ const server = mockServerPool.createServer();
+ const client = new SkyvernClient({ apiKey: "test", environment: server.baseUrl });
+ const rawRequestBody = { url: "url", action: { type: "ai_click" } };
+ const rawResponseBody = { workflow_run_id: "workflow_run_id", result: "result" };
+ server
+ .mockEndpoint()
+ .post("/v1/sdk/run_action")
+ .header("x-user-agent", "x-user-agent")
+ .jsonBody(rawRequestBody)
+ .respondWith()
+ .statusCode(200)
+ .jsonBody(rawResponseBody)
+ .build();
+
+ const response = await client.runSdkAction({
+ "x-user-agent": "x-user-agent",
+ url: "url",
+ action: {
+ type: "ai_click",
+ },
+ });
+ expect(response).toEqual({
+ workflow_run_id: "workflow_run_id",
+ result: "result",
+ });
+ });
+
+ test("run_sdk_action (2)", async () => {
+ const server = mockServerPool.createServer();
+ const client = new SkyvernClient({ apiKey: "test", environment: server.baseUrl });
+ const rawRequestBody = { url: "url", action: { type: "ai_click" } };
+ const rawResponseBody = { key: "value" };
+ server
+ .mockEndpoint()
+ .post("/v1/sdk/run_action")
+ .jsonBody(rawRequestBody)
+ .respondWith()
+ .statusCode(400)
+ .jsonBody(rawResponseBody)
+ .build();
+
+ await expect(async () => {
+ return await client.runSdkAction({
+ url: "url",
+ action: {
+ type: "ai_click",
+ },
+ });
+ }).rejects.toThrow(Skyvern.BadRequestError);
+ });
+
+ test("run_sdk_action (3)", async () => {
+ const server = mockServerPool.createServer();
+ const client = new SkyvernClient({ apiKey: "test", environment: server.baseUrl });
+ const rawRequestBody = { url: "url", action: { type: "ai_click" } };
+ const rawResponseBody = { key: "value" };
+ server
+ .mockEndpoint()
+ .post("/v1/sdk/run_action")
+ .jsonBody(rawRequestBody)
+ .respondWith()
+ .statusCode(403)
+ .jsonBody(rawResponseBody)
+ .build();
+
+ await expect(async () => {
+ return await client.runSdkAction({
+ url: "url",
+ action: {
+ type: "ai_click",
+ },
+ });
+ }).rejects.toThrow(Skyvern.ForbiddenError);
+ });
+
+ test("run_sdk_action (4)", async () => {
+ const server = mockServerPool.createServer();
+ const client = new SkyvernClient({ apiKey: "test", environment: server.baseUrl });
+ const rawRequestBody = { url: "url", action: { type: "ai_click" } };
+ const rawResponseBody = { key: "value" };
+ server
+ .mockEndpoint()
+ .post("/v1/sdk/run_action")
+ .jsonBody(rawRequestBody)
+ .respondWith()
+ .statusCode(404)
+ .jsonBody(rawResponseBody)
+ .build();
+
+ await expect(async () => {
+ return await client.runSdkAction({
+ url: "url",
+ action: {
+ type: "ai_click",
+ },
+ });
+ }).rejects.toThrow(Skyvern.NotFoundError);
+ });
+
+ test("run_sdk_action (5)", async () => {
+ const server = mockServerPool.createServer();
+ const client = new SkyvernClient({ apiKey: "test", environment: server.baseUrl });
+ const rawRequestBody = { url: "url", action: { type: "ai_click" } };
+ const rawResponseBody = { key: "value" };
+ server
+ .mockEndpoint()
+ .post("/v1/sdk/run_action")
+ .jsonBody(rawRequestBody)
+ .respondWith()
+ .statusCode(422)
+ .jsonBody(rawResponseBody)
+ .build();
+
+ await expect(async () => {
+ return await client.runSdkAction({
+ url: "url",
+ action: {
+ type: "ai_click",
+ },
+ });
+ }).rejects.toThrow(Skyvern.UnprocessableEntityError);
+ });
});
diff --git a/skyvern/client/__init__.py b/skyvern/client/__init__.py
index 8048b0950..b2b87372c 100644
--- a/skyvern/client/__init__.py
+++ b/skyvern/client/__init__.py
@@ -265,6 +265,7 @@ if typing.TYPE_CHECKING:
ProxyLocation,
RunEngine,
RunSdkActionResponse,
+ RunSdkActionResponseResult,
RunStatus,
Script,
ScriptFileCreate,
@@ -727,6 +728,7 @@ _dynamic_imports: typing.Dict[str, str] = {
"ProxyLocation": ".types",
"RunEngine": ".types",
"RunSdkActionResponse": ".types",
+ "RunSdkActionResponseResult": ".types",
"RunStatus": ".types",
"Script": ".types",
"ScriptFileCreate": ".types",
@@ -1212,6 +1214,7 @@ __all__ = [
"ProxyLocation",
"RunEngine",
"RunSdkActionResponse",
+ "RunSdkActionResponseResult",
"RunStatus",
"Script",
"ScriptFileCreate",
diff --git a/skyvern/client/raw_client.py b/skyvern/client/raw_client.py
index feb8b1ea0..87193ec51 100644
--- a/skyvern/client/raw_client.py
+++ b/skyvern/client/raw_client.py
@@ -2146,6 +2146,17 @@ class RawSkyvern:
),
),
)
+ if _response.status_code == 404:
+ raise NotFoundError(
+ headers=dict(_response.headers),
+ body=typing.cast(
+ typing.Optional[typing.Any],
+ parse_obj_as(
+ type_=typing.Optional[typing.Any], # type: ignore
+ object_=_response.json(),
+ ),
+ ),
+ )
if _response.status_code == 422:
raise UnprocessableEntityError(
headers=dict(_response.headers),
@@ -4267,6 +4278,17 @@ class AsyncRawSkyvern:
),
),
)
+ if _response.status_code == 404:
+ raise NotFoundError(
+ headers=dict(_response.headers),
+ body=typing.cast(
+ typing.Optional[typing.Any],
+ parse_obj_as(
+ type_=typing.Optional[typing.Any], # type: ignore
+ object_=_response.json(),
+ ),
+ ),
+ )
if _response.status_code == 422:
raise UnprocessableEntityError(
headers=dict(_response.headers),
diff --git a/skyvern/client/types/__init__.py b/skyvern/client/types/__init__.py
index 44001b535..0310da7b7 100644
--- a/skyvern/client/types/__init__.py
+++ b/skyvern/client/types/__init__.py
@@ -290,6 +290,7 @@ if typing.TYPE_CHECKING:
from .proxy_location import ProxyLocation
from .run_engine import RunEngine
from .run_sdk_action_response import RunSdkActionResponse
+ from .run_sdk_action_response_result import RunSdkActionResponseResult
from .run_status import RunStatus
from .script import Script
from .script_file_create import ScriptFileCreate
@@ -762,6 +763,7 @@ _dynamic_imports: typing.Dict[str, str] = {
"ProxyLocation": ".proxy_location",
"RunEngine": ".run_engine",
"RunSdkActionResponse": ".run_sdk_action_response",
+ "RunSdkActionResponseResult": ".run_sdk_action_response_result",
"RunStatus": ".run_status",
"Script": ".script",
"ScriptFileCreate": ".script_file_create",
@@ -1238,6 +1240,7 @@ __all__ = [
"ProxyLocation",
"RunEngine",
"RunSdkActionResponse",
+ "RunSdkActionResponseResult",
"RunStatus",
"Script",
"ScriptFileCreate",
diff --git a/skyvern/client/types/run_sdk_action_response.py b/skyvern/client/types/run_sdk_action_response.py
index 065f19a80..845d393f9 100644
--- a/skyvern/client/types/run_sdk_action_response.py
+++ b/skyvern/client/types/run_sdk_action_response.py
@@ -4,6 +4,7 @@ import typing
import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
+from .run_sdk_action_response_result import RunSdkActionResponseResult
class RunSdkActionResponse(UniversalBaseModel):
@@ -12,7 +13,7 @@ class RunSdkActionResponse(UniversalBaseModel):
The workflow run ID used for this action
"""
- result: typing.Optional[typing.Optional[typing.Any]] = pydantic.Field(default=None)
+ result: typing.Optional[RunSdkActionResponseResult] = pydantic.Field(default=None)
"""
The result from the action (e.g., selector, value, extracted data)
"""
diff --git a/skyvern/client/types/run_sdk_action_response_result.py b/skyvern/client/types/run_sdk_action_response_result.py
new file mode 100644
index 000000000..b771d2519
--- /dev/null
+++ b/skyvern/client/types/run_sdk_action_response_result.py
@@ -0,0 +1,7 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+RunSdkActionResponseResult = typing.Union[
+ str, typing.Dict[str, typing.Optional[typing.Any]], typing.List[typing.Optional[typing.Any]], float, bool
+]
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.py
index bfb9ecbbb..c6a2afbe4 100644
--- a/skyvern/forge/sdk/routes/sdk.py
+++ b/skyvern/forge/sdk/routes/sdk.py
@@ -1,4 +1,4 @@
-import json
+from typing import Any
import structlog
from fastapi import Depends, HTTPException, status
@@ -95,8 +95,8 @@ async def run_sdk_action(
task = await app.DATABASE.create_task(
organization_id=organization_id,
url=action_request.url,
- navigation_goal=None,
- navigation_payload=None,
+ navigation_goal=action.intention,
+ navigation_payload=action.data,
data_extraction_goal=None,
title=f"SDK Action Task: {action_request.action.type}",
workflow_run_id=workflow_run.workflow_run_id,
@@ -118,6 +118,18 @@ async def run_sdk_action(
task_id=task.task_id,
)
+ await app.WORKFLOW_CONTEXT_MANAGER.initialize_workflow_run_context(
+ organization,
+ workflow_run.workflow_run_id,
+ workflow.title,
+ workflow.workflow_id,
+ workflow.workflow_permanent_id,
+ [],
+ [],
+ [],
+ [],
+ )
+
context = skyvern_context.ensure_context()
skyvern_context.set(
SkyvernContext(
@@ -131,7 +143,7 @@ async def run_sdk_action(
workflow_run_id=workflow_run.workflow_run_id,
)
)
- result = None
+ result: Any | None = None
try:
scraped_page = await SkyvernPage.create_scraped_page(browser_session_id=browser_session_id)
page = await scraped_page._browser_state.must_get_working_page()
@@ -170,7 +182,7 @@ async def run_sdk_action(
intention=action.intention,
data=action.data,
)
- result = json.dumps(extract_result)
+ result = extract_result
finally:
skyvern_context.reset()
diff --git a/skyvern/library/SdkSkyvernPageAi.py b/skyvern/library/SdkSkyvernPageAi.py
index c3a412987..8c766b7c4 100644
--- a/skyvern/library/SdkSkyvernPageAi.py
+++ b/skyvern/library/SdkSkyvernPageAi.py
@@ -2,14 +2,9 @@ from typing import TYPE_CHECKING, Any
from playwright.async_api import Page
+from skyvern.client import SdkAction_AiClick, SdkAction_AiInputText, SdkAction_AiSelectOption, SdkAction_Extract
from skyvern.config import settings
from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi
-from skyvern.forge.sdk.schemas.sdk_actions import (
- ClickAction,
- ExtractAction,
- InputTextAction,
- SelectOptionAction,
-)
if TYPE_CHECKING:
from skyvern.library.skyvern_browser import SkyvernBrowser
@@ -35,18 +30,17 @@ class SdkSkyvernPageAi(SkyvernPageAi):
) -> str:
"""Click an element using AI via API call."""
- action = ClickAction(
- selector=selector,
- intention=intention,
- data=data,
- timeout=timeout,
- )
response = await self._browser.client.run_sdk_action(
url=self._page.url,
browser_session_id=self._browser.browser_session_id,
browser_address=self._browser.browser_address,
workflow_run_id=self._browser.workflow_run_id,
- action=action,
+ action=SdkAction_AiClick(
+ selector=selector,
+ intention=intention,
+ data=data,
+ timeout=timeout,
+ ),
)
self._browser.workflow_run_id = response.workflow_run_id
return response.result if response.result else selector
@@ -65,7 +59,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):
response = await self._browser.client.run_sdk_action(
url=self._page.url,
- action=InputTextAction(
+ action=SdkAction_AiInputText(
selector=selector,
value=value,
intention=intention,
@@ -93,7 +87,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):
response = await self._browser.client.run_sdk_action(
url=self._page.url,
- action=SelectOptionAction(
+ action=SdkAction_AiSelectOption(
selector=selector,
value=value,
intention=intention,
@@ -129,7 +123,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):
response = await self._browser.client.run_sdk_action(
url=self._page.url,
- action=ExtractAction(
+ action=SdkAction_Extract(
prompt=prompt,
extract_schema=schema,
error_code_mapping=error_code_mapping,
diff --git a/skyvern/library/skyvern_browser_page.py b/skyvern/library/skyvern_browser_page.py
index 447857426..de559398d 100644
--- a/skyvern/library/skyvern_browser_page.py
+++ b/skyvern/library/skyvern_browser_page.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, overload
from playwright.async_api import Page
@@ -230,24 +230,64 @@ class SkyvernBrowserPage:
self._ai = SdkSkyvernPageAi(browser, page)
self.run = SkyvernPageRun(browser, page)
+ @overload
async def click(
self,
+ selector: str,
*,
+ prompt: str | None = None,
+ ai: str | None = "fallback",
+ data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ ) -> str | None: ...
+
+ @overload
+ async def click(
+ self,
+ *,
+ prompt: str,
+ ai: str | None = "fallback",
+ data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ ) -> str | None: ...
+
+ async def click(
+ self,
selector: str | None = None,
- intention: str | None = None,
+ *,
+ prompt: str | None = None,
ai: str | None = "fallback",
data: str | dict[str, Any] | None = None,
timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
) -> str | None:
- """Click an element identified by ``selector``.
+ """Click an element using a CSS selector, AI-powered prompt matching, or both.
- When ``intention`` and ``data`` are provided a new click action is
- generated via the ``single-click-action`` prompt. The model returns a
- fresh "xpath=..." selector based on the current DOM and the updated data for this run.
- The browser then clicks the element using this newly generated xpath selector.
+ This method supports three modes:
+ - **Selector-based**: Click the element matching the CSS selector
+ - **AI-powered**: Use natural language to describe which element to click
+ - **Fallback mode** (default): Try the selector first, fall back to AI if it fails
- If the prompt generation or parsing fails for any reason we fall back to
- clicking the originally supplied ``selector``.
+ Args:
+ selector: CSS selector for the target element.
+ prompt: Natural language description of which element to click.
+ ai: AI behavior mode. Defaults to "fallback" which tries selector first, then AI.
+ data: Additional context data for AI processing.
+ timeout: Maximum time to wait for the click action in milliseconds.
+
+ Returns:
+ The selector string that was successfully used to click the element, or None.
+
+ Examples:
+ ```python
+ # Click using a CSS selector
+ await page.click("#open-invoice-button")
+
+ # Click using AI with natural language
+ await page.click(prompt="Click on the 'Open Invoice' button")
+
+ # Try selector first, fall back to AI if selector fails
+ await page.click("#open-invoice-button", prompt="Click on the 'Open Invoice' button")
+ ```
"""
if ai == "fallback":
@@ -262,10 +302,10 @@ class SkyvernBrowserPage:
error_to_raise = e
# if the original selector doesn't work, try to click the element with the ai generated selector
- if intention:
+ if prompt:
return await self._ai.ai_click(
selector=selector or "",
- intention=intention,
+ intention=prompt,
data=data,
timeout=timeout,
)
@@ -274,10 +314,10 @@ class SkyvernBrowserPage:
else:
return selector
elif ai == "proactive":
- if intention:
+ if prompt:
return await self._ai.ai_click(
selector=selector or "",
- intention=intention,
+ intention=prompt,
data=data,
timeout=timeout,
)
@@ -287,82 +327,87 @@ class SkyvernBrowserPage:
await locator.click(timeout=timeout)
return selector
- async def _input_text(
+ @overload
+ async def fill(
self,
selector: str,
value: str,
+ *,
+ prompt: str | None = None,
ai: str | None = "fallback",
- intention: str | None = None,
data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
totp_identifier: str | None = None,
totp_url: str | None = None,
- timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
- ) -> str:
- """Input text into an element identified by ``selector``.
-
- When ``intention`` and ``data`` are provided a new input text action is
- generated via the `script-generation-input-text-generation` prompt. The model returns a
- fresh text based on the current DOM and the updated data for this run.
- The browser then inputs the text using this newly generated text.
-
- If the prompt generation or parsing fails for any reason we fall back to
- inputting the originally supplied ``value``.
- """
-
- # format the text with the actual value of the parameter if it's a secret when running a workflow
- if ai == "fallback":
- error_to_raise = None
- try:
- locator = self._page.locator(selector)
- await handler_utils.input_sequentially(locator, value, timeout=timeout)
- return value
- except Exception as e:
- error_to_raise = e
+ ) -> str: ...
- if intention:
- return await self._ai.ai_input_text(
- selector=selector,
- value=value,
- intention=intention,
- data=data,
- totp_identifier=totp_identifier,
- totp_url=totp_url,
- timeout=timeout,
- )
- if error_to_raise:
- raise error_to_raise
- else:
- return value
- elif ai == "proactive" and intention:
- return await self._ai.ai_input_text(
- selector=selector,
- value=value,
- intention=intention,
- data=data,
- totp_identifier=totp_identifier,
- totp_url=totp_url,
- timeout=timeout,
- )
- locator = self._page.locator(selector)
- await handler_utils.input_sequentially(locator, value, timeout=timeout)
- return value
+ @overload
+ async def fill(
+ self,
+ *,
+ prompt: str,
+ value: str | None = None,
+ selector: str | None = None,
+ ai: str | None = "fallback",
+ data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ totp_identifier: str | None = None,
+ totp_url: str | None = None,
+ ) -> str: ...
async def fill(
self,
- selector: str,
- value: str,
+ selector: str | None = None,
+ value: str | None = None,
+ *,
+ prompt: str | None = None,
ai: str | None = "fallback",
- intention: str | None = None,
data: str | dict[str, Any] | None = None,
timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
totp_identifier: str | None = None,
totp_url: str | None = None,
) -> str:
+ """Fill an input field using a CSS selector, AI-powered prompt matching, or both.
+
+ This method supports three modes:
+ - **Selector-based**: Fill the input field with a value using CSS selector
+ - **AI-powered**: Use natural language prompt (AI extracts value from prompt)
+ - **Fallback mode** (default): Try the selector first, fall back to AI if it fails
+
+ Args:
+ selector: CSS selector for the target input element.
+ value: The text value to input into the field.
+ prompt: Natural language description of which field to fill and what value.
+ ai: AI behavior mode. Defaults to "fallback" which tries selector first, then AI.
+ data: Additional context data for AI processing.
+ timeout: Maximum time to wait for the fill action in milliseconds.
+ totp_identifier: TOTP identifier for time-based one-time password fields.
+ totp_url: URL to fetch TOTP codes from for authentication.
+
+ Returns:
+ The value that was successfully filled into the field.
+
+ Examples:
+ ```python
+ # Fill using selector and value (both positional)
+ await page.fill("#email-input", "[email protected]")
+
+ # Fill using AI with natural language (prompt only)
+ await page.fill(prompt="Fill '[email protected]' in the email address field")
+
+ # Try selector first, fall back to AI if selector fails
+ await page.fill(
+ "#email-input",
+ prompt="Fill the email address with [email protected]"
+ )
+ ```
+ """
return await self._input_text(
- selector=selector,
- value=value,
+ selector=selector or "",
+ value=value or "",
ai=ai,
- intention=intention,
+ intention=prompt,
data=data,
timeout=timeout,
totp_identifier=totp_identifier,
@@ -388,18 +433,118 @@ class SkyvernBrowserPage:
"""
await self._page.type(selector, text, **kwargs)
- async def select_option(self, selector: str, value: Any = None, **kwargs: Any) -> list[str]:
- """Select option(s) in a <select> element.
+ @overload
+ async def select_option(
+ self,
+ selector: str,
+ value: str,
+ *,
+ prompt: str | None = None,
+ ai: str | None = "fallback",
+ data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ ) -> str: ...
+
+ @overload
+ async def select_option(
+ self,
+ *,
+ prompt: str,
+ value: str | None = None,
+ selector: str | None = None,
+ ai: str | None = "fallback",
+ data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ ) -> str: ...
+
+ async def select_option(
+ self,
+ selector: str | None = None,
+ value: str | None = None,
+ *,
+ prompt: str | None = None,
+ ai: str | None = "fallback",
+ data: str | dict[str, Any] | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ ) -> str:
+ """Select an option from a dropdown using a CSS selector, AI-powered prompt matching, or both.
+
+ This method supports three modes:
+ - **Selector-based**: Select the option with a value using CSS selector
+ - **AI-powered**: Use natural language prompt (AI extracts value from prompt)
+ - **Fallback mode** (default): Try the selector first, fall back to AI if it fails
Args:
- selector: A selector to search for a select element.
- value: Option value(s) to select. Can be a string, list of strings, or dict with value/label/index.
- **kwargs: Additional options like timeout, force, no_wait_after, etc.
+ selector: CSS selector for the target select/dropdown element.
+ value: The option value to select.
+ prompt: Natural language description of which option to select.
+ ai: AI behavior mode. Defaults to "fallback" which tries selector first, then AI.
+ data: Additional context data for AI processing.
+ timeout: Maximum time to wait for the select action in milliseconds.
Returns:
- List of option values that have been successfully selected.
+ The value that was successfully selected.
+
+ Examples:
+ ```python
+ # Select using selector and value (both positional)
+ await page.select_option("#country", "us")
+
+ # Select using AI with natural language (prompt only)
+ await page.select_option(prompt="Select 'United States' from the country dropdown")
+
+ # Try selector first, fall back to AI if selector fails
+ await page.select_option(
+ "#country",
+ "us",
+ prompt="Select United States from country"
+ )
+ ```
"""
- return await self._page.select_option(selector, value, **kwargs)
+ value = value or ""
+ if ai == "fallback":
+ error_to_raise = None
+ if selector:
+ try:
+ locator = self._page.locator(selector)
+ await locator.select_option(value, timeout=timeout)
+ return value
+ except Exception as e:
+ error_to_raise = e
+ if prompt:
+ return await self._ai.ai_select_option(
+ selector=selector or "",
+ value=value,
+ intention=prompt,
+ data=data,
+ timeout=timeout,
+ )
+ if error_to_raise:
+ raise error_to_raise
+ else:
+ return value
+ elif ai == "proactive" and prompt:
+ return await self._ai.ai_select_option(
+ selector=selector or "",
+ value=value,
+ intention=prompt,
+ data=data,
+ timeout=timeout,
+ )
+ if selector:
+ locator = self._page.locator(selector)
+ await locator.select_option(value, timeout=timeout)
+ return value
+
+ async def extract(
+ self,
+ prompt: str,
+ schema: dict[str, Any] | list | str | None = None,
+ error_code_mapping: dict[str, str] | None = None,
+ intention: str | None = None,
+ data: str | dict[str, Any] | None = None,
+ ) -> dict[str, Any] | list | str | None:
+ return await self._ai.ai_extract(prompt, schema, error_code_mapping, intention, data)
async def reload(self, **kwargs: Any) -> None:
"""Reload the current page.
@@ -419,3 +564,63 @@ class SkyvernBrowserPage:
bytes: The screenshot as bytes (unless path is specified, then saves to file).
"""
return await self._page.screenshot(**kwargs)
+
+ async def _input_text(
+ self,
+ selector: str,
+ value: str,
+ ai: str | None = "fallback",
+ intention: str | None = None,
+ data: str | dict[str, Any] | None = None,
+ totp_identifier: str | None = None,
+ totp_url: str | None = None,
+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,
+ ) -> str:
+ """Input text into an element identified by ``selector``.
+
+ When ``intention`` and ``data`` are provided a new input text action is
+ generated via the `script-generation-input-text-generation` prompt. The model returns a
+ fresh text based on the current DOM and the updated data for this run.
+ The browser then inputs the text using this newly generated text.
+
+ If the prompt generation or parsing fails for any reason we fall back to
+ inputting the originally supplied ``value``.
+ """
+
+ # format the text with the actual value of the parameter if it's a secret when running a workflow
+ if ai == "fallback":
+ error_to_raise = None
+ try:
+ locator = self._page.locator(selector)
+ await handler_utils.input_sequentially(locator, value, timeout=timeout)
+ return value
+ except Exception as e:
+ error_to_raise = e
+
+ if intention:
+ return await self._ai.ai_input_text(
+ selector=selector,
+ value=value,
+ intention=intention,
+ data=data,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ timeout=timeout,
+ )
+ if error_to_raise:
+ raise error_to_raise
+ else:
+ return value
+ elif ai == "proactive" and intention:
+ return await self._ai.ai_input_text(
+ selector=selector,
+ value=value,
+ intention=intention,
+ data=data,
+ totp_identifier=totp_identifier,
+ totp_url=totp_url,
+ timeout=timeout,
+ )
+ locator = self._page.locator(selector)
+ await handler_utils.input_sequentially(locator, value, timeout=timeout)
+ return value