Skip to content

Automate browser based workflows with AI

current viewpoint screenshot and scrolling n screenshot (#2716)

Shuchang Zheng775da1887824
diff --git a/alembic/versions/2025_06_14_0649-2c6b27e8e961_add_screenshot_scroll_number.py b/alembic/versions/2025_06_14_0649-2c6b27e8e961_add_screenshot_scroll_number.py
new file mode 100644
index 000000000..10380859f
--- /dev/null
+++ b/alembic/versions/2025_06_14_0649-2c6b27e8e961_add_screenshot_scroll_number.py
@@ -0,0 +1,37 @@
+"""add_screenshot_scroll_number
+
+Revision ID: 2c6b27e8e961
+Revises: 1517a4ba63fa
+Create Date: 2025-06-14 06:49:43.628471+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "2c6b27e8e961"
+down_revision: Union[str, None] = "1517a4ba63fa"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("observer_cruises", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))
+ op.add_column("tasks", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))
+ op.add_column("workflow_runs", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))
+ op.add_column("workflows", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("workflows", "max_screenshot_scrolling_times")
+ op.drop_column("workflow_runs", "max_screenshot_scrolling_times")
+ op.drop_column("tasks", "max_screenshot_scrolling_times")
+ op.drop_column("observer_cruises", "max_screenshot_scrolling_times")
+ # ### end Alembic commands ###
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index cef3eb67c..f5def2e6d 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -141,6 +141,7 @@ export type CreateTaskRequest = {
totp_identifier?: string | null;
application?: string | null;
include_action_history_in_verification?: boolean | null;
+ max_screenshot_scrolling_times?: number | null;
};
export type User = {
@@ -293,6 +294,7 @@ export type WorkflowRunStatusApiResponse = {
total_cost: number | null;
task_v2: TaskV2 | null;
workflow_title: string | null;
+ max_screenshot_scrolling_times: number | null;
};
export type TaskGenerationApiResponse = {
diff --git a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx
index aa079dbac..565bbbdc2 100644
--- a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx
@@ -39,6 +39,7 @@ import {
} from "./taskFormTypes";
import { ProxySelector } from "@/components/ProxySelector";
import { Switch } from "@/components/ui/switch";
+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "@/routes/workflows/editor/nodes/Taskv2Node/types";
type Props = {
initialValues: CreateNewTaskFormValues;
};
@@ -80,6 +81,7 @@ function createTaskRequestObject(
extracted_information_schema: extractedInformationSchema,
totp_identifier: transform(formValues.totpIdentifier),
error_code_mapping: errorCodeMapping,
+ max_screenshot_scrolling_times: formValues.maxScreenshotScrollingTimes,
include_action_history_in_verification:
formValues.includeActionHistoryInVerification,
};
@@ -114,6 +116,8 @@ function CreateNewTaskForm({ initialValues }: Props) {
...initialValues,
maxStepsOverride: initialValues.maxStepsOverride ?? null,
proxyLocation: initialValues.proxyLocation ?? ProxyLocation.Residential,
+ maxScreenshotScrollingTimes:
+ initialValues.maxScreenshotScrollingTimes ?? null,
},
});
const { errors } = useFormState({ control: form.control });
@@ -557,6 +561,45 @@ function CreateNewTaskForm({ initialValues }: Props) {
);
}}
/>
+ <FormField
+ control={form.control}
+ name="maxScreenshotScrollingTimes"
+ render={({ field }) => (
+ <FormItem>
+ <div className="flex gap-16">
+ <FormLabel>
+ <div className="w-72">
+ <h1 className="text-lg">
+ Max Scrolling Screenshots
+ </h1>
+ <h2 className="text-base text-slate-400">
+ {`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}
+ </h2>
+ </div>
+ </FormLabel>
+ <div className="w-full">
+ <FormControl>
+ <Input
+ {...field}
+ type="number"
+ min={0}
+ value={field.value ?? ""}
+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}
+ onChange={(event) => {
+ const value =
+ event.target.value === ""
+ ? null
+ : Number(event.target.value);
+ field.onChange(value);
+ }}
+ />
+ </FormControl>
+ <FormMessage />
+ </div>
+ </div>
+ </FormItem>
+ )}
+ />
<Separator />
<FormField
control={form.control}
diff --git a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx
index 50f05b5b2..15c7d9650 100644
--- a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx
@@ -62,6 +62,7 @@ function CreateNewTaskFormPage() {
webhookCallbackUrl: null,
proxyLocation: null,
includeActionHistoryInVerification: null,
+ maxScreenshotScrollingTimes: null,
}}
/>
</div>
@@ -131,6 +132,7 @@ function CreateNewTaskFormPage() {
includeActionHistoryInVerification:
data.workflow_definition.blocks[0]
.include_action_history_in_verification,
+ maxScreenshotScrollingTimes: data.max_screenshot_scrolling_times,
}}
/>
</div>
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
index 21d8c4668..3254973ae 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
@@ -43,7 +43,10 @@ import {
generateUniqueEmail,
} from "../data/sampleTaskData";
import { ExampleCasePill } from "./ExampleCasePill";
-import { MAX_STEPS_DEFAULT } from "@/routes/workflows/editor/nodes/Taskv2Node/types";
+import {
+ MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT,
+ MAX_STEPS_DEFAULT,
+} from "@/routes/workflows/editor/nodes/Taskv2Node/types";
function createTemplateTaskFromTaskGenerationParameters(
values: TaskGenerationApiResponse,
@@ -153,6 +156,8 @@ function PromptBox() {
const [publishWorkflow, setPublishWorkflow] = useState(false);
const [totpIdentifier, setTotpIdentifier] = useState("");
const [maxStepsOverride, setMaxStepsOverride] = useState<string | null>(null);
+ const [maxScreenshotScrollingTimes, setMaxScreenshotScrollingTimes] =
+ useState<string | null>(null);
const [showAdvancedSettings, setShowAdvancedSettings] = useState(false);
const [dataSchema, setDataSchema] = useState<string | null>(null);
@@ -167,6 +172,7 @@ function PromptBox() {
proxy_location: proxyLocation,
totp_identifier: totpIdentifier,
publish_workflow: publishWorkflow,
+ max_screenshot_scrolling_times: maxScreenshotScrollingTimes,
extracted_information_schema: dataSchema
? (() => {
try {
@@ -438,6 +444,21 @@ function PromptBox() {
/>
</div>
</div>
+ <div className="flex gap-16">
+ <div className="w-48 shrink-0">
+ <div className="text-sm">Max Scrolling Screenshots</div>
+ <div className="text-xs text-slate-400">
+ {`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}
+ </div>
+ </div>
+ <Input
+ value={maxScreenshotScrollingTimes ?? ""}
+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}
+ onChange={(event) => {
+ setMaxScreenshotScrollingTimes(event.target.value);
+ }}
+ />
+ </div>
</div>
</div>
) : null}
diff --git a/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx b/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx
index 6755b5d1e..74bcdd300 100644
--- a/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx
@@ -44,6 +44,8 @@ function RetryTask() {
proxyLocation: task.request.proxy_location ?? null,
includeActionHistoryInVerification:
task.request.include_action_history_in_verification ?? false,
+ maxScreenshotScrollingTimes:
+ task.request.max_screenshot_scrolling_times ?? null,
}}
/>
</div>
diff --git a/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts b/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts
index efafcbcfe..e58afc691 100644
--- a/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts
+++ b/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts
@@ -15,6 +15,7 @@ const createNewTaskFormSchemaBase = z.object({
errorCodeMapping: z.string().or(z.null()),
proxyLocation: z.nativeEnum(ProxyLocation).or(z.null()),
includeActionHistoryInVerification: z.boolean().or(z.null()).default(false),
+ maxScreenshotScrollingTimes: z.number().or(z.null()).default(null),
});
const savedTaskFormSchemaBase = createNewTaskFormSchemaBase.extend({
diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
index da36beb4b..99648871a 100644
--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
@@ -28,12 +28,14 @@ import { WorkflowParameter } from "./types/workflowTypes";
import { WorkflowParameterInput } from "./WorkflowParameterInput";
import { AxiosError } from "axios";
import { getLabelForWorkflowParameterType } from "./editor/workflowEditorUtils";
+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "./editor/nodes/Taskv2Node/types";
type Props = {
workflowParameters: Array<WorkflowParameter>;
initialValues: Record<string, unknown>;
initialSettings: {
proxyLocation: ProxyLocation;
webhookCallbackUrl: string;
+ maxScreenshotScrollingTimes: number | null;
};
};
@@ -73,14 +75,20 @@ type RunWorkflowRequestBody = {
proxy_location: ProxyLocation | null;
webhook_callback_url?: string | null;
browser_session_id: string | null;
+ max_screenshot_scrolling_times?: number | null;
};
function getRunWorkflowRequestBody(
values: RunWorkflowFormType,
workflowParameters: Array<WorkflowParameter>,
): RunWorkflowRequestBody {
- const { webhookCallbackUrl, proxyLocation, browserSessionId, ...parameters } =
- values;
+ const {
+ webhookCallbackUrl,
+ proxyLocation,
+ browserSessionId,
+ maxScreenshotScrollingTimes,
+ ...parameters
+ } = values;
const parsedParameters = parseValuesForWorkflowRun(
parameters,
@@ -95,6 +103,10 @@ function getRunWorkflowRequestBody(
browser_session_id: bsi,
};
+ if (maxScreenshotScrollingTimes) {
+ body.max_screenshot_scrolling_times = maxScreenshotScrollingTimes;
+ }
+
if (webhookCallbackUrl) {
body.webhook_callback_url = webhookCallbackUrl;
}
@@ -106,6 +118,7 @@ type RunWorkflowFormType = Record<string, unknown> & {
webhookCallbackUrl: string;
proxyLocation: ProxyLocation;
browserSessionId: string | null;
+ maxScreenshotScrollingTimes: number | null;
};
function RunWorkflowForm({
@@ -127,6 +140,7 @@ function RunWorkflowForm({
webhookCallbackUrl: initialSettings.webhookCallbackUrl,
proxyLocation: initialSettings.proxyLocation,
browserSessionId: browserSessionIdDefault,
+ maxScreenshotScrollingTimes: initialSettings.maxScreenshotScrollingTimes,
},
});
const apiCredential = useApiCredential();
@@ -177,6 +191,7 @@ function RunWorkflowForm({
webhookCallbackUrl,
proxyLocation,
browserSessionId,
+ maxScreenshotScrollingTimes,
...parameters
} = values;
@@ -189,6 +204,7 @@ function RunWorkflowForm({
webhookCallbackUrl,
proxyLocation,
browserSessionId,
+ maxScreenshotScrollingTimes,
});
}
@@ -392,6 +408,48 @@ function RunWorkflowForm({
);
}}
/>
+ <FormField
+ key="maxScreenshotScrollingTimes"
+ control={form.control}
+ name="maxScreenshotScrollingTimes"
+ render={({ field }) => {
+ return (
+ <FormItem>
+ <div className="flex gap-16">
+ <FormLabel>
+ <div className="w-72">
+ <div className="flex items-center gap-2 text-lg">
+ Max Scrolling Screenshots
+ </div>
+ <h2 className="text-sm text-slate-400">
+ {`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}
+ </h2>
+ </div>
+ </FormLabel>
+ <div className="w-full space-y-2">
+ <FormControl>
+ <Input
+ {...field}
+ type="number"
+ min={0}
+ value={field.value ?? ""}
+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}
+ onChange={(event) => {
+ const value =
+ event.target.value === ""
+ ? null
+ : Number(event.target.value);
+ field.onChange(value);
+ }}
+ />
+ </FormControl>
+ <FormMessage />
+ </div>
+ </div>
+ </FormItem>
+ );
+ }}
+ />
</div>
<div className="flex justify-end gap-2">
<CopyApiCommandDropdown
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
index 5d2f59cc0..deb39eee5 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
@@ -100,6 +100,8 @@ function WorkflowRun() {
const parameters = workflowRun?.parameters ?? {};
const proxyLocation =
workflowRun?.proxy_location ?? ProxyLocation.Residential;
+ const maxScreenshotScrollingTimes =
+ workflowRun?.max_screenshot_scrolling_times ?? null;
const title = workflowIsLoading ? (
<Skeleton className="h-9 w-48" />
@@ -244,6 +246,7 @@ function WorkflowRun() {
data: parameters,
proxyLocation,
webhookCallbackUrl: workflowRun?.webhook_callback_url ?? "",
+ maxScreenshotScrollingTimes,
}}
>
<PlayIcon className="mr-2 h-4 w-4" />
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx
index 8de6821f4..2fe037d3e 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx
@@ -30,6 +30,8 @@ function WorkflowRunParameters() {
const proxyLocation = location.state
? (location.state.proxyLocation as ProxyLocation)
: null;
+ const maxScreenshotScrollingTimes =
+ location.state?.maxScreenshotScrollingTimes ?? null;
const webhookCallbackUrl = location.state
? (location.state.webhookCallbackUrl as string)
@@ -109,6 +111,10 @@ function WorkflowRunParameters() {
ProxyLocation.Residential,
webhookCallbackUrl:
webhookCallbackUrl ?? workflow.webhook_callback_url ?? "",
+ maxScreenshotScrollingTimes:
+ maxScreenshotScrollingTimes ??
+ workflow.max_screenshot_scrolling_times ??
+ null,
}}
/>
</div>
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index 357add8c1..a67653ef0 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -289,6 +289,8 @@ function FlowRenderer({
webhook_callback_url: data.settings.webhookCallbackUrl,
persist_browser_session: data.settings.persistBrowserSession,
model: data.settings.model,
+ max_screenshot_scrolling_times:
+ data.settings.maxScreenshotScrollingTimes,
totp_verification_url: workflow.totp_verification_url,
workflow_definition: {
parameters: data.parameters,
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
index 144ac78d2..322a3ef29 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
@@ -60,6 +60,7 @@ function WorkflowEditor() {
proxyLocation: workflow.proxy_location,
webhookCallbackUrl: workflow.webhook_callback_url,
model: workflow.model,
+ maxScreenshotScrollingTimes: workflow.max_screenshot_scrolling_times,
};
const elements = getElements(
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
index b2a2774d0..1e0a97f53 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
@@ -20,6 +20,7 @@ import { Separator } from "@/components/ui/separator";
import { ModelsResponse } from "@/api/types";
import { ModelSelector } from "@/components/ModelSelector";
import { WorkflowModel } from "@/routes/workflows/types/workflowTypes";
+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "../Taskv2Node/types";
function StartNode({ id, data }: NodeProps<StartNode>) {
const credentialGetter = useCredentialGetter();
@@ -51,6 +52,9 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
? data.persistBrowserSession
: false,
model: data.withWorkflowSettings ? data.model : workflowModel,
+ maxScreenshotScrollingTimes: data.withWorkflowSettings
+ ? data.maxScreenshotScrollingTimes
+ : null,
});
function handleChange(key: string, value: unknown) {
@@ -130,6 +134,26 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
/>
</div>
</div>
+ <div className="space-y-2">
+ <div className="flex items-center gap-2">
+ <Label>Max Scrolling Screenshots</Label>
+ <HelpTooltip
+ content={`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}
+ />
+ </div>
+ <Input
+ value={inputs.maxScreenshotScrollingTimes ?? ""}
+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}
+ onChange={(event) => {
+ const value =
+ event.target.value === ""
+ ? null
+ : Number(event.target.value);
+
+ handleChange("maxScreenshotScrollingTimes", value);
+ }}
+ />
+ </div>
</div>
</AccordionContent>
</AccordionItem>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts
index 11f54be49..6fa5ae31c 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts
@@ -9,6 +9,7 @@ export type WorkflowStartNodeData = {
proxyLocation: ProxyLocation;
persistBrowserSession: boolean;
model: WorkflowModel | null;
+ maxScreenshotScrollingTimes: number | null;
editable: boolean;
};
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts
index cafdea7da..f3cc549b1 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts
@@ -2,6 +2,7 @@ import { Node } from "@xyflow/react";
import { NodeBaseData } from "../types";
export const MAX_STEPS_DEFAULT = 25;
+export const MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT = 3;
export type Taskv2NodeData = NodeBaseData & {
prompt: string;
@@ -9,6 +10,7 @@ export type Taskv2NodeData = NodeBaseData & {
totpVerificationUrl: string | null;
totpIdentifier: string | null;
maxSteps: number | null;
+ maxScreenshotScrollingTimes: number | null;
};
export type Taskv2Node = Node<Taskv2NodeData, "taskv2">;
@@ -23,6 +25,7 @@ export const taskv2NodeDefaultData: Taskv2NodeData = {
totpVerificationUrl: null,
maxSteps: MAX_STEPS_DEFAULT,
model: null,
+ maxScreenshotScrollingTimes: null,
};
export function isTaskV2Node(node: Node): node is Taskv2Node {
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index cc2392950..ee2c0e2a1 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -251,6 +251,7 @@ function convertToNode(
maxSteps: block.max_steps,
totpIdentifier: block.totp_identifier,
totpVerificationUrl: block.totp_verification_url,
+ maxScreenshotScrollingTimes: null,
},
};
}
@@ -662,6 +663,7 @@ function getElements(
proxyLocation: settings.proxyLocation ?? ProxyLocation.Residential,
webhookCallbackUrl: settings.webhookCallbackUrl ?? "",
model: settings.model,
+ maxScreenshotScrollingTimes: settings.maxScreenshotScrollingTimes,
editable,
}),
);
@@ -1322,6 +1324,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {
proxyLocation: ProxyLocation.Residential,
webhookCallbackUrl: null,
model: null,
+ maxScreenshotScrollingTimes: null,
};
const startNodes = nodes.filter(isStartNode);
const startNodeWithWorkflowSettings = startNodes.find(
@@ -1337,6 +1340,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {
proxyLocation: data.proxyLocation,
webhookCallbackUrl: data.webhookCallbackUrl,
model: data.model,
+ maxScreenshotScrollingTimes: data.maxScreenshotScrollingTimes,
};
}
return defaultSettings;
@@ -1992,6 +1996,7 @@ function convert(workflow: WorkflowApiResponse): WorkflowCreateYAMLRequest {
persist_browser_session: workflow.persist_browser_session,
model: workflow.model,
totp_verification_url: workflow.totp_verification_url,
+ max_screenshot_scrolling_times: workflow.max_screenshot_scrolling_times,
workflow_definition: {
parameters: convertParametersToParameterYAML(userParameters),
blocks: convertBlocksToBlockYAML(workflow.workflow_definition.blocks),
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
index 430b72ad4..feef5c59a 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
@@ -470,6 +470,7 @@ export type WorkflowApiResponse = {
model: WorkflowModel | null;
totp_verification_url: string | null;
totp_identifier: string | null;
+ max_screenshot_scrolling_times: number | null;
created_at: string;
modified_at: string;
deleted_at: string | null;
@@ -480,6 +481,7 @@ export type WorkflowSettings = {
webhookCallbackUrl: string | null;
persistBrowserSession: boolean;
model: WorkflowModel | null;
+ maxScreenshotScrollingTimes: number | null;
};
export type WorkflowModel = JsonObjectExtendable<{ model_name: string }>;
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
index 9df7811df..e7a405538 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
@@ -12,6 +12,7 @@ export type WorkflowCreateYAMLRequest = {
totp_verification_url?: string | null;
workflow_definition: WorkflowDefinitionYAML;
is_saved_task?: boolean;
+ max_screenshot_scrolling_times?: number | null;
};
export type WorkflowDefinitionYAML = {
diff --git a/skyvern/constants.py b/skyvern/constants.py
index 52de64b10..d1a69a9b6 100644
--- a/skyvern/constants.py
+++ b/skyvern/constants.py
@@ -20,6 +20,7 @@ AUTO_COMPLETION_POTENTIAL_VALUES_COUNT = 3
DROPDOWN_MENU_MAX_DISTANCE = 100
BROWSER_DOWNLOADING_SUFFIX = ".crdownload"
MAX_UPLOAD_FILE_COUNT = 50
+DEFAULT_MAX_SCREENSHOT_SCROLLING_TIMES = 3
# reserved fields for navigation payload
SPECIAL_FIELD_VERIFICATION_CODE = "verification_code"
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 8dec96069..73ffd6fe7 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -19,6 +19,7 @@ from skyvern import analytics
from skyvern.config import settings
from skyvern.constants import (
BROWSER_DOWNLOADING_SUFFIX,
+ DEFAULT_MAX_SCREENSHOT_SCROLLING_TIMES,
GET_DOWNLOADED_FILES_TIMEOUT,
SAVE_DOWNLOADED_FILES_TIMEOUT,
SCRAPE_TYPE_ORDER,
@@ -181,6 +182,7 @@ class ForgeAgent:
error_code_mapping=task_block.error_code_mapping,
include_action_history_in_verification=task_block.include_action_history_in_verification,
model=task_block.model,
+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,
)
LOG.info(
"Created a new task for workflow run",
@@ -237,6 +239,7 @@ class ForgeAgent:
application=task_request.application,
include_action_history_in_verification=task_request.include_action_history_in_verification,
model=task_request.model,
+ max_screenshot_scrolling_times=task_request.max_screenshot_scrolling_times,
)
LOG.info(
"Created new task",
@@ -1650,12 +1653,22 @@ class ForgeAgent:
if not working_page:
raise BrowserStateMissingPage()
- fullpage_screenshot = True
+ context = skyvern_context.ensure_context()
+ scrolling_number = context.max_screenshot_scrolling_times
+ if scrolling_number is None:
+ scrolling_number = DEFAULT_MAX_SCREENSHOT_SCROLLING_TIMES
+
if engine in CUA_ENGINES:
- fullpage_screenshot = False
+ scrolling_number = 0
try:
- screenshot = await browser_state.take_screenshot(full_page=fullpage_screenshot)
+ screenshot = await browser_state.take_post_action_screenshot(
+ scrolling_number=scrolling_number,
+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "ENABLE_PLAYWRIGHT_FULLPAGE",
+ str(task.organization_id),
+ ),
+ )
await app.ARTIFACT_MANAGER.create_artifact(
step=step,
artifact_type=ArtifactType.SCREENSHOT_ACTION,
@@ -2135,7 +2148,12 @@ class ForgeAgent:
browser_state = app.BROWSER_MANAGER.get_for_task(task.task_id)
if browser_state is not None and await browser_state.get_working_page() is not None:
try:
- screenshot = await browser_state.take_screenshot(full_page=True)
+ screenshot = await browser_state.take_fullpage_screenshot(
+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "ENABLE_PLAYWRIGHT_FULLPAGE",
+ str(task.organization_id),
+ )
+ )
await app.ARTIFACT_MANAGER.create_artifact(
step=last_step,
artifact_type=ArtifactType.SCREENSHOT_FINAL,
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 819cb3ab8..d51b9fa8d 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -23,6 +23,7 @@ class SkyvernContext:
hashed_href_map: dict[str, str] = field(default_factory=dict)
refresh_working_page: bool = False
frame_index_map: dict[Frame, int] = field(default_factory=dict)
+ max_screenshot_scrolling_times: int | None = None
def __repr__(self) -> str:
return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override})"
diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.py
index 0f035b8b8..a3697c6ae 100644
--- a/skyvern/forge/sdk/db/client.py
+++ b/skyvern/forge/sdk/db/client.py
@@ -149,6 +149,7 @@ class AgentDB:
application: str | None = None,
include_action_history_in_verification: bool | None = None,
model: dict[str, Any] | None = None,
+ max_screenshot_scrolling_times: int | None = None,
) -> Task:
try:
async with self.Session() as session:
@@ -176,6 +177,7 @@ class AgentDB:
application=application,
include_action_history_in_verification=include_action_history_in_verification,
model=model,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
)
session.add(new_task)
await session.commit()
@@ -1217,6 +1219,7 @@ class AgentDB:
description: str | None = None,
proxy_location: ProxyLocation | None = None,
webhook_callback_url: str | None = None,
+ max_screenshot_scrolling_times: int | None = None,
totp_verification_url: str | None = None,
totp_identifier: str | None = None,
persist_browser_session: bool = False,
@@ -1236,6 +1239,7 @@ class AgentDB:
webhook_callback_url=webhook_callback_url,
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
persist_browser_session=persist_browser_session,
model=model,
is_saved_task=is_saved_task,
@@ -1479,6 +1483,7 @@ class AgentDB:
totp_verification_url: str | None = None,
totp_identifier: str | None = None,
parent_workflow_run_id: str | None = None,
+ max_screenshot_scrolling_times: int | None = None,
) -> WorkflowRun:
try:
async with self.Session() as session:
@@ -1492,6 +1497,7 @@ class AgentDB:
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
parent_workflow_run_id=parent_workflow_run_id,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
)
session.add(workflow_run)
await session.commit()
@@ -2436,6 +2442,7 @@ class AgentDB:
extracted_information_schema: dict | list | str | None = None,
error_code_mapping: dict | None = None,
model: dict[str, Any] | None = None,
+ max_screenshot_scrolling_times: int | None = None,
) -> TaskV2:
async with self.Session() as session:
new_task_v2 = TaskV2Model(
@@ -2452,6 +2459,7 @@ class AgentDB:
error_code_mapping=error_code_mapping,
organization_id=organization_id,
model=model,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
)
session.add(new_task_v2)
await session.commit()
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index a5cbf913f..b41c1b42e 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -88,6 +88,7 @@ class TaskModel(Base):
queued_at = Column(DateTime, nullable=True)
started_at = Column(DateTime, nullable=True)
finished_at = Column(DateTime, nullable=True)
+ max_screenshot_scrolling_times = Column(Integer, nullable=True)
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False, index=True)
modified_at = Column(
DateTime,
@@ -218,6 +219,7 @@ class WorkflowModel(Base):
workflow_definition = Column(JSON, nullable=False)
proxy_location = Column(String)
webhook_callback_url = Column(String)
+ max_screenshot_scrolling_times = Column(Integer, nullable=True)
totp_verification_url = Column(String)
totp_identifier = Column(String)
persist_browser_session = Column(Boolean, default=False, nullable=False)
@@ -254,6 +256,7 @@ class WorkflowRunModel(Base):
webhook_callback_url = Column(String)
totp_verification_url = Column(String)
totp_identifier = Column(String)
+ max_screenshot_scrolling_times = Column(Integer, nullable=True)
queued_at = Column(DateTime, nullable=True)
started_at = Column(DateTime, nullable=True)
@@ -621,6 +624,7 @@ class TaskV2Model(Base):
extracted_information_schema = Column(JSON, nullable=True)
error_code_mapping = Column(JSON, nullable=True)
max_steps = Column(Integer, nullable=True)
+ max_screenshot_scrolling_times = Column(Integer, nullable=True)
queued_at = Column(DateTime, nullable=True)
started_at = Column(DateTime, nullable=True)
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 5bfe3454b..441a07e32 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -142,6 +142,7 @@ def convert_to_task(task_obj: TaskModel, debug_enabled: bool = False, workflow_p
queued_at=task_obj.queued_at,
started_at=task_obj.started_at,
finished_at=task_obj.finished_at,
+ max_screenshot_scrolling_times=task_obj.max_screenshot_scrolling_times,
)
return task
@@ -238,6 +239,7 @@ def convert_to_workflow(workflow_model: WorkflowModel, debug_enabled: bool = Fal
persist_browser_session=workflow_model.persist_browser_session,
model=workflow_model.model,
proxy_location=(ProxyLocation(workflow_model.proxy_location) if workflow_model.proxy_location else None),
+ max_screenshot_scrolling_times=workflow_model.max_screenshot_scrolling_times,
version=workflow_model.version,
is_saved_task=workflow_model.is_saved_task,
description=workflow_model.description,
@@ -278,6 +280,7 @@ def convert_to_workflow_run(
created_at=workflow_run_model.created_at,
modified_at=workflow_run_model.modified_at,
workflow_title=workflow_title,
+ max_screenshot_scrolling_times=workflow_run_model.max_screenshot_scrolling_times,
)
diff --git a/skyvern/forge/sdk/executor/async_executor.py b/skyvern/forge/sdk/executor/async_executor.py
index a13aa161c..b579fafda 100644
--- a/skyvern/forge/sdk/executor/async_executor.py
+++ b/skyvern/forge/sdk/executor/async_executor.py
@@ -107,6 +107,7 @@ class BackgroundTaskExecutor(AsyncExecutor):
context.task_id = task.task_id
context.organization_id = organization_id
context.max_steps_override = max_steps_override
+ context.max_screenshot_scrolling_times = task.max_screenshot_scrolling_times
if background_tasks:
await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index e14417969..99a922822 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -166,6 +166,7 @@ async def run_task(
totp_identifier=run_request.totp_identifier,
include_action_history_in_verification=run_request.include_action_history_in_verification,
model=run_request.model,
+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,
)
task_v1_response = await task_v1_service.run_task(
task=task_v1_request,
@@ -203,6 +204,7 @@ async def run_task(
data_extraction_schema=task_v1_response.extracted_information_schema,
error_code_mapping=task_v1_response.error_code_mapping,
browser_session_id=run_request.browser_session_id,
+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,
),
)
if run_request.engine == RunEngine.skyvern_v2:
@@ -221,6 +223,7 @@ async def run_task(
error_code_mapping=run_request.error_code_mapping,
create_task_run=True,
model=run_request.model,
+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,
)
except MissingBrowserAddressError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
@@ -263,6 +266,7 @@ async def run_task(
error_code_mapping=task_v2.error_code_mapping,
data_extraction_schema=task_v2.extracted_information_schema,
publish_workflow=run_request.publish_workflow,
+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,
),
)
LOG.error("Invalid agent engine", engine=run_request.engine, organization_id=current_org.organization_id)
@@ -318,6 +322,7 @@ async def run_workflow(
totp_identifier=workflow_run_request.totp_identifier,
totp_url=workflow_run_request.totp_url,
browser_session_id=workflow_run_request.browser_session_id,
+ max_screenshot_scrolling_times=workflow_run_request.max_screenshot_scrolling_times,
)
try:
@@ -1765,6 +1770,7 @@ async def run_task_v2(
create_task_run=True,
extracted_information_schema=data.extracted_information_schema,
error_code_mapping=data.error_code_mapping,
+ max_screenshot_scrolling_times=data.max_screenshot_scrolling_times,
)
except MissingBrowserAddressError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
diff --git a/skyvern/forge/sdk/schemas/task_v2.py b/skyvern/forge/sdk/schemas/task_v2.py
index 1978242a2..8cbf418ac 100644
--- a/skyvern/forge/sdk/schemas/task_v2.py
+++ b/skyvern/forge/sdk/schemas/task_v2.py
@@ -48,6 +48,8 @@ class TaskV2(BaseModel):
queued_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
+ max_screenshot_scrolling_times: int | None = None
+
created_at: datetime
modified_at: datetime
@@ -147,6 +149,7 @@ class TaskV2Request(BaseModel):
publish_workflow: bool = False
extracted_information_schema: dict | list | str | None = None
error_code_mapping: dict[str, str] | None = None
+ max_screenshot_scrolling_times: int | None = None
@field_validator("url", "webhook_callback_url", "totp_verification_url")
@classmethod
diff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.py
index a8268f9f7..31b0a7e5b 100644
--- a/skyvern/forge/sdk/schemas/tasks.py
+++ b/skyvern/forge/sdk/schemas/tasks.py
@@ -96,6 +96,11 @@ class TaskBase(BaseModel):
description="Whether to include the action history when verifying the task is complete",
examples=[True, False],
)
+ max_screenshot_scrolling_times: int | None = Field(
+ default=None,
+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's None or 0, it takes the current viewpoint screenshot.",
+ examples=[10],
+ )
class TaskRequest(TaskBase):
@@ -314,6 +319,7 @@ class Task(TaskBase):
errors=self.errors,
max_steps_per_run=self.max_steps_per_run,
workflow_run_id=self.workflow_run_id,
+ max_screenshot_scrolling_times=self.max_screenshot_scrolling_times,
)
@@ -337,6 +343,7 @@ class TaskResponse(BaseModel):
queued_at: datetime | None = None
started_at: datetime | None = None
finished_at: datetime | None = None
+ max_screenshot_scrolling_times: int | None = None
class TaskOutput(BaseModel):
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 6b173a3ab..ed16d4524 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -307,7 +307,12 @@ class Block(BaseModel, abc.ABC):
if not browser_state:
LOG.warning("No browser state found when creating workflow_run_block", workflow_run_id=workflow_run_id)
else:
- screenshot = await browser_state.take_screenshot(full_page=True)
+ screenshot = await browser_state.take_fullpage_screenshot(
+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "ENABLE_PLAYWRIGHT_FULLPAGE",
+ str(organization_id),
+ )
+ )
if screenshot:
await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(
workflow_run_block=workflow_run_block,
@@ -569,8 +574,15 @@ class BaseTaskBlock(Block):
browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
workflow_run=workflow_run, url=self.url, browser_session_id=browser_session_id
)
+ # assert that the browser state is not None, otherwise we can't go through typing
+ assert browser_state is not None
# add screenshot artifact for the first task
- screenshot = await browser_state.take_screenshot(full_page=True)
+ screenshot = await browser_state.take_fullpage_screenshot(
+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "ENABLE_PLAYWRIGHT_FULLPAGE",
+ str(organization_id),
+ )
+ )
if screenshot:
await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(
workflow_run_block=workflow_run_block,
@@ -2486,6 +2498,7 @@ class TaskV2Block(Block):
proxy_location=workflow_run.proxy_location,
totp_identifier=self.totp_identifier,
totp_verification_url=self.totp_verification_url,
+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,
)
await app.DATABASE.update_task_v2(
task_v2.observer_cruise_id, status=TaskV2Status.queued, organization_id=organization_id
@@ -2517,6 +2530,7 @@ class TaskV2Block(Block):
workflow_permanent_id=workflow_run.workflow_permanent_id,
workflow_run_id=workflow_run_id,
browser_session_id=browser_session_id,
+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,
)
)
result_dict = None
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 1880446c4..0c9002dc5 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -22,6 +22,7 @@ class WorkflowRequestBody(BaseModel):
totp_verification_url: str | None = None
totp_identifier: str | None = None
browser_session_id: str | None = None
+ max_screenshot_scrolling_times: int | None = None
@field_validator("webhook_callback_url", "totp_verification_url")
@classmethod
@@ -76,6 +77,7 @@ class Workflow(BaseModel):
persist_browser_session: bool = False
model: dict[str, Any] | None = None
status: WorkflowStatus = WorkflowStatus.published
+ max_screenshot_scrolling_times: int | None = None
created_at: datetime
modified_at: datetime
@@ -115,6 +117,7 @@ class WorkflowRun(BaseModel):
failure_reason: str | None = None
parent_workflow_run_id: str | None = None
workflow_title: str | None = None
+ max_screenshot_scrolling_times: int | None = None
queued_at: datetime | None = None
started_at: datetime | None = None
@@ -162,3 +165,4 @@ class WorkflowRunResponseBase(BaseModel):
task_v2: TaskV2 | None = None
workflow_title: str | None = None
browser_session_id: str | None = None
+ max_screenshot_scrolling_times: int | None = None
diff --git a/skyvern/forge/sdk/workflow/models/yaml.py b/skyvern/forge/sdk/workflow/models/yaml.py
index a96cb38a0..d2ce6ae46 100644
--- a/skyvern/forge/sdk/workflow/models/yaml.py
+++ b/skyvern/forge/sdk/workflow/models/yaml.py
@@ -424,4 +424,5 @@ class WorkflowCreateYAMLRequest(BaseModel):
model: dict[str, Any] | None = None
workflow_definition: WorkflowDefinitionYAML
is_saved_task: bool = False
+ max_screenshot_scrolling_times: int | None = None
status: WorkflowStatus = WorkflowStatus.published
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 844d2a557..45819c275 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -169,6 +169,7 @@ class WorkflowService:
organization_id=workflow.organization_id,
proxy_location=workflow_request.proxy_location,
webhook_callback_url=workflow_request.webhook_callback_url,
+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolling_times,
)
skyvern_context.set(
SkyvernContext(
@@ -178,6 +179,7 @@ class WorkflowService:
workflow_id=workflow_id,
workflow_run_id=workflow_run.workflow_run_id,
max_steps_override=max_steps_override,
+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolling_times,
)
)
@@ -577,6 +579,7 @@ class WorkflowService:
workflow_definition: WorkflowDefinition,
description: str | None = None,
proxy_location: ProxyLocation | None = None,
+ max_screenshot_scrolling_times: int | None = None,
webhook_callback_url: str | None = None,
totp_verification_url: str | None = None,
totp_identifier: str | None = None,
@@ -594,6 +597,7 @@ class WorkflowService:
description=description,
proxy_location=proxy_location,
webhook_callback_url=webhook_callback_url,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
persist_browser_session=persist_browser_session,
@@ -767,6 +771,7 @@ class WorkflowService:
totp_verification_url=workflow_request.totp_verification_url,
totp_identifier=workflow_request.totp_identifier,
parent_workflow_run_id=parent_workflow_run_id,
+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolling_times,
)
async def mark_workflow_run_as_completed(self, workflow_run_id: str) -> WorkflowRun:
@@ -1180,6 +1185,7 @@ class WorkflowService:
total_steps=total_steps,
total_cost=total_cost,
workflow_title=workflow.title,
+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,
)
async def clean_up_workflow(
@@ -1453,6 +1459,7 @@ class WorkflowService:
totp_identifier=request.totp_identifier,
persist_browser_session=request.persist_browser_session,
model=request.model,
+ max_screenshot_scrolling_times=request.max_screenshot_scrolling_times,
workflow_permanent_id=workflow_permanent_id,
version=existing_version + 1,
is_saved_task=request.is_saved_task,
@@ -1470,6 +1477,7 @@ class WorkflowService:
totp_identifier=request.totp_identifier,
persist_browser_session=request.persist_browser_session,
model=request.model,
+ max_screenshot_scrolling_times=request.max_screenshot_scrolling_times,
is_saved_task=request.is_saved_task,
status=request.status,
)
diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py
index 116f1c672..dd809eb73 100644
--- a/skyvern/schemas/runs.py
+++ b/skyvern/schemas/runs.py
@@ -279,6 +279,10 @@ class TaskRunRequest(BaseModel):
include_action_history_in_verification: bool | None = Field(
default=False, description="Whether to include action history when verifying that the task is complete"
)
+ max_screenshot_scrolling_times: int | None = Field(
+ default=None,
+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's None or 0, it takes the current viewpoint screenshot.",
+ )
@field_validator("url", "webhook_url", "totp_url")
@classmethod
@@ -326,6 +330,10 @@ class WorkflowRunRequest(BaseModel):
default=None,
description="ID of a Skyvern browser session to reuse, having it continue from the current screen state",
)
+ max_screenshot_scrolling_times: int | None = Field(
+ default=None,
+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's None or 0, it takes the current viewpoint screenshot.",
+ )
@field_validator("webhook_url", "totp_url")
@classmethod
@@ -368,9 +376,11 @@ class BaseRunResponse(BaseModel):
examples=["https://app.skyvern.com/tasks/tsk_123", "https://app.skyvern.com/workflows/wpid_123/wr_123"],
)
browser_session_id: str | None = Field(
+ default=None, description="ID of the Skyvern persistent browser session used for this run", examples=["pbs_123"]
+ )
+ max_screenshot_scrolling_times: int | None = Field(
default=None,
- description="ID of the Skyvern persistent browser session used for this run",
- examples=["pbs_123"],
+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's NONE or 0, it takes the current view point screenshot.",
)
diff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.py
index a301dd24f..66f8d257f 100644
--- a/skyvern/services/run_service.py
+++ b/skyvern/services/run_service.py
@@ -67,6 +67,7 @@ async def get_run_response(run_id: str, organization_id: str | None = None) -> R
max_steps=task_v1_response.max_steps_per_run,
data_extraction_schema=task_v1_response.request.extracted_information_schema,
error_code_mapping=task_v1_response.request.error_code_mapping,
+ max_screenshot_scrolling_times=task_v1_response.request.max_screenshot_scrolling_times,
),
)
elif run.task_run_type == RunType.task_v2:
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 1b6bc7051..e11cdfa4a 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -165,6 +165,7 @@ async def initialize_task_v2(
error_code_mapping: dict | None = None,
create_task_run: bool = False,
model: dict[str, Any] | None = None,
+ max_screenshot_scrolling_times: int | None = None,
) -> TaskV2:
task_v2 = await app.DATABASE.create_task_v2(
prompt=user_prompt,
@@ -176,11 +177,13 @@ async def initialize_task_v2(
extracted_information_schema=extracted_information_schema,
error_code_mapping=error_code_mapping,
model=model,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
)
# set task_v2_id in context
context = skyvern_context.current()
if context:
context.task_v2_id = task_v2.observer_cruise_id
+ context.max_screenshot_scrolling_times = max_screenshot_scrolling_times
thought = await app.DATABASE.create_thought(
task_v2_id=task_v2.observer_cruise_id,
@@ -221,7 +224,9 @@ async def initialize_task_v2(
)
workflow_run = await app.WORKFLOW_SERVICE.setup_workflow_run(
request_id=None,
- workflow_request=WorkflowRequestBody(),
+ workflow_request=WorkflowRequestBody(
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ ),
workflow_permanent_id=new_workflow.workflow_permanent_id,
organization=organization,
version=None,
@@ -454,6 +459,7 @@ async def run_task_v2_helper(
request_id=request_id,
task_v2_id=task_v2_id,
browser_session_id=browser_session_id,
+ max_screenshot_scrolling_times=task_v2.max_screenshot_scrolling_times,
)
)
@@ -771,6 +777,7 @@ async def run_task_v2_helper(
proxy_location=task_v2.proxy_location or ProxyLocation.RESIDENTIAL,
workflow_definition=workflow_definition_yaml,
status=workflow.status,
+ max_screenshot_scrolling_times=task_v2.max_screenshot_scrolling_times,
)
LOG.info("Creating workflow from request", workflow_create_request=workflow_create_request)
workflow = await app.WORKFLOW_SERVICE.create_workflow_from_request(
diff --git a/skyvern/services/workflow_service.py b/skyvern/services/workflow_service.py
index fff09c9bb..47fd5ec0a 100644
--- a/skyvern/services/workflow_service.py
+++ b/skyvern/services/workflow_service.py
@@ -97,6 +97,7 @@ async def get_workflow_run_response(
webhook_url=workflow_run.webhook_callback_url or None,
totp_url=workflow_run.totp_verification_url or None,
totp_identifier=workflow_run.totp_identifier,
+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,
# TODO: add browser session id
),
)
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py
index d5ea7d106..76c0cc4b9 100644
--- a/skyvern/webeye/browser_factory.py
+++ b/skyvern/webeye/browser_factory.py
@@ -35,7 +35,7 @@ from skyvern.exceptions import (
from skyvern.forge.sdk.api.files import get_download_dir, make_temp_directory
from skyvern.forge.sdk.core.skyvern_context import current, ensure_context
from skyvern.schemas.runs import ProxyLocation, get_tzinfo_from_proxy
-from skyvern.webeye.utils.page import SkyvernFrame
+from skyvern.webeye.utils.page import ScreenshotMode, SkyvernFrame
LOG = structlog.get_logger()
@@ -865,6 +865,30 @@ class BrowserState:
except asyncio.TimeoutError:
LOG.error("Timeout to close playwright, might leave the broswer opening forever")
- async def take_screenshot(self, full_page: bool = False, file_path: str | None = None) -> bytes:
+ async def take_fullpage_screenshot(
+ self,
+ file_path: str | None = None,
+ use_playwright_fullpage: bool = False, # TODO: THIS IS ONLY FOR EXPERIMENT. will be removed after experiment.
+ ) -> bytes:
+ page = await self.__assert_page()
+ return await SkyvernFrame.take_scrolling_screenshot(
+ page=page,
+ file_path=file_path,
+ mode=ScreenshotMode.LITE,
+ use_playwright_fullpage=use_playwright_fullpage,
+ )
+
+ async def take_post_action_screenshot(
+ self,
+ scrolling_number: int,
+ file_path: str | None = None,
+ use_playwright_fullpage: bool = False, # TODO: THIS IS ONLY FOR EXPERIMENT. will be removed after experiment.
+ ) -> bytes:
page = await self.__assert_page()
- return await SkyvernFrame.take_screenshot(page=page, full_page=full_page, file_path=file_path)
+ return await SkyvernFrame.take_scrolling_screenshot(
+ page=page,
+ file_path=file_path,
+ mode=ScreenshotMode.LITE,
+ scrolling_number=scrolling_number,
+ use_playwright_fullpage=use_playwright_fullpage,
+ )
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 98b8f7168..5871b29a3 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -2115,13 +2115,14 @@ async function scrollToNextPage(
draw_boxes,
frame = "main.frame",
frame_index = undefined,
+ need_overlap = true,
) {
// remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
// return true if there is a next page, false otherwise
removeBoundingBoxes();
window.scrollBy({
left: 0,
- top: window.innerHeight - 200,
+ top: need_overlap ? window.innerHeight - 200 : window.innerHeight,
behavior: "instant",
});
if (draw_boxes) {
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index d1f9546db..7bf425303 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -2,9 +2,12 @@ from __future__ import annotations
import asyncio
import time
+from enum import StrEnum
+from io import BytesIO
from typing import Any
import structlog
+from PIL import Image
from playwright._impl._errors import TimeoutError
from playwright.async_api import ElementHandle, Frame, Page
@@ -31,17 +34,24 @@ def load_js_script() -> str:
JS_FUNCTION_DEFS = load_js_script()
+class ScreenshotMode(StrEnum):
+ LITE = "lite"
+ DETAILED = "detailed"
+
+
async def _current_viewpoint_screenshot_helper(
page: Page,
file_path: str | None = None,
full_page: bool = False,
timeout: float = settings.BROWSER_SCREENSHOT_TIMEOUT_MS,
+ mode: ScreenshotMode = ScreenshotMode.DETAILED,
) -> bytes:
if page.is_closed():
raise FailedToTakeScreenshot(error_message="Page is closed")
try:
- await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
- LOG.debug("Page is fully loaded, agent is about to take screenshots")
+ if mode == ScreenshotMode.DETAILED:
+ await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ LOG.debug("Page is fully loaded, agent is about to take screenshots")
start_time = time.time()
screenshot: bytes = b""
if file_path:
@@ -77,6 +87,7 @@ async def _scrolling_screenshots_helper(
url: str | None = None,
draw_boxes: bool = False,
max_number: int = settings.MAX_NUM_SCREENSHOTS,
+ mode: ScreenshotMode = ScreenshotMode.DETAILED,
) -> list[bytes]:
skyvern_page = await SkyvernFrame.create_instance(frame=page)
# page is the main frame and the index must be 0
@@ -84,6 +95,11 @@ async def _scrolling_screenshots_helper(
frame = "main.frame"
frame_index = 0
+ # when mode is lite, we don't draw bounding boxes
+ # since draw_boxes impacts the performance of processing
+ if mode == ScreenshotMode.LITE:
+ draw_boxes = False
+
screenshots: list[bytes] = []
if await skyvern_page.is_window_scrollable():
scroll_y_px_old = -30.0
@@ -92,12 +108,15 @@ async def _scrolling_screenshots_helper(
# We are checking the difference between the old and new scroll_y_px to determine if we have reached the end of the
# page. If the difference is less than 25, we assume we have reached the end of the page.
while abs(scroll_y_px_old - scroll_y_px) > 25 and len(screenshots) < max_number:
- screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame)
+ screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame, mode=mode)
screenshots.append(screenshot)
scroll_y_px_old = scroll_y_px
LOG.debug("Scrolling to next page", url=url, num_screenshots=len(screenshots))
scroll_y_px = await skyvern_page.scroll_to_next_page(
- draw_boxes=draw_boxes, frame=frame, frame_index=frame_index
+ draw_boxes=draw_boxes,
+ frame=frame,
+ frame_index=frame_index,
+ need_overlap=(mode == ScreenshotMode.DETAILED),
)
LOG.debug(
"Scrolled to next page",
@@ -107,15 +126,17 @@ async def _scrolling_screenshots_helper(
if draw_boxes:
await skyvern_page.remove_bounding_boxes()
await skyvern_page.scroll_to_top(draw_boxes=False, frame=frame, frame_index=frame_index)
- # wait until animation ends, which is triggered by scrolling
- LOG.debug("Waiting for 2 seconds until animation ends.")
- await asyncio.sleep(2)
+
+ if mode == ScreenshotMode.DETAILED:
+ # wait until animation ends, which is triggered by scrolling
+ LOG.debug("Waiting for 2 seconds until animation ends.")
+ await asyncio.sleep(2)
else:
if draw_boxes:
await skyvern_page.build_elements_and_draw_bounding_boxes(frame=frame, frame_index=frame_index)
LOG.debug("Page is not scrollable", url=url, num_screenshots=len(screenshots))
- screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame)
+ screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame, mode=mode)
screenshots.append(screenshot)
if draw_boxes:
@@ -144,28 +165,85 @@ class SkyvernFrame:
return await SkyvernFrame.evaluate(frame=frame, expression="() => document.location.href")
@staticmethod
- async def take_screenshot(
+ async def take_scrolling_screenshot(
page: Page,
- full_page: bool = False,
file_path: str | None = None,
timeout: float = settings.BROWSER_SCREENSHOT_TIMEOUT_MS,
+ mode: ScreenshotMode = ScreenshotMode.DETAILED,
+ scrolling_number: int = settings.MAX_NUM_SCREENSHOTS,
+ use_playwright_fullpage: bool = False, # TODO: THIS IS ONLY FOR EXPERIMENT. will be removed after experiment.
) -> bytes:
- return await _current_viewpoint_screenshot_helper(
- page=page, file_path=file_path, full_page=full_page, timeout=timeout
- )
+ if scrolling_number <= 0:
+ return await _current_viewpoint_screenshot_helper(
+ page=page, file_path=file_path, timeout=timeout, mode=mode
+ )
+
+ if use_playwright_fullpage:
+ return await _current_viewpoint_screenshot_helper(
+ page=page, file_path=file_path, timeout=timeout, full_page=True
+ )
+
+ if scrolling_number > settings.MAX_NUM_SCREENSHOTS:
+ LOG.warning(
+ "scrolling_number is greater than the max number of screenshots, setting it to the max number of screenshots",
+ scrolling_number=scrolling_number,
+ max_number=settings.MAX_NUM_SCREENSHOTS,
+ )
+ scrolling_number = settings.MAX_NUM_SCREENSHOTS
+
+ # use spilt screenshot with lite mode, isntead of fullpage screenshot from playwright
+ LOG.debug("Page is fully loaded, agent is about to generate the full page screenshot")
+ start_time = time.time()
+ async with asyncio.timeout(timeout):
+ screenshots = await _scrolling_screenshots_helper(page=page, mode=mode, max_number=scrolling_number)
+ images = []
+
+ for screenshot in screenshots:
+ with Image.open(BytesIO(screenshot)) as img:
+ img.load()
+ images.append(img)
+
+ total_height = sum(img.height for img in images)
+ max_width = max(img.width for img in images)
+
+ merged_img = Image.new("RGB", (max_width, total_height), color=(255, 255, 255))
+
+ current_y = 0
+ for img in images:
+ merged_img.paste(img, (0, current_y))
+ current_y += img.height
+
+ buffer = BytesIO()
+ merged_img.save(buffer, format="PNG")
+ buffer.seek(0)
+
+ img_data = buffer.read()
+ if file_path is not None:
+ with open(file_path, "wb") as f:
+ f.write(img_data)
+
+ end_time = time.time()
+ LOG.debug(
+ "Full page screenshot taking time",
+ screenshot_time=end_time - start_time,
+ file_path=file_path,
+ )
+ return img_data
@staticmethod
async def take_split_screenshots(
page: Page,
- url: str,
+ url: str | None = None,
draw_boxes: bool = False,
max_number: int = settings.MAX_NUM_SCREENSHOTS,
scroll: bool = True,
) -> list[bytes]:
if not scroll:
- return [await _current_viewpoint_screenshot_helper(page=page)]
+ return [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]
- return await _scrolling_screenshots_helper(page=page, url=url, max_number=max_number, draw_boxes=draw_boxes)
+ return await _scrolling_screenshots_helper(
+ page=page, url=url, max_number=max_number, draw_boxes=draw_boxes, mode=ScreenshotMode.DETAILED
+ )
@classmethod
async def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:
@@ -235,19 +313,21 @@ class SkyvernFrame:
)
return scroll_y_px
- async def scroll_to_next_page(self, draw_boxes: bool, frame: str, frame_index: int) -> float:
+ async def scroll_to_next_page(
+ self, draw_boxes: bool, frame: str, frame_index: int, need_overlap: bool = True
+ ) -> float:
"""
Scroll to the next page and take a screenshot.
:param drow_boxes: If True, draw bounding boxes around the elements.
:param page: Page instance to take the screenshot from.
:return: Screenshot of the page.
"""
- js_script = "async ([draw_boxes, frame, frame_index]) => await scrollToNextPage(draw_boxes, frame, frame_index)"
+ js_script = "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)"
scroll_y_px = await self.evaluate(
frame=self.frame,
expression=js_script,
timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS,
- arg=[draw_boxes, frame, frame_index],
+ arg=[draw_boxes, frame, frame_index, need_overlap],
)
return scroll_y_px