Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
current viewpoint screenshot and scrolling n screenshot (#2716)
Shuchang Zheng775da1887824
diff --git a/alembic/versions/2025_06_14_0649-2c6b27e8e961_add_screenshot_scroll_number.py b/alembic/versions/2025_06_14_0649-2c6b27e8e961_add_screenshot_scroll_number.pynew file mode 100644index 000000000..10380859f--- /dev/null+++ b/alembic/versions/2025_06_14_0649-2c6b27e8e961_add_screenshot_scroll_number.py@@ -0,0 +1,37 @@+"""add_screenshot_scroll_number++Revision ID: 2c6b27e8e961+Revises: 1517a4ba63fa+Create Date: 2025-06-14 06:49:43.628471+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "2c6b27e8e961"+down_revision: Union[str, None] = "1517a4ba63fa"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("observer_cruises", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))+ op.add_column("tasks", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))+ op.add_column("workflow_runs", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))+ op.add_column("workflows", sa.Column("max_screenshot_scrolling_times", sa.Integer(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("workflows", "max_screenshot_scrolling_times")+ op.drop_column("workflow_runs", "max_screenshot_scrolling_times")+ op.drop_column("tasks", "max_screenshot_scrolling_times")+ op.drop_column("observer_cruises", "max_screenshot_scrolling_times")+ # ### end Alembic commands ###diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex cef3eb67c..f5def2e6d 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -141,6 +141,7 @@ export type CreateTaskRequest = {totp_identifier?: string | null;application?: string | null;include_action_history_in_verification?: boolean | null;+ max_screenshot_scrolling_times?: number | null;};export type User = {@@ -293,6 +294,7 @@ export type WorkflowRunStatusApiResponse = {total_cost: number | null;task_v2: TaskV2 | null;workflow_title: string | null;+ max_screenshot_scrolling_times: number | null;};export type TaskGenerationApiResponse = {diff --git a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsxindex aa079dbac..565bbbdc2 100644--- a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx+++ b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskForm.tsx@@ -39,6 +39,7 @@ import {} from "./taskFormTypes";import { ProxySelector } from "@/components/ProxySelector";import { Switch } from "@/components/ui/switch";+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "@/routes/workflows/editor/nodes/Taskv2Node/types";type Props = {initialValues: CreateNewTaskFormValues;};@@ -80,6 +81,7 @@ function createTaskRequestObject(extracted_information_schema: extractedInformationSchema,totp_identifier: transform(formValues.totpIdentifier),error_code_mapping: errorCodeMapping,+ max_screenshot_scrolling_times: formValues.maxScreenshotScrollingTimes,include_action_history_in_verification:formValues.includeActionHistoryInVerification,};@@ -114,6 +116,8 @@ function CreateNewTaskForm({ initialValues }: Props) {...initialValues,maxStepsOverride: initialValues.maxStepsOverride ?? null,proxyLocation: initialValues.proxyLocation ?? ProxyLocation.Residential,+ maxScreenshotScrollingTimes:+ initialValues.maxScreenshotScrollingTimes ?? null,},});const { errors } = useFormState({ control: form.control });@@ -557,6 +561,45 @@ function CreateNewTaskForm({ initialValues }: Props) {);}}/>+ <FormField+ control={form.control}+ name="maxScreenshotScrollingTimes"+ render={({ field }) => (+ <FormItem>+ <div className="flex gap-16">+ <FormLabel>+ <div className="w-72">+ <h1 className="text-lg">+ Max Scrolling Screenshots+ </h1>+ <h2 className="text-base text-slate-400">+ {`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}+ </h2>+ </div>+ </FormLabel>+ <div className="w-full">+ <FormControl>+ <Input+ {...field}+ type="number"+ min={0}+ value={field.value ?? ""}+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}+ onChange={(event) => {+ const value =+ event.target.value === ""+ ? null+ : Number(event.target.value);+ field.onChange(value);+ }}+ />+ </FormControl>+ <FormMessage />+ </div>+ </div>+ </FormItem>+ )}+ /><Separator /><FormFieldcontrol={form.control}diff --git a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsxindex 50f05b5b2..15c7d9650 100644--- a/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx+++ b/skyvern-frontend/src/routes/tasks/create/CreateNewTaskFormPage.tsx@@ -62,6 +62,7 @@ function CreateNewTaskFormPage() {webhookCallbackUrl: null,proxyLocation: null,includeActionHistoryInVerification: null,+ maxScreenshotScrollingTimes: null,}}/></div>@@ -131,6 +132,7 @@ function CreateNewTaskFormPage() {includeActionHistoryInVerification:data.workflow_definition.blocks[0].include_action_history_in_verification,+ maxScreenshotScrollingTimes: data.max_screenshot_scrolling_times,}}/></div>diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsxindex 21d8c4668..3254973ae 100644--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx@@ -43,7 +43,10 @@ import {generateUniqueEmail,} from "../data/sampleTaskData";import { ExampleCasePill } from "./ExampleCasePill";-import { MAX_STEPS_DEFAULT } from "@/routes/workflows/editor/nodes/Taskv2Node/types";+import {+ MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT,+ MAX_STEPS_DEFAULT,+} from "@/routes/workflows/editor/nodes/Taskv2Node/types";function createTemplateTaskFromTaskGenerationParameters(values: TaskGenerationApiResponse,@@ -153,6 +156,8 @@ function PromptBox() {const [publishWorkflow, setPublishWorkflow] = useState(false);const [totpIdentifier, setTotpIdentifier] = useState("");const [maxStepsOverride, setMaxStepsOverride] = useState<string | null>(null);+ const [maxScreenshotScrollingTimes, setMaxScreenshotScrollingTimes] =+ useState<string | null>(null);const [showAdvancedSettings, setShowAdvancedSettings] = useState(false);const [dataSchema, setDataSchema] = useState<string | null>(null);@@ -167,6 +172,7 @@ function PromptBox() {proxy_location: proxyLocation,totp_identifier: totpIdentifier,publish_workflow: publishWorkflow,+ max_screenshot_scrolling_times: maxScreenshotScrollingTimes,extracted_information_schema: dataSchema? (() => {try {@@ -438,6 +444,21 @@ function PromptBox() {/></div></div>+ <div className="flex gap-16">+ <div className="w-48 shrink-0">+ <div className="text-sm">Max Scrolling Screenshots</div>+ <div className="text-xs text-slate-400">+ {`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}+ </div>+ </div>+ <Input+ value={maxScreenshotScrollingTimes ?? ""}+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}+ onChange={(event) => {+ setMaxScreenshotScrollingTimes(event.target.value);+ }}+ />+ </div></div></div>) : null}diff --git a/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx b/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsxindex 6755b5d1e..74bcdd300 100644--- a/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx+++ b/skyvern-frontend/src/routes/tasks/create/retry/RetryTask.tsx@@ -44,6 +44,8 @@ function RetryTask() {proxyLocation: task.request.proxy_location ?? null,includeActionHistoryInVerification:task.request.include_action_history_in_verification ?? false,+ maxScreenshotScrollingTimes:+ task.request.max_screenshot_scrolling_times ?? null,}}/></div>diff --git a/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts b/skyvern-frontend/src/routes/tasks/create/taskFormTypes.tsindex efafcbcfe..e58afc691 100644--- a/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts+++ b/skyvern-frontend/src/routes/tasks/create/taskFormTypes.ts@@ -15,6 +15,7 @@ const createNewTaskFormSchemaBase = z.object({errorCodeMapping: z.string().or(z.null()),proxyLocation: z.nativeEnum(ProxyLocation).or(z.null()),includeActionHistoryInVerification: z.boolean().or(z.null()).default(false),+ maxScreenshotScrollingTimes: z.number().or(z.null()).default(null),});const savedTaskFormSchemaBase = createNewTaskFormSchemaBase.extend({diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsxindex da36beb4b..99648871a 100644--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx@@ -28,12 +28,14 @@ import { WorkflowParameter } from "./types/workflowTypes";import { WorkflowParameterInput } from "./WorkflowParameterInput";import { AxiosError } from "axios";import { getLabelForWorkflowParameterType } from "./editor/workflowEditorUtils";+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "./editor/nodes/Taskv2Node/types";type Props = {workflowParameters: Array<WorkflowParameter>;initialValues: Record<string, unknown>;initialSettings: {proxyLocation: ProxyLocation;webhookCallbackUrl: string;+ maxScreenshotScrollingTimes: number | null;};};@@ -73,14 +75,20 @@ type RunWorkflowRequestBody = {proxy_location: ProxyLocation | null;webhook_callback_url?: string | null;browser_session_id: string | null;+ max_screenshot_scrolling_times?: number | null;};function getRunWorkflowRequestBody(values: RunWorkflowFormType,workflowParameters: Array<WorkflowParameter>,): RunWorkflowRequestBody {- const { webhookCallbackUrl, proxyLocation, browserSessionId, ...parameters } =- values;+ const {+ webhookCallbackUrl,+ proxyLocation,+ browserSessionId,+ maxScreenshotScrollingTimes,+ ...parameters+ } = values;const parsedParameters = parseValuesForWorkflowRun(parameters,@@ -95,6 +103,10 @@ function getRunWorkflowRequestBody(browser_session_id: bsi,};+ if (maxScreenshotScrollingTimes) {+ body.max_screenshot_scrolling_times = maxScreenshotScrollingTimes;+ }+if (webhookCallbackUrl) {body.webhook_callback_url = webhookCallbackUrl;}@@ -106,6 +118,7 @@ type RunWorkflowFormType = Record<string, unknown> & {webhookCallbackUrl: string;proxyLocation: ProxyLocation;browserSessionId: string | null;+ maxScreenshotScrollingTimes: number | null;};function RunWorkflowForm({@@ -127,6 +140,7 @@ function RunWorkflowForm({webhookCallbackUrl: initialSettings.webhookCallbackUrl,proxyLocation: initialSettings.proxyLocation,browserSessionId: browserSessionIdDefault,+ maxScreenshotScrollingTimes: initialSettings.maxScreenshotScrollingTimes,},});const apiCredential = useApiCredential();@@ -177,6 +191,7 @@ function RunWorkflowForm({webhookCallbackUrl,proxyLocation,browserSessionId,+ maxScreenshotScrollingTimes,...parameters} = values;@@ -189,6 +204,7 @@ function RunWorkflowForm({webhookCallbackUrl,proxyLocation,browserSessionId,+ maxScreenshotScrollingTimes,});}@@ -392,6 +408,48 @@ function RunWorkflowForm({);}}/>+ <FormField+ key="maxScreenshotScrollingTimes"+ control={form.control}+ name="maxScreenshotScrollingTimes"+ render={({ field }) => {+ return (+ <FormItem>+ <div className="flex gap-16">+ <FormLabel>+ <div className="w-72">+ <div className="flex items-center gap-2 text-lg">+ Max Scrolling Screenshots+ </div>+ <h2 className="text-sm text-slate-400">+ {`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}+ </h2>+ </div>+ </FormLabel>+ <div className="w-full space-y-2">+ <FormControl>+ <Input+ {...field}+ type="number"+ min={0}+ value={field.value ?? ""}+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}+ onChange={(event) => {+ const value =+ event.target.value === ""+ ? null+ : Number(event.target.value);+ field.onChange(value);+ }}+ />+ </FormControl>+ <FormMessage />+ </div>+ </div>+ </FormItem>+ );+ }}+ /></div><div className="flex justify-end gap-2"><CopyApiCommandDropdowndiff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsxindex 5d2f59cc0..deb39eee5 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx@@ -100,6 +100,8 @@ function WorkflowRun() {const parameters = workflowRun?.parameters ?? {};const proxyLocation =workflowRun?.proxy_location ?? ProxyLocation.Residential;+ const maxScreenshotScrollingTimes =+ workflowRun?.max_screenshot_scrolling_times ?? null;const title = workflowIsLoading ? (<Skeleton className="h-9 w-48" />@@ -244,6 +246,7 @@ function WorkflowRun() {data: parameters,proxyLocation,webhookCallbackUrl: workflowRun?.webhook_callback_url ?? "",+ maxScreenshotScrollingTimes,}}><PlayIcon className="mr-2 h-4 w-4" />diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsxindex 8de6821f4..2fe037d3e 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowRunParameters.tsx@@ -30,6 +30,8 @@ function WorkflowRunParameters() {const proxyLocation = location.state? (location.state.proxyLocation as ProxyLocation): null;+ const maxScreenshotScrollingTimes =+ location.state?.maxScreenshotScrollingTimes ?? null;const webhookCallbackUrl = location.state? (location.state.webhookCallbackUrl as string)@@ -109,6 +111,10 @@ function WorkflowRunParameters() {ProxyLocation.Residential,webhookCallbackUrl:webhookCallbackUrl ?? workflow.webhook_callback_url ?? "",+ maxScreenshotScrollingTimes:+ maxScreenshotScrollingTimes ??+ workflow.max_screenshot_scrolling_times ??+ null,}}/></div>diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsxindex 357add8c1..a67653ef0 100644--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx@@ -289,6 +289,8 @@ function FlowRenderer({webhook_callback_url: data.settings.webhookCallbackUrl,persist_browser_session: data.settings.persistBrowserSession,model: data.settings.model,+ max_screenshot_scrolling_times:+ data.settings.maxScreenshotScrollingTimes,totp_verification_url: workflow.totp_verification_url,workflow_definition: {parameters: data.parameters,diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsxindex 144ac78d2..322a3ef29 100644--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx@@ -60,6 +60,7 @@ function WorkflowEditor() {proxyLocation: workflow.proxy_location,webhookCallbackUrl: workflow.webhook_callback_url,model: workflow.model,+ maxScreenshotScrollingTimes: workflow.max_screenshot_scrolling_times,};const elements = getElements(diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsxindex b2a2774d0..1e0a97f53 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx@@ -20,6 +20,7 @@ import { Separator } from "@/components/ui/separator";import { ModelsResponse } from "@/api/types";import { ModelSelector } from "@/components/ModelSelector";import { WorkflowModel } from "@/routes/workflows/types/workflowTypes";+import { MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT } from "../Taskv2Node/types";function StartNode({ id, data }: NodeProps<StartNode>) {const credentialGetter = useCredentialGetter();@@ -51,6 +52,9 @@ function StartNode({ id, data }: NodeProps<StartNode>) {? data.persistBrowserSession: false,model: data.withWorkflowSettings ? data.model : workflowModel,+ maxScreenshotScrollingTimes: data.withWorkflowSettings+ ? data.maxScreenshotScrollingTimes+ : null,});function handleChange(key: string, value: unknown) {@@ -130,6 +134,26 @@ function StartNode({ id, data }: NodeProps<StartNode>) {/></div></div>+ <div className="space-y-2">+ <div className="flex items-center gap-2">+ <Label>Max Scrolling Screenshots</Label>+ <HelpTooltip+ content={`The maximum number of times to scroll down the page to take merged screenshots after action. Default is ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}. If it's set to 0, it will take the current viewport screenshot.`}+ />+ </div>+ <Input+ value={inputs.maxScreenshotScrollingTimes ?? ""}+ placeholder={`Default: ${MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT}`}+ onChange={(event) => {+ const value =+ event.target.value === ""+ ? null+ : Number(event.target.value);++ handleChange("maxScreenshotScrollingTimes", value);+ }}+ />+ </div></div></AccordionContent></AccordionItem>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.tsindex 11f54be49..6fa5ae31c 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/types.ts@@ -9,6 +9,7 @@ export type WorkflowStartNodeData = {proxyLocation: ProxyLocation;persistBrowserSession: boolean;model: WorkflowModel | null;+ maxScreenshotScrollingTimes: number | null;editable: boolean;};diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.tsindex cafdea7da..f3cc549b1 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/types.ts@@ -2,6 +2,7 @@ import { Node } from "@xyflow/react";import { NodeBaseData } from "../types";export const MAX_STEPS_DEFAULT = 25;+export const MAX_SCREENSHOT_SCROLLING_TIMES_DEFAULT = 3;export type Taskv2NodeData = NodeBaseData & {prompt: string;@@ -9,6 +10,7 @@ export type Taskv2NodeData = NodeBaseData & {totpVerificationUrl: string | null;totpIdentifier: string | null;maxSteps: number | null;+ maxScreenshotScrollingTimes: number | null;};export type Taskv2Node = Node<Taskv2NodeData, "taskv2">;@@ -23,6 +25,7 @@ export const taskv2NodeDefaultData: Taskv2NodeData = {totpVerificationUrl: null,maxSteps: MAX_STEPS_DEFAULT,model: null,+ maxScreenshotScrollingTimes: null,};export function isTaskV2Node(node: Node): node is Taskv2Node {diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex cc2392950..ee2c0e2a1 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -251,6 +251,7 @@ function convertToNode(maxSteps: block.max_steps,totpIdentifier: block.totp_identifier,totpVerificationUrl: block.totp_verification_url,+ maxScreenshotScrollingTimes: null,},};}@@ -662,6 +663,7 @@ function getElements(proxyLocation: settings.proxyLocation ?? ProxyLocation.Residential,webhookCallbackUrl: settings.webhookCallbackUrl ?? "",model: settings.model,+ maxScreenshotScrollingTimes: settings.maxScreenshotScrollingTimes,editable,}),);@@ -1322,6 +1324,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {proxyLocation: ProxyLocation.Residential,webhookCallbackUrl: null,model: null,+ maxScreenshotScrollingTimes: null,};const startNodes = nodes.filter(isStartNode);const startNodeWithWorkflowSettings = startNodes.find(@@ -1337,6 +1340,7 @@ function getWorkflowSettings(nodes: Array<AppNode>): WorkflowSettings {proxyLocation: data.proxyLocation,webhookCallbackUrl: data.webhookCallbackUrl,model: data.model,+ maxScreenshotScrollingTimes: data.maxScreenshotScrollingTimes,};}return defaultSettings;@@ -1992,6 +1996,7 @@ function convert(workflow: WorkflowApiResponse): WorkflowCreateYAMLRequest {persist_browser_session: workflow.persist_browser_session,model: workflow.model,totp_verification_url: workflow.totp_verification_url,+ max_screenshot_scrolling_times: workflow.max_screenshot_scrolling_times,workflow_definition: {parameters: convertParametersToParameterYAML(userParameters),blocks: convertBlocksToBlockYAML(workflow.workflow_definition.blocks),diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.tsindex 430b72ad4..feef5c59a 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts@@ -470,6 +470,7 @@ export type WorkflowApiResponse = {model: WorkflowModel | null;totp_verification_url: string | null;totp_identifier: string | null;+ max_screenshot_scrolling_times: number | null;created_at: string;modified_at: string;deleted_at: string | null;@@ -480,6 +481,7 @@ export type WorkflowSettings = {webhookCallbackUrl: string | null;persistBrowserSession: boolean;model: WorkflowModel | null;+ maxScreenshotScrollingTimes: number | null;};export type WorkflowModel = JsonObjectExtendable<{ model_name: string }>;diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.tsindex 9df7811df..e7a405538 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts@@ -12,6 +12,7 @@ export type WorkflowCreateYAMLRequest = {totp_verification_url?: string | null;workflow_definition: WorkflowDefinitionYAML;is_saved_task?: boolean;+ max_screenshot_scrolling_times?: number | null;};export type WorkflowDefinitionYAML = {diff --git a/skyvern/constants.py b/skyvern/constants.pyindex 52de64b10..d1a69a9b6 100644--- a/skyvern/constants.py+++ b/skyvern/constants.py@@ -20,6 +20,7 @@ AUTO_COMPLETION_POTENTIAL_VALUES_COUNT = 3DROPDOWN_MENU_MAX_DISTANCE = 100BROWSER_DOWNLOADING_SUFFIX = ".crdownload"MAX_UPLOAD_FILE_COUNT = 50+DEFAULT_MAX_SCREENSHOT_SCROLLING_TIMES = 3# reserved fields for navigation payloadSPECIAL_FIELD_VERIFICATION_CODE = "verification_code"diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 8dec96069..73ffd6fe7 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -19,6 +19,7 @@ from skyvern import analyticsfrom skyvern.config import settingsfrom skyvern.constants import (BROWSER_DOWNLOADING_SUFFIX,+ DEFAULT_MAX_SCREENSHOT_SCROLLING_TIMES,GET_DOWNLOADED_FILES_TIMEOUT,SAVE_DOWNLOADED_FILES_TIMEOUT,SCRAPE_TYPE_ORDER,@@ -181,6 +182,7 @@ class ForgeAgent:error_code_mapping=task_block.error_code_mapping,include_action_history_in_verification=task_block.include_action_history_in_verification,model=task_block.model,+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,)LOG.info("Created a new task for workflow run",@@ -237,6 +239,7 @@ class ForgeAgent:application=task_request.application,include_action_history_in_verification=task_request.include_action_history_in_verification,model=task_request.model,+ max_screenshot_scrolling_times=task_request.max_screenshot_scrolling_times,)LOG.info("Created new task",@@ -1650,12 +1653,22 @@ class ForgeAgent:if not working_page:raise BrowserStateMissingPage()- fullpage_screenshot = True+ context = skyvern_context.ensure_context()+ scrolling_number = context.max_screenshot_scrolling_times+ if scrolling_number is None:+ scrolling_number = DEFAULT_MAX_SCREENSHOT_SCROLLING_TIMES+if engine in CUA_ENGINES:- fullpage_screenshot = False+ scrolling_number = 0try:- screenshot = await browser_state.take_screenshot(full_page=fullpage_screenshot)+ screenshot = await browser_state.take_post_action_screenshot(+ scrolling_number=scrolling_number,+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "ENABLE_PLAYWRIGHT_FULLPAGE",+ str(task.organization_id),+ ),+ )await app.ARTIFACT_MANAGER.create_artifact(step=step,artifact_type=ArtifactType.SCREENSHOT_ACTION,@@ -2135,7 +2148,12 @@ class ForgeAgent:browser_state = app.BROWSER_MANAGER.get_for_task(task.task_id)if browser_state is not None and await browser_state.get_working_page() is not None:try:- screenshot = await browser_state.take_screenshot(full_page=True)+ screenshot = await browser_state.take_fullpage_screenshot(+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "ENABLE_PLAYWRIGHT_FULLPAGE",+ str(task.organization_id),+ )+ )await app.ARTIFACT_MANAGER.create_artifact(step=last_step,artifact_type=ArtifactType.SCREENSHOT_FINAL,diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 819cb3ab8..d51b9fa8d 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -23,6 +23,7 @@ class SkyvernContext:hashed_href_map: dict[str, str] = field(default_factory=dict)refresh_working_page: bool = Falseframe_index_map: dict[Frame, int] = field(default_factory=dict)+ max_screenshot_scrolling_times: int | None = Nonedef __repr__(self) -> str:return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override})"diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.pyindex 0f035b8b8..a3697c6ae 100644--- a/skyvern/forge/sdk/db/client.py+++ b/skyvern/forge/sdk/db/client.py@@ -149,6 +149,7 @@ class AgentDB:application: str | None = None,include_action_history_in_verification: bool | None = None,model: dict[str, Any] | None = None,+ max_screenshot_scrolling_times: int | None = None,) -> Task:try:async with self.Session() as session:@@ -176,6 +177,7 @@ class AgentDB:application=application,include_action_history_in_verification=include_action_history_in_verification,model=model,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,)session.add(new_task)await session.commit()@@ -1217,6 +1219,7 @@ class AgentDB:description: str | None = None,proxy_location: ProxyLocation | None = None,webhook_callback_url: str | None = None,+ max_screenshot_scrolling_times: int | None = None,totp_verification_url: str | None = None,totp_identifier: str | None = None,persist_browser_session: bool = False,@@ -1236,6 +1239,7 @@ class AgentDB:webhook_callback_url=webhook_callback_url,totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,persist_browser_session=persist_browser_session,model=model,is_saved_task=is_saved_task,@@ -1479,6 +1483,7 @@ class AgentDB:totp_verification_url: str | None = None,totp_identifier: str | None = None,parent_workflow_run_id: str | None = None,+ max_screenshot_scrolling_times: int | None = None,) -> WorkflowRun:try:async with self.Session() as session:@@ -1492,6 +1497,7 @@ class AgentDB:totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,parent_workflow_run_id=parent_workflow_run_id,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,)session.add(workflow_run)await session.commit()@@ -2436,6 +2442,7 @@ class AgentDB:extracted_information_schema: dict | list | str | None = None,error_code_mapping: dict | None = None,model: dict[str, Any] | None = None,+ max_screenshot_scrolling_times: int | None = None,) -> TaskV2:async with self.Session() as session:new_task_v2 = TaskV2Model(@@ -2452,6 +2459,7 @@ class AgentDB:error_code_mapping=error_code_mapping,organization_id=organization_id,model=model,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,)session.add(new_task_v2)await session.commit()diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex a5cbf913f..b41c1b42e 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -88,6 +88,7 @@ class TaskModel(Base):queued_at = Column(DateTime, nullable=True)started_at = Column(DateTime, nullable=True)finished_at = Column(DateTime, nullable=True)+ max_screenshot_scrolling_times = Column(Integer, nullable=True)created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False, index=True)modified_at = Column(DateTime,@@ -218,6 +219,7 @@ class WorkflowModel(Base):workflow_definition = Column(JSON, nullable=False)proxy_location = Column(String)webhook_callback_url = Column(String)+ max_screenshot_scrolling_times = Column(Integer, nullable=True)totp_verification_url = Column(String)totp_identifier = Column(String)persist_browser_session = Column(Boolean, default=False, nullable=False)@@ -254,6 +256,7 @@ class WorkflowRunModel(Base):webhook_callback_url = Column(String)totp_verification_url = Column(String)totp_identifier = Column(String)+ max_screenshot_scrolling_times = Column(Integer, nullable=True)queued_at = Column(DateTime, nullable=True)started_at = Column(DateTime, nullable=True)@@ -621,6 +624,7 @@ class TaskV2Model(Base):extracted_information_schema = Column(JSON, nullable=True)error_code_mapping = Column(JSON, nullable=True)max_steps = Column(Integer, nullable=True)+ max_screenshot_scrolling_times = Column(Integer, nullable=True)queued_at = Column(DateTime, nullable=True)started_at = Column(DateTime, nullable=True)diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 5bfe3454b..441a07e32 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -142,6 +142,7 @@ def convert_to_task(task_obj: TaskModel, debug_enabled: bool = False, workflow_pqueued_at=task_obj.queued_at,started_at=task_obj.started_at,finished_at=task_obj.finished_at,+ max_screenshot_scrolling_times=task_obj.max_screenshot_scrolling_times,)return task@@ -238,6 +239,7 @@ def convert_to_workflow(workflow_model: WorkflowModel, debug_enabled: bool = Falpersist_browser_session=workflow_model.persist_browser_session,model=workflow_model.model,proxy_location=(ProxyLocation(workflow_model.proxy_location) if workflow_model.proxy_location else None),+ max_screenshot_scrolling_times=workflow_model.max_screenshot_scrolling_times,version=workflow_model.version,is_saved_task=workflow_model.is_saved_task,description=workflow_model.description,@@ -278,6 +280,7 @@ def convert_to_workflow_run(created_at=workflow_run_model.created_at,modified_at=workflow_run_model.modified_at,workflow_title=workflow_title,+ max_screenshot_scrolling_times=workflow_run_model.max_screenshot_scrolling_times,)diff --git a/skyvern/forge/sdk/executor/async_executor.py b/skyvern/forge/sdk/executor/async_executor.pyindex a13aa161c..b579fafda 100644--- a/skyvern/forge/sdk/executor/async_executor.py+++ b/skyvern/forge/sdk/executor/async_executor.py@@ -107,6 +107,7 @@ class BackgroundTaskExecutor(AsyncExecutor):context.task_id = task.task_idcontext.organization_id = organization_idcontext.max_steps_override = max_steps_override+ context.max_screenshot_scrolling_times = task.max_screenshot_scrolling_timesif background_tasks:await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex e14417969..99a922822 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -166,6 +166,7 @@ async def run_task(totp_identifier=run_request.totp_identifier,include_action_history_in_verification=run_request.include_action_history_in_verification,model=run_request.model,+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,)task_v1_response = await task_v1_service.run_task(task=task_v1_request,@@ -203,6 +204,7 @@ async def run_task(data_extraction_schema=task_v1_response.extracted_information_schema,error_code_mapping=task_v1_response.error_code_mapping,browser_session_id=run_request.browser_session_id,+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,),)if run_request.engine == RunEngine.skyvern_v2:@@ -221,6 +223,7 @@ async def run_task(error_code_mapping=run_request.error_code_mapping,create_task_run=True,model=run_request.model,+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,)except MissingBrowserAddressError as e:raise HTTPException(status_code=400, detail=str(e)) from e@@ -263,6 +266,7 @@ async def run_task(error_code_mapping=task_v2.error_code_mapping,data_extraction_schema=task_v2.extracted_information_schema,publish_workflow=run_request.publish_workflow,+ max_screenshot_scrolling_times=run_request.max_screenshot_scrolling_times,),)LOG.error("Invalid agent engine", engine=run_request.engine, organization_id=current_org.organization_id)@@ -318,6 +322,7 @@ async def run_workflow(totp_identifier=workflow_run_request.totp_identifier,totp_url=workflow_run_request.totp_url,browser_session_id=workflow_run_request.browser_session_id,+ max_screenshot_scrolling_times=workflow_run_request.max_screenshot_scrolling_times,)try:@@ -1765,6 +1770,7 @@ async def run_task_v2(create_task_run=True,extracted_information_schema=data.extracted_information_schema,error_code_mapping=data.error_code_mapping,+ max_screenshot_scrolling_times=data.max_screenshot_scrolling_times,)except MissingBrowserAddressError as e:raise HTTPException(status_code=400, detail=str(e)) from ediff --git a/skyvern/forge/sdk/schemas/task_v2.py b/skyvern/forge/sdk/schemas/task_v2.pyindex 1978242a2..8cbf418ac 100644--- a/skyvern/forge/sdk/schemas/task_v2.py+++ b/skyvern/forge/sdk/schemas/task_v2.py@@ -48,6 +48,8 @@ class TaskV2(BaseModel):queued_at: datetime | None = Nonestarted_at: datetime | None = Nonefinished_at: datetime | None = None+ max_screenshot_scrolling_times: int | None = None+created_at: datetimemodified_at: datetime@@ -147,6 +149,7 @@ class TaskV2Request(BaseModel):publish_workflow: bool = Falseextracted_information_schema: dict | list | str | None = Noneerror_code_mapping: dict[str, str] | None = None+ max_screenshot_scrolling_times: int | None = None@field_validator("url", "webhook_callback_url", "totp_verification_url")@classmethoddiff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.pyindex a8268f9f7..31b0a7e5b 100644--- a/skyvern/forge/sdk/schemas/tasks.py+++ b/skyvern/forge/sdk/schemas/tasks.py@@ -96,6 +96,11 @@ class TaskBase(BaseModel):description="Whether to include the action history when verifying the task is complete",examples=[True, False],)+ max_screenshot_scrolling_times: int | None = Field(+ default=None,+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's None or 0, it takes the current viewpoint screenshot.",+ examples=[10],+ )class TaskRequest(TaskBase):@@ -314,6 +319,7 @@ class Task(TaskBase):errors=self.errors,max_steps_per_run=self.max_steps_per_run,workflow_run_id=self.workflow_run_id,+ max_screenshot_scrolling_times=self.max_screenshot_scrolling_times,)@@ -337,6 +343,7 @@ class TaskResponse(BaseModel):queued_at: datetime | None = Nonestarted_at: datetime | None = Nonefinished_at: datetime | None = None+ max_screenshot_scrolling_times: int | None = Noneclass TaskOutput(BaseModel):diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 6b173a3ab..ed16d4524 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -307,7 +307,12 @@ class Block(BaseModel, abc.ABC):if not browser_state:LOG.warning("No browser state found when creating workflow_run_block", workflow_run_id=workflow_run_id)else:- screenshot = await browser_state.take_screenshot(full_page=True)+ screenshot = await browser_state.take_fullpage_screenshot(+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "ENABLE_PLAYWRIGHT_FULLPAGE",+ str(organization_id),+ )+ )if screenshot:await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(workflow_run_block=workflow_run_block,@@ -569,8 +574,15 @@ class BaseTaskBlock(Block):browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(workflow_run=workflow_run, url=self.url, browser_session_id=browser_session_id)+ # assert that the browser state is not None, otherwise we can't go through typing+ assert browser_state is not None# add screenshot artifact for the first task- screenshot = await browser_state.take_screenshot(full_page=True)+ screenshot = await browser_state.take_fullpage_screenshot(+ use_playwright_fullpage=app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "ENABLE_PLAYWRIGHT_FULLPAGE",+ str(organization_id),+ )+ )if screenshot:await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(workflow_run_block=workflow_run_block,@@ -2486,6 +2498,7 @@ class TaskV2Block(Block):proxy_location=workflow_run.proxy_location,totp_identifier=self.totp_identifier,totp_verification_url=self.totp_verification_url,+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,)await app.DATABASE.update_task_v2(task_v2.observer_cruise_id, status=TaskV2Status.queued, organization_id=organization_id@@ -2517,6 +2530,7 @@ class TaskV2Block(Block):workflow_permanent_id=workflow_run.workflow_permanent_id,workflow_run_id=workflow_run_id,browser_session_id=browser_session_id,+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,))result_dict = Nonediff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex 1880446c4..0c9002dc5 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -22,6 +22,7 @@ class WorkflowRequestBody(BaseModel):totp_verification_url: str | None = Nonetotp_identifier: str | None = Nonebrowser_session_id: str | None = None+ max_screenshot_scrolling_times: int | None = None@field_validator("webhook_callback_url", "totp_verification_url")@classmethod@@ -76,6 +77,7 @@ class Workflow(BaseModel):persist_browser_session: bool = Falsemodel: dict[str, Any] | None = Nonestatus: WorkflowStatus = WorkflowStatus.published+ max_screenshot_scrolling_times: int | None = Nonecreated_at: datetimemodified_at: datetime@@ -115,6 +117,7 @@ class WorkflowRun(BaseModel):failure_reason: str | None = Noneparent_workflow_run_id: str | None = Noneworkflow_title: str | None = None+ max_screenshot_scrolling_times: int | None = Nonequeued_at: datetime | None = Nonestarted_at: datetime | None = None@@ -162,3 +165,4 @@ class WorkflowRunResponseBase(BaseModel):task_v2: TaskV2 | None = Noneworkflow_title: str | None = Nonebrowser_session_id: str | None = None+ max_screenshot_scrolling_times: int | None = Nonediff --git a/skyvern/forge/sdk/workflow/models/yaml.py b/skyvern/forge/sdk/workflow/models/yaml.pyindex a96cb38a0..d2ce6ae46 100644--- a/skyvern/forge/sdk/workflow/models/yaml.py+++ b/skyvern/forge/sdk/workflow/models/yaml.py@@ -424,4 +424,5 @@ class WorkflowCreateYAMLRequest(BaseModel):model: dict[str, Any] | None = Noneworkflow_definition: WorkflowDefinitionYAMLis_saved_task: bool = False+ max_screenshot_scrolling_times: int | None = Nonestatus: WorkflowStatus = WorkflowStatus.publisheddiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 844d2a557..45819c275 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -169,6 +169,7 @@ class WorkflowService:organization_id=workflow.organization_id,proxy_location=workflow_request.proxy_location,webhook_callback_url=workflow_request.webhook_callback_url,+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolling_times,)skyvern_context.set(SkyvernContext(@@ -178,6 +179,7 @@ class WorkflowService:workflow_id=workflow_id,workflow_run_id=workflow_run.workflow_run_id,max_steps_override=max_steps_override,+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolling_times,))@@ -577,6 +579,7 @@ class WorkflowService:workflow_definition: WorkflowDefinition,description: str | None = None,proxy_location: ProxyLocation | None = None,+ max_screenshot_scrolling_times: int | None = None,webhook_callback_url: str | None = None,totp_verification_url: str | None = None,totp_identifier: str | None = None,@@ -594,6 +597,7 @@ class WorkflowService:description=description,proxy_location=proxy_location,webhook_callback_url=webhook_callback_url,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,persist_browser_session=persist_browser_session,@@ -767,6 +771,7 @@ class WorkflowService:totp_verification_url=workflow_request.totp_verification_url,totp_identifier=workflow_request.totp_identifier,parent_workflow_run_id=parent_workflow_run_id,+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolling_times,)async def mark_workflow_run_as_completed(self, workflow_run_id: str) -> WorkflowRun:@@ -1180,6 +1185,7 @@ class WorkflowService:total_steps=total_steps,total_cost=total_cost,workflow_title=workflow.title,+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,)async def clean_up_workflow(@@ -1453,6 +1459,7 @@ class WorkflowService:totp_identifier=request.totp_identifier,persist_browser_session=request.persist_browser_session,model=request.model,+ max_screenshot_scrolling_times=request.max_screenshot_scrolling_times,workflow_permanent_id=workflow_permanent_id,version=existing_version + 1,is_saved_task=request.is_saved_task,@@ -1470,6 +1477,7 @@ class WorkflowService:totp_identifier=request.totp_identifier,persist_browser_session=request.persist_browser_session,model=request.model,+ max_screenshot_scrolling_times=request.max_screenshot_scrolling_times,is_saved_task=request.is_saved_task,status=request.status,)diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.pyindex 116f1c672..dd809eb73 100644--- a/skyvern/schemas/runs.py+++ b/skyvern/schemas/runs.py@@ -279,6 +279,10 @@ class TaskRunRequest(BaseModel):include_action_history_in_verification: bool | None = Field(default=False, description="Whether to include action history when verifying that the task is complete")+ max_screenshot_scrolling_times: int | None = Field(+ default=None,+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's None or 0, it takes the current viewpoint screenshot.",+ )@field_validator("url", "webhook_url", "totp_url")@classmethod@@ -326,6 +330,10 @@ class WorkflowRunRequest(BaseModel):default=None,description="ID of a Skyvern browser session to reuse, having it continue from the current screen state",)+ max_screenshot_scrolling_times: int | None = Field(+ default=None,+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's None or 0, it takes the current viewpoint screenshot.",+ )@field_validator("webhook_url", "totp_url")@classmethod@@ -368,9 +376,11 @@ class BaseRunResponse(BaseModel):examples=["https://app.skyvern.com/tasks/tsk_123", "https://app.skyvern.com/workflows/wpid_123/wr_123"],)browser_session_id: str | None = Field(+ default=None, description="ID of the Skyvern persistent browser session used for this run", examples=["pbs_123"]+ )+ max_screenshot_scrolling_times: int | None = Field(default=None,- description="ID of the Skyvern persistent browser session used for this run",- examples=["pbs_123"],+ description="Scroll down n times to get the merged screenshot of the page after taking an action. When it's NONE or 0, it takes the current view point screenshot.",)diff --git a/skyvern/services/run_service.py b/skyvern/services/run_service.pyindex a301dd24f..66f8d257f 100644--- a/skyvern/services/run_service.py+++ b/skyvern/services/run_service.py@@ -67,6 +67,7 @@ async def get_run_response(run_id: str, organization_id: str | None = None) -> Rmax_steps=task_v1_response.max_steps_per_run,data_extraction_schema=task_v1_response.request.extracted_information_schema,error_code_mapping=task_v1_response.request.error_code_mapping,+ max_screenshot_scrolling_times=task_v1_response.request.max_screenshot_scrolling_times,),)elif run.task_run_type == RunType.task_v2:diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex 1b6bc7051..e11cdfa4a 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -165,6 +165,7 @@ async def initialize_task_v2(error_code_mapping: dict | None = None,create_task_run: bool = False,model: dict[str, Any] | None = None,+ max_screenshot_scrolling_times: int | None = None,) -> TaskV2:task_v2 = await app.DATABASE.create_task_v2(prompt=user_prompt,@@ -176,11 +177,13 @@ async def initialize_task_v2(extracted_information_schema=extracted_information_schema,error_code_mapping=error_code_mapping,model=model,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,)# set task_v2_id in contextcontext = skyvern_context.current()if context:context.task_v2_id = task_v2.observer_cruise_id+ context.max_screenshot_scrolling_times = max_screenshot_scrolling_timesthought = await app.DATABASE.create_thought(task_v2_id=task_v2.observer_cruise_id,@@ -221,7 +224,9 @@ async def initialize_task_v2()workflow_run = await app.WORKFLOW_SERVICE.setup_workflow_run(request_id=None,- workflow_request=WorkflowRequestBody(),+ workflow_request=WorkflowRequestBody(+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,+ ),workflow_permanent_id=new_workflow.workflow_permanent_id,organization=organization,version=None,@@ -454,6 +459,7 @@ async def run_task_v2_helper(request_id=request_id,task_v2_id=task_v2_id,browser_session_id=browser_session_id,+ max_screenshot_scrolling_times=task_v2.max_screenshot_scrolling_times,))@@ -771,6 +777,7 @@ async def run_task_v2_helper(proxy_location=task_v2.proxy_location or ProxyLocation.RESIDENTIAL,workflow_definition=workflow_definition_yaml,status=workflow.status,+ max_screenshot_scrolling_times=task_v2.max_screenshot_scrolling_times,)LOG.info("Creating workflow from request", workflow_create_request=workflow_create_request)workflow = await app.WORKFLOW_SERVICE.create_workflow_from_request(diff --git a/skyvern/services/workflow_service.py b/skyvern/services/workflow_service.pyindex fff09c9bb..47fd5ec0a 100644--- a/skyvern/services/workflow_service.py+++ b/skyvern/services/workflow_service.py@@ -97,6 +97,7 @@ async def get_workflow_run_response(webhook_url=workflow_run.webhook_callback_url or None,totp_url=workflow_run.totp_verification_url or None,totp_identifier=workflow_run.totp_identifier,+ max_screenshot_scrolling_times=workflow_run.max_screenshot_scrolling_times,# TODO: add browser session id),)diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex d5ea7d106..76c0cc4b9 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -35,7 +35,7 @@ from skyvern.exceptions import (from skyvern.forge.sdk.api.files import get_download_dir, make_temp_directoryfrom skyvern.forge.sdk.core.skyvern_context import current, ensure_contextfrom skyvern.schemas.runs import ProxyLocation, get_tzinfo_from_proxy-from skyvern.webeye.utils.page import SkyvernFrame+from skyvern.webeye.utils.page import ScreenshotMode, SkyvernFrameLOG = structlog.get_logger()@@ -865,6 +865,30 @@ class BrowserState:except asyncio.TimeoutError:LOG.error("Timeout to close playwright, might leave the broswer opening forever")- async def take_screenshot(self, full_page: bool = False, file_path: str | None = None) -> bytes:+ async def take_fullpage_screenshot(+ self,+ file_path: str | None = None,+ use_playwright_fullpage: bool = False, # TODO: THIS IS ONLY FOR EXPERIMENT. will be removed after experiment.+ ) -> bytes:+ page = await self.__assert_page()+ return await SkyvernFrame.take_scrolling_screenshot(+ page=page,+ file_path=file_path,+ mode=ScreenshotMode.LITE,+ use_playwright_fullpage=use_playwright_fullpage,+ )++ async def take_post_action_screenshot(+ self,+ scrolling_number: int,+ file_path: str | None = None,+ use_playwright_fullpage: bool = False, # TODO: THIS IS ONLY FOR EXPERIMENT. will be removed after experiment.+ ) -> bytes:page = await self.__assert_page()- return await SkyvernFrame.take_screenshot(page=page, full_page=full_page, file_path=file_path)+ return await SkyvernFrame.take_scrolling_screenshot(+ page=page,+ file_path=file_path,+ mode=ScreenshotMode.LITE,+ scrolling_number=scrolling_number,+ use_playwright_fullpage=use_playwright_fullpage,+ )diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 98b8f7168..5871b29a3 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -2115,13 +2115,14 @@ async function scrollToNextPage(draw_boxes,frame = "main.frame",frame_index = undefined,+ need_overlap = true,) {// remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again// return true if there is a next page, false otherwiseremoveBoundingBoxes();window.scrollBy({left: 0,- top: window.innerHeight - 200,+ top: need_overlap ? window.innerHeight - 200 : window.innerHeight,behavior: "instant",});if (draw_boxes) {diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex d1f9546db..7bf425303 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -2,9 +2,12 @@ from __future__ import annotationsimport asyncioimport time+from enum import StrEnum+from io import BytesIOfrom typing import Anyimport structlog+from PIL import Imagefrom playwright._impl._errors import TimeoutErrorfrom playwright.async_api import ElementHandle, Frame, Page@@ -31,17 +34,24 @@ def load_js_script() -> str:JS_FUNCTION_DEFS = load_js_script()+class ScreenshotMode(StrEnum):+ LITE = "lite"+ DETAILED = "detailed"++async def _current_viewpoint_screenshot_helper(page: Page,file_path: str | None = None,full_page: bool = False,timeout: float = settings.BROWSER_SCREENSHOT_TIMEOUT_MS,+ mode: ScreenshotMode = ScreenshotMode.DETAILED,) -> bytes:if page.is_closed():raise FailedToTakeScreenshot(error_message="Page is closed")try:- await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)- LOG.debug("Page is fully loaded, agent is about to take screenshots")+ if mode == ScreenshotMode.DETAILED:+ await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)+ LOG.debug("Page is fully loaded, agent is about to take screenshots")start_time = time.time()screenshot: bytes = b""if file_path:@@ -77,6 +87,7 @@ async def _scrolling_screenshots_helper(url: str | None = None,draw_boxes: bool = False,max_number: int = settings.MAX_NUM_SCREENSHOTS,+ mode: ScreenshotMode = ScreenshotMode.DETAILED,) -> list[bytes]:skyvern_page = await SkyvernFrame.create_instance(frame=page)# page is the main frame and the index must be 0@@ -84,6 +95,11 @@ async def _scrolling_screenshots_helper(frame = "main.frame"frame_index = 0+ # when mode is lite, we don't draw bounding boxes+ # since draw_boxes impacts the performance of processing+ if mode == ScreenshotMode.LITE:+ draw_boxes = False+screenshots: list[bytes] = []if await skyvern_page.is_window_scrollable():scroll_y_px_old = -30.0@@ -92,12 +108,15 @@ async def _scrolling_screenshots_helper(# We are checking the difference between the old and new scroll_y_px to determine if we have reached the end of the# page. If the difference is less than 25, we assume we have reached the end of the page.while abs(scroll_y_px_old - scroll_y_px) > 25 and len(screenshots) < max_number:- screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame)+ screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame, mode=mode)screenshots.append(screenshot)scroll_y_px_old = scroll_y_pxLOG.debug("Scrolling to next page", url=url, num_screenshots=len(screenshots))scroll_y_px = await skyvern_page.scroll_to_next_page(- draw_boxes=draw_boxes, frame=frame, frame_index=frame_index+ draw_boxes=draw_boxes,+ frame=frame,+ frame_index=frame_index,+ need_overlap=(mode == ScreenshotMode.DETAILED),)LOG.debug("Scrolled to next page",@@ -107,15 +126,17 @@ async def _scrolling_screenshots_helper(if draw_boxes:await skyvern_page.remove_bounding_boxes()await skyvern_page.scroll_to_top(draw_boxes=False, frame=frame, frame_index=frame_index)- # wait until animation ends, which is triggered by scrolling- LOG.debug("Waiting for 2 seconds until animation ends.")- await asyncio.sleep(2)++ if mode == ScreenshotMode.DETAILED:+ # wait until animation ends, which is triggered by scrolling+ LOG.debug("Waiting for 2 seconds until animation ends.")+ await asyncio.sleep(2)else:if draw_boxes:await skyvern_page.build_elements_and_draw_bounding_boxes(frame=frame, frame_index=frame_index)LOG.debug("Page is not scrollable", url=url, num_screenshots=len(screenshots))- screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame)+ screenshot = await _current_viewpoint_screenshot_helper(page=skyvern_page.frame, mode=mode)screenshots.append(screenshot)if draw_boxes:@@ -144,28 +165,85 @@ class SkyvernFrame:return await SkyvernFrame.evaluate(frame=frame, expression="() => document.location.href")@staticmethod- async def take_screenshot(+ async def take_scrolling_screenshot(page: Page,- full_page: bool = False,file_path: str | None = None,timeout: float = settings.BROWSER_SCREENSHOT_TIMEOUT_MS,+ mode: ScreenshotMode = ScreenshotMode.DETAILED,+ scrolling_number: int = settings.MAX_NUM_SCREENSHOTS,+ use_playwright_fullpage: bool = False, # TODO: THIS IS ONLY FOR EXPERIMENT. will be removed after experiment.) -> bytes:- return await _current_viewpoint_screenshot_helper(- page=page, file_path=file_path, full_page=full_page, timeout=timeout- )+ if scrolling_number <= 0:+ return await _current_viewpoint_screenshot_helper(+ page=page, file_path=file_path, timeout=timeout, mode=mode+ )++ if use_playwright_fullpage:+ return await _current_viewpoint_screenshot_helper(+ page=page, file_path=file_path, timeout=timeout, full_page=True+ )++ if scrolling_number > settings.MAX_NUM_SCREENSHOTS:+ LOG.warning(+ "scrolling_number is greater than the max number of screenshots, setting it to the max number of screenshots",+ scrolling_number=scrolling_number,+ max_number=settings.MAX_NUM_SCREENSHOTS,+ )+ scrolling_number = settings.MAX_NUM_SCREENSHOTS++ # use spilt screenshot with lite mode, isntead of fullpage screenshot from playwright+ LOG.debug("Page is fully loaded, agent is about to generate the full page screenshot")+ start_time = time.time()+ async with asyncio.timeout(timeout):+ screenshots = await _scrolling_screenshots_helper(page=page, mode=mode, max_number=scrolling_number)+ images = []++ for screenshot in screenshots:+ with Image.open(BytesIO(screenshot)) as img:+ img.load()+ images.append(img)++ total_height = sum(img.height for img in images)+ max_width = max(img.width for img in images)++ merged_img = Image.new("RGB", (max_width, total_height), color=(255, 255, 255))++ current_y = 0+ for img in images:+ merged_img.paste(img, (0, current_y))+ current_y += img.height++ buffer = BytesIO()+ merged_img.save(buffer, format="PNG")+ buffer.seek(0)++ img_data = buffer.read()+ if file_path is not None:+ with open(file_path, "wb") as f:+ f.write(img_data)++ end_time = time.time()+ LOG.debug(+ "Full page screenshot taking time",+ screenshot_time=end_time - start_time,+ file_path=file_path,+ )+ return img_data@staticmethodasync def take_split_screenshots(page: Page,- url: str,+ url: str | None = None,draw_boxes: bool = False,max_number: int = settings.MAX_NUM_SCREENSHOTS,scroll: bool = True,) -> list[bytes]:if not scroll:- return [await _current_viewpoint_screenshot_helper(page=page)]+ return [await _current_viewpoint_screenshot_helper(page=page, mode=ScreenshotMode.DETAILED)]- return await _scrolling_screenshots_helper(page=page, url=url, max_number=max_number, draw_boxes=draw_boxes)+ return await _scrolling_screenshots_helper(+ page=page, url=url, max_number=max_number, draw_boxes=draw_boxes, mode=ScreenshotMode.DETAILED+ )@classmethodasync def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:@@ -235,19 +313,21 @@ class SkyvernFrame:)return scroll_y_px- async def scroll_to_next_page(self, draw_boxes: bool, frame: str, frame_index: int) -> float:+ async def scroll_to_next_page(+ self, draw_boxes: bool, frame: str, frame_index: int, need_overlap: bool = True+ ) -> float:"""Scroll to the next page and take a screenshot.:param drow_boxes: If True, draw bounding boxes around the elements.:param page: Page instance to take the screenshot from.:return: Screenshot of the page."""- js_script = "async ([draw_boxes, frame, frame_index]) => await scrollToNextPage(draw_boxes, frame, frame_index)"+ js_script = "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)"scroll_y_px = await self.evaluate(frame=self.frame,expression=js_script,timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS,- arg=[draw_boxes, frame, frame_index],+ arg=[draw_boxes, frame, frame_index, need_overlap],)return scroll_y_px