Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Let Copilot offer a one-click demonstration when the user says they'll show it (SKY-18725) (#8977)
Cindy Lifd35bea500de
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.demonstration.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.demonstration.test.tsxindex 33e19d23f..e1e89eb99 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.demonstration.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.demonstration.test.tsx@@ -280,6 +280,56 @@ afterEach(() => {});describe("WorkflowCopilotChat demonstration", () => {+ it("renders a persisted demonstration offer and starts recording from it", async () => {+ const onRecordTask = vi.fn();+ historyResponse.data.chat_history = [+ {+ sender: "ai",+ content: "Show me how you do it.",+ narrative_payload: {+ turnId: "turn-offer",+ terminal: "response",+ demonstrationOffer: true,+ },+ created_at: "2026-10-14T12:00:00Z",+ },+ ];++ await renderChat({ onRecordTask, canRecordTask: true });++ fireEvent.click(+ await screen.findByRole("button", { name: "Start demonstration" }),+ );+ expect(onRecordTask).toHaveBeenCalledTimes(1);+ });++ it("disables a demonstration offer once a later message follows it", async () => {+ historyResponse.data.chat_history = [+ {+ sender: "ai",+ content: "Show me how you do it.",+ narrative_payload: {+ turnId: "turn-offer",+ terminal: "response",+ demonstrationOffer: true,+ },+ created_at: "2026-10-14T12:00:00Z",+ },+ {+ sender: "user",+ content: "Actually, just build it.",+ created_at: "2026-10-14T12:01:00Z",+ },+ ];++ await renderChat({ onRecordTask: vi.fn(), canRecordTask: true });++ const offer = (await screen.findByRole("button", {+ name: "Start demonstration",+ })) as HTMLButtonElement;+ expect(offer.disabled).toBe(true);+ });+it("never looks for an active demonstration when the feature is off", async () => {demonstrationFlag.value = false;await renderChat({ liveBrowserSessionId: "browser-1" });@@ -453,9 +503,7 @@ describe("WorkflowCopilotChat demonstration", () => {useRecordingLauncherStore.getState().demonstrationRecordingBrowserSessionId,).toBeNull();- expect(- screen.getByText("I'll show you the task in the browser."),- ).toBeTruthy();+ expect(screen.getByText("Showing you in the browser now.")).toBeTruthy();expect(screen.getByRole("button", { name: "Attach a file" }).title).toBe("Use Build before attaching a file",);@@ -534,13 +582,11 @@ describe("WorkflowCopilotChat demonstration", () => {useRecordingLauncherStore.getState().startDemonstration?.();});expect(- await screen.findByText("I'll show you the task in the browser."),+ await screen.findByText("Showing you in the browser now."),).toBeTruthy();act(() => rejectStart?.(new Error("Already recording elsewhere")));await waitFor(() =>- expect(- screen.queryByText("I'll show you the task in the browser."),- ).toBeNull(),+ expect(screen.queryByText("Showing you in the browser now.")).toBeNull(),);streamCalls.length = 0;diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex fd004a21a..d3c0f4b92 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -452,7 +452,7 @@ const RECOVERY_POLL_SUPERSEDE_READS = 4;const RECOVERY_IN_PROGRESS_MESSAGE ="The connection dropped, so Copilot is checking whether this turn finished.";const TEST_END_TO_END_PROMPT = "Test this workflow end to end.";-const DEMONSTRATION_START_MESSAGE = "I'll show you the task in the browser.";+const DEMONSTRATION_START_MESSAGE = "Showing you in the browser now.";const DEMONSTRATION_DONE_MESSAGE ="I've shown you the task in the browser. Build a workflow that does it.";const diagnoseRunReceipt = (runId: string) =>@@ -1709,6 +1709,51 @@ const constrainPosition = (};};+function DemonstrateTaskButton({+ label = "Demonstrate task",+ disabled,+ disabledReason,+ onClick,+}: {+ label?: string;+ disabled: boolean;+ disabledReason: string;+ onClick: () => void;+}) {+ return (+ <TooltipProvider>+ <ControlTooltip+ content={+ disabled+ ? disabledReason+ : "Skyvern records your clicks, typing, and navigation, then turns them into workflow steps"+ }+ blocked={disabled}+ side="top"+ wrapperClassName="min-w-0 w-full"+ >+ <button+ type="button"+ aria-label={label}+ className="flex w-full min-w-0 items-center gap-3 rounded-lg border border-border bg-slate-elevation3 p-3 text-left transition-colors hover:bg-accent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50"+ disabled={disabled}+ onClick={onClick}+ >+ <span className="flex h-8 w-8 shrink-0 items-center justify-center rounded-md border border-border bg-slate-elevation2">+ <span className="h-2.5 w-2.5 rounded-full bg-red-500" />+ </span>+ <span className="min-w-0">+ <span className="block font-medium text-foreground">{label}</span>+ <span className="mt-0.5 block text-xs leading-snug text-muted-foreground">+ Show Copilot how in the browser+ </span>+ </span>+ </button>+ </ControlTooltip>+ </TooltipProvider>+ );+}+export function WorkflowCopilotChat({organizationId,captureProductEvent,@@ -9313,6 +9358,10 @@ export function WorkflowCopilotChat({supports_account_group_card: true,supports_credential_delete_card: true,supports_google_setup_card: true,+ supports_demonstration_offer:+ demonstrationEnabled &&+ Boolean(onRecordTask) &&+ !authoringUnavailableReason,} as WorkflowCopilotChatRequest,(payload) => {if (@@ -9741,7 +9790,10 @@ export function WorkflowCopilotChat({restoreSteersToComposer,captureCopilotEvent,createCanonicalRecovery,+ demonstrationEnabled,onWorkflowPersisted,+ onRecordTask,+ authoringUnavailableReason,reserveCopilotTurn,isCopilotTurnCurrent,isCopilotOwnerCurrent,@@ -11020,6 +11072,13 @@ export function WorkflowCopilotChat({};const recordTaskDisabled = !onRecordTask || !canRecordTask;+ const recordTaskUnavailableReason =+ authoringUnavailableReason ??+ "Demonstrate task is available when the browser is ready";+ const startRecordTask = () => {+ if (refuseMutationDuringYamlCommit()) return;+ onRecordTask?.();+ };const recordingChapterController = recordingAuthoringActive ? (<RecordingPanelkey="live-recording-chapter"@@ -11216,42 +11275,11 @@ export function WorkflowCopilotChat({</button></ControlTooltip></TooltipProvider>- <TooltipProvider>- <ControlTooltip- content={- recordTaskDisabled- ? (authoringUnavailableReason ??- "Demonstrate task is available when the browser is ready")- : "Skyvern records your clicks, typing, and navigation, then turns them into workflow steps"- }- blocked={recordTaskDisabled}- side="top"- wrapperClassName="min-w-0 w-full"- >- <button- type="button"- aria-label="Demonstrate task"- className="flex w-full min-w-0 items-center gap-3 rounded-lg border border-border bg-slate-elevation3 p-3 text-left transition-colors hover:bg-accent focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50"- disabled={recordTaskDisabled}- onClick={() => {- if (refuseMutationDuringYamlCommit()) return;- onRecordTask?.();- }}- >- <span className="flex h-8 w-8 shrink-0 items-center justify-center rounded-md border border-border bg-slate-elevation2">- <span className="h-2.5 w-2.5 rounded-full bg-red-500" />- </span>- <span className="min-w-0">- <span className="block font-medium text-foreground">- Demonstrate task- </span>- <span className="mt-0.5 block text-xs leading-snug text-muted-foreground">- Show Copilot how in the browser- </span>- </span>- </button>- </ControlTooltip>- </TooltipProvider>+ <DemonstrateTaskButton+ disabled={recordTaskDisabled}+ disabledReason={recordTaskUnavailableReason}+ onClick={startRecordTask}+ /></div></div></div>@@ -11594,6 +11622,29 @@ export function WorkflowCopilotChat({/>),)}+ {demonstrationEnabled &&+ message.narrative.demonstrationOffer ? (+ <div className="max-w-sm">+ <DemonstrateTaskButton+ label="Start demonstration"+ disabled={+ !isLastMessage ||+ isLoading ||+ hasPendingQuestion ||+ isDemonstrating ||+ recordTaskDisabled+ }+ disabledReason={+ recordTaskDisabled+ ? recordTaskUnavailableReason+ : !isLastMessage+ ? "Only the latest message can start a demonstration"+ : "Demonstrate task is unavailable right now"+ }+ onClick={startRecordTask}+ />+ </div>+ ) : null}{showReviewGate ? (<div className="space-y-2">{index === gateIndex && pendingProposalRun ? (diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.tsindex ab92ddd39..ab5bee4c3 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts@@ -470,6 +470,7 @@ export interface TurnNarrativeState {connectedAccountChoices: ConnectedAccountChoice[];outputFiles: DeliveredOutputFile[];googleConnectionNotices: GoogleConnectionNotice[];+ demonstrationOffer: boolean;review: ReviewProjection | null;turnFacts: TurnFacts | null;budgetExpiry: BudgetExpiryState | null;@@ -532,6 +533,7 @@ export const EMPTY_NARRATIVE: TurnNarrativeState = Object.freeze({connectedAccountChoices: [],outputFiles: [],googleConnectionNotices: [],+ demonstrationOffer: false,review: null,turnFacts: null,budgetExpiry: null,@@ -886,6 +888,7 @@ const TOOL_CALL_KINDS: Record<string, ToolCallKind> = {inspect_current_workflow: "other",list_integrations: "other",request_google_connection: "other",+ offer_demonstration: "other",read_google_sheet: "other",read_attached_pdf: "other",get_organization_usage_quota: "other",@@ -2097,6 +2100,7 @@ export function hydrateNarrativeFromPayload(googleConnectionNotices: parseGoogleConnectionNotices(payload.googleConnectionNotices,),+ demonstrationOffer: payload.demonstrationOffer === true,review: parseReviewProjection(payload.review),turnFacts,budgetExpiry,diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex c2bd07204..b171f0a1a 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -252,6 +252,7 @@ export interface WorkflowCopilotChatRequest {supports_account_group_card?: boolean;supports_credential_delete_card?: boolean;supports_google_setup_card?: boolean;+ supports_demonstration_offer?: boolean;}export type WorkflowCopilotCancelSource = "escape_key" | "stop_button" | "api";diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex c99f58f6b..f236bf123 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -294,6 +294,7 @@ from skyvern.webeye.actions.actions import (CompleteVerifyResult,DecisiveAction,DownloadFileAction,+ DragAction,ExtractAction,GotoUrlAction,HoverAction,@@ -501,6 +502,7 @@ def mark_pdf_source_downloaded(pdf_src: str) -> None:_TASKV3_TOOL_ACTION_TYPES = {"click": ActionType.CLICK,"hover": ActionType.HOVER,+ "drag": ActionType.DRAG,"type": ActionType.INPUT_TEXT,"select_option": ActionType.SELECT_OPTION,"select_combobox": ActionType.SELECT_OPTION,@@ -681,6 +683,8 @@ def _taskv3_action_for_tool_call(tool_name: str, args: dict[str, Any], **fields:return ClickAction(element_id=selector, **fields)if tool_name == "hover":return HoverAction(element_id=selector, **fields)+ if tool_name == "drag":+ return DragAction(element_id=selector, **fields)if tool_name == "type":return InputTextAction(element_id=selector, text=str(args.get("text") or ""), **fields)if tool_name in ("select_option", "select_combobox"):@@ -2945,6 +2949,7 @@ class ForgeAgent:args if isinstance(args, dict) else {}, secret_values, placeholder_ids)selector = tool_args.get("selector", "")+ drop_target = f" onto {tool_args['target']}" if name == "drag" and tool_args.get("target") else ""row_response = _taskv3_row_response(round_action.outcome) if round_action.outcome else Noneif row_response is not None:# The outcome carries page-supplied URLs the arg redaction above never saw, and a@@ -2980,7 +2985,7 @@ class ForgeAgent:# order) pairs).step_order=row_step_order,action_order=len(v3_persisted_actions),- description=f"{TASK_V3_ACTION_DESCRIPTION_PREFIX}{name} {selector}".strip(),+ description=f"{TASK_V3_ACTION_DESCRIPTION_PREFIX}{name} {selector}{drop_target}".strip(),screenshot_artifact_id=screenshot_artifact_id,reasoning=turn_reasoning,intention=_taskv3_row_intention(task, round_action, secret_values, tool_args),diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 78025903c..964823084 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -2124,6 +2124,8 @@ def _make_agent_result(notice_payloads = ctx.google_connection_notice_payloads()if notice_payloads:payload_updates["googleConnectionNotices"] = notice_payloads+ if ctx is not None and ctx.demonstration_offered and "demonstrationOffer" not in narrative_payload:+ payload_updates["demonstrationOffer"] = Trueif payload_updates or len(payload_base) != len(narrative_payload):kwargs["narrative_payload"] = {**payload_base, **payload_updates}if turn_facts is not None:@@ -2280,6 +2282,8 @@ def _build_narrative_payload(notice_payloads = ctx.google_connection_notice_payloads()if notice_payloads:payload["googleConnectionNotices"] = notice_payloads+ if ctx.demonstration_offered:+ payload["demonstrationOffer"] = Trueif ctx.demonstration_items:payload["demonstrationItems"] = list(ctx.demonstration_items)if ctx.staged_workflow_yaml is not None:@@ -5597,6 +5601,7 @@ async def _run_copilot_turn_impl(supports_question_tool=chat_request.supports_question_tool,supports_account_group_card=chat_request.supports_account_group_card,supports_google_setup_card=chat_request.supports_google_setup_card,+ supports_demonstration_offer=chat_request.supports_demonstration_offer,supports_credential_delete_card=chat_request.supports_credential_delete_card,browser_code_available=browser_code_mode != CopilotBrowserCodeMode.OFFand copilot_config.authoring_capability.code_blocks,diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 914af46d8..56abab4cd 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -89,6 +89,7 @@ _INTERNAL_TOOL_NAME_TOKENS: tuple[str, ...] = ("list_credentials","list_integrations","request_google_connection",+ "offer_demonstration","read_google_sheet",)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 35b87654e..aeea60e76 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -306,6 +306,7 @@ class TurnNarrativePayload(TypedDict):connectedAccountChoices: NotRequired[list[NarrativeConnectedAccountChoice]]outputFiles: NotRequired[list[NarrativeOutputFile]]googleConnectionNotices: NotRequired[list[GoogleConnectionNoticePayload]]+ demonstrationOffer: NotRequired[bool]designStarted: booldesignEnded: booldraft: NarrativeDraft | None@@ -1334,6 +1335,7 @@ class CopilotContext(AgentContext):demonstration_id: str | None = Nonedemonstration_insertion: dict[str, str | None] | None = Nonedemonstration_items: list[str] = field(default_factory=list)+ demonstration_offered: bool = False# Set while a ``request_credential`` ask is open, so tool calls issued alongside it in the same# model response wait for the user's answer instead of racing it.credential_pause_settled: asyncio.Event | None = Nonediff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.pyindex d41630150..1d905d755 100644--- a/skyvern/forge/sdk/copilot/narration.py+++ b/skyvern/forge/sdk/copilot/narration.py@@ -61,6 +61,7 @@ _TOOL_ACTIVITY_DISPLAY_LABELS = {"get_workflow_knowledge": "Looking up workflow guidance","list_integrations": "Checking connected integrations","request_google_connection": "Preparing a Google connection",+ "offer_demonstration": "Offering a demonstration","read_google_sheet": "Reading the Google Sheet","get_organization_usage_quota": "Checking account usage","inspect_current_workflow": "Inspecting workflow",diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 9321d094a..12774f1ae 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -1304,6 +1304,21 @@ async def list_integrations_tool(ctx: RunContextWrapper) -> str:GOOGLE_SETUP_TOOL_NAME = "request_google_connection"+DEMONSTRATION_OFFER_TOOL_NAME = "offer_demonstration"+++@function_tool(failure_error_function=copilot_tool_failure, name_override=DEMONSTRATION_OFFER_TOOL_NAME)+async def offer_demonstration_tool(ctx: RunContextWrapper) -> str:+ """Show the user a Start demonstration card in this chat. Use it when the user would rather show you the task in+ the browser themselves than describe it. The card appears under your reply. Recording starts only when the user+ clicks the card, and you see none of their actions while they record: the recorded steps reach you in a new turn+ after they press Done. This call does not record or wait (`status` is `offered`).+ """+ copilot_ctx = ctx.context+ copilot_ctx.demonstration_offered = True+ result = {"ok": True, "data": {"status": "offered"}}+ record_tool_step_result_for_ctx(copilot_ctx, DEMONSTRATION_OFFER_TOOL_NAME, {}, result)+ return json.dumps(result)@function_tool(failure_error_function=copilot_tool_failure, name_override=GOOGLE_SETUP_TOOL_NAME)@@ -2392,6 +2407,7 @@ NATIVE_TOOLS = [list_credentials_tool,list_integrations_tool,request_google_connection_tool,+ offer_demonstration_tool,read_google_sheet_tool,get_organization_usage_quota_tool,run_blocks_tool,@@ -2557,6 +2573,7 @@ def copilot_native_tools(supports_account_group_card: bool = False,supports_credential_delete_card: bool = False,supports_google_setup_card: bool = False,+ supports_demonstration_offer: bool = False,) -> list[FunctionTool]:capability = _normalized_authoring_capability(authoring_capability)both_families = capability.code_blocks and capability.agent_blocks@@ -2568,6 +2585,7 @@ def copilot_native_tools(or (tool.name == BROWSER_CODE_TOOL_NAME and not browser_code_available)or (tool.name in ACCOUNT_GROUP_TOOL_NAMES and not supports_account_group_card)or (tool.name == GOOGLE_SETUP_TOOL_NAME and not supports_google_setup_card)+ or (tool.name == DEMONSTRATION_OFFER_TOOL_NAME and not supports_demonstration_offer)or (tool.name == ACCOUNT_GROUP_SUBMIT_TOOL_NAME and not account_group_submit_enabled())or (tool.name == CREDENTIAL_DELETE_TOOL_NAMEdiff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex a8afebab0..730de3d07 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -475,6 +475,9 @@ class WorkflowCopilotChatRequest(BaseModel):supports_google_setup_card: bool = Field(False, description="The client can display Gmail and Google Sheets connection cards with a scope profile.")+ supports_demonstration_offer: bool = Field(+ False, description="The client can display the demonstration offer card and start a demonstration from it."+ )credential_recovery_token: SecretStr | None = Field(None,repr=False,diff --git a/skyvern/forge/taskv3/answer_basis_audit.py b/skyvern/forge/taskv3/answer_basis_audit.pyindex f9e53f147..761cd62fd 100644--- a/skyvern/forge/taskv3/answer_basis_audit.py+++ b/skyvern/forge/taskv3/answer_basis_audit.py@@ -140,7 +140,7 @@ def apply_answer_basis_audit(tools: list[ToolSpec], audit: AnswerBasisAudit | No)for spec in tools:- if spec.name not in ("click", "type", "select_option", "select_combobox", "press_key", CODE_TOOL_NAME):+ if spec.name not in ("click", "drag", "type", "select_option", "select_combobox", "press_key", CODE_TOOL_NAME):continueasync def wrapped(diff --git a/skyvern/forge/taskv3/input_dispatch.py b/skyvern/forge/taskv3/input_dispatch.pyindex 46d8369a4..816ef41a0 100644--- a/skyvern/forge/taskv3/input_dispatch.py+++ b/skyvern/forge/taskv3/input_dispatch.py@@ -1,7 +1,7 @@"""The one place Task V3 sends pointer and keyboard input to a page.Each gesture runs the Playwright call each tool made before this module existed, argument for argument. The-TASK_V3_LOGIN_PACE treatment holds the first click or Enter after a password fill until LOGIN_PACE_FLOOR_S after Task V3+TASK_V3_LOGIN_PACE treatment holds the first click, drag or Enter after a password fill until LOGIN_PACE_FLOOR_S after Task V3started the block."""@@ -235,6 +235,11 @@ async def hover(realm: Realm, selector: str, *, timeout: float | None = None) ->await realm.hover(selector, **_opts(timeout=timeout))+async def drag(realm: Realm, source: str, target: str, *, timeout: float | None = None) -> None:+ await _hold_submit(realm)+ await realm.drag_and_drop(source, target, **_opts(timeout=timeout))++async def js_click(realm: Realm, root_query_js: str, arg: Any) -> Any:"""Click a hidden native control from inside the page. No pointer events, as in v1."""await _hold_submit(realm)diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex e02ff3f12..cfc090fe1 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -321,6 +321,8 @@ PICKED_OPTION_DATA_KEY = "picked_option"# A list the control opened but could not settle a row in: the list stays open for a pick.OPENED_LIST_ERROR_CLASSES = frozenset({"list_no_match", "list_unread", "list_ambiguous", "identical_rows"})TARGET_KIND_DATA_KEY = "target_kind"+# The durable selector of the element a `drag` dropped onto, for the pending-submit watch.+DROP_TARGET_DATA_KEY = "drop_target"# The tool-result `data` key a recorded action's outcome rides on: the machine facts about what the# call achieved (`requested_url`, `url`, `http_status`, `page_transitioned`, `navigation_dead_end`),# carried verbatim to `RoundAction.outcome` for the caller to persist on the action row. Internal to@@ -1539,12 +1541,12 @@ CODE_TOOL_NAME = "execute_python"def _may_submit(tool_name: str, args: dict[str, Any]) -> bool:- """A click, an Enter press, or a type or select_combobox that pressed Enter: the loop cannot tell a submit from any of them.+ """A click or drag, an Enter press, or a type or select_combobox that pressed Enter: the loop cannot tell a submit from any of them.The code tool counts too: its body is opaque to the loop, so it is treated as possibly havingsubmitted rather than as certainly not having."""- return tool_name in ("click", CODE_TOOL_NAME) or _is_enter_submit(tool_name, args)+ return tool_name in ("click", "drag", CODE_TOOL_NAME) or _is_enter_submit(tool_name, args)# Secret values reach the model only as `placeholder_...` tokens, so the guard below keys on the@@ -1699,16 +1701,18 @@ class ActivityRecency:return self.last_trigger_turn is not None and (self.turn - self.last_trigger_turn) <= window-def _names_submit_control(tool_name: str, args: dict[str, Any], ok: bool) -> str | None:+def _names_submit_control(tool_name: str, args: dict[str, Any], ok: bool, data: dict[str, Any]) -> str | None:"""The selector of a control a successful action acted on directly, or None.- Only `click`. An Enter press and a type-that-pressed-Enter submit through a control they do not+ A `click`'s control, or the drop target a `drag` reports, since a drop handler can submit.++ An Enter press and a type-that-pressed-Enter submit through a control they do notname, and a captcha dispatch names none at all, so for all three "is that control still inflight" has no subject — and the selector they do carry is a text field, whose value is themodel's own typed text."""- if not ok or tool_name != "click":+ if not ok or tool_name not in ("click", "drag"):return None- selector = args.get("selector")+ selector = data.get(DROP_TARGET_DATA_KEY) if tool_name == "drag" else args.get("selector")return selector if isinstance(selector, str) and selector else None@@ -5281,7 +5285,7 @@ async def run_agent_tool_loop(if activity is not None and _arms_failure_evidence(tool_name, args, result.status == "ok"):activity.last_trigger_turn = st.turnsif submit_watch is not None:- submit_selector = _names_submit_control(tool_name, args, result.status == "ok")+ submit_selector = _names_submit_control(tool_name, args, result.status == "ok", result_data)if submit_selector is not None:submit_watch.record(submit_selector)diff --git a/skyvern/forge/taskv3/preflight.py b/skyvern/forge/taskv3/preflight.pyindex b0d5f950b..62d322e9c 100644--- a/skyvern/forge/taskv3/preflight.py+++ b/skyvern/forge/taskv3/preflight.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.browser_action_preflight import policy_observation_enablefrom skyvern.webeye.actions.actions import (Action,ClickAction,+ DragAction,GotoUrlAction,HoverAction,InputTextAction,@@ -39,7 +40,7 @@ LOG = structlog.get_logger()# `_build_action` maps. This is the single source of truth for which tool handlers get wrapped;# `build_browser_tools` imports it rather than keeping a second list that can drift.PREFLIGHT_TOOL_NAMES = frozenset(- {"click", "hover", "type", "select_combobox", "select_option", "press_key", "file_upload", "navigate"}+ {"click", "hover", "drag", "type", "select_combobox", "select_option", "press_key", "file_upload", "navigate"})@@ -49,6 +50,8 @@ def _build_action(tool_name: str, args: dict[str, Any]) -> Action | None:return ClickAction(element_id=selector)if tool_name == "hover":return HoverAction(element_id=selector)+ if tool_name == "drag":+ return DragAction(element_id=selector)if tool_name == "type":return InputTextAction(element_id=selector, text=str(args.get("text", "")))if tool_name == "select_combobox":diff --git a/skyvern/forge/taskv3/target_label.py b/skyvern/forge/taskv3/target_label.pyindex 0b2920af0..a11907b07 100644--- a/skyvern/forge/taskv3/target_label.py+++ b/skyvern/forge/taskv3/target_label.py@@ -150,6 +150,7 @@ def name_looks_like_a_label(raw: str) -> bool:TARGET_VERBS = {"click": "Clicked","hover": "Hovered over",+ "drag": "Dragged","type": "Typed into","select_option": "Selected an option in","select_combobox": "Selected an option in",@@ -211,6 +212,7 @@ _DEFAULT_KIND_BY_TOOL = {_FAILED_TARGET_VERBS = {"click": "Tried to click","hover": "Tried to hover over",+ "drag": "Tried to drag","type": "Tried to type into","select_option": "Tried to select an option in","select_combobox": "Tried to select an option in",diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 8b4be5ae2..833d610e4 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -85,6 +85,7 @@ from skyvern.forge.taskv3.loop import (ACTION_OUTCOME_DATA_KEY,CLICK_DISPATCHED_DATA_KEY,DOWNLOAD_FAILURE_MAX_LINES,+ DROP_TARGET_DATA_KEY,FILL_TOOLS,NAVIGATION_DEAD_END_STATUSES,OPENED_LIST_ERROR_CLASSES,@@ -474,6 +475,7 @@ _SELECTOR_GUARD_TOOL_NAMES = frozenset("get_html","click","hover",+ "drag","type","select_option","select_combobox",@@ -4685,7 +4687,9 @@ _TARGET_NAME_JS = (# Actions whose target has a name worth reading back. press_key and navigate are excluded: neither is# about an element a reader would recognize by name.-_TARGET_LABEL_TOOL_NAMES = frozenset({"click", "hover", "type", "select_option", "select_combobox", "file_upload"})+_TARGET_LABEL_TOOL_NAMES = frozenset(+ {"click", "hover", "drag", "type", "select_option", "select_combobox", "file_upload"}+)_REACH_PROBE_NEEDED_JS = (@@ -11269,6 +11273,7 @@ _TEXT_DELTA_TOOL_NAMES = frozenset("select_combobox","press_key","hover",+ "drag","file_upload","observe","scroll",@@ -15535,6 +15540,54 @@ def build_browser_tools(await input_dispatch.hover(page, selector, timeout=_ACTION_TIMEOUT_MS)return ToolResult.ok(f"hovered {selector}")+ async def drag(args: dict[str, Any]) -> ToolResult:+ realm, error = await _resolve_page()+ if error is not None:+ return error+ selector = args["selector"]+ target = _normalize_selector(str(args["target"]))+ # The address wrappers only see `selector`, so the drop target is resolved and checked here.+ match = REF_SELECTOR_RE.match(target)+ if match is not None:+ ref = int(match.group(1))+ resolved, target_error = await _resolve_ref(_current_page(), ref)+ entry = _observe_manifest.get(ref) or {}+ target_realm = entry.get("owner") or _current_page()+ target = resolved or target+ # The act token can leave with a node the page replaces, so the watch gets an address that re-resolves.+ durable_target = str(entry.get("selector") or "") or target+ else:+ try:+ await realm.locator(target).count()+ except Exception as exc:+ invalid = _invalid_selector_result(args["target"], exc)+ if invalid is None:+ raise+ return invalid+ target_realm, target_error = await _realm_for_typed_selector(_current_page(), target)+ durable_target = target+ if target_error is None and not await _holders(target_realm, target):+ raise ToolRefusal(+ f"{args['target']} matches nothing on the page. Re-observe and drop onto a ref from the observation.",+ error_class="stale_selector",+ )+ if target_error is not None:+ raise ToolRefusal.of(target_error)+ # Playwright drags within one frame; a drop target in another document is unreachable.+ if target_realm is not realm:+ raise ToolRefusal(+ f"The drop target {args['target']} is in a different frame than the dragged element; drag only "+ "moves an element onto a target in the same frame.",+ error_class="unreachable",+ )+ await _require_single_target(realm, selector)+ await _require_single_target(realm, target)+ await input_dispatch.drag(realm, selector, target, timeout=_ACTION_TIMEOUT_MS)+ # A drop handler can submit, so the drop target is what the pending-submit gate watches.+ result = ToolResult.ok(f"dragged {selector} onto {args['target']}", data={DROP_TARGET_DATA_KEY: durable_target})+ await _note_frame_work("drop", realm, durable_target, result)+ return result+async def _annotate_challenge_frame(realm: Any, top_page: Any, occluder: dict[str, Any] | None) -> None:# A clipped reading names no layer, so the cover marker this scan looks for was never# planted and the walk below can only ever answer "no" -- at the cost of a frame_element()@@ -21922,7 +21975,8 @@ def build_browser_tools(# Which tools leave work a reload would discard, and which leave something that may still be in# flight. The loop decides what counts as a submit; this only records WHERE the click landed.- _LEDGER_SUBMIT_TOOLS = frozenset({"click", "press_key"})+ # "drop" is the drag handler's own record of its drop target; the address wrapper only sees a drag's source.+ _LEDGER_SUBMIT_TOOLS = frozenset({"click", "drop", "press_key"})async def _note_frame_work(tool_name: str, realm: Any, selector: Any, result: ToolResult) -> None:"""Record realm-attributed work an action left in a child frame.@@ -22261,6 +22315,14 @@ def build_browser_tools(_obj({"selector": {"type": "string"}}, ["selector"]),hover,),+ _spec(+ "drag",+ "Drag an element onto another element (reorder a list, move a card to a column, drop into a "+ "dropzone). `selector` is the element to pick up and `target` is where to drop it; each is an "+ "observe ref (e.g. ref=12) or a CSS selector.",+ _obj({"selector": {"type": "string"}, "target": {"type": "string"}}, ["selector", "target"]),+ drag,+ ),_spec("type","Type text into an input/textarea by its observe ref (e.g. ref=12), a CSS selector, or mark=N from the last look(); "@@ -22378,6 +22440,7 @@ def build_browser_tools(if _tool_spec.name in ("click","hover",+ "drag","type","select_option","select_combobox",diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex 7e104fae4..2b985b9b5 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -80,6 +80,7 @@ _DISCOVERED_BROWSER_API_CALLS = {"click": 4,# A grid cell whose editor opens on a double-click (SKY-17868): one by selector, one by locator."dblclick": 2,+ "drag_and_drop": 1,"evaluate": 1,"fill": 3,"focus": 2,@@ -268,14 +269,14 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 211+ assert sum(sum(methods.values()) for methods in observed.values()) == 212handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATESassert sum(_NON_BROWSER_CANDIDATES.values()) == 7- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 204+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 205def test_every_raw_evaluate_call_is_classified() -> None:diff --git a/tests/unit/test_copilot_budget_drain.py b/tests/unit/test_copilot_budget_drain.pyindex 7ee3eef28..f3204e97e 100644--- a/tests/unit/test_copilot_budget_drain.py+++ b/tests/unit/test_copilot_budget_drain.py@@ -664,6 +664,7 @@ _DRAIN_KEPT_NATIVE_TOOLS = {"reply","request_credential","request_google_connection",+ "offer_demonstration","run_browser_code","search_web","set_work_plan",diff --git a/tests/unit/test_copilot_tools_list_integrations.py b/tests/unit/test_copilot_tools_list_integrations.pyindex 9abdddca4..ae8bf6797 100644--- a/tests/unit/test_copilot_tools_list_integrations.py+++ b/tests/unit/test_copilot_tools_list_integrations.py@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.tools import (copilot_native_tools,get_skyvern_mcp_alias_map,list_integrations_tool,+ offer_demonstration_tool,)from skyvern.forge.sdk.copilot.tools.integrations import (_list_integrations,@@ -653,6 +654,45 @@ def test_request_google_connection_is_offered_only_to_a_client_that_renders_the_assert ("request_google_connection" in names) is supports_google_setup_card[email protected]("supports_demonstration_offer", [False, True])+def test_offer_demonstration_is_offered_only_to_a_client_that_renders_the_card(+ supports_demonstration_offer: bool,+) -> None:+ names = {+ tool.name+ for tool in copilot_native_tools(+ supports_question_tool=True,+ browser_code_available=False,+ run_tools_available=True,+ supports_demonstration_offer=supports_demonstration_offer,+ )+ }++ assert ("offer_demonstration" in names) is supports_demonstration_offer+++async def test_offer_demonstration_sets_the_turn_flag_and_narrative_payload() -> None:+ ctx = make_copilot_ctx()++ await offer_demonstration_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="offer_demonstration"),+ "{}",+ )+ result = _make_agent_result(+ ctx,+ user_response="Show me.",+ updated_workflow=None,+ global_llm_context=None,+ narrative_payload={"turnId": "turn-1", "terminal": "response"},+ turn_outcome=TurnOutcome(response_kind=ResponseKind.ANSWER),+ )++ assert ctx.demonstration_offered is True+ assert result.narrative_payload is not None+ assert result.narrative_payload["demonstrationOffer"] is True++def _persisted_offer_turn(ctx: CopilotContext) -> TurnOutcome:result = _make_agent_result(ctx,diff --git a/tests/unit/test_taskv3_answer_basis_audit.py b/tests/unit/test_taskv3_answer_basis_audit.pyindex 9b88bd6aa..061415156 100644--- a/tests/unit/test_taskv3_answer_basis_audit.py+++ b/tests/unit/test_taskv3_answer_basis_audit.py@@ -70,7 +70,7 @@ def _tools(dispatched: list[tuple[str, dict[str, Any]]]) -> list[ToolSpec]:name=name, description="", parameters={}, handler=handler, address_probe=_address, answer_probe=probe)- names = ("click", "type", "select_option", "select_combobox", "press_key", CODE_TOOL_NAME, "observe")+ names = ("click", "drag", "type", "select_option", "select_combobox", "press_key", CODE_TOOL_NAME, "observe")return [make(n) for n in names]@@ -98,12 +98,13 @@ async def test_an_unsupported_answer_holds_every_moving_action_until_it_is_replaawait click({"selector": '[data-tv3-menu="2"]', "_row": "No"})message = await _held(tools, "click", submit)assert DEBARMENT in message and "#q1" in message- # Neither a resubmit, the next question's opener, a submit by ref, an Enter nor the code tool spends the hold: each- # is refused and Submit stays refused after them.+ # Neither a resubmit, the next question's opener, a submit by ref, an Enter, a drag nor the code tool spends the+ # hold: each is refused and Submit stays refused after them.await _held(tools, "click", submit)await _held(tools, "click", {"selector": "#q2", "_label": "Are you willing to relocate?"})await _held(tools, "click", {"selector": "ref=40"})await _held(tools, "press_key", {"key": "Enter"})+ await _held(tools, "drag", {"selector": "#slider", "target": "#end"})await _held(tools, CODE_TOOL_NAME, {"code": "await page.click('#submit')"})await _held(tools, "click", submit)assert [args for name, args in dispatched if name == "click"].count(submit) == 0diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.pyindex e90d7d576..d1b618d03 100644--- a/tests/unit/test_taskv3_loop.py+++ b/tests/unit/test_taskv3_loop.py@@ -7584,6 +7584,25 @@ async def test_the_loop_records_the_clicked_control_for_the_pending_gate() -> Noassert asked == ["#submit"]+async def test_the_loop_records_a_drags_drop_target_for_the_pending_gate() -> None:+ # A drop handler can start a save, so the gate watches the drop target the drag reports, not its source.+ probe, asked = _pending_probe(None)+ watch = SubmitWatch()+ drags: list[tuple[str, dict[str, Any]]] = []+ tools = [+ _billable_tool("drag", drags, data={"drop_target": "#done"}),+ make_finish_tool(pending_marker=probe, submit_watch=watch),+ ]+ script = [+ [("drag", {"selector": "#card", "target": "ref=4"})],+ [("finish", {"status": "completed", "reason": "moved the card"})],+ ]+ outcome, _ = await _run(script, tools, submit_watch=watch)+ assert outcome.status == "completed"+ assert asked == ["#done"]++@pytest.mark.asyncioasync def test_the_pending_gate_survives_a_long_wait_after_the_click() -> None:# Waiting for a slow submission is exactly what the gate asks the model to do, so a turn windowdiff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 4c7e7de64..8190be915 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -1713,6 +1713,7 @@ def test_preflight_tool_set_matches_builder() -> None:assert PREFLIGHT_TOOL_NAMES == {"click","hover",+ "drag","type","select_combobox","select_option",@@ -1723,6 +1724,7 @@ def test_preflight_tool_set_matches_builder() -> None:rep = {"click": {"selector": "#x"},"hover": {"selector": "#x"},+ "drag": {"selector": "#x", "target": "#y"},"type": {"selector": "#x", "text": "y"},"select_combobox": {"selector": "#x", "value": "v"},"select_option": {"selector": "#x", "value": "v"},@@ -3477,6 +3479,81 @@ async def test_hover_tool_dispatches_and_is_billable() -> None:assert spec.billable is True+_DRAG_LIST_HTML = """<!doctype html><html><body>+ <ul id="list">+ <li role="button" draggable="true" style="height:30px">Alpha</li>+ <li role="button" draggable="true" style="height:30px">Beta</li>+ <li role="button" draggable="true" style="height:30px">Gamma</li>+ </ul>+ <script>+ let dragged = null;+ for (const item of document.querySelectorAll('#list li')) {+ item.addEventListener('dragstart', (e) => { dragged = item; e.dataTransfer.setData('text/plain', ''); });+ item.addEventListener('dragover', (e) => e.preventDefault());+ item.addEventListener('drop', (e) => { e.preventDefault(); item.before(dragged); });+ }+ </script>+</body></html>"""+++@_skip_no_browser+async def test_drag_tool_moves_an_element_onto_its_target() -> None:+ # Copilot authors an agent block for a demonstrated drag when the org cannot author code blocks; with+ # no drag tool the block cannot repeat it. Both ends are observe refs, so each resolves server-side.+ async with _content_page(_DRAG_LIST_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ observed = await _tool(tools, "observe").handler({})+ spec = _tool(tools, "drag")+ result = await spec.handler(+ {"selector": _ref_line(observed.content, "Gamma"), "target": _ref_line(observed.content, "Alpha")}+ )+ assert result.status == "ok", result.content+ order = await page.evaluate("[...document.querySelectorAll('#list li')].map((li) => li.textContent)")+ assert order == ["Gamma", "Alpha", "Beta"]+ assert spec.billable is True+++@_skip_no_browser+async def test_drag_tool_drags_within_a_frame_and_refuses_a_target_in_another() -> None:+ # Playwright drags inside one document, so a pair inside one iframe must work and a drop target in+ # the parent document must be refused before anything is dispatched, by ref and by CSS selector.+ srcdoc = html.escape(_DRAG_LIST_HTML, quote=True)+ page_html = f'<!doctype html><html><body><button id="outside">Outside</button><iframe srcdoc="{srcdoc}" style="width:400px;height:300px"></iframe></body></html>'+ async with _content_page(page_html) as page:+ await page.wait_for_function(+ "document.querySelector('iframe').contentDocument?.querySelectorAll('li').length === 3"+ )+ tools = build_browser_tools(_fixed_page_provider(page))+ observed = await _tool(tools, "observe").handler({})+ drag = _tool(tools, "drag").handler+ gamma, alpha = _ref_line(observed.content, "Gamma"), _ref_line(observed.content, "Alpha")+ frame = page.frames[1]+ # A drop handler can start a save: the drop target inside the frame must then read as still in flight.+ await frame.evaluate(+ "[...document.querySelectorAll('li')][0].addEventListener('drop', (e) => { e.target.textContent = 'Processing'; })"+ )+ result = await drag({"selector": gamma, "target": alpha})+ assert result.status == "ok", result.content+ assert await pending_marker(page, result.data["drop_target"]) == "Processing"+ assert set(taskv3_tools._frame_work(page)["submitted"]) == {result.data["drop_target"]}+ assert await frame.evaluate("[...document.querySelectorAll('li')].map((li) => li.textContent)") == [+ "Gamma",+ "Processing",+ "Beta",+ ]+ observed = await _tool(tools, "observe").handler({})+ source = _ref_line(observed.content, "Beta")+ for target in (_ref_line(observed.content, "Outside"), "#outside"):+ refused = await drag({"selector": source, "target": target})+ assert refused.status == "error" and refused.error_class == "unreachable", (target, refused.content)+ missing = await drag({"selector": source, "target": "#nowhere"})+ assert missing.error_class == "stale_selector", missing.content+ invalid = await drag({"selector": source, "target": "li[data-x"})+ assert invalid.error_class == "invalid_selector" and "li[data-x" in invalid.content++def test_perception_tools_are_compactable_and_actions_are_not() -> None:# Compaction only elides tools flagged `compactable` on the real factory output. If a rename or a# refactor of the flag loop in tools.py stops flagging observe/get_html, the transcript stops being@@ -39654,6 +39731,7 @@ _SURFACE_OFF_TOOL_NAMES = frozenset("look","click","hover",+ "drag","type","select_option","select_combobox",