Skip to content

Automate browser based workflows with AI

Keep runner stalls from expiring broker test requests before they are forwarded (SKY-18688) (#8974)

Shuchang Zheng3c758c9a0c42
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 26c8757a9..fd004a21a 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -293,6 +293,10 @@ import { captureChatScope, type ChatScopeRefs } from "./chatReadScope";
import { WORKFLOW_COPILOT_DEMONSTRATION_FLAG } from "@/util/featureFlags";
import { safeHttpUrl } from "@/util/httpUrl";
import { useDemonstrationRecorder } from "./useDemonstrationRecorder";
+import {
+ DemonstrationFileMarker,
+ DemonstrationFileTray,
+} from "./cards/DemonstrationFileTray";
// Cap on retained per-turn snap-back snapshots. A typical session has a
// handful of turns; this ceiling guards a runaway long-running chat.
@@ -6978,7 +6982,8 @@ export function WorkflowCopilotChat({
const demonstrationRecorder = useDemonstrationRecorder({
workflowPermanentId,
browserSessionId: demonstrationEnabled ? liveBrowserSessionId : null,
- onDone: async ({ demonstrationId, actions }) => {
+ deleteFile: removeUnsentUpload,
+ onDone: async ({ demonstrationId, actions, files }) => {
const messageId =
demonstrationMessageIdRef.current ??
`demonstration-${crypto.randomUUID()}`;
@@ -7010,6 +7015,7 @@ export function WorkflowCopilotChat({
sent =
(await handleSendRef.current?.(DEMONSTRATION_DONE_MESSAGE, {
optimisticMessageId: messageId,
+ attachments: files.slice(0, ATTACHMENT_COUNT_LIMIT),
})) ?? false;
} finally {
if (!sent) {
@@ -7777,8 +7783,12 @@ export function WorkflowCopilotChat({
: "Type this sign-in yourself to keep its value private.",
}
: null;
+ const demonstrationFileRequest =
+ demonstrationRecorder.phase === "recording"
+ ? demonstrationRecorder.fileRequest
+ : null;
const attentionQueue: {
- kind: CopilotAttention | "demonstration_credential";
+ kind: CopilotAttention | "demonstration_credential" | "demonstration_file";
upNext?: string;
}[] = [];
if (trayQuestion) {
@@ -7799,6 +7809,12 @@ export function WorkflowCopilotChat({
upNext: "Fill this sign-in",
});
}
+ if (demonstrationFileRequest) {
+ attentionQueue.push({
+ kind: "demonstration_file",
+ upNext: "Choose a file",
+ });
+ }
if (isDemonstrating) {
attentionQueue.push({
kind: "demonstration",
@@ -7809,8 +7825,14 @@ export function WorkflowCopilotChat({
const headerAttention =
activeAttention === "demonstration_credential"
? "credential"
- : activeAttention;
- const attentionUpNext = attentionQueue[1]?.upNext ?? null;
+ : activeAttention === "demonstration_file"
+ ? "demonstration"
+ : activeAttention;
+ // The demonstration is already on screen as the composer strip, so it is never worth an "Up next" row.
+ const attentionUpNext =
+ attentionQueue[1]?.kind === "demonstration"
+ ? null
+ : (attentionQueue[1]?.upNext ?? null);
// Only the open tray replaces its transcript card; queued items stay actionable inline.
const dockedPauseFrame =
activeAttention === "credential" ? trayPauseFrame : null;
@@ -7823,7 +7845,10 @@ export function WorkflowCopilotChat({
? demonstrationCredentialFrame
: null;
const hasVisibleAttentionTray =
- activeAttention !== null && activeAttention !== "demonstration";
+ activeAttention !== null &&
+ activeAttention !== "demonstration" &&
+ activeAttention !== "demonstration_credential" &&
+ activeAttention !== "demonstration_file";
const [collapsedAttentionKey, setCollapsedAttentionKey] = useState<
string | null
>(null);
@@ -11746,6 +11771,18 @@ export function WorkflowCopilotChat({
}
/>
</div>
+ {message.id === demonstrationMessageIdRef.current &&
+ activeAttention === "demonstration_file" ? (
+ <div className={TURN_ROW_INSET}>
+ <DemonstrationFileMarker />
+ </div>
+ ) : null}
+ {message.id === demonstrationMessageIdRef.current &&
+ activeAttention === "demonstration_credential" ? (
+ <div className={TURN_ROW_INSET}>
+ <CredentialAskMarker />
+ </div>
+ ) : null}
</Fragment>
) : (
renderedMessage
@@ -12225,28 +12262,6 @@ export function WorkflowCopilotChat({
tray={attentionTray(`account:${dockedAccountChoice.turnId}`)}
/>
) : null}
- {dockedDemonstrationCredential && demonstrationSensitiveField ? (
- <CredentialCard
- key={`${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}:${demonstrationSensitiveField.url}`}
- frame={dockedDemonstrationCredential}
- mode="inline-pause"
- offerCredentials={demonstrationRecorder.canFillCredentials}
- reloadKey={credentialsReloadKey}
- onConnect={(credentialId) => {
- if (credentialId) {
- demonstrationRecorder.fillCredential(credentialId);
- return;
- }
- pendingCredentialConnect.current = null;
- pendingDemonstrationCredentialConnect.current = true;
- setCredentialModalOpen(true);
- }}
- onSkip={demonstrationRecorder.dismissSensitiveField}
- tray={attentionTray(
- `demonstration-credential:${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}`,
- )}
- />
- ) : null}
{showQueuedStrip && queuedPrompt ? (
<QueuedMessageStrip
text={queuedPrompt.content}
@@ -12349,6 +12364,45 @@ export function WorkflowCopilotChat({
</div>
</div>
) : null}
+ {dockedDemonstrationCredential && demonstrationSensitiveField ? (
+ <CredentialCard
+ key={`${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}:${demonstrationSensitiveField.url}`}
+ frame={dockedDemonstrationCredential}
+ mode="inline-pause"
+ offerCredentials={demonstrationRecorder.canFillCredentials}
+ reloadKey={credentialsReloadKey}
+ onConnect={(credentialId) => {
+ if (credentialId) {
+ demonstrationRecorder.fillCredential(credentialId);
+ return;
+ }
+ pendingCredentialConnect.current = null;
+ pendingDemonstrationCredentialConnect.current = true;
+ setCredentialModalOpen(true);
+ }}
+ onSkip={demonstrationRecorder.dismissSensitiveField}
+ tray={{
+ ...attentionTray(
+ `demonstration-credential:${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}`,
+ ),
+ embedded: true,
+ }}
+ />
+ ) : null}
+ {activeAttention === "demonstration_file" &&
+ demonstrationFileRequest ? (
+ <DemonstrationFileTray
+ label={demonstrationFileRequest.label}
+ busy={demonstrationRecorder.fileBusy}
+ onChoose={(file) => void demonstrationRecorder.chooseFile(file)}
+ tray={{
+ ...attentionTray(
+ `demonstration-file:${demonstrationFileRequest.label}`,
+ ),
+ embedded: true,
+ }}
+ />
+ ) : null}
{demonstrationRecorder.phase === "stopped" &&
demonstrationRecorder.browserRestarted ? (
<div className="border-b border-border px-3 py-1.5">
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx
index 79c7f3d06..fa55fab92 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx
@@ -6,6 +6,8 @@ import { cn } from "@/util/utils";
const TRAY_FRAME =
"rounded-t-lg border border-b-0 border-amber-500/50 bg-amber-500/[0.06]";
+// Inside the composer, under the demonstration strip, the composer's own border frames it.
+const EMBEDDED_TRAY_FRAME = "border-b border-amber-500/30 bg-amber-500/[0.06]";
const TRAY_TITLE = "font-semibold text-amber-700 dark:text-yellow-400";
// How the chat asks a request card to render itself as the docked tray.
@@ -13,6 +15,7 @@ export interface AttentionTrayPresentation {
collapsed: boolean;
onCollapsedChange: (collapsed: boolean) => void;
upNext?: string | null;
+ embedded?: boolean;
}
type AttentionTrayProps = Omit<HTMLAttributes<HTMLDivElement>, "title"> & {
@@ -28,6 +31,7 @@ type AttentionTrayProps = Omit<HTMLAttributes<HTMLDivElement>, "title"> & {
minimizeLabel: string;
// Another request waiting behind this one; the chat shows one at a time.
upNext?: string | null;
+ embedded?: boolean;
children: ReactNode;
};
@@ -43,18 +47,15 @@ export function AttentionTray({
onCollapsedChange,
minimizeLabel,
upNext,
+ embedded = false,
children,
className,
...groupProps
}: AttentionTrayProps) {
+ const frame = embedded ? EMBEDDED_TRAY_FRAME : TRAY_FRAME;
if (collapsed) {
return (
- <div
- className={cn(
- "flex items-center gap-2 px-3 py-1.5 text-xs",
- TRAY_FRAME,
- )}
- >
+ <div className={cn("flex items-center gap-2 px-3 py-1.5 text-xs", frame)}>
<span
aria-hidden
className="size-2 shrink-0 rounded-full bg-amber-500"
@@ -89,7 +90,7 @@ export function AttentionTray({
{...groupProps}
className={cn(
"flex max-h-[50vh] min-w-0 flex-col overflow-hidden",
- TRAY_FRAME,
+ frame,
className,
)}
>
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
index de31471ed..4678b5455 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
@@ -648,6 +648,7 @@ function AskChrome({
onCollapsedChange={tray.onCollapsedChange}
minimizeLabel="Minimize sign-in request"
upNext={tray.upNext}
+ embedded={tray.embedded}
>
<div className="flex min-h-0 flex-col gap-1 overflow-y-auto px-3 pb-2.5 pt-1">
{lines.map((line, index) => (
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/DemonstrationFileTray.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/DemonstrationFileTray.tsx
new file mode 100644
index 000000000..722ada2a9
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/DemonstrationFileTray.tsx
@@ -0,0 +1,88 @@
+import { FileIcon, UploadIcon } from "@radix-ui/react-icons";
+import { useRef } from "react";
+
+import { Button } from "@/components/ui/button";
+
+import {
+ AttentionMarker,
+ AttentionTray,
+ type AttentionTrayPresentation,
+} from "./AttentionTray";
+
+// The page opened a file chooser during a demonstration; the remote browser's own dialog is unusable.
+export function DemonstrationFileTray({
+ label,
+ busy,
+ onChoose,
+ tray,
+}: {
+ label: string;
+ busy: boolean;
+ onChoose: (file: File | null) => void;
+ tray: AttentionTrayPresentation;
+}) {
+ const inputRef = useRef<HTMLInputElement>(null);
+ const title = `Choose a file for “${label}”`;
+ return (
+ <AttentionTray
+ aria-label="File request"
+ title={title}
+ wrapTitle
+ collapsedTitle="The page wants a file"
+ collapsed={tray.collapsed}
+ onCollapsedChange={tray.onCollapsedChange}
+ minimizeLabel="Minimize file request"
+ upNext={tray.upNext}
+ embedded={tray.embedded}
+ >
+ <div className="px-3 pb-2.5 pt-1 text-[11px] leading-relaxed text-muted-foreground">
+ The page opened a file picker. Choose the file to attach here; Copilot
+ saves it as the default for the workflow&apos;s file input.
+ </div>
+ <div className="flex shrink-0 flex-wrap items-center gap-2 border-t border-border px-3 py-1.5">
+ <Button
+ type="button"
+ size="sm"
+ disabled={busy}
+ onClick={() => inputRef.current?.click()}
+ >
+ <UploadIcon className="mr-1.5 size-3.5" />
+ Choose file
+ </Button>
+ <input
+ ref={inputRef}
+ type="file"
+ className="sr-only"
+ tabIndex={-1}
+ aria-hidden
+ onChange={(event) => {
+ const file = event.target.files?.[0];
+ event.target.value = "";
+ if (file) onChoose(file);
+ }}
+ />
+ <div className="ml-auto">
+ <Button
+ type="button"
+ size="sm"
+ variant="outline"
+ disabled={busy}
+ onClick={() => onChoose(null)}
+ >
+ Skip
+ </Button>
+ </div>
+ </div>
+ </AttentionTray>
+ );
+}
+
+// Where the file request was raised in the transcript; the tray holds the actions.
+export function DemonstrationFileMarker() {
+ return (
+ <AttentionMarker
+ icon={<FileIcon aria-hidden className="size-3.5 shrink-0" />}
+ title="The page wants a file"
+ />
+ );
+}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts b/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts
index a55bbd480..a3990c227 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts
@@ -7,10 +7,12 @@ import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import type {
+ CopilotAttachedFile,
WorkflowCopilotActiveDemonstration,
WorkflowCopilotDemonstrationActionUpdate,
WorkflowCopilotDemonstrationControlRequest,
WorkflowCopilotDemonstrationControlResult,
+ WorkflowCopilotDemonstrationFileRequestedUpdate,
WorkflowCopilotDemonstrationInsertion,
WorkflowCopilotDemonstrationSensitiveFieldUpdate,
WorkflowCopilotDemonstrationUpdate,
@@ -26,6 +28,7 @@ type DemonstrationAction = Pick<
type DemonstrationDone = {
demonstrationId: string;
actions: DemonstrationAction[];
+ files: CopilotAttachedFile[];
};
type UseDemonstrationRecorderOptions = {
@@ -33,6 +36,7 @@ type UseDemonstrationRecorderOptions = {
browserSessionId: string | null | undefined;
onDone: (demonstration: DemonstrationDone) => boolean | Promise<boolean>;
onStartFailed: () => void;
+ deleteFile: (fileId: string) => Promise<unknown>;
};
export function useDemonstrationRecorder({
@@ -40,6 +44,7 @@ export function useDemonstrationRecorder({
browserSessionId,
onDone,
onStartFailed,
+ deleteFile,
}: UseDemonstrationRecorderOptions) {
const credentialGetter = useCredentialGetter();
const [demonstrationId, setDemonstrationId] = useState<string | null>(null);
@@ -49,6 +54,14 @@ export function useDemonstrationRecorder({
const [actionCount, setActionCount] = useState(0);
const [sensitiveField, setSensitiveField] =
useState<WorkflowCopilotDemonstrationSensitiveFieldUpdate | null>(null);
+ const [fileRequest, setFileRequest] =
+ useState<WorkflowCopilotDemonstrationFileRequestedUpdate | null>(null);
+ // Every file uploaded for this demonstration, so a discard can delete them.
+ const uploadedFileIdsRef = useRef<string[]>([]);
+ // The control only queues the attach; the tray stays locked until the recorder reports the outcome.
+ const [awaitingFileOutcome, setAwaitingFileOutcome] = useState(false);
+ // The files the recorder reports attached; they ride on the Done message.
+ const filesRef = useRef<CopilotAttachedFile[]>([]);
const [busy, setBusy] = useState(false);
const [canFillCredentials, setCanFillCredentials] = useState(false);
const [hasSkippedSensitiveField, setHasSkippedSensitiveField] =
@@ -64,6 +77,8 @@ export function useDemonstrationRecorder({
onDoneRef.current = onDone;
const onStartFailedRef = useRef(onStartFailed);
onStartFailedRef.current = onStartFailed;
+ const deleteFileRef = useRef(deleteFile);
+ deleteFileRef.current = deleteFile;
const sessionRef = useRef({ workflowPermanentId, browserSessionId });
const updateBusy = useCallback((next: boolean) => {
@@ -90,6 +105,10 @@ export function useDemonstrationRecorder({
setActions([]);
setActionCount(0);
setSensitiveField(null);
+ setFileRequest(null);
+ setAwaitingFileOutcome(false);
+ uploadedFileIdsRef.current = [];
+ filesRef.current = [];
setCanFillCredentials(false);
setHasSkippedSensitiveField(false);
setLimitReached(false);
@@ -116,6 +135,23 @@ export function useDemonstrationRecorder({
setActions(actionsRef.current);
setActionCount((current) => Math.max(current, frame.seq));
setSensitiveField(null);
+ if (frame.tool_name === "upload_attached_file") {
+ setAwaitingFileOutcome(false);
+ if (frame.file_id) {
+ filesRef.current = [
+ ...filesRef.current,
+ {
+ file_id: frame.file_id,
+ filename: frame.filename ?? "",
+ available: true,
+ },
+ ];
+ }
+ setFileRequest(null);
+ }
+ return false;
+ case "demonstration_file_requested":
+ setFileRequest(frame);
return false;
case "demonstration_sensitive_field":
if (frame.field === "other") {
@@ -125,6 +161,7 @@ export function useDemonstrationRecorder({
return false;
case "demonstration_notice":
if (frame.level === "error") {
+ setAwaitingFileOutcome(false);
toast({
title: "Could not update the demonstration",
description: frame.message,
@@ -138,6 +175,8 @@ export function useDemonstrationRecorder({
}
setActionCount(frame.action_count);
setSensitiveField(null);
+ setFileRequest(null);
+ setAwaitingFileOutcome(false);
setHasSkippedSensitiveField(false);
if (frame.reason === "time_limit") setLimitReached(true);
if (frame.reason === "event_limit") {
@@ -306,6 +345,56 @@ export function useDemonstrationRecorder({
[runControl, sensitiveField],
);
+ // The server stores the file for this demonstration and queues the attach itself, so the
+ // recorder never attaches a file the client merely names.
+ const uploadToDemonstration = useCallback(
+ async (file: File): Promise<string | null> => {
+ const id = demonstrationIdRef.current;
+ if (!id) throw new Error("No demonstration is active.");
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ const formData = new FormData();
+ formData.append("demonstration_id", id);
+ formData.append("file", file);
+ const response =
+ await client.post<WorkflowCopilotDemonstrationControlResult>(
+ "/workflow/copilot/demonstration/upload",
+ formData,
+ { headers: { "Content-Type": "multipart/form-data" } },
+ );
+ return response.data.file_id ?? null;
+ },
+ [credentialGetter],
+ );
+
+ // null skips the requested file; the recorder still logs that the page asked for one.
+ const chooseFile = useCallback(
+ async (file: File | null) => {
+ if (busyRef.current || awaitingFileOutcome) return;
+ updateBusy(true);
+ // Set before the request: the recorder's outcome frame can arrive before the response does.
+ setAwaitingFileOutcome(true);
+ try {
+ if (!file) {
+ await sendControl({ action: "upload_file" });
+ return;
+ }
+ const fileId = await uploadToDemonstration(file);
+ if (fileId) uploadedFileIdsRef.current.push(fileId);
+ } catch (error) {
+ setAwaitingFileOutcome(false);
+ toast({
+ title: "Could not upload the file",
+ description:
+ error instanceof Error ? error.message : "Please try again.",
+ variant: "destructive",
+ });
+ } finally {
+ updateBusy(false);
+ }
+ },
+ [awaitingFileOutcome, sendControl, updateBusy, uploadToDemonstration],
+ );
+
const dismissSensitiveField = useCallback(() => {
setSensitiveField(null);
}, []);
@@ -330,6 +419,7 @@ export function useDemonstrationRecorder({
}
stream?.abort();
setSensitiveField(null);
+ setFileRequest(null);
updatePhase("stopped");
setHasSkippedSensitiveField(false);
if (stopped?.action_count === 0) {
@@ -345,6 +435,7 @@ export function useDemonstrationRecorder({
const sent = await onDoneRef.current({
demonstrationId: id,
actions: recordedActions,
+ files: filesRef.current,
});
if (sent) {
clear();
@@ -364,6 +455,13 @@ export function useDemonstrationRecorder({
}
}, [clear, sendControl, updateBusy, updatePhase]);
+ // Discarded uploads were never sent with a message, so nothing else would delete them.
+ const discardUploads = useCallback(() => {
+ for (const fileId of uploadedFileIdsRef.current) {
+ void deleteFileRef.current(fileId);
+ }
+ }, []);
+
const cancel = useCallback(async (): Promise<boolean> => {
const id = demonstrationIdRef.current;
if (!id || busyRef.current) return false;
@@ -373,6 +471,7 @@ export function useDemonstrationRecorder({
} catch (error) {
const status = isAxiosError(error) ? error.response?.status : undefined;
if (status === 404 || status === 409) {
+ discardUploads();
clear();
return true;
}
@@ -385,9 +484,10 @@ export function useDemonstrationRecorder({
updateBusy(false);
return false;
}
+ discardUploads();
clear();
return true;
- }, [clear, sendControl, updateBusy]);
+ }, [clear, discardUploads, sendControl, updateBusy]);
useEffect(() => {
const previous = sessionRef.current;
@@ -506,6 +606,8 @@ export function useDemonstrationRecorder({
actions,
actionCount,
sensitiveField,
+ fileRequest,
+ fileBusy: busy || awaitingFileOutcome,
canFillCredentials,
hasSkippedSensitiveField,
limitReached,
@@ -515,6 +617,7 @@ export function useDemonstrationRecorder({
resume,
navigate,
fillCredential,
+ chooseFile,
dismissSensitiveField,
done,
cancel,
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index ceb39cd6f..a3483f721 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -583,6 +583,9 @@ export interface WorkflowCopilotDemonstrationActionUpdate {
| "navigate_browser"
| "upload_attached_file"
| "fill_credential_field";
+ // Present only on an upload the recorder attached; a skipped file request has none.
+ file_id?: string;
+ filename?: string;
}
export interface WorkflowCopilotDemonstrationSensitiveFieldUpdate {
@@ -592,6 +595,11 @@ export interface WorkflowCopilotDemonstrationSensitiveFieldUpdate {
url: string;
}
+export interface WorkflowCopilotDemonstrationFileRequestedUpdate {
+ type: "demonstration_file_requested";
+ label: string;
+}
+
export interface WorkflowCopilotDemonstrationNoticeUpdate {
type: "demonstration_notice";
message: string;
@@ -614,7 +622,7 @@ export interface WorkflowCopilotDemonstrationStoppedUpdate {
export interface WorkflowCopilotDemonstrationControlRequest {
demonstration_id: string;
- action: "navigate" | "fill_credential" | "stop" | "cancel";
+ action: "navigate" | "fill_credential" | "upload_file" | "stop" | "cancel";
url?: string | null;
selector?: string | null;
credential_id?: string | null;
@@ -625,12 +633,14 @@ export interface WorkflowCopilotDemonstrationControlRequest {
export interface WorkflowCopilotDemonstrationControlResult {
seq: number;
action_count?: number | null;
+ file_id?: string | null;
}
export type WorkflowCopilotDemonstrationUpdate =
| WorkflowCopilotDemonstrationStartedUpdate
| WorkflowCopilotDemonstrationActionUpdate
| WorkflowCopilotDemonstrationSensitiveFieldUpdate
+ | WorkflowCopilotDemonstrationFileRequestedUpdate
| WorkflowCopilotDemonstrationNoticeUpdate
| WorkflowCopilotDemonstrationStoppedUpdate;
diff --git a/skyvern/forge/sdk/copilot/demonstration_recorder.py b/skyvern/forge/sdk/copilot/demonstration_recorder.py
index 44a878e93..fc6fc4070 100644
--- a/skyvern/forge/sdk/copilot/demonstration_recorder.py
+++ b/skyvern/forge/sdk/copilot/demonstration_recorder.py
@@ -1,8 +1,10 @@
from __future__ import annotations
import asyncio
+import base64
import contextlib
import json
+import mimetypes
import secrets
import time
from collections.abc import Awaitable, Coroutine
@@ -19,6 +21,7 @@ from playwright.async_api import Page
from skyvern.cli.mcp_tools._localhost import is_localhost_url
from skyvern.config import settings
from skyvern.exceptions import BlockedHost, SkyvernHTTPException
+from skyvern.forge import app
from skyvern.forge.sdk.cache.base import BaseCache
from skyvern.forge.sdk.copilot.config import ALL_BLOCK_FAMILIES, AuthoringCapability
from skyvern.forge.sdk.copilot.context import CopilotContext
@@ -38,6 +41,7 @@ from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
from skyvern.forge.sdk.copilot.tools.credential_fill import fill_credential_field
from skyvern.forge.sdk.core.event_source_stream import EventSourceStream
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotDemonstrationControlRequest
+from skyvern.services import uploaded_file_service
from skyvern.utils.url_validators import validate_fetch_url
LOG = structlog.get_logger()
@@ -62,6 +66,21 @@ DEMONSTRATION_OWNER_LIVE_SECONDS = 20
DEMONSTRATION_MAX_SECONDS = 30 * 60
DEMONSTRATION_REPLAY_LEASE = timedelta(minutes=5)
+_DESCRIBE_FILE_INPUT = (
+ "function() { const tap = globalThis.__skyvernDemoTap; "
+ "return tap ? {selector: tap.selectorFor(this), label: tap.labelFor(this) || ''} : null; }"
+)
+# Built in the page rather than DOM.setFileInputFiles, which takes a path on the browser's host.
+_ATTACH_FILE = """function(name, type, data) {
+ const bytes = Uint8Array.from(atob(data), (char) => char.charCodeAt(0));
+ const transfer = new DataTransfer();
+ transfer.items.add(new File([bytes], name, {type}));
+ this.files = transfer.files;
+ this.dispatchEvent(new Event("input", {bubbles: true, composed: true}));
+ this.dispatchEvent(new Event("change", {bubbles: true}));
+ return this.files.length;
+}"""
+
DemonstrationStatus = Literal["recording", "stopping", "stopped", "replaying", "consumed", "cancelled"]
DemonstrationStopReason = Literal[
"stopped", "cancelled", "disconnected", "browser_gone", "time_limit", "event_limit", "consumed"
@@ -414,6 +433,7 @@ async def append_demonstration_control(
*,
organization_id: str,
request: WorkflowCopilotDemonstrationControlRequest,
+ file_id: str | None = None,
) -> int:
record_key = demonstration_record_key(organization_id, request.demonstration_id)
controls_key = demonstration_controls_key(organization_id, request.demonstration_id)
@@ -423,6 +443,9 @@ async def append_demonstration_control(
raise DemonstrationRejection(HTTPStatus.NOT_FOUND, "Unknown demonstration")
if record.status not in {"recording", "stopping", "stopped"}:
raise DemonstrationRejection(HTTPStatus.CONFLICT, "Demonstration has ended")
+ # A stopping recorder may already have drained its controls, so an upload queued now would never run.
+ if file_id is not None and record.status != "recording":
+ raise DemonstrationRejection(HTTPStatus.CONFLICT, "Demonstration is not recording")
if request.action == "stop":
if record.status == "recording":
record.status = "stopping"
@@ -448,6 +471,8 @@ async def append_demonstration_control(
raise DemonstrationRejection(HTTPStatus.CONFLICT, "Too many browser controls for this demonstration")
seq = len(controls) + 1
entry = {"seq": seq, **request.model_dump(exclude_none=True, exclude={"demonstration_id"})}
+ if file_id is not None:
+ entry["file_id"] = file_id
controls.append(entry)
await cache.set(controls_key, json.dumps(controls, separators=(",", ":")), ex=DEMONSTRATION_TTL)
return seq
@@ -532,6 +557,26 @@ async def demonstration_action_labels(
return labels
+async def demonstration_file_ids(
+ cache: BaseCache, organization_id: str, demonstration_id: str, event_count: int
+) -> list[str]:
+ events = await asyncio.gather(
+ *(
+ cache.get(demonstration_event_key(organization_id, demonstration_id, seq))
+ for seq in range(1, event_count + 1)
+ )
+ )
+ file_ids: list[str] = []
+ for raw in events:
+ try:
+ event = json.loads(raw)
+ except (TypeError, ValueError):
+ continue
+ if isinstance(event, dict) and event.get("kind") == "upload" and isinstance(event.get("file_id"), str):
+ file_ids.append(event["file_id"])
+ return file_ids
+
+
class DemonstrationRecorder:
def __init__(
self,
@@ -559,6 +604,9 @@ class DemonstrationRecorder:
self.sensitive_count = 0
self.event_limit_reached = False
self.credential_filled_selectors: set[str] = set()
+ # The page keeps running while a request is open, so later choosers wait their turn.
+ self.file_requests: list[dict[str, Any]] = []
+ self.intercepting_file_chooser = False
self.started_at = time.monotonic()
self.recording_ctx = CopilotContext(
organization_id=organization_id,
@@ -593,8 +641,11 @@ class DemonstrationRecorder:
self.page = pages[-1]
self.tap = PageTap(browser_context, max_events=DEMONSTRATION_MAX_EVENTS)
await self.tap.start(pages)
- with contextlib.suppress(Exception):
+ try:
await self.tap.set_intercept_file_chooser_dialog(True)
+ self.intercepting_file_chooser = True
+ except Exception:
+ LOG.warning("copilot_demonstration_file_chooser_intercept_failed", exc_info=True)
await self.stream.send(
{
"type": "demonstration_started",
@@ -701,6 +752,14 @@ class DemonstrationRecorder:
frame["path"] = path
kind = payload.get("kind")
selector = payload.get("selector")
+ if kind == "file_click":
+ # With interception on, the chooser this click opens arrives as its own file_chooser event.
+ if not self.intercepting_file_chooser:
+ await self._record_gap_event(event, frame, "file_chooser")
+ return
+ if kind == "file_chooser":
+ await self._request_file(event, frame)
+ return
if kind == "secret_focus":
field = payload.get("field")
await self.stream.send(
@@ -716,10 +775,7 @@ class DemonstrationRecorder:
return
event_kind = "secret" if kind in {"secret", "secret_select"} else kind
if event_kind not in {"click", "type", "select", "key", "navigate", "secret"}:
- gap_reason = (
- "file_chooser" if kind in {"file_click", "upload"} else f"unsupported_event:{kind or 'unknown'}"
- )
- await self._record_gap_event(event, frame, gap_reason)
+ await self._record_gap_event(event, frame, f"unsupported_event:{kind or 'unknown'}")
return
source_url = str(payload.get("url") or "") if event.is_main_frame else str(event.page.url or "")
if event_kind == "navigate":
@@ -779,6 +835,21 @@ class DemonstrationRecorder:
}
)
+ async def _request_file(self, event: TapEvent, frame: dict[str, Any]) -> None:
+ described = await self.tap.call_on_file_input(event, _DESCRIBE_FILE_INPUT) if self.tap is not None else None
+ if (
+ not isinstance(described, dict)
+ or not isinstance(described.get("selector"), str)
+ or not described["selector"]
+ ):
+ await self._record_gap_event(event, frame, "file_chooser")
+ return
+ selector = described["selector"]
+ field_label = safe_label(described.get("label")) or "file input"
+ self.file_requests.append({"event": event, "frame": frame, "selector": selector, "field_label": field_label})
+ if len(self.file_requests) == 1:
+ await self.stream.send({"type": "demonstration_file_requested", "label": field_label})
+
async def _record_gap_event(self, event: TapEvent, frame: dict[str, Any], reason: str) -> None:
stored = {
"ts": event.payload.get("ts"),
@@ -807,6 +878,8 @@ class DemonstrationRecorder:
await self._run_control_with_owner_heartbeat(self._navigate(entry))
elif action == "fill_credential":
await self._run_control_with_owner_heartbeat(self._fill_credential(entry))
+ elif action == "upload_file":
+ await self._run_control_with_owner_heartbeat(self._upload_file(entry))
return None
async def _run_control_with_owner_heartbeat(self, control: Awaitable[None]) -> None:
@@ -933,6 +1006,75 @@ class DemonstrationRecorder:
}
)
+ async def _upload_file(self, entry: dict[str, Any]) -> None:
+ file_id = entry.get("file_id")
+ if not self.file_requests or self.tap is None:
+ if isinstance(file_id, str):
+ await self._discard_upload(file_id)
+ await self._notice("The page is no longer waiting for a file.", level="error")
+ return
+ request = self.file_requests[0]
+ tap_event: TapEvent = request["event"]
+ field_label = request["field_label"]
+ event = _skipped_upload_event(request)
+ if isinstance(file_id, str):
+ uploaded = None
+ data: bytes | None = None
+ try:
+ uploaded = await app.DATABASE.uploaded_files.get_uploaded_file(
+ file_id=file_id, organization_id=self.organization_id
+ )
+ if uploaded is not None:
+ data = await app.STORAGE.download_managed_file(uploaded.storage_uri, self.organization_id)
+ except Exception:
+ # A failed read is the same outcome as a failed attach: the request stays open for a retry.
+ LOG.warning("copilot_demonstration_upload_read_failed", exc_info=True)
+ attached = (
+ uploaded is not None
+ and data is not None
+ and await self.tap.call_on_file_input(
+ tap_event,
+ _ATTACH_FILE,
+ uploaded.filename,
+ mimetypes.guess_type(uploaded.filename)[0] or "application/octet-stream",
+ base64.b64encode(data).decode(),
+ )
+ == 1
+ )
+ if not attached or uploaded is None or data is None:
+ await self._discard_upload(file_id)
+ await self._notice("The file could not be attached to the page.", level="error")
+ return
+ event.update(
+ file_id=file_id,
+ filename=uploaded.filename,
+ size_bytes=len(data),
+ label=f'Upload file to "{field_label}"'[:60],
+ )
+ seq = await self._append_event(event)
+ if seq is not None:
+ await self.stream.send(
+ {
+ "type": "demonstration_action",
+ "seq": seq,
+ "label": event["label"],
+ "tool_name": "upload_attached_file",
+ **({"file_id": event["file_id"], "filename": event["filename"]} if "file_id" in event else {}),
+ }
+ )
+ self.file_requests.pop(0)
+ if self.file_requests:
+ await self.stream.send(
+ {"type": "demonstration_file_requested", "label": self.file_requests[0]["field_label"]}
+ )
+
+ async def _discard_upload(self, file_id: str) -> None:
+ # The file was stored only for this attach, so nothing else references it once the attach is off.
+ try:
+ await uploaded_file_service.delete_uploaded_file(file_id=file_id, organization_id=self.organization_id)
+ except Exception:
+ LOG.warning("copilot_demonstration_upload_discard_failed", exc_info=True)
+
async def _notice(self, message: str, *, level: Literal["info", "error"] = "info") -> None:
await self.stream.send({"type": "demonstration_notice", "message": message, "level": level})
@@ -968,6 +1110,10 @@ class DemonstrationRecorder:
reason = await self._drain_tap() or reason
with contextlib.suppress(Exception):
await tap.set_intercept_file_chooser_dialog(False)
+ # A request still open at stop was never answered; replay states it rather than dropping the step.
+ for request in self.file_requests:
+ await self._append_event(_skipped_upload_event(request))
+ self.file_requests.clear()
final_url = str(self.page.url or "") if self.page is not None and not self.page.is_closed() else None
record_key = demonstration_record_key(self.organization_id, self.demonstration_id)
event_ttl: timedelta | None = None
@@ -1014,6 +1160,19 @@ class DemonstrationRecorder:
)
+def _skipped_upload_event(request: dict[str, Any]) -> dict[str, Any]:
+ tap_event: TapEvent = request["event"]
+ return {
+ "ts": datetime.now(UTC).timestamp() * 1000,
+ "kind": "upload",
+ "selector": request["selector"],
+ "source_url": str(tap_event.page.url or ""),
+ "frame": request["frame"],
+ "field_label": request["field_label"],
+ "label": f'Skip file for "{request["field_label"]}"'[:60],
+ }
+
+
def safe_label(value: str | None) -> str:
if not isinstance(value, str):
return ""
diff --git a/skyvern/forge/sdk/copilot/demonstration_replay.py b/skyvern/forge/sdk/copilot/demonstration_replay.py
index 03ea9f240..d5685b728 100644
--- a/skyvern/forge/sdk/copilot/demonstration_replay.py
+++ b/skyvern/forge/sdk/copilot/demonstration_replay.py
@@ -37,7 +37,7 @@ from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer
from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm, summarize_tool_result
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, screen_raw_secret_safety
from skyvern.forge.sdk.copilot.runtime import DemonstratedAction, ScoutedInteraction, staged_demonstrated_action
-from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt, redact_secretlike_filename
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
from skyvern.forge.sdk.copilot.streaming_adapter import parse_tool_output
from skyvern.forge.sdk.copilot.tools.scouting import record_scouted_interaction_fact
@@ -234,6 +234,31 @@ async def _replay_events(
)
await record_call("fill_credential_field", args, args, output)
continue
+ if kind == "upload":
+ selector = str(event.get("selector") or "")
+ file_id = event.get("file_id")
+ if not isinstance(file_id, str):
+ seq = event.get("seq")
+ step = f"Demonstration step {seq}" if isinstance(seq, int) else "A demonstration step"
+ field_label = safe_label(event.get("field_label")) or "file input"
+ history.append(
+ {
+ "role": "user",
+ "content": NUDGE_SENTINEL
+ + f'{step} opened a file chooser for "{field_label}" ({selector}); the user chose no file.',
+ }
+ )
+ continue
+ args = {"file_id": file_id, "selector": selector}
+ output = {
+ "ok": True,
+ "file_id": file_id,
+ "filename": redact_secretlike_filename(str(event.get("filename") or "")),
+ "size_bytes": event.get("size_bytes"),
+ "change_delivered": True,
+ }
+ await record_call("upload_attached_file", args, args, output)
+ continue
translated = _translate_action(
ctx,
diff --git a/skyvern/forge/sdk/copilot/demonstration_tap.js b/skyvern/forge/sdk/copilot/demonstration_tap.js
index 33ddc8812..b123fc732 100644
--- a/skyvern/forge/sdk/copilot/demonstration_tap.js
+++ b/skyvern/forge/sdk/copilot/demonstration_tap.js
@@ -1177,6 +1177,7 @@
waiter = finish;
}),
selectorFor: getDurableSelector,
+ labelFor: ownLabelText,
flush,
stop: () => {
if (stopped) return [];
diff --git a/skyvern/forge/sdk/copilot/demonstration_tap.py b/skyvern/forge/sdk/copilot/demonstration_tap.py
index 85e7d1974..1ec4f60f6 100644
--- a/skyvern/forge/sdk/copilot/demonstration_tap.py
+++ b/skyvern/forge/sdk/copilot/demonstration_tap.py
@@ -194,8 +194,58 @@ class PageTap:
self._reconcile_frames(key, tree)
session.on("Page.frameNavigated", lambda event: self._frame_navigated(key, event))
session.on("Page.frameRequestedNavigation", lambda event: self._frame_requested_navigation(key, event))
+ session.on("Page.fileChooserOpened", lambda event: self._file_chooser_opened(key, event))
return key
+ def _file_chooser_opened(self, session_key: str, event: dict[str, Any]) -> None:
+ state = self._sessions.get(session_key)
+ backend_node_id = event.get("backendNodeId")
+ if state is None or not isinstance(backend_node_id, int):
+ return
+ frame_id = str(event.get("frameId") or "")
+ self._enqueue_event(
+ TapEvent(
+ page=state["page"],
+ session_key=session_key,
+ frame_id=frame_id,
+ is_main_frame=bool(state["is_page_session"] and frame_id == state["main_frame_id"]),
+ frame_name=str(state["frames"].get(frame_id, {}).get("name") or ""),
+ payload={"kind": "file_chooser", "ts": time.time() * 1000, "backend_node_id": backend_node_id},
+ )
+ )
+
+ async def call_on_file_input(self, event: TapEvent, declaration: str, *args: Any) -> Any:
+ state = self._sessions.get(event.session_key)
+ if state is None:
+ return None
+ session = state["session"]
+ try:
+ world = await session.send(
+ "Page.createIsolatedWorld",
+ {"frameId": event.frame_id, "worldName": WORLD, "grantUniveralAccess": False},
+ )
+ resolved = await session.send(
+ "DOM.resolveNode",
+ {
+ "backendNodeId": event.payload["backend_node_id"],
+ "executionContextId": world["executionContextId"],
+ },
+ )
+ result = await session.send(
+ "Runtime.callFunctionOn",
+ {
+ "functionDeclaration": declaration,
+ "objectId": resolved["object"]["objectId"],
+ "arguments": [{"value": arg} for arg in args],
+ "returnByValue": True,
+ "awaitPromise": True,
+ },
+ )
+ except Exception:
+ LOG.info("copilot_demonstration_file_input_call_failed", exc_info=True)
+ return None
+ return result.get("result", {}).get("value")
+
def _frame_requested_navigation(self, session_key: str, event: dict[str, Any]) -> None:
state = self._sessions.get(session_key)
frame_id = str(event.get("frameId") or "")
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index ec11a0383..07357b7fc 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -84,6 +84,7 @@ from skyvern.forge.sdk.copilot.demonstration_recorder import (
append_demonstration_control,
claim_demonstration_recording,
demonstration_action_labels,
+ demonstration_file_ids,
demonstration_limit_reached,
demonstration_owner_is_live,
demonstration_time_limit_reached,
@@ -186,6 +187,7 @@ from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
COPILOT_PRIVATE_SETTINGS_KEY,
COPILOT_PROPOSAL_METADATA_KEY,
+ MAX_ATTACHED_FILES_PER_MESSAGE,
TURN_OPENER_SENDERS,
CopilotAttachedFile,
CopilotCancelSource,
@@ -246,6 +248,7 @@ from skyvern.schemas.workflows import (
WorkflowCreateYAMLRequest,
WorkflowDefinitionYAML,
)
+from skyvern.services import uploaded_file_service
from skyvern.services.browser_recording.evidence import build_recording_evidence
from skyvern.services.browser_recording.session_registry import interpretation_registry
from skyvern.utils.contained_effects import contained_effect
@@ -3857,7 +3860,7 @@ async def workflow_copilot_chat_post(
detail="demonstration_id is required for a Copilot demonstration",
)
try:
- await validate_demonstration_for_replay(
+ demonstration = await validate_demonstration_for_replay(
cache,
organization_id=organization.organization_id,
workflow_permanent_id=chat_request.workflow_permanent_id,
@@ -3865,6 +3868,13 @@ async def workflow_copilot_chat_post(
)
except DemonstrationRejection as rejection:
raise HTTPException(status_code=rejection.status_code, detail=rejection.detail) from rejection
+ # A reloaded card no longer holds the files it attached; the recorded upload events still do.
+ demonstrated_file_ids = await demonstration_file_ids(
+ cache, organization.organization_id, chat_request.demonstration_id, demonstration.event_count
+ )
+ chat_request.attached_file_ids = list(dict.fromkeys([*chat_request.attached_file_ids, *demonstrated_file_ids]))[
+ :MAX_ATTACHED_FILES_PER_MESSAGE
+ ]
raw_eval_mode = request.headers.get("x-copilot-eval-mode")
if raw_eval_mode is not None:
try:
@@ -4975,6 +4985,59 @@ async def workflow_copilot_demonstration_control(
return WorkflowCopilotDemonstrationControlResult(seq=seq)
+@base_router.post(
+ "/workflow/copilot/demonstration/upload",
+ include_in_schema=False,
+ response_model=WorkflowCopilotDemonstrationControlResult,
+)
+async def workflow_copilot_demonstration_upload(
+ demonstration_id: str = Form(..., max_length=64),
+ file: UploadFile = File(...),
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> WorkflowCopilotDemonstrationControlResult:
+ cache = await _demonstration_cache(organization.organization_id)
+ demonstration = await load_demonstration_record(cache, organization.organization_id, demonstration_id)
+ if demonstration is None:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Unknown demonstration")
+ await app.AGENT_FUNCTION.enforce_folder_access(
+ get_request_principal(),
+ "workflow_copilot.manage",
+ {"workflow_permanent_id": demonstration.workflow_permanent_id},
+ )
+ if demonstration.status != "recording":
+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Demonstration is not recording")
+ attached_file_ids = await demonstration_file_ids(
+ cache, organization.organization_id, demonstration_id, demonstration.event_count
+ )
+ # Done carries every attached file, so the message limit bounds them; failed attempts are not counted.
+ if len(attached_file_ids) >= MAX_ATTACHED_FILES_PER_MESSAGE:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=f"A demonstration can attach at most {MAX_ATTACHED_FILES_PER_MESSAGE} files",
+ )
+ await uploaded_file_service.validate_file_size(file, max_size_bytes=app.SETTINGS_MANAGER.MAX_UPLOAD_FILE_SIZE)
+ try:
+ uploaded, _ = await uploaded_file_service.save_uploaded_file(
+ file=file, organization_id=organization.organization_id
+ )
+ except uploaded_file_service.UploadStorageError as exc:
+ raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc
+ # The recorder attaches only files stored here for this demonstration, never an id the client names.
+ try:
+ seq = await append_demonstration_control(
+ cache,
+ organization_id=organization.organization_id,
+ request=WorkflowCopilotDemonstrationControlRequest(demonstration_id=demonstration_id, action="upload_file"),
+ file_id=uploaded.file_id,
+ )
+ except DemonstrationRejection as rejection:
+ await uploaded_file_service.delete_uploaded_file(
+ file_id=uploaded.file_id, organization_id=organization.organization_id
+ )
+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail) from rejection
+ return WorkflowCopilotDemonstrationControlResult(seq=seq, file_id=uploaded.file_id)
+
+
@base_router.get("/workflow/copilot/demonstration/active", include_in_schema=False)
async def workflow_copilot_demonstration_active(
workflow_permanent_id: str,
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index dcf995b55..a8afebab0 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -596,7 +596,7 @@ class WorkflowCopilotCredentialResponseRequest(BaseModel):
class WorkflowCopilotDemonstrationControlRequest(BaseModel):
demonstration_id: str = Field(max_length=64)
- action: Literal["navigate", "fill_credential", "stop", "cancel"]
+ action: Literal["navigate", "fill_credential", "upload_file", "stop", "cancel"]
url: str | None = Field(default=None, max_length=2048)
selector: str | None = Field(default=None, max_length=2000)
credential_id: str | None = Field(default=None, max_length=128)
@@ -608,6 +608,7 @@ class WorkflowCopilotDemonstrationControlRequest(BaseModel):
class WorkflowCopilotDemonstrationControlResult(BaseModel):
seq: int
action_count: int | None = None
+ file_id: str | None = None
class WorkflowCopilotDemonstrationRecordRequest(BaseModel):
diff --git a/skyvern/forge/sdk/services/route_authorization.py b/skyvern/forge/sdk/services/route_authorization.py
index 601d074e5..b3a7d8c5c 100644
--- a/skyvern/forge/sdk/services/route_authorization.py
+++ b/skyvern/forge/sdk/services/route_authorization.py
@@ -268,6 +268,7 @@ ROUTE_ACTIONS: Mapping[str, str] = {
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_active": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_control": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_record": "workflow_copilot.manage",
+ "skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_upload": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_disable_auto_accept": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_message_feedback": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_question_response": "workflow_copilot.manage",
diff --git a/tests/unit/browser_extension/test_broker_server.py b/tests/unit/browser_extension/test_broker_server.py
index 6f4d9d2e4..08c0555bc 100644
--- a/tests/unit/browser_extension/test_broker_server.py
+++ b/tests/unit/browser_extension/test_broker_server.py
@@ -1576,10 +1576,7 @@ async def test_client_enrollment_proceeds_when_extension_disconnects_during_rese
@pytest.mark.asyncio
-async def test_late_extension_reconnect_is_reset_before_successor_sees_snapshot(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(broker_server_module, "EXTENSION_RESET_TIMEOUT_SECONDS", 0.01)
+async def test_late_extension_reconnect_is_reset_before_successor_sees_snapshot() -> None:
server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())
relay = ControlledResetRelay(
"extension-secret",
@@ -1847,7 +1844,7 @@ async def test_reset_send_deadline_surfaces_structured_timeout(
@pytest.mark.asyncio
async def test_owner_release_allows_reenrollment_while_timed_out_request_awaits_terminal() -> None:
- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())
+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)
relay = ExtensionRelayServer(
"extension-secret",
19777,
@@ -2478,7 +2475,7 @@ async def test_retained_timed_out_extension_requests_remain_bounded(
limit_name: str,
) -> None:
monkeypatch.setattr(broker_server_module, limit_name, 1)
- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())
+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)
relay = ExtensionRelayServer(
"extension-secret",
19777,
@@ -2561,7 +2558,7 @@ async def test_retained_timed_out_extension_requests_remain_bounded(
async def test_retained_timed_out_request_holds_inbound_bytes_until_terminal(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())
+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)
relay = ExtensionRelayServer(
"extension-secret",
19777,
@@ -2624,7 +2621,7 @@ async def test_retained_timed_out_requests_enforce_per_tab_limit_until_terminal(
) -> None:
monkeypatch.setattr(broker_server_module, "MAX_REQUESTS_PER_TAB", 1)
monkeypatch.setattr(broker_server_module, "TAB_REQUEST_QUEUE_WAIT_SECONDS", 0.01)
- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())
+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)
relay = ExtensionRelayServer(
"extension-secret",
19777,
@@ -4988,6 +4985,12 @@ async def _ignore_event(_event: str, _params: dict) -> None:
return None
+def _frozen_broker_clock() -> float:
+ # Frozen so a runner stall before forwarding cannot expire a short request; the relay still
+ # times it out on real time.
+ return 0.0
+
+
async def _eventually(predicate: Callable[[], bool]) -> None:
for _ in range(100):
if predicate():
diff --git a/tests/unit/test_copilot_demonstration.py b/tests/unit/test_copilot_demonstration.py
index e9d801f82..5e0502012 100644
--- a/tests/unit/test_copilot_demonstration.py
+++ b/tests/unit/test_copilot_demonstration.py
@@ -129,6 +129,14 @@ class _FakeTap:
async def frame_path(self, event: TapEvent) -> list[dict[str, str]] | None:
return [] if event.is_main_frame else [{"name": event.frame_name}] if event.frame_name else None
+ async def call_on_file_input(self, event: TapEvent, declaration: str, *args: Any) -> Any:
+ if "selectorFor" in declaration:
+ return {"selector": "#resume", "label": "Resume"}
+ type(self).attached_files.append(args[:2])
+ return 1
+
+ attached_files: ClassVar[list[tuple[Any, ...]]] = []
+
def begin_control_navigation(self) -> None:
return None
@@ -273,6 +281,23 @@ async def test_replay_contract_never_touches_a_browser_and_consumes_once(monkeyp
"credential_field": "password",
},
},
+ {
+ "kind": "upload",
+ "selector": "#resume",
+ "file_id": "file-1",
+ "filename": "resume.pdf",
+ "size_bytes": 4,
+ "field_label": "Resume",
+ "source_url": "https://example.test/done",
+ "frame": {"is_main": True, "name": ""},
+ },
+ {
+ "kind": "upload",
+ "selector": "#cover",
+ "field_label": "Cover letter",
+ "source_url": "https://example.test/done",
+ "frame": {"is_main": True, "name": ""},
+ },
]
cache.store[demonstration_record_key("org-1", "demo-1")] = encode_demonstration_record(
DemonstrationRecord(
@@ -327,7 +352,9 @@ async def test_replay_contract_never_touches_a_browser_and_consumes_once(monkeyp
"type_text",
"type_text",
"fill_credential_field",
+ "upload_attached_file",
]
+ assert json.loads(calls[-1]["arguments"]) == {"file_id": "file-1", "selector": "#resume"}
serialized = json.dumps(history)
assert "Acme Corp" in serialized
assert token not in serialized
@@ -335,6 +362,7 @@ async def test_replay_contract_never_touches_a_browser_and_consumes_once(monkeyp
assert "<value not recorded>" in serialized
assert any("file chooser" in str(item.get("content")) for item in history if item.get("role") == "user")
assert any("disconnected" in str(item.get("content")) for item in history if item.get("role") == "user")
+ assert any("#cover" in str(item.get("content")) and "no file" in str(item.get("content")) for item in history)
gap_index = next(index for index, item in enumerate(history) if "file chooser" in str(item.get("content")))
first_type_index = next(index for index, item in enumerate(history) if item.get("name") == "type_text")
assert gap_index < first_type_index
@@ -481,9 +509,22 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pyt
[
{"seq": 1, "action": "navigate", "url": "http://127.0.0.1/private"},
{"seq": 2, "action": "navigate", "url": "https://example.test/next"},
+ {"seq": 3, "action": "upload_file", "file_id": "file-1"},
]
)
await cache.set(demonstration_browser_key("org-1", "pbs-1"), demonstration_id, ex=timedelta(minutes=1))
+ monkeypatch.setattr(
+ app,
+ "DATABASE",
+ SimpleNamespace(
+ uploaded_files=SimpleNamespace(
+ get_uploaded_file=AsyncMock(
+ return_value=SimpleNamespace(storage_uri="s3://bucket/org-1/resume.pdf", filename="resume.pdf")
+ )
+ )
+ ),
+ )
+ monkeypatch.setattr(app, "STORAGE", SimpleNamespace(download_managed_file=AsyncMock(return_value=b"%PDF")))
page = SimpleNamespace(url="https://example.test/start", goto=AsyncMock(), is_closed=lambda: False)
click = TapEvent(
page=page,
@@ -535,14 +576,31 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pyt
frame_name="",
payload={"kind": "file_click", "ts": 3, "url": page.url, "selector": "#upload"},
)
+ file_chooser = TapEvent(
+ page=page,
+ session_key="tap-1",
+ frame_id="main",
+ is_main_frame=True,
+ frame_name="",
+ payload={"kind": "file_chooser", "ts": 3.1, "backend_node_id": 7},
+ )
+ unanswered_chooser = TapEvent(
+ page=page,
+ session_key="tap-1",
+ frame_id="main",
+ is_main_frame=True,
+ frame_name="",
+ payload={"kind": "file_chooser", "ts": 3.2, "backend_node_id": 8},
+ )
browser_context = SimpleNamespace(pages=[page])
browser_state = SimpleNamespace(
browser_context=browser_context,
get_working_page=AsyncMock(return_value=page),
)
- _FakeTap.events = [click, password_focus, password, file_click]
+ _FakeTap.events = [click, password_focus, password, file_click, file_chooser, unanswered_chooser]
_FakeTap.flush_events = []
+ _FakeTap.attached_files = []
monkeypatch.setattr(recorder_module, "PageTap", _FakeTap)
monkeypatch.setattr(recorder_module, "resolve_persistent_browser_state", AsyncMock(return_value=browser_state))
@@ -574,18 +632,27 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pyt
assert [first["kind"], second["kind"], third["kind"], fourth["kind"]] == [
"click",
"secret",
- "gap",
"navigate",
+ "upload",
]
- assert third["reason"] == "file_chooser"
+ assert {key: fourth[key] for key in ("selector", "file_id", "filename", "size_bytes")} == {
+ "selector": "#resume",
+ "file_id": "file-1",
+ "filename": "resume.pdf",
+ "size_bytes": 4,
+ }
+ assert _FakeTap.attached_files == [("resume.pdf", "application/pdf")]
+ assert {"type": "demonstration_file_requested", "label": "Resume"} in stream.frames
+ unanswered = json.loads(cache.store[demonstration_event_key("org-1", demonstration_id, 5)])
+ assert unanswered["kind"] == "upload" and "file_id" not in unanswered
assert page.goto.await_args.args == ("https://example.test/next",)
assert any(frame.get("type") == "demonstration_notice" and frame.get("level") == "error" for frame in stream.frames)
action_frames = [frame for frame in stream.frames if frame.get("type") == "demonstration_action"]
assert [(frame["seq"], frame["tool_name"]) for frame in action_frames] == [
(1, "click"),
(2, "type_text"),
- (3, "gap"),
- (4, "navigate_browser"),
+ (3, "navigate_browser"),
+ (4, "upload_attached_file"),
]
assert action_frames[1]["label"] == second["label"] == "Fill password (value not recorded)"
sensitive_frame = next(frame for frame in stream.frames if frame.get("type") == "demonstration_sensitive_field")
@@ -596,13 +663,13 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pyt
"url": "https://example.test/start",
}
stopped = decode_demonstration_record(cache.store[demonstration_record_key("org-1", demonstration_id)])
- assert stopped is not None and stopped.status == "stopped" and stopped.event_count == 4
+ assert stopped is not None and stopped.status == "stopped" and stopped.event_count == 5
assert stopped.stop_reason == "disconnected"
assert stream.frames[-1] == {
"type": "demonstration_stopped",
"demonstration_id": "demo-1",
"reason": "disconnected",
- "action_count": 4,
+ "action_count": 5,
}
assert cache.expirations[demonstration_browser_key("org-1", "pbs-1")] == recorder_module.DEMONSTRATION_TTL
@@ -941,6 +1008,37 @@ async def test_discard_after_resume_commits_cancelled_even_when_the_stream_drops
assert active is None
+async def test_client_upload_control_cannot_name_a_file() -> None:
+ cache = _SharedCache()
+ cache.store[demonstration_record_key("org-1", "demo-1")] = encode_demonstration_record(
+ DemonstrationRecord(
+ workflow_permanent_id="wfp-1",
+ browser_session_id="pbs-1",
+ status="recording",
+ event_count=0,
+ owner="owner-1",
+ owner_seen_at=datetime.now(UTC),
+ final_url=None,
+ )
+ )
+ cache.store[demonstration_controls_key("org-1", "demo-1")] = "[]"
+ crafted = WorkflowCopilotDemonstrationControlRequest.model_validate(
+ {"demonstration_id": "demo-1", "action": "upload_file", "file_id": "file-from-elsewhere"}
+ )
+
+ await append_demonstration_control(cache, organization_id="org-1", request=crafted) # type: ignore[arg-type]
+ await append_demonstration_control(
+ cache, # type: ignore[arg-type]
+ organization_id="org-1",
+ request=WorkflowCopilotDemonstrationControlRequest(demonstration_id="demo-1", action="upload_file"),
+ file_id="file-stored-for-demo",
+ )
+
+ controls = json.loads(cache.store[demonstration_controls_key("org-1", "demo-1")])
+ assert [control.get("file_id") for control in controls] == [None, "file-stored-for-demo"]
+
+
@pytest.mark.asyncio
async def test_recorder_runs_controls_queued_before_stop(monkeypatch: pytest.MonkeyPatch) -> None:
cache = _SharedCache()