Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Keep runner stalls from expiring broker test requests before they are forwarded (SKY-18688) (#8974)
Shuchang Zheng3c758c9a0c42
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 26c8757a9..fd004a21a 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -293,6 +293,10 @@ import { captureChatScope, type ChatScopeRefs } from "./chatReadScope";import { WORKFLOW_COPILOT_DEMONSTRATION_FLAG } from "@/util/featureFlags";import { safeHttpUrl } from "@/util/httpUrl";import { useDemonstrationRecorder } from "./useDemonstrationRecorder";+import {+ DemonstrationFileMarker,+ DemonstrationFileTray,+} from "./cards/DemonstrationFileTray";// Cap on retained per-turn snap-back snapshots. A typical session has a// handful of turns; this ceiling guards a runaway long-running chat.@@ -6978,7 +6982,8 @@ export function WorkflowCopilotChat({const demonstrationRecorder = useDemonstrationRecorder({workflowPermanentId,browserSessionId: demonstrationEnabled ? liveBrowserSessionId : null,- onDone: async ({ demonstrationId, actions }) => {+ deleteFile: removeUnsentUpload,+ onDone: async ({ demonstrationId, actions, files }) => {const messageId =demonstrationMessageIdRef.current ??`demonstration-${crypto.randomUUID()}`;@@ -7010,6 +7015,7 @@ export function WorkflowCopilotChat({sent =(await handleSendRef.current?.(DEMONSTRATION_DONE_MESSAGE, {optimisticMessageId: messageId,+ attachments: files.slice(0, ATTACHMENT_COUNT_LIMIT),})) ?? false;} finally {if (!sent) {@@ -7777,8 +7783,12 @@ export function WorkflowCopilotChat({: "Type this sign-in yourself to keep its value private.",}: null;+ const demonstrationFileRequest =+ demonstrationRecorder.phase === "recording"+ ? demonstrationRecorder.fileRequest+ : null;const attentionQueue: {- kind: CopilotAttention | "demonstration_credential";+ kind: CopilotAttention | "demonstration_credential" | "demonstration_file";upNext?: string;}[] = [];if (trayQuestion) {@@ -7799,6 +7809,12 @@ export function WorkflowCopilotChat({upNext: "Fill this sign-in",});}+ if (demonstrationFileRequest) {+ attentionQueue.push({+ kind: "demonstration_file",+ upNext: "Choose a file",+ });+ }if (isDemonstrating) {attentionQueue.push({kind: "demonstration",@@ -7809,8 +7825,14 @@ export function WorkflowCopilotChat({const headerAttention =activeAttention === "demonstration_credential"? "credential"- : activeAttention;- const attentionUpNext = attentionQueue[1]?.upNext ?? null;+ : activeAttention === "demonstration_file"+ ? "demonstration"+ : activeAttention;+ // The demonstration is already on screen as the composer strip, so it is never worth an "Up next" row.+ const attentionUpNext =+ attentionQueue[1]?.kind === "demonstration"+ ? null+ : (attentionQueue[1]?.upNext ?? null);// Only the open tray replaces its transcript card; queued items stay actionable inline.const dockedPauseFrame =activeAttention === "credential" ? trayPauseFrame : null;@@ -7823,7 +7845,10 @@ export function WorkflowCopilotChat({? demonstrationCredentialFrame: null;const hasVisibleAttentionTray =- activeAttention !== null && activeAttention !== "demonstration";+ activeAttention !== null &&+ activeAttention !== "demonstration" &&+ activeAttention !== "demonstration_credential" &&+ activeAttention !== "demonstration_file";const [collapsedAttentionKey, setCollapsedAttentionKey] = useState<string | null>(null);@@ -11746,6 +11771,18 @@ export function WorkflowCopilotChat({}/></div>+ {message.id === demonstrationMessageIdRef.current &&+ activeAttention === "demonstration_file" ? (+ <div className={TURN_ROW_INSET}>+ <DemonstrationFileMarker />+ </div>+ ) : null}+ {message.id === demonstrationMessageIdRef.current &&+ activeAttention === "demonstration_credential" ? (+ <div className={TURN_ROW_INSET}>+ <CredentialAskMarker />+ </div>+ ) : null}</Fragment>) : (renderedMessage@@ -12225,28 +12262,6 @@ export function WorkflowCopilotChat({tray={attentionTray(`account:${dockedAccountChoice.turnId}`)}/>) : null}- {dockedDemonstrationCredential && demonstrationSensitiveField ? (- <CredentialCard- key={`${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}:${demonstrationSensitiveField.url}`}- frame={dockedDemonstrationCredential}- mode="inline-pause"- offerCredentials={demonstrationRecorder.canFillCredentials}- reloadKey={credentialsReloadKey}- onConnect={(credentialId) => {- if (credentialId) {- demonstrationRecorder.fillCredential(credentialId);- return;- }- pendingCredentialConnect.current = null;- pendingDemonstrationCredentialConnect.current = true;- setCredentialModalOpen(true);- }}- onSkip={demonstrationRecorder.dismissSensitiveField}- tray={attentionTray(- `demonstration-credential:${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}`,- )}- />- ) : null}{showQueuedStrip && queuedPrompt ? (<QueuedMessageStriptext={queuedPrompt.content}@@ -12349,6 +12364,45 @@ export function WorkflowCopilotChat({</div></div>) : null}+ {dockedDemonstrationCredential && demonstrationSensitiveField ? (+ <CredentialCard+ key={`${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}:${demonstrationSensitiveField.url}`}+ frame={dockedDemonstrationCredential}+ mode="inline-pause"+ offerCredentials={demonstrationRecorder.canFillCredentials}+ reloadKey={credentialsReloadKey}+ onConnect={(credentialId) => {+ if (credentialId) {+ demonstrationRecorder.fillCredential(credentialId);+ return;+ }+ pendingCredentialConnect.current = null;+ pendingDemonstrationCredentialConnect.current = true;+ setCredentialModalOpen(true);+ }}+ onSkip={demonstrationRecorder.dismissSensitiveField}+ tray={{+ ...attentionTray(+ `demonstration-credential:${demonstrationSensitiveField.field}:${demonstrationSensitiveField.selector}`,+ ),+ embedded: true,+ }}+ />+ ) : null}+ {activeAttention === "demonstration_file" &&+ demonstrationFileRequest ? (+ <DemonstrationFileTray+ label={demonstrationFileRequest.label}+ busy={demonstrationRecorder.fileBusy}+ onChoose={(file) => void demonstrationRecorder.chooseFile(file)}+ tray={{+ ...attentionTray(+ `demonstration-file:${demonstrationFileRequest.label}`,+ ),+ embedded: true,+ }}+ />+ ) : null}{demonstrationRecorder.phase === "stopped" &&demonstrationRecorder.browserRestarted ? (<div className="border-b border-border px-3 py-1.5">diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsxindex 79c7f3d06..fa55fab92 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/AttentionTray.tsx@@ -6,6 +6,8 @@ import { cn } from "@/util/utils";const TRAY_FRAME ="rounded-t-lg border border-b-0 border-amber-500/50 bg-amber-500/[0.06]";+// Inside the composer, under the demonstration strip, the composer's own border frames it.+const EMBEDDED_TRAY_FRAME = "border-b border-amber-500/30 bg-amber-500/[0.06]";const TRAY_TITLE = "font-semibold text-amber-700 dark:text-yellow-400";// How the chat asks a request card to render itself as the docked tray.@@ -13,6 +15,7 @@ export interface AttentionTrayPresentation {collapsed: boolean;onCollapsedChange: (collapsed: boolean) => void;upNext?: string | null;+ embedded?: boolean;}type AttentionTrayProps = Omit<HTMLAttributes<HTMLDivElement>, "title"> & {@@ -28,6 +31,7 @@ type AttentionTrayProps = Omit<HTMLAttributes<HTMLDivElement>, "title"> & {minimizeLabel: string;// Another request waiting behind this one; the chat shows one at a time.upNext?: string | null;+ embedded?: boolean;children: ReactNode;};@@ -43,18 +47,15 @@ export function AttentionTray({onCollapsedChange,minimizeLabel,upNext,+ embedded = false,children,className,...groupProps}: AttentionTrayProps) {+ const frame = embedded ? EMBEDDED_TRAY_FRAME : TRAY_FRAME;if (collapsed) {return (- <div- className={cn(- "flex items-center gap-2 px-3 py-1.5 text-xs",- TRAY_FRAME,- )}- >+ <div className={cn("flex items-center gap-2 px-3 py-1.5 text-xs", frame)}><spanaria-hiddenclassName="size-2 shrink-0 rounded-full bg-amber-500"@@ -89,7 +90,7 @@ export function AttentionTray({{...groupProps}className={cn("flex max-h-[50vh] min-w-0 flex-col overflow-hidden",- TRAY_FRAME,+ frame,className,)}>diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsxindex de31471ed..4678b5455 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx@@ -648,6 +648,7 @@ function AskChrome({onCollapsedChange={tray.onCollapsedChange}minimizeLabel="Minimize sign-in request"upNext={tray.upNext}+ embedded={tray.embedded}><div className="flex min-h-0 flex-col gap-1 overflow-y-auto px-3 pb-2.5 pt-1">{lines.map((line, index) => (diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/DemonstrationFileTray.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/DemonstrationFileTray.tsxnew file mode 100644index 000000000..722ada2a9--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/DemonstrationFileTray.tsx@@ -0,0 +1,88 @@+import { FileIcon, UploadIcon } from "@radix-ui/react-icons";+import { useRef } from "react";++import { Button } from "@/components/ui/button";++import {+ AttentionMarker,+ AttentionTray,+ type AttentionTrayPresentation,+} from "./AttentionTray";++// The page opened a file chooser during a demonstration; the remote browser's own dialog is unusable.+export function DemonstrationFileTray({+ label,+ busy,+ onChoose,+ tray,+}: {+ label: string;+ busy: boolean;+ onChoose: (file: File | null) => void;+ tray: AttentionTrayPresentation;+}) {+ const inputRef = useRef<HTMLInputElement>(null);+ const title = `Choose a file for “${label}”`;+ return (+ <AttentionTray+ aria-label="File request"+ title={title}+ wrapTitle+ collapsedTitle="The page wants a file"+ collapsed={tray.collapsed}+ onCollapsedChange={tray.onCollapsedChange}+ minimizeLabel="Minimize file request"+ upNext={tray.upNext}+ embedded={tray.embedded}+ >+ <div className="px-3 pb-2.5 pt-1 text-[11px] leading-relaxed text-muted-foreground">+ The page opened a file picker. Choose the file to attach here; Copilot+ saves it as the default for the workflow's file input.+ </div>+ <div className="flex shrink-0 flex-wrap items-center gap-2 border-t border-border px-3 py-1.5">+ <Button+ type="button"+ size="sm"+ disabled={busy}+ onClick={() => inputRef.current?.click()}+ >+ <UploadIcon className="mr-1.5 size-3.5" />+ Choose file+ </Button>+ <input+ ref={inputRef}+ type="file"+ className="sr-only"+ tabIndex={-1}+ aria-hidden+ onChange={(event) => {+ const file = event.target.files?.[0];+ event.target.value = "";+ if (file) onChoose(file);+ }}+ />+ <div className="ml-auto">+ <Button+ type="button"+ size="sm"+ variant="outline"+ disabled={busy}+ onClick={() => onChoose(null)}+ >+ Skip+ </Button>+ </div>+ </div>+ </AttentionTray>+ );+}++// Where the file request was raised in the transcript; the tray holds the actions.+export function DemonstrationFileMarker() {+ return (+ <AttentionMarker+ icon={<FileIcon aria-hidden className="size-3.5 shrink-0" />}+ title="The page wants a file"+ />+ );+}diff --git a/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts b/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.tsindex a55bbd480..a3990c227 100644--- a/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/useDemonstrationRecorder.ts@@ -7,10 +7,12 @@ import { toast } from "@/components/ui/use-toast";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import type {+ CopilotAttachedFile,WorkflowCopilotActiveDemonstration,WorkflowCopilotDemonstrationActionUpdate,WorkflowCopilotDemonstrationControlRequest,WorkflowCopilotDemonstrationControlResult,+ WorkflowCopilotDemonstrationFileRequestedUpdate,WorkflowCopilotDemonstrationInsertion,WorkflowCopilotDemonstrationSensitiveFieldUpdate,WorkflowCopilotDemonstrationUpdate,@@ -26,6 +28,7 @@ type DemonstrationAction = Pick<type DemonstrationDone = {demonstrationId: string;actions: DemonstrationAction[];+ files: CopilotAttachedFile[];};type UseDemonstrationRecorderOptions = {@@ -33,6 +36,7 @@ type UseDemonstrationRecorderOptions = {browserSessionId: string | null | undefined;onDone: (demonstration: DemonstrationDone) => boolean | Promise<boolean>;onStartFailed: () => void;+ deleteFile: (fileId: string) => Promise<unknown>;};export function useDemonstrationRecorder({@@ -40,6 +44,7 @@ export function useDemonstrationRecorder({browserSessionId,onDone,onStartFailed,+ deleteFile,}: UseDemonstrationRecorderOptions) {const credentialGetter = useCredentialGetter();const [demonstrationId, setDemonstrationId] = useState<string | null>(null);@@ -49,6 +54,14 @@ export function useDemonstrationRecorder({const [actionCount, setActionCount] = useState(0);const [sensitiveField, setSensitiveField] =useState<WorkflowCopilotDemonstrationSensitiveFieldUpdate | null>(null);+ const [fileRequest, setFileRequest] =+ useState<WorkflowCopilotDemonstrationFileRequestedUpdate | null>(null);+ // Every file uploaded for this demonstration, so a discard can delete them.+ const uploadedFileIdsRef = useRef<string[]>([]);+ // The control only queues the attach; the tray stays locked until the recorder reports the outcome.+ const [awaitingFileOutcome, setAwaitingFileOutcome] = useState(false);+ // The files the recorder reports attached; they ride on the Done message.+ const filesRef = useRef<CopilotAttachedFile[]>([]);const [busy, setBusy] = useState(false);const [canFillCredentials, setCanFillCredentials] = useState(false);const [hasSkippedSensitiveField, setHasSkippedSensitiveField] =@@ -64,6 +77,8 @@ export function useDemonstrationRecorder({onDoneRef.current = onDone;const onStartFailedRef = useRef(onStartFailed);onStartFailedRef.current = onStartFailed;+ const deleteFileRef = useRef(deleteFile);+ deleteFileRef.current = deleteFile;const sessionRef = useRef({ workflowPermanentId, browserSessionId });const updateBusy = useCallback((next: boolean) => {@@ -90,6 +105,10 @@ export function useDemonstrationRecorder({setActions([]);setActionCount(0);setSensitiveField(null);+ setFileRequest(null);+ setAwaitingFileOutcome(false);+ uploadedFileIdsRef.current = [];+ filesRef.current = [];setCanFillCredentials(false);setHasSkippedSensitiveField(false);setLimitReached(false);@@ -116,6 +135,23 @@ export function useDemonstrationRecorder({setActions(actionsRef.current);setActionCount((current) => Math.max(current, frame.seq));setSensitiveField(null);+ if (frame.tool_name === "upload_attached_file") {+ setAwaitingFileOutcome(false);+ if (frame.file_id) {+ filesRef.current = [+ ...filesRef.current,+ {+ file_id: frame.file_id,+ filename: frame.filename ?? "",+ available: true,+ },+ ];+ }+ setFileRequest(null);+ }+ return false;+ case "demonstration_file_requested":+ setFileRequest(frame);return false;case "demonstration_sensitive_field":if (frame.field === "other") {@@ -125,6 +161,7 @@ export function useDemonstrationRecorder({return false;case "demonstration_notice":if (frame.level === "error") {+ setAwaitingFileOutcome(false);toast({title: "Could not update the demonstration",description: frame.message,@@ -138,6 +175,8 @@ export function useDemonstrationRecorder({}setActionCount(frame.action_count);setSensitiveField(null);+ setFileRequest(null);+ setAwaitingFileOutcome(false);setHasSkippedSensitiveField(false);if (frame.reason === "time_limit") setLimitReached(true);if (frame.reason === "event_limit") {@@ -306,6 +345,56 @@ export function useDemonstrationRecorder({[runControl, sensitiveField],);+ // The server stores the file for this demonstration and queues the attach itself, so the+ // recorder never attaches a file the client merely names.+ const uploadToDemonstration = useCallback(+ async (file: File): Promise<string | null> => {+ const id = demonstrationIdRef.current;+ if (!id) throw new Error("No demonstration is active.");+ const client = await getClient(credentialGetter, "sans-api-v1");+ const formData = new FormData();+ formData.append("demonstration_id", id);+ formData.append("file", file);+ const response =+ await client.post<WorkflowCopilotDemonstrationControlResult>(+ "/workflow/copilot/demonstration/upload",+ formData,+ { headers: { "Content-Type": "multipart/form-data" } },+ );+ return response.data.file_id ?? null;+ },+ [credentialGetter],+ );++ // null skips the requested file; the recorder still logs that the page asked for one.+ const chooseFile = useCallback(+ async (file: File | null) => {+ if (busyRef.current || awaitingFileOutcome) return;+ updateBusy(true);+ // Set before the request: the recorder's outcome frame can arrive before the response does.+ setAwaitingFileOutcome(true);+ try {+ if (!file) {+ await sendControl({ action: "upload_file" });+ return;+ }+ const fileId = await uploadToDemonstration(file);+ if (fileId) uploadedFileIdsRef.current.push(fileId);+ } catch (error) {+ setAwaitingFileOutcome(false);+ toast({+ title: "Could not upload the file",+ description:+ error instanceof Error ? error.message : "Please try again.",+ variant: "destructive",+ });+ } finally {+ updateBusy(false);+ }+ },+ [awaitingFileOutcome, sendControl, updateBusy, uploadToDemonstration],+ );+const dismissSensitiveField = useCallback(() => {setSensitiveField(null);}, []);@@ -330,6 +419,7 @@ export function useDemonstrationRecorder({}stream?.abort();setSensitiveField(null);+ setFileRequest(null);updatePhase("stopped");setHasSkippedSensitiveField(false);if (stopped?.action_count === 0) {@@ -345,6 +435,7 @@ export function useDemonstrationRecorder({const sent = await onDoneRef.current({demonstrationId: id,actions: recordedActions,+ files: filesRef.current,});if (sent) {clear();@@ -364,6 +455,13 @@ export function useDemonstrationRecorder({}}, [clear, sendControl, updateBusy, updatePhase]);+ // Discarded uploads were never sent with a message, so nothing else would delete them.+ const discardUploads = useCallback(() => {+ for (const fileId of uploadedFileIdsRef.current) {+ void deleteFileRef.current(fileId);+ }+ }, []);+const cancel = useCallback(async (): Promise<boolean> => {const id = demonstrationIdRef.current;if (!id || busyRef.current) return false;@@ -373,6 +471,7 @@ export function useDemonstrationRecorder({} catch (error) {const status = isAxiosError(error) ? error.response?.status : undefined;if (status === 404 || status === 409) {+ discardUploads();clear();return true;}@@ -385,9 +484,10 @@ export function useDemonstrationRecorder({updateBusy(false);return false;}+ discardUploads();clear();return true;- }, [clear, sendControl, updateBusy]);+ }, [clear, discardUploads, sendControl, updateBusy]);useEffect(() => {const previous = sessionRef.current;@@ -506,6 +606,8 @@ export function useDemonstrationRecorder({actions,actionCount,sensitiveField,+ fileRequest,+ fileBusy: busy || awaitingFileOutcome,canFillCredentials,hasSkippedSensitiveField,limitReached,@@ -515,6 +617,7 @@ export function useDemonstrationRecorder({resume,navigate,fillCredential,+ chooseFile,dismissSensitiveField,done,cancel,diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex ceb39cd6f..a3483f721 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -583,6 +583,9 @@ export interface WorkflowCopilotDemonstrationActionUpdate {| "navigate_browser"| "upload_attached_file"| "fill_credential_field";+ // Present only on an upload the recorder attached; a skipped file request has none.+ file_id?: string;+ filename?: string;}export interface WorkflowCopilotDemonstrationSensitiveFieldUpdate {@@ -592,6 +595,11 @@ export interface WorkflowCopilotDemonstrationSensitiveFieldUpdate {url: string;}+export interface WorkflowCopilotDemonstrationFileRequestedUpdate {+ type: "demonstration_file_requested";+ label: string;+}+export interface WorkflowCopilotDemonstrationNoticeUpdate {type: "demonstration_notice";message: string;@@ -614,7 +622,7 @@ export interface WorkflowCopilotDemonstrationStoppedUpdate {export interface WorkflowCopilotDemonstrationControlRequest {demonstration_id: string;- action: "navigate" | "fill_credential" | "stop" | "cancel";+ action: "navigate" | "fill_credential" | "upload_file" | "stop" | "cancel";url?: string | null;selector?: string | null;credential_id?: string | null;@@ -625,12 +633,14 @@ export interface WorkflowCopilotDemonstrationControlRequest {export interface WorkflowCopilotDemonstrationControlResult {seq: number;action_count?: number | null;+ file_id?: string | null;}export type WorkflowCopilotDemonstrationUpdate =| WorkflowCopilotDemonstrationStartedUpdate| WorkflowCopilotDemonstrationActionUpdate| WorkflowCopilotDemonstrationSensitiveFieldUpdate+ | WorkflowCopilotDemonstrationFileRequestedUpdate| WorkflowCopilotDemonstrationNoticeUpdate| WorkflowCopilotDemonstrationStoppedUpdate;diff --git a/skyvern/forge/sdk/copilot/demonstration_recorder.py b/skyvern/forge/sdk/copilot/demonstration_recorder.pyindex 44a878e93..fc6fc4070 100644--- a/skyvern/forge/sdk/copilot/demonstration_recorder.py+++ b/skyvern/forge/sdk/copilot/demonstration_recorder.py@@ -1,8 +1,10 @@from __future__ import annotationsimport asyncio+import base64import contextlibimport json+import mimetypesimport secretsimport timefrom collections.abc import Awaitable, Coroutine@@ -19,6 +21,7 @@ from playwright.async_api import Pagefrom skyvern.cli.mcp_tools._localhost import is_localhost_urlfrom skyvern.config import settingsfrom skyvern.exceptions import BlockedHost, SkyvernHTTPException+from skyvern.forge import appfrom skyvern.forge.sdk.cache.base import BaseCachefrom skyvern.forge.sdk.copilot.config import ALL_BLOCK_FAMILIES, AuthoringCapabilityfrom skyvern.forge.sdk.copilot.context import CopilotContext@@ -38,6 +41,7 @@ from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structurefrom skyvern.forge.sdk.copilot.tools.credential_fill import fill_credential_fieldfrom skyvern.forge.sdk.core.event_source_stream import EventSourceStreamfrom skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotDemonstrationControlRequest+from skyvern.services import uploaded_file_servicefrom skyvern.utils.url_validators import validate_fetch_urlLOG = structlog.get_logger()@@ -62,6 +66,21 @@ DEMONSTRATION_OWNER_LIVE_SECONDS = 20DEMONSTRATION_MAX_SECONDS = 30 * 60DEMONSTRATION_REPLAY_LEASE = timedelta(minutes=5)+_DESCRIBE_FILE_INPUT = (+ "function() { const tap = globalThis.__skyvernDemoTap; "+ "return tap ? {selector: tap.selectorFor(this), label: tap.labelFor(this) || ''} : null; }"+)+# Built in the page rather than DOM.setFileInputFiles, which takes a path on the browser's host.+_ATTACH_FILE = """function(name, type, data) {+ const bytes = Uint8Array.from(atob(data), (char) => char.charCodeAt(0));+ const transfer = new DataTransfer();+ transfer.items.add(new File([bytes], name, {type}));+ this.files = transfer.files;+ this.dispatchEvent(new Event("input", {bubbles: true, composed: true}));+ this.dispatchEvent(new Event("change", {bubbles: true}));+ return this.files.length;+}"""+DemonstrationStatus = Literal["recording", "stopping", "stopped", "replaying", "consumed", "cancelled"]DemonstrationStopReason = Literal["stopped", "cancelled", "disconnected", "browser_gone", "time_limit", "event_limit", "consumed"@@ -414,6 +433,7 @@ async def append_demonstration_control(*,organization_id: str,request: WorkflowCopilotDemonstrationControlRequest,+ file_id: str | None = None,) -> int:record_key = demonstration_record_key(organization_id, request.demonstration_id)controls_key = demonstration_controls_key(organization_id, request.demonstration_id)@@ -423,6 +443,9 @@ async def append_demonstration_control(raise DemonstrationRejection(HTTPStatus.NOT_FOUND, "Unknown demonstration")if record.status not in {"recording", "stopping", "stopped"}:raise DemonstrationRejection(HTTPStatus.CONFLICT, "Demonstration has ended")+ # A stopping recorder may already have drained its controls, so an upload queued now would never run.+ if file_id is not None and record.status != "recording":+ raise DemonstrationRejection(HTTPStatus.CONFLICT, "Demonstration is not recording")if request.action == "stop":if record.status == "recording":record.status = "stopping"@@ -448,6 +471,8 @@ async def append_demonstration_control(raise DemonstrationRejection(HTTPStatus.CONFLICT, "Too many browser controls for this demonstration")seq = len(controls) + 1entry = {"seq": seq, **request.model_dump(exclude_none=True, exclude={"demonstration_id"})}+ if file_id is not None:+ entry["file_id"] = file_idcontrols.append(entry)await cache.set(controls_key, json.dumps(controls, separators=(",", ":")), ex=DEMONSTRATION_TTL)return seq@@ -532,6 +557,26 @@ async def demonstration_action_labels(return labels+async def demonstration_file_ids(+ cache: BaseCache, organization_id: str, demonstration_id: str, event_count: int+) -> list[str]:+ events = await asyncio.gather(+ *(+ cache.get(demonstration_event_key(organization_id, demonstration_id, seq))+ for seq in range(1, event_count + 1)+ )+ )+ file_ids: list[str] = []+ for raw in events:+ try:+ event = json.loads(raw)+ except (TypeError, ValueError):+ continue+ if isinstance(event, dict) and event.get("kind") == "upload" and isinstance(event.get("file_id"), str):+ file_ids.append(event["file_id"])+ return file_ids++class DemonstrationRecorder:def __init__(self,@@ -559,6 +604,9 @@ class DemonstrationRecorder:self.sensitive_count = 0self.event_limit_reached = Falseself.credential_filled_selectors: set[str] = set()+ # The page keeps running while a request is open, so later choosers wait their turn.+ self.file_requests: list[dict[str, Any]] = []+ self.intercepting_file_chooser = Falseself.started_at = time.monotonic()self.recording_ctx = CopilotContext(organization_id=organization_id,@@ -593,8 +641,11 @@ class DemonstrationRecorder:self.page = pages[-1]self.tap = PageTap(browser_context, max_events=DEMONSTRATION_MAX_EVENTS)await self.tap.start(pages)- with contextlib.suppress(Exception):+ try:await self.tap.set_intercept_file_chooser_dialog(True)+ self.intercepting_file_chooser = True+ except Exception:+ LOG.warning("copilot_demonstration_file_chooser_intercept_failed", exc_info=True)await self.stream.send({"type": "demonstration_started",@@ -701,6 +752,14 @@ class DemonstrationRecorder:frame["path"] = pathkind = payload.get("kind")selector = payload.get("selector")+ if kind == "file_click":+ # With interception on, the chooser this click opens arrives as its own file_chooser event.+ if not self.intercepting_file_chooser:+ await self._record_gap_event(event, frame, "file_chooser")+ return+ if kind == "file_chooser":+ await self._request_file(event, frame)+ returnif kind == "secret_focus":field = payload.get("field")await self.stream.send(@@ -716,10 +775,7 @@ class DemonstrationRecorder:returnevent_kind = "secret" if kind in {"secret", "secret_select"} else kindif event_kind not in {"click", "type", "select", "key", "navigate", "secret"}:- gap_reason = (- "file_chooser" if kind in {"file_click", "upload"} else f"unsupported_event:{kind or 'unknown'}"- )- await self._record_gap_event(event, frame, gap_reason)+ await self._record_gap_event(event, frame, f"unsupported_event:{kind or 'unknown'}")returnsource_url = str(payload.get("url") or "") if event.is_main_frame else str(event.page.url or "")if event_kind == "navigate":@@ -779,6 +835,21 @@ class DemonstrationRecorder:})+ async def _request_file(self, event: TapEvent, frame: dict[str, Any]) -> None:+ described = await self.tap.call_on_file_input(event, _DESCRIBE_FILE_INPUT) if self.tap is not None else None+ if (+ not isinstance(described, dict)+ or not isinstance(described.get("selector"), str)+ or not described["selector"]+ ):+ await self._record_gap_event(event, frame, "file_chooser")+ return+ selector = described["selector"]+ field_label = safe_label(described.get("label")) or "file input"+ self.file_requests.append({"event": event, "frame": frame, "selector": selector, "field_label": field_label})+ if len(self.file_requests) == 1:+ await self.stream.send({"type": "demonstration_file_requested", "label": field_label})+async def _record_gap_event(self, event: TapEvent, frame: dict[str, Any], reason: str) -> None:stored = {"ts": event.payload.get("ts"),@@ -807,6 +878,8 @@ class DemonstrationRecorder:await self._run_control_with_owner_heartbeat(self._navigate(entry))elif action == "fill_credential":await self._run_control_with_owner_heartbeat(self._fill_credential(entry))+ elif action == "upload_file":+ await self._run_control_with_owner_heartbeat(self._upload_file(entry))return Noneasync def _run_control_with_owner_heartbeat(self, control: Awaitable[None]) -> None:@@ -933,6 +1006,75 @@ class DemonstrationRecorder:})+ async def _upload_file(self, entry: dict[str, Any]) -> None:+ file_id = entry.get("file_id")+ if not self.file_requests or self.tap is None:+ if isinstance(file_id, str):+ await self._discard_upload(file_id)+ await self._notice("The page is no longer waiting for a file.", level="error")+ return+ request = self.file_requests[0]+ tap_event: TapEvent = request["event"]+ field_label = request["field_label"]+ event = _skipped_upload_event(request)+ if isinstance(file_id, str):+ uploaded = None+ data: bytes | None = None+ try:+ uploaded = await app.DATABASE.uploaded_files.get_uploaded_file(+ file_id=file_id, organization_id=self.organization_id+ )+ if uploaded is not None:+ data = await app.STORAGE.download_managed_file(uploaded.storage_uri, self.organization_id)+ except Exception:+ # A failed read is the same outcome as a failed attach: the request stays open for a retry.+ LOG.warning("copilot_demonstration_upload_read_failed", exc_info=True)+ attached = (+ uploaded is not None+ and data is not None+ and await self.tap.call_on_file_input(+ tap_event,+ _ATTACH_FILE,+ uploaded.filename,+ mimetypes.guess_type(uploaded.filename)[0] or "application/octet-stream",+ base64.b64encode(data).decode(),+ )+ == 1+ )+ if not attached or uploaded is None or data is None:+ await self._discard_upload(file_id)+ await self._notice("The file could not be attached to the page.", level="error")+ return+ event.update(+ file_id=file_id,+ filename=uploaded.filename,+ size_bytes=len(data),+ label=f'Upload file to "{field_label}"'[:60],+ )+ seq = await self._append_event(event)+ if seq is not None:+ await self.stream.send(+ {+ "type": "demonstration_action",+ "seq": seq,+ "label": event["label"],+ "tool_name": "upload_attached_file",+ **({"file_id": event["file_id"], "filename": event["filename"]} if "file_id" in event else {}),+ }+ )+ self.file_requests.pop(0)+ if self.file_requests:+ await self.stream.send(+ {"type": "demonstration_file_requested", "label": self.file_requests[0]["field_label"]}+ )++ async def _discard_upload(self, file_id: str) -> None:+ # The file was stored only for this attach, so nothing else references it once the attach is off.+ try:+ await uploaded_file_service.delete_uploaded_file(file_id=file_id, organization_id=self.organization_id)+ except Exception:+ LOG.warning("copilot_demonstration_upload_discard_failed", exc_info=True)+async def _notice(self, message: str, *, level: Literal["info", "error"] = "info") -> None:await self.stream.send({"type": "demonstration_notice", "message": message, "level": level})@@ -968,6 +1110,10 @@ class DemonstrationRecorder:reason = await self._drain_tap() or reasonwith contextlib.suppress(Exception):await tap.set_intercept_file_chooser_dialog(False)+ # A request still open at stop was never answered; replay states it rather than dropping the step.+ for request in self.file_requests:+ await self._append_event(_skipped_upload_event(request))+ self.file_requests.clear()final_url = str(self.page.url or "") if self.page is not None and not self.page.is_closed() else Nonerecord_key = demonstration_record_key(self.organization_id, self.demonstration_id)event_ttl: timedelta | None = None@@ -1014,6 +1160,19 @@ class DemonstrationRecorder:)+def _skipped_upload_event(request: dict[str, Any]) -> dict[str, Any]:+ tap_event: TapEvent = request["event"]+ return {+ "ts": datetime.now(UTC).timestamp() * 1000,+ "kind": "upload",+ "selector": request["selector"],+ "source_url": str(tap_event.page.url or ""),+ "frame": request["frame"],+ "field_label": request["field_label"],+ "label": f'Skip file for "{request["field_label"]}"'[:60],+ }++def safe_label(value: str | None) -> str:if not isinstance(value, str):return ""diff --git a/skyvern/forge/sdk/copilot/demonstration_replay.py b/skyvern/forge/sdk/copilot/demonstration_replay.pyindex 03ea9f240..d5685b728 100644--- a/skyvern/forge/sdk/copilot/demonstration_replay.py+++ b/skyvern/forge/sdk/copilot/demonstration_replay.py@@ -37,7 +37,7 @@ from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServerfrom skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm, summarize_tool_resultfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy, screen_raw_secret_safetyfrom skyvern.forge.sdk.copilot.runtime import DemonstratedAction, ScoutedInteraction, staged_demonstrated_action-from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt+from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt, redact_secretlike_filenamefrom skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structurefrom skyvern.forge.sdk.copilot.streaming_adapter import parse_tool_outputfrom skyvern.forge.sdk.copilot.tools.scouting import record_scouted_interaction_fact@@ -234,6 +234,31 @@ async def _replay_events()await record_call("fill_credential_field", args, args, output)continue+ if kind == "upload":+ selector = str(event.get("selector") or "")+ file_id = event.get("file_id")+ if not isinstance(file_id, str):+ seq = event.get("seq")+ step = f"Demonstration step {seq}" if isinstance(seq, int) else "A demonstration step"+ field_label = safe_label(event.get("field_label")) or "file input"+ history.append(+ {+ "role": "user",+ "content": NUDGE_SENTINEL+ + f'{step} opened a file chooser for "{field_label}" ({selector}); the user chose no file.',+ }+ )+ continue+ args = {"file_id": file_id, "selector": selector}+ output = {+ "ok": True,+ "file_id": file_id,+ "filename": redact_secretlike_filename(str(event.get("filename") or "")),+ "size_bytes": event.get("size_bytes"),+ "change_delivered": True,+ }+ await record_call("upload_attached_file", args, args, output)+ continuetranslated = _translate_action(ctx,diff --git a/skyvern/forge/sdk/copilot/demonstration_tap.js b/skyvern/forge/sdk/copilot/demonstration_tap.jsindex 33ddc8812..b123fc732 100644--- a/skyvern/forge/sdk/copilot/demonstration_tap.js+++ b/skyvern/forge/sdk/copilot/demonstration_tap.js@@ -1177,6 +1177,7 @@waiter = finish;}),selectorFor: getDurableSelector,+ labelFor: ownLabelText,flush,stop: () => {if (stopped) return [];diff --git a/skyvern/forge/sdk/copilot/demonstration_tap.py b/skyvern/forge/sdk/copilot/demonstration_tap.pyindex 85e7d1974..1ec4f60f6 100644--- a/skyvern/forge/sdk/copilot/demonstration_tap.py+++ b/skyvern/forge/sdk/copilot/demonstration_tap.py@@ -194,8 +194,58 @@ class PageTap:self._reconcile_frames(key, tree)session.on("Page.frameNavigated", lambda event: self._frame_navigated(key, event))session.on("Page.frameRequestedNavigation", lambda event: self._frame_requested_navigation(key, event))+ session.on("Page.fileChooserOpened", lambda event: self._file_chooser_opened(key, event))return key+ def _file_chooser_opened(self, session_key: str, event: dict[str, Any]) -> None:+ state = self._sessions.get(session_key)+ backend_node_id = event.get("backendNodeId")+ if state is None or not isinstance(backend_node_id, int):+ return+ frame_id = str(event.get("frameId") or "")+ self._enqueue_event(+ TapEvent(+ page=state["page"],+ session_key=session_key,+ frame_id=frame_id,+ is_main_frame=bool(state["is_page_session"] and frame_id == state["main_frame_id"]),+ frame_name=str(state["frames"].get(frame_id, {}).get("name") or ""),+ payload={"kind": "file_chooser", "ts": time.time() * 1000, "backend_node_id": backend_node_id},+ )+ )++ async def call_on_file_input(self, event: TapEvent, declaration: str, *args: Any) -> Any:+ state = self._sessions.get(event.session_key)+ if state is None:+ return None+ session = state["session"]+ try:+ world = await session.send(+ "Page.createIsolatedWorld",+ {"frameId": event.frame_id, "worldName": WORLD, "grantUniveralAccess": False},+ )+ resolved = await session.send(+ "DOM.resolveNode",+ {+ "backendNodeId": event.payload["backend_node_id"],+ "executionContextId": world["executionContextId"],+ },+ )+ result = await session.send(+ "Runtime.callFunctionOn",+ {+ "functionDeclaration": declaration,+ "objectId": resolved["object"]["objectId"],+ "arguments": [{"value": arg} for arg in args],+ "returnByValue": True,+ "awaitPromise": True,+ },+ )+ except Exception:+ LOG.info("copilot_demonstration_file_input_call_failed", exc_info=True)+ return None+ return result.get("result", {}).get("value")+def _frame_requested_navigation(self, session_key: str, event: dict[str, Any]) -> None:state = self._sessions.get(session_key)frame_id = str(event.get("frameId") or "")diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex ec11a0383..07357b7fc 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -84,6 +84,7 @@ from skyvern.forge.sdk.copilot.demonstration_recorder import (append_demonstration_control,claim_demonstration_recording,demonstration_action_labels,+ demonstration_file_ids,demonstration_limit_reached,demonstration_owner_is_live,demonstration_time_limit_reached,@@ -186,6 +187,7 @@ from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.workflow_copilot import (COPILOT_PRIVATE_SETTINGS_KEY,COPILOT_PROPOSAL_METADATA_KEY,+ MAX_ATTACHED_FILES_PER_MESSAGE,TURN_OPENER_SENDERS,CopilotAttachedFile,CopilotCancelSource,@@ -246,6 +248,7 @@ from skyvern.schemas.workflows import (WorkflowCreateYAMLRequest,WorkflowDefinitionYAML,)+from skyvern.services import uploaded_file_servicefrom skyvern.services.browser_recording.evidence import build_recording_evidencefrom skyvern.services.browser_recording.session_registry import interpretation_registryfrom skyvern.utils.contained_effects import contained_effect@@ -3857,7 +3860,7 @@ async def workflow_copilot_chat_post(detail="demonstration_id is required for a Copilot demonstration",)try:- await validate_demonstration_for_replay(+ demonstration = await validate_demonstration_for_replay(cache,organization_id=organization.organization_id,workflow_permanent_id=chat_request.workflow_permanent_id,@@ -3865,6 +3868,13 @@ async def workflow_copilot_chat_post()except DemonstrationRejection as rejection:raise HTTPException(status_code=rejection.status_code, detail=rejection.detail) from rejection+ # A reloaded card no longer holds the files it attached; the recorded upload events still do.+ demonstrated_file_ids = await demonstration_file_ids(+ cache, organization.organization_id, chat_request.demonstration_id, demonstration.event_count+ )+ chat_request.attached_file_ids = list(dict.fromkeys([*chat_request.attached_file_ids, *demonstrated_file_ids]))[+ :MAX_ATTACHED_FILES_PER_MESSAGE+ ]raw_eval_mode = request.headers.get("x-copilot-eval-mode")if raw_eval_mode is not None:try:@@ -4975,6 +4985,59 @@ async def workflow_copilot_demonstration_control(return WorkflowCopilotDemonstrationControlResult(seq=seq)+@base_router.post(+ "/workflow/copilot/demonstration/upload",+ include_in_schema=False,+ response_model=WorkflowCopilotDemonstrationControlResult,+)+async def workflow_copilot_demonstration_upload(+ demonstration_id: str = Form(..., max_length=64),+ file: UploadFile = File(...),+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowCopilotDemonstrationControlResult:+ cache = await _demonstration_cache(organization.organization_id)+ demonstration = await load_demonstration_record(cache, organization.organization_id, demonstration_id)+ if demonstration is None:+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Unknown demonstration")+ await app.AGENT_FUNCTION.enforce_folder_access(+ get_request_principal(),+ "workflow_copilot.manage",+ {"workflow_permanent_id": demonstration.workflow_permanent_id},+ )+ if demonstration.status != "recording":+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Demonstration is not recording")+ attached_file_ids = await demonstration_file_ids(+ cache, organization.organization_id, demonstration_id, demonstration.event_count+ )+ # Done carries every attached file, so the message limit bounds them; failed attempts are not counted.+ if len(attached_file_ids) >= MAX_ATTACHED_FILES_PER_MESSAGE:+ raise HTTPException(+ status_code=status.HTTP_409_CONFLICT,+ detail=f"A demonstration can attach at most {MAX_ATTACHED_FILES_PER_MESSAGE} files",+ )+ await uploaded_file_service.validate_file_size(file, max_size_bytes=app.SETTINGS_MANAGER.MAX_UPLOAD_FILE_SIZE)+ try:+ uploaded, _ = await uploaded_file_service.save_uploaded_file(+ file=file, organization_id=organization.organization_id+ )+ except uploaded_file_service.UploadStorageError as exc:+ raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc+ # The recorder attaches only files stored here for this demonstration, never an id the client names.+ try:+ seq = await append_demonstration_control(+ cache,+ organization_id=organization.organization_id,+ request=WorkflowCopilotDemonstrationControlRequest(demonstration_id=demonstration_id, action="upload_file"),+ file_id=uploaded.file_id,+ )+ except DemonstrationRejection as rejection:+ await uploaded_file_service.delete_uploaded_file(+ file_id=uploaded.file_id, organization_id=organization.organization_id+ )+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail) from rejection+ return WorkflowCopilotDemonstrationControlResult(seq=seq, file_id=uploaded.file_id)++@base_router.get("/workflow/copilot/demonstration/active", include_in_schema=False)async def workflow_copilot_demonstration_active(workflow_permanent_id: str,diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex dcf995b55..a8afebab0 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -596,7 +596,7 @@ class WorkflowCopilotCredentialResponseRequest(BaseModel):class WorkflowCopilotDemonstrationControlRequest(BaseModel):demonstration_id: str = Field(max_length=64)- action: Literal["navigate", "fill_credential", "stop", "cancel"]+ action: Literal["navigate", "fill_credential", "upload_file", "stop", "cancel"]url: str | None = Field(default=None, max_length=2048)selector: str | None = Field(default=None, max_length=2000)credential_id: str | None = Field(default=None, max_length=128)@@ -608,6 +608,7 @@ class WorkflowCopilotDemonstrationControlRequest(BaseModel):class WorkflowCopilotDemonstrationControlResult(BaseModel):seq: intaction_count: int | None = None+ file_id: str | None = Noneclass WorkflowCopilotDemonstrationRecordRequest(BaseModel):diff --git a/skyvern/forge/sdk/services/route_authorization.py b/skyvern/forge/sdk/services/route_authorization.pyindex 601d074e5..b3a7d8c5c 100644--- a/skyvern/forge/sdk/services/route_authorization.py+++ b/skyvern/forge/sdk/services/route_authorization.py@@ -268,6 +268,7 @@ ROUTE_ACTIONS: Mapping[str, str] = {"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_active": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_control": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_record": "workflow_copilot.manage",+ "skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_demonstration_upload": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_disable_auto_accept": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_message_feedback": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_question_response": "workflow_copilot.manage",diff --git a/tests/unit/browser_extension/test_broker_server.py b/tests/unit/browser_extension/test_broker_server.pyindex 6f4d9d2e4..08c0555bc 100644--- a/tests/unit/browser_extension/test_broker_server.py+++ b/tests/unit/browser_extension/test_broker_server.py@@ -1576,10 +1576,7 @@ async def test_client_enrollment_proceeds_when_extension_disconnects_during_rese@pytest.mark.asyncio-async def test_late_extension_reconnect_is_reset_before_successor_sees_snapshot(- monkeypatch: pytest.MonkeyPatch,-) -> None:- monkeypatch.setattr(broker_server_module, "EXTENSION_RESET_TIMEOUT_SECONDS", 0.01)+async def test_late_extension_reconnect_is_reset_before_successor_sees_snapshot() -> None:server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())relay = ControlledResetRelay("extension-secret",@@ -1847,7 +1844,7 @@ async def test_reset_send_deadline_surfaces_structured_timeout(@pytest.mark.asyncioasync def test_owner_release_allows_reenrollment_while_timed_out_request_awaits_terminal() -> None:- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)relay = ExtensionRelayServer("extension-secret",19777,@@ -2478,7 +2475,7 @@ async def test_retained_timed_out_extension_requests_remain_bounded(limit_name: str,) -> None:monkeypatch.setattr(broker_server_module, limit_name, 1)- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)relay = ExtensionRelayServer("extension-secret",19777,@@ -2561,7 +2558,7 @@ async def test_retained_timed_out_extension_requests_remain_bounded(async def test_retained_timed_out_request_holds_inbound_bytes_until_terminal(monkeypatch: pytest.MonkeyPatch,) -> None:- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)relay = ExtensionRelayServer("extension-secret",19777,@@ -2624,7 +2621,7 @@ async def test_retained_timed_out_requests_enforce_per_tab_limit_until_terminal() -> None:monkeypatch.setattr(broker_server_module, "MAX_REQUESTS_PER_TAB", 1)monkeypatch.setattr(broker_server_module, "TAB_REQUEST_QUEUE_WAIT_SECONDS", 0.01)- server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir())+ server = BrowserExtensionBrokerServer(19777, base_dir=_test_broker_base_dir(), time_source=_frozen_broker_clock)relay = ExtensionRelayServer("extension-secret",19777,@@ -4988,6 +4985,12 @@ async def _ignore_event(_event: str, _params: dict) -> None:return None+def _frozen_broker_clock() -> float:+ # Frozen so a runner stall before forwarding cannot expire a short request; the relay still+ # times it out on real time.+ return 0.0++async def _eventually(predicate: Callable[[], bool]) -> None:for _ in range(100):if predicate():diff --git a/tests/unit/test_copilot_demonstration.py b/tests/unit/test_copilot_demonstration.pyindex e9d801f82..5e0502012 100644--- a/tests/unit/test_copilot_demonstration.py+++ b/tests/unit/test_copilot_demonstration.py@@ -129,6 +129,14 @@ class _FakeTap:async def frame_path(self, event: TapEvent) -> list[dict[str, str]] | None:return [] if event.is_main_frame else [{"name": event.frame_name}] if event.frame_name else None+ async def call_on_file_input(self, event: TapEvent, declaration: str, *args: Any) -> Any:+ if "selectorFor" in declaration:+ return {"selector": "#resume", "label": "Resume"}+ type(self).attached_files.append(args[:2])+ return 1++ attached_files: ClassVar[list[tuple[Any, ...]]] = []+def begin_control_navigation(self) -> None:return None@@ -273,6 +281,23 @@ async def test_replay_contract_never_touches_a_browser_and_consumes_once(monkeyp"credential_field": "password",},},+ {+ "kind": "upload",+ "selector": "#resume",+ "file_id": "file-1",+ "filename": "resume.pdf",+ "size_bytes": 4,+ "field_label": "Resume",+ "source_url": "https://example.test/done",+ "frame": {"is_main": True, "name": ""},+ },+ {+ "kind": "upload",+ "selector": "#cover",+ "field_label": "Cover letter",+ "source_url": "https://example.test/done",+ "frame": {"is_main": True, "name": ""},+ },]cache.store[demonstration_record_key("org-1", "demo-1")] = encode_demonstration_record(DemonstrationRecord(@@ -327,7 +352,9 @@ async def test_replay_contract_never_touches_a_browser_and_consumes_once(monkeyp"type_text","type_text","fill_credential_field",+ "upload_attached_file",]+ assert json.loads(calls[-1]["arguments"]) == {"file_id": "file-1", "selector": "#resume"}serialized = json.dumps(history)assert "Acme Corp" in serializedassert token not in serialized@@ -335,6 +362,7 @@ async def test_replay_contract_never_touches_a_browser_and_consumes_once(monkeypassert "<value not recorded>" in serializedassert any("file chooser" in str(item.get("content")) for item in history if item.get("role") == "user")assert any("disconnected" in str(item.get("content")) for item in history if item.get("role") == "user")+ assert any("#cover" in str(item.get("content")) and "no file" in str(item.get("content")) for item in history)gap_index = next(index for index, item in enumerate(history) if "file chooser" in str(item.get("content")))first_type_index = next(index for index, item in enumerate(history) if item.get("name") == "type_text")assert gap_index < first_type_index@@ -481,9 +509,22 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pyt[{"seq": 1, "action": "navigate", "url": "http://127.0.0.1/private"},{"seq": 2, "action": "navigate", "url": "https://example.test/next"},+ {"seq": 3, "action": "upload_file", "file_id": "file-1"},])await cache.set(demonstration_browser_key("org-1", "pbs-1"), demonstration_id, ex=timedelta(minutes=1))+ monkeypatch.setattr(+ app,+ "DATABASE",+ SimpleNamespace(+ uploaded_files=SimpleNamespace(+ get_uploaded_file=AsyncMock(+ return_value=SimpleNamespace(storage_uri="s3://bucket/org-1/resume.pdf", filename="resume.pdf")+ )+ )+ ),+ )+ monkeypatch.setattr(app, "STORAGE", SimpleNamespace(download_managed_file=AsyncMock(return_value=b"%PDF")))page = SimpleNamespace(url="https://example.test/start", goto=AsyncMock(), is_closed=lambda: False)click = TapEvent(page=page,@@ -535,14 +576,31 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pytframe_name="",payload={"kind": "file_click", "ts": 3, "url": page.url, "selector": "#upload"},)+ file_chooser = TapEvent(+ page=page,+ session_key="tap-1",+ frame_id="main",+ is_main_frame=True,+ frame_name="",+ payload={"kind": "file_chooser", "ts": 3.1, "backend_node_id": 7},+ )+ unanswered_chooser = TapEvent(+ page=page,+ session_key="tap-1",+ frame_id="main",+ is_main_frame=True,+ frame_name="",+ payload={"kind": "file_chooser", "ts": 3.2, "backend_node_id": 8},+ )browser_context = SimpleNamespace(pages=[page])browser_state = SimpleNamespace(browser_context=browser_context,get_working_page=AsyncMock(return_value=page),)- _FakeTap.events = [click, password_focus, password, file_click]+ _FakeTap.events = [click, password_focus, password, file_click, file_chooser, unanswered_chooser]_FakeTap.flush_events = []+ _FakeTap.attached_files = []monkeypatch.setattr(recorder_module, "PageTap", _FakeTap)monkeypatch.setattr(recorder_module, "resolve_persistent_browser_state", AsyncMock(return_value=browser_state))@@ -574,18 +632,27 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pytassert [first["kind"], second["kind"], third["kind"], fourth["kind"]] == ["click","secret",- "gap","navigate",+ "upload",]- assert third["reason"] == "file_chooser"+ assert {key: fourth[key] for key in ("selector", "file_id", "filename", "size_bytes")} == {+ "selector": "#resume",+ "file_id": "file-1",+ "filename": "resume.pdf",+ "size_bytes": 4,+ }+ assert _FakeTap.attached_files == [("resume.pdf", "application/pdf")]+ assert {"type": "demonstration_file_requested", "label": "Resume"} in stream.frames+ unanswered = json.loads(cache.store[demonstration_event_key("org-1", demonstration_id, 5)])+ assert unanswered["kind"] == "upload" and "file_id" not in unansweredassert page.goto.await_args.args == ("https://example.test/next",)assert any(frame.get("type") == "demonstration_notice" and frame.get("level") == "error" for frame in stream.frames)action_frames = [frame for frame in stream.frames if frame.get("type") == "demonstration_action"]assert [(frame["seq"], frame["tool_name"]) for frame in action_frames] == [(1, "click"),(2, "type_text"),- (3, "gap"),- (4, "navigate_browser"),+ (3, "navigate_browser"),+ (4, "upload_attached_file"),]assert action_frames[1]["label"] == second["label"] == "Fill password (value not recorded)"sensitive_frame = next(frame for frame in stream.frames if frame.get("type") == "demonstration_sensitive_field")@@ -596,13 +663,13 @@ async def test_recorder_persists_tap_actions_controls_and_stops(monkeypatch: pyt"url": "https://example.test/start",}stopped = decode_demonstration_record(cache.store[demonstration_record_key("org-1", demonstration_id)])- assert stopped is not None and stopped.status == "stopped" and stopped.event_count == 4+ assert stopped is not None and stopped.status == "stopped" and stopped.event_count == 5assert stopped.stop_reason == "disconnected"assert stream.frames[-1] == {"type": "demonstration_stopped","demonstration_id": "demo-1","reason": "disconnected",- "action_count": 4,+ "action_count": 5,}assert cache.expirations[demonstration_browser_key("org-1", "pbs-1")] == recorder_module.DEMONSTRATION_TTL@@ -941,6 +1008,37 @@ async def test_discard_after_resume_commits_cancelled_even_when_the_stream_dropsassert active is None+async def test_client_upload_control_cannot_name_a_file() -> None:+ cache = _SharedCache()+ cache.store[demonstration_record_key("org-1", "demo-1")] = encode_demonstration_record(+ DemonstrationRecord(+ workflow_permanent_id="wfp-1",+ browser_session_id="pbs-1",+ status="recording",+ event_count=0,+ owner="owner-1",+ owner_seen_at=datetime.now(UTC),+ final_url=None,+ )+ )+ cache.store[demonstration_controls_key("org-1", "demo-1")] = "[]"+ crafted = WorkflowCopilotDemonstrationControlRequest.model_validate(+ {"demonstration_id": "demo-1", "action": "upload_file", "file_id": "file-from-elsewhere"}+ )++ await append_demonstration_control(cache, organization_id="org-1", request=crafted) # type: ignore[arg-type]+ await append_demonstration_control(+ cache, # type: ignore[arg-type]+ organization_id="org-1",+ request=WorkflowCopilotDemonstrationControlRequest(demonstration_id="demo-1", action="upload_file"),+ file_id="file-stored-for-demo",+ )++ controls = json.loads(cache.store[demonstration_controls_key("org-1", "demo-1")])+ assert [control.get("file_id") for control in controls] == [None, "file-stored-for-demo"]++@pytest.mark.asyncioasync def test_recorder_runs_controls_queued_before_stop(monkeypatch: pytest.MonkeyPatch) -> None:cache = _SharedCache()