Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fall back select-verification prompts to the secondary LLM instead of the main model (#7657)
pedrohsdb732e4618dd54
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsxindex d7682d6c7..4bbd91e06 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx@@ -330,3 +330,91 @@ describe("CredentialsModal edit-mode inline test", () => {}}, 15_000);});++describe("CredentialsModal copilot-context tested_url default", () => {+ function renderCopilotPasswordModal(opts: {+ defaultTestUrl?: string;+ onCredentialCreated?: (id: string, name?: string) => void;+ }) {+ const queryClient = new QueryClient({+ defaultOptions: {+ queries: { retry: false },+ mutations: { retry: false },+ },+ });+ return render(+ <QueryClientProvider client={queryClient}>+ <MemoryRouter>+ <CredentialsModal+ isOpen+ onOpenChange={vi.fn()}+ overrideType={CredentialModalTypes.PASSWORD}+ defaultTestUrl={opts.defaultTestUrl}+ onCredentialCreated={opts.onCredentialCreated}+ />+ </MemoryRouter>+ </QueryClientProvider>,+ );+ }++ async function fillUsernameAndPassword() {+ await waitFor(() => {+ expect(screen.getByDisplayValue("credentials")).toBeTruthy();+ });+ const usernameInput = Array.from(+ document.querySelectorAll<HTMLInputElement>("input"),+ ).find(+ (input) =>+ input.type === "text" && input.value === "" && input.placeholder === "",+ );+ expect(usernameInput).toBeTruthy();+ fireEvent.change(usernameInput as HTMLInputElement, {+ target: { value: "[email protected]" },+ });+ const passwordInput = document.querySelector('input[type="password"]');+ expect(passwordInput).toBeTruthy();+ fireEvent.change(passwordInput as HTMLInputElement, {+ target: { value: "password" },+ });+ }++ it("persists tested_url from defaultTestUrl on a plain create (no test run)", async () => {+ postMock.mockResolvedValueOnce({+ data: { credential_id: "cred-x", name: "credentials" },+ });+ patchMock.mockResolvedValue({ data: {} });+ const onCredentialCreated = vi.fn();+ renderCopilotPasswordModal({+ defaultTestUrl: "https://news.ycombinator.com/login",+ onCredentialCreated,+ });+ await fillUsernameAndPassword();+ fireEvent.click(screen.getByRole("button", { name: "Save" }));+ await waitFor(() =>+ expect(postMock).toHaveBeenCalledWith("/credentials", expect.anything()),+ );+ await waitFor(() =>+ expect(patchMock).toHaveBeenCalledWith(+ "/credentials/cred-x",+ expect.objectContaining({+ tested_url: "https://news.ycombinator.com/login",+ }),+ ),+ );+ expect(onCredentialCreated).toHaveBeenCalledWith("cred-x", "credentials");+ }, 10_000);++ it("sends no tested_url when defaultTestUrl is absent (modal from elsewhere)", async () => {+ postMock.mockResolvedValueOnce({+ data: { credential_id: "cred-y", name: "credentials" },+ });+ const onCredentialCreated = vi.fn();+ renderCopilotPasswordModal({ onCredentialCreated });+ await fillUsernameAndPassword();+ fireEvent.click(screen.getByRole("button", { name: "Save" }));+ await waitFor(() =>+ expect(onCredentialCreated).toHaveBeenCalledWith("cred-y", "credentials"),+ );+ expect(patchMock).not.toHaveBeenCalled();+ }, 10_000);+});diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsxindex 3fdbd941c..f291e1ec5 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx@@ -182,7 +182,7 @@ function buildMetadata(values: CreditCardCredentialValues) {}type Props = {- onCredentialCreated?: (id: string) => void;+ onCredentialCreated?: (id: string, name?: string) => void;/** Optional controlled mode: pass isOpen and onOpenChange to control modal state locally */isOpen?: boolean;onOpenChange?: (open: boolean) => void;@@ -190,6 +190,10 @@ type Props = {editingCredential?: CredentialApiResponse;/** Override the modal type (used in edit mode to set the correct form) */overrideType?: CredentialModalType;+ // Copilot-only: seed the login URL so a quick-added credential carries a+ // tested_url and matches later hostname-keyed asks. Create mode only; leaving+ // it undefined (every non-copilot caller) keeps the field empty as before.+ defaultTestUrl?: string;/** Called after a credential is saved with "Save browser session" checked to trigger an async test */onStartBackgroundTest?: (credentialId: string,@@ -217,6 +221,7 @@ function CredentialsModal({onOpenChange: controlledOnOpenChange,editingCredential,overrideType,+ defaultTestUrl,onStartBackgroundTest,}: Props) {const credentialGetter = useCredentialGetter();@@ -496,8 +501,11 @@ function CredentialsModal({...prev,name: defaultName,}));+ if (defaultTestUrl) {+ setTestUrl(defaultTestUrl);+ }}- }, [isOpen, credentials, isEditMode, editingCredential]);+ }, [isOpen, credentials, isEditMode, editingCredential, defaultTestUrl]);function reset() {setVaultType("default");@@ -757,7 +765,7 @@ function CredentialsModal({queryClient.invalidateQueries({queryKey: ["credentials"],});- onCredentialCreated?.(data.credential_id);+ onCredentialCreated?.(data.credential_id, data.name);reset();setIsOpen(false);@@ -923,7 +931,7 @@ function CredentialsModal({queryClient.invalidateQueries({queryKey: ["credentials"],});- onCredentialCreated?.(data.credential_id);+ onCredentialCreated?.(data.credential_id, data.name);reset();setIsOpen(false);toast({diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsxindex aa75a801c..8ffdc4bb9 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx@@ -34,6 +34,7 @@ const {credentialsData,credsFail,modalOverrideType,+ modalDefaultTestUrl,toastFn,flagMap,} = vi.hoisted(() => {@@ -93,6 +94,7 @@ const {credentialsData: creds,credsFail: fail,modalOverrideType: { current: undefined as string | undefined },+ modalDefaultTestUrl: { current: undefined as string | undefined },toastFn: vi.fn(),flagMap: { current: {} as Record<string, boolean> },};@@ -119,17 +121,20 @@ vi.mock("@/routes/credentials/CredentialsModal", () => ({isOpen,onCredentialCreated,overrideType,+ defaultTestUrl,}: {isOpen?: boolean;- onCredentialCreated?: (id: string) => void;+ onCredentialCreated?: (id: string, name?: string) => void;overrideType?: string;+ defaultTestUrl?: string;}) => {modalOverrideType.current = overrideType;+ modalDefaultTestUrl.current = defaultTestUrl;return isOpen ? (<buttontype="button"data-testid="mock-create-credential"- onClick={() => onCredentialCreated?.("new-cred-1")}+ onClick={() => onCredentialCreated?.("new-cred-1", "New Login")}>create credential</button>@@ -317,6 +322,7 @@ beforeEach(() => {credentialsData.current = [];credsFail.current = false;modalOverrideType.current = undefined;+ modalDefaultTestUrl.current = undefined;flagMap.current = {};historyResponse.data = {workflow_copilot_chat_id: "chat-1",@@ -575,6 +581,102 @@ describe("WorkflowCopilotChat — credential card wiring (flag on)", () => {).toBeNull();expect(screen.queryByRole("button", { name: "Skip for now" })).toBeNull();});++ it("terminal connect via the modal auto-continues once and morphs the receipt", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ await renderChat();+ await submit("who am I signing in as?");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart("turn-9"));+ streamCalls[0]!.onMessage(terminalPromptResponse("turn-9"));+ streamCalls[0]!.resolve();+ });+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Connect credential" }),+ );+ });+ const createBtn = await screen.findByTestId("mock-create-credential");+ await act(async () => {+ fireEvent.click(createBtn);+ });+ // A fresh turn is sent (exactly one) referencing the connected credential.+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(2));+ expect(streamCalls[1]!.body.message).toContain(+ "I've connected the credential 'New Login'",+ );+ expect(streamCalls[1]!.body.message).toContain("continue");+ expect(credentialResponsePosts()).toHaveLength(0);+ expect(+ await screen.findByText("Continuing with 'New Login'…"),+ ).toBeTruthy();+ });++ it("does not auto-continue while a turn is still in flight", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ await renderChat();+ await submit("who am I signing in as?");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ // Emit the terminal card but never resolve the stream — isLoading stays+ // true, so the connect must not fire a fresh turn.+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart("turn-9"));+ streamCalls[0]!.onMessage(terminalPromptResponse("turn-9"));+ });+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Connect credential" }),+ );+ });+ const createBtn = await screen.findByTestId("mock-create-credential");+ await act(async () => {+ fireEvent.click(createBtn);+ });+ expect(+ await screen.findByText("Credential 'New Login' added"),+ ).toBeTruthy();+ expect(postStreaming).toHaveBeenCalledTimes(1);+ });++ it("passes the pause frame's login URL to the modal as defaultTestUrl", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ await renderChat();+ await submit("build me a workflow");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart());+ streamCalls[0]!.onMessage(credentialFrame());+ });+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Connect credential" }),+ );+ });+ await screen.findByTestId("mock-create-credential");+ expect(modalDefaultTestUrl.current).toBe(+ "https://news.ycombinator.com/login",+ );+ });++ it("passes no defaultTestUrl for a terminal card (frame carries no URL)", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ await renderChat();+ await submit("who am I signing in as?");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart("turn-9"));+ streamCalls[0]!.onMessage(terminalPromptResponse("turn-9"));+ streamCalls[0]!.resolve();+ });+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Connect credential" }),+ );+ });+ await screen.findByTestId("mock-create-credential");+ expect(modalDefaultTestUrl.current).toBeUndefined();+ });});describe("WorkflowCopilotChat — credential card (flag off parity)", () => {diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 5777a77b9..1a38319a8 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -444,7 +444,11 @@ function historicalCredentialOutcome(};}-type CredentialResolution = CredentialPauseHistorical & { name?: string };+type CredentialResolution = CredentialPauseHistorical & {+ name?: string;+ // Terminal connect auto-sent a "continue" turn — drives the receipt copy.+ continued?: boolean;+};// Append a resolution keyed by turn, capping the map with oldest-eviction like// the sibling per-turn maps (turnSnapshots/turnOwnedRunIds). delete-then-set@@ -803,10 +807,18 @@ export function WorkflowCopilotChat({const credentialsFetchInFlight = useRef(false);// Routes the add-credential modal's onCredentialCreated back to the card that// opened it: a live frame (POST resume) or a terminal turn (local resolve).+ // isLastMessage is captured at open time so a terminal connect only+ // auto-continues from the conversation's tail, not a scrolled-back card.const pendingCredentialConnect = useRef<{frame: WorkflowCopilotCredentialRequiredUpdate | null;turnId: string;+ isLastMessage: boolean;} | null>(null);+ // Latest handleSend for callbacks defined before it (the terminal+ // auto-continue fires from the modal's onCredentialCreated).+ const handleSendRef = useRef<((messageOverride?: string) => void) | null>(+ null,+ );// Single-flight guard for the resume POST against a double-click.const credentialResponseInFlight = useRef(false);const streamingAbortController = useRef<AbortController | null>(null);@@ -1109,28 +1121,66 @@ export function WorkflowCopilotChat({// Terminal-mode cards have no resume_token — connect/skip is a local UI morph,// no network call.const resolveTerminalCredential = useCallback(- (turnId: string, action: "connected" | "skip", credentialId?: string) => {+ (+ turnId: string,+ action: "connected" | "skip",+ credentialId?: string,+ name?: string,+ continued?: boolean,+ ) => {setCredentialResolutions((prev) =>withCappedResolution(prev,turnId,action === "connected"- ? { outcome: "connected", credentialId }+ ? { outcome: "connected", credentialId, name, continued }: { outcome: "skipped" },),);},[],);+ // A terminal ask is a dead-end otherwise: the turn already ended, so connecting+ // a credential does nothing without a fresh turn. Auto-send one — but only from+ // the tail of an idle conversation (SKY-12384 gating), and only when we know+ // the credential's name to reference.+ const continueAfterTerminalConnect = useCallback(+ (+ turnId: string,+ credentialId: string,+ name: string | undefined,+ isLastMessage: boolean,+ ) => {+ const shouldContinue =+ isLastMessage && !isLoading && !isLoadingHistory && Boolean(name);+ resolveTerminalCredential(+ turnId,+ "connected",+ credentialId,+ name,+ shouldContinue,+ );+ if (shouldContinue) {+ void handleSendRef.current?.(+ `I've connected the credential '${name}' — continue.`,+ );+ }+ },+ [isLoading, isLoadingHistory, resolveTerminalCredential],+ );const openCredentialModal = useCallback(- (frame: WorkflowCopilotCredentialRequiredUpdate | null, turnId: string) => {- pendingCredentialConnect.current = { frame, turnId };+ (+ frame: WorkflowCopilotCredentialRequiredUpdate | null,+ turnId: string,+ isLastMessage = false,+ ) => {+ pendingCredentialConnect.current = { frame, turnId, isLastMessage };setCredentialModalOpen(true);},[],);const handleCredentialCreated = useCallback(- (credentialId: string) => {+ (credentialId: string, name?: string) => {const ctx = pendingCredentialConnect.current;pendingCredentialConnect.current = null;setCredentialModalOpen(false);@@ -1140,10 +1190,15 @@ export function WorkflowCopilotChat({if (ctx.frame) {void respondToCredentialPause(ctx.frame, "connected", credentialId);} else {- resolveTerminalCredential(ctx.turnId, "connected", credentialId);+ continueAfterTerminalConnect(+ ctx.turnId,+ credentialId,+ name,+ ctx.isLastMessage,+ );}},- [respondToCredentialPause, resolveTerminalCredential],+ [respondToCredentialPause, continueAfterTerminalConnect],);// Explore/Draft boundary is unobservable (the LLM writes code with no// frames emitted); after DRAFTING_GAP_MS of silence with no pending block@@ -2418,6 +2473,9 @@ export function WorkflowCopilotChat({workflowRunId,],);+ useEffect(() => {+ handleSendRef.current = handleSend;+ }, [handleSend]);// A code block's "Generate" button asks the copilot to (re)build that one block// from its goal. Force build + code mode, then fire the send on the next tick.@@ -3201,14 +3259,19 @@ export function WorkflowCopilotChat({localResolution ??historicalCredentialOutcome(message.narrative)}+ continued={Boolean(localResolution?.continued)}+ // Terminal cards surface no pre-connect matches today,+ // so only the modal branch fires; the id branch shares+ // the same helper for when terminal mode gains matches.onConnect={(credentialId) =>credentialId- ? resolveTerminalCredential(+ ? continueAfterTerminalConnect(turnId,- "connected",credentialId,+ localResolution?.name,+ isLastMessage,)- : openCredentialModal(null, turnId)+ : openCredentialModal(null, turnId, isLastMessage)}onSkip={() =>resolveTerminalCredential(turnId, "skip")@@ -3706,6 +3769,12 @@ export function WorkflowCopilotChat({// A sign-in pause always needs a password credential; force the form// so a lingering ?type=credit-card/secret param can't open the wrong one.overrideType={CredentialModalTypes.PASSWORD}+ // Seed tested_url from the pause frame's login page so a quick-add+ // credential matches later asks. Terminal frames carry no URL — the+ // field stays empty then.+ defaultTestUrl={+ pendingCredentialConnect.current?.frame?.login_page_urls?.[0]+ }onOpenChange={(open) => {setCredentialModalOpen(open);if (!open) pendingCredentialConnect.current = null;diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsxindex ffa3edef0..6b03ed5b6 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx@@ -585,6 +585,36 @@ describe("CredentialCard countdown (inline-pause vs terminal)", () => {});});+describe("CredentialCard terminal next-step hint", () => {+ it("shows the continue hint in terminal mode", () => {+ render(+ <CredentialCard+ frame={buildCredentialRequiredFrame()}+ mode="terminal"+ onConnect={vi.fn()}+ onSkip={vi.fn()}+ />,+ );+ expect(+ screen.getByText("Connect a credential and I'll continue."),+ ).toBeTruthy();+ });++ it("omits the continue hint in inline-pause mode", () => {+ render(+ <CredentialCard+ frame={buildCredentialRequiredFrame()}+ mode="inline-pause"+ onConnect={vi.fn()}+ onSkip={vi.fn()}+ />,+ );+ expect(+ screen.queryByText("Connect a credential and I'll continue."),+ ).toBeNull();+ });+});+describe("CredentialCard historical (resolvedOutcome) rendering", () => {it("renders a fallback instead of crashing on an out-of-union outcome value", () => {// Simulates untyped network data reaching a compile-time-exhaustive@@ -625,6 +655,37 @@ describe("CredentialCard historical (resolvedOutcome) rendering", () => {expect(onSkip).not.toHaveBeenCalled();});+ it("renders a 'Continuing' receipt when a terminal connect auto-continued", () => {+ render(+ <CredentialCard+ frame={buildCredentialRequiredFrame()}+ mode="terminal"+ matchingCredentials={ONE_MATCHING_CREDENTIAL}+ resolvedOutcome={RESOLVED_OUTCOME_CONNECTED}+ continued+ onConnect={vi.fn()}+ onSkip={vi.fn()}+ />,+ );+ expect(screen.getByText("Continuing with 'HN login'…")).toBeTruthy();+ expect(screen.queryByText(/added/)).toBeNull();+ });++ it("uses unnamed 'Continuing…' copy when the continued id has no match", () => {+ render(+ <CredentialCard+ frame={buildCredentialRequiredFrame()}+ mode="terminal"+ matchingCredentials={NO_MATCHING_CREDENTIALS}+ resolvedOutcome={RESOLVED_OUTCOME_CONNECTED}+ continued+ onConnect={vi.fn()}+ onSkip={vi.fn()}+ />,+ );+ expect(screen.getByText("Continuing…")).toBeTruthy();+ });+it("falls back to unnamed receipt copy when the connected id has no matching credential", () => {render(<CredentialCarddiff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsxindex 50eb7cec6..40c72cfd1 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx@@ -60,6 +60,10 @@ export interface CredentialCardProps {// id = the user picked an already-stored credential (chip or dropdown).onConnect: (credentialId?: string) => void;onSkip: () => void;+ // Terminal connect auto-sends a "continue" turn; the receipt says so instead+ // of the plain "added". Defaults false so inline-pause and every other caller+ // keep the existing copy.+ continued?: boolean;}const SIGN_IN_WHY_LINE =@@ -174,6 +178,7 @@ export function CredentialCard({resolvedOutcome,onConnect,onSkip,+ continued = false,}: Readonly<CredentialCardProps>) {// Terminal mode never expires by design: its signal carries no timeout/expiry// semantics at all, so there is nothing to compare "now" against. Only a@@ -197,11 +202,18 @@ export function CredentialCard({(credential) =>credential.credentialId === resolvedOutcome.credentialId,)?.name;+ const heading = continued+ ? name+ ? `Continuing with '${name}'…`+ : "Continuing…"+ : name+ ? `Credential '${name}' added`+ : "Credential added";return (<div className="rounded-lg border border-border bg-slate-elevation2 p-3"><div className="flex items-center gap-2 text-xs font-semibold text-foreground"><span className="text-success">✓</span>- {name ? `Credential '${name}' added` : "Credential added"}+ {heading}</div><p className="mt-1 text-xs text-muted-foreground">Stored encrypted · used to sign in on your behalf · never enters@@ -248,6 +260,11 @@ export function CredentialCard({<p className="mt-0.5 text-[11px] leading-relaxed text-muted-foreground">{CREDENTIAL_WHY_LINE_BY_REASON[frame.reason] ?? SIGN_IN_WHY_LINE}</p>+ {mode === "terminal" ? (+ <p className="mt-1 text-[11px] font-medium leading-relaxed text-foreground">+ Connect a credential and I'll continue.+ </p>+ ) : null}</div>{countdownActive ? (<>diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 856c06ee0..e9375a293 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -6992,12 +6992,14 @@ async def sequentially_select_from_dropdown(select_history=json.dumps(build_sequential_select_history(select_history)),local_datetime=datetime.now(ensure_context().tz_info).isoformat(),)+ # Fall back to the secondary (vision-capable) handler so this screenshot-based+ # verification stays sighted when the main model is vision-less.llm_api_handler = await resolve_prompt_type_handler_with_override("confirm-multi-selection-finish",task.llm_key,task.workflow_run_id if task.workflow_run_id else task.task_id,task.organization_id,- LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER),+ LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.SECONDARY_LLM_API_HANDLER),)json_response = await llm_api_handler(prompt=prompt, screenshots=[screenshot], step=step, prompt_name="confirm-multi-selection-finish"