Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-17489: Cannot reproduce See all 15; add expand/collapse coverage (#8744)
Aaron Perez166138ccdb30
diff --git a/skyvern-frontend/src/routes/discover/WorkflowTemplates.test.tsx b/skyvern-frontend/src/routes/discover/WorkflowTemplates.test.tsxindex 33edf6535..5e3d4ce95 100644--- a/skyvern-frontend/src/routes/discover/WorkflowTemplates.test.tsx+++ b/skyvern-frontend/src/routes/discover/WorkflowTemplates.test.tsx@@ -15,6 +15,10 @@ const templates = [title: "Invoice Downloading",},{ workflow_permanent_id: "wpid_c", title: "County - Business Registration" },+ ...Array.from({ length: 11 }, (_, index) => ({+ workflow_permanent_id: `wpid_template_${index}`,+ title: `Template ${index + 1}`,+ })),];vi.mock("../workflows/hooks/useGlobalWorkflowsQuery", () => ({@@ -100,4 +104,38 @@ describe("WorkflowTemplates", () => {"County - Business Registration",]);});++ it("expands all templates and collapses back to six", () => {+ render(+ <MemoryRouter>+ <WorkflowTemplates />+ </MemoryRouter>,+ );++ const cards = screen.getAllByRole("link");+ const isHiddenAtMd = (card: Element) =>+ card.classList.contains("md:hidden");+ const expandButton = screen.getByRole("button", { name: "See all 15" });++ expect(cards).toHaveLength(15);+ expect(expandButton.getAttribute("aria-expanded")).toBe("false");+ expect(cards.slice(0, 6).every((card) => !isHiddenAtMd(card))).toBe(true);+ expect(cards.slice(6).every(isHiddenAtMd)).toBe(true);++ fireEvent.click(expandButton);++ const collapseButton = screen.getByRole("button", { name: "Show fewer" });+ expect(collapseButton.getAttribute("aria-expanded")).toBe("true");+ expect(cards.every((card) => !isHiddenAtMd(card))).toBe(true);++ fireEvent.click(collapseButton);++ expect(+ screen+ .getByRole("button", { name: "See all 15" })+ .getAttribute("aria-expanded"),+ ).toBe("false");+ expect(cards.slice(0, 6).every((card) => !isHiddenAtMd(card))).toBe(true);+ expect(cards.slice(6).every(isHiddenAtMd)).toBe(true);+ });});diff --git a/skyvern-frontend/src/routes/tasks/create/CapabilityExamples.tsx b/skyvern-frontend/src/routes/tasks/create/CapabilityExamples.tsxindex b58a6c5a5..57ceaca82 100644--- a/skyvern-frontend/src/routes/tasks/create/CapabilityExamples.tsx+++ b/skyvern-frontend/src/routes/tasks/create/CapabilityExamples.tsx@@ -20,9 +20,26 @@ import {import "./capabilityExamples.css";+const SAMPLE_RESUME_PATH = "/sample-resume.pdf";++// Used when attachments can't reach the agent; an external URL is reachable from any+// execution browser, unlike the UI's own origin in self-hosted or protected deploys.+const SAMPLE_RESUME_PUBLIC_URL =+ "https://writing.colostate.edu/guides/documents/resume/functionalSample.pdf";++const JOB_APPLICATION_PROMPT =+ "Go to https://jobs.lever.co/leverdemo-8/45d39614-464a-4b62-a5cd-8683ce4fb80a/apply and fill out the job application using the attached resume. Upload the resume to the resume field, and fill every other field with information from the resume. Leave a field blank if the resume doesn't cover it; don't make anything up. For voluntary self-identification questions (gender, race, veteran, disability), choose \"Decline to self-identify\" or the closest option. Do not click Submit. Stop once the form is filled and list the fields you completed.";+type Capability = "forms" | "extract" | "login" | "monitor" | "schedule";-type Example = { id: string; label: string; prompt: string; icon: ReactNode };+type Example = {+ id: string;+ label: string;+ prompt: string;+ icon: ReactNode;+ // Public path of a file to attach alongside the prompt.+ attachment?: string;+};type Group = {capability: Capability;@@ -79,8 +96,8 @@ const GROUPS: ReadonlyArray<Group> = [id: "forms.apply_for_job",label: "Apply for a job",icon: <EnvelopeClosedIcon />,- prompt:- "Go to https://jobs.lever.co/leverdemo-8, find the first Solutions Engineer role in Toronto, and apply with the attached resume. Fill in the name and email, submit, and confirm the application went through.",+ prompt: JOB_APPLICATION_PROMPT,+ attachment: SAMPLE_RESUME_PATH,},{id: "forms.get_quote",@@ -280,12 +297,13 @@ type Selection = {capability: Capability;label: string;prompt: string;+ attachment?: string;};type Props = {disabled?: boolean;onSelect: (selection: Selection) => void;- onPreview?: (selection: Omit<Selection, "prompt">) => void;+ onPreview?: (selection: Omit<Selection, "prompt" | "attachment">) => void;};function CapabilityExamples({ disabled = false, onSelect, onPreview }: Props) {@@ -328,6 +346,7 @@ function CapabilityExamples({ disabled = false, onSelect, onPreview }: Props) {capability: group.capability,label: example.label,prompt: example.prompt,+ attachment: example.attachment,})}className="flex h-12 w-full items-center gap-2.5 rounded-[10px] border border-transparent bg-slate-elevation2 pl-4 pr-5 text-left text-sm text-foreground transition-colors hover:border-border hover:bg-slate-elevation3 disabled:pointer-events-none disabled:opacity-50 [&>svg]:size-[22px] [&>svg]:shrink-0"@@ -357,4 +376,9 @@ function CapabilityExamples({ disabled = false, onSelect, onPreview }: Props) {);}-export { CapabilityExamples };+export {+ CapabilityExamples,+ JOB_APPLICATION_PROMPT,+ SAMPLE_RESUME_PATH,+ SAMPLE_RESUME_PUBLIC_URL,+};diff --git a/skyvern-frontend/src/routes/tasks/create/CyclingPlaceholderTextarea.tsx b/skyvern-frontend/src/routes/tasks/create/CyclingPlaceholderTextarea.tsxindex 82484a5f4..4b46f81f5 100644--- a/skyvern-frontend/src/routes/tasks/create/CyclingPlaceholderTextarea.tsx+++ b/skyvern-frontend/src/routes/tasks/create/CyclingPlaceholderTextarea.tsx@@ -6,7 +6,7 @@ import { useCyclingPlaceholder } from "./useCyclingPlaceholder";type Props = Omit<ComponentProps<typeof AutoResizingTextarea>,- "placeholder"+ "placeholder" | "aria-label"> & {cycling: boolean;};@@ -16,7 +16,12 @@ const CyclingPlaceholderTextarea = forwardRef<HTMLTextAreaElement, Props>(function CyclingPlaceholderTextarea({ cycling, ...props }, ref) {const placeholder = useCyclingPlaceholder(cycling);return (- <AutoResizingTextarea ref={ref} placeholder={placeholder} {...props} />+ <AutoResizingTextarea+ ref={ref}+ {...props}+ aria-label="Prompt"+ placeholder={placeholder}+ />);},);diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsxindex 8d1ad4e7a..42ceb7ba9 100644--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx@@ -111,10 +111,9 @@ vi.mock("./CyclingPlaceholderTextarea", async () => {CyclingPlaceholderTextarea: React.forwardRef<HTMLTextAreaElement,TextareaHTMLAttributes<HTMLTextAreaElement> & { cycling?: boolean }- >(({ cycling, ...props }, ref) => {- void cycling;- return <textarea ref={ref} {...props} />;- }),+ >(({ cycling, ...props }, ref) => (+ <textarea ref={ref} data-cycling={cycling} {...props} />+ )),};});@@ -245,9 +244,15 @@ afterEach(() => {vi.unstubAllGlobals();});+function promptInput() {+ return document.getElementById(+ "discover-prompt-input",+ ) as HTMLTextAreaElement;+}+async function submitPrompt(text: string) {renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: text },});fireEvent.click(screen.getByLabelText("submit-prompt"));@@ -255,6 +260,20 @@ async function submitPrompt(text: string) {}describe("PromptBox", () => {+ test.each([+ ["legacy", false],+ ["redesigned", true],+ ])(+ "stops cycling example placeholders once the prompt is focused (%s)",+ (_label, minimal) => {+ renderPromptBox(false, undefined, minimal);+ const textarea = promptInput();+ expect(textarea.dataset.cycling).toBe("true");+ fireEvent.focus(textarea);+ expect(textarea.dataset.cycling).toBe("false");+ },+ );+test.each([["legacy", false, false, "RESIDENTIAL"],["redesigned", false, true, "RESIDENTIAL"],@@ -301,7 +320,7 @@ describe("PromptBox", () => {);renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Start" },});await act(async () => undefined);@@ -340,7 +359,7 @@ describe("PromptBox", () => {.mockResolvedValueOnce({ data: {} });renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "First identity" },});await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(1));@@ -349,7 +368,7 @@ describe("PromptBox", () => {authState.userId = "user-b";currentOrgState.organizationId = "org-b";renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Second identity" },});await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(2));@@ -372,7 +391,7 @@ describe("PromptBox", () => {.mockResolvedValueOnce({ data: {} });renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "First mount" },});await act(async () => undefined);@@ -383,7 +402,7 @@ describe("PromptBox", () => {await vi.advanceTimersByTimeAsync(PREWARM_DISPATCH_WAIT_TIMEOUT_MS);});renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Later mount" },});await act(async () => undefined);@@ -401,7 +420,7 @@ describe("PromptBox", () => {mockPost.mockResolvedValue({ data: {} });const ref = createRef<PromptBoxHandle>();renderPromptBox(false, ref);- const textarea = screen.getByPlaceholderText("Enter your prompt...");+ const textarea = promptInput();textarea.scrollIntoView = vi.fn();act(() =>@@ -481,7 +500,7 @@ describe("PromptBox", () => {});renderPromptBox();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Visit the docs" },});fireEvent.click(screen.getByLabelText("submit-prompt"));@@ -584,9 +603,7 @@ describe("PromptBox", () => {fireEvent.click(card);expect(card.getAttribute("aria-pressed")).toBe("true");- const textarea = screen.getByPlaceholderText(- "Enter your prompt...",- ) as HTMLTextAreaElement;+ const textarea = promptInput() as HTMLTextAreaElement;fireEvent.change(textarea, { target: { value: `${textarea.value} now` } });expect(card.getAttribute("aria-pressed")).toBe("false");});@@ -712,9 +729,7 @@ describe("PromptBox", () => {test("reuses the shipped sample prompts for onboarding intent keys", () => {const ref = createRef<PromptBoxHandle>();renderPromptBox(false, ref);- const textarea = screen.getByPlaceholderText(- "Enter your prompt...",- ) as HTMLTextAreaElement;+ const textarea = promptInput() as HTMLTextAreaElement;const cases = [["finditparts", "finditparts.com"],["contact_us_forms", "canadahvac.com/contact-hvac-canada"],@@ -742,7 +757,7 @@ describe("PromptBox", () => {expect(screen.queryByText("Skyvern 2.0")).toBeNull();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Visit the docs" },});fireEvent.click(screen.getByLabelText("submit-prompt"));@@ -811,7 +826,7 @@ describe("PromptBox", () => {expect(await screen.findByText("Max steps: 10")).toBeTruthy();expect(screen.getByLabelText("Advanced settings, 1 changed")).toBeTruthy();- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Visit the docs" },});fireEvent.click(screen.getByLabelText("submit-prompt"));@@ -878,6 +893,46 @@ describe("PromptBox", () => {expect(mockSetAutoplay).not.toHaveBeenCalled();});+ test("holds submit until the job example's sample resume is attached, then hands it off", async () => {+ let resolveFetch!: (response: Response) => void;+ vi.stubGlobal(+ "fetch",+ vi.fn(() => new Promise<Response>((resolve) => (resolveFetch = resolve))),+ );+ mockPost.mockImplementation((path: string) =>+ Promise.resolve({+ data:+ path === "/upload_file"+ ? { file_id: "file_resume" }+ : {+ workflow_permanent_id: "wpid_job",+ workflow_definition: { blocks: [] },+ },+ }),+ );++ renderPromptBox(true);+ fireEvent.click(screen.getByRole("button", { name: "Apply for a job" }));+ const submit = screen.getByLabelText("submit-prompt") as HTMLButtonElement;+ expect(submit.disabled).toBe(true);++ await waitFor(() =>+ expect(fetch).toHaveBeenCalledWith("/sample-resume.pdf"),+ );+ expect(submit.disabled).toBe(true);+ resolveFetch(new Response(new Blob(["%PDF"]), { status: 200 }));+ await waitFor(() => expect(submit.disabled).toBe(false));+ fireEvent.click(submit);++ await waitFor(() => expect(mockNavigate).toHaveBeenCalled());+ expect(mockNavigate.mock.calls[0]![1].state.copilotAttachedFiles).toEqual([+ expect.objectContaining({+ file_id: "file_resume",+ filename: "sample-resume.pdf",+ }),+ ]);+ });+test("hands Discover prompts to workflow studio with recoverable prompt state", async () => {studioState.enabled = true;mockPost.mockResolvedValue({@@ -889,7 +944,7 @@ describe("PromptBox", () => {renderPromptBox(true);- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Build this in studio" },});fireEvent.click(screen.getByLabelText("submit-prompt"));@@ -928,7 +983,7 @@ describe("PromptBox", () => {});renderPromptBox(handoff);- fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {+ fireEvent.change(promptInput(), {target: { value: "Visit the docs" },});fireEvent.click(screen.getByLabelText("submit-prompt"));diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsxindex ad84f0089..aedcec328 100644--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx@@ -4,7 +4,6 @@ import { Createv2TaskRequest } from "@/api/types";import { stringify as convertToYAML } from "yaml";import { WorkflowCreateYAMLRequest } from "@/routes/workflows/types/workflowYamlTypes";import img from "@/assets/promptBoxBg.png";-import { AutoResizingTextarea } from "@/components/AutoResizingTextarea/AutoResizingTextarea";import { CartIcon } from "@/components/icons/CartIcon";import { GraphIcon } from "@/components/icons/GraphIcon";import { InboxIcon } from "@/components/icons/InboxIcon";@@ -55,10 +54,11 @@ import {} from "react";import { Link, useNavigate } from "react-router-dom";import {- generatePhoneNumber,- generateUniqueEmail,-} from "../data/sampleTaskData";-import { CapabilityExamples } from "./CapabilityExamples";+ CapabilityExamples,+ JOB_APPLICATION_PROMPT,+ SAMPLE_RESUME_PATH,+ SAMPLE_RESUME_PUBLIC_URL,+} from "./CapabilityExamples";import { ExampleCasePill } from "./ExampleCasePill";import { CyclingPlaceholderTextarea } from "./CyclingPlaceholderTextarea";import {@@ -87,7 +87,8 @@ const exampleCases = [key: "job_application",hint: "jobs.lever.co",label: "Apply for a job",- prompt: `Go to https://jobs.lever.co/leverdemo-8/45d39614-464a-4b62-a5cd-8683ce4fb80a/apply, fill out the job application form and apply to the job. Fill out any public burden questions if they appear in the form. Your goal is complete when the page says you've successfully applied to the job. Terminate if you are unable to apply successfully. Here's the user information: {"name":"John Doe","email":"${generateUniqueEmail()}","phone":"${generatePhoneNumber()}","resume_url":"https://writing.colostate.edu/guides/documents/resume/functionalSample.pdf","cover_letter":"Generate a compelling cover letter for me"}`,+ prompt: JOB_APPLICATION_PROMPT,+ attachment: SAMPLE_RESUME_PATH,icon: <InboxIcon className="size-6" />,},{@@ -323,7 +324,14 @@ function PromptBoxImpl(exampleCases[0];if (!prompt.trim()) {cancelSpeech();- setPrompt(selectedExample.prompt);+ setPrompt(+ withExampleAttachment(+ selectedExample.prompt,+ "attachment" in selectedExample+ ? selectedExample.attachment+ : undefined,+ ),+ );setExampleAttribution({ id: selectedExample.key, edited: false });}textareaRef.current?.scrollIntoView?.({ block: "center" });@@ -340,7 +348,17 @@ function PromptBoxImpl(}));const uploadDocumentMutation = useMutation({- mutationFn: async (file: File) => {+ mutationFn: async (source: File | string) => {+ let file = source;+ if (typeof file === "string") {+ const response = await fetch(file);+ if (!response.ok) {+ throw new Error(`Failed to load ${file} (${response.status})`);+ }+ file = new File([await response.blob()], file.split("/").pop()!, {+ type: "application/pdf",+ });+ }const client = await getClient(credentialGetter);const formData = new FormData();formData.append("file", file);@@ -367,12 +385,36 @@ function PromptBoxImpl();textareaRef.current?.focus();},- onError: (error: AxiosError) => {+ onError: (error: unknown) => {HomeTelemetry.uploadDocumentFinished(false);showCreateErrorToast("Failed to upload file", error);},});+ // Returns the prompt to load. Attachments only reach the copilot handoff path,+ // so the plain task path gets the file's public URL in the prompt instead.+ const withExampleAttachment = (prompt: string, path?: string) => {+ const filename = path?.split("/").pop();+ if (!path || !filename) return prompt;+ if (!enableCopilotHandoff) {+ return `${prompt} The attached resume is at ${SAMPLE_RESUME_PUBLIC_URL}; download it from there.`;+ }+ if (attachedFiles.length >= MAX_HOME_ATTACHMENTS) {+ toast({+ variant: "destructive",+ title: "Too many attachments",+ description: "Remove an attachment to add the example's resume.",+ });+ return prompt;+ }+ // Starting the upload here (not after a fetch) flips isPending before the next render,+ // which disables submit and the example buttons until the resume is attached.+ if (!attachedFiles.some((file) => file.filename === filename)) {+ uploadDocumentMutation.mutate(path);+ }+ return prompt;+ };+const recordTaskMutation = useMutation({mutationFn: async () => {const client = await getClient(credentialGetter);@@ -786,14 +828,15 @@ function PromptBoxImpl(</div><div className="mt-6 flex w-full max-w-[45rem] flex-col md:mt-9"><div className="flex w-full flex-col rounded-2xl border border-input bg-background text-muted-foreground shadow-[0_12px_32px_rgba(0,0,0,0.06)] transition-[border-color,box-shadow] focus-within:border-foreground/20 focus-within:shadow-[0_0_0_4px_rgba(79,70,229,0.10),0_12px_32px_rgba(0,0,0,0.06)] dark:bg-slate-elevation1 dark:shadow-[0_12px_32px_rgba(0,0,0,0.35)] dark:focus-within:shadow-[0_0_0_4px_rgba(165,180,252,0.14),0_12px_32px_rgba(0,0,0,0.35)]">- <AutoResizingTextarea+ <CyclingPlaceholderTextarearef={textareaRef}id="discover-prompt-input"className="max-h-[14rem] min-h-[6rem] resize-none overflow-y-auto border-0 bg-transparent px-5 pb-1.5 pt-[18px] text-base leading-6 text-foreground shadow-none placeholder:text-muted-foreground hover:border-0 focus-visible:ring-0 md:text-[15px]"value={prompt}onChange={(e) => updatePrompt(e.target.value)}onKeyDown={handlePromptKeyDown}- placeholder="Enter your prompt..."+ onFocus={() => setPromptTouched(true)}+ cycling={!promptTouched}/>{enableCopilotHandoff ? (<div className="px-5">{attachmentChips}</div>@@ -813,6 +856,7 @@ function PromptBoxImpl(disabled={isSubmitting}onToggle={() => {HomeTelemetry.voiceToggled();+ setPromptTouched(true);toggleSpeech();}}className="size-11 rounded-full border-0 bg-transparent md:size-9"@@ -889,7 +933,12 @@ function PromptBoxImpl(label: example.label,});cancelSpeech();- setPrompt(example.prompt);+ setPrompt(+ withExampleAttachment(+ example.prompt,+ "attachment" in example ? example.attachment : undefined,+ ),+ );setExampleAttribution({ id: example.key, edited: false });textareaRef.current?.focus();}}@@ -960,6 +1009,7 @@ function PromptBoxImpl(disabled={isSubmitting}onToggle={() => {HomeTelemetry.voiceToggled();+ setPromptTouched(true);toggleSpeech();}}className="h-8 w-8 rounded-full border-0 bg-transparent"@@ -995,7 +1045,9 @@ function PromptBoxImpl(label: example.label,});cancelSpeech();- setPrompt(example.prompt);+ setPrompt(+ withExampleAttachment(example.prompt, example.attachment),+ );setExampleAttribution({ id: example.id, edited: false });setPromptTouched(true);textareaRef.current?.focus();diff --git a/skyvern/cli/skills/testing/SKILL.md b/skyvern/cli/skills/testing/SKILL.mdindex 0b87fb0b3..6f3d105a2 100644--- a/skyvern/cli/skills/testing/SKILL.md+++ b/skyvern/cli/skills/testing/SKILL.md@@ -41,7 +41,7 @@ No captured response means the backend is down.```skyvern_navigate(url="{{base_url}}/discover")skyvern_validate(prompt="The page does NOT show any error messages, error toasts, 'Something went wrong', a persistent loading spinner, a blank white screen, or a connection refused message")-skyvern_validate(prompt="The page shows 'What task would you like to accomplish?' as a heading, a text input area with 'Enter your prompt...' placeholder, an engine version selector, and a send/submit button icon")+skyvern_validate(prompt="The page shows 'What task would you like to accomplish?' as a heading, a prompt text input area, an engine version selector, and a send/submit button icon")skyvern_screenshot()skyvern_browser_session_close()```