Skip to content

Automate browser based workflows with AI

Validate the create-from-prompt response shape at the boundary (SKY-13154) (#7826)

Aaron Perez6d0d31af7ea6
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
index 77a8a921b..254f99676 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
@@ -15,6 +15,8 @@ import type {
} from "react";
import { afterEach, describe, expect, test, vi } from "vitest";
+import { toast } from "@/components/ui/use-toast";
+
import { PromptBox } from "./PromptBox";
const { mockNavigate, mockPost, mockSetAutoplay } = vi.hoisted(() => ({
@@ -149,8 +151,18 @@ afterEach(() => {
mockNavigate.mockReset();
mockPost.mockReset();
mockSetAutoplay.mockReset();
+ vi.mocked(toast).mockReset();
});
+async function submitPrompt(text: string) {
+ renderPromptBox();
+ fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {
+ target: { value: text },
+ });
+ fireEvent.click(screen.getByLabelText("submit-prompt"));
+ await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(1));
+}
+
describe("PromptBox", () => {
test("creates prompt-generated workflows as V1 agent runs", async () => {
mockPost.mockResolvedValue({
@@ -180,6 +192,62 @@ describe("PromptBox", () => {
expect(body.request.url).toBe("https://google.com");
});
+ // SKY-13154: a 2xx whose body fails JSON.parse arrives as a raw string, so
+ // `data.workflow_definition` was undefined and onSuccess threw a TypeError
+ // after the success toast had already fired.
+ test("rejects a 2xx whose body is not a workflow, without leaking the body", async () => {
+ const interstitial = "<html><body>Sign in to continue</body></html>";
+ mockPost.mockResolvedValue({
+ status: 200,
+ headers: { "content-type": "text/html" },
+ data: interstitial,
+ });
+
+ await submitPrompt("Visit the docs");
+
+ await waitFor(() =>
+ expect(vi.mocked(toast)).toHaveBeenCalledWith(
+ expect.objectContaining({ variant: "destructive" }),
+ ),
+ );
+
+ const descriptions = vi
+ .mocked(toast)
+ .mock.calls.map(([args]) => String(args.description ?? ""));
+ const diagnostic = descriptions.join(" ");
+
+ expect(diagnostic).toContain("status=200");
+ expect(diagnostic).toContain("content_type=text/html");
+ expect(diagnostic).toContain("parsed_as_json=false");
+ expect(diagnostic).toContain(`body_length=${interstitial.length}`);
+ // The body may be an auth interstitial or carry customer data.
+ expect(diagnostic).not.toContain("Sign in to continue");
+
+ expect(
+ vi.mocked(toast).mock.calls.some(([args]) => args.variant === "success"),
+ ).toBe(false);
+ expect(mockNavigate).not.toHaveBeenCalled();
+ });
+
+ // The agent exists server-side by this point, so an absent first block must
+ // not keep the user on /discover.
+ test("navigates to the new agent even when it has no blocks", async () => {
+ mockPost.mockResolvedValue({
+ status: 200,
+ headers: { "content-type": "application/json" },
+ data: {
+ workflow_permanent_id: "wpid_empty",
+ workflow_definition: { blocks: [] },
+ },
+ });
+
+ await submitPrompt("Visit the docs");
+
+ await waitFor(() => expect(mockNavigate).toHaveBeenCalledTimes(1));
+ expect(mockNavigate).toHaveBeenCalledWith("/agents/wpid_empty/build");
+ expect(mockSetAutoplay).not.toHaveBeenCalled();
+ });
+
test("hands Discover prompts to the legacy build path via route state only", async () => {
mockPost.mockResolvedValue({
data: {
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
index 54f76ef8b..db5d36789 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
@@ -26,7 +26,7 @@ import {
ReloadIcon,
} from "@radix-ui/react-icons";
import { useMutation, useQueryClient } from "@tanstack/react-query";
-import { AxiosError } from "axios";
+import { AxiosError, type AxiosResponse } from "axios";
import { useRef, useState } from "react";
import { useNavigate } from "react-router-dom";
import {
@@ -140,6 +140,35 @@ function buildBlankWorkflowRequest(
};
}
+function hasWorkflowShape(data: unknown): data is WorkflowApiResponse {
+ if (typeof data !== "object" || data === null) {
+ return false;
+ }
+ const candidate = data as Partial<WorkflowApiResponse>;
+ const definition = candidate.workflow_definition;
+ return (
+ typeof candidate.workflow_permanent_id === "string" &&
+ typeof definition === "object" &&
+ definition !== null &&
+ Array.isArray(definition.blocks)
+ );
+}
+
+// Axios yields a raw string for any 2xx whose body fails JSON.parse, and
+// `"".workflow_definition` is undefined — so an empty body or an edge/auth
+// interstitial reaches onSuccess looking like a workflow. Describe the
+// envelope only; the body may be an interstitial or carry customer data.
+function describeResponseEnvelope(response: AxiosResponse<unknown>): string {
+ const body: unknown = response.data;
+ const parsedAsJson = typeof body === "object" && body !== null;
+ const bodyLength =
+ typeof body === "string"
+ ? body.length
+ : JSON.stringify(body ?? null).length;
+ const contentType = String(response.headers?.["content-type"] ?? "unknown");
+ return `status=${response.status} content_type=${contentType} body_length=${bodyLength} parsed_as_json=${parsedAsJson}`;
+}
+
function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
const navigate = useNavigate();
const studioEnabled = useWorkflowStudioEnabled();
@@ -208,7 +237,7 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
const trimmedMaxStepsOverride = maxStepsOverride?.trim();
const result = await client.post<
Createv2TaskRequest,
- { data: WorkflowApiResponse }
+ AxiosResponse<unknown>
>(
"/workflows/create-from-prompt",
{
@@ -220,9 +249,15 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
: undefined,
);
- return result;
+ if (!hasWorkflowShape(result.data)) {
+ throw new Error(
+ `create-from-prompt returned an unexpected response shape (${describeResponseEnvelope(result)})`,
+ );
+ }
+
+ return result.data;
},
- onSuccess: ({ data: workflow }) => {
+ onSuccess: (workflow) => {
toast({
variant: "success",
title: "Agent Created",
@@ -233,6 +268,8 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
queryKey: ["workflows"],
});
+ // The agent already exists server-side, so nothing between here and
+ // navigate() may strand the user on /discover.
const firstBlock = workflow.workflow_definition.blocks[0];
if (firstBlock) {
@@ -243,7 +280,7 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
workflowEditorPath(workflow.workflow_permanent_id, studioEnabled),
);
},
- onError: (error: AxiosError) => {
+ onError: (error: Error) => {
toast({
variant: "destructive",
title: "Error creating agent from prompt",