Skip to content

Automate browser based workflows with AI

feat(frontend): add speech-to-text mic for prompt box and workflow copilot (#6395)

Cindy Li30f8ccfa20d3
diff --git a/skyvern-frontend/src/components/SpeechInputButton.tsx b/skyvern-frontend/src/components/SpeechInputButton.tsx
new file mode 100644
index 000000000..6a6c6125a
--- /dev/null
+++ b/skyvern-frontend/src/components/SpeechInputButton.tsx
@@ -0,0 +1,88 @@
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { cn } from "@/util/utils";
+
+function MicIcon({ className }: { className?: string }) {
+ return (
+ <svg
+ xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 15 15"
+ fill="currentColor"
+ className={className}
+ aria-hidden="true"
+ >
+ <path
+ fillRule="evenodd"
+ clipRule="evenodd"
+ d="M7.5 1.25a2.25 2.25 0 0 0-2.25 2.25v3.5a2.25 2.25 0 0 0 4.5 0V3.5A2.25 2.25 0 0 0 7.5 1.25Zm-3.75 5.25a3.75 3.75 0 0 0 7.5 0H10a4.75 4.75 0 0 1-9.5 0H3.75ZM7.5 10.25a.75.75 0 0 1 .75.75v1.5h1.25a.75.75 0 0 1 0 1.5H5.5a.75.75 0 0 1 0-1.5h1.25v-1.5A.75.75 0 0 1 7.5 10.25Z"
+ />
+ </svg>
+ );
+}
+
+type SpeechInputButtonProps = {
+ isSupported: boolean;
+ isListening: boolean;
+ isHearingSpeech: boolean;
+ disabled?: boolean;
+ onToggle: () => void;
+ className?: string;
+ iconClassName?: string;
+};
+
+function SpeechInputButton({
+ isSupported,
+ isListening,
+ isHearingSpeech,
+ disabled = false,
+ onToggle,
+ className,
+ iconClassName = "h-4 w-4",
+}: SpeechInputButtonProps) {
+ const isDisabled = disabled || !isSupported;
+ const tooltipLabel = !isSupported
+ ? "Voice input isn't supported in this browser. Try Chrome or Edge."
+ : isListening
+ ? "Stop dictating"
+ : "Dictate message";
+
+ const button = (
+ <button
+ type="button"
+ onClick={onToggle}
+ disabled={isDisabled}
+ aria-label={isListening ? "Stop dictating" : "Dictate message"}
+ aria-pressed={isListening}
+ className={cn(
+ "flex shrink-0 items-center justify-center rounded-md border",
+ isListening
+ ? "border-destructive bg-destructive/10 text-destructive"
+ : "border-input bg-slate-elevation2 text-muted-foreground hover:bg-accent hover:text-accent-foreground",
+ isListening &&
+ isHearingSpeech &&
+ "animate-pulse ring-2 ring-destructive/50",
+ "disabled:cursor-not-allowed disabled:opacity-50",
+ className,
+ )}
+ >
+ <MicIcon className={iconClassName} />
+ </button>
+ );
+
+ return (
+ <TooltipProvider>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ {isDisabled ? <span className="inline-flex">{button}</span> : button}
+ </TooltipTrigger>
+ <TooltipContent>{tooltipLabel}</TooltipContent>
+ </Tooltip>
+ </TooltipProvider>
+ );
+}
+
+export { SpeechInputButton };
diff --git a/skyvern-frontend/src/hooks/useSpeechToText.test.ts b/skyvern-frontend/src/hooks/useSpeechToText.test.ts
new file mode 100644
index 000000000..35403117f
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useSpeechToText.test.ts
@@ -0,0 +1,432 @@
+import { act, renderHook } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+ isSpeechRecognitionSupported,
+ useSpeechToText,
+} from "./useSpeechToText";
+
+type ResultHandler = (event: {
+ resultIndex: number;
+ results: Array<{ isFinal: boolean; transcript: string }>;
+}) => void;
+
+type ErrorHandler = (event: { error: string; message?: string }) => void;
+
+class MockSpeechRecognition {
+ continuous = false;
+ interimResults = false;
+ lang = "";
+ onresult: ResultHandler | null = null;
+ onerror: ErrorHandler | null = null;
+ onend: (() => void) | null = null;
+
+ private resultList: Array<{ isFinal: boolean; transcript: string }> = [];
+
+ start = vi.fn(() => {
+ MockSpeechRecognition.lastInstance = this;
+ this.resultList = [];
+ });
+
+ stop = vi.fn(() => {
+ this.onend?.();
+ });
+
+ abort = vi.fn(() => {
+ this.onend?.();
+ });
+
+ static lastInstance: MockSpeechRecognition | null = null;
+
+ emitResult(transcript: string, isFinal = true, resultIndex?: number) {
+ const index = resultIndex ?? this.resultList.length;
+ this.resultList[index] = { isFinal, transcript };
+
+ const results = {
+ length: this.resultList.length,
+ ...Object.fromEntries(
+ this.resultList.map((entry, i) => [
+ i,
+ {
+ length: 1,
+ isFinal: entry.isFinal,
+ 0: { transcript: entry.transcript },
+ },
+ ]),
+ ),
+ };
+
+ this.onresult?.({
+ resultIndex: index,
+ results: results as never,
+ });
+ }
+
+ emitError(error: string) {
+ this.onerror?.({ error });
+ }
+
+ emitEnd() {
+ this.onend?.();
+ }
+}
+
+describe("isSpeechRecognitionSupported", () => {
+ afterEach(() => {
+ delete (window as { SpeechRecognition?: unknown }).SpeechRecognition;
+ delete (window as { webkitSpeechRecognition?: unknown })
+ .webkitSpeechRecognition;
+ });
+
+ it("returns true when SpeechRecognition exists", () => {
+ (
+ window as { SpeechRecognition?: typeof MockSpeechRecognition }
+ ).SpeechRecognition = MockSpeechRecognition;
+ expect(isSpeechRecognitionSupported()).toBe(true);
+ });
+
+ it("returns false when SpeechRecognition is missing", () => {
+ expect(isSpeechRecognitionSupported()).toBe(false);
+ });
+});
+
+describe("useSpeechToText", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ MockSpeechRecognition.lastInstance = null;
+ (
+ window as { SpeechRecognition?: typeof MockSpeechRecognition }
+ ).SpeechRecognition = MockSpeechRecognition;
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ delete (window as { SpeechRecognition?: unknown }).SpeechRecognition;
+ delete (window as { webkitSpeechRecognition?: unknown })
+ .webkitSpeechRecognition;
+ });
+
+ it("reports support from the browser API", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+ expect(result.current.isSupported).toBe(true);
+ });
+
+ it("starts and stops recognition", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ expect(result.current.isListening).toBe(true);
+ expect(MockSpeechRecognition.lastInstance?.start).toHaveBeenCalled();
+ expect(MockSpeechRecognition.lastInstance?.continuous).toBe(true);
+ expect(MockSpeechRecognition.lastInstance?.interimResults).toBe(true);
+
+ act(() => {
+ result.current.stop();
+ });
+
+ expect(result.current.isListening).toBe(false);
+ expect(MockSpeechRecognition.lastInstance?.stop).toHaveBeenCalled();
+ });
+
+ it("toggles listening on and off", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+
+ act(() => {
+ result.current.toggle();
+ });
+ expect(result.current.isListening).toBe(true);
+
+ act(() => {
+ result.current.toggle();
+ });
+ expect(result.current.isListening).toBe(false);
+ });
+
+ it("appends final transcript chunks with spacing", () => {
+ const onTranscript = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ getBaseText: () => "existing prompt",
+ onTranscript,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("hello world");
+ });
+ expect(onTranscript).toHaveBeenLastCalledWith(
+ "existing prompt hello world",
+ );
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("more text", true, 1);
+ });
+ expect(onTranscript).toHaveBeenLastCalledWith(
+ "existing prompt hello world more text",
+ );
+ });
+
+ it("starts from an empty base text", () => {
+ const onTranscript = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ getBaseText: () => "",
+ onTranscript,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("first phrase");
+ });
+ expect(onTranscript).toHaveBeenLastCalledWith("first phrase");
+ });
+
+ it("surfaces permission errors", () => {
+ const onError = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onError,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitError("not-allowed");
+ });
+
+ expect(onError).toHaveBeenCalledWith(
+ "Microphone access was denied. Allow microphone permission and try again.",
+ );
+ expect(result.current.isListening).toBe(false);
+ });
+
+ it("stops when enabled becomes false", () => {
+ const { result, rerender } = renderHook(
+ ({ enabled }: { enabled: boolean }) =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ enabled,
+ }),
+ { initialProps: { enabled: true } },
+ );
+
+ act(() => {
+ result.current.start();
+ });
+ expect(result.current.isListening).toBe(true);
+
+ rerender({ enabled: false });
+
+ expect(result.current.isListening).toBe(false);
+ });
+
+ it("cleans up active recognition on unmount", () => {
+ const { result, unmount } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ const instance = MockSpeechRecognition.lastInstance;
+ unmount();
+
+ expect(instance?.abort).toHaveBeenCalled();
+ });
+
+ it("updates transcript with interim results", () => {
+ const onTranscript = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ getBaseText: () => "base",
+ onTranscript,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("hel", false);
+ });
+ expect(onTranscript).toHaveBeenLastCalledWith("base hel");
+ });
+
+ it("pulses isHearingSpeech while interim speech is detected", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("hel", false);
+ });
+ expect(result.current.isHearingSpeech).toBe(true);
+
+ act(() => {
+ vi.advanceTimersByTime(400);
+ });
+ expect(result.current.isHearingSpeech).toBe(false);
+ });
+
+ it("clears isHearingSpeech when dictation stops", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("hel", false);
+ });
+ expect(result.current.isHearingSpeech).toBe(true);
+
+ act(() => {
+ result.current.stop();
+ });
+ expect(result.current.isHearingSpeech).toBe(false);
+ });
+
+ it("restarts recognition when the browser ends the session while listening", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({ onTranscript: vi.fn() }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ const instance = MockSpeechRecognition.lastInstance;
+ expect(instance?.start).toHaveBeenCalledTimes(1);
+
+ act(() => {
+ instance?.emitEnd();
+ });
+
+ expect(instance?.start).toHaveBeenCalledTimes(1);
+
+ act(() => {
+ vi.advanceTimersByTime(300);
+ });
+
+ expect(instance?.start).toHaveBeenCalledTimes(2);
+ expect(result.current.isListening).toBe(true);
+ });
+
+ it("stops listening after too many rapid auto-restarts", () => {
+ const onError = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onError,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ const instance = MockSpeechRecognition.lastInstance;
+ expect(instance).not.toBeNull();
+
+ for (let i = 0; i < 9; i += 1) {
+ act(() => {
+ instance?.emitEnd();
+ vi.advanceTimersByTime(300);
+ });
+ }
+
+ expect(result.current.isListening).toBe(false);
+ expect(onError).toHaveBeenCalledWith(
+ "Voice input stopped after repeated failures. Try again.",
+ );
+ });
+
+ it("ignores no-speech errors while listening", () => {
+ const onError = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onError,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitError("no-speech");
+ });
+
+ expect(onError).not.toHaveBeenCalled();
+ expect(result.current.isListening).toBe(true);
+ });
+
+ it("does not start when enabled is false", () => {
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ enabled: false,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ expect(result.current.isListening).toBe(false);
+ expect(MockSpeechRecognition.lastInstance).toBeNull();
+ });
+
+ it("uses cached finalized chunks for results before resultIndex", () => {
+ const onTranscript = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ getBaseText: () => "",
+ onTranscript,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("hello world");
+ });
+ act(() => {
+ MockSpeechRecognition.lastInstance?.emitResult("more text", true, 1);
+ });
+
+ expect(onTranscript).toHaveBeenLastCalledWith("hello world more text");
+ });
+});
diff --git a/skyvern-frontend/src/hooks/useSpeechToText.ts b/skyvern-frontend/src/hooks/useSpeechToText.ts
new file mode 100644
index 000000000..811ecfb4e
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useSpeechToText.ts
@@ -0,0 +1,414 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+
+interface SpeechRecognitionResultItem {
+ transcript: string;
+}
+
+interface SpeechRecognitionResultList {
+ readonly length: number;
+ [index: number]: {
+ readonly length: number;
+ [index: number]: SpeechRecognitionResultItem;
+ isFinal: boolean;
+ };
+}
+
+interface SpeechRecognitionEventLike extends Event {
+ resultIndex: number;
+ results: SpeechRecognitionResultList;
+}
+
+interface SpeechRecognitionErrorEventLike extends Event {
+ error: string;
+ message?: string;
+}
+
+interface SpeechRecognitionLike extends EventTarget {
+ continuous: boolean;
+ interimResults: boolean;
+ lang: string;
+ onresult: ((event: SpeechRecognitionEventLike) => void) | null;
+ onerror: ((event: SpeechRecognitionErrorEventLike) => void) | null;
+ onend: (() => void) | null;
+ start: () => void;
+ stop: () => void;
+ abort: () => void;
+}
+
+type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
+
+declare global {
+ interface Window {
+ SpeechRecognition?: SpeechRecognitionConstructor;
+ webkitSpeechRecognition?: SpeechRecognitionConstructor;
+ }
+}
+
+const HEARING_PULSE_MS = 400;
+const MIN_RESTART_INTERVAL_MS = 300;
+const MAX_RAPID_RESTARTS = 8;
+
+function getSpeechRecognitionConstructor():
+ | SpeechRecognitionConstructor
+ | undefined {
+ if (typeof window === "undefined") {
+ return undefined;
+ }
+ return window.SpeechRecognition ?? window.webkitSpeechRecognition;
+}
+
+export function isSpeechRecognitionSupported(): boolean {
+ return getSpeechRecognitionConstructor() !== undefined;
+}
+
+function mergeTranscript(baseText: string, dictatedText: string): string {
+ const base = baseText.trimEnd();
+ const dictated = dictatedText.trim();
+ if (!dictated) {
+ return baseText;
+ }
+ if (!base) {
+ return dictated;
+ }
+ return `${base} ${dictated}`;
+}
+
+function buildSessionTranscript(
+ results: SpeechRecognitionResultList,
+ resultIndex: number,
+ finalizedChunks: Map<number, string>,
+): {
+ finalText: string;
+ interimText: string;
+} {
+ const finalParts: string[] = [];
+
+ for (let i = 0; i < resultIndex; i += 1) {
+ const cached = finalizedChunks.get(i);
+ if (cached) {
+ finalParts.push(cached);
+ }
+ }
+
+ const interimParts: string[] = [];
+ for (let i = resultIndex; i < results.length; i += 1) {
+ const result = results[i];
+ if (!result) {
+ continue;
+ }
+ const chunk = (result[0]?.transcript ?? "").trim();
+ if (!chunk) {
+ continue;
+ }
+ if (result.isFinal) {
+ finalParts.push(chunk);
+ finalizedChunks.set(i, chunk);
+ } else {
+ interimParts.push(chunk);
+ }
+ }
+ return {
+ finalText: finalParts.join(" "),
+ interimText: interimParts.join(" "),
+ };
+}
+
+function combineDictated(finalText: string, interimText: string): string {
+ if (finalText && interimText) {
+ return `${finalText} ${interimText}`;
+ }
+ return finalText || interimText;
+}
+
+export interface UseSpeechToTextOptions {
+ /** Snapshot of textarea content when dictation starts. */
+ getBaseText?: () => string;
+ onTranscript: (text: string) => void;
+ onError?: (message: string) => void;
+ lang?: string;
+ /** When false, active recognition is stopped (e.g. panel closed). */
+ enabled?: boolean;
+}
+
+export interface UseSpeechToTextResult {
+ isSupported: boolean;
+ isListening: boolean;
+ /** Briefly true while interim speech is being recognized. */
+ isHearingSpeech: boolean;
+ start: () => void;
+ stop: () => void;
+ toggle: () => void;
+}
+
+const ERROR_MESSAGES: Record<string, string> = {
+ "not-allowed":
+ "Microphone access was denied. Allow microphone permission and try again.",
+ "service-not-allowed":
+ "Microphone access was denied. Allow microphone permission and try again.",
+ network: "Voice input failed due to a network error. Try again.",
+ "audio-capture": "No microphone was found. Check your device and try again.",
+};
+
+function errorMessageForCode(error: string): string {
+ return ERROR_MESSAGES[error] ?? "Voice input failed. Try again.";
+}
+
+export function useSpeechToText(
+ options: UseSpeechToTextOptions,
+): UseSpeechToTextResult {
+ const { getBaseText, onTranscript, onError, lang, enabled = true } = options;
+
+ const isSupported = isSpeechRecognitionSupported();
+ const [isListening, setIsListening] = useState(false);
+ const [isHearingSpeech, setIsHearingSpeech] = useState(false);
+
+ const recognitionRef = useRef<SpeechRecognitionLike | null>(null);
+ const baseTextRef = useRef("");
+ const isListeningRef = useRef(false);
+ const shouldKeepListeningRef = useRef(false);
+ const enabledRef = useRef(enabled);
+ const finalizedChunksRef = useRef<Map<number, string>>(new Map());
+ const hearingTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+ const restartTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+ const lastStartAttemptAtRef = useRef(0);
+ const rapidRestartCountRef = useRef(0);
+
+ const onTranscriptRef = useRef(onTranscript);
+ const onErrorRef = useRef(onError);
+ const getBaseTextRef = useRef(getBaseText);
+
+ useEffect(() => {
+ onTranscriptRef.current = onTranscript;
+ onErrorRef.current = onError;
+ getBaseTextRef.current = getBaseText;
+ }, [onTranscript, onError, getBaseText]);
+
+ useEffect(() => {
+ enabledRef.current = enabled;
+ }, [enabled]);
+
+ const clearHearingSpeech = useCallback(() => {
+ if (hearingTimeoutRef.current !== null) {
+ clearTimeout(hearingTimeoutRef.current);
+ hearingTimeoutRef.current = null;
+ }
+ setIsHearingSpeech(false);
+ }, []);
+
+ const markHearingSpeech = useCallback(() => {
+ setIsHearingSpeech(true);
+ if (hearingTimeoutRef.current !== null) {
+ clearTimeout(hearingTimeoutRef.current);
+ }
+ hearingTimeoutRef.current = setTimeout(() => {
+ setIsHearingSpeech(false);
+ hearingTimeoutRef.current = null;
+ }, HEARING_PULSE_MS);
+ }, []);
+
+ const clearRestartTimeout = useCallback(() => {
+ if (restartTimeoutRef.current !== null) {
+ clearTimeout(restartTimeoutRef.current);
+ restartTimeoutRef.current = null;
+ }
+ }, []);
+
+ const teardownRecognition = useCallback(() => {
+ clearRestartTimeout();
+ const recognition = recognitionRef.current;
+ if (!recognition) {
+ return;
+ }
+ recognition.onresult = null;
+ recognition.onerror = null;
+ recognition.onend = null;
+ try {
+ recognition.abort();
+ } catch {
+ // ignore abort errors during cleanup
+ }
+ recognitionRef.current = null;
+ }, [clearRestartTimeout]);
+
+ const stop = useCallback(() => {
+ shouldKeepListeningRef.current = false;
+ isListeningRef.current = false;
+ setIsListening(false);
+ clearHearingSpeech();
+ clearRestartTimeout();
+ const recognition = recognitionRef.current;
+ if (recognition) {
+ try {
+ recognition.stop();
+ } catch {
+ teardownRecognition();
+ }
+ }
+ }, [clearHearingSpeech, clearRestartTimeout, teardownRecognition]);
+
+ const start = useCallback(() => {
+ if (!isSupported || !enabledRef.current || isListeningRef.current) {
+ return;
+ }
+
+ const SpeechRecognitionCtor = getSpeechRecognitionConstructor();
+ if (!SpeechRecognitionCtor) {
+ return;
+ }
+
+ teardownRecognition();
+ clearHearingSpeech();
+ finalizedChunksRef.current.clear();
+ rapidRestartCountRef.current = 0;
+
+ baseTextRef.current = getBaseTextRef.current?.() ?? "";
+
+ const recognition = new SpeechRecognitionCtor();
+ recognition.continuous = true;
+ recognition.interimResults = true;
+ recognition.lang =
+ lang ?? (typeof navigator !== "undefined" ? navigator.language : "en-US");
+
+ recognition.onresult = (event: SpeechRecognitionEventLike) => {
+ const { finalText, interimText } = buildSessionTranscript(
+ event.results,
+ event.resultIndex,
+ finalizedChunksRef.current,
+ );
+ const dictated = combineDictated(finalText, interimText);
+ onTranscriptRef.current(mergeTranscript(baseTextRef.current, dictated));
+ rapidRestartCountRef.current = 0;
+ if (interimText) {
+ markHearingSpeech();
+ }
+ };
+
+ recognition.onerror = (event: SpeechRecognitionErrorEventLike) => {
+ if (event.error === "aborted") {
+ return;
+ }
+ if (event.error === "no-speech" && shouldKeepListeningRef.current) {
+ return;
+ }
+ shouldKeepListeningRef.current = false;
+ isListeningRef.current = false;
+ setIsListening(false);
+ clearHearingSpeech();
+ onErrorRef.current?.(errorMessageForCode(event.error));
+ teardownRecognition();
+ };
+
+ const attemptAutoRestart = (): boolean => {
+ if (
+ !shouldKeepListeningRef.current ||
+ recognitionRef.current !== recognition
+ ) {
+ return false;
+ }
+
+ rapidRestartCountRef.current += 1;
+ if (rapidRestartCountRef.current > MAX_RAPID_RESTARTS) {
+ shouldKeepListeningRef.current = false;
+ isListeningRef.current = false;
+ setIsListening(false);
+ clearHearingSpeech();
+ recognitionRef.current = null;
+ onErrorRef.current?.(
+ "Voice input stopped after repeated failures. Try again.",
+ );
+ return false;
+ }
+
+ try {
+ lastStartAttemptAtRef.current = Date.now();
+ recognition.start();
+ return true;
+ } catch {
+ shouldKeepListeningRef.current = false;
+ return false;
+ }
+ };
+
+ recognition.onend = () => {
+ if (
+ shouldKeepListeningRef.current &&
+ recognitionRef.current === recognition
+ ) {
+ const elapsed = Date.now() - lastStartAttemptAtRef.current;
+ if (elapsed < MIN_RESTART_INTERVAL_MS) {
+ clearRestartTimeout();
+ restartTimeoutRef.current = setTimeout(() => {
+ restartTimeoutRef.current = null;
+ attemptAutoRestart();
+ }, MIN_RESTART_INTERVAL_MS - elapsed);
+ return;
+ }
+
+ if (attemptAutoRestart()) {
+ return;
+ }
+ }
+
+ isListeningRef.current = false;
+ setIsListening(false);
+ clearHearingSpeech();
+ recognitionRef.current = null;
+ };
+
+ recognitionRef.current = recognition;
+
+ try {
+ lastStartAttemptAtRef.current = Date.now();
+ recognition.start();
+ shouldKeepListeningRef.current = true;
+ isListeningRef.current = true;
+ setIsListening(true);
+ } catch {
+ shouldKeepListeningRef.current = false;
+ isListeningRef.current = false;
+ setIsListening(false);
+ clearHearingSpeech();
+ teardownRecognition();
+ onErrorRef.current?.("Voice input failed to start. Try again.");
+ }
+ }, [
+ clearHearingSpeech,
+ clearRestartTimeout,
+ isSupported,
+ lang,
+ markHearingSpeech,
+ teardownRecognition,
+ ]);
+
+ const toggle = useCallback(() => {
+ if (isListeningRef.current) {
+ stop();
+ } else {
+ start();
+ }
+ }, [start, stop]);
+
+ useEffect(() => {
+ if (!enabled && isListeningRef.current) {
+ stop();
+ }
+ }, [enabled, stop]);
+
+ useEffect(() => {
+ return () => {
+ shouldKeepListeningRef.current = false;
+ isListeningRef.current = false;
+ clearHearingSpeech();
+ teardownRecognition();
+ };
+ }, [clearHearingSpeech, teardownRecognition]);
+
+ return {
+ isSupported,
+ isListening,
+ isHearingSpeech,
+ start,
+ stop,
+ toggle,
+ };
+}
diff --git a/skyvern-frontend/src/hooks/useSpeechToTextField.ts b/skyvern-frontend/src/hooks/useSpeechToTextField.ts
new file mode 100644
index 000000000..6d41c622e
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useSpeechToTextField.ts
@@ -0,0 +1,34 @@
+import { useCallback } from "react";
+
+import { toast } from "@/components/ui/use-toast";
+
+import { useSpeechToText, type UseSpeechToTextResult } from "./useSpeechToText";
+
+type UseSpeechToTextFieldOptions = {
+ value: string;
+ onChange: (text: string) => void;
+ enabled?: boolean;
+};
+
+export function useSpeechToTextField({
+ value,
+ onChange,
+ enabled = true,
+}: UseSpeechToTextFieldOptions): UseSpeechToTextResult {
+ const handleSpeechError = useCallback((message: string) => {
+ toast({
+ title: "Voice input failed",
+ description: message,
+ variant: "destructive",
+ });
+ }, []);
+
+ const getBaseText = useCallback(() => value, [value]);
+
+ return useSpeechToText({
+ getBaseText,
+ onTranscript: onChange,
+ onError: handleSpeechError,
+ enabled,
+ });
+}
diff --git a/skyvern-frontend/src/routes/discover/DiscoverPage.tsx b/skyvern-frontend/src/routes/discover/DiscoverPage.tsx
index aa4ced68e..02fb8e33b 100644
--- a/skyvern-frontend/src/routes/discover/DiscoverPage.tsx
+++ b/skyvern-frontend/src/routes/discover/DiscoverPage.tsx
@@ -1,15 +1,14 @@
import { useFeatureFlag } from "@/hooks/useFeatureFlag";
import { PromptBox } from "../tasks/create/PromptBox";
import { WorkflowTemplates } from "./WorkflowTemplates";
-import { useCreateWorkflowMutation } from "../workflows/hooks/useCreateWorkflowMutation";
import { Button } from "@/components/ui/button";
-import { ReloadIcon } from "@radix-ui/react-icons";
-import { defaultWorkflowRequest } from "../workflows/defaultWorkflowRequest";
+import { useNavigate } from "react-router-dom";
+import { navigateToBlankAgentEditor } from "../workflows/blankAgentNavigation";
function DiscoverPage() {
const enableCopilotHandoff =
useFeatureFlag("ENABLE_DISCOVER_COPILOT_HANDOFF") === true;
- const createWorkflowMutation = useCreateWorkflowMutation();
+ const navigate = useNavigate();
return (
<div className="space-y-10">
@@ -20,17 +19,10 @@ function DiscoverPage() {
variant="ghost"
size="sm"
className="text-slate-400 hover:text-slate-200"
- disabled={createWorkflowMutation.isPending}
onClick={() =>
- createWorkflowMutation.mutate({
- ...defaultWorkflowRequest,
- _via: "blank",
- })
+ navigateToBlankAgentEditor(navigate, { via: "blank" })
}
>
- {createWorkflowMutation.isPending && (
- <ReloadIcon className="mr-2 h-3 w-3 animate-spin" />
- )}
Skip — start with blank canvas →
</Button>
</div>
diff --git a/skyvern-frontend/src/routes/root/SideNav.test.tsx b/skyvern-frontend/src/routes/root/SideNav.test.tsx
index abd02cbe5..6d8797493 100644
--- a/skyvern-frontend/src/routes/root/SideNav.test.tsx
+++ b/skyvern-frontend/src/routes/root/SideNav.test.tsx
@@ -6,7 +6,6 @@ import { SideNav } from "./SideNav";
import { useSidebarStore } from "@/store/SidebarStore";
const capture = vi.fn();
-const mutate = vi.fn();
function LocationProbe() {
const location = useLocation();
@@ -25,13 +24,6 @@ vi.mock("posthog-js/react", () => ({
}),
}));
-vi.mock("@/routes/workflows/hooks/useCreateWorkflowMutation", () => ({
- useCreateWorkflowMutation: () => ({
- isPending: false,
- mutate,
- }),
-}));
-
describe("SideNav", () => {
function setViewportHeight(height: number) {
Object.defineProperty(window, "innerHeight", {
@@ -46,28 +38,21 @@ describe("SideNav", () => {
window.localStorage.clear();
useSidebarStore.setState({ collapsed: false });
setViewportHeight(1024);
- mutate.mockClear();
capture.mockClear();
});
- it("creates a new agent from the sidebar", () => {
+ it("opens the draft agent builder from the sidebar", () => {
render(
<MemoryRouter>
<SideNav />
+ <LocationProbe />
</MemoryRouter>,
);
fireEvent.click(screen.getByRole("button", { name: "New Agent" }));
- expect(mutate).toHaveBeenCalledWith(
- expect.objectContaining({
- _via: "sidebar",
- title: "New Agent",
- workflow_definition: expect.objectContaining({
- blocks: [],
- parameters: [],
- }),
- }),
+ expect(screen.getByTestId("location").textContent).toBe(
+ "/workflows/new/build?via=sidebar",
);
});
diff --git a/skyvern-frontend/src/routes/root/SideNav.tsx b/skyvern-frontend/src/routes/root/SideNav.tsx
index 16d923502..3131ef41d 100644
--- a/skyvern-frontend/src/routes/root/SideNav.tsx
+++ b/skyvern-frontend/src/routes/root/SideNav.tsx
@@ -13,7 +13,6 @@ import {
PersonIcon,
PlusIcon,
ReaderIcon,
- ReloadIcon,
Share1Icon,
} from "@radix-ui/react-icons";
@@ -32,10 +31,10 @@ import {
SidebarTreeNav,
type SidebarNavItem,
} from "@/components/SidebarTreeNav";
-import { defaultWorkflowRequest } from "@/routes/workflows/defaultWorkflowRequest";
-import { useCreateWorkflowMutation } from "@/routes/workflows/hooks/useCreateWorkflowMutation";
+import { navigateToBlankAgentEditor } from "@/routes/workflows/blankAgentNavigation";
import { shouldDefaultRecipesOpen } from "./sidebarDefaults";
import { usePostHog } from "posthog-js/react";
+import { useNavigate } from "react-router-dom";
const recipeAnalyticsByPath: Record<string, { beta: boolean; badge?: string }> =
{
@@ -55,7 +54,7 @@ type Props = {
};
function SideNav({ collapsed }: Props = {}) {
- const createWorkflowMutation = useCreateWorkflowMutation();
+ const navigate = useNavigate();
const postHog = usePostHog();
const captureRecipeClick = (label: string, to: string) => {
const analytics = recipeAnalyticsByPath[to] ?? { beta: false };
@@ -81,22 +80,11 @@ function SideNav({ collapsed }: Props = {}) {
icon: <LightningBoltIcon className="size-4" />,
children: [
{
- label: createWorkflowMutation.isPending ? "Creating..." : "New Agent",
- icon: createWorkflowMutation.isPending ? (
- <ReloadIcon className="size-3.5 animate-spin" />
- ) : (
- <PlusIcon className="size-3.5" />
- ),
+ label: "New Agent",
+ icon: <PlusIcon className="size-3.5" />,
onClick: () => {
- if (createWorkflowMutation.isPending) {
- return;
- }
- createWorkflowMutation.mutate({
- ...defaultWorkflowRequest,
- _via: "sidebar",
- });
+ navigateToBlankAgentEditor(navigate, { via: "sidebar" });
},
- disabled: createWorkflowMutation.isPending,
},
{
label: "All Agents",
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
index 977c6ac96..1f7a5ab3e 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
@@ -170,14 +170,7 @@ describe("PromptBox", () => {
expect(body.request.url).toBe("https://google.com");
});
- test("hands Discover prompts to workflow copilot with agent execution", async () => {
- mockPost.mockResolvedValue({
- data: {
- workflow_permanent_id: "wpid_copilot",
- workflow_definition: { blocks: [] },
- },
- });
-
+ test("hands Discover prompts to the draft builder without creating an agent", async () => {
renderPromptBox(true);
fireEvent.change(screen.getByPlaceholderText("Enter your prompt..."), {
@@ -185,16 +178,16 @@ describe("PromptBox", () => {
});
fireEvent.click(screen.getByLabelText("submit-prompt"));
- await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(1));
- const call = mockPost.mock.calls[0];
- expect(call).toBeDefined();
- const [path, yaml] = call!;
-
- expect(path).toBe("/workflows");
- expect(yaml).toContain("run_with: agent");
- expect(yaml).not.toContain("run_with: code");
- expect(mockNavigate).toHaveBeenCalledWith("/workflows/wpid_copilot/build", {
- state: { copilotMessage: "Build this workflow" },
- });
+ await waitFor(() => expect(mockPost).not.toHaveBeenCalled());
+ expect(mockNavigate).toHaveBeenCalledWith(
+ "/workflows/new/build?via=copilot",
+ {
+ state: {
+ copilotMessage: "Build this workflow",
+ draftTitle: "Build this workflow",
+ draftRunWith: "agent",
+ },
+ },
+ );
});
});
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
index d83e9c69a..2ca2f8713 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
@@ -1,7 +1,6 @@
import { getClient } from "@/api/AxiosClient";
import { Createv2TaskRequest, ProxyLocation } from "@/api/types";
-import { stringify as convertToYAML } from "yaml";
-import { WorkflowCreateYAMLRequest } from "@/routes/workflows/types/workflowYamlTypes";
+import { buildBlankAgentBuildPath } from "@/routes/workflows/blankAgentNavigation";
import img from "@/assets/promptBoxBg.png";
import { AutoResizingTextarea } from "@/components/AutoResizingTextarea/AutoResizingTextarea";
import { CartIcon } from "@/components/icons/CartIcon";
@@ -41,7 +40,9 @@ import {
import { useAutoplayStore } from "@/store/useAutoplayStore";
import { TestWebhookDialog } from "@/components/TestWebhookDialog";
import { ImprovePrompt } from "@/components/ImprovePrompt";
+import { SpeechInputButton } from "@/components/SpeechInputButton";
import { cn } from "@/util/utils";
+import { useSpeechToTextField } from "@/hooks/useSpeechToTextField";
const exampleCases = [
{
@@ -117,24 +118,6 @@ function deriveHandoffTitle(prompt: string): string {
return `${collapsed.slice(0, HANDOFF_TITLE_MAX_LEN - 1).trimEnd()}…`;
}
-function buildBlankWorkflowRequest(
- title: string,
- runWith: "agent" | "code" = "agent",
-): WorkflowCreateYAMLRequest {
- return {
- title,
- description: "",
- ai_fallback: true,
- code_version: 2,
- run_with: runWith,
- workflow_definition: {
- version: 2,
- blocks: [],
- parameters: [],
- },
- };
-}
-
function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
const navigate = useNavigate();
const [prompt, setPrompt] = useState<string>("");
@@ -247,50 +230,18 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
},
});
- const handoffWorkflowMutation = useMutation({
- mutationFn: async ({
- prompt,
- runWith,
- }: {
- prompt: string;
- runWith: "agent" | "code";
- }) => {
- const client = await getClient(credentialGetter);
- const yaml = convertToYAML(
- buildBlankWorkflowRequest(deriveHandoffTitle(prompt), runWith),
- );
- const result = await client.post<string, { data: WorkflowApiResponse }>(
- "/workflows",
- yaml,
- {
- headers: {
- "Content-Type": "text/plain",
- },
- },
- );
- return { data: result.data, prompt };
- },
- onSuccess: ({ data: workflow, prompt }) => {
- queryClient.invalidateQueries({ queryKey: ["workflows"] });
- queryClient.invalidateQueries({ queryKey: ["folders"] });
- navigate(`/workflows/${workflow.workflow_permanent_id}/build`, {
- state: { copilotMessage: prompt },
- });
- },
- onError: (error: AxiosError) => {
- toast({
- variant: "destructive",
- title: "Error creating agent",
- description: error.message,
- });
- },
- onSettled: () => {
- submitInFlightRef.current = false;
- },
- });
+ const isSubmitting = generateWorkflowMutation.isPending;
- const isSubmitting =
- generateWorkflowMutation.isPending || handoffWorkflowMutation.isPending;
+ const {
+ isSupported: isSpeechSupported,
+ isListening: isSpeechListening,
+ isHearingSpeech: isSpeechHearing,
+ toggle: toggleSpeech,
+ } = useSpeechToTextField({
+ value: prompt,
+ onChange: setPrompt,
+ enabled: !promptImprovalIsPending && !isSubmitting,
+ });
const submitPrompt = ({ prompt }: { prompt: string }) => {
if (submitInFlightRef.current || isSubmitting) {
@@ -298,7 +249,14 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
}
submitInFlightRef.current = true;
if (enableCopilotHandoff) {
- handoffWorkflowMutation.mutate({ prompt, runWith: "agent" });
+ navigate(buildBlankAgentBuildPath({ via: "copilot" }), {
+ state: {
+ copilotMessage: prompt,
+ draftTitle: deriveHandoffTitle(prompt),
+ draftRunWith: "agent",
+ },
+ });
+ submitInFlightRef.current = false;
return;
}
generateWorkflowMutation.mutate({ prompt });
@@ -325,6 +283,15 @@ function PromptBox({ enableCopilotHandoff = false }: PromptBoxProps) {
},
)}
>
+ <SpeechInputButton
+ isSupported={isSpeechSupported}
+ isListening={isSpeechListening}
+ isHearingSpeech={isSpeechHearing}
+ disabled={promptImprovalIsPending || isSubmitting}
+ onToggle={toggleSpeech}
+ className="ml-2 h-9 w-9 border-0 bg-transparent shadow-none hover:bg-muted"
+ iconClassName="h-5 w-5"
+ />
<AutoResizingTextarea
className="min-h-0 resize-none border-0 bg-transparent px-4 py-0 leading-5 text-foreground shadow-none placeholder:text-muted-foreground hover:border-0 focus-visible:ring-0"
value={prompt}
diff --git a/skyvern-frontend/src/routes/workflows/Workflows.tsx b/skyvern-frontend/src/routes/workflows/Workflows.tsx
index f22b72cc7..03c2a30dd 100644
--- a/skyvern-frontend/src/routes/workflows/Workflows.tsx
+++ b/skyvern-frontend/src/routes/workflows/Workflows.tsx
@@ -54,6 +54,7 @@ import { CreateFromTemplateDialog } from "./components/CreateFromTemplateDialog"
import { ViewAllFoldersDialog } from "./components/ViewAllFoldersDialog";
import { WorkflowFolderSelector } from "./components/WorkflowFolderSelector";
import { HighlightText } from "./components/HighlightText";
+import { navigateToBlankAgentEditor } from "./blankAgentNavigation";
import { useCreateWorkflowMutation } from "./hooks/useCreateWorkflowMutation";
import { useFoldersQuery } from "./hooks/useFoldersQuery";
import { useActiveImportsPolling } from "./hooks/useActiveImportsPolling";
@@ -68,7 +69,6 @@ import { ParameterDisplayInline } from "./components/ParameterDisplayInline";
import { useKeywordSearch } from "./hooks/useKeywordSearch";
import { useParameterExpansion } from "./hooks/useParameterExpansion";
import { Folder } from "./types/folderTypes";
-import { defaultWorkflowRequest } from "./defaultWorkflowRequest";
// Utility function to create URL-safe folder slugs from folder names
function slugifyFolderName(name: string): string {
@@ -507,12 +507,8 @@ function Workflows() {
/>
<DropdownMenu modal={false}>
<DropdownMenuTrigger asChild>
- <Button disabled={createWorkflowMutation.isPending}>
- {createWorkflowMutation.isPending ? (
- <ReloadIcon className="mr-2 h-4 w-4 animate-spin" />
- ) : (
- <PlusIcon className="mr-2 h-4 w-4" />
- )}
+ <Button>
+ <PlusIcon className="mr-2 h-4 w-4" />
Create
<ChevronDownIcon className="ml-2 h-4 w-4" />
</Button>
@@ -520,10 +516,9 @@ function Workflows() {
<DropdownMenuContent align="end">
<DropdownMenuItem
onSelect={() => {
- createWorkflowMutation.mutate({
- ...defaultWorkflowRequest,
- folder_id: selectedFolderId,
- _via: "blank",
+ navigateToBlankAgentEditor(navigate, {
+ via: "blank",
+ folderId: selectedFolderId,
});
}}
>
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowsPageBanner.tsx b/skyvern-frontend/src/routes/workflows/WorkflowsPageBanner.tsx
index 2bb40be7f..94a3a7aef 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowsPageBanner.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowsPageBanner.tsx
@@ -1,11 +1,11 @@
import { Button } from "@/components/ui/button";
-import { PlusIcon, ReloadIcon } from "@radix-ui/react-icons";
-import { useCreateWorkflowMutation } from "./hooks/useCreateWorkflowMutation";
+import { PlusIcon } from "@radix-ui/react-icons";
+import { useNavigate } from "react-router-dom";
import { ImportWorkflowButton } from "./ImportWorkflowButton";
-import { defaultWorkflowRequest } from "./defaultWorkflowRequest";
+import { navigateToBlankAgentEditor } from "./blankAgentNavigation";
function WorkflowsPageBanner() {
- const createNewWorkflowMutation = useCreateWorkflowMutation();
+ const navigate = useNavigate();
return (
<div className="space-y-8 bg-slate-elevation1 p-12">
@@ -15,16 +15,11 @@ function WorkflowsPageBanner() {
<div className="flex justify-center gap-4">
<ImportWorkflowButton />
<Button
- disabled={createNewWorkflowMutation.isPending}
onClick={() => {
- createNewWorkflowMutation.mutate(defaultWorkflowRequest);
+ navigateToBlankAgentEditor(navigate, { via: "banner" });
}}
>
- {createNewWorkflowMutation.isPending ? (
- <ReloadIcon className="mr-2 h-4 w-4 animate-spin" />
- ) : (
- <PlusIcon className="mr-2 h-4 w-4" />
- )}
+ <PlusIcon className="mr-2 h-4 w-4" />
Create Agent
</Button>
</div>
diff --git a/skyvern-frontend/src/routes/workflows/blankAgentNavigation.test.ts b/skyvern-frontend/src/routes/workflows/blankAgentNavigation.test.ts
new file mode 100644
index 000000000..0cc765206
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/blankAgentNavigation.test.ts
@@ -0,0 +1,19 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+ buildBlankAgentBuildPath,
+ navigateToBlankAgentEditor,
+} from "./blankAgentNavigation";
+
+describe("blankAgentNavigation", () => {
+ it("builds the draft builder path with via and folder_id", () => {
+ expect(
+ buildBlankAgentBuildPath({ via: "blank", folderId: "fld_123" }),
+ ).toBe("/workflows/new/build?via=blank&folder_id=fld_123");
+ });
+
+ it("navigates to the draft builder", () => {
+ const navigate = vi.fn();
+ navigateToBlankAgentEditor(navigate, { via: "sidebar" });
+ expect(navigate).toHaveBeenCalledWith("/workflows/new/build?via=sidebar");
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/blankAgentNavigation.ts b/skyvern-frontend/src/routes/workflows/blankAgentNavigation.ts
new file mode 100644
index 000000000..da69600bb
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/blankAgentNavigation.ts
@@ -0,0 +1,35 @@
+import type { NavigateFunction } from "react-router-dom";
+import { DRAFT_WORKFLOW_PERMANENT_ID } from "./draftWorkflow";
+
+type NavigateToBlankAgentOptions = {
+ via?: string;
+ folderId?: string | null;
+};
+
+function buildBlankAgentBuildPath({
+ via,
+ folderId,
+}: NavigateToBlankAgentOptions = {}) {
+ const params = new URLSearchParams();
+ if (via) {
+ params.set("via", via);
+ }
+ if (folderId) {
+ params.set("folder_id", folderId);
+ }
+ const search = params.toString();
+ return `/workflows/${DRAFT_WORKFLOW_PERMANENT_ID}/build${search ? `?${search}` : ""}`;
+}
+
+function navigateToBlankAgentEditor(
+ navigate: NavigateFunction,
+ options: NavigateToBlankAgentOptions = {},
+) {
+ navigate(buildBlankAgentBuildPath(options));
+}
+
+export {
+ buildBlankAgentBuildPath,
+ navigateToBlankAgentEditor,
+ type NavigateToBlankAgentOptions,
+};
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.draft.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.draft.test.tsx
new file mode 100644
index 000000000..2d8ac8157
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.draft.test.tsx
@@ -0,0 +1,181 @@
+import { act, cleanup, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import type { WorkflowCopilotStreamResponseUpdate } from "./workflowCopilotTypes";
+
+type StreamCall = {
+ body: {
+ message: string;
+ workflow_id: string;
+ workflow_permanent_id: string;
+ };
+ onMessage: (payload: unknown) => boolean;
+ resolve: () => void;
+ reject: (error: unknown) => void;
+};
+
+const { streamCalls, postStreaming, mutateAsync, toast } = vi.hoisted(() => {
+ const calls: StreamCall[] = [];
+ const streaming = vi.fn(
+ (
+ _path: string,
+ body: StreamCall["body"],
+ onMessage: (payload: unknown) => boolean,
+ ) =>
+ new Promise<void>((resolve, reject) => {
+ calls.push({ body, onMessage, resolve, reject });
+ }),
+ );
+ return {
+ streamCalls: calls,
+ postStreaming: streaming,
+ mutateAsync: vi.fn(),
+ toast: vi.fn(),
+ };
+});
+
+vi.mock("@/api/sse", () => ({
+ getSseClient: vi.fn().mockResolvedValue({ postStreaming }),
+}));
+
+vi.mock("@/api/AxiosClient", () => ({
+ getClient: vi.fn().mockResolvedValue({
+ get: vi.fn().mockResolvedValue({
+ data: {
+ workflow_copilot_chat_id: null,
+ chat_history: [],
+ proposed_workflow: null,
+ auto_accept: false,
+ },
+ }),
+ post: vi.fn().mockResolvedValue({}),
+ }),
+}));
+
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => null,
+}));
+
+vi.mock("@/components/ui/use-toast", () => ({ toast }));
+
+vi.mock("react-router-dom", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("react-router-dom")>();
+ return {
+ ...actual,
+ useParams: () => ({
+ workflowPermanentId: "new",
+ workflowRunId: undefined,
+ }),
+ };
+});
+
+const draftSaveData = {
+ title: "Handoff title",
+ workflow: {
+ workflow_id: "new",
+ workflow_permanent_id: "new",
+ description: "",
+ totp_verification_url: null,
+ is_saved_task: false,
+ status: "draft",
+ },
+ settings: {
+ proxyLocation: null,
+ webhookCallbackUrl: null,
+ persistBrowserSession: false,
+ browserProfileId: null,
+ model: null,
+ maxScreenshotScrolls: null,
+ extraHttpHeaders: null,
+ runWith: "agent",
+ scriptCacheKey: "",
+ aiFallback: true,
+ codeVersion: 2,
+ runSequentially: false,
+ sequentialKey: null,
+ },
+ parameters: [],
+ blocks: [],
+ workflowDefinitionVersion: 1,
+};
+
+const persistedWorkflow = {
+ ...draftSaveData.workflow,
+ workflow_id: "wf_real",
+ workflow_permanent_id: "wpid_real",
+ status: "published",
+};
+
+vi.mock("@/store/WorkflowHasChangesStore", () => ({
+ useWorkflowHasChangesStore: () => ({
+ getSaveData: () => draftSaveData,
+ }),
+ useWorkflowSave: () => ({
+ isPending: false,
+ mutateAsync,
+ }),
+}));
+
+import { WorkflowCopilotChat } from "./WorkflowCopilotChat";
+
+const terminalResponse = (
+ message: string,
+): WorkflowCopilotStreamResponseUpdate => ({
+ type: "response",
+ workflow_copilot_chat_id: "chat-1",
+ message,
+ updated_workflow: null,
+ response_time: "2026-05-25T00:00:05Z",
+ proposal_disposition: "no_proposal",
+});
+
+const turnStart = () => ({
+ type: "turn_start" as const,
+ turn_id: "turn-1",
+ turn_index: 0,
+ mode: "build",
+ timestamp: "2026-05-25T00:00:00Z",
+});
+
+beforeEach(() => {
+ HTMLElement.prototype.scrollIntoView = vi.fn();
+ HTMLElement.prototype.scrollTo = vi.fn();
+ streamCalls.length = 0;
+ postStreaming.mockClear();
+ toast.mockClear();
+ mutateAsync.mockResolvedValue({
+ saveData: draftSaveData,
+ createdWorkflow: persistedWorkflow,
+ isDraft: true,
+ });
+});
+
+afterEach(() => {
+ cleanup();
+});
+
+describe("WorkflowCopilotChat — draft auto-send", () => {
+ it("persists a draft agent before auto-sending the Discover handoff prompt", async () => {
+ render(<WorkflowCopilotChat initialMessage="Build this workflow" />);
+
+ await waitFor(() => expect(mutateAsync).toHaveBeenCalledTimes(1));
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(toast).not.toHaveBeenCalledWith(
+ expect.objectContaining({ title: "Save your agent first" }),
+ );
+
+ const streamBody = streamCalls[0]?.body;
+ expect(streamBody?.message).toBe("Build this workflow");
+ expect(streamBody?.workflow_id).toBe("wf_real");
+ expect(streamBody?.workflow_permanent_id).toBe("wpid_real");
+
+ await act(async () => {
+ streamCalls[0]?.onMessage(turnStart());
+ streamCalls[0]?.onMessage(terminalResponse("Done"));
+ streamCalls[0]?.resolve();
+ });
+
+ expect(await screen.findByText("Done")).toBeTruthy();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx
index c87225c82..e07d7fd14 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx
@@ -114,6 +114,10 @@ const saveData = {
vi.mock("@/store/WorkflowHasChangesStore", () => ({
useWorkflowHasChangesStore: () => ({ getSaveData: () => saveData }),
+ useWorkflowSave: () => ({
+ isPending: false,
+ mutateAsync: vi.fn(),
+ }),
}));
import { WorkflowCopilotChat } from "./WorkflowCopilotChat";
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.scroll.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.scroll.test.tsx
index c9bf44d06..e4b16165f 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.scroll.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.scroll.test.tsx
@@ -57,6 +57,10 @@ vi.mock("react-router-dom", async (importOriginal) => {
vi.mock("@/store/WorkflowHasChangesStore", () => ({
useWorkflowHasChangesStore: () => ({ getSaveData: () => ({}) }),
+ useWorkflowSave: () => ({
+ isPending: false,
+ mutateAsync: vi.fn(),
+ }),
}));
import { WorkflowCopilotChat } from "./WorkflowCopilotChat";
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index e89cadfd2..666a54834 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -10,9 +10,13 @@ import {
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useParams } from "react-router-dom";
+import { isDraftWorkflowPermanentId } from "@/routes/workflows/draftWorkflow";
import { ReloadIcon, Cross2Icon, ChevronDownIcon } from "@radix-ui/react-icons";
import { stringify as convertToYAML } from "yaml";
-import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
+import {
+ useWorkflowHasChangesStore,
+ useWorkflowSave,
+} from "@/store/WorkflowHasChangesStore";
import { WorkflowCreateYAMLRequest } from "@/routes/workflows/types/workflowYamlTypes";
import { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";
import { toast } from "@/components/ui/use-toast";
@@ -54,6 +58,8 @@ import {
parseUtcIsoMs,
} from "./narrativeState";
import { computeFollowSignature, useStickToBottom } from "./useStickToBottom";
+import { useSpeechToTextField } from "@/hooks/useSpeechToTextField";
+import { SpeechInputButton } from "@/components/SpeechInputButton";
// Cap on retained per-turn snap-back snapshots. A typical session has a
// handful of turns; this ceiling guards a runaway long-running chat.
@@ -385,6 +391,7 @@ export function WorkflowCopilotChat({
const { workflowRunId, workflowPermanentId } = useParams();
const textareaRef = useRef<HTMLTextAreaElement>(null);
const { getSaveData } = useWorkflowHasChangesStore();
+ const saveWorkflowMutation = useWorkflowSave();
const hasInitializedPosition = useRef(false);
const hasAutoSentRef = useRef(false);
const isWaitingForLiveBrowser = shouldWaitForLiveBrowser({
@@ -439,6 +446,17 @@ export function WorkflowCopilotChat({
adjustTextareaHeight();
}, [adjustTextareaHeight, inputValue]);
+ const {
+ isSupported: isSpeechSupported,
+ isListening: isSpeechListening,
+ isHearingSpeech: isSpeechHearing,
+ toggle: toggleSpeech,
+ } = useSpeechToTextField({
+ value: inputValue,
+ onChange: setInputValue,
+ enabled: isOpen && !queuedPrompt,
+ });
+
const updateQueuedPrompt = useCallback((next: QueuedPrompt | null) => {
queuedPromptRef.current = next;
setQueuedPrompt(next);
@@ -840,6 +858,23 @@ export function WorkflowCopilotChat({
});
return;
}
+ let persistedWorkflow: WorkflowApiResponse | null = null;
+ if (isDraftWorkflowPermanentId(workflowPermanentId)) {
+ if (saveWorkflowMutation.isPending) {
+ return;
+ }
+ try {
+ const saveResult = await saveWorkflowMutation.mutateAsync();
+ if (!saveResult) {
+ return;
+ }
+ persistedWorkflow = saveResult.createdWorkflow;
+ } catch {
+ return;
+ }
+ }
+ const activeWorkflowPermanentId =
+ persistedWorkflow?.workflow_permanent_id ?? workflowPermanentId;
if (action === "queue_working" || action === "queue_live_browser") {
const reason: QueuedPromptReason =
action === "queue_working" ? "working" : "live_browser";
@@ -903,10 +938,11 @@ export function WorkflowCopilotChat({
try {
const saveData = getSaveData();
- const workflowId = saveData?.workflow.workflow_id;
+ const workflowId =
+ persistedWorkflow?.workflow_id ?? saveData?.workflow.workflow_id;
let workflowYaml = "";
- if (!workflowId) {
+ if (!workflowId || !activeWorkflowPermanentId) {
toast({
title: "Missing agent",
description: "Agent ID is required to chat.",
@@ -1116,7 +1152,7 @@ export function WorkflowCopilotChat({
"/workflow/copilot/chat-post",
{
workflow_id: workflowId,
- workflow_permanent_id: workflowPermanentId,
+ workflow_permanent_id: activeWorkflowPermanentId,
workflow_copilot_chat_id: workflowCopilotChatId,
workflow_run_id: workflowRunId,
browser_session_id: liveBrowserSessionId ?? null,
@@ -1236,6 +1272,7 @@ export function WorkflowCopilotChat({
liveBrowserSessionId,
requiresLiveBrowser,
updateQueuedPrompt,
+ saveWorkflowMutation,
workflowCopilotChatId,
workflowPermanentId,
workflowRunId,
@@ -1517,6 +1554,11 @@ export function WorkflowCopilotChat({
: isWaitingForLiveBrowser
? "Live browser is starting. Send now to queue your prompt."
: null;
+ const inputStatusText = isSpeechListening
+ ? browserStatusText
+ ? `Listening… · ${browserStatusText}`
+ : "Listening…"
+ : browserStatusText;
return (
<div
@@ -1724,12 +1766,23 @@ export function WorkflowCopilotChat({
{/* Input */}
<div className="border-t border-border p-3">
- {browserStatusText ? (
- <div className="mb-2 text-xs text-muted-foreground">
- {browserStatusText}
+ {inputStatusText ? (
+ <div
+ className="mb-2 text-xs text-muted-foreground"
+ aria-live="polite"
+ >
+ {inputStatusText}
</div>
) : null}
<div className="flex items-end gap-2">
+ <SpeechInputButton
+ isSupported={isSpeechSupported}
+ isListening={isSpeechListening}
+ isHearingSpeech={isSpeechHearing}
+ disabled={inputDisabled}
+ onToggle={toggleSpeech}
+ className="h-9 w-9"
+ />
<textarea
ref={textareaRef}
placeholder={
diff --git a/skyvern-frontend/src/routes/workflows/draftWorkflow.test.ts b/skyvern-frontend/src/routes/workflows/draftWorkflow.test.ts
new file mode 100644
index 000000000..6bc8ea7fa
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/draftWorkflow.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import {
+ DRAFT_WORKFLOW_PERMANENT_ID,
+ buildDraftWorkflowApiResponse,
+ isDraftWorkflowPermanentId,
+} from "./draftWorkflow";
+
+describe("draftWorkflow", () => {
+ it("detects the draft workflow permanent id", () => {
+ expect(isDraftWorkflowPermanentId("new")).toBe(true);
+ expect(isDraftWorkflowPermanentId("wpid_abc")).toBe(false);
+ expect(isDraftWorkflowPermanentId(undefined)).toBe(false);
+ });
+
+ it("keeps the default title when overrides are undefined", () => {
+ const workflow = buildDraftWorkflowApiResponse({
+ title: undefined,
+ run_with: undefined,
+ folder_id: undefined,
+ });
+
+ expect(workflow.title).toBe("New Agent");
+ });
+
+ it("builds an in-memory workflow payload without persisting", () => {
+ const workflow = buildDraftWorkflowApiResponse({
+ title: "Handoff title",
+ folder_id: "fld_1",
+ run_with: "code",
+ });
+
+ expect(workflow.workflow_permanent_id).toBe(DRAFT_WORKFLOW_PERMANENT_ID);
+ expect(workflow.title).toBe("Handoff title");
+ expect(workflow.folder_id).toBe("fld_1");
+ expect(workflow.run_with).toBe("code");
+ expect(workflow.workflow_definition.blocks).toEqual([]);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/draftWorkflow.ts b/skyvern-frontend/src/routes/workflows/draftWorkflow.ts
new file mode 100644
index 000000000..15cd620f0
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/draftWorkflow.ts
@@ -0,0 +1,82 @@
+import type { WorkflowCreateYAMLRequest } from "./types/workflowYamlTypes";
+import type { WorkflowApiResponse } from "./types/workflowTypes";
+import { defaultWorkflowRequest } from "./defaultWorkflowRequest";
+
+const DRAFT_WORKFLOW_PERMANENT_ID = "new";
+
+type DraftWorkflowOverrides = Partial<
+ Pick<WorkflowCreateYAMLRequest, "title" | "run_with" | "folder_id">
+>;
+
+function isDraftWorkflowPermanentId(
+ workflowPermanentId: string | undefined,
+): workflowPermanentId is typeof DRAFT_WORKFLOW_PERMANENT_ID {
+ return workflowPermanentId === DRAFT_WORKFLOW_PERMANENT_ID;
+}
+
+function omitUndefinedOverrides(
+ overrides: DraftWorkflowOverrides,
+): DraftWorkflowOverrides {
+ return Object.fromEntries(
+ Object.entries(overrides).filter(([, value]) => value !== undefined),
+ ) as DraftWorkflowOverrides;
+}
+
+function buildDraftWorkflowApiResponse(
+ overrides: DraftWorkflowOverrides = {},
+): WorkflowApiResponse {
+ const request: WorkflowCreateYAMLRequest = {
+ ...defaultWorkflowRequest,
+ ...omitUndefinedOverrides(overrides),
+ };
+ const now = new Date().toISOString();
+
+ return {
+ workflow_id: DRAFT_WORKFLOW_PERMANENT_ID,
+ organization_id: "",
+ is_saved_task: false,
+ is_template: false,
+ title: request.title,
+ workflow_permanent_id: DRAFT_WORKFLOW_PERMANENT_ID,
+ version: 1,
+ description: request.description ?? "",
+ workflow_definition: {
+ version: request.workflow_definition.version,
+ blocks: request.workflow_definition
+ .blocks as WorkflowApiResponse["workflow_definition"]["blocks"],
+ parameters: request.workflow_definition
+ .parameters as WorkflowApiResponse["workflow_definition"]["parameters"],
+ },
+ proxy_location: null,
+ webhook_callback_url: null,
+ extra_http_headers: null,
+ cdp_connect_headers: null,
+ persist_browser_session: false,
+ browser_profile_id: null,
+ model: null,
+ totp_verification_url: null,
+ totp_identifier: null,
+ max_screenshot_scrolls: null,
+ max_elapsed_time_minutes: null,
+ status: "draft",
+ created_at: now,
+ modified_at: now,
+ deleted_at: null,
+ run_with: request.run_with ?? "agent",
+ cache_key: null,
+ ai_fallback: request.ai_fallback ?? true,
+ adaptive_caching: null,
+ code_version: request.code_version ?? null,
+ run_sequentially: false,
+ sequential_key: null,
+ folder_id: request.folder_id ?? null,
+ import_error: null,
+ };
+}
+
+export {
+ DRAFT_WORKFLOW_PERMANENT_ID,
+ buildDraftWorkflowApiResponse,
+ isDraftWorkflowPermanentId,
+ type DraftWorkflowOverrides,
+};
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
index 2639d73b3..40e41030a 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
@@ -79,7 +79,7 @@ function WorkflowEditor() {
persistBrowserSession: workflow.persist_browser_session,
browserProfileId: workflow.browser_profile_id ?? null,
proxyLocation: workflow.proxy_location,
- webhookCallbackUrl: workflow.webhook_callback_url,
+ webhookCallbackUrl: workflow.webhook_callback_url ?? "",
model: workflow.model,
maxScreenshotScrolls: workflow.max_screenshot_scrolls,
maxElapsedTimeMinutes: workflow.max_elapsed_time_minutes ?? null,
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
index bc2c9f108..b41e7200f 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
@@ -156,7 +156,7 @@ const getPayload = (opts: {
workflowPermanentId: string;
workflowSettings: WorkflowSettingsState;
}): Payload | null => {
- const webhook_url = opts.workflowSettings.webhookCallbackUrl.trim();
+ const webhook_url = (opts.workflowSettings.webhookCallbackUrl ?? "").trim();
let extraHttpHeaders = null;
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts
index ed53cc7e3..8f41922cf 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts
@@ -1,6 +1,7 @@
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery } from "@tanstack/react-query";
+import { isDraftWorkflowPermanentId } from "../draftWorkflow";
import { ScriptBlocksResponse } from "../types/scriptTypes";
type Props = {
@@ -49,7 +50,8 @@ function useBlockScriptsQuery({
}
return Math.max(2000, pollIntervalMs);
},
- enabled: !!workflowPermanentId,
+ enabled:
+ !!workflowPermanentId && !isDraftWorkflowPermanentId(workflowPermanentId),
});
}
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts
index baa60b72c..cffcd0604 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useCacheKeyValuesQuery.ts
@@ -2,6 +2,7 @@ import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery, keepPreviousData } from "@tanstack/react-query";
import { useEffect, useState } from "react";
+import { isDraftWorkflowPermanentId } from "../draftWorkflow";
import { CacheKeyValuesResponse } from "../types/scriptTypes";
type Props = {
@@ -55,7 +56,11 @@ function useCacheKeyValuesQuery({
return result;
},
- enabled: !!workflowPermanentId && !!cacheKey && cacheKey.length > 0,
+ enabled:
+ !!workflowPermanentId &&
+ !isDraftWorkflowPermanentId(workflowPermanentId) &&
+ !!cacheKey &&
+ cacheKey.length > 0,
placeholderData: keepPreviousData,
staleTime: 5 * 60 * 1000,
});
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.ts
index d85c7d6ec..8c21fa33c 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useDebugSessionQuery.ts
@@ -2,6 +2,7 @@ import { getClient } from "@/api/AxiosClient";
import { DebugSessionApiResponse } from "@/api/types";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery } from "@tanstack/react-query";
+import { isDraftWorkflowPermanentId } from "../draftWorkflow";
interface Opts {
workflowPermanentId?: string;
@@ -17,9 +18,10 @@ function useDebugSessionQuery({
const credentialGetter = useCredentialGetter();
const baseEnabled =
- enabled !== undefined
+ (enabled !== undefined
? enabled && !!workflowPermanentId
- : !!workflowPermanentId;
+ : !!workflowPermanentId) &&
+ !isDraftWorkflowPermanentId(workflowPermanentId);
return useQuery<DebugSessionApiResponse>({
queryKey: ["debugSession", workflowPermanentId],
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowQuery.ts
index 555394760..42c1f481a 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowQuery.ts
@@ -1,20 +1,48 @@
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery } from "@tanstack/react-query";
+import { useLocation, useSearchParams } from "react-router-dom";
import { WorkflowApiResponse } from "../types/workflowTypes";
import { useGlobalWorkflowsQuery } from "./useGlobalWorkflowsQuery";
+import {
+ buildDraftWorkflowApiResponse,
+ isDraftWorkflowPermanentId,
+} from "../draftWorkflow";
type Props = {
workflowPermanentId?: string;
};
+type DraftLocationState = {
+ draftTitle?: string;
+ draftRunWith?: "agent" | "code";
+};
+
function useWorkflowQuery({ workflowPermanentId }: Props) {
const { data: globalWorkflows } = useGlobalWorkflowsQuery();
const credentialGetter = useCredentialGetter();
+ const [searchParams] = useSearchParams();
+ const location = useLocation();
+ const isDraft = isDraftWorkflowPermanentId(workflowPermanentId);
return useQuery<WorkflowApiResponse>({
- queryKey: ["workflow", workflowPermanentId],
+ queryKey: [
+ "workflow",
+ workflowPermanentId,
+ isDraft ? searchParams.toString() : null,
+ isDraft ? location.state : null,
+ ],
queryFn: async () => {
+ if (isDraft) {
+ const locationState = location.state as DraftLocationState | null;
+ const folderId = searchParams.get("folder_id");
+ return buildDraftWorkflowApiResponse({
+ title: locationState?.draftTitle,
+ run_with: locationState?.draftRunWith,
+ folder_id: folderId ?? undefined,
+ });
+ }
+
const client = await getClient(credentialGetter);
const isGlobalWorkflow = globalWorkflows?.some(
(workflow) => workflow.workflow_permanent_id === workflowPermanentId,
@@ -27,8 +55,7 @@ function useWorkflowQuery({ workflowPermanentId }: Props) {
.get(`/workflows/${workflowPermanentId}`, { params })
.then((response) => response.data);
},
- enabled: !!globalWorkflows && !!workflowPermanentId,
- // Keep showing stale data during refetch to prevent flash of empty content
+ enabled: isDraft ? true : !!globalWorkflows && !!workflowPermanentId,
placeholderData: (previousData) => previousData,
});
}
diff --git a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
index 5890504e2..a62bf9421 100644
--- a/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
+++ b/skyvern-frontend/src/store/WorkflowHasChangesStore.ts
@@ -4,10 +4,12 @@ import { create } from "zustand";
import { useMutation, useQueryClient } from "@tanstack/react-query";
import { stringify as convertToYAML } from "yaml";
import { usePostHog } from "posthog-js/react";
+import { useLocation, useNavigate, useSearchParams } from "react-router-dom";
import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { isDraftWorkflowPermanentId } from "@/routes/workflows/draftWorkflow";
import {
type BlockYAML,
type ParameterYAML,
@@ -86,6 +88,9 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
const credentialGetter = useCredentialGetter();
const queryClient = useQueryClient();
const postHog = usePostHog();
+ const navigate = useNavigate();
+ const location = useLocation();
+ const [searchParams] = useSearchParams();
const {
getSaveData,
saidOkToCodeCacheDeletion,
@@ -100,9 +105,13 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
if (!saveData) {
setHasChanges(false);
- return;
+ return null;
}
+ const isDraft = isDraftWorkflowPermanentId(
+ saveData.workflow.workflow_permanent_id,
+ );
+
const client = await getClient(credentialGetter);
const extraHttpHeaders: Record<string, string> = {};
@@ -207,11 +216,30 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
status: opts?.status ?? saveData.workflow.status,
run_sequentially: saveData.settings.runSequentially,
sequential_key: saveData.settings.sequentialKey,
+ ...(isDraft && saveData.workflow.folder_id
+ ? { folder_id: saveData.workflow.folder_id }
+ : {}),
};
const yaml = convertToYAML(requestBody);
- return client.put<string, WorkflowApiResponse>(
+ if (isDraft) {
+ const created = await client.post<
+ string,
+ { data: WorkflowApiResponse }
+ >("/workflows", yaml, {
+ headers: {
+ "Content-Type": "text/plain",
+ },
+ });
+ return {
+ saveData,
+ createdWorkflow: created.data,
+ isDraft: true as const,
+ };
+ }
+
+ const updated = await client.put<string, WorkflowApiResponse>(
`/workflows/${saveData.workflow.workflow_permanent_id}`,
yaml,
{
@@ -225,37 +253,58 @@ const useWorkflowSave = (opts?: WorkflowSaveOpts) => {
},
},
);
+ return { saveData, createdWorkflow: updated, isDraft: false as const };
},
- onSuccess: () => {
- const saveData = getSaveData();
-
- if (!saveData) {
+ onSuccess: (result) => {
+ if (!result) {
return;
}
+ const { saveData, createdWorkflow, isDraft } = result;
+ const workflowPermanentId = createdWorkflow.workflow_permanent_id;
+
postHog.capture("builder.workflow.saved", {
- org_id: saveData.workflow.organization_id,
- workflow_permanent_id: saveData.workflow.workflow_permanent_id,
+ org_id:
+ createdWorkflow.organization_id || saveData.workflow.organization_id,
+ workflow_permanent_id: workflowPermanentId,
block_count: saveData.blocks.length,
block_types: saveData.blocks.map((b) => b.block_type),
});
toast({
- title: "Changes saved",
- description: "Your changes have been saved",
+ title: isDraft ? "Agent created" : "Changes saved",
+ description: isDraft
+ ? "Your agent has been saved"
+ : "Your changes have been saved",
variant: "success",
});
- queryClient.invalidateQueries({
- queryKey: ["workflow", saveData.workflow.workflow_permanent_id],
- });
+ if (isDraft) {
+ const via = searchParams.get("via");
+ const nextSearch = via ? `?via=${encodeURIComponent(via)}` : "";
+ navigate(`/workflows/${workflowPermanentId}/build${nextSearch}`, {
+ replace: true,
+ state: location.state,
+ });
+ queryClient.setQueryData(
+ ["workflow", workflowPermanentId],
+ createdWorkflow,
+ );
+ } else {
+ queryClient.invalidateQueries({
+ queryKey: ["workflow", workflowPermanentId],
+ });
+ }
queryClient.invalidateQueries({
queryKey: ["workflows"],
});
+ queryClient.invalidateQueries({
+ queryKey: ["folders"],
+ });
queryClient.invalidateQueries({
- queryKey: ["block-scripts", saveData.workflow.workflow_permanent_id],
+ queryKey: ["block-scripts", workflowPermanentId],
});
setHasChanges(false);
diff --git a/skyvern-frontend/src/store/WorkflowTitleStore.ts b/skyvern-frontend/src/store/WorkflowTitleStore.ts
index 167b75e1f..70ce50970 100644
--- a/skyvern-frontend/src/store/WorkflowTitleStore.ts
+++ b/skyvern-frontend/src/store/WorkflowTitleStore.ts
@@ -20,15 +20,18 @@ const useWorkflowTitleStore = create<WorkflowTitleStore>((set, get) => {
return get().title === DEFAULT_WORKFLOW_TITLE;
},
setTitle: (title: string) => {
- set({ title: title.trim(), titleHasBeenGenerated: true });
+ const normalizedTitle = (title ?? "").trim();
+ set({ title: normalizedTitle, titleHasBeenGenerated: true });
},
setTitleFromGeneration: (title: string) => {
- set({ title: title.trim(), titleHasBeenGenerated: true });
+ const normalizedTitle = (title ?? "").trim();
+ set({ title: normalizedTitle, titleHasBeenGenerated: true });
},
initializeTitle: (title: string) => {
+ const normalizedTitle = (title ?? "").trim();
set({
- title: title.trim(),
- titleHasBeenGenerated: title.trim() !== DEFAULT_WORKFLOW_TITLE,
+ title: normalizedTitle,
+ titleHasBeenGenerated: normalizedTitle !== DEFAULT_WORKFLOW_TITLE,
});
},
resetTitle: () => {