Skip to content

Automate browser based workflows with AI

SKY-10827: persist copilot artifact-metadata violation batches as durable telemetry (#6520)

Andrew Neilson559fea0bb3e7
diff --git a/alembic/versions/2026_06_11_2030-a8c4e2f9d6b1_add_audio_artifact_id_to_workflow_copilot_chat_messages.py b/alembic/versions/2026_06_11_2030-a8c4e2f9d6b1_add_audio_artifact_id_to_workflow_copilot_chat_messages.py
new file mode 100644
index 000000000..97cd66d49
--- /dev/null
+++ b/alembic/versions/2026_06_11_2030-a8c4e2f9d6b1_add_audio_artifact_id_to_workflow_copilot_chat_messages.py
@@ -0,0 +1,45 @@
+"""add audio_artifact_id to workflow_copilot_chat_messages
+
+Revision ID: a8c4e2f9d6b1
+Revises: f3a9c2b7e1d4
+Create Date: 2026-06-11T20:30:00.000000+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "a8c4e2f9d6b1"
+down_revision: Union[str, None] = "f3a9c2b7e1d4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+
+ columns = {col["name"] for col in inspector.get_columns("workflow_copilot_chat_messages")}
+ if "audio_artifact_id" in columns:
+ return
+
+ if conn.dialect.name == "postgresql":
+ op.execute(sa.text("SET lock_timeout = '2s'"))
+
+ op.add_column(
+ "workflow_copilot_chat_messages",
+ sa.Column("audio_artifact_id", sa.String(), nullable=True),
+ )
+
+
+def downgrade() -> None:
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ columns = {col["name"] for col in inspector.get_columns("workflow_copilot_chat_messages")}
+ if "audio_artifact_id" not in columns:
+ return
+ op.drop_column("workflow_copilot_chat_messages", "audio_artifact_id")
diff --git a/skyvern-frontend/src/hooks/useFeatureFlag.ts b/skyvern-frontend/src/hooks/useFeatureFlag.ts
index 01c726a7c..e665d3b15 100644
--- a/skyvern-frontend/src/hooks/useFeatureFlag.ts
+++ b/skyvern-frontend/src/hooks/useFeatureFlag.ts
@@ -13,3 +13,16 @@ export function useFeatureFlag(flagName: string): boolean | undefined {
const evaluate = useContext(FeatureFlagContext);
return evaluate(flagName);
}
+
+/**
+ * Context for multivariate (string variant) feature flags. Mirrors
+ * FeatureFlagContext; the cloud build supplies the real evaluator.
+ */
+export const FeatureFlagValueContext = createContext<
+ (flagName: string) => string | undefined
+>(() => undefined);
+
+export function useFeatureFlagValue(flagName: string): string | undefined {
+ const evaluate = useContext(FeatureFlagValueContext);
+ return evaluate(flagName);
+}
diff --git a/skyvern-frontend/src/hooks/useSpeechToText.test.ts b/skyvern-frontend/src/hooks/useSpeechToText.test.ts
index 35403117f..7992680c7 100644
--- a/skyvern-frontend/src/hooks/useSpeechToText.test.ts
+++ b/skyvern-frontend/src/hooks/useSpeechToText.test.ts
@@ -71,6 +71,35 @@ class MockSpeechRecognition {
}
}
+class MockMediaRecorder {
+ static lastInstance: MockMediaRecorder | null = null;
+
+ mimeType = "audio/webm";
+ state: RecordingState = "inactive";
+ ondataavailable: ((event: BlobEvent) => void) | null = null;
+ onstop: (() => void) | null = null;
+
+ constructor(public stream: MediaStream) {
+ MockMediaRecorder.lastInstance = this;
+ }
+
+ start = vi.fn(() => {
+ this.state = "recording";
+ });
+
+ stop = vi.fn(() => {
+ this.state = "inactive";
+ this.ondataavailable?.({
+ data: new Blob(["audio"], { type: "audio/webm" }),
+ } as BlobEvent);
+ this.onstop?.();
+ });
+
+ emitData(data: Blob) {
+ this.ondataavailable?.({ data } as BlobEvent);
+ }
+}
+
describe("isSpeechRecognitionSupported", () => {
afterEach(() => {
delete (window as { SpeechRecognition?: unknown }).SpeechRecognition;
@@ -91,12 +120,30 @@ describe("isSpeechRecognitionSupported", () => {
});
describe("useSpeechToText", () => {
+ const trackStop = vi.fn();
+
beforeEach(() => {
vi.useFakeTimers();
MockSpeechRecognition.lastInstance = null;
+ MockMediaRecorder.lastInstance = null;
+ trackStop.mockClear();
(
window as { SpeechRecognition?: typeof MockSpeechRecognition }
).SpeechRecognition = MockSpeechRecognition;
+ Object.defineProperty(globalThis, "MediaRecorder", {
+ configurable: true,
+ value: MockMediaRecorder,
+ });
+ Object.defineProperty(navigator, "mediaDevices", {
+ configurable: true,
+ value: {
+ getUserMedia: vi.fn(() =>
+ Promise.resolve({
+ getTracks: () => [{ stop: trackStop }],
+ } as unknown as MediaStream),
+ ),
+ },
+ });
});
afterEach(() => {
@@ -104,6 +151,8 @@ describe("useSpeechToText", () => {
delete (window as { SpeechRecognition?: unknown }).SpeechRecognition;
delete (window as { webkitSpeechRecognition?: unknown })
.webkitSpeechRecognition;
+ delete (globalThis as { MediaRecorder?: unknown }).MediaRecorder;
+ delete (navigator as { mediaDevices?: unknown }).mediaDevices;
});
it("reports support from the browser API", () => {
@@ -135,6 +184,72 @@ describe("useSpeechToText", () => {
expect(MockSpeechRecognition.lastInstance?.stop).toHaveBeenCalled();
});
+ it("captures an audio blob while dictating", async () => {
+ const onAudioCaptured = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onAudioCaptured,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ await act(async () => {
+ await result.current.stop();
+ });
+
+ expect(onAudioCaptured).toHaveBeenCalledTimes(1);
+ const audioBlob = onAudioCaptured.mock.calls[0]?.[0] as Blob;
+ expect(audioBlob.type).toBe("audio/webm");
+ expect(result.current.takeAudioBlob()).toBe(audioBlob);
+ expect(trackStop).toHaveBeenCalled();
+ });
+
+ it("keeps buffered audio chunks when the recorder is already inactive", async () => {
+ const onAudioCaptured = vi.fn();
+ const { result } = renderHook(() =>
+ useSpeechToText({
+ onTranscript: vi.fn(),
+ onAudioCaptured,
+ }),
+ );
+
+ act(() => {
+ result.current.start();
+ });
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ const recorder = MockMediaRecorder.lastInstance;
+ const audioChunk = new Blob(["partial-audio"], { type: "audio/webm" });
+ act(() => {
+ recorder?.emitData(audioChunk);
+ if (recorder) {
+ recorder.state = "inactive";
+ }
+ });
+
+ let stoppedBlob: Blob | null = null;
+ await act(async () => {
+ stoppedBlob = await result.current.stop();
+ });
+
+ const capturedBlob = stoppedBlob as unknown as Blob;
+ expect(capturedBlob).toBeInstanceOf(Blob);
+ expect(capturedBlob.type).toBe("audio/webm");
+ expect(await capturedBlob.text()).toBe("partial-audio");
+ expect(onAudioCaptured).toHaveBeenCalledWith(capturedBlob);
+ expect(recorder?.stop).not.toHaveBeenCalled();
+ expect(trackStop).toHaveBeenCalled();
+ });
+
it("toggles listening on and off", () => {
const { result } = renderHook(() =>
useSpeechToText({ onTranscript: vi.fn() }),
diff --git a/skyvern-frontend/src/hooks/useSpeechToText.ts b/skyvern-frontend/src/hooks/useSpeechToText.ts
index 811ecfb4e..f6c0ec442 100644
--- a/skyvern-frontend/src/hooks/useSpeechToText.ts
+++ b/skyvern-frontend/src/hooks/useSpeechToText.ts
@@ -47,6 +47,7 @@ declare global {
const HEARING_PULSE_MS = 400;
const MIN_RESTART_INTERVAL_MS = 300;
const MAX_RAPID_RESTARTS = 8;
+const DEFAULT_AUDIO_MIME_TYPE = "audio/webm";
function getSpeechRecognitionConstructor():
| SpeechRecognitionConstructor
@@ -120,11 +121,18 @@ function combineDictated(finalText: string, interimText: string): string {
return finalText || interimText;
}
+function buildAudioBlob(chunks: Blob[], mimeType: string): Blob | null {
+ return chunks.length > 0
+ ? new Blob(chunks, { type: mimeType || DEFAULT_AUDIO_MIME_TYPE })
+ : null;
+}
+
export interface UseSpeechToTextOptions {
/** Snapshot of textarea content when dictation starts. */
getBaseText?: () => string;
onTranscript: (text: string) => void;
onError?: (message: string) => void;
+ onAudioCaptured?: (audio: Blob) => void;
lang?: string;
/** When false, active recognition is stopped (e.g. panel closed). */
enabled?: boolean;
@@ -136,8 +144,10 @@ export interface UseSpeechToTextResult {
/** Briefly true while interim speech is being recognized. */
isHearingSpeech: boolean;
start: () => void;
- stop: () => void;
+ /** Resolves with captured audio when available; callers may ignore it. */
+ stop: () => Promise<Blob | null>;
toggle: () => void;
+ takeAudioBlob: () => Blob | null;
}
const ERROR_MESSAGES: Record<string, string> = {
@@ -156,7 +166,14 @@ function errorMessageForCode(error: string): string {
export function useSpeechToText(
options: UseSpeechToTextOptions,
): UseSpeechToTextResult {
- const { getBaseText, onTranscript, onError, lang, enabled = true } = options;
+ const {
+ getBaseText,
+ onTranscript,
+ onError,
+ onAudioCaptured,
+ lang,
+ enabled = true,
+ } = options;
const isSupported = isSpeechRecognitionSupported();
const [isListening, setIsListening] = useState(false);
@@ -172,16 +189,26 @@ export function useSpeechToText(
const restartTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const lastStartAttemptAtRef = useRef(0);
const rapidRestartCountRef = useRef(0);
+ const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+ const mediaStreamRef = useRef<MediaStream | null>(null);
+ const audioChunksRef = useRef<Blob[]>([]);
+ const latestAudioBlobRef = useRef<Blob | null>(null);
+ const isAudioStartingRef = useRef(false);
+ const audioStopResolverRef = useRef<((blob: Blob | null) => void) | null>(
+ null,
+ );
const onTranscriptRef = useRef(onTranscript);
const onErrorRef = useRef(onError);
+ const onAudioCapturedRef = useRef(onAudioCaptured);
const getBaseTextRef = useRef(getBaseText);
useEffect(() => {
onTranscriptRef.current = onTranscript;
onErrorRef.current = onError;
+ onAudioCapturedRef.current = onAudioCaptured;
getBaseTextRef.current = getBaseText;
- }, [onTranscript, onError, getBaseText]);
+ }, [onTranscript, onError, onAudioCaptured, getBaseText]);
useEffect(() => {
enabledRef.current = enabled;
@@ -213,6 +240,109 @@ export function useSpeechToText(
}
}, []);
+ const cleanupAudioStream = useCallback(() => {
+ mediaStreamRef.current?.getTracks().forEach((track) => {
+ track.stop();
+ });
+ mediaStreamRef.current = null;
+ }, []);
+
+ const resolveAudioStop = useCallback((blob: Blob | null) => {
+ const resolver = audioStopResolverRef.current;
+ audioStopResolverRef.current = null;
+ resolver?.(blob);
+ }, []);
+
+ const finishAudioCapture = useCallback(
+ (blob: Blob | null) => {
+ mediaRecorderRef.current = null;
+ cleanupAudioStream();
+ if (blob && blob.size > 0) {
+ latestAudioBlobRef.current = blob;
+ onAudioCapturedRef.current?.(blob);
+ resolveAudioStop(blob);
+ return;
+ }
+ resolveAudioStop(null);
+ },
+ [cleanupAudioStream, resolveAudioStop],
+ );
+
+ const startAudioCapture = useCallback(() => {
+ if (
+ typeof navigator === "undefined" ||
+ !navigator.mediaDevices?.getUserMedia ||
+ typeof MediaRecorder === "undefined"
+ ) {
+ return;
+ }
+ if (isAudioStartingRef.current || mediaRecorderRef.current) {
+ return;
+ }
+
+ audioChunksRef.current = [];
+ latestAudioBlobRef.current = null;
+ isAudioStartingRef.current = true;
+
+ navigator.mediaDevices
+ .getUserMedia({ audio: true })
+ .then((stream) => {
+ isAudioStartingRef.current = false;
+ if (!shouldKeepListeningRef.current) {
+ stream.getTracks().forEach((track) => {
+ track.stop();
+ });
+ return;
+ }
+ mediaStreamRef.current = stream;
+ const recorder = new MediaRecorder(stream);
+ mediaRecorderRef.current = recorder;
+ recorder.ondataavailable = (event) => {
+ if (event.data.size > 0) {
+ audioChunksRef.current.push(event.data);
+ }
+ };
+ recorder.onstop = () => {
+ const blob = buildAudioBlob(
+ audioChunksRef.current,
+ recorder.mimeType,
+ );
+ audioChunksRef.current = [];
+ finishAudioCapture(blob);
+ };
+ recorder.start();
+ })
+ .catch(() => {
+ isAudioStartingRef.current = false;
+ onErrorRef.current?.(
+ "Audio recording failed to start. Check microphone permission and try again.",
+ );
+ });
+ }, [finishAudioCapture]);
+
+ const stopAudioCapture = useCallback((): Promise<Blob | null> => {
+ const recorder = mediaRecorderRef.current;
+ if (!recorder) {
+ cleanupAudioStream();
+ return Promise.resolve(null);
+ }
+
+ return new Promise((resolve) => {
+ audioStopResolverRef.current = resolve;
+ if (recorder.state === "inactive") {
+ const blob = buildAudioBlob(audioChunksRef.current, recorder.mimeType);
+ audioChunksRef.current = [];
+ finishAudioCapture(blob);
+ return;
+ }
+ try {
+ recorder.stop();
+ } catch {
+ finishAudioCapture(null);
+ }
+ });
+ }, [cleanupAudioStream, finishAudioCapture]);
+
const teardownRecognition = useCallback(() => {
clearRestartTimeout();
const recognition = recognitionRef.current;
@@ -230,7 +360,7 @@ export function useSpeechToText(
recognitionRef.current = null;
}, [clearRestartTimeout]);
- const stop = useCallback(() => {
+ const stop = useCallback(async () => {
shouldKeepListeningRef.current = false;
isListeningRef.current = false;
setIsListening(false);
@@ -244,7 +374,13 @@ export function useSpeechToText(
teardownRecognition();
}
}
- }, [clearHearingSpeech, clearRestartTimeout, teardownRecognition]);
+ return await stopAudioCapture();
+ }, [
+ clearHearingSpeech,
+ clearRestartTimeout,
+ stopAudioCapture,
+ teardownRecognition,
+ ]);
const start = useCallback(() => {
if (!isSupported || !enabledRef.current || isListeningRef.current) {
@@ -260,6 +396,7 @@ export function useSpeechToText(
clearHearingSpeech();
finalizedChunksRef.current.clear();
rapidRestartCountRef.current = 0;
+ startAudioCapture();
baseTextRef.current = getBaseTextRef.current?.() ?? "";
@@ -369,6 +506,7 @@ export function useSpeechToText(
setIsListening(false);
clearHearingSpeech();
teardownRecognition();
+ void stopAudioCapture();
onErrorRef.current?.("Voice input failed to start. Try again.");
}
}, [
@@ -377,12 +515,14 @@ export function useSpeechToText(
isSupported,
lang,
markHearingSpeech,
+ startAudioCapture,
+ stopAudioCapture,
teardownRecognition,
]);
const toggle = useCallback(() => {
if (isListeningRef.current) {
- stop();
+ void stop();
} else {
start();
}
@@ -390,7 +530,7 @@ export function useSpeechToText(
useEffect(() => {
if (!enabled && isListeningRef.current) {
- stop();
+ void stop();
}
}, [enabled, stop]);
@@ -400,8 +540,15 @@ export function useSpeechToText(
isListeningRef.current = false;
clearHearingSpeech();
teardownRecognition();
+ void stopAudioCapture();
};
- }, [clearHearingSpeech, teardownRecognition]);
+ }, [clearHearingSpeech, stopAudioCapture, teardownRecognition]);
+
+ const takeAudioBlob = useCallback(() => {
+ const blob = latestAudioBlobRef.current;
+ latestAudioBlobRef.current = null;
+ return blob;
+ }, []);
return {
isSupported,
@@ -410,5 +557,6 @@ export function useSpeechToText(
start,
stop,
toggle,
+ takeAudioBlob,
};
}
diff --git a/skyvern-frontend/src/hooks/useSpeechToTextField.ts b/skyvern-frontend/src/hooks/useSpeechToTextField.ts
index 6d41c622e..d8e15b2bd 100644
--- a/skyvern-frontend/src/hooks/useSpeechToTextField.ts
+++ b/skyvern-frontend/src/hooks/useSpeechToTextField.ts
@@ -25,10 +25,12 @@ export function useSpeechToTextField({
const getBaseText = useCallback(() => value, [value]);
- return useSpeechToText({
+ const speech = useSpeechToText({
getBaseText,
onTranscript: onChange,
onError: handleSpeechError,
enabled,
});
+
+ return speech;
}
diff --git a/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.test.tsx b/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.test.tsx
new file mode 100644
index 000000000..6c5169def
--- /dev/null
+++ b/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.test.tsx
@@ -0,0 +1,85 @@
+// @vitest-environment jsdom
+
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { act, renderHook, waitFor } from "@testing-library/react";
+import type { ReactNode } from "react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { getClient } from "@/api/AxiosClient";
+import { useCredentialTestStore } from "@/store/useCredentialTestStore";
+import { useBackgroundCredentialTest } from "./useBackgroundCredentialTest";
+
+vi.mock("@/api/AxiosClient", () => ({ getClient: vi.fn() }));
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => null,
+}));
+vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));
+
+const mockedGetClient = vi.mocked(getClient);
+
+function makeTest(overrides: Partial<Record<string, unknown>> = {}) {
+ return {
+ credentialId: "cred_1",
+ workflowRunId: "wr_1",
+ url: "https://example.com/login",
+ startTime: Date.now() - 1000,
+ ...overrides,
+ };
+}
+
+afterEach(() => {
+ useCredentialTestStore.getState().clearActiveTest();
+ localStorage.clear();
+ vi.clearAllMocks();
+});
+
+describe("useBackgroundCredentialTest adoption (SKY-10855)", () => {
+ function setup() {
+ const queryClient = new QueryClient();
+ const invalidateSpy = vi.spyOn(queryClient, "invalidateQueries");
+ const wrapper = ({ children }: { children: ReactNode }) => (
+ <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
+ );
+ const hook = renderHook(() => useBackgroundCredentialTest(), { wrapper });
+ return { hook, invalidateSpy };
+ }
+
+ it("starts polling when a test appears in the store after mount (started by another tab)", async () => {
+ const get = vi.fn().mockResolvedValue({
+ data: {
+ status: "completed",
+ browser_profile_id: "bp_1",
+ tested_url: "https://example.com/login",
+ },
+ });
+ mockedGetClient.mockResolvedValue({ get } as never);
+
+ const { invalidateSpy } = setup();
+ act(() => {
+ useCredentialTestStore.getState().setActiveTest(makeTest());
+ });
+
+ await waitFor(() => {
+ expect(useCredentialTestStore.getState().activeTest).toBeNull();
+ });
+ expect(get).toHaveBeenCalledWith("/credentials/cred_1/test/wr_1");
+ expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["credentials"] });
+ });
+
+ it("clears an expired persisted test instead of polling it", async () => {
+ const get = vi.fn();
+ mockedGetClient.mockResolvedValue({ get } as never);
+
+ setup();
+ act(() => {
+ useCredentialTestStore
+ .getState()
+ .setActiveTest(makeTest({ startTime: Date.now() - 11 * 60 * 1000 }));
+ });
+
+ await waitFor(() => {
+ expect(useCredentialTestStore.getState().activeTest).toBeNull();
+ });
+ expect(get).not.toHaveBeenCalled();
+ });
+});
diff --git a/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts b/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts
index 5e2f87b84..922004074 100644
--- a/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts
+++ b/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts
@@ -31,9 +31,10 @@ type ActiveTest = {
* The hook polls the backend, shows toast notifications on completion/failure,
* and invalidates the credentials query so the list updates.
*
- * Active test state is persisted in a zustand store backed by sessionStorage
- * so it survives both SPA navigation and full page reloads. On mount, the hook
- * checks for persisted state and resumes polling if a test was in progress.
+ * Active test state is persisted in a zustand store backed by localStorage so
+ * it survives reloads and is visible across tabs (e.g. the noopener watch-run
+ * tab). The hook adopts the persisted test on mount and whenever the slot
+ * changes (a test started in another tab), resuming polling either way.
*
* Instantiate this in a component that outlives the modal (e.g. CredentialsPage)
* so polling survives modal close.
@@ -47,11 +48,13 @@ function useBackgroundCredentialTest() {
// Full cleanup: stop polling AND clear the persisted store.
// Used when a test reaches a terminal state (completed/failed/timeout).
const cleanup = useCallback(() => {
- if (activeTestRef.current?.timeoutId) {
- clearTimeout(activeTestRef.current.timeoutId);
+ const current = activeTestRef.current;
+ if (current?.timeoutId) {
+ clearTimeout(current.timeoutId);
}
activeTestRef.current = null;
- clearActiveTest();
+ // Scoped to this tab's run so it can't wipe a newer test from another tab.
+ clearActiveTest(current?.workflowRunId);
}, [clearActiveTest]);
// On unmount, only stop the timer — don't clear the store so the test
@@ -98,10 +101,12 @@ function useBackgroundCredentialTest() {
);
const data = response.data;
- // Reset error counter on successful poll
- if (activeTestRef.current) {
- activeTestRef.current.errorCount = 0;
+ // A newer test may have superseded this run while the request was in
+ // flight — its own poll chain owns the state now.
+ if (activeTestRef.current?.workflowRunId !== test.workflowRunId) {
+ return;
}
+ activeTestRef.current.errorCount = 0;
if (data.status === "completed") {
// The backend sets browser_profile_id in a separate background task
@@ -165,6 +170,9 @@ function useBackgroundCredentialTest() {
activeTestRef.current.timeoutId = setTimeout(poll, POLL_INTERVAL_MS);
}
} catch {
+ if (activeTestRef.current?.workflowRunId !== test.workflowRunId) {
+ return;
+ }
// Network error — increment counter and bail if too many consecutive failures
if (activeTestRef.current) {
activeTestRef.current.errorCount++;
@@ -183,33 +191,40 @@ function useBackgroundCredentialTest() {
}
}, [credentialGetter, queryClient, cleanup]);
- // Rehydrate polling from persisted store after a full page reload.
- // If a test was in progress before the reload, resume polling so the
- // link stays visible and the toast fires when the test finishes.
- const rehydratedRef = useRef(false);
+ // Adopt the persisted test whenever this tab isn't already polling it —
+ // covers mount rehydration AND tests started in other tabs (storage sync).
+ const storedTest = useCredentialTestStore((s) => s.activeTest);
useEffect(() => {
- if (rehydratedRef.current) return;
- rehydratedRef.current = true;
+ if (!storedTest) return;
- const stored = useCredentialTestStore.getState().activeTest;
- if (!stored || activeTestRef.current) return;
+ const current = activeTestRef.current;
+ if (
+ current?.workflowRunId === storedTest.workflowRunId &&
+ current.timeoutId !== null
+ ) {
+ return;
+ }
// If the persisted test already exceeded the timeout, clean up
- if (Date.now() - stored.startTime > MAX_POLL_DURATION_MS) {
- clearActiveTest();
+ if (Date.now() - storedTest.startTime > MAX_POLL_DURATION_MS) {
+ clearActiveTest(storedTest.workflowRunId);
return;
}
+ if (current?.timeoutId) {
+ clearTimeout(current.timeoutId);
+ }
activeTestRef.current = {
- credentialId: stored.credentialId,
- workflowRunId: stored.workflowRunId,
- url: stored.url,
- startTime: stored.startTime,
- timeoutId: setTimeout(poll, POLL_INTERVAL_MS),
+ credentialId: storedTest.credentialId,
+ workflowRunId: storedTest.workflowRunId,
+ url: storedTest.url,
+ startTime: storedTest.startTime,
+ // Poll immediately, not after a full interval: the test may have already
+ // finished, and the credentials list shouldn't sit stale any longer.
+ timeoutId: setTimeout(poll, 0),
errorCount: 0,
};
- // eslint-disable-next-line react-hooks/exhaustive-deps -- intentionally runs once on mount
- }, []);
+ }, [storedTest, poll, clearActiveTest]);
const startBackgroundTest = useCallback(
async (credentialId: string, url: string, userContext?: string) => {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.composerMode.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.composerMode.test.tsx
new file mode 100644
index 000000000..881a6f1cd
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.composerMode.test.tsx
@@ -0,0 +1,281 @@
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+ FeatureFlagContext,
+ FeatureFlagValueContext,
+} from "@/hooks/useFeatureFlag";
+
+type StreamBody = {
+ message: string;
+ mode: string | null;
+ code_block: boolean | null;
+};
+type StreamCall = {
+ body: StreamBody;
+ onMessage: (payload: unknown) => boolean;
+ resolve: () => void;
+ reject: (error: unknown) => void;
+};
+
+const { streamCalls, postStreaming, cancelPost, historyResponse } = vi.hoisted(
+ () => {
+ const calls: StreamCall[] = [];
+ const post = vi.fn().mockResolvedValue({});
+ const streaming = vi.fn(
+ (
+ _path: string,
+ body: StreamBody,
+ onMessage: (payload: unknown) => boolean,
+ ) =>
+ new Promise<void>((resolve, reject) => {
+ calls.push({ body, onMessage, resolve, reject });
+ }),
+ );
+ const history = {
+ data: {
+ workflow_copilot_chat_id: null as string | null,
+ chat_history: [] as unknown[],
+ proposed_workflow: null as Record<string, unknown> | null,
+ auto_accept: false,
+ },
+ };
+ return {
+ streamCalls: calls,
+ postStreaming: streaming,
+ cancelPost: post,
+ historyResponse: history,
+ };
+ },
+);
+
+vi.mock("@/api/sse", () => ({
+ getSseClient: vi.fn().mockResolvedValue({ postStreaming }),
+}));
+
+vi.mock("@/api/AxiosClient", () => ({
+ getClient: vi.fn().mockResolvedValue({
+ get: vi.fn().mockImplementation(() => Promise.resolve(historyResponse)),
+ post: cancelPost,
+ }),
+}));
+
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => null,
+}));
+
+vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));
+
+vi.mock("react-router-dom", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("react-router-dom")>();
+ return {
+ ...actual,
+ useParams: () => ({
+ workflowPermanentId: "wpid_1",
+ workflowRunId: undefined,
+ }),
+ };
+});
+
+const saveData = {
+ title: "Test WF",
+ workflow: {
+ workflow_id: "wf_1",
+ workflow_permanent_id: "wpid_1",
+ description: "",
+ totp_verification_url: null,
+ is_saved_task: false,
+ status: "published",
+ },
+ settings: {
+ proxyLocation: null,
+ webhookCallbackUrl: null,
+ persistBrowserSession: false,
+ browserProfileId: null,
+ model: null,
+ maxScreenshotScrolls: null,
+ extraHttpHeaders: null,
+ runWith: "agent",
+ scriptCacheKey: "",
+ aiFallback: true,
+ codeVersion: 2,
+ runSequentially: false,
+ sequentialKey: null,
+ },
+ parameters: [],
+ blocks: [],
+ workflowDefinitionVersion: 1,
+};
+
+vi.mock("@/store/WorkflowHasChangesStore", () => ({
+ useWorkflowHasChangesStore: () => ({ getSaveData: () => saveData }),
+}));
+
+import { WorkflowCopilotChat } from "./WorkflowCopilotChat";
+
+type FlagConfig = {
+ copilotV2?: boolean;
+ codeBlockMode?: boolean;
+ defaultMode?: string;
+};
+
+async function renderChat(flags: FlagConfig) {
+ const booleanFlags: Record<string, boolean> = {
+ ENABLE_WORKFLOW_COPILOT_V2: flags.copilotV2 ?? false,
+ WORKFLOW_COPILOT_CODE_BLOCK_MODE: flags.codeBlockMode ?? false,
+ CODE_BLOCK_ACCESS: flags.codeBlockMode ?? false,
+ };
+ const valueFlags: Record<string, string | undefined> = {
+ WORKFLOW_COPILOT_DEFAULT_MODE: flags.defaultMode,
+ };
+ const view = render(
+ <FeatureFlagContext.Provider value={(name) => booleanFlags[name]}>
+ <FeatureFlagValueContext.Provider value={(name) => valueFlags[name]}>
+ <WorkflowCopilotChat />
+ </FeatureFlagValueContext.Provider>
+ </FeatureFlagContext.Provider>,
+ );
+ await waitFor(() =>
+ expect(screen.getByPlaceholderText(/Message Skyvern Copilot/)).toBeTruthy(),
+ );
+ return view;
+}
+
+function textarea(): HTMLTextAreaElement {
+ return screen.getByRole("textbox") as HTMLTextAreaElement;
+}
+
+async function submit(value: string) {
+ fireEvent.change(textarea(), { target: { value } });
+ await act(async () => {
+ fireEvent.keyDown(textarea(), { key: "Enter" });
+ });
+}
+
+async function switchToBuild() {
+ await act(async () => {
+ fireEvent.pointerDown(screen.getByRole("button", { name: "Switch mode" }), {
+ button: 0,
+ ctrlKey: false,
+ });
+ });
+ const buildItem = await screen.findByRole("menuitem", { name: /Build/ });
+ await act(async () => {
+ fireEvent.click(buildItem);
+ });
+}
+
+beforeEach(() => {
+ HTMLElement.prototype.scrollIntoView = vi.fn();
+ HTMLElement.prototype.scrollTo = vi.fn();
+ streamCalls.length = 0;
+ postStreaming.mockClear();
+ cancelPost.mockClear();
+ historyResponse.data = {
+ workflow_copilot_chat_id: null,
+ chat_history: [],
+ proposed_workflow: null,
+ auto_accept: false,
+ };
+});
+
+afterEach(() => {
+ cleanup();
+});
+
+describe("WorkflowCopilotChat — composer default mode variant", () => {
+ it("defaults to Build with code OFF when the variant is unset (new baseline)", async () => {
+ await renderChat({ copilotV2: true, codeBlockMode: true });
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("build");
+ expect(streamCalls[0]?.body.code_block).toBe(false);
+ });
+
+ it("sends code_block=true for the build_code override variant", async () => {
+ await renderChat({
+ copilotV2: true,
+ codeBlockMode: true,
+ defaultMode: "build_code",
+ });
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("build");
+ expect(streamCalls[0]?.body.code_block).toBe(true);
+ });
+
+ it("sends code_block=false for the build_no_code variant", async () => {
+ await renderChat({
+ copilotV2: true,
+ codeBlockMode: true,
+ defaultMode: "build_no_code",
+ });
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("build");
+ expect(streamCalls[0]?.body.code_block).toBe(false);
+ });
+
+ it("defaults to Ask for the ask variant and sends code_block=null", async () => {
+ await renderChat({
+ copilotV2: true,
+ codeBlockMode: true,
+ defaultMode: "ask",
+ });
+ await submit("answer a question");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("ask");
+ expect(streamCalls[0]?.body.code_block).toBe(null);
+ });
+
+ it("defaults to Ask for the ask_code variant and sends code_block=null while in Ask", async () => {
+ await renderChat({
+ copilotV2: true,
+ codeBlockMode: true,
+ defaultMode: "ask_code",
+ });
+ await submit("answer a question");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("ask");
+ expect(streamCalls[0]?.body.code_block).toBe(null);
+ });
+
+ it("lands on code ON when ask_code switches to Build", async () => {
+ await renderChat({
+ copilotV2: true,
+ codeBlockMode: true,
+ defaultMode: "ask_code",
+ });
+ await switchToBuild();
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("build");
+ expect(streamCalls[0]?.body.code_block).toBe(true);
+ });
+
+ it("sends code_block=null when the code-block flag is off, ignoring the variant", async () => {
+ await renderChat({
+ copilotV2: true,
+ codeBlockMode: false,
+ defaultMode: "build_no_code",
+ });
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+
+ expect(streamCalls[0]?.body.mode).toBe("build");
+ expect(streamCalls[0]?.body.code_block).toBe(null);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx
index 7230f6673..fbc499d02 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.queue.test.tsx
@@ -18,8 +18,8 @@ type StreamCall = {
resolve: () => void;
reject: (error: unknown) => void;
};
-const { streamCalls, postStreaming, cancelPost, historyResponse } = vi.hoisted(
- () => {
+const { streamCalls, postStreaming, cancelPost, historyResponse, speechState } =
+ vi.hoisted(() => {
const calls: StreamCall[] = [];
const post = vi.fn().mockResolvedValue({});
const streaming = vi.fn(
@@ -45,14 +45,23 @@ const { streamCalls, postStreaming, cancelPost, historyResponse } = vi.hoisted(
auto_accept: false,
},
};
+ const speech = {
+ isSupported: false,
+ isListening: false,
+ isHearingSpeech: false,
+ start: vi.fn(),
+ stop: vi.fn<() => Promise<Blob | null>>().mockResolvedValue(null),
+ toggle: vi.fn(),
+ takeAudioBlob: vi.fn<() => Blob | null>().mockReturnValue(null),
+ };
return {
streamCalls: calls,
postStreaming: streaming,
cancelPost: post,
historyResponse: history,
+ speechState: speech,
};
- },
-);
+ });
vi.mock("@/api/sse", () => ({
getSseClient: vi.fn().mockResolvedValue({ postStreaming }),
@@ -69,6 +78,10 @@ vi.mock("@/hooks/useCredentialGetter", () => ({
useCredentialGetter: () => null,
}));
+vi.mock("@/hooks/useSpeechToTextField", () => ({
+ useSpeechToTextField: () => speechState,
+}));
+
vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));
vi.mock("react-router-dom", async (importOriginal) => {
@@ -183,6 +196,15 @@ beforeEach(() => {
streamCalls.length = 0;
postStreaming.mockClear();
cancelPost.mockClear();
+ speechState.isSupported = false;
+ speechState.isListening = false;
+ speechState.isHearingSpeech = false;
+ speechState.start.mockClear();
+ speechState.stop.mockClear();
+ speechState.stop.mockResolvedValue(null);
+ speechState.toggle.mockClear();
+ speechState.takeAudioBlob.mockClear();
+ speechState.takeAudioBlob.mockReturnValue(null);
historyResponse.data = {
workflow_copilot_chat_id: null,
chat_history: [],
@@ -205,6 +227,28 @@ describe("WorkflowCopilotChat — keep the chat live during a turn", () => {
expect(screen.getByRole("button", { name: "Cancel run" })).toBeTruthy();
});
+ it("still sends the message when dictation audio upload fails", async () => {
+ await renderChat();
+ speechState.takeAudioBlob.mockReturnValueOnce(
+ new Blob(["audio"], { type: "audio/webm" }),
+ );
+ cancelPost.mockRejectedValueOnce(new Error("upload failed"));
+
+ await submit("dictated prompt");
+
+ await waitFor(() => expect(cancelPost).toHaveBeenCalledTimes(1));
+ expect(cancelPost).toHaveBeenCalledWith(
+ "/workflow/copilot/chat-audio",
+ expect.any(FormData),
+ expect.any(Object),
+ );
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+ expect(streamCalls[0]?.body).toMatchObject({
+ message: "dictated prompt",
+ audio_artifact_id: null,
+ });
+ });
+
it("queues a second submit instead of starting a concurrent stream", async () => {
await renderChat();
await submit("first message");
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 7932ea3f2..50f9869c2 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -41,6 +41,7 @@ import {
WorkflowCopilotChatRequest,
WorkflowCopilotClearProposedWorkflowRequest,
WorkflowCopilotApplyProposedWorkflowRequest,
+ WorkflowCopilotAudioUploadResponse,
} from "./workflowCopilotTypes";
import { shouldWaitForLiveBrowser } from "./browserReadiness";
import {
@@ -61,7 +62,7 @@ import {
import { computeFollowSignature, useStickToBottom } from "./useStickToBottom";
import { useSpeechToTextField } from "@/hooks/useSpeechToTextField";
import { SpeechInputButton } from "@/components/SpeechInputButton";
-import { useFeatureFlag } from "@/hooks/useFeatureFlag";
+import { useFeatureFlag, useFeatureFlagValue } from "@/hooks/useFeatureFlag";
import {
DropdownMenu,
DropdownMenuTrigger,
@@ -80,6 +81,27 @@ import { cn } from "@/util/utils";
// handful of turns; this ceiling guards a runaway long-running chat.
const MAX_TURN_SNAPSHOTS = 20;
+type ComposerDefaultVariant =
+ | "build"
+ | "build_code"
+ | "build_no_code"
+ | "ask"
+ | "ask_code";
+
+function normalizeComposerDefaultVariant(
+ variant: string | undefined,
+): ComposerDefaultVariant {
+ if (
+ variant === "ask" ||
+ variant === "ask_code" ||
+ variant === "build_code" ||
+ variant === "build_no_code"
+ ) {
+ return variant;
+ }
+ return "build";
+}
+
function formatElapsedSeconds(ms: number): string {
const seconds = Math.max(0, Math.round(ms / 1000));
const m = Math.floor(seconds / 60);
@@ -194,11 +216,13 @@ type QueuedPrompt = {
id: string;
content: string;
reason: QueuedPromptReason;
+ audioBlob?: Blob | null;
};
type SendOptions = {
queuedMessageId?: string;
skipQueue?: boolean;
+ audioBlob?: Blob | null;
};
type WorkflowCopilotSsePayload =
@@ -350,15 +374,56 @@ export function WorkflowCopilotChat({
initialMessage,
onInitialMessageConsumed,
}: WorkflowCopilotChatProps = {}) {
- const copilotV2Enabled =
- useFeatureFlag("ENABLE_WORKFLOW_COPILOT_V2") === true;
+ const copilotV2Flag = useFeatureFlag("ENABLE_WORKFLOW_COPILOT_V2");
+ const codeBlockModeFlag = useFeatureFlag("WORKFLOW_COPILOT_CODE_BLOCK_MODE");
+ const codeBlockAccessFlag = useFeatureFlag("CODE_BLOCK_ACCESS");
+ const copilotV2Enabled = copilotV2Flag === true;
const codeBlockModeEnabled =
- useFeatureFlag("WORKFLOW_COPILOT_CODE_BLOCK_MODE") === true;
- const [composerMode, setComposerMode] = useState<"ask" | "build">("build");
- const [codeWorkflow, setCodeWorkflow] = useState(true);
+ codeBlockModeFlag === true && codeBlockAccessFlag === true;
+ const defaultModeVariant = useFeatureFlagValue(
+ "WORKFLOW_COPILOT_DEFAULT_MODE",
+ );
+ // The variant configures the initial default only when both gating flags are on.
+ const effectiveDefaultVariant: ComposerDefaultVariant =
+ copilotV2Enabled && codeBlockModeEnabled
+ ? normalizeComposerDefaultVariant(defaultModeVariant)
+ : "build";
+ const [composerMode, setComposerMode] = useState<"ask" | "build">(() =>
+ effectiveDefaultVariant === "ask" || effectiveDefaultVariant === "ask_code"
+ ? "ask"
+ : "build",
+ );
+ const [codeWorkflow, setCodeWorkflow] = useState(
+ () =>
+ effectiveDefaultVariant === "build_code" ||
+ effectiveDefaultVariant === "ask_code",
+ );
+ // Flags arrive asynchronously from /customer; seed the default once they resolve, never again.
+ const composerSeededRef = useRef(false);
+ const flagsResolved =
+ copilotV2Flag !== undefined &&
+ codeBlockModeFlag !== undefined &&
+ codeBlockAccessFlag !== undefined;
+ useEffect(() => {
+ if (composerSeededRef.current || !flagsResolved) {
+ return;
+ }
+ composerSeededRef.current = true;
+ setComposerMode(
+ effectiveDefaultVariant === "ask" ||
+ effectiveDefaultVariant === "ask_code"
+ ? "ask"
+ : "build",
+ );
+ setCodeWorkflow(
+ effectiveDefaultVariant === "build_code" ||
+ effectiveDefaultVariant === "ask_code",
+ );
+ }, [flagsResolved, effectiveDefaultVariant]);
// Build can never be active unless the V2 flag is on.
const isBuild = copilotV2Enabled && composerMode === "build";
- const showCodeToggle = isBuild && codeBlockModeEnabled;
+ const codeToggleAllowed = effectiveDefaultVariant !== "build_no_code";
+ const showCodeToggle = isBuild && codeBlockModeEnabled && codeToggleAllowed;
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [proposedWorkflow, setProposedWorkflow] =
useState<WorkflowApiResponse | null>(null);
@@ -540,7 +605,9 @@ export function WorkflowCopilotChat({
isSupported: isSpeechSupported,
isListening: isSpeechListening,
isHearingSpeech: isSpeechHearing,
+ stop: stopSpeech,
toggle: toggleSpeech,
+ takeAudioBlob: takeSpeechAudioBlob,
} = useSpeechToTextField({
value: inputValue,
onChange: setInputValue,
@@ -694,6 +761,37 @@ export function WorkflowCopilotChat({
return latestChatId;
};
+ const uploadDictationAudio = useCallback(
+ async (audioBlob: Blob): Promise<WorkflowCopilotAudioUploadResponse> => {
+ if (!workflowPermanentId) {
+ throw new Error("Missing workflow permanent ID for audio upload.");
+ }
+
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ const formData = new FormData();
+ formData.append("workflow_permanent_id", workflowPermanentId);
+ const chatId = workflowCopilotChatIdRef.current?.trim();
+ if (chatId) {
+ formData.append("workflow_copilot_chat_id", chatId);
+ }
+ formData.append("file", audioBlob, `dictation-${Date.now()}.webm`);
+
+ const response = await client.post<WorkflowCopilotAudioUploadResponse>(
+ "/workflow/copilot/chat-audio",
+ formData,
+ {
+ headers: {
+ "Content-Type": "multipart/form-data",
+ },
+ },
+ );
+ setWorkflowCopilotChatId(response.data.workflow_copilot_chat_id);
+ workflowCopilotChatIdRef.current = response.data.workflow_copilot_chat_id;
+ return response.data;
+ },
+ [credentialGetter, workflowPermanentId],
+ );
+
const clearProposedWorkflow = async (autoAcceptValue: boolean) => {
const clearProposalByChatId = async (chatId: string) => {
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -948,11 +1046,25 @@ export function WorkflowCopilotChat({
});
return;
}
+
+ let messageAudioBlob = options.audioBlob ?? null;
+ if (!messageAudioBlob && messageOverride === undefined) {
+ if (isSpeechListening) {
+ messageAudioBlob = await stopSpeech();
+ }
+ messageAudioBlob = messageAudioBlob ?? takeSpeechAudioBlob();
+ }
+
if (action === "queue_working" || action === "queue_live_browser") {
const reason: QueuedPromptReason =
action === "queue_working" ? "working" : "live_browser";
const queuedId = options.queuedMessageId ?? crypto.randomUUID();
- updateQueuedPrompt({ id: queuedId, content: candidate, reason });
+ updateQueuedPrompt({
+ id: queuedId,
+ content: candidate,
+ reason,
+ audioBlob: messageAudioBlob,
+ });
// First queue adds the user bubble; a re-queue (a working drain that
// then had to wait for the browser) reuses the existing bubble.
if (!options.queuedMessageId) {
@@ -1013,6 +1125,8 @@ export function WorkflowCopilotChat({
const saveData = getSaveData();
const workflowId = saveData?.workflow.workflow_id;
let workflowYaml = "";
+ let chatIdForRequest = workflowCopilotChatId;
+ let audioArtifactId: string | null = null;
if (!workflowId) {
toast({
@@ -1100,6 +1214,16 @@ export function WorkflowCopilotChat({
} as WorkflowApiResponse;
}
+ if (messageAudioBlob) {
+ try {
+ const uploadResponse = await uploadDictationAudio(messageAudioBlob);
+ chatIdForRequest = uploadResponse.workflow_copilot_chat_id;
+ audioArtifactId = uploadResponse.audio_artifact_id;
+ } catch (error) {
+ console.warn("Failed to upload dictation audio:", error);
+ }
+ }
+
const handleProcessingUpdate = (
payload: WorkflowCopilotProcessingUpdate,
) => {
@@ -1225,13 +1349,14 @@ export function WorkflowCopilotChat({
{
workflow_id: workflowId,
workflow_permanent_id: workflowPermanentId,
- workflow_copilot_chat_id: workflowCopilotChatId,
+ workflow_copilot_chat_id: chatIdForRequest,
workflow_run_id: workflowRunId,
browser_session_id: liveBrowserSessionId ?? null,
message: messageContent,
+ audio_artifact_id: audioArtifactId,
workflow_yaml: workflowYaml,
mode: copilotV2Enabled ? composerMode : null,
- code_block: showCodeToggle ? codeWorkflow : null,
+ code_block: isBuild && codeBlockModeEnabled ? codeWorkflow : null,
cancel_token: cancelToken,
} as WorkflowCopilotChatRequest,
(payload) => {
@@ -1339,17 +1464,22 @@ export function WorkflowCopilotChat({
applyStoredNarrativeEvent,
applyWorkflowUpdate,
autoAccept,
+ codeBlockModeEnabled,
codeWorkflow,
composerMode,
copilotV2Enabled,
credentialGetter,
getSaveData,
inputValue,
+ isSpeechListening,
+ isBuild,
isLiveBrowserReady,
liveBrowserSessionId,
requiresLiveBrowser,
- showCodeToggle,
+ stopSpeech,
+ takeSpeechAudioBlob,
updateQueuedPrompt,
+ uploadDictationAudio,
workflowCopilotChatId,
workflowPermanentId,
workflowRunId,
@@ -1385,6 +1515,7 @@ export function WorkflowCopilotChat({
handleSend(promptToSend.content, {
queuedMessageId: promptToSend.id,
skipQueue: drainAction === "drain_skip_queue",
+ audioBlob: promptToSend.audioBlob,
}).catch((error) => {
console.error("Queued send failed:", error);
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index 4d37ca8c7..7afd68ffa 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -24,6 +24,7 @@ export interface WorkflowCopilotChatMessage {
workflow_copilot_chat_id: string;
sender: WorkflowCopilotChatSender;
content: string;
+ audio_artifact_id?: string | null;
global_llm_context: string | null;
created_at: string;
modified_at: string;
@@ -36,6 +37,7 @@ export interface WorkflowCopilotChatRequest {
workflow_run_id?: string | null;
browser_session_id?: string | null;
message: string;
+ audio_artifact_id?: string | null;
workflow_yaml: string;
mode?: "ask" | "build" | null;
code_block?: boolean | null;
@@ -49,6 +51,7 @@ export interface WorkflowCopilotCancelRequest {
export interface WorkflowCopilotChatHistoryMessage {
sender: WorkflowCopilotChatSender;
content: string;
+ audio_artifact_id?: string | null;
created_at: string;
narrative_payload?: Record<string, unknown> | null;
}
@@ -70,6 +73,11 @@ export interface WorkflowCopilotApplyProposedWorkflowRequest {
auto_accept: boolean;
}
+export interface WorkflowCopilotAudioUploadResponse {
+ workflow_copilot_chat_id: string;
+ audio_artifact_id: string;
+}
+
export type WorkflowCopilotStreamMessageType =
| "processing_update"
| "response"
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts
index 82b59edf5..93d0ce945 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowRunTypes.ts
@@ -165,11 +165,17 @@ export function findUnexecutedDefinedBlocks(
definedBlocks: Array<WorkflowBlock>,
timelineItems: Array<WorkflowRunTimelineItem>,
): Array<WorkflowBlock> {
+ // Walk the whole tree: defined blocks executed inside a conditional's
+ // branch appear as nested children, not roots, and must not be reported
+ // as unexecuted.
const executedLabels = new Set<string>();
- for (const item of timelineItems) {
+ const stack = [...timelineItems];
+ while (stack.length > 0) {
+ const item = stack.pop()!;
if (isBlockItem(item) && item.block.label !== null) {
executedLabels.add(item.block.label);
}
+ stack.push(...item.children);
}
return definedBlocks.filter((block) => !executedLabels.has(block.label));
}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.test.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.test.tsx
new file mode 100644
index 000000000..b7c74a31c
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.test.tsx
@@ -0,0 +1,353 @@
+// @vitest-environment jsdom
+
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { type ReactNode } from "react";
+
+import { Status } from "@/api/types";
+import type {
+ WorkflowRunBlock,
+ WorkflowRunTimelineBlockItem as TimelineBlockItem,
+ WorkflowRunTimelineItem,
+} from "../types/workflowRunTypes";
+import type { WorkflowBlock } from "../types/workflowTypes";
+import type { WorkflowRunOverviewActiveElement } from "./WorkflowRunOverview";
+import { WorkflowRunTimeline } from "./WorkflowRunTimeline";
+
+const mocks = vi.hoisted(() => ({
+ workflowRun: undefined as unknown,
+ timeline: undefined as unknown,
+}));
+
+vi.mock("../hooks/useWorkflowRunWithWorkflowQuery", () => ({
+ useWorkflowRunWithWorkflowQuery: () => ({
+ data: mocks.workflowRun,
+ isLoading: false,
+ }),
+}));
+vi.mock("../hooks/useWorkflowRunTimelineQuery", () => ({
+ useWorkflowRunTimelineQuery: () => ({
+ data: mocks.timeline,
+ isLoading: false,
+ }),
+}));
+// Radix ScrollArea needs ResizeObserver, which jsdom doesn't provide.
+vi.mock("@/components/ui/scroll-area", () => ({
+ ScrollArea: ({ children }: { children?: ReactNode }) => <div>{children}</div>,
+ ScrollAreaViewport: ({ children }: { children?: ReactNode }) => (
+ <div>{children}</div>
+ ),
+}));
+
+function buildBlock(
+ overrides: Partial<WorkflowRunBlock> = {},
+): WorkflowRunBlock {
+ return {
+ workflow_run_block_id: "wrb_default",
+ workflow_run_id: "wr_default",
+ parent_workflow_run_block_id: null,
+ block_type: "http_request",
+ label: null,
+ description: null,
+ title: null,
+ status: Status.Completed,
+ failure_reason: null,
+ output: null,
+ continue_on_failure: false,
+ task_id: null,
+ url: null,
+ navigation_goal: null,
+ navigation_payload: null,
+ data_extraction_goal: null,
+ data_schema: null,
+ terminate_criterion: null,
+ complete_criterion: null,
+ include_action_history_in_verification: null,
+ engine: null,
+ actions: null,
+ created_at: "2026-01-01T00:00:00Z",
+ modified_at: "2026-01-01T00:00:00Z",
+ duration: null,
+ loop_values: null,
+ current_value: null,
+ current_index: null,
+ ...overrides,
+ };
+}
+
+function buildBlockItem(
+ block: WorkflowRunBlock,
+ children: Array<WorkflowRunTimelineItem> = [],
+): TimelineBlockItem {
+ return {
+ type: "block",
+ block,
+ children,
+ thought: null,
+ created_at: block.created_at,
+ modified_at: block.modified_at,
+ };
+}
+
+const noop = () => {};
+
+function renderTimeline(activeItem: WorkflowRunOverviewActiveElement) {
+ return render(
+ <WorkflowRunTimeline
+ activeItem={activeItem}
+ onLiveStreamSelected={noop}
+ onActionItemSelected={noop}
+ onBlockItemSelected={noop}
+ onThoughtItemSelected={noop}
+ onIterationSelected={noop}
+ />,
+ );
+}
+
+function expectDomOrder(labels: Array<string>) {
+ const nodes = labels.map((label) => screen.getByText(label));
+ for (let i = 0; i < nodes.length - 1; i++) {
+ expect(
+ nodes[i]!.compareDocumentPosition(nodes[i + 1]!) &
+ Node.DOCUMENT_POSITION_FOLLOWING,
+ `expected "${labels[i]}" to render before "${labels[i + 1]}"`,
+ ).toBeTruthy();
+ }
+}
+
+afterEach(() => {
+ cleanup();
+ mocks.workflowRun = undefined;
+ mocks.timeline = undefined;
+});
+
+describe("WorkflowRunTimeline", () => {
+ it("renders blocks in global execution order, not branch-tree order", () => {
+ // Regression: block_8/block_12 are branch children of conditional
+ // block_2 but executed after root loop block_5. The tree rendering used
+ // to print them above the loop, implying the run continued past the
+ // terminated block.
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_block_2",
+ block_type: "conditional",
+ label: "block_2",
+ created_at: "2026-06-10T07:16:29Z",
+ });
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_block_5",
+ block_type: "for_loop",
+ label: "block_5",
+ loop_values: ["account_1"],
+ created_at: "2026-06-10T07:19:06Z",
+ });
+ const loopChild = buildBlock({
+ workflow_run_block_id: "wrb_goto_viewbill",
+ block_type: "navigation",
+ label: "goto_viewbill",
+ parent_workflow_run_block_id: "wrb_block_5",
+ created_at: "2026-06-10T07:19:11Z",
+ current_index: 0,
+ });
+ const branchConditional = buildBlock({
+ workflow_run_block_id: "wrb_block_8",
+ block_type: "conditional",
+ label: "block_8",
+ parent_workflow_run_block_id: "wrb_block_2",
+ created_at: "2026-06-10T07:29:32Z",
+ });
+ const terminated = buildBlock({
+ workflow_run_block_id: "wrb_block_12",
+ block_type: "navigation",
+ label: "block_12",
+ status: Status.Terminated,
+ parent_workflow_run_block_id: "wrb_block_8",
+ created_at: "2026-06-10T07:39:31Z",
+ });
+
+ mocks.workflowRun = {
+ status: Status.Terminated,
+ total_steps: 0,
+ credits_used: 0,
+ cached_credits_used: 0,
+ workflow: {
+ workflow_definition: { blocks: [], finally_block_label: null },
+ },
+ };
+ mocks.timeline = [
+ buildBlockItem(conditional, [
+ buildBlockItem(branchConditional, [buildBlockItem(terminated)]),
+ ]),
+ buildBlockItem(loop, [buildBlockItem(loopChild)]),
+ ];
+
+ // Selecting the loop child keeps the loop and its iteration expanded so
+ // the nested row is visible for the ordering assertion.
+ renderTimeline(loopChild);
+
+ expectDomOrder([
+ "block_2",
+ "block_5",
+ "goto_viewbill",
+ "block_8",
+ "block_12",
+ ]);
+ });
+
+ it("renders the terminated block as the last row", () => {
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "branch_check",
+ created_at: "2026-06-10T07:16:29Z",
+ });
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_loop",
+ block_type: "for_loop",
+ label: "download_loop",
+ created_at: "2026-06-10T07:19:06Z",
+ });
+ const terminated = buildBlock({
+ workflow_run_block_id: "wrb_terminated",
+ block_type: "navigation",
+ label: "final_navigation",
+ status: Status.Terminated,
+ parent_workflow_run_block_id: "wrb_cond",
+ created_at: "2026-06-10T07:39:31Z",
+ });
+
+ mocks.workflowRun = {
+ status: Status.Terminated,
+ total_steps: 0,
+ credits_used: 0,
+ cached_credits_used: 0,
+ workflow: {
+ workflow_definition: { blocks: [], finally_block_label: null },
+ },
+ };
+ mocks.timeline = [
+ buildBlockItem(conditional, [buildBlockItem(terminated)]),
+ buildBlockItem(loop),
+ ];
+
+ renderTimeline(null);
+
+ expectDomOrder(["branch_check", "download_loop", "final_navigation"]);
+ });
+
+ it("does not show a 'did not execute' ghost for blocks that ran inside a branch", () => {
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "branch_check",
+ created_at: "2026-06-10T07:16:29Z",
+ });
+ const branchChild = buildBlock({
+ workflow_run_block_id: "wrb_branch_child",
+ block_type: "navigation",
+ label: "block_8",
+ parent_workflow_run_block_id: "wrb_cond",
+ created_at: "2026-06-10T07:29:32Z",
+ });
+
+ mocks.workflowRun = {
+ status: Status.Completed,
+ total_steps: 0,
+ credits_used: 0,
+ cached_credits_used: 0,
+ workflow: {
+ workflow_definition: {
+ finally_block_label: null,
+ blocks: [
+ { block_type: "navigation", label: "block_8" },
+ { block_type: "navigation", label: "never_ran" },
+ ] as unknown as Array<WorkflowBlock>,
+ },
+ },
+ };
+ mocks.timeline = [
+ buildBlockItem(conditional, [buildBlockItem(branchChild)]),
+ ];
+
+ renderTimeline(null);
+
+ // block_8 executed (inside the branch) — exactly one row, no ghost.
+ expect(screen.getAllByText("block_8")).toHaveLength(1);
+ // never_ran is the only unexecuted defined block.
+ expect(screen.getAllByText("did not execute")).toHaveLength(1);
+ expect(screen.getByText("never_ran")).toBeDefined();
+ });
+
+ it("labels not-taken branch ghosts as skipped and unreached ones as did not execute", () => {
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "branch_check",
+ created_at: "2026-06-10T07:16:29Z",
+ output: {
+ evaluations: [
+ { is_matched: true, next_block_label: "block_8" },
+ { is_matched: false, next_block_label: "other_path" },
+ ],
+ } as WorkflowRunBlock["output"],
+ });
+ const takenChild = buildBlock({
+ workflow_run_block_id: "wrb_block_8",
+ block_type: "navigation",
+ label: "block_8",
+ status: Status.Terminated,
+ parent_workflow_run_block_id: "wrb_cond",
+ created_at: "2026-06-10T07:29:32Z",
+ });
+
+ mocks.workflowRun = {
+ status: Status.Terminated,
+ total_steps: 0,
+ credits_used: 0,
+ cached_credits_used: 0,
+ workflow: {
+ workflow_definition: {
+ finally_block_label: null,
+ blocks: [
+ {
+ block_type: "conditional",
+ label: "branch_check",
+ branch_conditions: [
+ {
+ id: "br_taken",
+ next_block_label: "block_8",
+ is_default: false,
+ },
+ {
+ id: "br_other",
+ next_block_label: "other_path",
+ is_default: true,
+ },
+ ],
+ },
+ {
+ block_type: "navigation",
+ label: "block_8",
+ next_block_label: "tail_block",
+ },
+ { block_type: "navigation", label: "other_path" },
+ { block_type: "navigation", label: "tail_block" },
+ ] as unknown as Array<WorkflowBlock>,
+ },
+ },
+ };
+ mocks.timeline = [
+ buildBlockItem(conditional, [buildBlockItem(takenChild)]),
+ ];
+
+ renderTimeline(null);
+
+ const skippedBadge = screen.getByText("skipped");
+ expect(skippedBadge.closest("div.min-w-0")?.textContent).toContain(
+ "other_path",
+ );
+ const notReachedBadge = screen.getByText("did not execute");
+ expect(notReachedBadge.closest("div.min-w-0")?.textContent).toContain(
+ "tail_block",
+ );
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.tsx
index 07d80b52f..043e71271 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimeline.tsx
@@ -9,7 +9,6 @@ import { useWorkflowRunTimelineQuery } from "../hooks/useWorkflowRunTimelineQuer
import {
countActionsInTimeline,
countCompletedTopLevelBlocks,
- findUnexecutedDefinedBlocks,
isBlockItem,
isObserverThought,
isThoughtItem,
@@ -24,6 +23,10 @@ import {
import { ThoughtCard } from "./ThoughtCard";
import { WorkflowRunTimelineBlockItem } from "./WorkflowRunTimelineBlockItem";
import { WorkflowRunTimelineUnexecutedBlockItem } from "./WorkflowRunTimelineUnexecutedBlockItem";
+import {
+ classifyUnexecutedDefinedBlocks,
+ flattenTimelineChronologically,
+} from "./workflowTimelineUtils";
type Props = {
activeItem: WorkflowRunOverviewActiveElement;
@@ -74,26 +77,6 @@ function buildBlockOrderIndex(
return new Map(blocks.map((block, index) => [block.id, index + 1]));
}
-function toTimelineTime(value: string): number {
- const time = new Date(value).getTime();
- return Number.isNaN(time) ? Number.MAX_SAFE_INTEGER : time;
-}
-
-function sortTimelineTopDown(
- items: Array<WorkflowRunTimelineItem>,
-): Array<WorkflowRunTimelineItem> {
- if (items.length === 0) return items;
- return items
- .map((item) => ({
- ...item,
- children: sortTimelineTopDown(item.children),
- }))
- .sort(
- (left, right) =>
- toTimelineTime(left.created_at) - toTimelineTime(right.created_at),
- );
-}
-
function WorkflowRunTimeline({
activeItem,
activeIteration = null,
@@ -109,7 +92,7 @@ function WorkflowRunTimeline({
const { data: workflowRunTimeline, isLoading: workflowRunTimelineIsLoading } =
useWorkflowRunTimelineQuery();
const displayTimeline = useMemo(
- () => sortTimelineTopDown(workflowRunTimeline ?? []),
+ () => flattenTimelineChronologically(workflowRunTimeline ?? []),
[workflowRunTimeline],
);
const blockOrder = useMemo(
@@ -158,7 +141,7 @@ function WorkflowRunTimeline({
const totalBlocks = definedBlocks.length;
const completedBlocks = countCompletedTopLevelBlocks(workflowRunTimeline);
const unexecutedBlocks = workflowRunIsFinalized
- ? findUnexecutedDefinedBlocks(definedBlocks, workflowRunTimeline)
+ ? classifyUnexecutedDefinedBlocks(definedBlocks, workflowRunTimeline)
: [];
return (
@@ -281,10 +264,11 @@ function WorkflowRunTimeline({
}
return null;
})}
- {unexecutedBlocks.map((block) => (
+ {unexecutedBlocks.map(({ block, reason }) => (
<WorkflowRunTimelineUnexecutedBlockItem
key={`unexecuted-${block.label}`}
block={block}
+ reason={reason}
/>
))}
</div>
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineUnexecutedBlockItem.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineUnexecutedBlockItem.tsx
index d3cdf54db..a8b1b869d 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineUnexecutedBlockItem.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunTimelineUnexecutedBlockItem.tsx
@@ -1,9 +1,27 @@
import { workflowBlockTitle } from "../editor/nodes/types";
import { WorkflowBlockIcon } from "../editor/nodes/WorkflowBlockIcon";
import { WorkflowBlock } from "../types/workflowTypes";
+import { type UnexecutedBlockReason } from "./workflowTimelineUtils";
type Props = {
block: WorkflowBlock;
+ reason?: UnexecutedBlockReason;
+};
+
+const reasonBadge: Record<
+ UnexecutedBlockReason,
+ { label: string; title: string }
+> = {
+ branch_not_taken: {
+ label: "skipped",
+ title:
+ "This block did not execute because its conditional branch was not taken",
+ },
+ not_reached: {
+ label: "did not execute",
+ title:
+ "This block did not execute because the workflow ended before reaching it",
+ },
};
function getTypeLabel(block: WorkflowBlock): string {
@@ -24,9 +42,13 @@ function getTypeLabel(block: WorkflowBlock): string {
}
}
-function WorkflowRunTimelineUnexecutedBlockItem({ block }: Props) {
+function WorkflowRunTimelineUnexecutedBlockItem({
+ block,
+ reason = "not_reached",
+}: Props) {
const typeLabel = getTypeLabel(block);
const blockTypeTitle = workflowBlockTitle[block.block_type];
+ const badge = reasonBadge[reason];
return (
<div className="min-w-0 opacity-60">
<div className="flex min-h-[28px] items-stretch text-xs">
@@ -51,9 +73,9 @@ function WorkflowRunTimelineUnexecutedBlockItem({ block }: Props) {
</span>
<span
className="ml-auto shrink-0 rounded bg-slate-800 px-1 text-[10px] uppercase tracking-wide text-slate-400"
- title="This block did not execute because the workflow ended before reaching it"
+ title={badge.title}
>
- did not execute
+ {badge.label}
</span>
</div>
</div>
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts
index 30b37a54f..242e36094 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts
@@ -8,12 +8,15 @@ import {
type WorkflowRunTimelineBlockItem,
type WorkflowRunTimelineItem,
} from "../types/workflowRunTypes";
+import type { WorkflowBlock } from "../types/workflowTypes";
import {
aggregateIterationStatus,
+ classifyUnexecutedDefinedBlocks,
findActiveItem,
findBlockSurroundingThought,
findLastExecutedBlock,
findRunningBlock,
+ flattenTimelineChronologically,
resolveScreenshotBlockId,
} from "./workflowTimelineUtils";
@@ -439,6 +442,470 @@ describe("aggregateIterationStatus", () => {
});
});
+describe("flattenTimelineChronologically", () => {
+ function rootIds(items: Array<WorkflowRunTimelineItem>): Array<string> {
+ return items.map((item) =>
+ item.type === "block" ? item.block.workflow_run_block_id : "thought",
+ );
+ }
+
+ test("hoists conditional branch chains so visible order matches execution order", () => {
+ // Regression: a conditional's branch blocks executed AFTER a root-level
+ // loop, but tree rendering printed them above it (the terminated block
+ // looked like it ran before the loop).
+ // root block_2 conditional 07:16
+ // root block_5 for_loop 07:19 (children 07:19–07:27)
+ // child block_8 (parent block_2) 07:29
+ // child block_12 (parent block_8) 07:39, terminated
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_block_2",
+ block_type: "conditional",
+ created_at: "2026-06-10T07:16:29Z",
+ });
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_block_5",
+ block_type: "for_loop",
+ created_at: "2026-06-10T07:19:06Z",
+ });
+ const loopChild = buildBlock({
+ workflow_run_block_id: "wrb_loop_child",
+ parent_workflow_run_block_id: "wrb_block_5",
+ created_at: "2026-06-10T07:19:11Z",
+ current_index: 0,
+ });
+ const branchConditional = buildBlock({
+ workflow_run_block_id: "wrb_block_8",
+ block_type: "conditional",
+ parent_workflow_run_block_id: "wrb_block_2",
+ created_at: "2026-06-10T07:29:32Z",
+ });
+ const terminated = buildBlock({
+ workflow_run_block_id: "wrb_block_12",
+ block_type: "navigation",
+ status: Status.Terminated,
+ parent_workflow_run_block_id: "wrb_block_8",
+ created_at: "2026-06-10T07:39:31Z",
+ });
+
+ const flattened = flattenTimelineChronologically([
+ buildBlockItem(conditional, [
+ buildBlockItem(branchConditional, [buildBlockItem(terminated)]),
+ ]),
+ buildBlockItem(loop, [buildBlockItem(loopChild)]),
+ ]);
+
+ expect(rootIds(flattened)).toEqual([
+ "wrb_block_2",
+ "wrb_block_5",
+ "wrb_block_8",
+ "wrb_block_12",
+ ]);
+ // The loop keeps its iteration children nested; hoisted conditional rows
+ // carry none.
+ const loopRow = flattened.find(
+ (item) =>
+ item.type === "block" &&
+ item.block.workflow_run_block_id === "wrb_block_5",
+ );
+ expect(rootIds(loopRow?.children ?? [])).toEqual(["wrb_loop_child"]);
+ const conditionalRow = flattened.find(
+ (item) =>
+ item.type === "block" &&
+ item.block.workflow_run_block_id === "wrb_block_2",
+ );
+ expect(conditionalRow?.children).toEqual([]);
+ });
+
+ test("interleaves a conditional's scoped blocks around a root loop by created_at", () => {
+ // The same conditional parents blocks both before (07:17) and after
+ // (07:28) the root loop ran (07:19) — they must straddle the loop row.
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ created_at: "2026-06-10T07:16:29Z",
+ });
+ const beforeLoop = buildBlock({
+ workflow_run_block_id: "wrb_before_loop",
+ parent_workflow_run_block_id: "wrb_cond",
+ created_at: "2026-06-10T07:17:40Z",
+ });
+ const afterLoop = buildBlock({
+ workflow_run_block_id: "wrb_after_loop",
+ block_type: "wait",
+ parent_workflow_run_block_id: "wrb_cond",
+ created_at: "2026-06-10T07:28:55Z",
+ });
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_loop",
+ block_type: "for_loop",
+ created_at: "2026-06-10T07:19:06Z",
+ });
+
+ const flattened = flattenTimelineChronologically([
+ buildBlockItem(conditional, [
+ buildBlockItem(beforeLoop),
+ buildBlockItem(afterLoop),
+ ]),
+ buildBlockItem(loop),
+ ]);
+
+ expect(rootIds(flattened)).toEqual([
+ "wrb_cond",
+ "wrb_before_loop",
+ "wrb_loop",
+ "wrb_after_loop",
+ ]);
+ });
+
+ test("keeps loop children nested and sorts them ascending by created_at", () => {
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_loop",
+ block_type: "for_loop",
+ created_at: "2026-06-10T07:00:00Z",
+ });
+ const later = buildBlock({
+ workflow_run_block_id: "wrb_later",
+ parent_workflow_run_block_id: "wrb_loop",
+ created_at: "2026-06-10T07:05:00Z",
+ current_index: 1,
+ });
+ const earlier = buildBlock({
+ workflow_run_block_id: "wrb_earlier",
+ parent_workflow_run_block_id: "wrb_loop",
+ created_at: "2026-06-10T07:01:00Z",
+ current_index: 0,
+ });
+
+ const flattened = flattenTimelineChronologically([
+ buildBlockItem(loop, [buildBlockItem(later), buildBlockItem(earlier)]),
+ ]);
+
+ expect(rootIds(flattened)).toEqual(["wrb_loop"]);
+ expect(rootIds(flattened[0]!.children)).toEqual([
+ "wrb_earlier",
+ "wrb_later",
+ ]);
+ });
+
+ test("hoists a conditional's branch chain inside a loop into the loop's children", () => {
+ // Inside a loop iteration, branch targets are parented to the in-loop
+ // conditional. They keep their current_index, so hoisting them up to the
+ // loop level preserves iteration grouping.
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_loop",
+ block_type: "for_loop",
+ created_at: "2026-06-10T07:00:00Z",
+ });
+ const inLoopConditional = buildBlock({
+ workflow_run_block_id: "wrb_in_loop_cond",
+ block_type: "conditional",
+ parent_workflow_run_block_id: "wrb_loop",
+ created_at: "2026-06-10T07:01:00Z",
+ current_index: 0,
+ });
+ const branchTask = buildBlock({
+ workflow_run_block_id: "wrb_branch_task",
+ parent_workflow_run_block_id: "wrb_in_loop_cond",
+ created_at: "2026-06-10T07:02:00Z",
+ current_index: 0,
+ });
+
+ const flattened = flattenTimelineChronologically([
+ buildBlockItem(loop, [
+ buildBlockItem(inLoopConditional, [buildBlockItem(branchTask)]),
+ ]),
+ ]);
+
+ expect(rootIds(flattened)).toEqual(["wrb_loop"]);
+ expect(rootIds(flattened[0]!.children)).toEqual([
+ "wrb_in_loop_cond",
+ "wrb_branch_task",
+ ]);
+ const hoistedBranchTask = flattened[0]!.children[1]!;
+ expect(
+ hoistedBranchTask.type === "block" &&
+ hoistedBranchTask.block.current_index,
+ ).toBe(0);
+ });
+
+ test("sorts thought items among blocks chronologically", () => {
+ const thought = buildThoughtItem({
+ thought_id: "thought_mid",
+ created_at: "2026-06-10T07:01:00Z",
+ });
+ const first = buildBlock({
+ workflow_run_block_id: "wrb_first",
+ created_at: "2026-06-10T07:00:00Z",
+ });
+ const last = buildBlock({
+ workflow_run_block_id: "wrb_last",
+ created_at: "2026-06-10T07:02:00Z",
+ });
+
+ const flattened = flattenTimelineChronologically([
+ buildBlockItem(last),
+ thought,
+ buildBlockItem(first),
+ ]);
+
+ expect(rootIds(flattened)).toEqual(["wrb_first", "thought", "wrb_last"]);
+ });
+});
+
+describe("classifyUnexecutedDefinedBlocks", () => {
+ function buildDefinedBlock(
+ overrides: Partial<{
+ label: string;
+ block_type: string;
+ next_block_label: string | null;
+ branch_conditions: Array<{
+ id: string;
+ next_block_label: string | null;
+ is_default: boolean;
+ }>;
+ }>,
+ ): WorkflowBlock {
+ return {
+ label: "defined_default",
+ block_type: "navigation",
+ next_block_label: null,
+ ...overrides,
+ } as unknown as WorkflowBlock;
+ }
+
+ function reasonsByLabel(
+ result: ReturnType<typeof classifyUnexecutedDefinedBlocks>,
+ ): Record<string, string> {
+ return Object.fromEntries(result.map((r) => [r.block.label, r.reason]));
+ }
+
+ test("labels a not-taken branch chain as branch_not_taken using runtime evaluations", () => {
+ const defined = [
+ buildDefinedBlock({
+ label: "cond",
+ block_type: "conditional",
+ branch_conditions: [
+ { id: "br_a", next_block_label: "block_a", is_default: false },
+ { id: "br_b", next_block_label: "block_b", is_default: true },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_a", next_block_label: null }),
+ buildDefinedBlock({ label: "block_b", next_block_label: "block_b2" }),
+ buildDefinedBlock({ label: "block_b2", next_block_label: null }),
+ ];
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "cond",
+ output: {
+ evaluations: [
+ { is_matched: true, next_block_label: "block_a" },
+ { is_matched: false, next_block_label: "block_b" },
+ ],
+ } as WorkflowRunBlock["output"],
+ });
+ const taken = buildBlock({
+ workflow_run_block_id: "wrb_a",
+ label: "block_a",
+ parent_workflow_run_block_id: "wrb_cond",
+ });
+
+ const result = classifyUnexecutedDefinedBlocks(defined, [
+ buildBlockItem(conditional, [buildBlockItem(taken)]),
+ ]);
+
+ expect(reasonsByLabel(result)).toEqual({
+ block_b: "branch_not_taken",
+ block_b2: "branch_not_taken",
+ });
+ });
+
+ test("labels an unexecuted block on the TAKEN branch as not_reached (run ended first)", () => {
+ const defined = [
+ buildDefinedBlock({
+ label: "cond",
+ block_type: "conditional",
+ branch_conditions: [
+ { id: "br_a", next_block_label: "block_a", is_default: false },
+ { id: "br_b", next_block_label: "block_b", is_default: true },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_a", next_block_label: "block_a2" }),
+ buildDefinedBlock({ label: "block_a2", next_block_label: null }),
+ buildDefinedBlock({ label: "block_b", next_block_label: null }),
+ ];
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "cond",
+ output: {
+ evaluations: [
+ { is_matched: true, next_block_label: "block_a" },
+ { is_matched: false, next_block_label: "block_b" },
+ ],
+ } as WorkflowRunBlock["output"],
+ });
+ const taken = buildBlock({
+ workflow_run_block_id: "wrb_a",
+ label: "block_a",
+ status: Status.Terminated,
+ parent_workflow_run_block_id: "wrb_cond",
+ });
+
+ const result = classifyUnexecutedDefinedBlocks(defined, [
+ buildBlockItem(conditional, [buildBlockItem(taken)]),
+ ]);
+
+ // block_a2 was on the taken path but the run terminated at block_a.
+ expect(reasonsByLabel(result)).toEqual({
+ block_a2: "not_reached",
+ block_b: "branch_not_taken",
+ });
+ });
+
+ test("labels everything not_reached when the conditional itself never executed", () => {
+ const defined = [
+ buildDefinedBlock({ label: "first", next_block_label: "cond" }),
+ buildDefinedBlock({
+ label: "cond",
+ block_type: "conditional",
+ branch_conditions: [
+ { id: "br_a", next_block_label: "block_a", is_default: false },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_a", next_block_label: null }),
+ ];
+ const first = buildBlock({
+ workflow_run_block_id: "wrb_first",
+ label: "first",
+ status: Status.Terminated,
+ });
+
+ const result = classifyUnexecutedDefinedBlocks(defined, [
+ buildBlockItem(first),
+ ]);
+
+ expect(reasonsByLabel(result)).toEqual({
+ cond: "not_reached",
+ block_a: "not_reached",
+ });
+ });
+
+ test("descends into an unexecuted inner conditional inside a not-taken branch", () => {
+ const defined = [
+ buildDefinedBlock({
+ label: "outer",
+ block_type: "conditional",
+ branch_conditions: [
+ { id: "br_a", next_block_label: "block_a", is_default: false },
+ { id: "br_inner", next_block_label: "inner", is_default: true },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_a", next_block_label: null }),
+ buildDefinedBlock({
+ label: "inner",
+ block_type: "conditional",
+ next_block_label: "inner_merge",
+ branch_conditions: [
+ { id: "br_x", next_block_label: "block_x", is_default: false },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_x", next_block_label: null }),
+ buildDefinedBlock({ label: "inner_merge", next_block_label: null }),
+ ];
+ const outer = buildBlock({
+ workflow_run_block_id: "wrb_outer",
+ block_type: "conditional",
+ label: "outer",
+ output: {
+ evaluations: [
+ { is_matched: true, next_block_label: "block_a" },
+ { is_matched: false, next_block_label: "inner" },
+ ],
+ } as WorkflowRunBlock["output"],
+ });
+ const taken = buildBlock({
+ workflow_run_block_id: "wrb_a",
+ label: "block_a",
+ parent_workflow_run_block_id: "wrb_outer",
+ });
+
+ const result = classifyUnexecutedDefinedBlocks(defined, [
+ buildBlockItem(outer, [buildBlockItem(taken)]),
+ ]);
+
+ expect(reasonsByLabel(result)).toEqual({
+ inner: "branch_not_taken",
+ block_x: "branch_not_taken",
+ inner_merge: "branch_not_taken",
+ });
+ });
+
+ test("falls back to executed_branch_next_block when output has no evaluations", () => {
+ const defined = [
+ buildDefinedBlock({
+ label: "cond",
+ block_type: "conditional",
+ branch_conditions: [
+ { id: "br_a", next_block_label: "block_a", is_default: false },
+ { id: "br_b", next_block_label: "block_b", is_default: true },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_a", next_block_label: null }),
+ buildDefinedBlock({ label: "block_b", next_block_label: null }),
+ ];
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "cond",
+ executed_branch_next_block: "block_a",
+ });
+ const taken = buildBlock({
+ workflow_run_block_id: "wrb_a",
+ label: "block_a",
+ parent_workflow_run_block_id: "wrb_cond",
+ });
+
+ const result = classifyUnexecutedDefinedBlocks(defined, [
+ buildBlockItem(conditional, [buildBlockItem(taken)]),
+ ]);
+
+ expect(reasonsByLabel(result)).toEqual({ block_b: "branch_not_taken" });
+ });
+
+ test("stays conservative (not_reached) when the taken branch is unknowable", () => {
+ const defined = [
+ buildDefinedBlock({
+ label: "cond",
+ block_type: "conditional",
+ branch_conditions: [
+ { id: "br_a", next_block_label: "block_a", is_default: false },
+ { id: "br_b", next_block_label: "block_b", is_default: true },
+ ],
+ }),
+ buildDefinedBlock({ label: "block_a", next_block_label: null }),
+ buildDefinedBlock({ label: "block_b", next_block_label: null }),
+ ];
+ // Legacy run: no evaluations in output, no executed_branch_next_block.
+ const conditional = buildBlock({
+ workflow_run_block_id: "wrb_cond",
+ block_type: "conditional",
+ label: "cond",
+ });
+ const taken = buildBlock({
+ workflow_run_block_id: "wrb_a",
+ label: "block_a",
+ parent_workflow_run_block_id: "wrb_cond",
+ });
+
+ const result = classifyUnexecutedDefinedBlocks(defined, [
+ buildBlockItem(conditional, [buildBlockItem(taken)]),
+ ]);
+
+ expect(reasonsByLabel(result)).toEqual({ block_b: "not_reached" });
+ });
+});
+
describe("resolveScreenshotBlockId", () => {
test("returns the block's own id when it isn't a container", () => {
const leaf = buildBlock({
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts
index 6044fb2ad..cbbe4b66d 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts
@@ -1,12 +1,15 @@
import { Status } from "@/api/types";
import { statusIsFinalized } from "@/routes/tasks/types";
import {
+ findUnexecutedDefinedBlocks,
+ hasEvaluations,
isBlockItem,
isThoughtItem,
ObserverThought,
WorkflowRunBlock,
WorkflowRunTimelineItem,
} from "../types/workflowRunTypes";
+import { WorkflowBlock } from "../types/workflowTypes";
import { WorkflowRunOverviewActiveElement } from "./WorkflowRunOverview";
const containerBlockTypes = new Set(["for_loop", "while_loop", "conditional"]);
@@ -17,6 +20,180 @@ function parseActiveIterationParam(value: string | null): number | null {
return Number.isSafeInteger(parsed) ? parsed : null;
}
+function toTimelineTime(value: string): number {
+ const time = new Date(value).getTime();
+ return Number.isNaN(time) ? Number.MAX_SAFE_INTEGER : time;
+}
+
+/**
+ * Order timeline items by actual execution chronology.
+ *
+ * A conditional's children are the blocks its taken branch jumped to —
+ * lineage, not containment — so they interleave in time with the
+ * conditional's own siblings (e.g. a root-level loop can run between two of
+ * them). Hoist them to the conditional's level so depth-first rendering
+ * can't show a later branch block above an earlier sibling. Loop and
+ * task_v2 children do execute within their parent's lifespan and stay
+ * nested. Every sibling group is sorted ascending by created_at.
+ */
+function flattenTimelineChronologically(
+ items: Array<WorkflowRunTimelineItem>,
+): Array<WorkflowRunTimelineItem> {
+ if (items.length === 0) return items;
+
+ const rows: Array<WorkflowRunTimelineItem> = [];
+ const visit = (item: WorkflowRunTimelineItem) => {
+ if (isBlockItem(item) && item.block.block_type === "conditional") {
+ rows.push({ ...item, children: [] });
+ item.children.forEach(visit);
+ return;
+ }
+ rows.push({
+ ...item,
+ children: flattenTimelineChronologically(item.children),
+ });
+ };
+ items.forEach(visit);
+
+ return rows.sort(
+ (left, right) =>
+ toTimelineTime(left.created_at) - toTimelineTime(right.created_at),
+ );
+}
+
+type UnexecutedBlockReason = "branch_not_taken" | "not_reached";
+
+type UnexecutedDefinedBlock = {
+ block: WorkflowBlock;
+ reason: UnexecutedBlockReason;
+};
+
+function collectExecutedConditionals(
+ timelineItems: Array<WorkflowRunTimelineItem>,
+): Array<WorkflowRunBlock> {
+ const conditionals: Array<WorkflowRunBlock> = [];
+ const stack = [...timelineItems];
+ while (stack.length > 0) {
+ const item = stack.pop()!;
+ if (isBlockItem(item) && item.block.block_type === "conditional") {
+ conditionals.push(item.block);
+ }
+ stack.push(...item.children);
+ }
+ return conditionals;
+}
+
+/**
+ * Branch targets a conditional execution did / did not take. Prefer the
+ * runtime evaluations (they record the matched branch even when it has no
+ * redirect target); fall back to executed_branch_next_block against the
+ * definition's branch list. Returns null when the taken branch is unknowable
+ * (legacy runs) so callers can skip classification instead of guessing.
+ */
+function getBranchOutcome(
+ runtimeBlock: WorkflowRunBlock,
+ definedBlock: WorkflowBlock | undefined,
+): { takenTargets: Array<string>; notTakenTargets: Array<string> } | null {
+ if (hasEvaluations(runtimeBlock.output)) {
+ const evaluations = runtimeBlock.output.evaluations ?? [];
+ return {
+ takenTargets: evaluations
+ .filter((e) => e.is_matched && e.next_block_label)
+ .map((e) => e.next_block_label!),
+ notTakenTargets: evaluations
+ .filter((e) => !e.is_matched && e.next_block_label)
+ .map((e) => e.next_block_label!),
+ };
+ }
+ const taken = runtimeBlock.executed_branch_next_block;
+ if (!taken || !definedBlock || definedBlock.block_type !== "conditional") {
+ return null;
+ }
+ return {
+ takenTargets: [taken],
+ notTakenTargets: definedBlock.branch_conditions
+ .map((branch) => branch.next_block_label)
+ .filter((label): label is string => !!label && label !== taken),
+ };
+}
+
+/**
+ * Collect the chain of defined-block labels reachable from `start` via
+ * next_block_label. Executed conditionals end the walk (their own outcome is
+ * classified separately); unexecuted conditionals are descended into —
+ * every branch they could have taken shares the parent chain's fate.
+ */
+function traceDefinitionChain(
+ start: string,
+ blocksByLabel: ReadonlyMap<string, WorkflowBlock>,
+ executedLabels: ReadonlySet<string>,
+ into: Set<string>,
+): void {
+ const queue = [start];
+ while (queue.length > 0) {
+ const label = queue.pop()!;
+ if (into.has(label)) continue;
+ const block = blocksByLabel.get(label);
+ if (!block) continue;
+ into.add(label);
+ if (block.block_type === "conditional") {
+ if (executedLabels.has(label)) continue;
+ for (const branch of block.branch_conditions) {
+ if (branch.next_block_label) queue.push(branch.next_block_label);
+ }
+ }
+ if (block.next_block_label) queue.push(block.next_block_label);
+ }
+}
+
+/**
+ * Split unexecuted defined blocks into "skipped because a conditional took a
+ * different branch" and "never reached" (the run ended first). Labels on a
+ * taken branch win over not-taken membership: a block that the taken path
+ * should have reached but didn't is "not reached", not "skipped".
+ */
+function classifyUnexecutedDefinedBlocks(
+ definedBlocks: Array<WorkflowBlock>,
+ timelineItems: Array<WorkflowRunTimelineItem>,
+): Array<UnexecutedDefinedBlock> {
+ const unexecuted = findUnexecutedDefinedBlocks(definedBlocks, timelineItems);
+ if (unexecuted.length === 0) return [];
+
+ const blocksByLabel = new Map(
+ definedBlocks.map((block) => [block.label, block]),
+ );
+ const unexecutedLabels = new Set(unexecuted.map((block) => block.label));
+ const executedLabels = new Set(
+ definedBlocks
+ .filter((block) => !unexecutedLabels.has(block.label))
+ .map((block) => block.label),
+ );
+
+ const skipped = new Set<string>();
+ const onTakenPath = new Set<string>();
+ for (const conditional of collectExecutedConditionals(timelineItems)) {
+ const definedBlock = conditional.label
+ ? blocksByLabel.get(conditional.label)
+ : undefined;
+ const outcome = getBranchOutcome(conditional, definedBlock);
+ if (!outcome) continue;
+ for (const target of outcome.notTakenTargets) {
+ traceDefinitionChain(target, blocksByLabel, executedLabels, skipped);
+ }
+ for (const target of outcome.takenTargets) {
+ traceDefinitionChain(target, blocksByLabel, executedLabels, onTakenPath);
+ }
+ }
+
+ return unexecuted.map((block) => ({
+ block,
+ reason:
+ skipped.has(block.label) && !onTakenPath.has(block.label)
+ ? "branch_not_taken"
+ : "not_reached",
+ }));
+}
+
function findBlockSurroundingAction(
timeline: Array<WorkflowRunTimelineItem>,
actionId: string,
@@ -433,12 +610,15 @@ function aggregateIterationStatus(
export {
aggregateIterationStatus,
+ classifyUnexecutedDefinedBlocks,
findActiveItem,
findBlockSurroundingAction,
findBlockSurroundingThought,
findLastExecutedBlock,
findRunningBlock,
findThoughtsForBlock,
+ flattenTimelineChronologically,
parseActiveIterationParam,
resolveScreenshotBlockId,
};
+export type { UnexecutedBlockReason, UnexecutedDefinedBlock };
diff --git a/skyvern-frontend/src/store/useCredentialTestStore.test.ts b/skyvern-frontend/src/store/useCredentialTestStore.test.ts
new file mode 100644
index 000000000..f47a03fbb
--- /dev/null
+++ b/skyvern-frontend/src/store/useCredentialTestStore.test.ts
@@ -0,0 +1,76 @@
+// @vitest-environment jsdom
+
+import { afterEach, describe, expect, it } from "vitest";
+
+import { useCredentialTestStore } from "./useCredentialTestStore";
+
+const TEST = {
+ credentialId: "cred_1",
+ workflowRunId: "wr_1",
+ url: "https://example.com/login",
+ startTime: 1700000000000,
+};
+
+const NEWER_TEST = {
+ credentialId: "cred_2",
+ workflowRunId: "wr_2",
+ url: "https://example.org/login",
+ startTime: 1700000001000,
+};
+
+function persistedActiveTest() {
+ const raw = localStorage.getItem("credential-test");
+ return raw ? JSON.parse(raw).state.activeTest : undefined;
+}
+
+afterEach(() => {
+ useCredentialTestStore.getState().clearActiveTest();
+ localStorage.clear();
+});
+
+describe("useCredentialTestStore persistence (SKY-10855)", () => {
+ it("persists the active test to localStorage so other tabs (noopener run tab) can rehydrate it", () => {
+ useCredentialTestStore.getState().setActiveTest(TEST);
+
+ expect(persistedActiveTest()).toEqual(TEST);
+ expect(sessionStorage.getItem("credential-test")).toBeNull();
+ });
+
+ it("clears the persisted test when it reaches a terminal state", () => {
+ useCredentialTestStore.getState().setActiveTest(TEST);
+ useCredentialTestStore.getState().clearActiveTest();
+
+ expect(persistedActiveTest()).toBeNull();
+ });
+
+ it("scoped clear ignores the slot when another run owns it", () => {
+ useCredentialTestStore.getState().setActiveTest(NEWER_TEST);
+ useCredentialTestStore.getState().clearActiveTest(TEST.workflowRunId);
+
+ expect(useCredentialTestStore.getState().activeTest).toEqual(NEWER_TEST);
+ expect(persistedActiveTest()).toEqual(NEWER_TEST);
+ });
+
+ it("scoped clear removes the slot when the run matches", () => {
+ useCredentialTestStore.getState().setActiveTest(TEST);
+ useCredentialTestStore.getState().clearActiveTest(TEST.workflowRunId);
+
+ expect(useCredentialTestStore.getState().activeTest).toBeNull();
+ });
+
+ it("rehydrates from a storage event so tabs track each other's writes", () => {
+ useCredentialTestStore.getState().setActiveTest(TEST);
+
+ // Simulate another tab replacing the slot: write storage directly, then
+ // dispatch the storage event (browsers only fire it in non-writer tabs).
+ localStorage.setItem(
+ "credential-test",
+ JSON.stringify({ state: { activeTest: NEWER_TEST }, version: 0 }),
+ );
+ window.dispatchEvent(
+ new StorageEvent("storage", { key: "credential-test" }),
+ );
+
+ expect(useCredentialTestStore.getState().activeTest).toEqual(NEWER_TEST);
+ });
+});
diff --git a/skyvern-frontend/src/store/useCredentialTestStore.ts b/skyvern-frontend/src/store/useCredentialTestStore.ts
index 1f9096558..40b31e030 100644
--- a/skyvern-frontend/src/store/useCredentialTestStore.ts
+++ b/skyvern-frontend/src/store/useCredentialTestStore.ts
@@ -11,7 +11,8 @@ type ActiveCredentialTest = {
type CredentialTestStore = {
activeTest: ActiveCredentialTest | null;
setActiveTest: (test: ActiveCredentialTest) => void;
- clearActiveTest: () => void;
+ /** With a run id, clears only if the slot still holds that run (another tab may have started a newer test). */
+ clearActiveTest: (workflowRunId?: string) => void;
};
export type { ActiveCredentialTest };
@@ -21,11 +22,28 @@ export const useCredentialTestStore = create<CredentialTestStore>()(
(set) => ({
activeTest: null,
setActiveTest: (test) => set({ activeTest: test }),
- clearActiveTest: () => set({ activeTest: null }),
+ clearActiveTest: (workflowRunId) =>
+ set((state) =>
+ workflowRunId && state.activeTest?.workflowRunId !== workflowRunId
+ ? state
+ : { activeTest: null },
+ ),
}),
{
name: "credential-test",
- storage: createJSONStorage(() => sessionStorage),
+ // localStorage, not sessionStorage: the watch-run link opens a noopener tab
+ // whose sessionStorage starts empty, which hid in-flight tests from that tab.
+ storage: createJSONStorage(() => localStorage),
},
),
);
+
+// Sync the shared slot across tabs; otherwise a tab's persist write (e.g. on its
+// own test finishing) would silently clobber a newer test started in another tab.
+if (typeof window !== "undefined") {
+ window.addEventListener("storage", (event) => {
+ if (event.key === "credential-test") {
+ void useCredentialTestStore.persist.rehydrate();
+ }
+ });
+}
diff --git a/skyvern/config.py b/skyvern/config.py
index b0bf2a83e..4038b3a73 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -95,6 +95,7 @@ class Settings(BaseSettings):
VIDEO_COMPRESSION_CRF: int = 28
VIDEO_COMPRESSION_PRESET: str = "veryfast"
VIDEO_COMPRESSION_TIMEOUT_SECONDS: float = 300.0
+ VIDEO_FINAL_SYNC_TIMEOUT_SECONDS: float = 750.0
HAR_PATH: str | None = "./har"
LOG_PATH: str = "./log"
TEMP_PATH: str = "./temp"
@@ -139,6 +140,9 @@ class Settings(BaseSettings):
EXECUTE_ALL_STEPS: bool = True
JSON_LOGGING: bool = False
LOG_RAW_API_REQUESTS: bool = True
+ # Successful (<400) GET/HEAD/OPTIONS are skipped by default: they dominate
+ # log volume (health checks, polling) while carrying no mutation to audit.
+ LOG_RAW_API_REQUESTS_SUCCESSFUL_READS: bool = False
LOG_LEVEL: str = "INFO"
COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS: float = 12.0
# Gate copilot verified-success on per-criterion outcome evidence.
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index ea15fdb00..71abae9eb 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -1469,6 +1469,11 @@ def _build_block_fn(
ActionType.COMPLETE,
ActionType.TERMINATE,
ActionType.NULL_ACTION,
+ # Tab management isn't represented in generated scripts yet (no ACTION_MAP
+ # entry); skip so cached replay of a multi-tab run falls back to the agent
+ # instead of raising KeyError and breaking script generation.
+ ActionType.NEW_TAB,
+ ActionType.SWITCH_TAB,
]:
continue
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 1a2f8329b..038ae80d8 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -3475,6 +3475,8 @@ class ForgeAgent:
verification_code_check: bool,
show_close_page_action: bool,
complete_criterion: str | None,
+ show_new_tab_action: bool = False,
+ show_switch_tab_action: bool = False,
enriched_tree_enabled: bool = False,
llm_screenshots_enabled: bool = True,
slim_output: str | None = None,
@@ -3489,6 +3491,10 @@ class ForgeAgent:
variant_parts.append("vc")
if show_close_page_action:
variant_parts.append("cp")
+ if show_new_tab_action:
+ variant_parts.append("nt")
+ if show_switch_tab_action:
+ variant_parts.append("st")
if enriched_tree_enabled:
variant_parts.append("et")
if enriched_tree_enabled and not llm_screenshots_enabled:
@@ -3763,6 +3769,11 @@ class ForgeAgent:
open_tabs_context = await _build_open_tabs_context(browser_state, page)
show_close_page_action = open_tabs_context is not None
+ # NEW_TAB/SWITCH_TAB are gated behind a flag (default off) so single-tab tasks are
+ # unaffected. SWITCH_TAB additionally requires >= 2 open tabs, like CLOSE_PAGE.
+ multi_tab_enabled = await self._is_multi_tab_control_enabled(context)
+ show_new_tab_action = multi_tab_enabled
+ show_switch_tab_action = multi_tab_enabled and open_tabs_context is not None
llm_screenshots_enabled = context.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index)
enriched_tree_enabled = context.enriched_tree_enabled()
@@ -3790,6 +3801,8 @@ class ForgeAgent:
"complete_criterion": task.complete_criterion.strip() if task.complete_criterion else None,
"terminate_criterion": task.terminate_criterion.strip() if task.terminate_criterion else None,
"show_close_page_action": show_close_page_action,
+ "show_new_tab_action": show_new_tab_action,
+ "show_switch_tab_action": show_switch_tab_action,
"open_tabs_context": open_tabs_context,
"recent_dialog_messages_str": recent_dialog_messages_str,
"llm_screenshots_enabled": llm_screenshots_enabled,
@@ -3799,6 +3812,8 @@ class ForgeAgent:
cache_variant = self._build_extract_action_cache_variant(
verification_code_check=verification_code_check,
show_close_page_action=show_close_page_action,
+ show_new_tab_action=show_new_tab_action,
+ show_switch_tab_action=show_switch_tab_action,
complete_criterion=task.complete_criterion.strip() if task.complete_criterion else None,
enriched_tree_enabled=enriched_tree_enabled,
llm_screenshots_enabled=llm_screenshots_enabled,
@@ -3888,6 +3903,8 @@ class ForgeAgent:
complete_criterion=task.complete_criterion.strip() if task.complete_criterion else None,
terminate_criterion=task.terminate_criterion.strip() if task.terminate_criterion else None,
show_close_page_action=show_close_page_action,
+ show_new_tab_action=show_new_tab_action,
+ show_switch_tab_action=show_switch_tab_action,
open_tabs_context=open_tabs_context,
recent_dialog_messages_str=recent_dialog_messages_str,
llm_screenshots_enabled=llm_screenshots_enabled,
@@ -3963,6 +3980,26 @@ class ForgeAgent:
return context.prompt_caching_settings
+ async def _is_multi_tab_control_enabled(self, context: SkyvernContext) -> bool:
+ distinct_id = context.run_id or context.workflow_run_id or context.task_id
+ organization_id = context.organization_id
+ if not distinct_id or not organization_id:
+ return False
+ try:
+ return await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "MULTI_TAB_CONTROL",
+ distinct_id,
+ properties={"organization_id": organization_id},
+ )
+ except Exception as exc:
+ LOG.warning(
+ "Failed to evaluate multi-tab control experiment; defaulting to disabled",
+ distinct_id=distinct_id,
+ organization_id=organization_id,
+ error=str(exc),
+ )
+ return False
+
def _should_process_totp(self, scraped_page: ScrapedPage | None) -> bool:
"""Detect TOTP pages by checking for multiple input fields or verification keywords."""
if not scraped_page:
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 7bbd57827..431e9a7d5 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1076,8 +1076,11 @@ class AgentFunction:
return cleanup_element_tree_func
+ async def has_code_block_access(self, organization_id: str | None = None) -> bool:
+ return settings.ENABLE_CODE_BLOCK
+
async def validate_code_block(self, organization_id: str | None = None) -> None:
- if not settings.ENABLE_CODE_BLOCK:
+ if not await self.has_code_block_access(organization_id):
raise DisabledBlockExecutionError("CodeBlock is disabled")
# TODO: Remove these methods if nothing calls them after verifying in production
diff --git a/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2 b/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
index b8c4a8cc2..12136d26c 100644
--- a/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
@@ -35,7 +35,7 @@ User details:
```
{% if open_tabs_context %}
-Open browser tabs (CLOSE_PAGE always closes the tab marked [current] — other tabs are shown for context only):
+Open browser tabs ({% if show_switch_tab_action %}use SWITCH_TAB with the tab's number to make a different tab active, and CLOSE_PAGE closes the tab marked [current]{% else %}CLOSE_PAGE always closes the tab marked [current] — other tabs are shown for context only{% endif %}):
```
{{ open_tabs_context }}
```
diff --git a/skyvern/forge/prompts/skyvern/extract-action-static.j2 b/skyvern/forge/prompts/skyvern/extract-action-static.j2
index cb3e4d401..6f0f1e091 100644
--- a/skyvern/forge/prompts/skyvern/extract-action-static.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action-static.j2
@@ -21,13 +21,13 @@ Reply in JSON format with the following keys:
"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts.
"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
- "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if show_close_page_action else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a visible captcha to solve on the screen. Captcha tokens expire quickly (~2 minutes), so fill all form fields first, then SOLVE_CAPTCHA immediately before the final submit or protected click — do not place SOLVE_CAPTCHA before a long form-fill sequence. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current tab and return to a previous tab. Use it when the page instructs you to close it or return to the original page, or when the user goal explicitly requires closing this tab. Do not use CLOSE_PAGE simply because the goal cannot be achieved — use TERMINATE for that.' if show_close_page_action else ''}}
+ "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "NEW_TAB"' if show_new_tab_action else ""}}{{', "SWITCH_TAB"' if show_switch_tab_action else ""}}{{', "CLOSE_PAGE"' if show_close_page_action else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a visible captcha to solve on the screen. Captcha tokens expire quickly (~2 minutes), so fill all form fields first, then SOLVE_CAPTCHA immediately before the final submit or protected click — do not place SOLVE_CAPTCHA before a long form-fill sequence. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current tab and return to a previous tab. Use it when the page instructs you to close it or return to the original page, or when the user goal explicitly requires closing this tab. Do not use CLOSE_PAGE simply because the goal cannot be achieved — use TERMINATE for that.' if show_close_page_action else ''}}{{' "NEW_TAB" opens a new browser tab at a given URL and makes it the active tab, keeping existing tabs open. Use it to open a page in a separate tab while keeping the current tab open (for example, to compare two pages). NEW_TAB requires the "url" field set to a fully-qualified http/https URL. Do not use NEW_TAB to follow a normal link on the page when a clickable element is available. NEW_TAB does not require an element id.' if show_new_tab_action else ''}}{{' "SWITCH_TAB" makes another already-open tab the active tab so you can see and act on it. SWITCH_TAB requires the "tab_index" field set to the number shown for the target tab in the open browser tabs list. Use it to return to an earlier tab without closing the current one. SWITCH_TAB does not require an element id.' if show_switch_tab_action else ''}}
"id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null.
"captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER".
"text": str, // Text for INPUT_TEXT action only
"key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". For "0"-"9" and "a"-"z", use KEYPRESS only when no suitable element is available for INPUT_TEXT. null if not KEYPRESS action.
"repeat": int, // For KEYPRESS only: how many times to press the key in rapid succession without re-evaluating the page between presses. Use this when navigating transient menus (e.g., ArrowDown 3 times to reach an item). Defaults to 1. null if not KEYPRESS action.
- "direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action.
+ "direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action.{{'\n "url": str, // The fully-qualified http/https URL to open for NEW_TAB action only (required for NEW_TAB). null if not NEW_TAB action.' if show_new_tab_action else ''}}{{'\n "tab_index": int, // The tab number to switch to for SWITCH_TAB action only, taken from the open browser tabs list (required for SWITCH_TAB). null if not SWITCH_TAB action.' if show_switch_tab_action else ''}}
"file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise.
"double_click": bool, // For CLICK action only. True only when the element requires a double-click to activate (e.g., opening a file in a file manager, entering edit mode in a spreadsheet cell, or expanding a tree node). Defaults to false for normal clicks. Do not use double-click for regular buttons, links, form elements, clickable grid rows, icons, or dashboard cards.
"download": bool, // Can only be true for CLICK or SELECT_OPTION actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.
diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2
index 1f1158502..0c681eb0d 100644
--- a/skyvern/forge/prompts/skyvern/extract-action.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action.j2
@@ -21,13 +21,13 @@ Reply in JSON format with the following keys:
"user_detail_query": str, // Think of this value as a Jeopardy question and the intention behind the action. Ask the user for the details you need for executing this action. IMPORTANT: This question must be user information agnostic and must NOT contain specific user data (like names, IDs, specific values, etc.). Ask what information is needed generically. Ask the question even if the details are disclosed in user goal or user details. If it's a text field, ask for the text. If it's a file upload, ask for the file. If it's a dropdown, ask for the relevant information. If you are clicking on something specific, ask about what the intention is behind the click and what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Examples are: "What product ID should I input into the search bar?", "What file should I upload?", "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?", "Which member should I take action on?". NEVER include specific user data in this question - keep it generic so it can be answered with different user contexts.
"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
- "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if show_close_page_action else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a visible captcha to solve on the screen. Captcha tokens expire quickly (~2 minutes), so fill all form fields first, then SOLVE_CAPTCHA immediately before the final submit or protected click — do not place SOLVE_CAPTCHA before a long form-fill sequence. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current tab and return to a previous tab. Use it when the page instructs you to close it or return to the original page, or when the user goal explicitly requires closing this tab. Do not use CLOSE_PAGE simply because the goal cannot be achieved — use TERMINATE for that.' if show_close_page_action else ''}}
+ "action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "NEW_TAB"' if show_new_tab_action else ""}}{{', "SWITCH_TAB"' if show_switch_tab_action else ""}}{{', "CLOSE_PAGE"' if show_close_page_action else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a visible captcha to solve on the screen. Captcha tokens expire quickly (~2 minutes), so fill all form fields first, then SOLVE_CAPTCHA immediately before the final submit or protected click — do not place SOLVE_CAPTCHA before a long form-fill sequence. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current tab and return to a previous tab. Use it when the page instructs you to close it or return to the original page, or when the user goal explicitly requires closing this tab. Do not use CLOSE_PAGE simply because the goal cannot be achieved — use TERMINATE for that.' if show_close_page_action else ''}}{{' "NEW_TAB" opens a new browser tab at a given URL and makes it the active tab, keeping existing tabs open. Use it to open a page in a separate tab while keeping the current tab open (for example, to compare two pages). NEW_TAB requires the "url" field set to a fully-qualified http/https URL. Do not use NEW_TAB to follow a normal link on the page when a clickable element is available. NEW_TAB does not require an element id.' if show_new_tab_action else ''}}{{' "SWITCH_TAB" makes another already-open tab the active tab so you can see and act on it. SWITCH_TAB requires the "tab_index" field set to the number shown for the target tab in the open browser tabs list. Use it to return to an earlier tab without closing the current one. SWITCH_TAB does not require an element id.' if show_switch_tab_action else ''}}
"id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null.
"captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER".
"text": str, // Text for INPUT_TEXT action only
"key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". For "0"-"9" and "a"-"z", use KEYPRESS only when no suitable element is available for INPUT_TEXT. null if not KEYPRESS action.
"repeat": int, // For KEYPRESS only: how many times to press the key in rapid succession without re-evaluating the page between presses. Use this when navigating transient menus (e.g., ArrowDown 3 times to reach an item). Defaults to 1. null if not KEYPRESS action.
- "direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action.
+ "direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action.{{'\n "url": str, // The fully-qualified http/https URL to open for NEW_TAB action only (required for NEW_TAB). null if not NEW_TAB action.' if show_new_tab_action else ''}}{{'\n "tab_index": int, // The tab number to switch to for SWITCH_TAB action only, taken from the open browser tabs list (required for SWITCH_TAB). null if not SWITCH_TAB action.' if show_switch_tab_action else ''}}
"file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise.
"double_click": bool, // For CLICK action only. True only when the element requires a double-click to activate (e.g., opening a file in a file manager, entering edit mode in a spreadsheet cell, or expanding a tree node). Defaults to false for normal clicks. Do not use double-click for regular buttons, links, form elements, clickable grid rows, icons, or dashboard cards.
"download": bool, // Can only be true for CLICK or SELECT_OPTION actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.
@@ -98,7 +98,7 @@ User details:
```
{% if open_tabs_context %}
-Open browser tabs (CLOSE_PAGE always closes the tab marked [current] — other tabs are shown for context only):
+Open browser tabs ({% if show_switch_tab_action %}use SWITCH_TAB with the tab's number to make a different tab active, and CLOSE_PAGE closes the tab marked [current]{% else %}CLOSE_PAGE always closes the tab marked [current] — other tabs are shown for context only{% endif %}):
```
{{ open_tabs_context }}
```
diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.py
index fe415e197..d81896ed4 100644
--- a/skyvern/forge/request_logging.py
+++ b/skyvern/forge/request_logging.py
@@ -28,6 +28,7 @@ _SENSITIVE_ENDPOINTS = {
"GET /v1/credentials/totp",
}
_MAX_BODY_LENGTH = 1000
+_READ_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
_MAX_RESPONSE_READ_BYTES = 1024 * 1024 # 1 MB — skip logging bodies larger than this
_BINARY_PLACEHOLDER = "<binary>"
_REDACTED = "****"
@@ -76,8 +77,12 @@ def _client_ip_from_headers(headers: typing.Mapping[str, str]) -> str | None:
return first_hop or None
+def _is_sensitive_endpoint(request: Request) -> bool:
+ return f"{request.method.upper()} {request.url.path.rstrip('/')}" in _SENSITIVE_ENDPOINTS
+
+
def _sanitize_body(request: Request, body: bytes, content_type: str | None) -> str:
- if f"{request.method.upper()} {request.url.path.rstrip('/')}" in _SENSITIVE_ENDPOINTS:
+ if _is_sensitive_endpoint(request):
return _REDACTED
if not body:
return ""
@@ -124,7 +129,7 @@ def _is_loggable_content_type(content_type: str | None) -> bool:
def _sanitize_response_body(request: Request, body_str: str | None, content_type: str | None) -> str:
- if f"{request.method.upper()} {request.url.path.rstrip('/')}" in _SENSITIVE_ENDPOINTS:
+ if _is_sensitive_endpoint(request):
return _REDACTED
if body_str is None:
return _BINARY_PLACEHOLDER
@@ -186,6 +191,19 @@ async def log_raw_request_middleware(request: Request, call_next: Callable[[Requ
try:
response = await call_next(request)
+ # Skip successful reads before buffering the response body: they are
+ # the bulk of request volume (health checks, status polling) and the
+ # 4xx/5xx and mutating paths below retain all audit/debugging value.
+ # Sensitive endpoints (credential/TOTP reads) always keep their
+ # redacted audit line.
+ if (
+ response.status_code < 400
+ and http_method in _READ_METHODS
+ and not settings.LOG_RAW_API_REQUESTS_SUCCESSFUL_READS
+ and not _is_sensitive_endpoint(request)
+ ):
+ return response
+
if response.status_code >= 500:
log_method = LOG.error
elif response.status_code >= 400:
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index fb015bcac..abcef9f92 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -1789,6 +1789,25 @@ if settings.ENABLE_VERTEX_AI:
),
),
)
+ LLMConfigRegistry.register_config(
+ "VERTEX_GEMINI_3.5_FLASH_FLEX",
+ LLMConfig(
+ "vertex_ai/gemini-3.5-flash",
+ [],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ max_completion_tokens=65536,
+ litellm_params=LiteLLMParams(
+ api_base=f"{api_base}/gemini-3.5-flash" if api_base else None,
+ vertex_location=settings.VERTEX_LOCATION,
+ thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",
+ vertex_credentials=settings.VERTEX_CREDENTIALS,
+ service_tier="SERVICE_TIER_FLEX",
+ extra_headers={"X-Vertex-AI-LLM-Shared-Request-Type": "flex", "X-Vertex-AI-LLM-Request-Type": "shared"},
+ timeout=FLEX_EXECUTION_TIMEOUT_SECONDS,
+ ),
+ ),
+ )
# Register old keys as aliases to prevent breaking existing tasks
LLMConfigRegistry.register_config(
"VERTEX_GEMINI_2.5_FLASH_PREVIEW_09_2025",
diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.py
index d74b20e47..df620f72c 100644
--- a/skyvern/forge/sdk/artifact/manager.py
+++ b/skyvern/forge/sdk/artifact/manager.py
@@ -310,6 +310,7 @@ class ArtifactManager:
workflow_run_id=workflow_run_id,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ file_size=len(data) if data is not None else _safe_file_size_from_path(path),
data=data,
path=path,
)
diff --git a/skyvern/forge/sdk/artifact/models.py b/skyvern/forge/sdk/artifact/models.py
index 7689883c4..63a3b1ee3 100644
--- a/skyvern/forge/sdk/artifact/models.py
+++ b/skyvern/forge/sdk/artifact/models.py
@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, field_serializer
class ArtifactType(StrEnum):
RECORDING = "recording"
+ AUDIO = "audio"
SESSION_REPLAY = "session_replay"
BROWSER_CONSOLE_LOG = "browser_console_log"
@@ -111,3 +112,4 @@ class LogEntityType(StrEnum):
WORKFLOW_RUN = "workflow_run"
WORKFLOW_RUN_BLOCK = "workflow_run_block"
TASK_V2 = "task_v2"
+ WORKFLOW_COPILOT_CHAT = "workflow_copilot_chat"
diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py
index 87b27c0f6..194767a49 100644
--- a/skyvern/forge/sdk/artifact/storage/base.py
+++ b/skyvern/forge/sdk/artifact/storage/base.py
@@ -63,6 +63,7 @@ async def _file_infos_from_download_artifacts(artifacts: list[Artifact]) -> list
# TODO: This should be a part of the ArtifactType model
FILE_EXTENTSION_MAP: dict[ArtifactType, str] = {
ArtifactType.RECORDING: "webm",
+ ArtifactType.AUDIO: "webm",
ArtifactType.SESSION_REPLAY: "mp4",
ArtifactType.BROWSER_CONSOLE_LOG: "log",
ArtifactType.SCREENSHOT_LLM: "png",
diff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.py
index fe30205e2..8ddf3ceba 100644
--- a/skyvern/forge/sdk/artifact/storage/gcs.py
+++ b/skyvern/forge/sdk/artifact/storage/gcs.py
@@ -1,3 +1,4 @@
+import asyncio
import os
import shutil
import uuid
@@ -26,7 +27,11 @@ from skyvern.forge.sdk.artifact.storage.base import (
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
)
-from skyvern.forge.sdk.artifact.storage.run_recording_clips import RUN_RECORDING_PATH_SEGMENT, sync_run_recording_clips
+from skyvern.forge.sdk.artifact.storage.run_recording_clips import (
+ RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS,
+ RUN_RECORDING_PATH_SEGMENT,
+ sync_run_recording_clips,
+)
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -781,13 +786,14 @@ class GcsStorage(BaseStorage):
return clip_uri
try:
- await sync_run_recording_clips(
- organization_id=organization_id,
- browser_session_id=browser_session_id,
- source_path=local_file_path,
- upload_clip=_upload_clip,
- now=recording_finalized_at,
- )
+ async with asyncio.timeout(RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS):
+ await sync_run_recording_clips(
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ source_path=local_file_path,
+ upload_clip=_upload_clip,
+ now=recording_finalized_at,
+ )
except Exception:
LOG.warning(
"Run recording clip generation failed", browser_session_id=browser_session_id, exc_info=True
diff --git a/skyvern/forge/sdk/artifact/storage/run_recording_clips.py b/skyvern/forge/sdk/artifact/storage/run_recording_clips.py
index bed41ae3e..ead4b040a 100644
--- a/skyvern/forge/sdk/artifact/storage/run_recording_clips.py
+++ b/skyvern/forge/sdk/artifact/storage/run_recording_clips.py
@@ -17,6 +17,10 @@ LOG = structlog.get_logger()
# unbounded number of ffmpeg jobs at close time.
MAX_RUNS_PER_SESSION = 200
+# Total wall-clock budget for one session's clip pass, enforced by callers so clip
+# generation can never starve the budget that delivers the session recording itself.
+RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS = 300.0
+
# Artifact-type folder for per-run clips. Kept distinct from "videos" so the session recording
# listing (which scans .../videos) never picks up clips, and used as the URI marker that
# identifies a clip created by this path (vs. a run's own full recording).
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index cdefdbccd..25db431cf 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -30,7 +30,11 @@ from skyvern.forge.sdk.artifact.storage.base import (
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
)
-from skyvern.forge.sdk.artifact.storage.run_recording_clips import RUN_RECORDING_PATH_SEGMENT, sync_run_recording_clips
+from skyvern.forge.sdk.artifact.storage.run_recording_clips import (
+ RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS,
+ RUN_RECORDING_PATH_SEGMENT,
+ sync_run_recording_clips,
+)
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -912,13 +916,14 @@ class S3Storage(BaseStorage):
return clip_uri
try:
- await sync_run_recording_clips(
- organization_id=organization_id,
- browser_session_id=browser_session_id,
- source_path=upload_file_path,
- upload_clip=_upload_clip,
- now=recording_finalized_at,
- )
+ async with asyncio.timeout(RUN_RECORDING_CLIPS_SYNC_TIMEOUT_SECONDS):
+ await sync_run_recording_clips(
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ source_path=upload_file_path,
+ upload_clip=_upload_clip,
+ now=recording_finalized_at,
+ )
except Exception:
LOG.warning(
"Run recording clip generation failed", browser_session_id=browser_session_id, exc_info=True
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 5541ff25e..fee8cc92d 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -53,7 +53,7 @@ from skyvern.forge.sdk.copilot.context import (
TurnNarrativePayload,
finalize_discovery_counter_in_global_llm_context,
)
-from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified
+from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified, verified_goal_satisfied_context
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY
from skyvern.forge.sdk.copilot.outcome_verification_trace import (
finalize_outcome_verification_trace,
@@ -910,6 +910,10 @@ def _make_agent_result(
payload_updates["responseType"] = response_type
if proposal_disposition is not None and "proposalDisposition" not in narrative_payload:
payload_updates["proposalDisposition"] = proposal_disposition
+ if turn_outcome is not None and "responseKind" not in narrative_payload:
+ payload_updates["responseKind"] = turn_outcome.response_kind.value
+ if ctx is not None and "verifiedSuccess" not in narrative_payload:
+ payload_updates["verifiedSuccess"] = bool(verified_goal_satisfied_context(ctx))
if payload_updates:
kwargs["narrative_payload"] = {**narrative_payload, **payload_updates}
return AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs)
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 54728c4f7..0e2af2583 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import re
from collections.abc import Mapping
+from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Literal, Protocol
@@ -157,12 +158,37 @@ def to_trace_data(signal: CopilotToolBlockerSignal) -> dict[str, Any]:
}
-class _BlockerSignalCtx(Protocol):
+@dataclass(frozen=True)
+class LoopBlockerEvidence:
+ outcome_gate_reason: str | None = None
+ anti_bot_blocked: bool = False
+ has_draft: bool = False
+
+
+class _LoopEvidenceCtx(Protocol):
+ last_outcome_gate_reason: str | None
+ last_test_anti_bot: str | None
+ staged_workflow: Any | None
+ staged_workflow_yaml: str | None
+ has_staged_proposal: bool
+
+
+class _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):
blocker_signal: CopilotToolBlockerSignal | None
latest_tool_blocker_signal: CopilotToolBlockerSignal | None
tool_blocker_signals: list[CopilotToolBlockerSignal]
+def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence:
+ return LoopBlockerEvidence(
+ outcome_gate_reason=ctx.last_outcome_gate_reason,
+ anti_bot_blocked=bool(ctx.last_test_anti_bot),
+ has_draft=ctx.staged_workflow is not None
+ or ctx.staged_workflow_yaml is not None
+ or bool(ctx.has_staged_proposal),
+ )
+
+
_LOOP_PROGRESS_TOOL_SUCCESS_REASON_CODES = frozenset(
{
"loop_detected_credential_or_parameter_misconfig",
@@ -246,35 +272,167 @@ def stash_blocker_signal(ctx: _BlockerSignalCtx, signal: CopilotToolBlockerSigna
extra["replaced_reason_code"] = existing.internal_reason_code
extra["replaced_blocker_kind"] = existing.blocker_kind
LOG.info("copilot tool blocker signal", **extra, **to_trace_data(signal))
+ if not stashed:
+ refresh_held_loop_blocker_evidence(ctx)
return build_llm_tool_error_payload(signal)
-def build_loop_blocker_signal(loop_message: str, *, tool_name: str) -> CopilotToolBlockerSignal:
+_LOOP_CREDENTIAL_TEMPLATE = (
+ "I couldn't run this with the current credential or parameter setup. Update them and ask me to try again."
+)
+_LOOP_BRANCH_COPY: dict[str, tuple[str, str]] = {
+ "loop_detected_repeated_failed_step": (
+ "I retried without making progress.",
+ "Tell me what to change and I'll try a different approach.",
+ ),
+ "loop_detected_consecutive_same_tool": (
+ "I'm stuck retrying the same step.",
+ "Tell me what to change and I'll try a different approach.",
+ ),
+ "loop_detected_generic": (
+ "I couldn't keep going on this turn.",
+ "Tell me what to change and I'll try again.",
+ ),
+}
+_LOOP_ANTI_BOT_BLOCKER_COPY = "The site's verification challenge was still keeping the submit/search control disabled."
+_LOOP_VERDICT_MAX_CHARS = 240
+_LOOP_VERDICT_FAILED_PREFIX = "failed:"
+# Fixed agent-advice tails appended by the recorded-reason producers; repair
+# vocabulary aimed at the agent, never user information.
+_LOOP_VERDICT_ADVICE_MARKERS = ("Add an end-state confirmation", "Re-run to verify the outcome")
+# Raw runtime-error text carries none of the deny-listed tokens, so the verdict
+# tier is dropped on error-output shapes rather than relying on the token gate.
+_LOOP_VERDICT_RAW_ERROR_RE = re.compile(
+ r"={3,}"
+ r"|\b[A-Z][a-z]\w*(?:Error|Exception)\b"
+ r"|(?i:\btraceback \(most recent call last\))"
+ r"|(?i:\bfailed to execute\b)"
+ r"|(?i:\btimeout \d+\s*ms\b)"
+)
+
+
+def _sanitize_loop_verdict_reason(reason: str | None) -> str | None:
+ if not reason:
+ return None
+ text = " ".join(reason.split())
+ if _LOOP_VERDICT_RAW_ERROR_RE.search(text):
+ return None
+ if text.lower().startswith(_LOOP_VERDICT_FAILED_PREFIX):
+ text = text[len(_LOOP_VERDICT_FAILED_PREFIX) :].strip()
+ for marker in _LOOP_VERDICT_ADVICE_MARKERS:
+ index = text.find(marker)
+ if index != -1:
+ text = text[:index].strip()
+ if len(text) > _LOOP_VERDICT_MAX_CHARS:
+ text = text[:_LOOP_VERDICT_MAX_CHARS].rstrip() + "…"
+ if not text:
+ return None
+ if text[-1] not in ".!?…":
+ text += "."
+ return text
+
+
+def compose_loop_blocker_user_facing_reason(
+ internal_reason_code: str | None,
+ evidence: LoopBlockerEvidence | None,
+ *,
+ blocked_tool: str | None = None,
+) -> tuple[str, tuple[str, ...]]:
+ """Deterministic tier composition; leaky candidates degrade tier-by-tier to
+ the branch template instead of raising out of signal construction."""
+ draft_tier = ("draft",) if evidence is not None and evidence.has_draft else ()
+ if internal_reason_code == "loop_detected_credential_or_parameter_misconfig":
+ return _LOOP_CREDENTIAL_TEMPLATE, draft_tier
+ framing, ask = _LOOP_BRANCH_COPY.get(internal_reason_code or "", _LOOP_BRANCH_COPY["loop_detected_generic"])
+ template = f"{framing} {ask}"
+ if evidence is None:
+ return template, draft_tier
+ verdict = _sanitize_loop_verdict_reason(evidence.outcome_gate_reason)
+ blocker = _LOOP_ANTI_BOT_BLOCKER_COPY if evidence.anti_bot_blocked else None
+ for include_verdict in (True, False):
+ parts = [framing]
+ tiers: list[str] = []
+ if include_verdict and verdict is not None:
+ parts.append(verdict)
+ tiers.append("verdict")
+ if blocker is not None:
+ parts.append(blocker)
+ tiers.append("anti_bot")
+ parts.append(ask)
+ candidate = " ".join(parts)
+ if candidate == template:
+ return template, draft_tier
+ try:
+ assert_clean_user_facing_text(candidate, blocked_tool=blocked_tool)
+ except ValueError:
+ continue
+ return candidate, (*tiers, *draft_tier)
+ return template, draft_tier
+
+
+def build_loop_blocker_signal(
+ loop_message: str,
+ *,
+ tool_name: str,
+ evidence: LoopBlockerEvidence | None = None,
+) -> CopilotToolBlockerSignal:
# Category markers come first because credential/parameter messages also contain "has already failed".
if "with CREDENTIAL_ERROR" in loop_message or "with PARAMETER_BINDING_ERROR" in loop_message:
internal = "loop_detected_credential_or_parameter_misconfig"
- user_facing = (
- "I couldn't run this with the current credential or parameter setup. Update them and ask me to try again."
- )
recovery_hint: RecoveryHint = "ask_user_clarifying"
elif "has already failed" in loop_message:
internal = "loop_detected_repeated_failed_step"
- user_facing = "I retried without making progress. Tell me what to change and I'll try a different approach."
recovery_hint = "report_blocker_to_user"
elif "has been called" in loop_message:
internal = "loop_detected_consecutive_same_tool"
- user_facing = "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."
recovery_hint = "report_blocker_to_user"
else:
internal = "loop_detected_generic"
- user_facing = "I couldn't keep going on this turn. Tell me what to change and I'll try again."
recovery_hint = "report_blocker_to_user"
+ user_facing, tiers = compose_loop_blocker_user_facing_reason(internal, evidence, blocked_tool=tool_name)
return CopilotToolBlockerSignal(
blocker_kind="loop_detected",
agent_steering_text=loop_message,
user_facing_reason=user_facing,
recovery_hint=recovery_hint,
cleared_by_tools=frozenset(),
+ preserves_workflow_draft=evidence is not None and evidence.has_draft,
internal_reason_code=internal,
blocked_tool=tool_name,
+ extra={"loop_evidence_tiers": list(tiers)} if tiers else {},
)
+
+
+def refresh_held_loop_blocker_evidence(ctx: _BlockerSignalCtx) -> None:
+ """Recompose the held loop signal's user-facing copy, draft flag, and extra
+ from current ctx evidence; identity and lifecycle fields never change."""
+ held = getattr(ctx, "blocker_signal", None)
+ if not isinstance(held, CopilotToolBlockerSignal) or held.blocker_kind != "loop_detected":
+ return
+ evidence = loop_blocker_evidence_from_ctx(ctx)
+ user_facing, tiers = compose_loop_blocker_user_facing_reason(
+ held.internal_reason_code, evidence, blocked_tool=held.blocked_tool
+ )
+ if not tiers:
+ return
+ if user_facing == held.user_facing_reason and evidence.has_draft == held.preserves_workflow_draft:
+ return
+ try:
+ refreshed = CopilotToolBlockerSignal(
+ blocker_kind=held.blocker_kind,
+ agent_steering_text=held.agent_steering_text,
+ user_facing_reason=user_facing,
+ recovery_hint=held.recovery_hint,
+ cleared_by_tools=held.cleared_by_tools,
+ preserves_workflow_draft=evidence.has_draft,
+ renders_final_reply=held.renders_final_reply,
+ internal_reason_code=held.internal_reason_code,
+ blocked_tool=held.blocked_tool,
+ classifier_mode=held.classifier_mode,
+ exception_type=held.exception_type,
+ extra={"loop_evidence_tiers": list(tiers)} if tiers else {},
+ )
+ except ValueError:
+ return
+ ctx.blocker_signal = refreshed
+ LOG.info("copilot loop blocker signal evidence refreshed", **to_trace_data(refreshed))
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 6a4b4ddce..fa238994b 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -57,6 +57,11 @@ class TurnNarrativePayload(TypedDict):
responseType: NotRequired[ResponseType]
cancelled: NotRequired[bool]
proposalDisposition: NotRequired[ProposalDisposition]
+ # TurnOutcome.response_kind value: "build" | "clarify" | "diagnose" | "refuse" | "recover".
+ responseKind: NotRequired[str]
+ # The ADR-0005 terminal adjudication (enforcement.verified_goal_satisfied_context):
+ # True only when outcome evidence authorizes a tested-success claim.
+ verifiedSuccess: NotRequired[bool]
designStarted: bool
designEnded: bool
draft: NarrativeDraft | None
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index de2525c3d..684601430 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -22,6 +22,8 @@ from mcp.types import (
from skyvern.forge.sdk.copilot.blocker_signal import (
build_loop_blocker_signal,
+ loop_blocker_evidence_from_ctx,
+ refresh_held_loop_blocker_evidence,
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
@@ -74,7 +76,8 @@ def _hide_code_only_tool(ctx: CopilotContext, tool_name: str) -> bool:
def _stash_and_emit_loop_blocker(ctx: Any, loop_message: str, tool_name: str) -> str:
- return stash_blocker_signal(ctx, build_loop_blocker_signal(loop_message, tool_name=tool_name))
+ signal = build_loop_blocker_signal(loop_message, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx))
+ return stash_blocker_signal(ctx, signal)
def _apply_schema_overlay(
@@ -234,6 +237,7 @@ class SkyvernOverlayMCPServer(MCPServer):
record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, {"ok": False, "error": payload})
return _copilot_to_call_tool_result({"ok": False, "error": payload})
+ refresh_held_loop_blocker_evidence(copilot_ctx)
loop_error = detect_failed_tool_step_loop_for_ctx(copilot_ctx, tool_name, arguments)
if loop_error:
LOG.warning(
diff --git a/skyvern/forge/sdk/copilot/outcome_verification_trace.py b/skyvern/forge/sdk/copilot/outcome_verification_trace.py
index c8bc4fcfe..e3bf6da32 100644
--- a/skyvern/forge/sdk/copilot/outcome_verification_trace.py
+++ b/skyvern/forge/sdk/copilot/outcome_verification_trace.py
@@ -19,6 +19,9 @@ LOG = structlog.get_logger()
_SNAPSHOT_ATTR = "outcome_verification_trace_snapshot"
_COMPLETION_PREFIX = "completion_verification_"
+_CODE_ARTIFACT_VIOLATIONS_KEY = "copilot.code_artifact_violations"
+_CODE_ARTIFACT_VIOLATION_LABELS_KEY = "copilot.code_artifact_violation_block_labels"
+_CODE_ARTIFACT_VIOLATION_COUNT_KEY = "copilot.code_artifact_violation_count"
def _snapshot(ctx: Any) -> dict[str, Any] | None:
@@ -42,6 +45,26 @@ def record_gate_decision(ctx: Any, fields: dict[str, Any]) -> None:
LOG.warning("failed to record copilot gate decision telemetry", exc_info=True)
+def record_code_artifact_violations(ctx: Any, violations: list[str], offending_labels: list[str]) -> None:
+ """Persist the code-artifact-metadata violation batch onto the turn snapshot.
+
+ The strings carry schema field names and the graph's structural identifiers
+ (block labels, claim/dependency/expectation ids) but never free-text content,
+ so they ride the prod span verbatim without value-scrubbing.
+ """
+ try:
+ if not violations:
+ return
+ snapshot = _snapshot(ctx)
+ if snapshot is None:
+ return
+ snapshot[_CODE_ARTIFACT_VIOLATIONS_KEY] = [str(message) for message in violations]
+ snapshot[_CODE_ARTIFACT_VIOLATION_LABELS_KEY] = sorted({str(label) for label in offending_labels if str(label)})
+ snapshot[_CODE_ARTIFACT_VIOLATION_COUNT_KEY] = len(violations)
+ except Exception:
+ LOG.warning("failed to record copilot code artifact violation telemetry", exc_info=True)
+
+
def record_completion_verification(ctx: Any, result: Any | None) -> None:
"""Refresh the snapshot's completion-verification block for the latest run.
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 5be680246..6ebc1a3b8 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -170,6 +170,9 @@ class AgentContext:
last_test_suspicious_success: bool = False
last_test_anti_bot: str | None = None
last_test_failure_reason: str | None = None
+ # Latest evaluated outcome-gate verdict this turn. Deliberately not reset
+ # per-run: a later run that fails before verification keeps the verdict.
+ last_outcome_gate_reason: str | None = None
last_failure_category_top: str | None = None
last_update_block_count: int | None = None
last_failed_workflow_yaml: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index cc3d5292c..f861150d0 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -74,6 +74,9 @@ from .blockers import _post_budget_terminal_challenge_signal as _post_budget_ter
from .blockers import _post_run_terminal_challenge_reason as _post_run_terminal_challenge_reason
from .blockers import (
_record_per_tool_budget_problem_blocks_from_results,
+)
+from .blockers import _run_blocks_structured_blocker_message as _run_blocks_structured_blocker_message
+from .blockers import (
_tool_loop_error,
)
from .blockers import _trusted_post_drain_status as _trusted_post_drain_status
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 0c9b43a19..288ed33df 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -282,16 +282,63 @@ def _current_workflow_has_evidence_block(ctx: object) -> bool:
workflow = getattr(ctx, "last_workflow", None)
blocks = getattr(getattr(workflow, "workflow_definition", None), "blocks", None)
if blocks:
- return any(_block_type_name(block) in _OUTCOME_EVIDENCE_BLOCK_TYPES for block in blocks)
+ if any(_block_type_name(block) in _OUTCOME_EVIDENCE_BLOCK_TYPES for block in blocks):
+ return True
+ code_labels = [
+ getattr(block, "label", None) for block in blocks if _block_type_name(block) == BlockType.CODE.value
+ ]
+ return _code_artifact_metadata_covers_terminal_criterion(ctx, code_labels)
workflow_yaml = getattr(ctx, "last_workflow_yaml", None)
if not isinstance(workflow_yaml, str):
return False
+ parsed_blocks = [block for block in (_parse_workflow_blocks(workflow_yaml) or []) if isinstance(block, dict)]
+ if any(_enum_or_string_name(block.get("block_type")) in _OUTCOME_EVIDENCE_BLOCK_TYPES for block in parsed_blocks):
+ return True
+ code_labels = [
+ _block_label_from_yaml(block)
+ for block in parsed_blocks
+ if _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value
+ ]
+ return _code_artifact_metadata_covers_terminal_criterion(ctx, code_labels)
+
+
+def _code_artifact_metadata_covers_terminal_criterion(ctx: object, labels: list[str | None]) -> bool:
+ metadata = getattr(ctx, "code_artifact_metadata", None)
+ if not isinstance(metadata, dict):
+ return False
return any(
- isinstance(block, dict) and _enum_or_string_name(block.get("block_type")) in _OUTCOME_EVIDENCE_BLOCK_TYPES
- for block in (_parse_workflow_blocks(workflow_yaml) or [])
+ isinstance(metadata.get(label), dict) and _artifact_entry_claims_terminal_criterion(metadata[label])
+ for label in labels
+ if isinstance(label, str)
)
+def _artifact_entry_claims_terminal_criterion(entry: dict[str, Any]) -> bool:
+ criteria = entry.get("completion_criteria")
+ criteria_rows = [row for row in criteria if isinstance(row, dict)] if isinstance(criteria, list) else []
+ terminal_ids = {
+ str(row.get("id") or "").strip()
+ for row in criteria_rows
+ if row.get("terminal") is True or str(row.get("level") or "").strip() == "terminal"
+ } - {""}
+ if not terminal_ids:
+ return False
+ claims = entry.get("claimed_outcomes")
+ if not isinstance(claims, list):
+ return False
+ for claim in claims:
+ if not isinstance(claim, dict):
+ continue
+ covered: set[str] = set()
+ for field_name in ("covered_criteria", "criteria_ids"):
+ values = claim.get(field_name)
+ if isinstance(values, list):
+ covered.update(str(item).strip() for item in values)
+ if covered & terminal_ids:
+ return True
+ return False
+
+
def _unverified_current_workflow_labels(ctx: object) -> list[str]:
labels = _current_workflow_block_labels(ctx)
verified = set(getattr(ctx, "verified_prefix_labels", []) or [])
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index 870e9903c..9fd24c14c 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -12,6 +12,8 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
build_loop_blocker_signal,
+ loop_blocker_evidence_from_ctx,
+ refresh_held_loop_blocker_evidence,
)
from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controls
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
@@ -567,6 +569,24 @@ _ANTI_BOT_BLOCKER_TERMS: tuple[str, ...] = (
"human verification",
"verify you are human",
)
+# Strict subset of ``_STRUCTURED_BLOCKER_KEY_TERMS`` for the flag/status rules that
+# scan arbitrary code-block JSON; broad terms like ``verification`` stay string-only.
+_STRICT_BLOCKER_FLAG_TERMS: frozenset[str] = frozenset(
+ {
+ "blocker",
+ "blocked",
+ "captcha",
+ "challenge",
+ "human_verification",
+ }
+)
+_BLOCKER_STATUS_KEYS: frozenset[str] = frozenset({"status", "state"})
+_BLOCKER_SIBLING_MESSAGE_KEYS: frozenset[str] = frozenset({"reason", "message", "error", "failure_reason"})
+_MAX_BLOCKER_STATUS_VALUE_LEN = 80
+
+
+def _is_code_block_type(block_type: object) -> bool:
+ return isinstance(block_type, str) and block_type.strip().upper() == BlockType.CODE.value.upper()
def _normalize_structured_key(value: object) -> str:
@@ -578,7 +598,7 @@ def _looks_like_anti_bot_blocker(text: str) -> bool:
return any(term in lowered for term in _ANTI_BOT_BLOCKER_TERMS)
-def _structured_blocker_message(value: object, *, depth: int = 0) -> str | None:
+def _structured_blocker_message(value: object, *, depth: int = 0, include_flag_keys: bool = False) -> str | None:
if depth > 5:
return None
if isinstance(value, dict):
@@ -594,18 +614,44 @@ def _structured_blocker_message(value: object, *, depth: int = 0) -> str | None:
and _looks_like_anti_bot_blocker(item)
):
return item.strip()[:240]
+ if include_flag_keys:
+ flagged = _blocker_flag_or_status_message(value)
+ if flagged:
+ return flagged
for item in value.values():
- nested = _structured_blocker_message(item, depth=depth + 1)
+ nested = _structured_blocker_message(item, depth=depth + 1, include_flag_keys=include_flag_keys)
if nested:
return nested
elif isinstance(value, list):
for item in value:
- nested = _structured_blocker_message(item, depth=depth + 1)
+ nested = _structured_blocker_message(item, depth=depth + 1, include_flag_keys=include_flag_keys)
if nested:
return nested
return None
+def _blocker_flag_or_status_message(value: dict[str, Any]) -> str | None:
+ for key, item in value.items():
+ normalized_key = _normalize_structured_key(key)
+ if item is True and any(term in normalized_key for term in _STRICT_BLOCKER_FLAG_TERMS):
+ for sibling_key, sibling_item in value.items():
+ if (
+ _normalize_structured_key(sibling_key) in _BLOCKER_SIBLING_MESSAGE_KEYS
+ and isinstance(sibling_item, str)
+ and sibling_item.strip()
+ ):
+ return sibling_item.strip()[:240]
+ return f"The run output flagged {normalized_key.replace('_', ' ')}."
+ if (
+ isinstance(item, str)
+ and normalized_key in _BLOCKER_STATUS_KEYS
+ and 0 < len(item.strip()) <= _MAX_BLOCKER_STATUS_VALUE_LEN
+ and any(term in item.strip().lower() for term in _STRICT_BLOCKER_FLAG_TERMS)
+ ):
+ return f"The run output reported status '{item.strip()}'."
+ return None
+
+
def _run_blocks_structured_blocker_message(result: dict[str, Any]) -> str | None:
data = result.get("data")
if not isinstance(data, dict):
@@ -620,15 +666,62 @@ def _run_blocks_structured_blocker_message(result: dict[str, Any]) -> str | None
if not isinstance(block, dict):
continue
block_type = block.get("block_type")
- if block_type not in _DATA_PRODUCING_BLOCK_TYPES or block.get("status") != "completed":
+ if block.get("status") != "completed":
+ continue
+ if _is_code_block_type(block_type):
+ blocker = _structured_blocker_message(block.get("extracted_data"), include_flag_keys=True)
+ elif block_type in _DATA_PRODUCING_BLOCK_TYPES:
+ payload = _block_data_payload(block.get("extracted_data"), block_type)
+ blocker = _structured_blocker_message(payload)
+ else:
continue
- payload = _block_data_payload(block.get("extracted_data"), block_type)
- blocker = _structured_blocker_message(payload)
if blocker:
return blocker
return None
+def _is_blocker_term_key(key: object) -> bool:
+ normalized_key = _normalize_structured_key(key)
+ return any(term in normalized_key for term in _STRUCTURED_BLOCKER_KEY_TERMS)
+
+
+def _code_output_contains_collection(value: Any, *, depth: int = 0) -> bool:
+ if depth > 5:
+ return False
+ if isinstance(value, (list, tuple, set)):
+ return True
+ if isinstance(value, dict):
+ return any(
+ _code_output_contains_collection(item, depth=depth + 1)
+ for key, item in value.items()
+ if not _is_blocker_term_key(key)
+ )
+ return False
+
+
+def _code_output_has_goal_content(value: Any, *, depth: int = 0) -> bool:
+ """Goal content in a code block's output: a non-empty string, truthy number, or
+ non-empty collection surviving after blocker-term, status, and boolean entries
+ are stripped (status/state values are machine shape, not goal data)."""
+ if depth > 5:
+ return False
+ if isinstance(value, bool):
+ return False
+ if isinstance(value, (int, float)):
+ return bool(value)
+ if isinstance(value, str):
+ return bool(value.strip())
+ if isinstance(value, (list, tuple, set)):
+ return len(value) > 0
+ if isinstance(value, dict):
+ return any(
+ _code_output_has_goal_content(item, depth=depth + 1)
+ for key, item in value.items()
+ if not _is_blocker_term_key(key) and _normalize_structured_key(key) not in _BLOCKER_STATUS_KEYS
+ )
+ return False
+
+
def _active_block_run_budget_seconds(ctx: AgentContext) -> int:
remaining = _copilot_seconds_remaining(ctx)
if remaining is None:
@@ -828,9 +921,13 @@ def _challenge_gated_anti_bot_rerun_signal(
def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:
+ refresh_held_loop_blocker_evidence(ctx)
detected = detect_failed_tool_step_loop_for_ctx(ctx, tool_name, arguments or {})
if detected is not None:
- return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
+ return _emit_tool_blocker_signal(
+ ctx,
+ _build_loop_blocker_signal(detected, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx)),
+ )
# Consecutive same-name guard: false-positives on the intended iterative
# build (one new block per update_and_run_blocks). Block-running tools
@@ -839,7 +936,10 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
if isinstance(tracker, list) and tool_name not in BLOCK_RUNNING_TOOLS:
detected = detect_tool_loop(tracker, tool_name)
if detected is not None:
- return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
+ return _emit_tool_blocker_signal(
+ ctx,
+ _build_loop_blocker_signal(detected, tool_name=tool_name, evidence=loop_blocker_evidence_from_ctx(ctx)),
+ )
if tool_name == "update_workflow" or tool_name in BLOCK_RUNNING_TOOLS:
active_terminal_signal = _active_run_terminal_evidence_signal(ctx, tool_name)
@@ -1016,7 +1116,9 @@ def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[
if isinstance(reason, str):
texts_to_scan.append(reason)
block_type = block.get("block_type")
- if block_type in _DATA_PRODUCING_BLOCK_TYPES and block.get("status") == "completed":
+ if block.get("status") != "completed":
+ continue
+ if block_type in _DATA_PRODUCING_BLOCK_TYPES:
has_data_blocks = True
payload = _block_data_payload(block.get("extracted_data"), block_type)
structured_blocker = _structured_blocker_message(payload)
@@ -1024,6 +1126,19 @@ def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[
texts_to_scan.append(structured_blocker)
if _is_meaningful_extracted_data(payload):
any_data_output = True
+ elif _is_code_block_type(block_type):
+ extracted = block.get("extracted_data")
+ if extracted is None:
+ continue
+ structured_blocker = _structured_blocker_message(extracted, include_flag_keys=True)
+ if structured_blocker:
+ texts_to_scan.append(structured_blocker)
+ # A code output joins the emptiness denominator only when it declares a
+ # collection shape; action-only outputs are exempt.
+ if _code_output_contains_collection(extracted):
+ has_data_blocks = True
+ if _code_output_has_goal_content(extracted):
+ any_data_output = True
combined = "\n".join(texts_to_scan)
categories = classify_from_failure_reason(combined)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 2e89f4287..de139b906 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -36,7 +36,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
from skyvern.forge.sdk.copilot.narration import NarratorState
from skyvern.forge.sdk.copilot.narration import handler_available as narration_handler_available
from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
-from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
+from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification, record_gate_decision
from skyvern.forge.sdk.copilot.output_utils import (
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
build_run_blocks_response,
@@ -1400,6 +1400,8 @@ def _record_run_blocks_result(
copilot_ctx.last_full_workflow_test_ok = False
copilot_ctx.last_unverified_block_labels = _unverified_current_workflow_labels(copilot_ctx)
copilot_ctx.last_test_failure_reason = None
+ if completion_verification is not None and completion_verification.status == "evaluated":
+ copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
copilot_ctx.last_test_suspicious_success = False
copilot_ctx.suspicious_success_nudge_count = 0
copilot_ctx.last_test_anti_bot = None
@@ -1414,6 +1416,13 @@ def _record_run_blocks_result(
structured_blocker = _run_blocks_structured_blocker_message(result)
anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result)
+ record_gate_decision(
+ copilot_ctx,
+ {
+ "run_output_blocker_detected": bool(structured_blocker),
+ "run_output_empty_data_blocks": bool(empty_data_blocks),
+ },
+ )
if not anti_bot_match:
anti_bot_match = _composition_anti_bot_reason(copilot_ctx)
if anti_bot_match:
@@ -1479,7 +1488,7 @@ def _record_run_blocks_result(
copilot_ctx.null_data_streak_count = getattr(copilot_ctx, "null_data_streak_count", 0) + 1
copilot_ctx.last_test_failure_reason = (
"All blocks completed but data-producing blocks "
- "(extraction/text_prompt) produced no meaningful output "
+ "produced no meaningful output "
"(missing, empty, or all-null fields). "
"The workflow may not be working correctly."
)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index ed86894dd..01620b4d1 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import re
from collections.abc import Mapping
-from typing import Annotated, Any, Literal
+from typing import Annotated, Any, Literal, NamedTuple
import structlog
import yaml
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_goal_likely_needs_more_blocks,
)
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
+from skyvern.forge.sdk.copilot.outcome_verification_trace import record_code_artifact_violations
from skyvern.forge.sdk.copilot.output_policy import (
evaluate_output_policy,
format_output_policy_tool_error,
@@ -232,6 +233,23 @@ def _format_code_artifact_violations(violations: list[str]) -> str:
return f"Artifact metadata has {len(violations)} contract violations; fix all of them in one update:\n{numbered}"
+def _code_artifact_validation_error_message(exc: ValidationError) -> str:
+ # Build from loc/type only; pydantic's str(exc) embeds input_value, which would
+ # carry submitted metadata values onto the scrubbing-exempt durable span.
+ parts = [
+ f"{'.'.join(str(loc) for loc in err.get('loc', ()))}: {err.get('type', 'invalid')}" for err in exc.errors()
+ ]
+ detail = "; ".join(part for part in parts if part) or "schema validation failed"
+ return f"Artifact metadata is malformed ({detail})."
+
+
+class CodeArtifactNormalization(NamedTuple):
+ normalized: dict[str, dict[str, Any]]
+ error: str | None
+ violations: list[str]
+ offending_labels: list[str]
+
+
def _normalize_code_artifact_metadata(
raw_metadata: Any,
workflow_yaml: str,
@@ -239,18 +257,36 @@ def _normalize_code_artifact_metadata(
impose_defaults: bool = False,
scout_trajectory: list[ScoutedInteraction] | None = None,
) -> tuple[dict[str, dict[str, Any]], str | None]:
+ result = _normalize_code_artifact_metadata_detailed(
+ raw_metadata,
+ workflow_yaml,
+ impose_defaults=impose_defaults,
+ scout_trajectory=scout_trajectory,
+ )
+ return result.normalized, result.error
+
+
+def _normalize_code_artifact_metadata_detailed(
+ raw_metadata: Any,
+ workflow_yaml: str,
+ *,
+ impose_defaults: bool = False,
+ scout_trajectory: list[ScoutedInteraction] | None = None,
+) -> CodeArtifactNormalization:
"""Normalize submitted artifact metadata at the persist seam.
Entries keyed to a missing code-block label are re-keyed to the single
uncovered block or dropped, never rejected. When imposing, mechanical graph
fields are server-defaulted (and uncovered labels get a deterministic
- skeleton) so only contradictions reject."""
+ skeleton) so only contradictions reject. Returns the per-violation list and
+ offending labels alongside the batched error for durable telemetry."""
if raw_metadata in (None, [], {}):
- return {}, None
+ return CodeArtifactNormalization({}, None, [], [])
items = _code_artifact_metadata_items(raw_metadata)
code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
trajectory: list[ScoutedInteraction] = scout_trajectory or []
violations: list[str] = []
+ offending_labels: list[str] = []
anchored: list[dict[str, Any]] = []
unanchored: list[dict[str, Any]] = []
seen_labels: set[str] = set()
@@ -262,7 +298,7 @@ def _normalize_code_artifact_metadata(
else CodeArtifactMetadata.model_validate(raw_item)
)
except ValidationError as exc:
- violations.append(f"Artifact metadata is malformed: {exc}")
+ violations.append(_code_artifact_validation_error_message(exc))
continue
dumped = metadata.model_dump(mode="json", exclude_none=True)
label = str(dumped.get("block_label") or "").strip()
@@ -327,14 +363,17 @@ def _normalize_code_artifact_metadata(
item_violations.extend(_code_artifact_metadata_shape_errors(label, dumped))
if item_violations:
violations.extend(item_violations)
+ offending_labels.append(label)
continue
normalized[label] = dumped
if impose_defaults:
for label in uncovered:
normalized[label] = _imposed_artifact_skeleton(label, code_blocks[label], trajectory)
if violations:
- return normalized, _format_code_artifact_violations(violations)
- return normalized, None
+ return CodeArtifactNormalization(
+ normalized, _format_code_artifact_violations(violations), violations, offending_labels
+ )
+ return CodeArtifactNormalization(normalized, None, [], [])
def _artifact_label_fragment(label: str) -> str:
@@ -716,12 +755,16 @@ async def _update_workflow(
ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
scout_trajectory = getattr(ctx, "scout_trajectory", None)
- code_artifact_metadata, code_artifact_metadata_error = _normalize_code_artifact_metadata(
+ normalization = _normalize_code_artifact_metadata_detailed(
params.get("code_artifact_metadata"),
workflow_yaml,
impose_defaults=_copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER,
scout_trajectory=scout_trajectory if isinstance(scout_trajectory, list) else None,
)
+ code_artifact_metadata = normalization.normalized
+ code_artifact_metadata_error = normalization.error
+ if code_artifact_metadata_error is not None:
+ record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)
code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)
# Per-label salvage keeps conforming metadata across a rejection; a
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 493223ccb..225ee713e 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -1488,6 +1488,7 @@ class WorkflowCopilotChatMessageModel(Base):
organization_id = Column(String, nullable=False)
sender = Column(String, nullable=False)
content = Column(UnicodeText, nullable=False)
+ audio_artifact_id = Column(String, nullable=True)
global_llm_context = Column(UnicodeText, nullable=True)
turn_outcome = Column(JSON, nullable=True)
narrative_payload = Column(JSON, nullable=True)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index 1534a4741..23c6f0996 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -520,6 +520,7 @@ class WorkflowParametersRepository(BaseRepository):
workflow_copilot_chat_id: str,
sender: WorkflowCopilotChatSender,
content: str,
+ audio_artifact_id: str | None = None,
global_llm_context: str | None = None,
turn_outcome: TurnOutcome | None = None,
narrative_payload: TurnNarrativePayload | dict[str, Any] | None = None,
@@ -530,6 +531,7 @@ class WorkflowParametersRepository(BaseRepository):
organization_id=organization_id,
sender=sender,
content=content,
+ audio_artifact_id=audio_artifact_id,
global_llm_context=global_llm_context,
turn_outcome=turn_outcome.model_dump(mode="json") if turn_outcome is not None else None,
narrative_payload=narrative_payload,
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 7e0b44d49..b22852967 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -89,11 +89,13 @@ from skyvern.webeye.actions.actions import (
KeypressAction,
LeftMouseAction,
MoveAction,
+ NewTabAction,
NullAction,
ReloadPageAction,
ScrollAction,
SelectOptionAction,
SolveCaptchaAction,
+ SwitchTabAction,
TerminateAction,
UploadFileAction,
VerificationCodeAction,
@@ -223,6 +225,8 @@ ACTION_TYPE_TO_CLASS = {
ActionType.SOLVE_CAPTCHA: SolveCaptchaAction,
ActionType.RELOAD_PAGE: ReloadPageAction,
ActionType.CLOSE_PAGE: ClosePageAction,
+ ActionType.NEW_TAB: NewTabAction,
+ ActionType.SWITCH_TAB: SwitchTabAction,
ActionType.EXTRACT: ExtractAction,
ActionType.SCROLL: ScrollAction,
ActionType.KEYPRESS: KeypressAction,
@@ -387,6 +391,7 @@ def convert_to_workflow_copilot_chat_message(
workflow_copilot_chat_id=message_model.workflow_copilot_chat_id,
sender=message_model.sender,
content=message_model.content,
+ audio_artifact_id=message_model.audio_artifact_id,
global_llm_context=message_model.global_llm_context,
turn_outcome=parsed_outcome,
narrative_payload=parsed_narrative,
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 95074b0e0..5fbd73ba2 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -2113,6 +2113,7 @@ _ARTIFACT_CONTENT_TYPES: dict[ArtifactType, str] = {
ArtifactType.SCREENSHOT_ACTION: "image/png",
ArtifactType.SCREENSHOT_FINAL: "image/png",
ArtifactType.RECORDING: "video/webm",
+ ArtifactType.AUDIO: "audio/webm",
ArtifactType.SESSION_REPLAY: "video/mp4",
ArtifactType.DOWNLOAD: "application/octet-stream",
}
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index e788ba4de..5f228047d 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -11,7 +11,7 @@ from typing import Any, Iterator
import structlog
import yaml
-from fastapi import Depends, HTTPException, Request, status
+from fastapi import Depends, File, Form, HTTPException, Request, UploadFile, status
from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from sse_starlette import EventSourceResponse
@@ -22,7 +22,7 @@ from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
-from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
from skyvern.forge.sdk.copilot.agent import run_copilot_agent
from skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write, resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
@@ -43,7 +43,9 @@ from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotApplyProposedWorkflowRequest,
+ WorkflowCopilotAudioUploadResponse,
WorkflowCopilotCancelRequest,
+ WorkflowCopilotChat,
WorkflowCopilotChatHistoryMessage,
WorkflowCopilotChatHistoryResponse,
WorkflowCopilotChatMessage,
@@ -78,6 +80,15 @@ from skyvern.utils.yaml_loader import safe_load_no_dates
WORKFLOW_KNOWLEDGE_BASE_PATH = Path("skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt")
CHAT_HISTORY_CONTEXT_MESSAGES = 10
+ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {
+ "audio/mp4",
+ "audio/mpeg",
+ "audio/ogg",
+ "audio/wav",
+ "audio/wave",
+ "audio/webm",
+ "audio/x-wav",
+}
LOG = structlog.get_logger()
@@ -425,6 +436,7 @@ async def _persist_cancel_turn(
original_workflow: Workflow | None,
user_message: str,
agent_result: AgentResult | None,
+ audio_artifact_id: str | None = None,
turn_id: str | None = None,
) -> None:
"""Persist a cancelled turn and emit a terminal SSE response frame.
@@ -486,6 +498,7 @@ async def _persist_cancel_turn(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
sender=WorkflowCopilotChatSender.USER,
content=user_message,
+ audio_artifact_id=audio_artifact_id,
)
)
assistant_message = await asyncio.shield(
@@ -592,6 +605,7 @@ async def _finalise_normal_turn(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
sender=WorkflowCopilotChatSender.USER,
content=chat_request.message,
+ audio_artifact_id=chat_request.audio_artifact_id,
)
assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
@@ -1515,22 +1529,17 @@ async def _new_copilot_chat_post(
)
)
- if chat_request.workflow_copilot_chat_id:
- chat = await app.DATABASE.workflow_params.get_workflow_copilot_chat_by_id(
- organization_id=organization.organization_id,
- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
- )
- if not chat:
- raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Chat not found")
- if chat_request.workflow_permanent_id != chat.workflow_permanent_id:
- raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Wrong workflow permanent ID")
- else:
- chat = await app.DATABASE.workflow_params.create_workflow_copilot_chat(
- organization_id=organization.organization_id,
- workflow_permanent_id=chat_request.workflow_permanent_id,
- )
-
+ chat = await _get_or_create_workflow_copilot_chat(
+ organization_id=organization.organization_id,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ )
chat_request.workflow_copilot_chat_id = chat.workflow_copilot_chat_id
+ chat_request.audio_artifact_id = await _validate_copilot_audio_artifact_id(
+ audio_artifact_id=chat_request.audio_artifact_id,
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ )
chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
@@ -1695,6 +1704,7 @@ async def _new_copilot_chat_post(
original_workflow=original_workflow,
user_message=chat_request.message,
agent_result=agent_result,
+ audio_artifact_id=chat_request.audio_artifact_id,
turn_id=turn_id,
)
terminal_frame_emitted = True
@@ -1816,6 +1826,7 @@ async def _new_copilot_chat_post(
original_workflow=None,
user_message=chat_request.message,
agent_result=None,
+ audio_artifact_id=chat_request.audio_artifact_id,
turn_id=turn_id,
)
)
@@ -1933,6 +1944,120 @@ async def _should_use_copilot_v2(
return False
+async def _get_or_create_workflow_copilot_chat(
+ *,
+ organization_id: str,
+ workflow_permanent_id: str,
+ workflow_copilot_chat_id: str | None,
+) -> WorkflowCopilotChat:
+ if workflow_copilot_chat_id:
+ chat = await app.DATABASE.workflow_params.get_workflow_copilot_chat_by_id(
+ organization_id=organization_id,
+ workflow_copilot_chat_id=workflow_copilot_chat_id,
+ )
+ if not chat:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Chat not found")
+ if workflow_permanent_id != chat.workflow_permanent_id:
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Wrong workflow permanent ID")
+ return chat
+
+ return await app.DATABASE.workflow_params.create_workflow_copilot_chat(
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ )
+
+
+async def _validate_copilot_audio_artifact_id(
+ *,
+ audio_artifact_id: str | None,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+) -> str | None:
+ if not audio_artifact_id:
+ return None
+
+ artifact = await app.DATABASE.artifacts.get_artifact_by_id(
+ audio_artifact_id,
+ organization_id=organization_id,
+ )
+ if not artifact:
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Audio artifact not found")
+ if artifact.artifact_type != ArtifactType.AUDIO:
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid audio artifact type")
+
+ # Audio artifacts are written through create_log_artifact for this chat, so
+ # the log URI carries the chat id. Keep this in sync with artifact storage
+ # path construction if that layout changes.
+ log_marker = f"/logs/{LogEntityType.WORKFLOW_COPILOT_CHAT}/{workflow_copilot_chat_id}/"
+ if log_marker not in (artifact.uri or ""):
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail="Audio artifact is not linked to this chat",
+ )
+ return audio_artifact_id
+
+
+@base_router.post("/workflow/copilot/chat-audio", include_in_schema=False)
+async def workflow_copilot_chat_audio(
+ workflow_permanent_id: str = Form(...),
+ workflow_copilot_chat_id: str | None = Form(None),
+ file: UploadFile = File(...),
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> WorkflowCopilotAudioUploadResponse:
+ content_type = (file.content_type or "").split(";", 1)[0].strip().lower()
+ if content_type not in ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES:
+ await file.close()
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Unsupported audio format")
+
+ max_upload_bytes = settings.MAX_UPLOAD_FILE_SIZE
+ try:
+ audio_bytes = await file.read(max_upload_bytes + 1)
+ except Exception as exc:
+ LOG.exception("Failed to read workflow copilot dictation audio upload")
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to read audio file"
+ ) from exc
+ finally:
+ await file.close()
+
+ if not audio_bytes:
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Audio file is empty")
+
+ if len(audio_bytes) > max_upload_bytes:
+ raise HTTPException(
+ status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
+ detail=f"Audio file exceeds the maximum allowed size ({max_upload_bytes / 1024 / 1024:.0f} MB)",
+ )
+
+ chat = await _get_or_create_workflow_copilot_chat(
+ organization_id=organization.organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ workflow_copilot_chat_id=workflow_copilot_chat_id,
+ )
+
+ LOG.info(
+ "Workflow copilot dictation audio upload",
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=organization.organization_id,
+ audio_bytes=len(audio_bytes),
+ )
+
+ audio_artifact_id = await app.ARTIFACT_MANAGER.create_log_artifact(
+ log_entity_type=LogEntityType.WORKFLOW_COPILOT_CHAT,
+ log_entity_id=chat.workflow_copilot_chat_id,
+ artifact_type=ArtifactType.AUDIO,
+ organization_id=organization.organization_id,
+ data=audio_bytes,
+ )
+ await app.ARTIFACT_MANAGER.wait_for_upload_aiotasks([chat.workflow_copilot_chat_id])
+
+ return WorkflowCopilotAudioUploadResponse(
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ audio_artifact_id=audio_artifact_id,
+ )
+
+
@base_router.post("/workflow/copilot/chat-post", include_in_schema=False)
async def workflow_copilot_chat_post(
request: Request,
@@ -1962,20 +2087,17 @@ async def workflow_copilot_chat_post(
)
)
- if chat_request.workflow_copilot_chat_id:
- chat = await app.DATABASE.workflow_params.get_workflow_copilot_chat_by_id(
- organization_id=organization.organization_id,
- workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
- )
- if not chat:
- raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Chat not found")
- if chat_request.workflow_permanent_id != chat.workflow_permanent_id:
- raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Wrong workflow permanent ID")
- else:
- chat = await app.DATABASE.workflow_params.create_workflow_copilot_chat(
- organization_id=organization.organization_id,
- workflow_permanent_id=chat_request.workflow_permanent_id,
- )
+ chat = await _get_or_create_workflow_copilot_chat(
+ organization_id=organization.organization_id,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
+ )
+ chat_request.workflow_copilot_chat_id = chat.workflow_copilot_chat_id
+ chat_request.audio_artifact_id = await _validate_copilot_audio_artifact_id(
+ audio_artifact_id=chat_request.audio_artifact_id,
+ organization_id=organization.organization_id,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ )
chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
@@ -2042,6 +2164,7 @@ async def workflow_copilot_chat_post(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
sender=WorkflowCopilotChatSender.USER,
content=chat_request.message,
+ audio_artifact_id=chat_request.audio_artifact_id,
)
assistant_message = await app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
@@ -2255,6 +2378,7 @@ def convert_to_history_messages(
WorkflowCopilotChatHistoryMessage(
sender=message.sender,
content=message.content,
+ audio_artifact_id=message.audio_artifact_id,
turn_outcome=message.turn_outcome,
created_at=message.created_at,
narrative_payload=message.narrative_payload,
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 47ef8356c..d69dbadb2 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -47,6 +47,7 @@ class WorkflowCopilotChatMessage(BaseModel):
workflow_copilot_chat_id: str = Field(..., description="ID of the parent workflow copilot chat")
sender: WorkflowCopilotChatSender = Field(..., description="Message sender")
content: str = Field(..., description="Message content")
+ audio_artifact_id: str | None = Field(None, description="Artifact ID for audio captured during dictation")
global_llm_context: str | None = Field(None, description="Optional global LLM context for the message")
turn_outcome: TurnOutcome | None = Field(None, description="Typed turn outcome (assistant rows)")
narrative_payload: TurnNarrativePayload | None = Field(
@@ -67,6 +68,10 @@ class WorkflowCopilotChatRequest(BaseModel):
description="Optional persistent browser session ID to reuse instead of creating a new one.",
)
message: str = Field(..., description="The message that user sends")
+ audio_artifact_id: str | None = Field(
+ None,
+ description="Artifact ID for audio captured while dictating this message.",
+ )
workflow_yaml: str = Field(..., description="Current workflow YAML including unsaved changes")
mode: Literal["ask", "build"] | None = Field(
None, description="Per-request copilot path selector; None falls back to feature flags."
@@ -103,6 +108,7 @@ class WorkflowCopilotApplyProposedWorkflowRequest(BaseModel):
class WorkflowCopilotChatHistoryMessage(BaseModel):
sender: WorkflowCopilotChatSender = Field(..., description="Message sender")
content: str = Field(..., description="Message content")
+ audio_artifact_id: str | None = Field(None, description="Artifact ID for captured dictation audio")
turn_outcome: TurnOutcome | None = Field(None, description="Typed turn outcome (assistant rows only)")
narrative_payload: TurnNarrativePayload | None = Field(
None,
@@ -118,6 +124,11 @@ class WorkflowCopilotChatHistoryResponse(BaseModel):
auto_accept: bool | None = Field(None, description="Whether copilot auto-accepts workflow updates")
+class WorkflowCopilotAudioUploadResponse(BaseModel):
+ workflow_copilot_chat_id: str = Field(..., description="Chat ID the audio artifact is associated with")
+ audio_artifact_id: str = Field(..., description="Stored audio artifact ID")
+
+
class WorkflowCopilotStreamMessageType(StrEnum):
PROCESSING_UPDATE = "processing_update"
RESPONSE = "response"
diff --git a/skyvern/webeye/actions/action_types.py b/skyvern/webeye/actions/action_types.py
index b45de0c89..3abe3d31c 100644
--- a/skyvern/webeye/actions/action_types.py
+++ b/skyvern/webeye/actions/action_types.py
@@ -19,6 +19,8 @@ class ActionType(StrEnum):
COMPLETE = "complete"
RELOAD_PAGE = "reload_page"
CLOSE_PAGE = "close_page"
+ NEW_TAB = "new_tab"
+ SWITCH_TAB = "switch_tab"
EXTRACT = "extract"
VERIFICATION_CODE = "verification_code"
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index 88fdea879..306963713 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -218,6 +218,10 @@ class Action(BaseModel):
return GoForwardAction.model_validate(value)
elif action_type is ActionType.CLOSE_PAGE:
return ClosePageAction.model_validate(value)
+ elif action_type is ActionType.NEW_TAB:
+ return NewTabAction.model_validate(value)
+ elif action_type is ActionType.SWITCH_TAB:
+ return SwitchTabAction.model_validate(value)
elif action_type is ActionType.EXECUTE_JS:
return ExecuteJsAction.model_validate(value)
else:
@@ -253,6 +257,16 @@ class ClosePageAction(Action):
action_type: ActionType = ActionType.CLOSE_PAGE
+class NewTabAction(Action):
+ action_type: ActionType = ActionType.NEW_TAB
+ url: str
+
+
+class SwitchTabAction(Action):
+ action_type: ActionType = ActionType.SWITCH_TAB
+ tab_index: int
+
+
class ClickAction(WebAction):
action_type: ActionType = ActionType.CLICK
file_url: str | None = None
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index cba5bdc8d..2f5435e19 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -3161,6 +3161,71 @@ async def handle_close_page_action(
return [ActionSuccess()]
+@traced(name="skyvern.agent.action.new_tab")
+async def handle_new_tab_action(
+ action: actions.NewTabAction,
+ page: Page,
+ scraped_page: ScrapedPage,
+ task: Task,
+ step: Step,
+) -> list[ActionResult]:
+ browser_state = app.BROWSER_MANAGER.get_for_task(task.task_id, workflow_run_id=task.workflow_run_id)
+ if browser_state is None:
+ return [ActionFailure(Exception("No browser state found for the task"), stop_execution_on_failure=False)]
+ new_page = await browser_state.new_page()
+ try:
+ await browser_state.navigate_to_url(page=new_page, url=action.url)
+ except Exception as e:
+ # Don't leave a blank/failed tab as the newest page — the next scrape would fail it.
+ try:
+ await new_page.close()
+ except Exception:
+ LOG.debug("Failed to close new tab after navigation failure", exc_info=True)
+ return [ActionFailure(e, stop_execution_on_failure=False)]
+ await browser_state.set_active_page(new_page)
+ try:
+ await new_page.bring_to_front()
+ except Exception:
+ LOG.debug("Failed to bring new tab to front", exc_info=True)
+ # The remaining batch was planned against the old tab's scraped page; stop here so the
+ # next step re-scrapes the newly active tab.
+ result = ActionSuccess()
+ result.skip_remaining_actions = True
+ return [result]
+
+
+@traced(name="skyvern.agent.action.switch_tab")
+async def handle_switch_tab_action(
+ action: actions.SwitchTabAction,
+ page: Page,
+ scraped_page: ScrapedPage,
+ task: Task,
+ step: Step,
+) -> list[ActionResult]:
+ browser_state = app.BROWSER_MANAGER.get_for_task(task.task_id, workflow_run_id=task.workflow_run_id)
+ if browser_state is None:
+ return [ActionFailure(Exception("No browser state found for the task"), stop_execution_on_failure=False)]
+ pages = await browser_state.list_valid_pages()
+ if action.tab_index < 0 or action.tab_index >= len(pages):
+ return [
+ ActionFailure(
+ Exception(f"SWITCH_TAB tab_index {action.tab_index} is out of range (0-{len(pages) - 1})"),
+ stop_execution_on_failure=False,
+ )
+ ]
+ target_page = pages[action.tab_index]
+ await browser_state.set_active_page(target_page)
+ try:
+ await target_page.bring_to_front()
+ except Exception:
+ LOG.debug("Failed to bring switched tab to front", exc_info=True)
+ # The remaining batch was planned against the previous tab; stop so the next step
+ # re-scrapes the now-active tab.
+ result = ActionSuccess()
+ result.skip_remaining_actions = True
+ return [result]
+
+
async def handle_execute_js_action(
action: actions.ExecuteJsAction,
page: Page,
@@ -3198,6 +3263,8 @@ ActionHandler.register_action_type(ActionType.VERIFICATION_CODE, handle_verifica
ActionHandler.register_action_type(ActionType.LEFT_MOUSE, handle_left_mouse_action)
ActionHandler.register_action_type(ActionType.GOTO_URL, handle_goto_url_action)
ActionHandler.register_action_type(ActionType.CLOSE_PAGE, handle_close_page_action)
+ActionHandler.register_action_type(ActionType.NEW_TAB, handle_new_tab_action)
+ActionHandler.register_action_type(ActionType.SWITCH_TAB, handle_switch_tab_action)
ActionHandler.register_action_type(ActionType.GO_BACK, handle_go_back_action)
ActionHandler.register_action_type(ActionType.GO_FORWARD, handle_go_forward_action)
ActionHandler.register_action_type(ActionType.RELOAD_PAGE, handle_reload_page_action)
diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.py
index f1ceaee23..1363e5b73 100644
--- a/skyvern/webeye/actions/parse_actions.py
+++ b/skyvern/webeye/actions/parse_actions.py
@@ -1,6 +1,7 @@
import ast
import re
from typing import Any, Dict, Match
+from urllib.parse import urlparse
import structlog
from openai.types.responses.response import Response as OpenAIResponse
@@ -9,7 +10,12 @@ from pydantic import ValidationError
from skyvern.constants import EXTRACT_ACTION_SCROLL_AMOUNT, SCROLL_AMOUNT_MULTIPLIER
from skyvern.errors.errors import GetTOTPVerificationCodeError, MissingTOTPSourceError
-from skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFound, UnsupportedActionType
+from skyvern.exceptions import (
+ FailedToGetTOTPVerificationCode,
+ InvalidUrl,
+ NoTOTPVerificationCodeFound,
+ UnsupportedActionType,
+)
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.core import skyvern_context
@@ -23,7 +29,7 @@ from skyvern.services.otp_service import (
resolve_otp_value,
)
from skyvern.utils.image_resizer import Resolution, scale_coordinates
-from skyvern.utils.url_validators import strip_query_params
+from skyvern.utils.url_validators import is_blocked_host, prepend_scheme_and_validate_url, strip_query_params
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
Action,
@@ -42,11 +48,13 @@ from skyvern.webeye.actions.actions import (
KeypressAction,
LeftMouseAction,
MoveAction,
+ NewTabAction,
NullAction,
ScrollAction,
SelectOption,
SelectOptionAction,
SolveCaptchaAction,
+ SwitchTabAction,
TerminateAction,
UploadFileAction,
VerificationCodeAction,
@@ -248,6 +256,34 @@ def parse_action(
if action_type == ActionType.CLOSE_PAGE:
return ClosePageAction(**base_action_dict)
+ if action_type == ActionType.NEW_TAB:
+ url = action.get("url")
+ if not url or not isinstance(url, str):
+ LOG.warning("NEW_TAB action returned without a url, skipping action", raw_action=action)
+ return NullAction(**base_action_dict)
+ try:
+ validated_url = prepend_scheme_and_validate_url(url.strip())
+ except InvalidUrl:
+ LOG.warning("NEW_TAB action returned with an invalid url, skipping action", url=url)
+ return NullAction(**base_action_dict)
+ host = urlparse(validated_url).hostname
+ if not host or is_blocked_host(host):
+ LOG.warning("NEW_TAB action targets a blocked host, skipping action", url=validated_url)
+ return NullAction(**base_action_dict)
+ return NewTabAction(**base_action_dict, url=validated_url)
+
+ if action_type == ActionType.SWITCH_TAB:
+ raw_index = action.get("tab_index")
+ if raw_index is None:
+ LOG.warning("SWITCH_TAB action returned without a tab_index, skipping action", raw_action=action)
+ return NullAction(**base_action_dict)
+ try:
+ tab_index = int(raw_index)
+ except (TypeError, ValueError):
+ LOG.warning("SWITCH_TAB action returned with a non-integer tab_index, skipping action", raw_action=action)
+ return NullAction(**base_action_dict)
+ return SwitchTabAction(**base_action_dict, tab_index=tab_index)
+
raise UnsupportedActionType(action_type=action_type)
diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.py
index fe55e0b55..bda49f059 100644
--- a/skyvern/webeye/browser_state.py
+++ b/skyvern/webeye/browser_state.py
@@ -39,6 +39,8 @@ class BrowserState(Protocol):
async def set_working_page(self, page: Page | None, index: int = 0) -> None: ...
+ async def set_active_page(self, page: Page) -> None: ...
+
async def navigate_to_url(
self,
page: Page,
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index a62ecd3ea..11d2b6b38 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -59,6 +59,13 @@ class RealBrowserState(BrowserState):
browser_cleanup: BrowserCleanupFunc = None,
):
self.__page = page
+ # An explicitly selected tab (set by NEW_TAB/SWITCH_TAB). When set, it overrides the
+ # last-page default in get_working_page so multi-tab targeting is deterministic.
+ self.__active_page: Page | None = None
+ # Snapshot of the valid pages present when the active tab was pinned. If a page appears
+ # that was not in this set, a new tab opened and auto-takes focus (legacy behavior),
+ # so the pin is dropped.
+ self.__active_page_known_pages: set[Page] = set()
self.pw = pw
self.browser_context = browser_context
self.browser_artifacts = browser_artifacts
@@ -170,18 +177,31 @@ class RealBrowserState(BrowserState):
await app.AGENT_FUNCTION.wait_for_challenge_solver(page=page)
async def get_working_page(self) -> Page | None:
- # HACK: currently, assuming the last page is always the working page.
- # Need to refactor this logic when we want to manipulate multi pages together
- # TODO: do not use index of pages, it should be more robust if we want to fully support multi pages manipulation
if self.__page is None or self.browser_context is None:
return None
- # pick the last and http/https page as the working page
pages = await self.list_valid_pages()
if len(pages) == 0:
LOG.info("No http, https or blank page found in the browser context, return None")
return None
+ # Honor a tab explicitly selected via NEW_TAB/SWITCH_TAB while it is still open and no
+ # new tab has appeared since selection. A newly-opened tab (any page not in the snapshot)
+ # auto-takes focus, preserving the legacy last-page behavior; closing an unrelated tab
+ # does not drop the pin.
+ active_page = self.__active_page
+ if (
+ active_page is not None
+ and not active_page.is_closed()
+ and active_page in pages
+ and all(page in self.__active_page_known_pages for page in pages)
+ ):
+ self.__page = active_page
+ return active_page
+
+ # No (or stale) pin: fall back to the last http/https page as the working page.
+ self.__active_page = None
+ self.__active_page_known_pages = set()
last_page = pages[-1]
if self.__page == last_page:
return self.__page
@@ -257,6 +277,14 @@ class RealBrowserState(BrowserState):
async def set_working_page(self, page: Page | None, index: int = 0) -> None:
self.__page = page
+ if page is None:
+ self.__active_page = None
+ self.__active_page_known_pages = set()
+
+ async def set_active_page(self, page: Page) -> None:
+ self.__active_page = page
+ self.__page = page
+ self.__active_page_known_pages = set(await self.list_valid_pages())
async def get_or_create_page(
self,
diff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.py
index 81aa1f062..bba896595 100644
--- a/tests/unit/test_actions.py
+++ b/tests/unit/test_actions.py
@@ -7,8 +7,10 @@ from skyvern.webeye.actions.actions import (
Action,
ClickAction,
KeypressAction,
+ NewTabAction,
NullAction,
SelectOptionAction,
+ SwitchTabAction,
WebAction,
)
from skyvern.webeye.actions.parse_actions import parse_action
@@ -225,3 +227,81 @@ def test_parse_click_download_field(download_value: bool | None) -> None:
assert isinstance(action, ClickAction)
expected = download_value if download_value is not None else False
assert action.download is expected
+
+
+def test_parse_new_tab_action_with_url() -> None:
+ action = parse_action(
+ action={"action_type": "NEW_TAB", "url": "https://example.test/page", "reasoning": "open a separate tab"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, NewTabAction)
+ assert action.url == "https://example.test/page"
+ assert action.element_id is None
+
+
+def test_parse_new_tab_action_prepends_scheme() -> None:
+ action = parse_action(
+ action={"action_type": "NEW_TAB", "url": "example.test/page", "reasoning": "test"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, NewTabAction)
+ assert action.url == "https://example.test/page"
+
+
+def test_parse_new_tab_action_missing_url_returns_null() -> None:
+ action = parse_action(
+ action={"action_type": "NEW_TAB", "reasoning": "test"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, NullAction)
+
+
+def test_parse_new_tab_action_blocked_host_returns_null() -> None:
+ action = parse_action(
+ action={"action_type": "NEW_TAB", "url": "http://localhost:8000/admin", "reasoning": "test"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, NullAction)
+
+
+def test_parse_switch_tab_action_valid_index() -> None:
+ action = parse_action(
+ action={"action_type": "SWITCH_TAB", "tab_index": 1, "reasoning": "go back to first tab"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, SwitchTabAction)
+ assert action.tab_index == 1
+ assert action.element_id is None
+
+
+def test_parse_switch_tab_action_coerces_string_index() -> None:
+ action = parse_action(
+ action={"action_type": "SWITCH_TAB", "tab_index": "2", "reasoning": "test"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, SwitchTabAction)
+ assert action.tab_index == 2
+
+
+def test_parse_switch_tab_action_missing_index_returns_null() -> None:
+ action = parse_action(
+ action={"action_type": "SWITCH_TAB", "reasoning": "test"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, NullAction)
+
+
+def test_parse_switch_tab_action_non_integer_index_returns_null() -> None:
+ action = parse_action(
+ action={"action_type": "SWITCH_TAB", "tab_index": "not-a-number", "reasoning": "test"},
+ scraped_page=_mock_scraped_page(),
+ )
+ assert isinstance(action, NullAction)
+
+
+def test_tab_actions_registered_for_db_hydration() -> None:
+ from skyvern.forge.sdk.db.utils import ACTION_TYPE_TO_CLASS
+ from skyvern.webeye.actions.action_types import ActionType
+
+ assert ACTION_TYPE_TO_CLASS[ActionType.NEW_TAB] is NewTabAction
+ assert ACTION_TYPE_TO_CLASS[ActionType.SWITCH_TAB] is SwitchTabAction
diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.py
index f6e159259..188cb8a43 100644
--- a/tests/unit/test_copilot_blocker_signal.py
+++ b/tests/unit/test_copilot_blocker_signal.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+from types import SimpleNamespace
+
import pytest
from skyvern.forge.sdk.copilot.blocker_signal import (
@@ -8,11 +10,14 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
build_llm_tool_error_payload,
+ build_loop_blocker_signal,
clear_blocker_signal_for_reason_codes,
maybe_clear_blocker_signal_on_tool_success,
+ refresh_held_loop_blocker_evidence,
stash_blocker_signal,
to_trace_data,
)
+from skyvern.forge.sdk.copilot.context import CopilotContext
def _make(
@@ -315,3 +320,84 @@ def test_agent_context_and_copilot_context_blocker_signal_defaults_match() -> No
assert agent_ctx.latest_tool_blocker_signal == copilot_ctx.latest_tool_blocker_signal
assert agent_ctx.tool_blocker_signals == []
assert copilot_ctx.tool_blocker_signals == []
+
+
+def _copilot_ctx() -> CopilotContext:
+ return CopilotContext(
+ organization_id="org",
+ workflow_id="wf",
+ workflow_permanent_id="wfp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+
+
+_CONSECUTIVE_LOOP_MESSAGE = "LOOP DETECTED: 'evaluate' has been called 3 times consecutively."
+_LATE_RECORDED_REASON = (
+ "Failed: The run completed but did not demonstrate the goal outcome(s): the requested record is checked "
+ "on a public registry site with a search form and expandable result rows. "
+ "Add an end-state confirmation (an extraction or validation block) that observes the outcome, then re-run."
+)
+
+
+def test_stash_refreshes_held_loop_signal_with_evidence_recorded_after_the_stash() -> None:
+ ctx = _copilot_ctx()
+ loop_signal = build_loop_blocker_signal(_CONSECUTIVE_LOOP_MESSAGE, tool_name="evaluate")
+ stash_blocker_signal(ctx, loop_signal)
+ assert ctx.blocker_signal is loop_signal
+ assert dict(loop_signal.extra) == {}
+
+ ctx.last_outcome_gate_reason = _LATE_RECORDED_REASON
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ ctx.has_staged_proposal = True
+
+ late_error = _make(kind="tool_error", internal_reason_code="tool_error_late_block_running")
+ payload = stash_blocker_signal(ctx, late_error)
+ assert payload == late_error.agent_steering_text
+ assert ctx.latest_tool_blocker_signal is late_error
+
+ held = ctx.blocker_signal
+ assert isinstance(held, CopilotToolBlockerSignal)
+ assert held is not loop_signal
+ assert held.internal_reason_code == "loop_detected_consecutive_same_tool"
+ assert held.agent_steering_text == loop_signal.agent_steering_text
+ assert held.recovery_hint == loop_signal.recovery_hint
+ assert held.blocked_tool == loop_signal.blocked_tool
+ assert held.cleared_by_tools == loop_signal.cleared_by_tools
+ assert "did not demonstrate the goal outcome" in held.user_facing_reason
+ assert "Add an end-state confirmation" not in held.user_facing_reason
+ assert "verification challenge" in held.user_facing_reason
+ assert held.preserves_workflow_draft is True
+ assert dict(held.extra) == {"loop_evidence_tiers": ["verdict", "anti_bot", "draft"]}
+
+
+def test_stash_keeps_non_loop_held_signal_unrefreshed() -> None:
+ ctx = _copilot_ctx()
+ held = _make()
+ stash_blocker_signal(ctx, held)
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ stash_blocker_signal(ctx, _make(internal_reason_code="second"))
+ assert ctx.blocker_signal is held
+
+
+def test_refresh_held_loop_blocker_evidence_is_idempotent() -> None:
+ ctx = _copilot_ctx()
+ ctx.blocker_signal = build_loop_blocker_signal(_CONSECUTIVE_LOOP_MESSAGE, tool_name="evaluate")
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+
+ refresh_held_loop_blocker_evidence(ctx)
+ refreshed = ctx.blocker_signal
+ assert isinstance(refreshed, CopilotToolBlockerSignal)
+ assert "verification challenge" in refreshed.user_facing_reason
+ assert dict(refreshed.extra) == {"loop_evidence_tiers": ["anti_bot"]}
+
+ refresh_held_loop_blocker_evidence(ctx)
+ assert ctx.blocker_signal is refreshed
+
+
+def test_refresh_with_no_held_signal_is_a_no_op() -> None:
+ ctx = _copilot_ctx()
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ refresh_held_loop_blocker_evidence(ctx)
+ assert ctx.blocker_signal is None
diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.py
index a10d9de81..c2595d8c9 100644
--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py
+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py
@@ -8,7 +8,13 @@ from __future__ import annotations
import re
import textwrap
+from skyvern.forge.sdk.copilot.outcome_verification_trace import (
+ finalize_outcome_verification_trace,
+ record_code_artifact_violations,
+)
+from skyvern.forge.sdk.copilot.output_utils import _sanitize_failure_text
from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
+from skyvern.forge.sdk.copilot.tools.workflow_update import _normalize_code_artifact_metadata_detailed
def _code_block_yaml(label: str) -> str:
@@ -426,3 +432,96 @@ class TestStaleLabelRekey:
normalized, error = _normalize_code_artifact_metadata([first, second], _code_block_yaml("my_block"))
assert error is None
assert normalized["my_block"]["declared_goal"] == "g"
+
+
+def _two_code_block_yaml(first: str, second: str) -> str:
+ return textwrap.dedent(
+ f"""
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {first}
+ code: |
+ await page.goto("https://example.com/")
+ - block_type: code
+ label: {second}
+ code: |
+ await page.goto("https://example.com/")
+ """
+ ).strip()
+
+
+class _FakeSpan:
+ def __init__(self) -> None:
+ self.attrs: dict = {}
+
+ def set_attributes(self, fields: dict) -> None:
+ self.attrs.update(fields)
+
+
+def _record_and_flush(violations: list[str], offending_labels: list[str]) -> dict:
+ ctx = type("Ctx", (), {})()
+ record_code_artifact_violations(ctx, violations, offending_labels)
+ span = _FakeSpan()
+ finalize_outcome_verification_trace(ctx, span)
+ return span.attrs
+
+
+class TestViolationBatchIsDurablyRecoverable:
+ def test_full_batch_recoverable_from_span_even_with_credential_labels(self) -> None:
+ yaml = _two_code_block_yaml("credential_login", "credential_vault")
+ result = _normalize_code_artifact_metadata_detailed(
+ [_broken_metadata("credential_login"), _broken_metadata("credential_vault")], yaml
+ )
+ assert result.error is not None
+ attrs = _record_and_flush(result.violations, result.offending_labels)
+
+ assert attrs["copilot.code_artifact_violations"] == result.violations
+ assert attrs["copilot.code_artifact_violation_count"] == len(result.violations)
+ assert attrs["copilot.code_artifact_violation_block_labels"] == ["credential_login", "credential_vault"]
+ # Every numbered line from the batched error survives as its own element.
+ numbered = [line.split(". ", 1)[1] for line in result.error.splitlines() if re.match(r"^\d+\.", line)]
+ assert numbered == result.violations
+
+ def test_malformed_only_batch_records_count_without_labels_or_values(self) -> None:
+ secret = "SUPER_SECRET_VALUE_12345"
+ result = _normalize_code_artifact_metadata_detailed(
+ [{"block_label": "credential_x", "claimed_outcomes": secret}], _code_block_yaml("credential_x")
+ )
+ assert result.error is not None
+ assert result.offending_labels == []
+ assert all(secret not in violation for violation in result.violations)
+ attrs = _record_and_flush(result.violations, result.offending_labels)
+ assert attrs["copilot.code_artifact_violation_count"] == len(result.violations)
+ assert attrs["copilot.code_artifact_violation_block_labels"] == []
+ assert all(secret not in violation for violation in attrs["copilot.code_artifact_violations"])
+
+ def test_span_keeps_violations_the_backend_log_summary_truncates_away(self) -> None:
+ yaml = _two_code_block_yaml("credential_login", "credential_vault")
+ result = _normalize_code_artifact_metadata_detailed(
+ [_broken_metadata("credential_login"), _broken_metadata("credential_vault")], yaml
+ )
+ bounded = _sanitize_failure_text(result.error)
+ assert len(bounded) <= 120
+ assert len(result.violations) > 1
+ attrs = _record_and_flush(result.violations, result.offending_labels)
+ # The bounded summary loses all but the first violation; the span keeps them all.
+ assert attrs["copilot.code_artifact_violations"][-1] not in bounded
+ assert len(attrs["copilot.code_artifact_violations"]) == len(result.violations)
+
+ def test_empty_batch_is_a_noop(self) -> None:
+ ctx = type("Ctx", (), {})()
+ record_code_artifact_violations(ctx, [], [])
+ span = _FakeSpan()
+ finalize_outcome_verification_trace(ctx, span)
+ assert "copilot.code_artifact_violations" not in span.attrs
+
+ def test_latest_batch_wins_on_retry(self) -> None:
+ ctx = type("Ctx", (), {})()
+ record_code_artifact_violations(ctx, ["v1", "v2", "v3"], ["a"])
+ record_code_artifact_violations(ctx, ["only_one"], ["b"])
+ span = _FakeSpan()
+ finalize_outcome_verification_trace(ctx, span)
+ assert span.attrs["copilot.code_artifact_violations"] == ["only_one"]
+ assert span.attrs["copilot.code_artifact_violation_count"] == 1
+ assert span.attrs["copilot.code_artifact_violation_block_labels"] == ["b"]
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index a99f89d2e..6e771b474 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -479,6 +479,47 @@ def test_record_run_blocks_downgrades_on_contradiction_without_confirmation_bloc
assert ctx.last_full_workflow_test_ok is False
+def _failed_code_block_result() -> dict:
+ raw = (
+ "code block failed. failure reason: Failed to execute code block. Reason: TimeoutError: "
+ "Timeout 30000ms exceeded. =========================== logs =========================== "
+ '"load" event fired ============================================================'
+ )
+ return {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_x",
+ "overall_status": "failed",
+ "executed_block_labels": ["b0"],
+ "blocks": [{"label": "b0", "block_type": "code", "status": "failed", "failure_reason": raw}],
+ },
+ }
+
+
+def test_failed_run_records_gate_reason_separately_from_raw_block_failure() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ _record_run_blocks_result(ctx, _failed_code_block_result(), completion_verification=_evaluated(("c0", False)))
+ assert "item in cart" in (ctx.last_outcome_gate_reason or "")
+ assert "TimeoutError" not in (ctx.last_outcome_gate_reason or "")
+ assert "TimeoutError" in (ctx.last_test_failure_reason or "")
+
+
+def test_gate_reason_survives_a_later_run_without_verification() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
+ assert "item in cart" in (ctx.last_outcome_gate_reason or "")
+ _record_run_blocks_result(ctx, _failed_code_block_result(), completion_verification=None)
+ assert "item in cart" in (ctx.last_outcome_gate_reason or "")
+
+
+def test_gate_reason_cleared_when_outcome_verified() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
+ assert ctx.last_outcome_gate_reason is not None
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
+ assert ctx.last_outcome_gate_reason is None
+
+
def test_record_run_blocks_keeps_success_when_outcome_verified() -> None:
ctx = _ctx_with_blocks("extraction")
_record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index f80fcedb6..110c62314 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -4,9 +4,20 @@ from types import SimpleNamespace
import pytest
-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.agent import _ensure_unvalidated_proposal_affordance
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ _INTERNAL_TOOL_NAME_TOKENS,
+ _LOOP_PROGRESS_TOOLS,
+ CopilotToolBlockerSignal,
+ LoopBlockerEvidence,
+ assert_clean_user_facing_text,
+ loop_blocker_evidence_from_ctx,
+)
from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer, _stash_and_emit_loop_blocker
+from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy
from skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_error
+from skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map
_LEAK_TOKENS = ("safe_reason_code", "LOOP DETECTED:")
@@ -135,3 +146,311 @@ def test_native_and_mcp_paths_produce_equivalent_loop_signal() -> None:
assert mcp_ctx.blocker_signal.recovery_hint == native_signal.recovery_hint
assert mcp_ctx.blocker_signal.cleared_by_tools == native_signal.cleared_by_tools
assert mcp_ctx.blocker_signal.blocked_tool == native_signal.blocked_tool
+
+
+_FULL_EVIDENCE_REASON = (
+ "Failed: The run completed but did not demonstrate the goal outcome(s): The credential information for the "
+ "requested person is checked on a public registry site with a search form and expandable result rows. "
+ "Add an end-state confirmation (an extraction or validation block) that observes the outcome, then re-run."
+)
+_FULL_EVIDENCE = LoopBlockerEvidence(
+ outcome_gate_reason=_FULL_EVIDENCE_REASON,
+ anti_bot_blocked=True,
+ has_draft=True,
+)
+_BRANCH_MESSAGES = {
+ "loop_detected_credential_or_parameter_misconfig": (
+ "LOOP DETECTED: 'update_and_run_blocks' has already failed 3 times with CREDENTIAL_ERROR; blocking attempt #4."
+ ),
+ "loop_detected_repeated_failed_step": (
+ "LOOP DETECTED: 'update_workflow' has already failed 3 consecutive times with these arguments; "
+ "blocking attempt #4."
+ ),
+ "loop_detected_consecutive_same_tool": "LOOP DETECTED: 'evaluate' has been called 3 times consecutively.",
+ "loop_detected_generic": "something unfamiliar happened",
+}
+_BRANCH_TEMPLATES = {
+ "loop_detected_credential_or_parameter_misconfig": (
+ "I couldn't run this with the current credential or parameter setup. Update them and ask me to try again."
+ ),
+ "loop_detected_repeated_failed_step": (
+ "I retried without making progress. Tell me what to change and I'll try a different approach."
+ ),
+ "loop_detected_consecutive_same_tool": (
+ "I'm stuck retrying the same step. Tell me what to change and I'll try a different approach."
+ ),
+ "loop_detected_generic": "I couldn't keep going on this turn. Tell me what to change and I'll try again.",
+}
+_FULL_TIER_BRANCHES = (
+ "loop_detected_repeated_failed_step",
+ "loop_detected_consecutive_same_tool",
+ "loop_detected_generic",
+)
+
+
[email protected]("reason_code", _FULL_TIER_BRANCHES)
+def test_full_evidence_composition_names_verdict_blocker_and_draft(reason_code: str) -> None:
+ signal = _build_loop_blocker_signal(_BRANCH_MESSAGES[reason_code], tool_name="evaluate", evidence=_FULL_EVIDENCE)
+ assert signal.internal_reason_code == reason_code
+ assert "did not demonstrate the goal outcome" in signal.user_facing_reason
+ assert "verification challenge" in signal.user_facing_reason
+ assert "Add an end-state confirmation" not in signal.user_facing_reason
+ assert "Failed:" not in signal.user_facing_reason
+ assert signal.preserves_workflow_draft is True
+ assert dict(signal.extra) == {"loop_evidence_tiers": ["verdict", "anti_bot", "draft"]}
+ assert signal.agent_steering_text == _BRANCH_MESSAGES[reason_code]
+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")
+
+
[email protected]("reason_code", sorted(_BRANCH_MESSAGES))
[email protected]("evidence", [None, LoopBlockerEvidence()], ids=["none", "all_empty"])
+def test_no_recorded_state_keeps_templates_byte_identical(
+ reason_code: str, evidence: LoopBlockerEvidence | None
+) -> None:
+ message = _BRANCH_MESSAGES[reason_code]
+ baseline = _build_loop_blocker_signal(message, tool_name="update_and_run_blocks")
+ signal = _build_loop_blocker_signal(message, tool_name="update_and_run_blocks", evidence=evidence)
+ assert signal.user_facing_reason == _BRANCH_TEMPLATES[reason_code]
+ assert signal.user_facing_reason == baseline.user_facing_reason
+ assert signal.internal_reason_code == reason_code
+ assert signal.recovery_hint == baseline.recovery_hint
+ assert signal.agent_steering_text == message
+ assert signal.cleared_by_tools == frozenset()
+ assert signal.preserves_workflow_draft is False
+ assert dict(signal.extra) == {}
+
+
+def test_credential_branch_keeps_copy_and_gains_only_the_draft_flag() -> None:
+ signal = _build_loop_blocker_signal(
+ _BRANCH_MESSAGES["loop_detected_credential_or_parameter_misconfig"],
+ tool_name="update_and_run_blocks",
+ evidence=_FULL_EVIDENCE,
+ )
+ assert signal.user_facing_reason == _BRANCH_TEMPLATES["loop_detected_credential_or_parameter_misconfig"]
+ assert signal.recovery_hint == "ask_user_clarifying"
+ assert signal.preserves_workflow_draft is True
+ assert dict(signal.extra) == {"loop_evidence_tiers": ["draft"]}
+
+
+ "adversarial_reason",
+ [
+ "Run wr_123456789012345678 did not finish; outcome unknown.",
+ "update_and_run_blocks exhausted its retries against the search form.",
+ "do not retry this step; the form stayed blocked.",
+ "The run exceeded the per-tool-call budget while the search stayed disabled.",
+ ],
+)
+def test_adversarial_recorded_reason_drops_free_text_tier_without_raising(adversarial_reason: str) -> None:
+ evidence = LoopBlockerEvidence(outcome_gate_reason=adversarial_reason, anti_bot_blocked=True, has_draft=False)
+ signal = _build_loop_blocker_signal(
+ _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate", evidence=evidence
+ )
+ lowered = signal.user_facing_reason.lower()
+ for token in ("wr_", "update_and_run_blocks", "do not retry", "per-tool-call budget"):
+ assert token not in lowered
+ assert "verification challenge" in signal.user_facing_reason
+ assert dict(signal.extra) == {"loop_evidence_tiers": ["anti_bot"]}
+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")
+
+
+ "raw_error_reason",
+ [
+ (
+ "Failed to execute code block. Reason: TimeoutError: Timeout 30000ms exceeded. "
+ '=========================== logs =========================== "load" event fired '
+ "============================================================"
+ ),
+ "code block failed. failure reason: Failed to execute code block.",
+ "Traceback (most recent call last): ValueError: bad input",
+ "ElementNotFoundException: selector did not resolve",
+ ],
+)
+def test_raw_runtime_error_reason_drops_verdict_tier(raw_error_reason: str) -> None:
+ evidence = LoopBlockerEvidence(outcome_gate_reason=raw_error_reason, anti_bot_blocked=True, has_draft=True)
+ signal = _build_loop_blocker_signal(
+ _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate", evidence=evidence
+ )
+ for fragment in ("TimeoutError", "Failed to execute", "===", "Traceback", "Exception"):
+ assert fragment not in signal.user_facing_reason
+ assert "verification challenge" in signal.user_facing_reason
+ assert signal.preserves_workflow_draft is True
+ assert dict(signal.extra) == {"loop_evidence_tiers": ["anti_bot", "draft"]}
+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")
+
+
+def test_evidence_verdict_sources_only_from_the_outcome_gate_field() -> None:
+ ctx = _ctx()
+ ctx.last_test_failure_reason = "Failed to execute code block. Reason: TimeoutError: Timeout 30000ms exceeded."
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ assert loop_blocker_evidence_from_ctx(ctx).outcome_gate_reason is None
+
+ gate_reason = "The run completed but did not demonstrate the goal outcome(s): the requested record is checked."
+ ctx.last_outcome_gate_reason = gate_reason
+ assert loop_blocker_evidence_from_ctx(ctx).outcome_gate_reason == gate_reason
+
+
+def test_blocked_tool_named_by_common_word_collides_safely() -> None:
+ evidence = LoopBlockerEvidence(
+ outcome_gate_reason=(
+ "The run completed but did not demonstrate the goal outcome(s): each result row is expanded with a click."
+ ),
+ anti_bot_blocked=True,
+ has_draft=True,
+ )
+ signal = _build_loop_blocker_signal(
+ "LOOP DETECTED: 'click' has been called 3 times consecutively.", tool_name="click", evidence=evidence
+ )
+ assert "click" not in signal.user_facing_reason.lower()
+ assert "verification challenge" in signal.user_facing_reason
+ assert signal.preserves_workflow_draft is True
+ assert dict(signal.extra) == {"loop_evidence_tiers": ["anti_bot", "draft"]}
+
+
+_LOOP_PRONE_TOOL_NAMES = sorted(
+ set(_LOOP_PROGRESS_TOOLS) | set(_INTERNAL_TOOL_NAME_TOKENS) | set(get_skyvern_mcp_alias_map())
+)
+
+
[email protected]("reason_code", sorted(_BRANCH_MESSAGES))
[email protected]("blocked_tool", _LOOP_PRONE_TOOL_NAMES)
+def test_fixed_tier_copy_is_clean_for_every_loop_prone_tool(blocked_tool: str, reason_code: str) -> None:
+ evidence = LoopBlockerEvidence(anti_bot_blocked=True, has_draft=True)
+ signal = _build_loop_blocker_signal(_BRANCH_MESSAGES[reason_code], tool_name=blocked_tool, evidence=evidence)
+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool=blocked_tool)
+
+
+def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> None:
+ def _prepped_ctx() -> CopilotContext:
+ ctx = _ctx(consecutive_tool_tracker=["list_credentials", "list_credentials"])
+ ctx.last_outcome_gate_reason = (
+ "The run completed but did not demonstrate the goal outcome(s): the requested record is checked "
+ "on a public registry site with a search form and expandable result rows."
+ )
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ ctx.has_staged_proposal = True
+ return ctx
+
+ native_ctx = _prepped_ctx()
+ native_payload = _tool_loop_error(native_ctx, "list_credentials", None)
+ assert native_payload is not None
+ native_signal = native_ctx.blocker_signal
+ assert isinstance(native_signal, CopilotToolBlockerSignal)
+
+ mcp_ctx = _prepped_ctx()
+ mcp_payload = _stash_and_emit_loop_blocker(mcp_ctx, native_signal.agent_steering_text, "list_credentials")
+ mcp_signal = mcp_ctx.blocker_signal
+ assert isinstance(mcp_signal, CopilotToolBlockerSignal)
+
+ assert mcp_payload == native_payload
+ assert mcp_signal.user_facing_reason == native_signal.user_facing_reason
+ assert "did not demonstrate the goal outcome" in mcp_signal.user_facing_reason
+ assert "verification challenge" in mcp_signal.user_facing_reason
+ assert native_signal.preserves_workflow_draft is True
+ assert mcp_signal.preserves_workflow_draft is True
+ assert dict(native_signal.extra) == {"loop_evidence_tiers": ["verdict", "anti_bot", "draft"]}
+ assert dict(mcp_signal.extra) == dict(native_signal.extra)
+ assert mcp_signal.internal_reason_code == native_signal.internal_reason_code
+ assert mcp_signal.agent_steering_text == native_signal.agent_steering_text
+ assert mcp_signal.recovery_hint == native_signal.recovery_hint
+ assert mcp_signal.blocked_tool == native_signal.blocked_tool
+
+
+def test_tool_loop_error_entry_refreshes_stale_held_loop_signal() -> None:
+ ctx = _ctx()
+ stale = _build_loop_blocker_signal(_BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate")
+ ctx.blocker_signal = stale
+ ctx.last_outcome_gate_reason = (
+ "The run completed but did not demonstrate the goal outcome(s): the requested record is checked."
+ )
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ ctx.staged_workflow_yaml = "blocks: []"
+
+ assert _tool_loop_error(ctx, "list_credentials", None) is None
+
+ refreshed = ctx.blocker_signal
+ assert isinstance(refreshed, CopilotToolBlockerSignal)
+ assert refreshed is not stale
+ assert "did not demonstrate the goal outcome" in refreshed.user_facing_reason
+ assert "verification challenge" in refreshed.user_facing_reason
+ assert refreshed.preserves_workflow_draft is True
+ assert refreshed.agent_steering_text == stale.agent_steering_text
+ assert refreshed.internal_reason_code == stale.internal_reason_code
+ assert refreshed.blocked_tool == stale.blocked_tool
+
+
+async def test_mcp_call_tool_entry_refreshes_stale_held_loop_signal() -> None:
+ ctx = _ctx()
+ stale = _build_loop_blocker_signal(_BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate")
+ ctx.blocker_signal = stale
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ ctx.has_staged_proposal = True
+
+ class _StubResult:
+ structured_content = {"ok": False, "error": "page state unchanged"}
+ is_error = False
+ content: list = []
+
+ class _StubClient:
+ async def call_tool(self, name: str, args: dict, raise_on_error: bool = False) -> _StubResult:
+ return _StubResult()
+
+ server = SkyvernOverlayMCPServer(
+ transport=None,
+ overlays={},
+ alias_map={},
+ allowlist=frozenset({"evaluate"}),
+ context_provider=lambda: ctx,
+ )
+ server._client = _StubClient() # type: ignore[assignment]
+
+ await server.call_tool("evaluate", {})
+
+ refreshed = ctx.blocker_signal
+ assert isinstance(refreshed, CopilotToolBlockerSignal)
+ assert refreshed is not stale
+ assert "verification challenge" in refreshed.user_facing_reason
+ assert refreshed.preserves_workflow_draft is True
+
+
+def test_composed_loop_reply_passes_output_policy_allow_verdict() -> None:
+ signal = _build_loop_blocker_signal(
+ _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"],
+ tool_name="evaluate",
+ evidence=LoopBlockerEvidence(outcome_gate_reason=_FULL_EVIDENCE_REASON, anti_bot_blocked=True, has_draft=False),
+ )
+ verdict = evaluate_output_policy(
+ request_policy=None,
+ response_type="REPLY",
+ user_response=signal.user_facing_reason,
+ global_llm_context=None,
+ workflow_yaml=None,
+ has_workflow_proposal=False,
+ workflow_was_persisted=False,
+ workflow_attempted=False,
+ unvalidated=False,
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ )
+ assert verdict.allowed, [code.value for code in verdict.reason_codes]
+
+
+def test_composed_loop_reply_with_draft_affordance_passes_output_policy() -> None:
+ signal = _build_loop_blocker_signal(
+ _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate", evidence=_FULL_EVIDENCE
+ )
+ reply = _ensure_unvalidated_proposal_affordance(signal.user_facing_reason)
+ verdict = evaluate_output_policy(
+ request_policy=None,
+ response_type="REPLY",
+ user_response=reply,
+ global_llm_context=None,
+ workflow_yaml="title: Example workflow\nblocks: []",
+ has_workflow_proposal=True,
+ workflow_was_persisted=False,
+ workflow_attempted=False,
+ unvalidated=True,
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ )
+ assert verdict.allowed, [code.value for code in verdict.reason_codes]
diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.py
new file mode 100644
index 000000000..7681ee269
--- /dev/null
+++ b/tests/unit/test_copilot_make_agent_result_backfill.py
@@ -0,0 +1,121 @@
+"""`_make_agent_result` back-fills the typed terminal adjudication onto the
+narrative payload: ``responseKind`` from ``TurnOutcome.response_kind`` and
+``verifiedSuccess`` from ``enforcement.verified_goal_satisfied_context``."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from skyvern.forge.sdk.copilot.agent import _make_agent_result
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisInput,
+ DiagnosisRepairContract,
+ DiagnosisResult,
+ RepairDecision,
+ RepairNextAction,
+ VerificationResult,
+)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
+
+
+def _ctx(**overrides: object) -> CopilotContext:
+ defaults: dict = dict(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wfp-1",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=MagicMock(),
+ )
+ defaults.update(overrides)
+ return CopilotContext(**defaults)
+
+
+def _verified_goal_ctx() -> CopilotContext:
+ return _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
+ diagnosis_result=DiagnosisResult(),
+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),
+ verification_result=VerificationResult(
+ user_goal_satisfied=True,
+ completion_contract_satisfied=True,
+ ),
+ ),
+ )
+
+
+def _outcome(kind: ResponseKind) -> TurnOutcome:
+ return TurnOutcome(response_kind=kind)
+
+
+def _payload(**overrides: object) -> dict:
+ base: dict = {
+ "turnId": "turn-1",
+ "turnIndex": 0,
+ "mode": "build",
+ "designStarted": True,
+ "designEnded": True,
+ "draft": None,
+ "blocks": [],
+ "terminal": "response",
+ "terminalMessage": "done",
+ "narrativeSummary": None,
+ "priorBlockCount": None,
+ "designActivity": [],
+ "startedAt": None,
+ "endedAt": None,
+ }
+ base.update(overrides)
+ return base
+
+
+def _result(ctx: CopilotContext | None, **kwargs: object):
+ kwargs.setdefault("user_response", "ok")
+ kwargs.setdefault("updated_workflow", None)
+ kwargs.setdefault("global_llm_context", None)
+ return _make_agent_result(ctx, **kwargs)
+
+
+def test_backfill_writes_both_fields_together() -> None:
+ result = _result(_ctx(), turn_outcome=_outcome(ResponseKind.CLARIFY), narrative_payload=_payload())
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["responseKind"] == "clarify"
+ assert result.narrative_payload["verifiedSuccess"] is False
+
+
+def test_backfill_verified_success_true_when_gate_passes() -> None:
+ result = _result(_verified_goal_ctx(), turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=_payload())
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["responseKind"] == "build"
+ assert result.narrative_payload["verifiedSuccess"] is True
+
+
+def test_backfill_never_overwrites_explicit_values() -> None:
+ payload = _payload(responseKind="refuse", verifiedSuccess=True)
+ result = _result(_ctx(), turn_outcome=_outcome(ResponseKind.CLARIFY), narrative_payload=payload)
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["responseKind"] == "refuse"
+ assert result.narrative_payload["verifiedSuccess"] is True
+
+
+def test_backfill_tolerates_turn_outcome_none() -> None:
+ result = _result(_ctx(), turn_outcome=None, narrative_payload=_payload())
+ assert result.narrative_payload is not None
+ assert "responseKind" not in result.narrative_payload
+ assert result.narrative_payload["verifiedSuccess"] is False
+
+
+def test_backfill_tolerates_ctx_none() -> None:
+ result = _result(None, turn_outcome=_outcome(ResponseKind.REFUSE), narrative_payload=_payload())
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["responseKind"] == "refuse"
+ assert "verifiedSuccess" not in result.narrative_payload
+
+
+def test_backfill_tolerates_missing_payload() -> None:
+ result = _result(_ctx(), turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)
+ assert result.narrative_payload is None
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
new file mode 100644
index 000000000..80be36fbf
--- /dev/null
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -0,0 +1,332 @@
+"""Deterministic content gate over completed code-block run outputs.
+
+Fixtures model a public registry site with a search form and expandable
+result rows; domains and person names are generic placeholders.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.tools import (
+ _analyze_run_blocks,
+ _current_workflow_has_evidence_block,
+ _is_outcome_evidence_candidate,
+ _is_unfinished_run_verification_candidate,
+ _record_run_blocks_result,
+ _run_blocks_structured_blocker_message,
+)
+
+
+def _code_block(label: str, extracted: Any, *, block_type: str = "CODE") -> dict[str, Any]:
+ return {"label": label, "block_type": block_type, "status": "completed", "extracted_data": extracted}
+
+
+def _run_result(blocks: list[dict[str, Any]], *, ok: bool = True) -> dict[str, Any]:
+ return {
+ "ok": ok,
+ "data": {
+ "workflow_run_id": "wr_test",
+ "overall_status": "completed" if ok else "failed",
+ "current_url": "https://registry.example.com/search",
+ "blocks": blocks,
+ },
+ }
+
+
+def _ctx(blocks: list[dict[str, Any]] | None = None) -> CopilotContext:
+ ctx = CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="blocks: []",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ user_message="search the public registry for a person and expand their result rows",
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[CompletionCriterion(id="c0", outcome="result rows extracted")]
+ )
+ labels = [block["label"] for block in (blocks or [])]
+ workflow_blocks = [SimpleNamespace(block_type="code", label=label) for label in labels]
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=workflow_blocks)) # type: ignore[assignment]
+ ctx.last_workflow_yaml = "blocks: []"
+ ctx.verified_prefix_labels = labels
+ return ctx
+
+
+def _no_evidence(cid: str) -> CompletionVerificationResult:
+ verdict = CriterionVerdict(criterion_id=cid, satisfied=False, reason_code="no_evidence")
+ return CompletionVerificationResult(status="evaluated", criterion_ids=[cid], verdicts=[verdict])
+
+
+def _blocked_flag_run_result() -> dict[str, Any]:
+ return _run_result(
+ [
+ _code_block("open_registry_search", {"submit_button_enabled": False}),
+ _code_block(
+ "search_registry_person",
+ {
+ "anti_bot_blocked": True,
+ "blocker": "The search form is gated by a human verification challenge; the search never ran.",
+ "has_results": False,
+ "records": [],
+ },
+ ),
+ ]
+ )
+
+
+def _blocked_status_run_result(block_type: str = "CODE") -> dict[str, Any]:
+ return _run_result(
+ [
+ _code_block(
+ "search_registry_person",
+ {"status": "blocked_by_challenge", "records": []},
+ block_type=block_type,
+ )
+ ]
+ )
+
+
+def _genuine_success_run_result() -> dict[str, Any]:
+ return _run_result(
+ [
+ _code_block(
+ "search_registry_person",
+ {"result_row_count": 1, "visible_results_evidence": "DOE, JANE - Status: Active"},
+ ),
+ _code_block(
+ "expand_result_rows",
+ {
+ "results_found": 2,
+ "records": [
+ {"name": "DOE, JANE", "detail": "Row A", "status": "Active"},
+ {"name": "DOE, JANE", "detail": "Row B", "status": "Active"},
+ ],
+ },
+ ),
+ ]
+ )
+
+
+def _terminal_metadata_entry(label: str = "search_registry_person") -> dict[str, Any]:
+ return {
+ "block_label": label,
+ "declared_goal": "extract result rows for the requested person",
+ "claimed_outcomes": [
+ {
+ "id": "claim:goal",
+ "scope": "outcome",
+ "text": "result rows extracted",
+ "status": "observed_not_verified",
+ "covered_criteria": ["criterion:goal_0"],
+ }
+ ],
+ "completion_criteria": [
+ {"id": "criterion:goal_0", "text": "result rows extracted", "level": "terminal", "terminal": True}
+ ],
+ }
+
+
+def test_blocked_flag_run_reports_structured_blocker() -> None:
+ blocker = _run_blocks_structured_blocker_message(_blocked_flag_run_result())
+ assert blocker is not None
+ assert "human verification challenge" in blocker
+
+
+def test_blocked_flag_run_records_suspicious_success() -> None:
+ result = _blocked_flag_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+
+ assert ctx.last_test_ok is False
+ assert ctx.last_test_suspicious_success is True
+ assert ctx.last_full_workflow_test_ok is False
+ assert "reported a blocker" in (ctx.last_test_failure_reason or "")
+ assert ctx.last_failed_workflow_yaml == "blocks: []"
+ categories = result["data"]["failure_categories"]
+ assert any(category["category"] == "ANTI_BOT_DETECTION" for category in categories)
+ assert verified_goal_satisfied_context(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+ snapshot = getattr(ctx, "outcome_verification_trace_snapshot", {})
+ assert snapshot.get("run_output_blocker_detected") is True
+
+
+def test_blocked_flag_run_is_never_a_judge_candidate() -> None:
+ result = _blocked_flag_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+ assert _is_outcome_evidence_candidate(ctx, result) is False
+
+
+def test_candidacy_and_recording_agree_on_blocked_run() -> None:
+ result = _blocked_flag_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+
+ assert _is_outcome_evidence_candidate(ctx, result) is False
+ failed_variant = {**result, "ok": False}
+ assert _is_unfinished_run_verification_candidate(ctx, failed_variant) is False
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+ assert ctx.last_test_ok is False
+ assert ctx.last_test_suspicious_success is True
+
+
[email protected]("block_type", ["CODE", "code"])
+def test_blocked_status_value_rejected_deterministically(block_type: str) -> None:
+ result = _blocked_status_run_result(block_type)
+ ctx = _ctx(result["data"]["blocks"])
+
+ blocker = _run_blocks_structured_blocker_message(result)
+ assert blocker is not None
+ assert "blocked_by_challenge" in blocker
+ assert _is_outcome_evidence_candidate(ctx, result) is False
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+ assert ctx.last_test_ok is False
+ assert ctx.last_test_suspicious_success is True
+ assert ctx.last_full_workflow_test_ok is False
+
+
+def test_genuine_success_run_keeps_clean_path() -> None:
+ result = _genuine_success_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+
+ assert _run_blocks_structured_blocker_message(result) is None
+ _, empty_data_blocks, _ = _analyze_run_blocks(result)
+ assert empty_data_blocks is False
+ assert _is_outcome_evidence_candidate(ctx, result) is True
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+ assert ctx.last_test_ok is True
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_full_workflow_test_ok is True
+
+
+def test_empty_goal_collections_without_blocker_are_flagged() -> None:
+ result = _run_result([_code_block("search_registry_person", {"records": [], "result_count": 0})])
+ ctx = _ctx(result["data"]["blocks"])
+
+ assert _run_blocks_structured_blocker_message(result) is None
+ _, empty_data_blocks, _ = _analyze_run_blocks(result)
+ assert empty_data_blocks is True
+ assert _is_outcome_evidence_candidate(ctx, result) is False
+
+ _record_run_blocks_result(ctx, result, completion_verification=None)
+ assert ctx.last_test_ok is None
+ assert ctx.last_test_suspicious_success is True
+ assert ctx.last_full_workflow_test_ok is False
+
+
+def test_neutral_status_string_does_not_redeem_empty_collections() -> None:
+ result = _run_result([_code_block("search_registry_person", {"status": "completed", "records": []})])
+ _, empty_data_blocks, _ = _analyze_run_blocks(result)
+ assert empty_data_blocks is True
+
+
+def test_falsy_blocker_flags_and_action_only_outputs_do_not_trip() -> None:
+ result = _run_result(
+ [
+ _code_block("open_registry_search", {"anti_bot_blocked": False, "clicked": True}),
+ _code_block("accept_terms", {"clicked": True}),
+ {"label": "scroll_results", "block_type": "CODE", "status": "completed"},
+ ]
+ )
+ assert _run_blocks_structured_blocker_message(result) is None
+ _, empty_data_blocks, _ = _analyze_run_blocks(result)
+ assert empty_data_blocks is False
+
+
+def test_flag_rule_requires_strict_blocker_terms() -> None:
+ benign = _run_result([_code_block("notify", {"verification_passed": True})])
+ assert _run_blocks_structured_blocker_message(benign) is None
+
+ string_rule_parity = _run_result([_code_block("notify", {"verification_code_sent": "yes"})])
+ assert _run_blocks_structured_blocker_message(string_rule_parity) == "yes"
+
+
+def test_flag_rule_synthesizes_message_and_prefers_sibling_reason() -> None:
+ flag_only = _run_result([_code_block("search", {"captcha_required": True, "clicked": True})])
+ blocker = _run_blocks_structured_blocker_message(flag_only)
+ assert blocker is not None
+ assert "captcha" in blocker
+
+ flag_with_reason = _run_result(
+ [_code_block("search", {"blocked_by_challenge": True, "reason": "The submit control stayed disabled."})]
+ )
+ assert _run_blocks_structured_blocker_message(flag_with_reason) == "The submit control stayed disabled."
+
+
+def test_positive_status_value_with_strict_term_still_trips_blocker() -> None:
+ result = _run_result([_code_block("search", {"status": "captcha_solved", "records": [{"name": "DOE, JANE"}]})])
+ blocker = _run_blocks_structured_blocker_message(result)
+ assert blocker is not None
+ assert "captcha_solved" in blocker
+
+
+def test_status_rule_ignores_long_values_and_matches_state_key() -> None:
+ long_value = _run_result([_code_block("search", {"status": "x" * 100 + " challenge"})])
+ assert _run_blocks_structured_blocker_message(long_value) is None
+
+ state_value = _run_result([_code_block("search", {"state": "captcha_pending"})])
+ blocker = _run_blocks_structured_blocker_message(state_value)
+ assert blocker is not None
+ assert "captcha_pending" in blocker
+
+
+def test_extraction_payload_flag_semantics_unchanged() -> None:
+ block = {
+ "label": "extract_rows",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": {"extracted_information": {"blocked": True}},
+ }
+ assert _run_blocks_structured_blocker_message(_run_result([block])) is None
+
+
+def test_code_only_workflow_with_seam_metadata_counts_as_evidence_block() -> None:
+ blocks = [_code_block("search_registry_person", {})]
+ ctx = _ctx(blocks)
+ assert _current_workflow_has_evidence_block(ctx) is False
+
+ ctx.code_artifact_metadata = {"search_registry_person": _terminal_metadata_entry()}
+ assert _current_workflow_has_evidence_block(ctx) is True
+
+
+def test_metadata_without_terminal_coverage_is_not_an_evidence_block() -> None:
+ entry = _terminal_metadata_entry()
+ entry["completion_criteria"][0]["level"] = "prefix"
+ entry["completion_criteria"][0]["terminal"] = False
+ ctx = _ctx([_code_block("search_registry_person", {})])
+ ctx.code_artifact_metadata = {"search_registry_person": entry}
+ assert _current_workflow_has_evidence_block(ctx) is False
+
+
+def test_judge_no_evidence_warrants_repair_for_code_only_workflow_with_metadata() -> None:
+ result = _genuine_success_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+ ctx.code_artifact_metadata = {"search_registry_person": _terminal_metadata_entry()}
+
+ _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))
+
+ assert ctx.last_test_suspicious_success is True
+ assert ctx.last_full_workflow_test_ok is False
+
+
+def test_judge_no_evidence_keeps_building_without_metadata() -> None:
+ result = _genuine_success_run_result()
+ ctx = _ctx(result["data"]["blocks"])
+
+ _record_run_blocks_result(ctx, result, completion_verification=_no_evidence("c0"))
+
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_full_workflow_test_ok is True
diff --git a/tests/unit/test_copilot_tool_loop_error_block_running.py b/tests/unit/test_copilot_tool_loop_error_block_running.py
index 1552a0cd9..156bce834 100644
--- a/tests/unit/test_copilot_tool_loop_error_block_running.py
+++ b/tests/unit/test_copilot_tool_loop_error_block_running.py
@@ -27,6 +27,12 @@ def _ctx(**overrides: Any) -> Any:
"last_workflow_yaml": None,
"last_test_ok": None,
"copilot_run_start_monotonic": None,
+ "last_test_failure_reason": None,
+ "last_outcome_gate_reason": None,
+ "last_test_anti_bot": None,
+ "staged_workflow": None,
+ "staged_workflow_yaml": None,
+ "has_staged_proposal": False,
}
base.update(overrides)
return SimpleNamespace(**base)
diff --git a/tests/unit/test_downloaded_files_artifact_urls.py b/tests/unit/test_downloaded_files_artifact_urls.py
index fccb088a6..f7df77bdb 100644
--- a/tests/unit/test_downloaded_files_artifact_urls.py
+++ b/tests/unit/test_downloaded_files_artifact_urls.py
@@ -403,6 +403,22 @@ def test_content_endpoint_non_download_stays_inline():
assert disposition == "inline"
+def test_content_endpoint_audio_artifact_serves_audio_webm():
+ from skyvern.forge.sdk.routes.agent_protocol import _artifact_response_config
+
+ audio = Artifact(
+ artifact_id="a_audio",
+ artifact_type=ArtifactType.AUDIO,
+ uri="s3://skyvern-artifacts/.../dictation.webm",
+ organization_id="o_1",
+ created_at="2026-04-23T00:00:00Z",
+ modified_at="2026-04-23T00:00:00Z",
+ )
+ media_type, disposition = _artifact_response_config(audio)
+ assert media_type == "audio/webm"
+ assert disposition == "inline"
+
+
def test_content_endpoint_session_replay_falls_back_to_mp4_for_unknown_extension():
from skyvern.forge.sdk.routes.agent_protocol import _artifact_response_config
diff --git a/tests/unit/test_extract_action_open_tabs_context.py b/tests/unit/test_extract_action_open_tabs_context.py
index 057730f7e..473f23eee 100644
--- a/tests/unit/test_extract_action_open_tabs_context.py
+++ b/tests/unit/test_extract_action_open_tabs_context.py
@@ -131,6 +131,93 @@ class TestCacheVariant:
assert "cp" in result
+class TestNewTabSwitchTabGate:
+ @pytest.mark.parametrize("template", ["extract-action", "extract-action-static"])
+ def test_new_tab_shown_when_enabled(self, template: str) -> None:
+ rendered = prompt_engine.load_prompt(
+ template,
+ show_close_page_action=False,
+ show_new_tab_action=True,
+ show_switch_tab_action=False,
+ open_tabs_context=None,
+ **_BASE_KWARGS,
+ )
+ assert '"NEW_TAB"' in rendered
+ assert '"url"' in rendered
+ assert '"SWITCH_TAB"' not in rendered
+
+ @pytest.mark.parametrize("template", ["extract-action", "extract-action-static"])
+ def test_switch_tab_and_tab_index_key_shown_when_enabled(self, template: str) -> None:
+ rendered = prompt_engine.load_prompt(
+ template,
+ show_close_page_action=True,
+ show_new_tab_action=True,
+ show_switch_tab_action=True,
+ open_tabs_context="Tab 0: https://a.test\nTab 1 [current]: https://b.test",
+ **_BASE_KWARGS,
+ )
+ assert '"SWITCH_TAB"' in rendered
+ assert '"tab_index"' in rendered
+
+ @pytest.mark.parametrize("template", ["extract-action", "extract-action-static"])
+ def test_no_tab_actions_when_disabled(self, template: str) -> None:
+ rendered = prompt_engine.load_prompt(
+ template,
+ show_close_page_action=False,
+ show_new_tab_action=False,
+ show_switch_tab_action=False,
+ open_tabs_context=None,
+ **_BASE_KWARGS,
+ )
+ assert '"NEW_TAB"' not in rendered
+ assert '"SWITCH_TAB"' not in rendered
+ assert '"tab_index"' not in rendered
+
+ def test_switch_tab_changes_open_tabs_framing(self) -> None:
+ switch_on = prompt_engine.load_prompt(
+ "extract-action-dynamic",
+ show_close_page_action=True,
+ show_new_tab_action=True,
+ show_switch_tab_action=True,
+ open_tabs_context="Tab 0: https://a.test\nTab 1 [current]: https://b.test",
+ **_BASE_KWARGS,
+ )
+ assert "use SWITCH_TAB" in switch_on
+ assert "context only" not in switch_on
+
+ switch_off = prompt_engine.load_prompt(
+ "extract-action-dynamic",
+ show_close_page_action=True,
+ show_new_tab_action=False,
+ show_switch_tab_action=False,
+ open_tabs_context="Tab 0: https://a.test\nTab 1 [current]: https://b.test",
+ **_BASE_KWARGS,
+ )
+ assert "use SWITCH_TAB" not in switch_off
+ assert "context only" in switch_off
+
+
+class TestTabCacheVariant:
+ def test_nt_and_st_tags_present_when_enabled(self) -> None:
+ result = ForgeAgent._build_extract_action_cache_variant(
+ verification_code_check=False,
+ show_close_page_action=False,
+ complete_criterion=None,
+ show_new_tab_action=True,
+ show_switch_tab_action=True,
+ )
+ assert "nt" in result
+ assert "st" in result
+
+ def test_tab_tags_absent_by_default(self) -> None:
+ result = ForgeAgent._build_extract_action_cache_variant(
+ verification_code_check=False,
+ show_close_page_action=False,
+ complete_criterion=None,
+ )
+ assert result == "std"
+
+
class TestBuildOpenTabsContext:
@pytest.mark.asyncio
async def test_returns_none_when_working_page_is_none(self) -> None:
diff --git a/tests/unit/test_real_browser_state_active_page.py b/tests/unit/test_real_browser_state_active_page.py
new file mode 100644
index 000000000..e6e323d94
--- /dev/null
+++ b/tests/unit/test_real_browser_state_active_page.py
@@ -0,0 +1,100 @@
+"""Tests for the deterministic active-tab pin in RealBrowserState.get_working_page."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.webeye.real_browser_state import RealBrowserState
+
+
+def _mock_page(url: str = "https://example.test", closed: bool = False) -> MagicMock:
+ page = MagicMock()
+ page.url = url
+ page.is_closed = MagicMock(return_value=closed)
+ return page
+
+
+def _make_state() -> RealBrowserState:
+ return RealBrowserState(pw=MagicMock(), browser_context=MagicMock())
+
+
+async def test_no_pin_returns_last_page() -> None:
+ page0, page1 = _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ await state.set_working_page(page0)
+
+ assert await state.get_working_page() is page1
+
+
+async def test_pin_overrides_last_page() -> None:
+ page0, page1 = _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ await state.set_active_page(page0)
+
+ # page0 is the pinned (earlier) tab, not the last page — the pin must win.
+ assert await state.get_working_page() is page0
+
+
+async def test_pin_dropped_when_pinned_page_closed() -> None:
+ page0, page1 = _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ await state.set_active_page(page0)
+
+ page0.is_closed.return_value = True
+ assert await state.get_working_page() is page1
+
+
+async def test_pin_dropped_when_new_tab_opens() -> None:
+ page0, page1 = _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ await state.set_active_page(page0)
+
+ # A brand-new tab appears (count grows past pin time) -> auto-follow the newest tab.
+ page2 = _mock_page()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1, page2])
+ assert await state.get_working_page() is page2
+
+
+async def test_pin_survives_when_other_tab_closes() -> None:
+ page0, page1, page2 = _mock_page(), _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1, page2])
+ await state.set_active_page(page0)
+
+ # A different tab closes (count shrinks) -> the pin still holds.
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ assert await state.get_working_page() is page0
+
+
+async def test_pin_dropped_when_tab_closed_then_new_tab_opens() -> None:
+ # A close+open leaves the tab count unchanged, but a genuinely new tab must still
+ # take focus rather than being masked by the pin.
+ page0, page1, page2 = _mock_page(), _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1, page2])
+ await state.set_active_page(page0)
+
+ page3 = _mock_page()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page2, page3])
+ assert await state.get_working_page() is page3
+
+
+async def test_reset_working_page_clears_pin() -> None:
+ page0, page1 = _mock_page(), _mock_page()
+ state = _make_state()
+ state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ await state.set_active_page(page0)
+
+ await state.set_working_page(None)
+ assert await state.get_working_page() is None
diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.py
index 3232d891d..692373920 100644
--- a/tests/unit/test_request_logging.py
+++ b/tests/unit/test_request_logging.py
@@ -4,7 +4,10 @@ import json
from unittest.mock import MagicMock
import pytest
+from fastapi import FastAPI, HTTPException
+from fastapi.testclient import TestClient
+from skyvern.forge import request_logging
from skyvern.forge.request_logging import (
_BINARY_PLACEHOLDER,
_MAX_BODY_LENGTH,
@@ -14,6 +17,7 @@ from skyvern.forge.request_logging import (
_is_sensitive_key,
_sanitize_body,
_sanitize_response_body,
+ log_raw_request_middleware,
redact_sensitive_fields,
)
@@ -344,3 +348,113 @@ class TestSanitizeBody:
body = b'{"totp_identifier": "[email protected]", "content": "Your code is 123456"}'
result = _sanitize_body(request, body, "application/json")
assert result == _REDACTED
+
+
+# ---------------------------------------------------------------------------
+# log_raw_request_middleware — which requests produce an api.raw_request log
+# ---------------------------------------------------------------------------
+
+
+def _make_app() -> FastAPI:
+ app = FastAPI()
+ app.middleware("http")(log_raw_request_middleware)
+
+ @app.get("/heartbeat")
+ async def heartbeat() -> dict:
+ return {"ok": True}
+
+ @app.post("/tasks")
+ async def create_task() -> dict:
+ return {"created": True}
+
+ @app.get("/missing")
+ async def missing() -> dict:
+ raise HTTPException(status_code=404, detail="not found")
+
+ @app.get("/protected")
+ async def protected() -> dict:
+ raise HTTPException(status_code=403, detail="Invalid credentials")
+
+ @app.get("/v1/credentials/totp")
+ async def get_totp() -> dict:
+ return {"code": "123456", "content": "Your code is 123456"}
+
+ @app.get("/boom")
+ async def boom() -> dict:
+ raise ValueError("kaboom")
+
+ return app
+
+
+def log_mock(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ mock = MagicMock()
+ monkeypatch.setattr(request_logging, "LOG", mock)
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", True)
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS_SUCCESSFUL_READS", False)
+ return mock
+
+
+class TestMiddlewareLogVolume:
+ def test_successful_sensitive_get_keeps_redacted_audit_line(self, log_mock: MagicMock) -> None:
+ """OTP/credential reads must leave an audit trail even though they are successful reads."""
+ client = TestClient(_make_app())
+ response = client.get("/v1/credentials/totp")
+ assert response.status_code == 200
+ log_mock.info.assert_called_once()
+ assert log_mock.info.call_args.args[0] == "api.raw_request"
+ assert log_mock.info.call_args.kwargs["response_body"] == _REDACTED
+
+ def test_successful_get_is_not_logged(self, log_mock: MagicMock) -> None:
+ client = TestClient(_make_app())
+ response = client.get("/heartbeat")
+ assert response.status_code == 200
+ log_mock.info.assert_not_called()
+ log_mock.warning.assert_not_called()
+ log_mock.error.assert_not_called()
+
+ def test_successful_get_logged_when_reads_enabled(
+ self, log_mock: MagicMock, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS_SUCCESSFUL_READS", True)
+ client = TestClient(_make_app())
+ response = client.get("/heartbeat")
+ assert response.status_code == 200
+ log_mock.info.assert_called_once()
+ assert log_mock.info.call_args.args[0] == "api.raw_request"
+
+ def test_successful_post_is_logged(self, log_mock: MagicMock) -> None:
+ client = TestClient(_make_app())
+ response = client.post("/tasks")
+ assert response.status_code == 200
+ log_mock.info.assert_called_once()
+ assert log_mock.info.call_args.args[0] == "api.raw_request"
+
+ def test_failed_get_is_logged_as_warning(self, log_mock: MagicMock) -> None:
+ client = TestClient(_make_app())
+ response = client.get("/missing")
+ assert response.status_code == 404
+ log_mock.warning.assert_called_once()
+ assert log_mock.warning.call_args.args[0] == "api.raw_request"
+
+ def test_403_get_keeps_datadog_monitor_contract(self, log_mock: MagicMock) -> None:
+ """The 403-spike monitors query api.raw_request status:warn @status_code:403."""
+ client = TestClient(_make_app())
+ response = client.get("/protected")
+ assert response.status_code == 403
+ log_mock.warning.assert_called_once()
+ assert log_mock.warning.call_args.args[0] == "api.raw_request"
+ assert log_mock.warning.call_args.kwargs["status_code"] == 403
+
+ def test_exception_path_is_logged_as_error(self, log_mock: MagicMock) -> None:
+ client = TestClient(_make_app(), raise_server_exceptions=False)
+ response = client.get("/boom")
+ assert response.status_code == 500
+ log_mock.error.assert_called_once()
+ assert log_mock.error.call_args.args[0] == "api.raw_request"
+
+ def test_disabled_middleware_logs_nothing(self, log_mock: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", False)
+ client = TestClient(_make_app())
+ client.post("/tasks")
+ log_mock.info.assert_not_called()
diff --git a/tests/unit/test_tab_action_handlers.py b/tests/unit/test_tab_action_handlers.py
new file mode 100644
index 000000000..aa5328dfd
--- /dev/null
+++ b/tests/unit/test_tab_action_handlers.py
@@ -0,0 +1,96 @@
+"""Tests for the NEW_TAB / SWITCH_TAB action handlers."""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.webeye.actions import actions
+from skyvern.webeye.actions.handler import handle_new_tab_action, handle_switch_tab_action
+from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+
+
+def _task() -> MagicMock:
+ task = MagicMock()
+ task.task_id = "tsk_test"
+ task.workflow_run_id = None
+ return task
+
+
+def _mock_app(browser_state: MagicMock) -> MagicMock:
+ mock_app = MagicMock()
+ # get_for_task is synchronous and returns the browser state for the task.
+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state
+ return mock_app
+
+
+async def test_new_tab_opens_navigates_pins_and_stops_batch() -> None:
+ new_page = MagicMock()
+ new_page.bring_to_front = AsyncMock()
+ browser_state = MagicMock()
+ browser_state.new_page = AsyncMock(return_value=new_page)
+ browser_state.navigate_to_url = AsyncMock()
+ browser_state.set_active_page = AsyncMock()
+
+ action = actions.NewTabAction(url="https://example.test/page")
+ with patch("skyvern.webeye.actions.handler.app", _mock_app(browser_state)):
+ result = await handle_new_tab_action(action, MagicMock(), MagicMock(), _task(), MagicMock())
+
+ assert len(result) == 1
+ assert isinstance(result[0], ActionSuccess)
+ assert result[0].skip_remaining_actions is True
+ browser_state.navigate_to_url.assert_awaited_once_with(page=new_page, url="https://example.test/page")
+ browser_state.set_active_page.assert_awaited_once_with(new_page)
+
+
+async def test_new_tab_closes_tab_and_fails_when_navigation_fails() -> None:
+ new_page = MagicMock()
+ new_page.close = AsyncMock()
+ browser_state = MagicMock()
+ browser_state.new_page = AsyncMock(return_value=new_page)
+ browser_state.navigate_to_url = AsyncMock(side_effect=Exception("net::ERR_NAME_NOT_RESOLVED"))
+ browser_state.set_active_page = AsyncMock()
+
+ action = actions.NewTabAction(url="https://does-not-exist.test")
+ with patch("skyvern.webeye.actions.handler.app", _mock_app(browser_state)):
+ result = await handle_new_tab_action(action, MagicMock(), MagicMock(), _task(), MagicMock())
+
+ assert isinstance(result[0], ActionFailure)
+ # The failed/blank tab must be closed so the next scrape doesn't fail the task.
+ new_page.close.assert_awaited_once()
+ browser_state.set_active_page.assert_not_awaited()
+
+
+async def test_switch_tab_pins_target_and_stops_batch() -> None:
+ page0, page1 = MagicMock(), MagicMock()
+ page1.bring_to_front = AsyncMock()
+ browser_state = MagicMock()
+ browser_state.list_valid_pages = AsyncMock(return_value=[page0, page1])
+ browser_state.set_active_page = AsyncMock()
+
+ with patch("skyvern.webeye.actions.handler.app", _mock_app(browser_state)):
+ result = await handle_switch_tab_action(
+ actions.SwitchTabAction(tab_index=1), MagicMock(), MagicMock(), _task(), MagicMock()
+ )
+
+ assert isinstance(result[0], ActionSuccess)
+ assert result[0].skip_remaining_actions is True
+ browser_state.set_active_page.assert_awaited_once_with(page1)
+
+
+async def test_switch_tab_out_of_range_fails_without_stopping_step() -> None:
+ browser_state = MagicMock()
+ browser_state.list_valid_pages = AsyncMock(return_value=[MagicMock()])
+ browser_state.set_active_page = AsyncMock()
+
+ with patch("skyvern.webeye.actions.handler.app", _mock_app(browser_state)):
+ result = await handle_switch_tab_action(
+ actions.SwitchTabAction(tab_index=5), MagicMock(), MagicMock(), _task(), MagicMock()
+ )
+
+ assert isinstance(result[0], ActionFailure)
+ assert result[0].stop_execution_on_failure is False
+ browser_state.set_active_page.assert_not_awaited()
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 042d59c87..373c6c4c5 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -20,13 +20,20 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
+from fastapi import HTTPException
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
+from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
-from skyvern.forge.sdk.routes.workflow_copilot import COPILOT_V2_FLAG_KEY, workflow_copilot_chat_post
+from skyvern.forge.sdk.routes.workflow_copilot import (
+ COPILOT_V2_FLAG_KEY,
+ _validate_copilot_audio_artifact_id,
+ workflow_copilot_chat_audio,
+ workflow_copilot_chat_post,
+)
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatRequest,
WorkflowCopilotStreamErrorUpdate,
@@ -47,6 +54,142 @@ def _make_chat_request(mode: str | None = None, code_block: bool | None = None)
)
+async def test_chat_audio_upload_stores_artifact_for_existing_chat(monkeypatch: pytest.MonkeyPatch) -> None:
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ )
+ monkeypatch.setattr(
+ app.DATABASE,
+ "workflow_params",
+ SimpleNamespace(
+ get_workflow_copilot_chat_by_id=AsyncMock(return_value=chat),
+ create_workflow_copilot_chat=AsyncMock(),
+ ),
+ )
+ artifact_manager = SimpleNamespace(
+ create_log_artifact=AsyncMock(return_value="a_audio"),
+ wait_for_upload_aiotasks=AsyncMock(),
+ )
+ monkeypatch.setattr(app, "ARTIFACT_MANAGER", artifact_manager)
+ file = SimpleNamespace(
+ content_type="audio/webm",
+ read=AsyncMock(return_value=b"audio-bytes"),
+ close=AsyncMock(),
+ )
+
+ response = await workflow_copilot_chat_audio(
+ workflow_permanent_id="wpid-1",
+ workflow_copilot_chat_id="chat-1",
+ file=file,
+ organization=SimpleNamespace(organization_id="org-1"),
+ )
+
+ assert response.workflow_copilot_chat_id == "chat-1"
+ assert response.audio_artifact_id == "a_audio"
+ app.DATABASE.workflow_params.create_workflow_copilot_chat.assert_not_called()
+ artifact_manager.create_log_artifact.assert_awaited_once()
+ artifact_manager.wait_for_upload_aiotasks.assert_awaited_once_with(["chat-1"])
+ file.read.assert_awaited_once_with(settings.MAX_UPLOAD_FILE_SIZE + 1)
+ file.close.assert_awaited_once()
+
+
+async def test_chat_audio_upload_rejects_unsupported_audio_content_type() -> None:
+ file = SimpleNamespace(
+ content_type="audio/fake-binary",
+ read=AsyncMock(return_value=b"audio-bytes"),
+ close=AsyncMock(),
+ )
+
+ with pytest.raises(HTTPException) as exc_info:
+ await workflow_copilot_chat_audio(
+ workflow_permanent_id="wpid-1",
+ workflow_copilot_chat_id="chat-1",
+ file=file,
+ organization=SimpleNamespace(organization_id="org-1"),
+ )
+
+ assert exc_info.value.status_code == 400
+ assert exc_info.value.detail == "Unsupported audio format"
+ file.read.assert_not_awaited()
+ file.close.assert_awaited_once()
+
+
+async def test_chat_audio_upload_rejects_oversized_audio_without_reading_past_cap(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(settings, "MAX_UPLOAD_FILE_SIZE", 4)
+ file = SimpleNamespace(
+ content_type="audio/webm;codecs=opus",
+ read=AsyncMock(return_value=b"12345"),
+ close=AsyncMock(),
+ )
+
+ with pytest.raises(HTTPException) as exc_info:
+ await workflow_copilot_chat_audio(
+ workflow_permanent_id="wpid-1",
+ workflow_copilot_chat_id="chat-1",
+ file=file,
+ organization=SimpleNamespace(organization_id="org-1"),
+ )
+
+ assert exc_info.value.status_code == 413
+ file.read.assert_awaited_once_with(5)
+ file.close.assert_awaited_once()
+
+
+async def test_validate_copilot_audio_artifact_id_rejects_foreign_artifact(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ artifact = SimpleNamespace(
+ artifact_type=ArtifactType.AUDIO,
+ uri="s3://bucket/v1/local/org-1/logs/workflow_copilot_chat/chat-2/t.webm",
+ )
+ monkeypatch.setattr(
+ app.DATABASE,
+ "artifacts",
+ SimpleNamespace(get_artifact_by_id=AsyncMock(return_value=artifact)),
+ )
+
+ with pytest.raises(HTTPException) as exc_info:
+ await _validate_copilot_audio_artifact_id(
+ audio_artifact_id="a_audio",
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ )
+
+ assert exc_info.value.status_code == 400
+ assert "not linked" in exc_info.value.detail
+
+
+async def test_validate_copilot_audio_artifact_id_accepts_chat_scoped_artifact(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ artifact = SimpleNamespace(
+ artifact_type=ArtifactType.AUDIO,
+ uri="s3://bucket/v1/local/org-1/logs/workflow_copilot_chat/chat-1/t.webm",
+ )
+ monkeypatch.setattr(
+ app.DATABASE,
+ "artifacts",
+ SimpleNamespace(get_artifact_by_id=AsyncMock(return_value=artifact)),
+ )
+
+ validated = await _validate_copilot_audio_artifact_id(
+ audio_artifact_id="a_audio",
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ )
+
+ assert validated == "a_audio"
+
+
def test_terminal_narrative_metadata_preserves_payload_and_adds_contract_fields() -> None:
payload = {
"turnId": "turn-1",