Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-15740 Use question-response URLs for credential handoff (#8507)
Andrew Neilson6a7472179972
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsxindex 33d3cac16..2ff0523f3 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx@@ -6,14 +6,21 @@ import {render,screen,waitFor,+ within,} from "@testing-library/react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import type { WorkflowCopilotCredentialRequiredUpdate } from "./workflowCopilotTypes";+import type {+ QuestionInteraction,+ WorkflowCopilotCredentialRequiredUpdate,+} from "./workflowCopilotTypes";+import { ensureCredentialRecoveryToken } from "./credentialRecovery";type StreamBody = {message: string;supports_credential_pause?: boolean;+ supports_credential_pause_recovery?: boolean;+ credential_recovery_token?: string;};type StreamCall = {body: StreamBody;@@ -53,6 +60,9 @@ const {chat_history: [] as unknown[],proposed_workflow: null as Record<string, unknown> | null,auto_accept: false,+ pending_credential_requests:+ [] as WorkflowCopilotCredentialRequiredUpdate[],+ question_interactions: [] as QuestionInteraction[],},};const creds = {@@ -265,7 +275,10 @@ const saveData = {};vi.mock("@/store/WorkflowHasChangesStore", () => ({- useWorkflowHasChangesStore: () => ({ getSaveData: () => saveData }),+ useWorkflowHasChangesStore: Object.assign(+ () => ({ getSaveData: () => saveData }),+ { getState: () => ({ hasChanges: true }) },+ ),}));vi.mock("@/routes/workflows/hooks/useWorkflowRunQuery", () => ({@@ -428,6 +441,7 @@ function credentialsGets() {}beforeEach(() => {+ sessionStorage.clear();HTMLElement.prototype.scrollIntoView = vi.fn();HTMLElement.prototype.scrollTo = vi.fn();streamCalls.length = 0;@@ -447,6 +461,8 @@ beforeEach(() => {chat_history: [],proposed_workflow: null,auto_accept: false,+ pending_credential_requests: [],+ question_interactions: [],};});@@ -484,11 +500,113 @@ describe("WorkflowCopilotChat — activity log", () => {});describe("WorkflowCopilotChat — credential card wiring", () => {- it("sends supports_credential_pause on the request", async () => {+ it("keeps a credential pause recovered from history in ownership", async () => {+ ensureCredentialRecoveryToken("wpid_1");+ historyResponse.data.pending_credential_requests = [credentialFrame()];+ credentialsData.current = [+ {+ credential_id: "cred-hn",+ name: "HN Login",+ tested_url: "https://news.ycombinator.com/login",+ },+ ];++ await renderChat();+ expect(await screen.findByRole("combobox")).toBeTruthy();++ await submit("start a second turn");++ expect(postStreaming).not.toHaveBeenCalled();+ });++ it("recovers a credential card without SSE and resumes its original turn", async () => {+ const recoveryToken = ensureCredentialRecoveryToken("wpid_1");+ const question: QuestionInteraction = {+ interaction_id: "question-1",+ turn_id: "turn-1",+ tool_call_id: "ask-1",+ parts: [{ part_id: "url", prompt: "Sign-in URL?", choices: [] }],+ status: "pending",+ response: null,+ created_at: new Date().toISOString(),+ resolved_at: null,+ };+ historyResponse.data.question_interactions = [question];+ credentialsData.current = [+ {+ credential_id: "cred-hn",+ name: "HN Login",+ tested_url: "https://news.ycombinator.com/login",+ },+ ];+ await renderChat();+ expect(apiGet).toHaveBeenCalledWith(+ "/workflow/copilot/chat-history",+ expect.objectContaining({+ headers: {+ "X-Copilot-Credential-Recovery-Token": recoveryToken,+ },+ }),+ );+ const resolved: QuestionInteraction = {+ ...question,+ status: "resolved",+ response: { text: "https://news.ycombinator.com/login" },+ resolved_at: new Date().toISOString(),+ };+ sansApiPost.mockResolvedValueOnce({ data: resolved });+ historyResponse.data.question_interactions = [resolved];+ historyResponse.data.pending_credential_requests = [credentialFrame()];+ fireEvent.change(await screen.findByLabelText("Your response"), {+ target: { value: "https://news.ycombinator.com/login" },+ });+ fireEvent.click(+ within(screen.getByRole("group", { name: "Question parts" })).getByRole(+ "button",+ { name: "Send" },+ ),+ );+ const recoveredCredentialPicker = await screen.findByRole(+ "combobox",+ {},+ { timeout: 10000 },+ );+ await submit("start a different turn");+ expect(postStreaming).not.toHaveBeenCalled();+ fireEvent.click(recoveredCredentialPicker);+ fireEvent.click(await screen.findByRole("button", { name: "HN Login" }));+ await waitFor(() => expect(credentialResponsePosts()).toHaveLength(1));+ expect(credentialResponsePosts()[0]![1]).toMatchObject({+ turn_id: "turn-1",+ workflow_copilot_chat_id: "chat-1",+ resume_token: "rt-abc",+ action: "connected",+ credential_id: "cred-hn",+ });+ expect(postStreaming).not.toHaveBeenCalled();+ historyResponse.data.pending_credential_requests = [];+ historyResponse.data.chat_history = [+ {+ sender: "ai",+ content: "Recovered continuation completed",+ created_at: new Date().toISOString(),+ narrative_payload: { turnId: "turn-1", terminal: "response" },+ },+ ];+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1), {+ timeout: 10000,+ });+ }, 15000);++ it("sends a recoverable credential capability with the request", async () => {await renderChat();await submit("build me a workflow");await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));expect(streamCalls[0]!.body.supports_credential_pause).toBe(true);+ expect(streamCalls[0]!.body.supports_credential_pause_recovery).toBe(true);+ expect(streamCalls[0]!.body.credential_recovery_token).toMatch(+ /^[0-9a-f]{64}$/,+ );});it("renders the inline-pause card when a credential_required frame arrives", async () => {@@ -509,6 +627,21 @@ describe("WorkflowCopilotChat — credential card wiring", () => {).toBeTruthy();});+ it("keeps a dropped credential-pause turn owned while recovery polls", async () => {+ await renderChat();+ await submit("build me a workflow");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart());+ streamCalls[0]!.onMessage(credentialFrame());+ streamCalls[0]!.reject(new Error("stream disconnected"));+ });++ await submit("start a second turn");++ expect(postStreaming).toHaveBeenCalledTimes(1);+ });+it("skip POSTs a credential-response with action skip", async () => {await renderChat();await submit("build me a workflow");diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsxindex ea5aaf1e2..706aba9be 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx@@ -10,11 +10,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { FeatureFlagContext } from "@/hooks/useFeatureFlag";-import type { QuestionInteraction } from "./workflowCopilotTypes";+import type {+ QuestionInteraction,+ WorkflowCopilotCredentialRequiredUpdate,+} from "./workflowCopilotTypes";type HistoryData = {question_interactions?: QuestionInteraction[];pending_question_cancel_token?: string | null;+ pending_credential_requests?: WorkflowCopilotCredentialRequiredUpdate[];workflow_copilot_chat_id: string | null;chat_history: unknown[];proposed_workflow: Record<string, unknown> | null;@@ -712,6 +716,36 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream closexpect(renderedText()).toContain(capturedAiText);});+ it("claims ownership when recovery discovers a credential pause", async () => {+ await startTurn();+ await closeStreamWithoutTerminal();+ await advance(2_000);+ await resolveNextHistory(+ historyData({+ pending_credential_requests: [+ {+ type: "credential_required",+ turn_id: turnId,+ workflow_copilot_chat_id: "chat-1",+ resume_token: "resume-token",+ reason: "workflow_credential_inputs_unbound",+ message: "",+ login_page_urls: ["https://example.com/login"],+ credential_refs: [],+ timeout_seconds: 300,+ expires_at: new Date(Date.now() + 300_000).toISOString(),+ timestamp: new Date().toISOString(),+ },+ ],+ }),+ );++ await submit("another one");+ await advance(10);++ expect(streamCalls.length).toBe(1);+ });+it("does not re-read history when the stream ends on a terminal frame", async () => {await startTurn();await emitTerminalResponse();diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 224078a7d..bfc90cb22 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -97,6 +97,10 @@ import { QuestionPartsCard } from "./cards/QuestionPartsCard";import { WorkPlanCard } from "./cards/WorkPlanCard";import { nextAnsweringMessage, previousAskingMessage } from "./cardAdjacency";import { composerPlaceholder } from "./composerPlaceholder";+import {+ ensureCredentialRecoveryToken,+ readCredentialRecoveryHistory,+} from "./credentialRecovery";import { connectedAccountChoiceLabel } from "./cards/connectedAccountChoiceLabel";import { shouldShowDiffCard } from "./cards/DiffCard";import { ReviewGateCard, getReviewGateVerdict } from "./cards/ReviewGateCard";@@ -1039,6 +1043,9 @@ export function WorkflowCopilotChat({// turn_start and at every terminal so a dead resume_token can never render.const [livePauseFrame, setLivePauseFrame] =useState<WorkflowCopilotCredentialRequiredUpdate | null>(null);+ const [recoveredPauseFrames, setRecoveredPauseFrames] = useState<+ WorkflowCopilotCredentialRequiredUpdate[]+ >([]);// Local credential-card resolutions keyed by turn_id: live pauses after a// successful resume POST, and terminal-mode connect/skip (which never POST).// name is captured at connect so the receipt keeps showing it after the turn@@ -1125,6 +1132,9 @@ export function WorkflowCopilotChat({const cancelInFlightController = useRef<AbortController | null>(null);const recoveryPolls = useRef(new Map<string, () => void>());const recoveryGeneration = useRef(0);+ // A turn resumed from saved human input has no live SSE controller, but it+ // still owns the composer until history proves that continuation finished.+ const recoveredTurnOwnerRef = useRef<string | null>(null);// The poll is declared before the reconcile it calls on a recovered row.const reconcileCanonicalWorkflowRef = useRef<(() => Promise<void>) | null>(null,@@ -1438,6 +1448,9 @@ export function WorkflowCopilotChat({location.state,],);+ const recoverCredentialTurn = useRef<+ (chatId: string, turnId: string) => void+ >(() => {});const respondToCredentialPause = useCallback(async (frame: WorkflowCopilotCredentialRequiredUpdate,@@ -1447,6 +1460,8 @@ export function WorkflowCopilotChat({) => {if (credentialResponseInFlight.current) return;credentialResponseInFlight.current = true;+ const generation = recoveryGeneration.current;+ const chatId = workflowCopilotChatIdRef.current;try {// Copilot routes live on base_router (no /api/v1 prefix), like cancel.const client = await getClient(credentialGetter, "sans-api-v1");@@ -1457,6 +1472,19 @@ export function WorkflowCopilotChat({action,credential_id: action === "connected" ? credentialId : undefined,});+ if (+ recoveryGeneration.current !== generation ||+ workflowCopilotChatIdRef.current !== chatId ||+ chatId !== frame.workflow_copilot_chat_id+ ) {+ return;+ }+ if (!streamingAbortController.current) {+ recoverCredentialTurn.current(+ frame.workflow_copilot_chat_id,+ frame.turn_id,+ );+ }setCredentialResolutions((prev) =>withCappedResolution(prev,@@ -1741,6 +1769,7 @@ export function WorkflowCopilotChat({streamingAbortController.current = null;inFlightRef.current = false;setIsLoading(false);+ setRecoveredPauseFrames([]);setQuestionInteractions([]);setQuestionCancelToken(null);stopRecoveryPolls();@@ -1767,6 +1796,7 @@ export function WorkflowCopilotChat({data: WorkflowCopilotChatHistoryResponse,carryForwardLifecycle = true,) => {+ setRecoveredPauseFrames(data.pending_credential_requests ?? []);setQuestionInteractions(data.question_interactions ?? []);setQuestionCancelToken(data.pending_question_cancel_token ?? null);const historyMessages = data.chat_history.map((message, index) => ({@@ -1832,10 +1862,11 @@ export function WorkflowCopilotChat({recoveryGeneration.current += 1;recoveryPolls.current.forEach((stop) => stop());recoveryPolls.current.clear();+ recoveredTurnOwnerRef.current = null;}, []);const startRecoveryPoll = useCallback(- (chatId: string | null, turnId: string) => {+ (chatId: string | null, turnId: string, ownsTurn = false) => {if (!workflowPermanentId) {return;}@@ -1875,6 +1906,10 @@ export function WorkflowCopilotChat({if (recoveryPolls.current.get(turnId) === finish) {recoveryPolls.current.delete(turnId);}+ if (recoveredTurnOwnerRef.current === turnId) {+ recoveredTurnOwnerRef.current = null;+ setIsLoading(false);+ }}// Ending without the turn's row leaves a notice describing work that has@@ -1892,6 +1927,10 @@ export function WorkflowCopilotChat({recoveryPolls.current.get(turnId)?.();recoveryPolls.current.set(turnId, finish);+ if (ownsTurn) {+ recoveredTurnOwnerRef.current = turnId;+ setIsLoading(true);+ }// A read that never returns leaves schedule() unreached, so the budget// needs a timer of its own or the deadline can never fire.deadlineTimer = setTimeout(giveUp, RECOVERY_POLL_BUDGET_MS);@@ -1900,8 +1939,11 @@ export function WorkflowCopilotChat({clearTimer();// finish() aborts the in-flight read, which surfaces in tick's catch;// without this the rejection would reschedule a stopped poll.- if (stopped || Date.now() >= deadline) {- finish();+ if (stopped) {+ return;+ }+ if (Date.now() >= deadline) {+ giveUp();return;}const delay = RECOVERY_POLL_DELAYS_MS[step] ?? RECOVERY_POLL_STEADY_MS;@@ -1922,15 +1964,17 @@ export function WorkflowCopilotChat({const client = await getClient(credentialGetter, "sans-api-v1");const controller = new AbortController();inFlight = controller;- const response = await client.get<WorkflowCopilotChatHistoryResponse>(- "/workflow/copilot/chat-history",- {- params: chatId- ? { workflow_copilot_chat_id: chatId }- : { workflow_permanent_id: workflowPermanentId },- signal: controller.signal,- },- );+ const response =+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(+ client,+ workflowPermanentId,+ {+ params: chatId+ ? { workflow_copilot_chat_id: chatId }+ : { workflow_permanent_id: workflowPermanentId },+ signal: controller.signal,+ },+ );if (inFlight === controller) {inFlight = null;}@@ -1944,6 +1988,20 @@ export function WorkflowCopilotChat({finish();return;}+ const pendingCredentialRequests =+ response.data.pending_credential_requests ?? [];+ setRecoveredPauseFrames(pendingCredentialRequests);+ // The backend can persist the pause before its SSE frame reaches the+ // browser. Once history proves this poll owns a credential wait, it+ // must block another turn just like a poll armed from the live frame.+ if (+ !ownsTurn &&+ pendingCredentialRequests.some((item) => item.turn_id === turnId)+ ) {+ ownsTurn = true;+ recoveredTurnOwnerRef.current = turnId;+ setIsLoading(true);+ }const pendingQuestion = response.data.question_interactions?.find((item) => item.turn_id === turnId && item.status === "pending",);@@ -2040,6 +2098,16 @@ export function WorkflowCopilotChat({[applyHistoryResponse, credentialGetter, workflowPermanentId],);+ const adoptRecoveredCredentialPause = useCallback(+ (data: WorkflowCopilotChatHistoryResponse) => {+ const pendingRequests = data.pending_credential_requests ?? [];+ const pending = pendingRequests[pendingRequests.length - 1];+ if (!pending) return;+ startRecoveryPoll(data.workflow_copilot_chat_id, pending.turn_id, true);+ },+ [startRecoveryPoll],+ );+const loadChatInPlace = useCallback(async (chatId: string) => {if (!workflowPermanentId) return;@@ -2061,16 +2129,19 @@ export function WorkflowCopilotChat({repin();try {const client = await getClient(credentialGetter, "sans-api-v1");- const response = await client.get<WorkflowCopilotChatHistoryResponse>(- "/workflow/copilot/chat-history",- {- params: {- workflow_permanent_id: workflowPermanentId,- workflow_copilot_chat_id: chatId,+ const response =+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(+ client,+ workflowPermanentId,+ {+ params: {+ workflow_permanent_id: workflowPermanentId,+ workflow_copilot_chat_id: chatId,+ },},- },- );+ );applyHistoryResponse(response.data, false);+ adoptRecoveredCredentialPause(response.data);// Mark history loaded for this workflow so the mount effect won't reload// the latest chat over the one the user just selected.historyLoadedForRef.current = workflowPermanentId;@@ -2085,12 +2156,16 @@ export function WorkflowCopilotChat({credentialGetter,workflowPermanentId,applyHistoryResponse,+ adoptRecoveredCredentialPause,stopRecoveryPolls,repin,discardQueuedPrompt,],);+ recoverCredentialTurn.current = (chatId, turnId) =>+ startRecoveryPoll(chatId, turnId, true);+const handleSelectHistoryChat = useCallback((chat: WorkflowCopilotChatSummary) => {if (chat.workflow_copilot_chat_id === workflowCopilotChatIdRef.current) {@@ -2291,12 +2366,14 @@ export function WorkflowCopilotChat({return null;}const client = await getClient(credentialGetter, "sans-api-v1");- const response = await client.get<WorkflowCopilotChatHistoryResponse>(- "/workflow/copilot/chat-history",- {- params: { workflow_permanent_id: workflowPermanentId },- },- );+ const response =+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(+ client,+ workflowPermanentId,+ {+ params: { workflow_permanent_id: workflowPermanentId },+ },+ );const latestChatId = response.data.workflow_copilot_chat_id ?? null;setWorkflowCopilotChatId(latestChatId);return latestChatId;@@ -2689,9 +2766,19 @@ export function WorkflowCopilotChat({if (historyLoadedForRef.current === workflowPermanentId) {return;}- // Reached only when this workflow's transcript is about to replace another's,- // so a poll armed against the outgoing chat must not apply history here.+ const isWorkflowSwitch = historyLoadedForRef.current !== null;+ // A poll armed against the outgoing chat must not apply history to the new+ // workflow. The state reset is only needed after a workflow was loaded;+ // writing fresh empty arrays on first mount can restart this async effect+ // before its history request records completion.stopRecoveryPolls();+ if (isWorkflowSwitch) {+ setRecoveredPauseFrames([]);+ setQuestionInteractions([]);+ setQuestionCancelToken(null);+ setWorkflowCopilotChatId(null);+ workflowCopilotChatIdRef.current = null;+ }let isMounted = true;@@ -2700,16 +2787,20 @@ export function WorkflowCopilotChat({repin();try {const client = await getClient(credentialGetter, "sans-api-v1");- const response = await client.get<WorkflowCopilotChatHistoryResponse>(- "/workflow/copilot/chat-history",- {- params: { workflow_permanent_id: workflowPermanentId },- },- );+ const response =+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(+ client,+ workflowPermanentId,+ {+ params: { workflow_permanent_id: workflowPermanentId },+ },+ { retryTransientFailure: true },+ );if (!isMounted) return;applyHistoryResponse(response.data);+ adoptRecoveredCredentialPause(response.data);historyLoadedForRef.current = workflowPermanentId;} catch (error) {console.error("Failed to load chat history:", error);@@ -2727,6 +2818,7 @@ export function WorkflowCopilotChat({};}, [credentialGetter,+ adoptRecoveredCredentialPause,repin,stopRecoveryPolls,updateQueuedPrompt,@@ -2923,8 +3015,9 @@ export function WorkflowCopilotChat({),);setIsLoading(true);- if (!streamingAbortController.current)- startRecoveryPoll(workflowCopilotChatId, interaction.turn_id);+ if (!streamingAbortController.current) {+ startRecoveryPoll(workflowCopilotChatId, interaction.turn_id, true);+ }return true;} catch {toast({@@ -3046,12 +3139,14 @@ export function WorkflowCopilotChat({const refresh = async () => {try {const client = await getClient(credentialGetter, "sans-api-v1");- const { data } = await client.get<WorkflowCopilotChatHistoryResponse>(- "/workflow/copilot/chat-history",- {- params: { workflow_copilot_chat_id: workflowCopilotChatId },- },- );+ const { data } =+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(+ client,+ workflowPermanentId,+ {+ params: { workflow_copilot_chat_id: workflowCopilotChatId },+ },+ );if (disposed) return;const interactions = data.question_interactions ?? [];setQuestionInteractions(interactions);@@ -3064,8 +3159,7 @@ export function WorkflowCopilotChat({.reverse().find((item) => item.status === "resolved");if (resolved) {- setIsLoading(true);- startRecoveryPoll(workflowCopilotChatId, resolved.turn_id);+ startRecoveryPoll(workflowCopilotChatId, resolved.turn_id, true);} else {setIsLoading(false);inFlightRef.current = false;@@ -3083,6 +3177,7 @@ export function WorkflowCopilotChat({}, [hasPendingQuestion,workflowCopilotChatId,+ workflowPermanentId,credentialGetter,startRecoveryPoll,]);@@ -3116,7 +3211,7 @@ export function WorkflowCopilotChat({);};const action = resolveSendAction({- inFlight: inFlightRef.current,+ inFlight: inFlightRef.current || recoveredTurnOwnerRef.current !== null,hasQueuedPrompt: Boolean(queuedPromptRef.current),requiresLiveBrowser,isLiveBrowserReady,@@ -3395,6 +3490,7 @@ export function WorkflowCopilotChat({};let streamChatId: string | null = null;let sawTerminalFrame = false;+ let sawCredentialPause = false;const shouldArmRecovery = () =>streamTurnId !== null &&!sawTerminalFrame &&@@ -3772,6 +3868,8 @@ export function WorkflowCopilotChat({for (const attached of sentAttachments) {inFlightFileIds.current.add(attached.file_id);}+ const credentialRecoveryToken =+ ensureCredentialRecoveryToken(workflowPermanentId);await client.postStreaming<WorkflowCopilotSsePayload>("/workflow/copilot/chat-post",{@@ -3797,6 +3895,10 @@ export function WorkflowCopilotChat({selected_block_label: readSelectedBlockLabel(),keep_pending_proposal: Boolean(pendingProposalTurnId),supports_credential_pause: true,+ supports_credential_pause_recovery: Boolean(+ credentialRecoveryToken,+ ),+ credential_recovery_token: credentialRecoveryToken ?? undefined,supports_question_tool: true,} as WorkflowCopilotChatRequest,(payload) => {@@ -3882,6 +3984,7 @@ export function WorkflowCopilotChat({.setTitleFromCopilotIfDefault(payload.title);return false;case "credential_required":+ sawCredentialPause = true;setLivePauseFrame(payload);return false;case "turn_start": {@@ -4055,6 +4158,7 @@ export function WorkflowCopilotChat({startRecoveryPoll(streamChatId ?? workflowCopilotChatIdRef.current,streamTurnId,+ sawCredentialPause,);}}@@ -4244,7 +4348,7 @@ export function WorkflowCopilotChat({const lastTurn = lastTurnRef.current;const drainAction = resolveDrainAction({queuedReason: queuedPrompt.reason,- inFlight: isLoading,+ inFlight: isLoading || recoveredTurnOwnerRef.current !== null,hasLiveBrowserSession: Boolean(liveBrowserSessionId),hasWorkflowPermanentId: Boolean(workflowPermanentId),queuedContent: queuedPrompt.content,@@ -5190,6 +5294,33 @@ export function WorkflowCopilotChat({RESPONSE has frozen the narrative into the latest AI message —otherwise the same turn would render twice.*/}+ {!isLoadingHistory &&+ recoveredPauseFrames+ .filter(+ (frame) =>+ frame.workflow_copilot_chat_id === workflowCopilotChatId &&+ frame.turn_id !== livePauseFrame?.turn_id,+ )+ .map((frame) => (+ <CredentialCard+ key={frame.turn_id}+ frame={liveFrameToCardFrame(frame)}+ mode="inline-pause"+ reloadKey={credentialsReloadKey}+ resolvedOutcome={credentialResolutions[frame.turn_id]}+ onConnect={(credentialId, name) =>+ credentialId+ ? void respondToCredentialPause(+ frame,+ "connected",+ credentialId,+ name,+ )+ : openCredentialModal(frame, frame.turn_id)+ }+ onSkip={() => void respondToCredentialPause(frame, "skip")}+ />+ ))}{narrative.turnId !== null && narrative.terminal === null && (<divclassName="flex flex-col gap-2"diff --git a/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.test.ts b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.test.tsnew file mode 100644index 000000000..e5ae2e0d9--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.test.ts@@ -0,0 +1,75 @@+import type { AxiosInstance } from "axios";+import { beforeEach, describe, expect, it, vi } from "vitest";++import {+ CredentialRecoveryHistoryError,+ credentialRecoveryHeaders,+ currentCredentialRecoveryToken,+ readCredentialRecoveryHistory,+ ensureCredentialRecoveryToken,+} from "./credentialRecovery";++describe("credentialRecovery", () => {+ beforeEach(() => sessionStorage.clear());++ it("retains one capability for every recoverable turn in the workflow", () => {+ const first = ensureCredentialRecoveryToken("wpid-1");+ expect(first).toMatch(/^[0-9a-f]{64}$/);+ expect(currentCredentialRecoveryToken("wpid-1")).toBe(first);+ expect(credentialRecoveryHeaders("wpid-1")).toEqual({+ "X-Copilot-Credential-Recovery-Token": first,+ });++ const second = ensureCredentialRecoveryToken("wpid-1");+ expect(second).toMatch(/^[0-9a-f]{64}$/);+ expect(second).toBe(first);+ expect(currentCredentialRecoveryToken("wpid-1")).toBe(second);+ });++ it("keeps capabilities scoped by workflow", () => {+ ensureCredentialRecoveryToken("wpid-1");+ expect(currentCredentialRecoveryToken("wpid-2")).toBeNull();+ expect(credentialRecoveryHeaders(undefined)).toEqual({});+ });++ it("does not expose the capability when a history read fails", async () => {+ const token = ensureCredentialRecoveryToken("wpid-1");+ const client = {+ get: async () => {+ throw {+ response: { status: 403 },+ config: { headers: { "X-Copilot-Credential-Recovery-Token": token } },+ };+ },+ };++ const error = await readCredentialRecoveryHistory(+ client,+ "wpid-1",+ {},+ ).catch((caught: unknown) => caught);++ expect(error).toBeInstanceOf(CredentialRecoveryHistoryError);+ expect(error).toMatchObject({ status: 403 });+ expect(String(error)).not.toContain(token);+ expect(error).not.toHaveProperty("config");+ });++ it("retries one transient authenticated history failure", async () => {+ ensureCredentialRecoveryToken("wpid-1");+ let reads = 0;+ const get = vi.fn(async () => {+ reads += 1;+ if (reads === 1) throw new Error("temporary network failure");+ return { data: { workflow_copilot_chat_id: "chat-1" } };+ });+ const client = { get } as unknown as Pick<AxiosInstance, "get">;++ const response = await readCredentialRecoveryHistory<{+ workflow_copilot_chat_id: string;+ }>(client, "wpid-1", {}, { retryTransientFailure: true });++ expect(reads).toBe(2);+ expect(response.data.workflow_copilot_chat_id).toBe("chat-1");+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.ts b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.tsnew file mode 100644index 000000000..6a569e090--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.ts@@ -0,0 +1,92 @@+import type { AxiosInstance, AxiosRequestConfig, AxiosResponse } from "axios";++const KEY_PREFIX = "copilot-credential-recovery";++function storageKey(workflowId: string): string {+ return `${KEY_PREFIX}:${workflowId}`;+}++export function currentCredentialRecoveryToken(+ workflowId: string | undefined,+): string | null {+ if (!workflowId) return null;+ try {+ return sessionStorage.getItem(storageKey(workflowId));+ } catch {+ return null;+ }+}++export function ensureCredentialRecoveryToken(+ workflowId: string | undefined,+): string | null {+ if (!workflowId) return null;+ try {+ // Keep one capability for this workflow for the lifetime of the browser+ // tab. Each server pause still binds its digest to one org/chat/turn and+ // requires its own single-use resume token.+ const current = sessionStorage.getItem(storageKey(workflowId));+ if (current) return current;+ const token = Array.from(+ crypto.getRandomValues(new Uint8Array(32)),+ (byte) => byte.toString(16).padStart(2, "0"),+ ).join("");+ sessionStorage.setItem(storageKey(workflowId), token);+ return token;+ } catch {+ return null;+ }+}++export function credentialRecoveryHeaders(+ workflowId: string | undefined,+): Record<string, string> {+ const token = currentCredentialRecoveryToken(workflowId);+ return token ? { "X-Copilot-Credential-Recovery-Token": token } : {};+}++export class CredentialRecoveryHistoryError extends Error {+ readonly status: number | null;++ constructor(error: unknown) {+ const status =+ (error as { response?: { status?: unknown } })?.response?.status ?? null;+ const safeStatus = typeof status === "number" ? status : null;+ super(+ safeStatus === null+ ? "Copilot history request failed"+ : `Copilot history request failed (${safeStatus})`,+ );+ this.name = "CredentialRecoveryHistoryError";+ this.status = safeStatus;+ }+}++export async function readCredentialRecoveryHistory<T>(+ client: Pick<AxiosInstance, "get">,+ workflowId: string | undefined,+ config: AxiosRequestConfig,+ options: { retryTransientFailure?: boolean } = {},+): Promise<AxiosResponse<T>> {+ const attempts = options.retryTransientFailure ? 2 : 1;+ for (let attempt = 0; attempt < attempts; attempt += 1) {+ try {+ return await client.get<T>("/workflow/copilot/chat-history", {+ ...config,+ headers: {+ ...config.headers,+ ...credentialRecoveryHeaders(workflowId),+ },+ });+ } catch (error) {+ // Axios errors retain request headers. Replace them before callers log+ // the failure so the capability cannot reach the browser console.+ const safeError = new CredentialRecoveryHistoryError(error);+ const retryable = safeError.status === null || safeError.status >= 500;+ if (attempt + 1 >= attempts || !retryable || config.signal?.aborted) {+ throw safeError;+ }+ }+ }+ throw new CredentialRecoveryHistoryError(null);+}diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex ed81336c6..9a56cb633 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -131,6 +131,8 @@ export interface WorkflowCopilotChatRequest {// Opt-in: only clients that can render the credential_required frame set// this, so the backend never pauses a turn a client would silently drop.supports_credential_pause?: boolean;+ credential_recovery_token?: string;+ supports_credential_pause_recovery?: boolean;supports_question_tool?: boolean;}@@ -163,6 +165,7 @@ export interface WorkflowCopilotChatHistoryMessage {}export interface WorkflowCopilotChatHistoryResponse {+ pending_credential_requests?: WorkflowCopilotCredentialRequiredUpdate[];question_interactions?: QuestionInteraction[];pending_question_cancel_token?: string | null;workflow_copilot_chat_id: string | null;diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 1114f0350..1b276e63b 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -1487,6 +1487,10 @@ class AgentFunction:async def browser_context_route_handlers_allowed(self, **_: Any) -> bool:return True+ async def has_retained_browser_egress_guard(self, browser_context: Any) -> bool:+ """Whether cloud-only retained init scripts still require this context's guard."""+ return False+async def setup_browser_context_extensions(self, browser_context: Any, **kwargs: Any) -> None:"""Attach cloud-only listeners/route handlers to a fresh BrowserContext. OSS no-op."""diff --git a/skyvern/forge/log_redaction.py b/skyvern/forge/log_redaction.pyindex db1ef07e6..70ee5f631 100644--- a/skyvern/forge/log_redaction.py+++ b/skyvern/forge/log_redaction.py@@ -40,6 +40,7 @@ SENSITIVE_HEADERS = {"set-cookie","x-api-key","x-posthog-attribution",+ "x-copilot-credential-recovery-token",}# Exact field names that are always redacted. Use a set for O(1) lookup@@ -51,6 +52,8 @@ SENSITIVE_FIELDS: set[str] = {"password","secret","token",+ "credential_recovery_token",+ "resume_token","api_key","apikey","api-key",diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex b7f0c9d1c..d13de0702 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -114,6 +114,7 @@ from skyvern.forge.sdk.copilot.context import (record_proposed_credential_in_global_llm_context,sanitize_global_llm_context_for_prompt,)+from skyvern.forge.sdk.copilot.credential_pause import credential_recovery_token_digestfrom skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failurefrom skyvern.forge.sdk.copilot.enforcement import (CopilotNonRetriableNavError,@@ -683,10 +684,7 @@ def _apply_raw_secret_turn_transition(returnif policy.raw_secret_handling == "redacted_draft" and policy.raw_secret_safety_status != "blocked":- policy.testing_intent = "skip_test"- policy.allow_run_blocks = False- policy.allow_missing_credentials_in_draft = True- policy.credential_draft_deferred_explicitly = True+ policy.apply_raw_secret_redacted_draft()else:policy.raw_secret_handling = "block"policy.user_response_policy = "ask_clarification"@@ -5102,6 +5100,12 @@ async def _run_copilot_turn_impl(target_block_label=getattr(chat_request, "target_block_label", None),selected_block_label=getattr(chat_request, "selected_block_label", None),client_supports_credential_pause=getattr(chat_request, "supports_credential_pause", False),+ client_supports_credential_pause_recovery=chat_request.supports_credential_pause_recovery,+ credential_recovery_token_digest=credential_recovery_token_digest(+ chat_request.credential_recovery_token.get_secret_value()+ if chat_request.credential_recovery_token+ else None+ ),executed_block_fingerprints={label: set(fingerprints) for label, fingerprints in (prior_executed_block_fingerprints or {}).items()},diff --git a/skyvern/forge/sdk/copilot/ask_user.py b/skyvern/forge/sdk/copilot/ask_user.pyindex a29c0dee2..4469183e0 100644--- a/skyvern/forge/sdk/copilot/ask_user.py+++ b/skyvern/forge/sdk/copilot/ask_user.py@@ -57,6 +57,10 @@ class QuestionResponse(BaseModel):answers: list[QuestionAnswer] = Field(default_factory=list)text: str | None = Noneskipped: bool = False+ # Server-owned result of screening the free-text answer. It is persisted+ # with the interaction so active and reconstructed policies enforce the+ # same trust floor without retaining the detected literal.+ raw_secret_detected: bool = Field(default=False, exclude_if=lambda detected: not detected)class QuestionInteraction(BaseModel):@@ -194,6 +198,14 @@ async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_iwhile True:recorded = await repo.poll_copilot_question(ctx.organization_id, chat_id, interaction.interaction_id)if recorded.status == "resolved":+ if ctx.request_policy is not None:+ ctx.request_policy.project_question_response_sites(recorded)+ ctx.allow_untested_workflow_draft = (+ ctx.request_policy.raw_secret_detected+ and ctx.request_policy.raw_secret_handling == "redacted_draft"+ )+ if recorded.response is not None and not recorded.response.skipped:+ ctx.credential_recovery_armed = Trueawait ctx.stream.send({"type": "question_resolved",diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex f16e629f4..b8bec2852 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -1145,6 +1145,9 @@ class CopilotContext(AgentContext):# is set from the chat request at construction; the rest are owned by maybe_credential_pause.last_run_skipped_unbound_credentials: bool = Falseclient_supports_credential_pause: bool = False+ client_supports_credential_pause_recovery: bool = False+ credential_recovery_token_digest: str | None = field(default=None, repr=False)+ credential_recovery_armed: bool = Falsecredential_pause_used: bool = False# True only while a card is on screen. credential_pause_used stays true for the rest of the# turn once one has been raised, which cannot tell a concurrent sibling ask from a later one.diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.pyindex a86d0c4d2..9aa894c6f 100644--- a/skyvern/forge/sdk/copilot/credential_pause.py+++ b/skyvern/forge/sdk/copilot/credential_pause.py@@ -10,8 +10,8 @@ itself is the poller rather than a task racing the handler.The resume path is not authorized by org auth + ``turn_id`` alone. Establishingthe pause writes an *active-pause record*, keyed by (org, chat, turn), that-carries a one-time ``resume_token`` delivered only in the ``credential_required``-frame. ``resolve_credential_pause`` -- the only writer of the loop-facing+carries a one-time ``resume_token`` delivered in the ``credential_required``+frame and authenticated chat history for clients advertising card recovery. ``resolve_credential_pause`` -- the only writer of the loop-facingresponse flag -- refuses to store a decision unless the caller presents thattoken against a still-pending record, and consumes the record on the firstaccepted response so a leaked or replayed ``turn_id`` can't resolve the pause.@@ -20,6 +20,7 @@ accepted response so a leaked or replayed ``turn_id`` can't resolve the pause.from __future__ import annotationsimport asyncio+import hashlibimport jsonimport reimport secrets@@ -27,9 +28,10 @@ import timefrom dataclasses import dataclassfrom datetime import datetime, timedelta, timezonefrom http import HTTPStatus-from typing import TYPE_CHECKING, Any, Literal+from typing import TYPE_CHECKING, Any, Literal, Protocolimport structlog+from pydantic import ValidationErrorfrom skyvern.config import settingsfrom skyvern.forge import app@@ -99,14 +101,25 @@ class _ActivePauseRecord:resume_token: strstatus: Literal["pending", "consumed"]expires_at: datetime+ card: WorkflowCopilotCredentialRequiredUpdate | None = None+ recovery_token_digest: str | None = None-def _encode_active_pause(resume_token: str, expires_at: datetime, *, consumed: bool = False) -> str:+def _encode_active_pause(+ resume_token: str,+ expires_at: datetime,+ *,+ consumed: bool = False,+ card: WorkflowCopilotCredentialRequiredUpdate | None = None,+ recovery_token_digest: str | None = None,+) -> str:return json.dumps({"resume_token": resume_token,"status": "consumed" if consumed else "pending","expires_at": expires_at.isoformat(),+ "card": card.model_dump(mode="json") if card is not None else None,+ "recovery_token_digest": recovery_token_digest,})@@ -126,7 +139,18 @@ def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:expires_at = datetime.fromisoformat(str(data.get("expires_at")))except ValueError:return None- return _ActivePauseRecord(resume_token=token, status=record_status, expires_at=expires_at)+ try:+ card = WorkflowCopilotCredentialRequiredUpdate.model_validate(data["card"]) if data.get("card") else None+ except ValidationError:+ card = None+ digest = data.get("recovery_token_digest")+ return _ActivePauseRecord(+ resume_token=token,+ status=record_status,+ expires_at=expires_at,+ card=card,+ recovery_token_digest=digest if isinstance(digest, str) else None,+ )def _validate_pending_pause(record: _ActivePauseRecord | None, resume_token: str) -> _ActivePauseRecord:@@ -158,6 +182,86 @@ def _validate_pending_pause(record: _ActivePauseRecord | None, resume_token: strreturn record+def credential_recovery_token_digest(token: str | None) -> str | None:+ """Hash a 32-byte browser capability without retaining the bearer value."""+ if not isinstance(token, str) or len(token) != 64:+ return None+ try:+ decoded = bytes.fromhex(token)+ except ValueError:+ return None+ if len(decoded) != 32:+ return None+ return hashlib.sha256(token.encode()).hexdigest()+++class _CredentialRecoveryContext(Protocol):+ credential_recovery_armed: bool+ client_supports_credential_pause_recovery: bool+ credential_recovery_token_digest: str | None+++def _credential_recovery_enabled(ctx: _CredentialRecoveryContext) -> bool:+ return bool(+ ctx.credential_recovery_armed+ and ctx.client_supports_credential_pause_recovery+ and ctx.credential_recovery_token_digest+ )+++async def pending_credential_requests(+ organization_id: str,+ chat_id: str,+ turn_ids: list[str],+ recovery_token: str | None,+) -> list[WorkflowCopilotCredentialRequiredUpdate]:+ digest = credential_recovery_token_digest(recovery_token)+ cache = app.CACHE+ if cache is None or digest is None:+ return []+ cards: list[WorkflowCopilotCredentialRequiredUpdate] = []+ for turn_id in turn_ids:+ try:+ raw = await cache.get(credential_pause_active_key(organization_id, chat_id, turn_id))+ except Exception:+ LOG.warning("Failed to recover Copilot credential pause", exc_info=True)+ raise+ record = _decode_active_pause(raw)+ if (+ record is None+ or record.card is None+ or record.recovery_token_digest is None+ or not secrets.compare_digest(digest, record.recovery_token_digest)+ ):+ continue+ try:+ _validate_pending_pause(record, record.resume_token)+ except CredentialPauseRejection:+ continue+ card = record.card+ if (+ card.turn_id == turn_id+ and card.workflow_copilot_chat_id == chat_id+ and card.resume_token == record.resume_token+ and card.expires_at == record.expires_at+ ):+ cards.append(card)+ return cards+++async def credential_pause_is_active(organization_id: str, chat_id: str, turn_id: str) -> bool | None:+ """Report whether a live waiter owns the turn, or ``None`` when Redis is unavailable."""+ cache = app.CACHE+ if cache is None:+ return False+ try:+ record = _decode_active_pause(await cache.get(credential_pause_active_key(organization_id, chat_id, turn_id)))+ except Exception:+ LOG.warning("Failed to inspect Copilot credential pause", exc_info=True)+ return None+ return bool(record is not None and record.status == "pending" and datetime.now(timezone.utc) < record.expires_at)++async def check_credential_pause_resumable(cache: Any,*,@@ -394,6 +498,7 @@ async def _wait_for_credential_response(stream: EventSourceStream,timeout_seconds: int,) -> CredentialPauseResolution | None:+ recovery_enabled = _credential_recovery_enabled(ctx)# Check once before the sleep loop so a card response posted in the brief# window before the first poll doesn't cost a full extra poll interval.first = await _try_resolve_credential_response(response_key, ctx.organization_id)@@ -409,7 +514,7 @@ async def _wait_for_credential_response(resolved = await _try_resolve_credential_response(response_key, ctx.organization_id)if resolved != "pending":return resolved- if await stream.is_disconnected():+ if not recovery_enabled and await stream.is_disconnected():return Nonereturn None@@ -427,23 +532,20 @@ async def _run_credential_pause(Returns the resolution, or None to let the caller proceed without one(kill-switch off, client can't render the frame, already paused once this- turn, no cache configured or not shared across workers, client gone, or- the wait timed out).+ turn, no shared cache, unrecoverable client disconnect, or timeout).+ Recovery-capable clients restore the same active card through chat history."""if not credential_pause_transport_ready(ctx, copilot_config):return None- # Predicted true (the sync guard chain passed) but bailing below anyway --- # latch it now so this iteration can't loop back into the same prediction,- # and tag the outcome "declined" (distinct from "timeout") so the caller- # knows no frame was ever sent and can fall back to a normal nudge instead- # of a premature finalize. credential_pause_transport_ready's own docstring- # notes it excludes the async-only disconnect check, which is the gap here.+ # Latch before async checks so a declined transport cannot trigger another+ # pause. Recovery-capable clients can retrieve the card after disconnect.ctx.credential_pause_used = Truecache = getattr(app, "CACHE", None)if cache is None:ctx.credential_pause_outcome = "declined"return None- if await stream.is_disconnected():+ recovery_enabled = _credential_recovery_enabled(ctx)+ if not recovery_enabled and await stream.is_disconnected():ctx.credential_pause_outcome = "declined"return Nonepolicy = getattr(ctx, "request_policy", None)@@ -465,27 +567,30 @@ async def _run_credential_pause(# response endpoint refuses to resolve a turn that has no pending record.# expires_at is the same deadline the frame tells the client -- the record's# own TTL is a separate infra grace, not a resolve-after-timeout allowance.+ card = WorkflowCopilotCredentialRequiredUpdate(+ type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,+ turn_id=turn_id,+ workflow_copilot_chat_id=chat_id,+ resume_token=resume_token,+ reason=reason,+ message=message,+ login_page_urls=login_page_urls,+ credential_refs=credential_refs,+ timeout_seconds=timeout_seconds,+ expires_at=expires_at,+ timestamp=now,+ )await cache.set(credential_pause_active_key(organization_id, chat_id, turn_id),- _encode_active_pause(resume_token, expires_at),+ _encode_active_pause(+ resume_token,+ expires_at,+ card=card if recovery_enabled else None,+ recovery_token_digest=ctx.credential_recovery_token_digest if recovery_enabled else None,+ ),ex=_credential_pause_record_ttl(timeout_seconds),)-- await stream.send(- WorkflowCopilotCredentialRequiredUpdate(- type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,- turn_id=turn_id,- workflow_copilot_chat_id=chat_id,- resume_token=resume_token,- reason=reason,- message=message,- login_page_urls=login_page_urls,- credential_refs=credential_refs,- timeout_seconds=timeout_seconds,- expires_at=expires_at,- timestamp=now,- )- )+ await stream.send(card)async def _invalidate_active_pause_record() -> CredentialPauseResolution | None:# The waiter can exit (disconnect, genuine timeout, cancellation, or an@@ -510,12 +615,9 @@ async def _run_credential_pause(response_key = credential_response_cache_key(organization_id, chat_id, turn_id)- if await stream.is_disconnected():- # send() can return True even when the client is already gone (its own- # protocol contract: "queued for delivery or dropped because the client- # is gone") -- a disconnect racing the send itself would otherwise wait- # out the full timeout for a card nobody ever saw. Re-check right after- # send and treat it the same as the pre-send disconnect guard above.+ if not recovery_enabled and await stream.is_disconnected():+ # Legacy clients cannot restore a dropped frame, so preserve their+ # immediate decline when disconnect races the send.await _invalidate_active_pause_record()ctx.credential_pause_outcome = "declined"return Nonediff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 8b8cac3d1..80dc0bcdc 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -18,6 +18,7 @@ from skyvern.forge import appfrom skyvern.forge.log_redaction import redact_sensitive_fieldsfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler+from skyvern.forge.sdk.copilot.ask_user import QuestionInteractionfrom skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.context import ProposedCredential, StructuredContextfrom skyvern.forge.sdk.copilot.credential_resolution import (@@ -874,6 +875,27 @@ class LivePageResolutionRecord:page_url: str = ""+@dataclass(frozen=True)+class UserMessageSiteURLSource:+ message_index: int+ kind: Literal["user_message"] = field(default="user_message", init=False)+++@dataclass(frozen=True)+class QuestionResponseSiteURLSource:+ interaction_id: str+ kind: Literal["question_response"] = field(default="question_response", init=False)+++SiteURLSource = UserMessageSiteURLSource | QuestionResponseSiteURLSource+++@dataclass(frozen=True)+class _SiteURLText:+ text: str+ source: SiteURLSource++@dataclassclass RequestPolicy:testing_intent: str = "unspecified"@@ -936,8 +958,9 @@ class RequestPolicy:# only be released onto one of these (or a vault/tested match); a site only a model produced is# never eligible.user_provided_site_urls: list[str] = field(default_factory=list)- # Which user message (1-based) each of those URLs came from, so a release records its provenance.- user_site_url_sources: dict[str, int] = field(default_factory=dict)+ # The ordinary user message or accepted interactive response each URL came from, so a release+ # records truthful provenance without retaining or logging the response text.+ user_site_url_sources: dict[str, SiteURLSource] = field(default_factory=dict)existing_workflow_credential_ids: list[str] = field(default_factory=list)# Read from the saved workflow row, never from the submitted YAML. The submission is the live# canvas, which carries a copilot proposal the user has not accepted, so it cannot grant a run.@@ -964,6 +987,18 @@ class RequestPolicy:canonical_user_message: str = field(default="", repr=False, compare=False)_authoring_pending: bool = field(default=False, repr=False, compare=False)+ def project_question_response_sites(self, interaction: QuestionInteraction) -> None:+ project_question_response_sites(self, interaction)++ def apply_raw_secret_redacted_draft(self) -> None:+ self.raw_secret_detected = True+ self.raw_secret_handling = "redacted_draft"+ self.raw_secret_safety_status = "detected"+ self.testing_intent = "skip_test"+ self.allow_run_blocks = False+ self.allow_missing_credentials_in_draft = True+ self.credential_draft_deferred_explicitly = True+def graded_completion_criteria(self) -> list[CompletionCriterion]:return [criterion for criterion in self.completion_criteria if not criterion.method_mandated]@@ -4115,40 +4150,87 @@ def credential_candidate_label(credential: Credential) -> str:return f"{label} - {'; '.join(facts)}"-def _ground_user_provided_sites(+def _accepted_question_response_url_texts(interaction: QuestionInteraction) -> list[_SiteURLText]:+ response = interaction.response+ if interaction.status != "resolved" or response is None or response.skipped or response.raw_secret_detected:+ return []+ source = QuestionResponseSiteURLSource(interaction_id=interaction.interaction_id)+ texts = [response.text] if response.text else []+ texts.extend(answer.text for answer in response.answers if answer.text)+ return [_SiteURLText(text=text, source=source) for text in texts]+++def _persisted_question_response_url_texts(raw_interaction: dict[str, Any]) -> list[_SiteURLText]:+ try:+ interaction = QuestionInteraction.model_validate(raw_interaction)+ except ValidationError:+ return []+ return _accepted_question_response_url_texts(interaction)+++def _project_user_provided_sites(policy: RequestPolicy,- user_message: str,- full_chat_history: Sequence[WorkflowCopilotChatHistoryMessage],+ url_texts: Sequence[_SiteURLText],+ *,+ reset: bool,) -> None:- """Record every site the user themselves gave this chat, so the fill seam can release a credential- onto one of them and never onto a site only a model produced.-- Linking a later "log into pathfold" back to a URL from an earlier turn is the agent's job, not- this function's: it reads the whole conversation. What is recorded here is only the deterministic- part — the origins the user actually wrote.- """- # USER only, never TURN_OPENER_SENDERS: this set releases credentials, so a site must have- # been written by the person, not by a row the product authored on their behalf.- user_texts = [- message.content- for message in full_chat_history- if message.sender == WorkflowCopilotChatSender.USER and message.content- ]- user_texts.append(user_message or "")- seen_origins: set[str] = set()- urls: list[str] = []- sources: dict[str, int] = {}- for index, text in enumerate(user_texts, start=1):- for candidate in URL_CANDIDATE_RE.findall(text):+ """Project literal user-authored URL facts while preserving first-origin provenance."""+ if reset:+ policy.user_provided_site_urls = []+ policy.user_site_url_sources = {}+ seen_origins = {parts[2] for url in policy.user_provided_site_urls if (parts := _url_parts(url)) is not None}+ for item in url_texts:+ for candidate in URL_CANDIDATE_RE.findall(item.text):cleaned = candidate.rstrip(".,;:!?")parts = _url_parts(cleaned)if parts is None or parts[2] in seen_origins:continueseen_origins.add(parts[2])- urls.append(cleaned)- sources[cleaned] = index- policy.user_provided_site_urls = urls- policy.user_site_url_sources = sources+ policy.user_provided_site_urls.append(cleaned)+ policy.user_site_url_sources[cleaned] = item.source+++def project_question_response_sites(policy: RequestPolicy, interaction: QuestionInteraction) -> None:+ if interaction.response is not None and interaction.response.raw_secret_detected:+ policy.apply_raw_secret_redacted_draft()+ return+ _project_user_provided_sites(policy, _accepted_question_response_url_texts(interaction), reset=False)+++def _ground_user_provided_sites(+ policy: RequestPolicy,+ user_message: str,+ full_chat_history: Sequence[WorkflowCopilotChatHistoryMessage],+) -> None:+ """Rebuild the URL facts the person supplied in USER rows and accepted question responses.++ Linking a later site name back to a URL is the agent's job. This projection only verifies URLs+ in the two structured user-authored text surfaces; prompts, choices, rendered history, and+ PRODUCT or AI prose never become credential-origin authority.+ """+ url_texts: list[_SiteURLText] = []+ user_message_index = 0+ for message in full_chat_history:+ if message.sender == WorkflowCopilotChatSender.USER and message.content:+ user_message_index += 1+ url_texts.append(+ _SiteURLText(+ text=message.content,+ source=UserMessageSiteURLSource(message_index=user_message_index),+ )+ )+ if message.sender != WorkflowCopilotChatSender.AI or message.narrative_payload is None:+ continue+ for raw_interaction in message.narrative_payload.get("questionInteractions", []):+ url_texts.extend(_persisted_question_response_url_texts(raw_interaction))+ if user_message:+ url_texts.append(+ _SiteURLText(+ text=user_message,+ source=UserMessageSiteURLSource(message_index=user_message_index + 1),+ )+ )+ _project_user_provided_sites(policy, url_texts, reset=True)def _prior_approved_connection_ids(global_llm_context: str) -> set[str]:diff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.pyindex 8babc6fd2..cebe26d2d 100644--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py@@ -25,7 +25,10 @@ from skyvern.forge.sdk.copilot.credential_resolution import is_resolved_page_urlfrom skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctxfrom skyvern.forge.sdk.copilot.page_identity import safe_page_originfrom skyvern.forge.sdk.copilot.request_policy import (+ QuestionResponseSiteURLSource,RequestPolicy,+ SiteURLSource,+ UserMessageSiteURLSource,admit_credential_for_live_page,loggable_origin,)@@ -343,13 +346,23 @@ def _missing_credential_origin_error(credential_id: str, page_url: str | None) -)-def _log_fill_grant(route: str, url: str, credential_id: str, source_message: int | None = None) -> None:+def _log_fill_grant(+ route: str,+ url: str,+ credential_id: str,+ source: SiteURLSource | None = None,+) -> None:+ source_fields: dict[str, str | int] = {}+ if isinstance(source, UserMessageSiteURLSource):+ source_fields = {"source_kind": source.kind, "source_user_message": source.message_index}+ elif isinstance(source, QuestionResponseSiteURLSource):+ source_fields = {"source_kind": source.kind, "source_interaction_id": source.interaction_id}LOG.info("copilot credential fill grant",route=route,page_origin=loggable_origin(url),credential_id=credential_id,- source_user_message=source_message,+ **source_fields,)@@ -538,7 +551,12 @@ async def _credential_fill_origin_grant(if settled or credential_id == await _sole_org_password_credential_id(load_once):# An origin-only match (no registrable site) keeps the origin-scoped grant so the# release guard can still compare it; site matches travel the whole site.- _log_fill_grant("user_url", page_url, credential_id, policy.user_site_url_sources.get(matched_url))+ _log_fill_grant(+ "user_url",+ page_url,+ credential_id,+ policy.user_site_url_sources.get(matched_url),+ )return _CredentialFillOriginGrant(page_url, whole_site=site_level), Nonereturn None, _ambiguous_unbound_credential_steer(credential_id, page_url)return None, _missing_credential_origin_error(credential_id, page_url or None)diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 7eb1df50a..d234ec2a4 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -14,7 +14,7 @@ from urllib.parse import urlparseimport structlogimport yaml-from fastapi import Depends, File, Form, HTTPException, Request, UploadFile, status+from fastapi import Depends, File, Form, Header, HTTPException, Request, UploadFile, statusfrom opentelemetry import trace as otel_tracefrom pydantic import ValidationErrorfrom sse_starlette import EventSourceResponse@@ -42,6 +42,8 @@ from skyvern.forge.sdk.copilot.context import (from skyvern.forge.sdk.copilot.credential_pause import (CredentialPauseRejection,check_credential_pause_resumable,+ credential_pause_is_active,+ pending_credential_requests,resolve_credential_pause,)from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS@@ -2771,6 +2773,27 @@ async def _reconcile_interrupted_copilot_turns(chat: WorkflowCopilotChat, organi)if last_activity > abandoned_before:continue+ credential_pause_active = await credential_pause_is_active(+ organization_id,+ chat.workflow_copilot_chat_id,+ entry.turn_id,+ )+ if credential_pause_active is True:+ continue+ has_recovery_handoff = any(+ item.status == "resolved" and item.response is not None and not item.response.skipped+ for item in entry.question_interactions+ )+ if (+ credential_pause_active is None+ and has_recovery_handoff+ and last_activity+ > abandoned_before - timedelta(seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS)+ ):+ # A transient Redis failure must not reclaim a possible second+ # credential wait, while an unrelated stale turn must eventually+ # reconcile even if the auxiliary cache remains unavailable.+ continueclaimed = await app.DATABASE.workflow_params.claim_pending_copilot_turn(organization_id=organization_id,workflow_copilot_chat_id=chat.workflow_copilot_chat_id,@@ -2792,6 +2815,7 @@ async def _reconcile_interrupted_copilot_turns(chat: WorkflowCopilotChat, organi@base_router.get("/workflow/copilot/chat-history", include_in_schema=False)async def workflow_copilot_chat_history(+ credential_recovery_token: str | None = Header(None, alias="X-Copilot-Credential-Recovery-Token"),workflow_permanent_id: str | None = None,workflow_copilot_chat_id: str | None = None,organization: Organization = Depends(org_auth_service.get_current_org),@@ -2825,6 +2849,14 @@ async def workflow_copilot_chat_history(else:chat_messages = []return WorkflowCopilotChatHistoryResponse(+ pending_credential_requests=await pending_credential_requests(+ organization.organization_id,+ chat.workflow_copilot_chat_id,+ list(chat.pending_turns),+ credential_recovery_token,+ )+ if chat+ else [],workflow_copilot_chat_id=chat.workflow_copilot_chat_id if chat else None,question_interactions=list({@@ -2899,6 +2931,8 @@ async def workflow_copilot_question_response(raise HTTPException(status_code=503, detail="The safety screen is unavailable. Please retry your answer.")+ if safety.status == "detected":+ response.raw_secret_detected = Truereturn safety.canonical_user_messageif response.text is not None:diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex c4b6c2ba6..cbb60daf2 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -3,7 +3,7 @@ from datetime import datetimefrom enum import StrEnumfrom typing import Annotated, Any, Literal-from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, StringConstraints, field_validator+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, SecretStr, StringConstraints, field_validatorfrom skyvern.forge.sdk.copilot.ask_user import QuestionInteraction, QuestionResponsefrom skyvern.forge.sdk.copilot.code_write_diff import CodeWriteDiff@@ -250,6 +250,15 @@ class WorkflowCopilotChatRequest(BaseModel):),)supports_question_tool: bool = Field(False, description="The client can display and answer ask_user requests.")+ credential_recovery_token: SecretStr | None = Field(+ None,+ repr=False,+ exclude=True,+ description="Client-held capability for restoring this turn's pending credential card.",+ )+ supports_credential_pause_recovery: bool = Field(+ False, description="The client restores pending credential cards from chat history."+ )supports_credential_pause: bool = Field(False,description=(@@ -336,16 +345,6 @@ class WorkflowCopilotChatHistoryMessage(BaseModel):created_at: datetime = Field(..., description="When the message was created")-class WorkflowCopilotChatHistoryResponse(BaseModel):- question_interactions: list[QuestionInteraction] = Field(default_factory=list)- pending_question_cancel_token: str | None = None- workflow_copilot_chat_id: str | None = Field(None, description="Latest chat ID for the workflow")- chat_history: list[WorkflowCopilotChatHistoryMessage] = Field(default_factory=list, description="Chat messages")- proposed_workflow: dict | None = Field(None, description="Latest workflow proposed by the copilot")- auto_accept: bool | None = Field(None, description="Whether copilot auto-accepts workflow updates")- work_plan: list[str] = Field(default_factory=list, description="Latest work plan the copilot model wrote")--class WorkflowCopilotChatSummary(BaseModel):model_config = ConfigDict(from_attributes=True)@@ -711,6 +710,18 @@ class WorkflowCopilotCredentialRequiredUpdate(BaseModel):timestamp: datetime = Field(..., description="Server timestamp")+class WorkflowCopilotChatHistoryResponse(BaseModel):+ pending_credential_requests: list[WorkflowCopilotCredentialRequiredUpdate] = Field(default_factory=list)+ question_interactions: list[QuestionInteraction] = Field(default_factory=list)+ pending_question_cancel_token: str | None = None+ workflow_copilot_chat_id: str | None = Field(None, description="Latest chat ID for the workflow")+ chat_history: list[WorkflowCopilotChatHistoryMessage] = Field(default_factory=list, description="Chat messages")+ proposed_workflow: dict | None = Field(None, description="Latest workflow proposed by the copilot")+ auto_accept: bool | None = Field(None, description="Whether copilot auto-accepts workflow updates")++ work_plan: list[str] = Field(default_factory=list, description="Latest work plan the copilot model wrote")++class WorkflowCopilotCodegenProgressUpdate(BaseModel):"""Live-only drafting progress while the LLM streams authoring-tool arguments. Not persisted; theworkflow_draft frame supersedes it."""diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex bd2a985ff..af52da5f0 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -3247,7 +3247,19 @@ class ForLoopBlock(Block):browser_session_id: str | None,) -> BrowserState | None:if browser_session_id:- return await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(browser_session_id, organization_id)+ context = skyvern_context.current()+ expected_runnable_id = (+ context.browser_session_runnable_id+ if context and context.browser_session_runnable_id+ else workflow_run_id+ )+ return await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(+ browser_session_id,+ organization_id,+ expected_runnable_id=expected_runnable_id,+ expected_runnable_generation_id=(context.browser_session_runnable_generation_id if context else None),+ workflow_run_id=workflow_run_id,+ )return app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)async def _snapshot_loop_baseline_pages(diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 0400095ad..571b8c3ae 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -2165,6 +2165,7 @@ async def _recover_download_page(browser_address=browser_address,browser_profile_id=browser_state.browser_artifacts.applied_browser_profile_id,browser_session_id=task.browser_session_id,+ stale_context_is_unusable=True,)recovered_page = await browser_state.get_working_page()if recovered_page is None:diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.pyindex 592a788e4..c709f7a99 100644--- a/skyvern/webeye/browser_state.py+++ b/skyvern/webeye/browser_state.py@@ -69,6 +69,7 @@ class BrowserState(Protocol):browser_address: str | None = None,browser_profile_id: str | None = None,browser_session_id: str | None = None,+ stale_context_is_unusable: bool = False,) -> None: ...async def get_working_page(self) -> Page | None: ...diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex f7f7d5dd2..cb5534518 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -53,6 +53,10 @@ SETTLE_JITTER_MS = 500RECOVERABLE_BLANK_PAGE_URLS = {":"}+class _BrowserConnectionProbeFailed(str):+ """Diagnostic reason whose stale driver may be stopped before replacement."""++def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:if not left or not right:return False@@ -758,7 +762,7 @@ class RealBrowserState(BrowserState):try:connected = bool(browser.is_connected())except Exception as exc:- return False, f"browser_connection_probe_failed:{type(exc).__name__}"+ return False, _BrowserConnectionProbeFailed(f"browser_connection_probe_failed:{type(exc).__name__}")return connected, None if connected else "browser_context_disconnected"def is_connected(self) -> bool:@@ -780,10 +784,23 @@ class RealBrowserState(BrowserState):browser_address: str | None = None,browser_profile_id: str | None = None,browser_session_id: str | None = None,+ stale_context_is_unusable: bool = False,) -> None:- # The old driver pipe is gone, so check_and_fix_state must not reuse self.pw; start a- # fresh Playwright driver and reconnect to the same (still-alive) remote browser.+ # Rebuild through a fresh Playwright driver only after the old connection is known dead or+ # an explicitly unusable connection has completed bounded shutdown.stale_pw = self.pw+ stale_context = self.browser_context+ stale_page = self.__page+ stale_active_page = self.__active_page+ stale_active_page_known_pages = self.__active_page_known_pages.copy()+ stale_browser_artifacts = self.browser_artifacts+ stale_browser_cleanup = self.browser_cleanup+ stale_route_policy_url = self.browser_context_route_policy_url+ stale_sessionless_init_script_registrations = self._sessionless_init_script_registrations.copy()+ stale_disconnect_listener_browser = self._disconnect_listener_browser+ stale_diagnostic = self._browser_state_diagnostic+ stale_ever_connected = self._ever_connected+ stale_close_requested = self._close_requested# check_and_fix_state rebuilds through the factory; forward this session's download binding so the# creator seam preserves the provider-selected destination on reconnect. The binding is carried# forward, never overridden after the fact, so a genuine provider change is not mislabeled.@@ -797,32 +814,93 @@ class RealBrowserState(BrowserState):display_recording_owner_id = recorder.owner_id if isinstance(recorder, DisplayRecorder) else Noneeffective_route_policy_url = browser_context_route_policy_url or self.browser_context_route_policy_url_stale_driver_connected, stale_driver_disconnect_reason = self._connection_status()+ stale_driver_connection_probe_failed = isinstance(stale_driver_disconnect_reason, _BrowserConnectionProbeFailed)stale_driver_is_known_disconnected = stale_driver_disconnect_reason in {"playwright_driver_connection_closed","browser_context_disconnected",}+ stale_context_is_known_unusable = (+ stale_driver_disconnect_reason+ in {+ "browser_context_missing",+ "browser_context_close_called",+ "browser_context_closed",+ }+ or stale_driver_connection_probe_failed+ )+ stale_driver_may_be_live = not stale_driver_is_known_disconnected+ if stale_driver_may_be_live and not (stale_context_is_unusable or stale_context_is_known_unusable):+ raise RuntimeError("Cannot replace a Playwright driver while its connection may still be live")+ retain_guard_until_replacement = (+ stale_driver_may_be_live+ and (stale_context_is_unusable or stale_driver_connection_probe_failed)+ and await app.AGENT_FUNCTION.has_retained_browser_egress_guard(stale_context)+ )+ stop_stale_before_replacement = stale_driver_may_be_live and not retain_guard_until_replacement+ retry_stale_shutdown_after_replacement = False+ if stop_stale_before_replacement:+ stale_driver_pre_shutdown_error: BaseException | None = None++ async def stop_stale_driver_before_replacement() -> None:+ nonlocal stale_driver_pre_shutdown_error+ try:+ await stale_pw.stop()+ except BaseException as exc:+ stale_driver_pre_shutdown_error = exc+ raise+ finally:+ # A cancellation-resistant stop may deliver the close event after+ # reconnect has already propagated cancellation to its caller.+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations.copy()++ try:+ stale_driver_stopped = await self._run_bounded_detachable(+ stop_stale_driver_before_replacement(),+ BROWSER_CLOSE_TIMEOUT,+ "unusable stale Playwright driver shutdown before reconnect",+ accept_failure_as_completion=True,+ )+ finally:+ # The stop can synchronously deliver this context's close event before it+ # times out, fails, or observes caller cancellation. Preserve the snapshot+ # for this wrapper's next reconnect even when no replacement is started.+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations.copy()+ if stale_driver_pre_shutdown_error is not None:+ retry_stale_shutdown_after_replacement = True+ elif not stale_driver_stopped:+ raise RuntimeError("Failed to stop unusable stale Playwright driver before reconnect")++ def restore_stale_state() -> None:+ self.pw = stale_pw+ self.browser_context = stale_context+ self.__page = stale_page+ self.__active_page = stale_active_page+ self.__active_page_known_pages = stale_active_page_known_pages+ self.browser_artifacts = stale_browser_artifacts+ self.browser_cleanup = stale_browser_cleanup+ self.browser_context_route_policy_url = stale_route_policy_url+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations+ self._disconnect_listener_browser = stale_disconnect_listener_browser+ self._browser_state_diagnostic = stale_diagnostic+ self._ever_connected = stale_ever_connected+ self._close_requested = stale_close_requested++ try:+ if self.engine_selection is not None:+ fresh_pw = await self.engine_selection.start_driver()+ else:+ fresh_pw = await async_playwright().start()+ close_driver_connection_on_transport_loss(fresh_pw)+ except BaseException:+ restore_stale_state()+ raise++ # Reconciliation installs the replacement guard before replaying idempotent registrations.+ # A retained guard remains attached until its replacement is ready so an unusable context+ # cannot expose scripts during recovery. Unguarded unusable drivers are stopped above.+ self.pw = fresh_pwself.browser_context = Noneawait self.set_working_page(None)- # Detach the stale driver before the replacement connects. Playwright can otherwise replay- # context init scripts through both CDP clients, duplicating effects such as keepalive timers.- # A timeout or unknown liveness aborts because overlap is not safe to guess through. When the- # transport is positively disconnected, a completed stop that raises keeps the legacy recovery- # behavior: start a fresh driver rather than converting the transport loss into MissingBrowserState.- stale_driver_stopped = await self._run_bounded_detachable(- stale_pw.stop(),- BROWSER_CLOSE_TIMEOUT,- "stale Playwright driver shutdown before reconnect",- accept_failure_as_completion=stale_driver_is_known_disconnected,- )- if not stale_driver_stopped:- raise RuntimeError("Failed to stop stale Playwright driver before reconnect")- # Reconnect on the SAME engine this state was pinned to at creation; never silently switch- # engines underneath a live run. States built outside the per-run seam keep the stock driver.- if self.engine_selection is not None:- self.pw = await self.engine_selection.start_driver()- else:- self.pw = await async_playwright().start()- close_driver_connection_on_transport_loss(self.pw)try:await self.check_and_fix_state(proxy_location=proxy_location,@@ -839,14 +917,63 @@ class RealBrowserState(BrowserState):download_binding=prior_download_binding,display_recording_owner_id=display_recording_owner_id,reconcile_persistent_init_scripts=True,- sessionless_init_script_registrations=self.sessionless_init_script_registrations,+ sessionless_init_script_registrations=tuple(stale_sessionless_init_script_registrations),)- except Exception:- # The caller abandons this state on failure, so stop the just-started driver too or it leaks.+ except BaseException:+ # Restore the old wrapper before any await so cancellation during replacement cleanup+ # cannot leave the partially initialized replacement published on this state.+ restore_stale_state()await self._run_bounded_detachable(- self.pw.stop(), BROWSER_CLOSE_TIMEOUT, "new Playwright driver shutdown after failed reconnect"+ fresh_pw.stop(), BROWSER_CLOSE_TIMEOUT, "new Playwright driver shutdown after failed reconnect")raise+ # Pre-handoff shutdown can synchronously deliver the stale context's close event while it+ # is still this state's current context, clearing the live list. The replacement replayed+ # the snapshot above, so retain that same durable state for later reconnects.+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations.copy()+ if stop_stale_before_replacement and not retry_stale_shutdown_after_replacement:+ return++ async def stop_stale_driver() -> None:+ await stale_pw.stop()++ stale_driver_shutdown_retry_scheduled = False++ def schedule_stale_driver_shutdown_retry() -> None:+ nonlocal stale_driver_shutdown_retry_scheduled+ if stale_driver_shutdown_retry_scheduled:+ return+ stale_driver_shutdown_retry_scheduled = True+ LOG.warning("Scheduling stale Playwright driver shutdown retry after reconnect")++ async def retry_stale_driver_shutdown() -> None:+ await self._run_bounded_detachable(+ stale_pw.stop(),+ BROWSER_CLOSE_TIMEOUT,+ "retry stale Playwright driver shutdown after reconnect",+ accept_failure_as_completion=True,+ )++ retry_task = asyncio.create_task(retry_stale_driver_shutdown())+ self._own_detached_task(retry_task, "retry stale Playwright driver shutdown after reconnect")++ def retry_failed_stale_driver_shutdown(finished: asyncio.Task[None]) -> None:+ if finished.cancelled() or finished.exception() is not None:+ # Scheduling from completion serializes the calls even when the first stop ignores+ # cancellation, while a late successful completion needs no retry.+ schedule_stale_driver_shutdown_retry()++ stale_driver_shutdown_task = asyncio.create_task(stop_stale_driver())+ stale_driver_shutdown_task.add_done_callback(retry_failed_stale_driver_shutdown)+ await self._run_bounded_detachable(+ stale_driver_shutdown_task,+ BROWSER_CLOSE_TIMEOUT,+ "stale Playwright driver shutdown after guarded reconnect",+ # The replacement is already authoritative. The done callback owns any needed+ # retry, so a stale-driver stop failure must not turn the successful handoff into+ # a reconnect failure.+ accept_failure_as_completion=True,+ )async def close_current_open_page(self) -> bool:try:diff --git a/tests/unit/test_copilot_ask_user.py b/tests/unit/test_copilot_ask_user.pyindex 36aee0594..e5b4b81c3 100644--- a/tests/unit/test_copilot_ask_user.py+++ b/tests/unit/test_copilot_ask_user.py@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.ask_user import (from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.human_input_wait import HumanInputWait, pause_human_inputfrom skyvern.forge.sdk.copilot.output_utils import summarize_tool_result+from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDBfrom skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepositoryfrom skyvern.forge.sdk.schemas.workflow_copilot import CopilotPendingTurn@@ -613,6 +614,55 @@ async def test_invalid_and_duplicate_replies_do_not_call_secret_provider(sqlite_resolver.assert_not_awaited()+async def test_question_reply_persists_raw_secret_detection(sqlite_engine, monkeypatch):+ from skyvern.forge.sdk.routes import workflow_copilot as routes++ _, client, ctx, frames = await setup_question_chat(sqlite_engine, monkeypatch)+ ctx.request_policy = RequestPolicy()+ monkeypatch.setattr(+ routes,+ "_screen_raw_secret_safety",+ AsyncMock(+ return_value=SimpleNamespace(+ status="detected",+ canonical_user_message="https://portal.example.com/login [REDACTED]",+ )+ ),+ )++ async with client:+ task = asyncio.create_task(+ ask_user(+ ctx,+ AskUserArguments.model_validate({"parts": [{"prompt": "Sign-in URL?"}]}),+ "call",+ )+ )+ question = (await asyncio.wait_for(frames.get(), 5))["interactions"][0]+ result = await client.post(+ "/reply",+ json={+ "workflow_copilot_chat_id": ctx.workflow_copilot_chat_id,+ "interaction_id": question["interaction_id"],+ "text": "https://portal.example.com/login password=hunter2",+ },+ )+ await asyncio.wait_for(task, 5)++ assert result.status_code == 200+ assert result.json()["response"] == {+ "answers": [],+ "text": "https://portal.example.com/login [REDACTED]",+ "skipped": False,+ "raw_secret_detected": True,+ }+ assert ctx.request_policy.raw_secret_detected is True+ assert ctx.request_policy.testing_intent == "skip_test"+ assert ctx.request_policy.allow_run_blocks is False+ assert ctx.allow_untested_workflow_draft is True++@pytest.mark.asyncioasync def test_client_checkins_extend_wait_and_abandoned_handler_exits(sqlite_engine, monkeypatch):from skyvern.forge.sdk.db.repositories import workflow_parameters as recordsdiff --git a/tests/unit/test_copilot_credential_pause_reload.py b/tests/unit/test_copilot_credential_pause_reload.pynew file mode 100644index 000000000..37a1e568a--- /dev/null+++ b/tests/unit/test_copilot_credential_pause_reload.py@@ -0,0 +1,203 @@+import asyncio+from datetime import UTC+from types import SimpleNamespace+from unittest.mock import AsyncMock++import pytest++from skyvern.forge import app+from skyvern.forge.sdk.copilot import credential_pause as pauses+from skyvern.forge.sdk.copilot.config import CopilotConfig+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, _ground_user_provided_sites+from skyvern.forge.sdk.copilot.tools.credential_fill import _request_credential+from skyvern.forge.sdk.routes import workflow_copilot as routes+from tests.unit.test_copilot_ask_user import setup_question_chat+from tests.unit.test_copilot_credential_pause import _FakeCache, _make_credential, _make_stream+++async def test_recovery_capability_does_not_keep_a_direct_credential_ask_alive(sqlite_engine, monkeypatch):+ _, _, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)+ monkeypatch.setattr(app, "CACHE", _FakeCache())+ ctx.client_supports_credential_pause = True+ ctx.client_supports_credential_pause_recovery = True+ ctx.credential_recovery_token_digest = pauses.credential_recovery_token_digest("a" * 64)+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)+ ctx.request_policy = RequestPolicy()+ ctx.stream = _make_stream(disconnected=True)++ result = await _request_credential("https://portal.example.com/login", "Login", ctx)++ assert result["status"] == "unanswered"+ assert result["outcome"] == "declined"+ ctx.stream.send.assert_not_awaited()+++async def test_disconnected_card_recovers_through_authenticated_history_and_resumes_same_call(+ sqlite_engine, monkeypatch+):+ repo, client, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)+ cache = _FakeCache()+ monkeypatch.setattr(app, "CACHE", cache)+ monkeypatch.setattr(+ app.DATABASE,+ "credentials",+ SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()])),+ )+ monkeypatch.setattr(pauses, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)+ monkeypatch.setattr(routes, "RECONCILE_ABANDON_AFTER_SECONDS", -1)+ ctx.client_supports_credential_pause = True+ ctx.client_supports_credential_pause_recovery = True+ ctx.credential_recovery_armed = True+ recovery_token = "a" * 64+ ctx.credential_recovery_token_digest = pauses.credential_recovery_token_digest(recovery_token)+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)+ ctx.request_policy = RequestPolicy()+ _ground_user_provided_sites(ctx.request_policy, "https://portal.example.com/login", [])+ ctx.stream = _make_stream(disconnected=True)+ emitted = asyncio.Event()+ ctx.stream.send.side_effect = lambda _: emitted.set()+ client._transport.app.add_api_route(+ "/credential-response", routes.workflow_copilot_credential_response, methods=["POST"]+ )+ async with client:+ invocation = asyncio.create_task(_request_credential("https://portal.example.com/login", "Login", ctx))+ try:+ await asyncio.wait_for(emitted.wait(), 1)+ await asyncio.sleep(0.03)+ assert not invocation.done()+ missing_proof = await client.get(+ "/history", params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id}+ )+ assert missing_proof.json()["pending_credential_requests"] == []+ wrong_proof = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},+ headers={"X-Copilot-Credential-Recovery-Token": "b" * 64},+ )+ assert wrong_proof.json()["pending_credential_requests"] == []+ history = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},+ )+ card = history.json()["pending_credential_requests"][0]+ assert not invocation.done()+ assert card == ctx.stream.send.await_args.args[0].model_dump(mode="json")+ assert card["turn_id"] == "turn"+ foreign = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},+ headers={+ "test-org": "other",+ "X-Copilot-Credential-Recovery-Token": recovery_token,+ },+ )+ assert foreign.json()["pending_credential_requests"] == []+ other_chat = await repo.create_workflow_copilot_chat(organization_id="org", workflow_permanent_id="other")+ other = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": other_chat.workflow_copilot_chat_id},+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},+ )+ assert other.json()["pending_credential_requests"] == []+ body = {+ "workflow_copilot_chat_id": card["workflow_copilot_chat_id"],+ "turn_id": card["turn_id"],+ "resume_token": card["resume_token"],+ "action": "connected",+ "credential_id": "cred_1",+ }+ invalid = await client.post("/credential-response", json={**body, "resume_token": "invalid"})+ assert invalid.status_code == 403+ accepted = await client.post("/credential-response", json=body)+ assert accepted.status_code == 200+ result = await asyncio.wait_for(invocation, 1)+ assert result["status"] == "connected"+ assert result["credential_id"] == "cred_1"+ reloaded = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},+ )+ assert reloaded.json()["pending_credential_requests"] == []+ duplicate = await client.post("/credential-response", json=body)+ assert duplicate.status_code == 409+ finally:+ if not invocation.done():+ invocation.cancel()+ await asyncio.gather(invocation, return_exceptions=True)++[email protected]("ending", ["expired", "cancelled"])+async def test_history_hides_expired_or_cancelled_recoverable_card(sqlite_engine, monkeypatch, ending):+ from datetime import datetime, timedelta++ _, client, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)+ cache = _FakeCache()+ monkeypatch.setattr(app, "CACHE", cache)+ monkeypatch.setattr(pauses, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)+ ctx.client_supports_credential_pause = True+ ctx.client_supports_credential_pause_recovery = True+ ctx.credential_recovery_armed = True+ recovery_token = "a" * 64+ recovery_digest = pauses.credential_recovery_token_digest(recovery_token)+ ctx.credential_recovery_token_digest = recovery_digest+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)+ ctx.request_policy = RequestPolicy()+ _ground_user_provided_sites(ctx.request_policy, "https://portal.example.com/login", [])+ ctx.stream = _make_stream(disconnected=True)+ emitted = asyncio.Event()+ ctx.stream.send.side_effect = lambda _: emitted.set()+ async with client:+ invocation = asyncio.create_task(_request_credential("https://portal.example.com/login", "Login", ctx))+ try:+ await asyncio.wait_for(emitted.wait(), 1)+ card = ctx.stream.send.await_args.args[0]+ if ending == "expired":+ expires_at = datetime.now(UTC) - timedelta(seconds=1)+ cache.store[pauses.credential_pause_active_key("org", ctx.workflow_copilot_chat_id, "turn")] = (+ pauses._encode_active_pause(+ card.resume_token,+ expires_at,+ card=card.model_copy(update={"expires_at": expires_at}),+ recovery_token_digest=recovery_digest,+ )+ )+ else:+ invocation.cancel()+ with pytest.raises(asyncio.CancelledError):+ await invocation+ history = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},+ )+ assert history.json()["pending_credential_requests"] == []+ with pytest.raises(pauses.CredentialPauseRejection) as rejection:+ await pauses.resolve_credential_pause(+ cache,+ organization_id="org",+ workflow_copilot_chat_id=ctx.workflow_copilot_chat_id,+ turn_id="turn",+ resume_token=card.resume_token,+ action="skip",+ credential_id=None,+ )+ assert rejection.value.status_code == (410 if ending == "expired" else 409)+ finally:+ if not invocation.done():+ invocation.cancel()+ await asyncio.gather(invocation, return_exceptions=True)+++async def test_pending_credential_requests_propagates_redis_failure(monkeypatch):+ cache = AsyncMock()+ cache.get.side_effect = RuntimeError("redis unavailable")+ monkeypatch.setattr(app, "CACHE", cache)++ with pytest.raises(RuntimeError, match="redis unavailable"):+ await pauses.pending_credential_requests("org", "chat", ["turn"], "a" * 64)diff --git a/tests/unit/test_copilot_question_url_authority.py b/tests/unit/test_copilot_question_url_authority.pynew file mode 100644index 000000000..e71f4753f--- /dev/null+++ b/tests/unit/test_copilot_question_url_authority.py@@ -0,0 +1,353 @@+from __future__ import annotations++import asyncio+from datetime import UTC, datetime+from types import SimpleNamespace+from unittest.mock import AsyncMock, patch++import pytest+from structlog.testing import capture_logs++from skyvern.forge import app+from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module+from skyvern.forge.sdk.copilot.ask_user import AskUserArguments, QuestionInteraction, ask_user+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig+from skyvern.forge.sdk.copilot.request_policy import (+ QuestionResponseSiteURLSource,+ RequestPolicy,+ UserMessageSiteURLSource,+ _ground_user_provided_sites,+)+from skyvern.forge.sdk.copilot.tools.credential_fill import (+ _credential_fill_origin_grant,+ _request_credential,+)+from skyvern.forge.sdk.copilot.tools.discovery import _user_provided_entry_url+from skyvern.forge.sdk.routes.workflow_copilot import _make_error_narrative_payload, _persist_turn_messages+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChatHistoryMessage,+ WorkflowCopilotChatSender,+ WorkflowCopilotStreamMessageType,+)+from tests.unit.test_copilot_ask_user import setup_question_chat+from tests.unit.test_copilot_credential_pause import (+ _FakeCache,+ _make_credential,+ _make_stream,+)++_LOGIN_URL = "https://portal.example.com/login"++[email protected]("form", ["text", "answer_text"])+async def test_authenticated_question_url_reaches_the_real_card_and_rebuilds_from_history(+ sqlite_engine, monkeypatch: pytest.MonkeyPatch, form: str+) -> None:+ repo, client, ctx, frames = await setup_question_chat(sqlite_engine, monkeypatch)+ ctx.request_policy = RequestPolicy()+ _ground_user_provided_sites(ctx.request_policy, "Repair the login workflow", [])++ async with client:+ waiting = asyncio.create_task(+ ask_user(+ ctx,+ AskUserArguments.model_validate({"parts": [{"prompt": "Paste the sign-in URL"}]}),+ "url-call",+ )+ )+ try:+ required = await asyncio.wait_for(frames.get(), 5)+ question = required["interactions"][0]+ body = {+ "workflow_copilot_chat_id": ctx.workflow_copilot_chat_id,+ "interaction_id": question["interaction_id"],+ }+ if form == "text":+ body["text"] = _LOGIN_URL+ else:+ body["answers"] = [{"part_id": question["parts"][0]["part_id"], "text": _LOGIN_URL}]++ accepted = await client.post("/reply", json=body)+ assert accepted.status_code == 200+ duplicate = await client.post("/reply", json={**body, "text": "https://retry.example.net"})+ assert duplicate.json() == accepted.json()+ await asyncio.wait_for(waiting, 5)++ source = ctx.request_policy.user_site_url_sources[_LOGIN_URL]+ assert source == QuestionResponseSiteURLSource(interaction_id=question["interaction_id"])+ assert ctx.request_policy.user_provided_site_urls == [_LOGIN_URL]+ assert _user_provided_entry_url(ctx) == _LOGIN_URL++ chat = await repo.get_workflow_copilot_chat_by_id("org", ctx.workflow_copilot_chat_id)+ await _persist_turn_messages(+ chat=chat,+ turn_id="turn",+ user_message="Repair the login workflow",+ audio_artifact_id=None,+ user_row_already_persisted=True,+ sender=WorkflowCopilotChatSender.USER,+ assistant_content="Response received",+ global_llm_context=None,+ turn_outcome=None,+ narrative_payload=None,+ )+ saved = await client.get("/history", params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id})+ persisted_history = [+ WorkflowCopilotChatHistoryMessage.model_validate(message) for message in saved.json()["chat_history"]+ ]+ rebuilt = RequestPolicy()+ _ground_user_provided_sites(rebuilt, "", persisted_history)+ assert rebuilt.user_provided_site_urls == ctx.request_policy.user_provided_site_urls+ assert rebuilt.user_site_url_sources == ctx.request_policy.user_site_url_sources++ cache = _FakeCache()+ cache.store[+ credential_pause_module.credential_response_cache_key("org", ctx.workflow_copilot_chat_id, "turn")+ ] = credential_pause_module.encode_credential_response("connected", "cred_1")+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)+ monkeypatch.setattr(+ app,+ "DATABASE",+ SimpleNamespace(+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))+ ),+ )+ ctx.client_supports_credential_pause = True+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)+ ctx.stream = _make_stream()++ card_result = await _request_credential(_LOGIN_URL, "Login required", ctx)++ assert card_result["status"] == "connected"+ assert card_result["credential_id"] == "cred_1"+ card = ctx.stream.send.await_args_list[0].args[0]+ assert card.type is WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED+ assert card.login_page_urls == [_LOGIN_URL]+ finally:+ if not waiting.done():+ waiting.cancel()+ await asyncio.gather(waiting, return_exceptions=True)+++def test_projection_preserves_first_origin_order_and_ignores_non_response_text() -> None:+ interaction = QuestionInteraction.model_validate(+ {+ "interaction_id": "interaction-1",+ "turn_id": "turn-1",+ "tool_call_id": "call-1",+ "parts": [+ {+ "part_id": "part-1",+ "prompt": "Prompt https://prompt.example.net",+ "choices": [{"choice_id": "choice-1", "text": "https://choice.example.net"}],+ },+ {"part_id": "part-2", "prompt": "Second", "choices": []},+ ],+ "status": "resolved",+ "response": {+ "text": "https://question.example.com/top",+ "answers": [+ {"part_id": "part-1", "choice_id": "choice-1", "text": None},+ {"part_id": "part-2", "text": "https://answer.example.org/path"},+ ],+ "skipped": False,+ },+ "created_at": datetime.now(UTC),+ "resolved_at": datetime.now(UTC),+ }+ )+ skipped = interaction.model_copy(+ update={+ "interaction_id": "interaction-skipped",+ "response": interaction.response.model_copy(+ update={"text": "https://skipped.example.net", "skipped": True}+ ),+ }+ )+ unresolved = interaction.model_copy(+ update={"interaction_id": "interaction-pending", "status": "pending", "response": None}+ )+ interrupted = interaction.model_copy(+ update={"interaction_id": "interaction-interrupted", "status": "interrupted", "response": None}+ )+ redacted = interaction.model_copy(+ update={+ "interaction_id": "interaction-redacted",+ "response": interaction.response.model_copy(update={"text": "[REDACTED]", "answers": []}),+ }+ )+ narrative_payload = _make_error_narrative_payload("turn-1", None, "done")+ narrative_payload["questionInteractions"] = [+ interaction.model_dump(mode="json"),+ skipped.model_dump(mode="json"),+ unresolved.model_dump(mode="json"),+ interrupted.model_dump(mode="json"),+ redacted.model_dump(mode="json"),+ ]+ history = [+ WorkflowCopilotChatHistoryMessage(+ sender=WorkflowCopilotChatSender.USER,+ content="Start at https://first.example.com/a",+ created_at=datetime.now(UTC),+ ),+ WorkflowCopilotChatHistoryMessage(+ sender=WorkflowCopilotChatSender.PRODUCT,+ content="https://product.example.net",+ created_at=datetime.now(UTC),+ ),+ WorkflowCopilotChatHistoryMessage(+ sender=WorkflowCopilotChatSender.AI,+ content="https://assistant.example.net",+ narrative_payload=narrative_payload,+ created_at=datetime.now(UTC),+ ),+ ]+ policy = RequestPolicy()++ _ground_user_provided_sites(+ policy,+ "Repeat https://question.example.com/other then https://last.example.edu/report",+ history,+ )++ assert policy.user_provided_site_urls == [+ "https://first.example.com/a",+ "https://question.example.com/top",+ "https://answer.example.org/path",+ "https://last.example.edu/report",+ ]+ assert policy.user_site_url_sources == {+ "https://first.example.com/a": UserMessageSiteURLSource(message_index=1),+ "https://question.example.com/top": QuestionResponseSiteURLSource(interaction_id="interaction-1"),+ "https://answer.example.org/path": QuestionResponseSiteURLSource(interaction_id="interaction-1"),+ "https://last.example.edu/report": UserMessageSiteURLSource(message_index=2),+ }+++def test_immediate_question_projection_appends_without_replacing_composer_urls() -> None:+ policy = RequestPolicy(+ user_provided_site_urls=["https://first.example.com/login"],+ user_site_url_sources={+ "https://first.example.com/login": UserMessageSiteURLSource(message_index=1),+ },+ )+ interaction = QuestionInteraction.model_validate(+ {+ "interaction_id": "interaction-1",+ "turn_id": "turn-1",+ "tool_call_id": "call-1",+ "parts": [{"part_id": "part-1", "prompt": "Backup URL?", "choices": []}],+ "status": "resolved",+ "response": {"text": "https://second.example.net/login", "skipped": False},+ "created_at": datetime.now(UTC),+ "resolved_at": datetime.now(UTC),+ }+ )++ policy.project_question_response_sites(interaction)++ assert policy.user_provided_site_urls == [+ "https://first.example.com/login",+ "https://second.example.net/login",+ ]+ assert policy.user_site_url_sources["https://first.example.com/login"] == UserMessageSiteURLSource(message_index=1)+ assert policy.user_site_url_sources["https://second.example.net/login"] == QuestionResponseSiteURLSource(+ interaction_id="interaction-1"+ )+++def test_question_response_with_detected_raw_secret_grants_no_url_authority() -> None:+ policy = RequestPolicy()+ interaction = QuestionInteraction.model_validate(+ {+ "interaction_id": "interaction-1",+ "turn_id": "turn-1",+ "tool_call_id": "call-1",+ "parts": [{"part_id": "part-1", "prompt": "Sign-in URL?", "choices": []}],+ "status": "resolved",+ "response": {+ "text": "https://portal.example.com/login",+ "skipped": False,+ "raw_secret_detected": True,+ },+ "created_at": datetime.now(UTC),+ "resolved_at": datetime.now(UTC),+ }+ )++ policy.project_question_response_sites(interaction)++ assert policy.user_provided_site_urls == []+ assert policy.user_site_url_sources == {}+ assert policy.raw_secret_detected is True+ assert policy.raw_secret_safety_status == "detected"+ assert policy.testing_intent == "skip_test"+ assert policy.allow_run_blocks is False+ assert policy.allow_missing_credentials_in_draft is True+ assert policy.credential_draft_deferred_explicitly is True+++async def test_question_source_reaches_fill_boundary_without_weakening_origin_guard(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ source = QuestionResponseSiteURLSource(interaction_id="interaction-1")+ policy = RequestPolicy(+ resolved_credentials=[SimpleNamespace(credential_id="cred_1", tested_url=None)],+ current_turn_named_credential_ids={"cred_1"},+ user_provided_site_urls=[_LOGIN_URL],+ user_site_url_sources={_LOGIN_URL: source},+ )+ ctx = SimpleNamespace(+ organization_id="org",+ request_policy=policy,+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ org_credentials_for_turn=None,+ vault_login_uris_by_credential_id={},+ persisted_workflow_yaml="",+ )+ with (+ patch(+ "skyvern.forge.sdk.copilot.tools.credential_fill._live_working_page_url",+ AsyncMock(return_value=_LOGIN_URL),+ ),+ patch(+ "skyvern.forge.sdk.copilot.tools.credential_fill._vault_named_sites",+ AsyncMock(return_value=[]),+ ),+ capture_logs() as logs,+ ):+ grant, error = await _credential_fill_origin_grant(ctx, "cred_1")++ assert error is None+ assert grant is not None+ event = next(log for log in logs if log.get("event") == "copilot credential fill grant")+ assert event["source_kind"] == "question_response"+ assert event["source_interaction_id"] == "interaction-1"+ assert "source_user_message" not in event+ assert "/login" not in str(event)++ with patch(+ "skyvern.forge.sdk.copilot.tools.credential_fill._live_working_page_url",+ AsyncMock(return_value="https://unrelated.example.net/login"),+ ):+ unrelated_grant, unrelated_error = await _credential_fill_origin_grant(ctx, "cred_1")+ assert unrelated_grant is None+ assert unrelated_error is not None+++def test_malformed_persisted_question_record_cannot_supply_origin() -> None:+ payload = _make_error_narrative_payload("turn", None, "done")+ payload["questionInteractions"] = [+ {"interaction_id": "malformed", "status": "resolved", "response": {"text": _LOGIN_URL}}+ ]+ history = [+ WorkflowCopilotChatHistoryMessage(+ sender=WorkflowCopilotChatSender.AI, content="", narrative_payload=payload, created_at=datetime.now(UTC)+ )+ ]+ policy = RequestPolicy()+ _ground_user_provided_sites(policy, "", history)+ assert policy.user_provided_site_urls == []diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex 190b17e9a..00d72bcf5 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -1471,6 +1471,7 @@ async def test_handle_action_reconnects_context_closed_during_download_wait(browser_address=task.browser_address,browser_profile_id=None,browser_session_id=task.browser_session_id,+ stale_context_is_unusable=True,)browser_state.navigate_to_url.assert_awaited_once_with(page=recovered_page, url=page.url)browser_state.set_active_page.assert_awaited_once_with(recovered_page)diff --git a/tests/unit/test_forloop_incremental_persist.py b/tests/unit/test_forloop_incremental_persist.pyindex 038a6545d..4b7580206 100644--- a/tests/unit/test_forloop_incremental_persist.py+++ b/tests/unit/test_forloop_incremental_persist.py@@ -420,6 +420,30 @@ class TestResetBrowserTabsBetweenIterations:loop_blocks=[TaskBlock(label="inner", output_parameter=_make_output_param("inner"))],)+ @pytest.mark.asyncio+ async def test_persistent_loop_browser_lookup_presents_workflow_owner(self) -> None:+ loop_block = self._loop_block()+ browser_state = MagicMock(spec=RealBrowserState)+ context = MagicMock()+ context.browser_session_runnable_id = "wr_owner"+ context.browser_session_runnable_generation_id = "generation_owner"+ with (+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch.object(block_module.skyvern_context, "current", return_value=context),+ ):+ mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state = AsyncMock(return_value=browser_state)++ result = await loop_block._get_loop_browser_state("wr_test", "org_test", "pbs_test")++ assert result is browser_state+ mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with(+ "pbs_test",+ "org_test",+ expected_runnable_id="wr_owner",+ expected_runnable_generation_id="generation_owner",+ workflow_run_id="wr_test",+ )+@pytest.mark.asyncioasync def test_baseline_snapshot_once_and_reset_between_iterations(self) -> None:loop_block = self._loop_block()@@ -478,7 +502,13 @@ class TestResetBrowserTabsBetweenIterations:mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state = AsyncMock(return_value=mock_bs)await loop_block._reset_browser_tabs_for_iteration("wr_test", "org_test", "pbs_test", baseline)reset_browser_tabs.assert_awaited_once_with("org_test")- mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with("pbs_test", "org_test")+ mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with(+ "pbs_test",+ "org_test",+ expected_runnable_id="wr_test",+ expected_runnable_generation_id=None,+ workflow_run_id="wr_test",+ )mock_bs.close_pages_opened_after.assert_awaited_once_with(baseline)@pytest.mark.asynciodiff --git a/tests/unit/test_real_browser_state_reload.py b/tests/unit/test_real_browser_state_reload.pyindex 4851d0684..eb3d018ea 100644--- a/tests/unit/test_real_browser_state_reload.py+++ b/tests/unit/test_real_browser_state_reload.py@@ -207,6 +207,7 @@ async def test_scrape_with_type_normal_no_reload_call() -> None:def _reconnect_state(binding: DownloadBinding) -> RealBrowserState:state = RealBrowserState(pw=AsyncMock(), browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))state.engine_selection = MagicMock()state.engine_selection.start_driver = AsyncMock(return_value=AsyncMock())state.set_working_page = AsyncMock()@@ -239,6 +240,7 @@ async def test_reconnect_keeps_run_dir_binding() -> None:def _reconnect_state_capturing_check(binding: DownloadBinding, captured: dict[str, object]) -> RealBrowserState:state = RealBrowserState(pw=AsyncMock(), browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))state.engine_selection = MagicMock()state.engine_selection.start_driver = AsyncMock(return_value=AsyncMock())state.set_working_page = AsyncMock()diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.pyindex 10f36f479..65dd20960 100644--- a/tests/unit/test_request_logging.py+++ b/tests/unit/test_request_logging.py@@ -96,6 +96,9 @@ class TestIsSensitiveKey:"Cookie","x-api-key","X-Api-Key",+ "X-Copilot-Credential-Recovery-Token",+ "credential_recovery_token",+ "resume_token",# Whole header dicts, masked wholesale because their inner key names are# caller-chosen and therefore unmatchable."extra_http_headers",@@ -1178,3 +1181,55 @@ class TestRawRequestLogLevel:assert log_mock.warning.call_args.args[0] == "api.raw_request"assert log_mock.warning.call_args.kwargs["status_code"] == status_codelog_mock.info.assert_not_called()++[email protected]("status_code", [200, 403, 500])+def test_copilot_recovery_capabilities_never_enter_request_logs(+ log_mock: MagicMock, monkeypatch: pytest.MonkeyPatch, status_code: int+) -> None:+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS_SUCCESSFUL_READS", True)+ app = _make_app()+ recovery_proof = "a" * 64+ selection_token = "synthetic-selection-capability"++ @app.get("/v1/workflow/copilot/chat-history")+ async def history() -> dict:+ if status_code == 403:+ raise HTTPException(status_code=403, detail="Forbidden")+ if status_code == 500:+ raise RuntimeError("Synthetic history failure")+ return {"pending_credential_requests": [{"resume_token": selection_token, "turn_id": "turn"}]}++ response = TestClient(app, raise_server_exceptions=False).get(+ "/v1/workflow/copilot/chat-history",+ headers={"X-Copilot-Credential-Recovery-Token": recovery_proof, "X-Request-ID": "request"},+ )+ assert response.status_code == status_code+ logger = log_mock.info if status_code == 200 else log_mock.warning if status_code == 403 else log_mock.error+ logger.assert_called_once()+ fields = logger.call_args.kwargs+ assert fields["headers"]["x-request-id"] == "request"+ assert "x-copilot-credential-recovery-token" not in fields["headers"]+ assert recovery_proof not in str(log_mock.mock_calls)+ assert selection_token not in str(log_mock.mock_calls)+ if status_code == 200:+ assert response.json()["pending_credential_requests"][0]["resume_token"] == selection_token+ assert json.loads(fields["response_body"])["pending_credential_requests"] == [+ {"resume_token": REDACTED, "turn_id": "turn"}+ ]+++def test_structured_recovery_capabilities_are_redacted_without_masking_turn_identity() -> None:+ assert redact_sensitive_fields(+ {+ "headers": {"X-Copilot-Credential-Recovery-Token": "synthetic-proof"},+ "credential_recovery_token": "synthetic-proof",+ "resume_token": "synthetic-selection",+ "turn_id": "turn",+ }+ ) == {+ "headers": {"X-Copilot-Credential-Recovery-Token": REDACTED},+ "credential_recovery_token": REDACTED,+ "resume_token": REDACTED,+ "turn_id": "turn",+ }diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 97595e182..de4e07f08 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -25,6 +25,7 @@ from skyvern.forge import appfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot.ask_user import QuestionInteraction, QuestionResponsefrom skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalModefrom skyvern.forge.sdk.copilot.build_test_connect_failure import (SUPERSEDED_BY_NEWER_TEST_REASON,@@ -3492,6 +3493,77 @@ async def test_chat_history_marks_abandoned_turn_interrupted_not_cancelled(restore_mock.assert_not_awaited()+async def test_redis_failure_does_not_protect_an_unrelated_stale_turn(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ chat = _make_persisted_chat([_make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 60)])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")+ monkeypatch.setattr(workflow_copilot_route, "credential_pause_is_active", AsyncMock(return_value=None))++ await _load_history()++ assert store.claim_calls == ["turn-a"]+ assert chat.pending_turns == {}+++async def test_redis_failure_briefly_protects_a_recent_question_handoff(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ resolved_at = datetime.now(timezone.utc) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS + 60)+ interaction = QuestionInteraction(+ interaction_id="question-1",+ turn_id="turn-a",+ tool_call_id="call-1",+ parts=[],+ status="resolved",+ response=QuestionResponse(text="https://portal.example.com/login"),+ resolved_at=resolved_at,+ )+ entry = _make_pending_turn(+ "turn-a",+ RECONCILE_ABANDON_AFTER_SECONDS + 120,+ question_interactions=[interaction],+ )+ chat = _make_persisted_chat([entry])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")+ monkeypatch.setattr(workflow_copilot_route, "credential_pause_is_active", AsyncMock(return_value=None))++ await _load_history()++ assert store.claim_calls == []+ assert list(chat.pending_turns) == ["turn-a"]+++async def test_redis_failure_eventually_releases_an_old_question_handoff(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ age = RECONCILE_ABANDON_AFTER_SECONDS + settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 60+ interaction = QuestionInteraction(+ interaction_id="question-1",+ turn_id="turn-a",+ tool_call_id="call-1",+ parts=[],+ status="resolved",+ response=QuestionResponse(text="https://portal.example.com/login"),+ resolved_at=datetime.now(timezone.utc) - timedelta(seconds=age),+ )+ entry = _make_pending_turn("turn-a", age + 60, question_interactions=[interaction])+ chat = _make_persisted_chat([entry])+ store, _ = _install_reconcile_store(monkeypatch, chat)+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")+ monkeypatch.setattr(workflow_copilot_route, "credential_pause_is_active", AsyncMock(return_value=None))++ await _load_history()++ assert store.claim_calls == ["turn-a"]+ assert chat.pending_turns == {}++@pytest.mark.asyncioasync def test_the_recovered_row_carries_what_is_known_and_never_says_why(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/workflow/test_block_browser_state_recovery.py b/tests/unit/workflow/test_block_browser_state_recovery.pyindex 8e58caad0..d7f35d777 100644--- a/tests/unit/workflow/test_block_browser_state_recovery.py+++ b/tests/unit/workflow/test_block_browser_state_recovery.py@@ -425,7 +425,7 @@ async def test_is_connected_false_after_real_driver_stop(tmp_path: Path) -> None@pytest.mark.asyncio-async def test_reconnect_starts_fresh_driver_and_stops_stale_one(monkeypatch: pytest.MonkeyPatch) -> None:+async def test_reconnect_restores_replacement_before_stopping_stale_driver(monkeypatch: pytest.MonkeyPatch) -> None:events: list[str] = []stale_pw = MagicMock()stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))@@ -437,15 +437,15 @@ async def test_reconnect_starts_fresh_driver_and_stops_stale_one(monkeypatch: pyreturn fresh_pwmonkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())-state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild and replay scripts"))monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)await state.reconnect(browser_address="ws://remote-browser")assert state.pw is fresh_pw- assert events == ["stop stale driver", "start fresh driver", "rebuild and replay scripts"]+ assert events == ["start fresh driver", "rebuild and replay scripts", "stop stale driver"]stale_pw.stop.assert_awaited_once()assert check_and_fix.await_args.kwargs["browser_address"] == "ws://remote-browser"@@ -464,14 +464,154 @@ async def test_reconnect_stops_fresh_driver_when_state_rebuild_fails(monkeypatchmonkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=RuntimeError("cdp handshake failed")))with pytest.raises(RuntimeError, match="cdp handshake failed"):await state.reconnect(browser_address="ws://remote-browser")- # A failed rebuild must stop both drivers so it never orphans the freshly started one.+ # The failed replacement is stopped while the still-guarded stale driver remains attached.fresh_pw.stop.assert_awaited_once()- stale_pw.stop.assert_awaited_once()+ stale_pw.stop.assert_not_awaited()+ assert state.pw is stale_pw+++async def test_reconnect_cancellation_restores_still_guarded_stale_state(monkeypatch: pytest.MonkeyPatch) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(return_value=None)+ fresh_pw = MagicMock()+ fresh_pw.stop = AsyncMock(return_value=None)++ class _FakeAsyncPlaywright:+ async def start(self) -> object:+ return fresh_pw++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())++ stale_context = MagicMock()+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))+ stale_listener_browser = MagicMock()+ state._disconnect_listener_browser = stale_listener_browser+ registration = object()+ state.add_sessionless_init_script_registration(registration)++ async def cancel_after_replacement_context_was_published(**_kwargs: object) -> None:+ replacement_context = MagicMock()+ state.browser_context = replacement_context+ state._disconnect_listener_browser = MagicMock()+ state._on_browser_context_closed(replacement_context)+ raise asyncio.CancelledError++ monkeypatch.setattr(+ state,+ "check_and_fix_state",+ AsyncMock(side_effect=cancel_after_replacement_context_was_published),+ )++ with pytest.raises(asyncio.CancelledError):+ await state.reconnect(browser_address="ws://remote-browser")++ assert state.pw is stale_pw+ assert state.browser_context is stale_context+ assert state._disconnect_listener_browser is stale_listener_browser+ assert state.sessionless_init_script_registrations == (registration,)+ fresh_pw.stop.assert_awaited_once()+ stale_pw.stop.assert_not_awaited()+++async def test_successful_pre_handoff_reconnect_preserves_sessionless_registrations_for_next_reconnect(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ initial_pw = MagicMock()+ first_replacement_pw = MagicMock()+ second_replacement_pw = MagicMock()+ start_fresh = AsyncMock(side_effect=[first_replacement_pw, second_replacement_pw])++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))++ initial_context = MagicMock()+ replacement_contexts = [MagicMock(), MagicMock()]+ state = RealBrowserState(pw=initial_pw, browser_context=initial_context)+ state._connection_status = MagicMock(return_value=(True, None))+ registration = object()+ state.add_sessionless_init_script_registration(registration)+ replayed_registrations: list[tuple[object, ...]] = []++ async def stop_current_driver() -> None:+ current_context = state.browser_context+ assert current_context is not None+ state._on_browser_context_closed(current_context)++ initial_pw.stop = AsyncMock(side_effect=stop_current_driver)+ first_replacement_pw.stop = AsyncMock(side_effect=stop_current_driver)++ async def rebuild(**kwargs: object) -> None:+ replayed_registrations.append(kwargs["sessionless_init_script_registrations"]) # type: ignore[arg-type]+ state.browser_context = replacement_contexts.pop(0)++ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=rebuild))++ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)++ assert replayed_registrations == [(registration,), (registration,)]+ assert state.sessionless_init_script_registrations == (registration,)+++async def test_reconnect_restores_stale_state_before_cancelled_replacement_shutdown(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(return_value=None)+ fresh_pw = MagicMock()+ shutdown_started = asyncio.Event()++ async def wait_during_replacement_shutdown() -> None:+ shutdown_started.set()+ await asyncio.Event().wait()++ fresh_pw.stop = AsyncMock(side_effect=wait_during_replacement_shutdown)++ class _FakeAsyncPlaywright:+ async def start(self) -> object:+ return fresh_pw++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())++ stale_context = MagicMock()+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))+ replacement_context = MagicMock()++ async def fail_after_replacement_context_was_published(**_kwargs: object) -> None:+ state.browser_context = replacement_context+ raise RuntimeError("replacement setup failed")++ monkeypatch.setattr(+ state,+ "check_and_fix_state",+ AsyncMock(side_effect=fail_after_replacement_context_was_published),+ )++ reconnect = asyncio.create_task(state.reconnect(browser_address="ws://remote-browser"))+ await asyncio.wait_for(shutdown_started.wait(), timeout=1)+ reconnect.cancel()+ with pytest.raises(asyncio.CancelledError):+ await reconnect++ assert state.pw is stale_pw+ assert state.browser_context is stale_context+ fresh_pw.stop.assert_awaited_once()+ stale_pw.stop.assert_not_awaited()+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)@pytest.mark.asyncio@@ -497,6 +637,7 @@ async def test_reconnect_bounds_fresh_driver_shutdown_when_state_rebuild_fails(monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=RuntimeError("cdp handshake failed")))with pytest.raises(RuntimeError, match="cdp handshake failed"):@@ -504,7 +645,8 @@ async def test_reconnect_bounds_fresh_driver_shutdown_when_state_rebuild_fails(assert stop_started.is_set()fresh_pw.stop.assert_awaited_once()- stale_pw.stop.assert_awaited_once()+ stale_pw.stop.assert_not_awaited()+ assert state.pw is stale_pw@pytest.mark.asyncio@@ -526,17 +668,22 @@ async def test_reconnect_recovers_when_stale_driver_shutdown_has_already_failed(monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)await state.reconnect(browser_address="ws://remote-browser")+ await asyncio.sleep(0)+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)+ await asyncio.sleep(0)+ assert stale_pw.stop.await_count == 2start_fresh.assert_awaited_once()check_and_fix.assert_awaited_once()+ assert state._detached_teardown_tasks == set()@pytest.mark.asyncio-async def test_reconnect_does_not_overlap_driver_when_stale_connection_may_still_be_live(+async def test_reconnect_refuses_to_overlap_driver_when_stale_connection_may_still_be_live(monkeypatch: pytest.MonkeyPatch,) -> None:stale_pw = MagicMock()- stale_pw.stop = AsyncMock(side_effect=RuntimeError("stale driver still attached"))+ stale_pw.stop = AsyncMock()start_fresh = AsyncMock()class _FakeAsyncPlaywright:@@ -545,28 +692,261 @@ async def test_reconnect_does_not_overlap_driver_when_stale_connection_may_stillmonkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())- state._connection_status = MagicMock(side_effect=[(True, None), (False, "browser_context_missing")])+ state._connection_status = MagicMock(return_value=(True, None))check_and_fix = AsyncMock()monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)- with pytest.raises(RuntimeError, match="stale Playwright driver"):+ with pytest.raises(RuntimeError, match="may still be live"):await state.reconnect(browser_address="ws://remote-browser")+ stale_pw.stop.assert_not_awaited()start_fresh.assert_not_awaited()check_and_fix.assert_not_awaited()- # reconnect clears the context before attempting stop. A retry must treat that missing context- # as unknown rather than proof that the first driver detached.- with pytest.raises(RuntimeError, match="stale Playwright driver"):- await state.reconnect(browser_address="ws://remote-browser")+[email protected]("disconnect_reason", ["browser_context_missing", "browser_context_closed"])+async def test_reconnect_stops_known_unusable_context_before_starting_replacement(+ monkeypatch: pytest.MonkeyPatch,+ disconnect_reason: str,+) -> None:+ events: list[str] = []+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, disconnect_reason))+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild"))+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser")++ assert events == ["stop stale driver", "start fresh driver", "rebuild"]+++async def test_reconnect_stops_driver_after_connection_probe_failure_before_starting_replacement(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[str] = []+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))++ stale_context = MagicMock()+ stale_context._impl_obj = None+ stale_context.browser.is_connected.side_effect = RuntimeError("connection probe failed")+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild"))+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser")++ assert events == ["stop stale driver", "start fresh driver", "rebuild"]+++async def test_reconnect_keeps_guard_through_connection_probe_failure_handoff(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[str] = []+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))++ stale_context = MagicMock()+ stale_context._impl_obj = None+ stale_context.browser.is_connected.side_effect = RuntimeError("connection probe failed")+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild and restore guard"))+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser")++ assert events == ["start fresh driver", "rebuild and restore guard", "stop stale driver"]+++async def test_reconnect_stops_explicitly_unusable_live_context_before_starting_replacement(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[str] = []+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(True, None))+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild"))+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)++ assert events == ["stop stale driver", "start fresh driver", "rebuild"]+++async def test_reconnect_replaces_guarded_live_context_before_stopping_stale_driver(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[str] = []+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(True, None))+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild and replay scripts"))+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)++ assert events == ["start fresh driver", "rebuild and replay scripts", "stop stale driver"]+++async def test_reconnect_recovers_when_unusable_live_driver_stop_raises(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=[RuntimeError("driver still attached"), None])+ start_fresh = AsyncMock()++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(True, None))+ check_and_fix = AsyncMock()+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)assert stale_pw.stop.await_count == 2+ start_fresh.assert_awaited_once()+ check_and_fix.assert_awaited_once()+++async def test_reconnect_aborts_when_unguarded_unusable_driver_shutdown_times_out(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def hang_during_stop() -> None:+ state._on_browser_context_closed(stale_context)+ await asyncio.Event().wait()++ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=hang_during_stop)+ start_fresh = AsyncMock()++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))++ stale_context = MagicMock()+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)+ state._connection_status = MagicMock(return_value=(True, None))+ registration = object()+ state.add_sessionless_init_script_registration(registration)+ check_and_fix = AsyncMock()+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ with pytest.raises(RuntimeError, match="Failed to stop unusable stale Playwright driver"):+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)+start_fresh.assert_not_awaited()check_and_fix.assert_not_awaited()+ assert state.sessionless_init_script_registrations == (registration,)+++async def test_reconnect_cancellation_during_pre_handoff_shutdown_restores_sessionless_registrations(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stop_started = asyncio.Event()+ release_stop = asyncio.Event()++ async def stop_after_context_close() -> None:+ stop_started.set()+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ await release_stop.wait()+ state._on_browser_context_closed(stale_context)++ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=stop_after_context_close)+ start_fresh = AsyncMock()++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))++ stale_context = MagicMock()+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)+ state._connection_status = MagicMock(return_value=(True, None))+ registration = object()+ state.add_sessionless_init_script_registration(registration)+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())++ reconnect = asyncio.create_task(+ state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)+ )+ await asyncio.wait_for(stop_started.wait(), timeout=1)+ reconnect.cancel()++ with pytest.raises(asyncio.CancelledError):+ await reconnect++ start_fresh.assert_not_awaited()+ assert state.sessionless_init_script_registrations == (registration,)++ release_stop.set()+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)+ assert state.sessionless_init_script_registrations == (registration,)@pytest.mark.asyncio-async def test_reconnect_bounds_stale_driver_shutdown_before_starting_fresh_driver(+async def test_reconnect_bounds_stale_driver_shutdown_after_guarded_replacement(monkeypatch: pytest.MonkeyPatch,) -> None:stop_started = asyncio.Event()@@ -586,15 +966,236 @@ async def test_reconnect_bounds_stale_driver_shutdown_before_starting_fresh_drivmonkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))check_and_fix = AsyncMock()monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)- with pytest.raises(RuntimeError, match="stale Playwright driver"):- await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)+ await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)+ await asyncio.sleep(0)+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)assert stop_started.is_set()- start_fresh.assert_not_awaited()- check_and_fix.assert_not_awaited()+ assert stale_pw.stop.await_count == 2+ start_fresh.assert_awaited_once()+ check_and_fix.assert_awaited_once()+ assert state._detached_teardown_tasks == set()+++async def test_reconnect_does_not_retry_until_cancellation_resistant_stale_shutdown_completes(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ release_stop = asyncio.Event()++ async def cancellation_resistant_stop() -> None:+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ await release_stop.wait()++ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=cancellation_resistant_stop)+ start_fresh = AsyncMock()++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())++ await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)+ await asyncio.sleep(0.02)++ # The first stop remains owned without racing a second stop.+ assert stale_pw.stop.await_count == 1+ assert len(state._detached_teardown_tasks) == 1++ release_stop.set()+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)+ await asyncio.sleep(0)+ assert state._detached_teardown_tasks == set()+++async def test_reconnect_retries_after_cancellation_resistant_stale_shutdown_eventually_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ release_stop = asyncio.Event()+ stop_attempts = 0++ async def cancellation_resistant_stop() -> None:+ nonlocal stop_attempts+ stop_attempts += 1+ if stop_attempts != 1:+ return+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ await release_stop.wait()+ raise RuntimeError("late shutdown failure")++ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=cancellation_resistant_stop)+ start_fresh = AsyncMock()++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())++ await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)+ await asyncio.sleep(0.02)+ assert stale_pw.stop.await_count == 1++ release_stop.set()++ async def wait_for_retry() -> None:+ while stale_pw.stop.await_count < 2:+ await asyncio.sleep(0)++ await asyncio.wait_for(wait_for_retry(), timeout=0.1)+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)+ await asyncio.sleep(0)++ assert stale_pw.stop.await_count == 2+ assert state._detached_teardown_tasks == set()+++async def test_reconnect_retries_post_replacement_stale_driver_stop_exception(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=[RuntimeError("driver still attached"), None])+ fresh_pw = MagicMock()+ start_fresh = AsyncMock(return_value=fresh_pw)++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(True, None))+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())++ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)+ await asyncio.gather(*list(state._detached_teardown_tasks))+ await asyncio.sleep(0)++ assert state.pw is fresh_pw+ assert stale_pw.stop.await_count == 2+ assert state._detached_teardown_tasks == set()+++async def test_reconnect_cancellation_during_post_replacement_shutdown_retains_retry(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ first_stop_started = asyncio.Event()+ stop_attempts = 0++ async def cancellation_sensitive_stop() -> None:+ nonlocal stop_attempts+ stop_attempts += 1+ if stop_attempts == 1:+ first_stop_started.set()+ await asyncio.Event().wait()++ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(side_effect=cancellation_sensitive_stop)+ fresh_pw = MagicMock()+ start_fresh = AsyncMock(return_value=fresh_pw)++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(True, None))+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())++ reconnect = asyncio.create_task(+ state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)+ )+ await asyncio.wait_for(first_stop_started.wait(), timeout=1)+ reconnect.cancel()+ with pytest.raises(asyncio.CancelledError):+ await reconnect+ await asyncio.sleep(0)+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)+ await asyncio.sleep(0)++ assert state.pw is fresh_pw+ assert stale_pw.stop.await_count == 2+ assert state._detached_teardown_tasks == set()+++async def test_reconnect_cancellation_before_stale_shutdown_starts_wakes_retry(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock()+ fresh_pw = MagicMock()+ start_fresh = AsyncMock(return_value=fresh_pw)++ class _FakeAsyncPlaywright:+ start = start_fresh++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ state._connection_status = MagicMock(return_value=(True, None))+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())+ bounded_calls = 0++ async def cancel_first_phase_before_start(+ phase: object,+ _timeout: float,+ _description: str,+ *,+ accept_failure_as_completion: bool = False,+ ) -> bool:+ nonlocal bounded_calls+ del accept_failure_as_completion+ bounded_calls += 1+ if bounded_calls == 1:+ phase_task = asyncio.ensure_future(phase) # type: ignore[arg-type]+ phase_task.cancel()+ await asyncio.gather(phase_task, return_exceptions=True)+ raise asyncio.CancelledError+ await phase # type: ignore[misc]+ return True++ monkeypatch.setattr(state, "_run_bounded_detachable", cancel_first_phase_before_start)++ with pytest.raises(asyncio.CancelledError):+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)+ detached = list(state._detached_teardown_tasks)+ done, pending = await asyncio.wait(detached, timeout=0.1)+ for task in pending:+ task.cancel()+ await asyncio.gather(*pending, return_exceptions=True)+ await asyncio.sleep(0)++ assert state.pw is fresh_pw+ assert len(done) == len(detached)+ stale_pw.stop.assert_awaited_once()+ assert state._detached_teardown_tasks == set()@pytest.mark.asyncio