Skip to content

Automate browser based workflows with AI

SKY-15740 Use question-response URLs for credential handoff (#8507)

Andrew Neilson6a7472179972
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
index 33d3cac16..2ff0523f3 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
@@ -6,14 +6,21 @@ import {
render,
screen,
waitFor,
+ within,
} from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import type { WorkflowCopilotCredentialRequiredUpdate } from "./workflowCopilotTypes";
+import type {
+ QuestionInteraction,
+ WorkflowCopilotCredentialRequiredUpdate,
+} from "./workflowCopilotTypes";
+import { ensureCredentialRecoveryToken } from "./credentialRecovery";
type StreamBody = {
message: string;
supports_credential_pause?: boolean;
+ supports_credential_pause_recovery?: boolean;
+ credential_recovery_token?: string;
};
type StreamCall = {
body: StreamBody;
@@ -53,6 +60,9 @@ const {
chat_history: [] as unknown[],
proposed_workflow: null as Record<string, unknown> | null,
auto_accept: false,
+ pending_credential_requests:
+ [] as WorkflowCopilotCredentialRequiredUpdate[],
+ question_interactions: [] as QuestionInteraction[],
},
};
const creds = {
@@ -265,7 +275,10 @@ const saveData = {
};
vi.mock("@/store/WorkflowHasChangesStore", () => ({
- useWorkflowHasChangesStore: () => ({ getSaveData: () => saveData }),
+ useWorkflowHasChangesStore: Object.assign(
+ () => ({ getSaveData: () => saveData }),
+ { getState: () => ({ hasChanges: true }) },
+ ),
}));
vi.mock("@/routes/workflows/hooks/useWorkflowRunQuery", () => ({
@@ -428,6 +441,7 @@ function credentialsGets() {
}
beforeEach(() => {
+ sessionStorage.clear();
HTMLElement.prototype.scrollIntoView = vi.fn();
HTMLElement.prototype.scrollTo = vi.fn();
streamCalls.length = 0;
@@ -447,6 +461,8 @@ beforeEach(() => {
chat_history: [],
proposed_workflow: null,
auto_accept: false,
+ pending_credential_requests: [],
+ question_interactions: [],
};
});
@@ -484,11 +500,113 @@ describe("WorkflowCopilotChat — activity log", () => {
});
describe("WorkflowCopilotChat — credential card wiring", () => {
- it("sends supports_credential_pause on the request", async () => {
+ it("keeps a credential pause recovered from history in ownership", async () => {
+ ensureCredentialRecoveryToken("wpid_1");
+ historyResponse.data.pending_credential_requests = [credentialFrame()];
+ credentialsData.current = [
+ {
+ credential_id: "cred-hn",
+ name: "HN Login",
+ tested_url: "https://news.ycombinator.com/login",
+ },
+ ];
+
+ await renderChat();
+ expect(await screen.findByRole("combobox")).toBeTruthy();
+
+ await submit("start a second turn");
+
+ expect(postStreaming).not.toHaveBeenCalled();
+ });
+
+ it("recovers a credential card without SSE and resumes its original turn", async () => {
+ const recoveryToken = ensureCredentialRecoveryToken("wpid_1");
+ const question: QuestionInteraction = {
+ interaction_id: "question-1",
+ turn_id: "turn-1",
+ tool_call_id: "ask-1",
+ parts: [{ part_id: "url", prompt: "Sign-in URL?", choices: [] }],
+ status: "pending",
+ response: null,
+ created_at: new Date().toISOString(),
+ resolved_at: null,
+ };
+ historyResponse.data.question_interactions = [question];
+ credentialsData.current = [
+ {
+ credential_id: "cred-hn",
+ name: "HN Login",
+ tested_url: "https://news.ycombinator.com/login",
+ },
+ ];
+ await renderChat();
+ expect(apiGet).toHaveBeenCalledWith(
+ "/workflow/copilot/chat-history",
+ expect.objectContaining({
+ headers: {
+ "X-Copilot-Credential-Recovery-Token": recoveryToken,
+ },
+ }),
+ );
+ const resolved: QuestionInteraction = {
+ ...question,
+ status: "resolved",
+ response: { text: "https://news.ycombinator.com/login" },
+ resolved_at: new Date().toISOString(),
+ };
+ sansApiPost.mockResolvedValueOnce({ data: resolved });
+ historyResponse.data.question_interactions = [resolved];
+ historyResponse.data.pending_credential_requests = [credentialFrame()];
+ fireEvent.change(await screen.findByLabelText("Your response"), {
+ target: { value: "https://news.ycombinator.com/login" },
+ });
+ fireEvent.click(
+ within(screen.getByRole("group", { name: "Question parts" })).getByRole(
+ "button",
+ { name: "Send" },
+ ),
+ );
+ const recoveredCredentialPicker = await screen.findByRole(
+ "combobox",
+ {},
+ { timeout: 10000 },
+ );
+ await submit("start a different turn");
+ expect(postStreaming).not.toHaveBeenCalled();
+ fireEvent.click(recoveredCredentialPicker);
+ fireEvent.click(await screen.findByRole("button", { name: "HN Login" }));
+ await waitFor(() => expect(credentialResponsePosts()).toHaveLength(1));
+ expect(credentialResponsePosts()[0]![1]).toMatchObject({
+ turn_id: "turn-1",
+ workflow_copilot_chat_id: "chat-1",
+ resume_token: "rt-abc",
+ action: "connected",
+ credential_id: "cred-hn",
+ });
+ expect(postStreaming).not.toHaveBeenCalled();
+ historyResponse.data.pending_credential_requests = [];
+ historyResponse.data.chat_history = [
+ {
+ sender: "ai",
+ content: "Recovered continuation completed",
+ created_at: new Date().toISOString(),
+ narrative_payload: { turnId: "turn-1", terminal: "response" },
+ },
+ ];
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1), {
+ timeout: 10000,
+ });
+ }, 15000);
+
+ it("sends a recoverable credential capability with the request", async () => {
await renderChat();
await submit("build me a workflow");
await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
expect(streamCalls[0]!.body.supports_credential_pause).toBe(true);
+ expect(streamCalls[0]!.body.supports_credential_pause_recovery).toBe(true);
+ expect(streamCalls[0]!.body.credential_recovery_token).toMatch(
+ /^[0-9a-f]{64}$/,
+ );
});
it("renders the inline-pause card when a credential_required frame arrives", async () => {
@@ -509,6 +627,21 @@ describe("WorkflowCopilotChat — credential card wiring", () => {
).toBeTruthy();
});
+ it("keeps a dropped credential-pause turn owned while recovery polls", async () => {
+ await renderChat();
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+ await act(async () => {
+ streamCalls[0]!.onMessage(turnStart());
+ streamCalls[0]!.onMessage(credentialFrame());
+ streamCalls[0]!.reject(new Error("stream disconnected"));
+ });
+
+ await submit("start a second turn");
+
+ expect(postStreaming).toHaveBeenCalledTimes(1);
+ });
+
it("skip POSTs a credential-response with action skip", async () => {
await renderChat();
await submit("build me a workflow");
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx
index ea5aaf1e2..706aba9be 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx
@@ -10,11 +10,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { FeatureFlagContext } from "@/hooks/useFeatureFlag";
-import type { QuestionInteraction } from "./workflowCopilotTypes";
+import type {
+ QuestionInteraction,
+ WorkflowCopilotCredentialRequiredUpdate,
+} from "./workflowCopilotTypes";
type HistoryData = {
question_interactions?: QuestionInteraction[];
pending_question_cancel_token?: string | null;
+ pending_credential_requests?: WorkflowCopilotCredentialRequiredUpdate[];
workflow_copilot_chat_id: string | null;
chat_history: unknown[];
proposed_workflow: Record<string, unknown> | null;
@@ -712,6 +716,36 @@ describe("WorkflowCopilotChat — recovery poll after a non-terminal stream clos
expect(renderedText()).toContain(capturedAiText);
});
+ it("claims ownership when recovery discovers a credential pause", async () => {
+ await startTurn();
+ await closeStreamWithoutTerminal();
+ await advance(2_000);
+ await resolveNextHistory(
+ historyData({
+ pending_credential_requests: [
+ {
+ type: "credential_required",
+ turn_id: turnId,
+ workflow_copilot_chat_id: "chat-1",
+ resume_token: "resume-token",
+ reason: "workflow_credential_inputs_unbound",
+ message: "",
+ login_page_urls: ["https://example.com/login"],
+ credential_refs: [],
+ timeout_seconds: 300,
+ expires_at: new Date(Date.now() + 300_000).toISOString(),
+ timestamp: new Date().toISOString(),
+ },
+ ],
+ }),
+ );
+
+ await submit("another one");
+ await advance(10);
+
+ expect(streamCalls.length).toBe(1);
+ });
+
it("does not re-read history when the stream ends on a terminal frame", async () => {
await startTurn();
await emitTerminalResponse();
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 224078a7d..bfc90cb22 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -97,6 +97,10 @@ import { QuestionPartsCard } from "./cards/QuestionPartsCard";
import { WorkPlanCard } from "./cards/WorkPlanCard";
import { nextAnsweringMessage, previousAskingMessage } from "./cardAdjacency";
import { composerPlaceholder } from "./composerPlaceholder";
+import {
+ ensureCredentialRecoveryToken,
+ readCredentialRecoveryHistory,
+} from "./credentialRecovery";
import { connectedAccountChoiceLabel } from "./cards/connectedAccountChoiceLabel";
import { shouldShowDiffCard } from "./cards/DiffCard";
import { ReviewGateCard, getReviewGateVerdict } from "./cards/ReviewGateCard";
@@ -1039,6 +1043,9 @@ export function WorkflowCopilotChat({
// turn_start and at every terminal so a dead resume_token can never render.
const [livePauseFrame, setLivePauseFrame] =
useState<WorkflowCopilotCredentialRequiredUpdate | null>(null);
+ const [recoveredPauseFrames, setRecoveredPauseFrames] = useState<
+ WorkflowCopilotCredentialRequiredUpdate[]
+ >([]);
// Local credential-card resolutions keyed by turn_id: live pauses after a
// successful resume POST, and terminal-mode connect/skip (which never POST).
// name is captured at connect so the receipt keeps showing it after the turn
@@ -1125,6 +1132,9 @@ export function WorkflowCopilotChat({
const cancelInFlightController = useRef<AbortController | null>(null);
const recoveryPolls = useRef(new Map<string, () => void>());
const recoveryGeneration = useRef(0);
+ // A turn resumed from saved human input has no live SSE controller, but it
+ // still owns the composer until history proves that continuation finished.
+ const recoveredTurnOwnerRef = useRef<string | null>(null);
// The poll is declared before the reconcile it calls on a recovered row.
const reconcileCanonicalWorkflowRef = useRef<(() => Promise<void>) | null>(
null,
@@ -1438,6 +1448,9 @@ export function WorkflowCopilotChat({
location.state,
],
);
+ const recoverCredentialTurn = useRef<
+ (chatId: string, turnId: string) => void
+ >(() => {});
const respondToCredentialPause = useCallback(
async (
frame: WorkflowCopilotCredentialRequiredUpdate,
@@ -1447,6 +1460,8 @@ export function WorkflowCopilotChat({
) => {
if (credentialResponseInFlight.current) return;
credentialResponseInFlight.current = true;
+ const generation = recoveryGeneration.current;
+ const chatId = workflowCopilotChatIdRef.current;
try {
// Copilot routes live on base_router (no /api/v1 prefix), like cancel.
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -1457,6 +1472,19 @@ export function WorkflowCopilotChat({
action,
credential_id: action === "connected" ? credentialId : undefined,
});
+ if (
+ recoveryGeneration.current !== generation ||
+ workflowCopilotChatIdRef.current !== chatId ||
+ chatId !== frame.workflow_copilot_chat_id
+ ) {
+ return;
+ }
+ if (!streamingAbortController.current) {
+ recoverCredentialTurn.current(
+ frame.workflow_copilot_chat_id,
+ frame.turn_id,
+ );
+ }
setCredentialResolutions((prev) =>
withCappedResolution(
prev,
@@ -1741,6 +1769,7 @@ export function WorkflowCopilotChat({
streamingAbortController.current = null;
inFlightRef.current = false;
setIsLoading(false);
+ setRecoveredPauseFrames([]);
setQuestionInteractions([]);
setQuestionCancelToken(null);
stopRecoveryPolls();
@@ -1767,6 +1796,7 @@ export function WorkflowCopilotChat({
data: WorkflowCopilotChatHistoryResponse,
carryForwardLifecycle = true,
) => {
+ setRecoveredPauseFrames(data.pending_credential_requests ?? []);
setQuestionInteractions(data.question_interactions ?? []);
setQuestionCancelToken(data.pending_question_cancel_token ?? null);
const historyMessages = data.chat_history.map((message, index) => ({
@@ -1832,10 +1862,11 @@ export function WorkflowCopilotChat({
recoveryGeneration.current += 1;
recoveryPolls.current.forEach((stop) => stop());
recoveryPolls.current.clear();
+ recoveredTurnOwnerRef.current = null;
}, []);
const startRecoveryPoll = useCallback(
- (chatId: string | null, turnId: string) => {
+ (chatId: string | null, turnId: string, ownsTurn = false) => {
if (!workflowPermanentId) {
return;
}
@@ -1875,6 +1906,10 @@ export function WorkflowCopilotChat({
if (recoveryPolls.current.get(turnId) === finish) {
recoveryPolls.current.delete(turnId);
}
+ if (recoveredTurnOwnerRef.current === turnId) {
+ recoveredTurnOwnerRef.current = null;
+ setIsLoading(false);
+ }
}
// Ending without the turn's row leaves a notice describing work that has
@@ -1892,6 +1927,10 @@ export function WorkflowCopilotChat({
recoveryPolls.current.get(turnId)?.();
recoveryPolls.current.set(turnId, finish);
+ if (ownsTurn) {
+ recoveredTurnOwnerRef.current = turnId;
+ setIsLoading(true);
+ }
// A read that never returns leaves schedule() unreached, so the budget
// needs a timer of its own or the deadline can never fire.
deadlineTimer = setTimeout(giveUp, RECOVERY_POLL_BUDGET_MS);
@@ -1900,8 +1939,11 @@ export function WorkflowCopilotChat({
clearTimer();
// finish() aborts the in-flight read, which surfaces in tick's catch;
// without this the rejection would reschedule a stopped poll.
- if (stopped || Date.now() >= deadline) {
- finish();
+ if (stopped) {
+ return;
+ }
+ if (Date.now() >= deadline) {
+ giveUp();
return;
}
const delay = RECOVERY_POLL_DELAYS_MS[step] ?? RECOVERY_POLL_STEADY_MS;
@@ -1922,15 +1964,17 @@ export function WorkflowCopilotChat({
const client = await getClient(credentialGetter, "sans-api-v1");
const controller = new AbortController();
inFlight = controller;
- const response = await client.get<WorkflowCopilotChatHistoryResponse>(
- "/workflow/copilot/chat-history",
- {
- params: chatId
- ? { workflow_copilot_chat_id: chatId }
- : { workflow_permanent_id: workflowPermanentId },
- signal: controller.signal,
- },
- );
+ const response =
+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(
+ client,
+ workflowPermanentId,
+ {
+ params: chatId
+ ? { workflow_copilot_chat_id: chatId }
+ : { workflow_permanent_id: workflowPermanentId },
+ signal: controller.signal,
+ },
+ );
if (inFlight === controller) {
inFlight = null;
}
@@ -1944,6 +1988,20 @@ export function WorkflowCopilotChat({
finish();
return;
}
+ const pendingCredentialRequests =
+ response.data.pending_credential_requests ?? [];
+ setRecoveredPauseFrames(pendingCredentialRequests);
+ // The backend can persist the pause before its SSE frame reaches the
+ // browser. Once history proves this poll owns a credential wait, it
+ // must block another turn just like a poll armed from the live frame.
+ if (
+ !ownsTurn &&
+ pendingCredentialRequests.some((item) => item.turn_id === turnId)
+ ) {
+ ownsTurn = true;
+ recoveredTurnOwnerRef.current = turnId;
+ setIsLoading(true);
+ }
const pendingQuestion = response.data.question_interactions?.find(
(item) => item.turn_id === turnId && item.status === "pending",
);
@@ -2040,6 +2098,16 @@ export function WorkflowCopilotChat({
[applyHistoryResponse, credentialGetter, workflowPermanentId],
);
+ const adoptRecoveredCredentialPause = useCallback(
+ (data: WorkflowCopilotChatHistoryResponse) => {
+ const pendingRequests = data.pending_credential_requests ?? [];
+ const pending = pendingRequests[pendingRequests.length - 1];
+ if (!pending) return;
+ startRecoveryPoll(data.workflow_copilot_chat_id, pending.turn_id, true);
+ },
+ [startRecoveryPoll],
+ );
+
const loadChatInPlace = useCallback(
async (chatId: string) => {
if (!workflowPermanentId) return;
@@ -2061,16 +2129,19 @@ export function WorkflowCopilotChat({
repin();
try {
const client = await getClient(credentialGetter, "sans-api-v1");
- const response = await client.get<WorkflowCopilotChatHistoryResponse>(
- "/workflow/copilot/chat-history",
- {
- params: {
- workflow_permanent_id: workflowPermanentId,
- workflow_copilot_chat_id: chatId,
+ const response =
+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(
+ client,
+ workflowPermanentId,
+ {
+ params: {
+ workflow_permanent_id: workflowPermanentId,
+ workflow_copilot_chat_id: chatId,
+ },
},
- },
- );
+ );
applyHistoryResponse(response.data, false);
+ adoptRecoveredCredentialPause(response.data);
// Mark history loaded for this workflow so the mount effect won't reload
// the latest chat over the one the user just selected.
historyLoadedForRef.current = workflowPermanentId;
@@ -2085,12 +2156,16 @@ export function WorkflowCopilotChat({
credentialGetter,
workflowPermanentId,
applyHistoryResponse,
+ adoptRecoveredCredentialPause,
stopRecoveryPolls,
repin,
discardQueuedPrompt,
],
);
+ recoverCredentialTurn.current = (chatId, turnId) =>
+ startRecoveryPoll(chatId, turnId, true);
+
const handleSelectHistoryChat = useCallback(
(chat: WorkflowCopilotChatSummary) => {
if (chat.workflow_copilot_chat_id === workflowCopilotChatIdRef.current) {
@@ -2291,12 +2366,14 @@ export function WorkflowCopilotChat({
return null;
}
const client = await getClient(credentialGetter, "sans-api-v1");
- const response = await client.get<WorkflowCopilotChatHistoryResponse>(
- "/workflow/copilot/chat-history",
- {
- params: { workflow_permanent_id: workflowPermanentId },
- },
- );
+ const response =
+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(
+ client,
+ workflowPermanentId,
+ {
+ params: { workflow_permanent_id: workflowPermanentId },
+ },
+ );
const latestChatId = response.data.workflow_copilot_chat_id ?? null;
setWorkflowCopilotChatId(latestChatId);
return latestChatId;
@@ -2689,9 +2766,19 @@ export function WorkflowCopilotChat({
if (historyLoadedForRef.current === workflowPermanentId) {
return;
}
- // Reached only when this workflow's transcript is about to replace another's,
- // so a poll armed against the outgoing chat must not apply history here.
+ const isWorkflowSwitch = historyLoadedForRef.current !== null;
+ // A poll armed against the outgoing chat must not apply history to the new
+ // workflow. The state reset is only needed after a workflow was loaded;
+ // writing fresh empty arrays on first mount can restart this async effect
+ // before its history request records completion.
stopRecoveryPolls();
+ if (isWorkflowSwitch) {
+ setRecoveredPauseFrames([]);
+ setQuestionInteractions([]);
+ setQuestionCancelToken(null);
+ setWorkflowCopilotChatId(null);
+ workflowCopilotChatIdRef.current = null;
+ }
let isMounted = true;
@@ -2700,16 +2787,20 @@ export function WorkflowCopilotChat({
repin();
try {
const client = await getClient(credentialGetter, "sans-api-v1");
- const response = await client.get<WorkflowCopilotChatHistoryResponse>(
- "/workflow/copilot/chat-history",
- {
- params: { workflow_permanent_id: workflowPermanentId },
- },
- );
+ const response =
+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(
+ client,
+ workflowPermanentId,
+ {
+ params: { workflow_permanent_id: workflowPermanentId },
+ },
+ { retryTransientFailure: true },
+ );
if (!isMounted) return;
applyHistoryResponse(response.data);
+ adoptRecoveredCredentialPause(response.data);
historyLoadedForRef.current = workflowPermanentId;
} catch (error) {
console.error("Failed to load chat history:", error);
@@ -2727,6 +2818,7 @@ export function WorkflowCopilotChat({
};
}, [
credentialGetter,
+ adoptRecoveredCredentialPause,
repin,
stopRecoveryPolls,
updateQueuedPrompt,
@@ -2923,8 +3015,9 @@ export function WorkflowCopilotChat({
),
);
setIsLoading(true);
- if (!streamingAbortController.current)
- startRecoveryPoll(workflowCopilotChatId, interaction.turn_id);
+ if (!streamingAbortController.current) {
+ startRecoveryPoll(workflowCopilotChatId, interaction.turn_id, true);
+ }
return true;
} catch {
toast({
@@ -3046,12 +3139,14 @@ export function WorkflowCopilotChat({
const refresh = async () => {
try {
const client = await getClient(credentialGetter, "sans-api-v1");
- const { data } = await client.get<WorkflowCopilotChatHistoryResponse>(
- "/workflow/copilot/chat-history",
- {
- params: { workflow_copilot_chat_id: workflowCopilotChatId },
- },
- );
+ const { data } =
+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(
+ client,
+ workflowPermanentId,
+ {
+ params: { workflow_copilot_chat_id: workflowCopilotChatId },
+ },
+ );
if (disposed) return;
const interactions = data.question_interactions ?? [];
setQuestionInteractions(interactions);
@@ -3064,8 +3159,7 @@ export function WorkflowCopilotChat({
.reverse()
.find((item) => item.status === "resolved");
if (resolved) {
- setIsLoading(true);
- startRecoveryPoll(workflowCopilotChatId, resolved.turn_id);
+ startRecoveryPoll(workflowCopilotChatId, resolved.turn_id, true);
} else {
setIsLoading(false);
inFlightRef.current = false;
@@ -3083,6 +3177,7 @@ export function WorkflowCopilotChat({
}, [
hasPendingQuestion,
workflowCopilotChatId,
+ workflowPermanentId,
credentialGetter,
startRecoveryPoll,
]);
@@ -3116,7 +3211,7 @@ export function WorkflowCopilotChat({
);
};
const action = resolveSendAction({
- inFlight: inFlightRef.current,
+ inFlight: inFlightRef.current || recoveredTurnOwnerRef.current !== null,
hasQueuedPrompt: Boolean(queuedPromptRef.current),
requiresLiveBrowser,
isLiveBrowserReady,
@@ -3395,6 +3490,7 @@ export function WorkflowCopilotChat({
};
let streamChatId: string | null = null;
let sawTerminalFrame = false;
+ let sawCredentialPause = false;
const shouldArmRecovery = () =>
streamTurnId !== null &&
!sawTerminalFrame &&
@@ -3772,6 +3868,8 @@ export function WorkflowCopilotChat({
for (const attached of sentAttachments) {
inFlightFileIds.current.add(attached.file_id);
}
+ const credentialRecoveryToken =
+ ensureCredentialRecoveryToken(workflowPermanentId);
await client.postStreaming<WorkflowCopilotSsePayload>(
"/workflow/copilot/chat-post",
{
@@ -3797,6 +3895,10 @@ export function WorkflowCopilotChat({
selected_block_label: readSelectedBlockLabel(),
keep_pending_proposal: Boolean(pendingProposalTurnId),
supports_credential_pause: true,
+ supports_credential_pause_recovery: Boolean(
+ credentialRecoveryToken,
+ ),
+ credential_recovery_token: credentialRecoveryToken ?? undefined,
supports_question_tool: true,
} as WorkflowCopilotChatRequest,
(payload) => {
@@ -3882,6 +3984,7 @@ export function WorkflowCopilotChat({
.setTitleFromCopilotIfDefault(payload.title);
return false;
case "credential_required":
+ sawCredentialPause = true;
setLivePauseFrame(payload);
return false;
case "turn_start": {
@@ -4055,6 +4158,7 @@ export function WorkflowCopilotChat({
startRecoveryPoll(
streamChatId ?? workflowCopilotChatIdRef.current,
streamTurnId,
+ sawCredentialPause,
);
}
}
@@ -4244,7 +4348,7 @@ export function WorkflowCopilotChat({
const lastTurn = lastTurnRef.current;
const drainAction = resolveDrainAction({
queuedReason: queuedPrompt.reason,
- inFlight: isLoading,
+ inFlight: isLoading || recoveredTurnOwnerRef.current !== null,
hasLiveBrowserSession: Boolean(liveBrowserSessionId),
hasWorkflowPermanentId: Boolean(workflowPermanentId),
queuedContent: queuedPrompt.content,
@@ -5190,6 +5294,33 @@ export function WorkflowCopilotChat({
RESPONSE has frozen the narrative into the latest AI message —
otherwise the same turn would render twice.
*/}
+ {!isLoadingHistory &&
+ recoveredPauseFrames
+ .filter(
+ (frame) =>
+ frame.workflow_copilot_chat_id === workflowCopilotChatId &&
+ frame.turn_id !== livePauseFrame?.turn_id,
+ )
+ .map((frame) => (
+ <CredentialCard
+ key={frame.turn_id}
+ frame={liveFrameToCardFrame(frame)}
+ mode="inline-pause"
+ reloadKey={credentialsReloadKey}
+ resolvedOutcome={credentialResolutions[frame.turn_id]}
+ onConnect={(credentialId, name) =>
+ credentialId
+ ? void respondToCredentialPause(
+ frame,
+ "connected",
+ credentialId,
+ name,
+ )
+ : openCredentialModal(frame, frame.turn_id)
+ }
+ onSkip={() => void respondToCredentialPause(frame, "skip")}
+ />
+ ))}
{narrative.turnId !== null && narrative.terminal === null && (
<div
className="flex flex-col gap-2"
diff --git a/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.test.ts b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.test.ts
new file mode 100644
index 000000000..e5ae2e0d9
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.test.ts
@@ -0,0 +1,75 @@
+import type { AxiosInstance } from "axios";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+ CredentialRecoveryHistoryError,
+ credentialRecoveryHeaders,
+ currentCredentialRecoveryToken,
+ readCredentialRecoveryHistory,
+ ensureCredentialRecoveryToken,
+} from "./credentialRecovery";
+
+describe("credentialRecovery", () => {
+ beforeEach(() => sessionStorage.clear());
+
+ it("retains one capability for every recoverable turn in the workflow", () => {
+ const first = ensureCredentialRecoveryToken("wpid-1");
+ expect(first).toMatch(/^[0-9a-f]{64}$/);
+ expect(currentCredentialRecoveryToken("wpid-1")).toBe(first);
+ expect(credentialRecoveryHeaders("wpid-1")).toEqual({
+ "X-Copilot-Credential-Recovery-Token": first,
+ });
+
+ const second = ensureCredentialRecoveryToken("wpid-1");
+ expect(second).toMatch(/^[0-9a-f]{64}$/);
+ expect(second).toBe(first);
+ expect(currentCredentialRecoveryToken("wpid-1")).toBe(second);
+ });
+
+ it("keeps capabilities scoped by workflow", () => {
+ ensureCredentialRecoveryToken("wpid-1");
+ expect(currentCredentialRecoveryToken("wpid-2")).toBeNull();
+ expect(credentialRecoveryHeaders(undefined)).toEqual({});
+ });
+
+ it("does not expose the capability when a history read fails", async () => {
+ const token = ensureCredentialRecoveryToken("wpid-1");
+ const client = {
+ get: async () => {
+ throw {
+ response: { status: 403 },
+ config: { headers: { "X-Copilot-Credential-Recovery-Token": token } },
+ };
+ },
+ };
+
+ const error = await readCredentialRecoveryHistory(
+ client,
+ "wpid-1",
+ {},
+ ).catch((caught: unknown) => caught);
+
+ expect(error).toBeInstanceOf(CredentialRecoveryHistoryError);
+ expect(error).toMatchObject({ status: 403 });
+ expect(String(error)).not.toContain(token);
+ expect(error).not.toHaveProperty("config");
+ });
+
+ it("retries one transient authenticated history failure", async () => {
+ ensureCredentialRecoveryToken("wpid-1");
+ let reads = 0;
+ const get = vi.fn(async () => {
+ reads += 1;
+ if (reads === 1) throw new Error("temporary network failure");
+ return { data: { workflow_copilot_chat_id: "chat-1" } };
+ });
+ const client = { get } as unknown as Pick<AxiosInstance, "get">;
+
+ const response = await readCredentialRecoveryHistory<{
+ workflow_copilot_chat_id: string;
+ }>(client, "wpid-1", {}, { retryTransientFailure: true });
+
+ expect(reads).toBe(2);
+ expect(response.data.workflow_copilot_chat_id).toBe("chat-1");
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.ts b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.ts
new file mode 100644
index 000000000..6a569e090
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/credentialRecovery.ts
@@ -0,0 +1,92 @@
+import type { AxiosInstance, AxiosRequestConfig, AxiosResponse } from "axios";
+
+const KEY_PREFIX = "copilot-credential-recovery";
+
+function storageKey(workflowId: string): string {
+ return `${KEY_PREFIX}:${workflowId}`;
+}
+
+export function currentCredentialRecoveryToken(
+ workflowId: string | undefined,
+): string | null {
+ if (!workflowId) return null;
+ try {
+ return sessionStorage.getItem(storageKey(workflowId));
+ } catch {
+ return null;
+ }
+}
+
+export function ensureCredentialRecoveryToken(
+ workflowId: string | undefined,
+): string | null {
+ if (!workflowId) return null;
+ try {
+ // Keep one capability for this workflow for the lifetime of the browser
+ // tab. Each server pause still binds its digest to one org/chat/turn and
+ // requires its own single-use resume token.
+ const current = sessionStorage.getItem(storageKey(workflowId));
+ if (current) return current;
+ const token = Array.from(
+ crypto.getRandomValues(new Uint8Array(32)),
+ (byte) => byte.toString(16).padStart(2, "0"),
+ ).join("");
+ sessionStorage.setItem(storageKey(workflowId), token);
+ return token;
+ } catch {
+ return null;
+ }
+}
+
+export function credentialRecoveryHeaders(
+ workflowId: string | undefined,
+): Record<string, string> {
+ const token = currentCredentialRecoveryToken(workflowId);
+ return token ? { "X-Copilot-Credential-Recovery-Token": token } : {};
+}
+
+export class CredentialRecoveryHistoryError extends Error {
+ readonly status: number | null;
+
+ constructor(error: unknown) {
+ const status =
+ (error as { response?: { status?: unknown } })?.response?.status ?? null;
+ const safeStatus = typeof status === "number" ? status : null;
+ super(
+ safeStatus === null
+ ? "Copilot history request failed"
+ : `Copilot history request failed (${safeStatus})`,
+ );
+ this.name = "CredentialRecoveryHistoryError";
+ this.status = safeStatus;
+ }
+}
+
+export async function readCredentialRecoveryHistory<T>(
+ client: Pick<AxiosInstance, "get">,
+ workflowId: string | undefined,
+ config: AxiosRequestConfig,
+ options: { retryTransientFailure?: boolean } = {},
+): Promise<AxiosResponse<T>> {
+ const attempts = options.retryTransientFailure ? 2 : 1;
+ for (let attempt = 0; attempt < attempts; attempt += 1) {
+ try {
+ return await client.get<T>("/workflow/copilot/chat-history", {
+ ...config,
+ headers: {
+ ...config.headers,
+ ...credentialRecoveryHeaders(workflowId),
+ },
+ });
+ } catch (error) {
+ // Axios errors retain request headers. Replace them before callers log
+ // the failure so the capability cannot reach the browser console.
+ const safeError = new CredentialRecoveryHistoryError(error);
+ const retryable = safeError.status === null || safeError.status >= 500;
+ if (attempt + 1 >= attempts || !retryable || config.signal?.aborted) {
+ throw safeError;
+ }
+ }
+ }
+ throw new CredentialRecoveryHistoryError(null);
+}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index ed81336c6..9a56cb633 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -131,6 +131,8 @@ export interface WorkflowCopilotChatRequest {
// Opt-in: only clients that can render the credential_required frame set
// this, so the backend never pauses a turn a client would silently drop.
supports_credential_pause?: boolean;
+ credential_recovery_token?: string;
+ supports_credential_pause_recovery?: boolean;
supports_question_tool?: boolean;
}
@@ -163,6 +165,7 @@ export interface WorkflowCopilotChatHistoryMessage {
}
export interface WorkflowCopilotChatHistoryResponse {
+ pending_credential_requests?: WorkflowCopilotCredentialRequiredUpdate[];
question_interactions?: QuestionInteraction[];
pending_question_cancel_token?: string | null;
workflow_copilot_chat_id: string | null;
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 1114f0350..1b276e63b 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1487,6 +1487,10 @@ class AgentFunction:
async def browser_context_route_handlers_allowed(self, **_: Any) -> bool:
return True
+ async def has_retained_browser_egress_guard(self, browser_context: Any) -> bool:
+ """Whether cloud-only retained init scripts still require this context's guard."""
+ return False
+
async def setup_browser_context_extensions(self, browser_context: Any, **kwargs: Any) -> None:
"""Attach cloud-only listeners/route handlers to a fresh BrowserContext. OSS no-op."""
diff --git a/skyvern/forge/log_redaction.py b/skyvern/forge/log_redaction.py
index db1ef07e6..70ee5f631 100644
--- a/skyvern/forge/log_redaction.py
+++ b/skyvern/forge/log_redaction.py
@@ -40,6 +40,7 @@ SENSITIVE_HEADERS = {
"set-cookie",
"x-api-key",
"x-posthog-attribution",
+ "x-copilot-credential-recovery-token",
}
# Exact field names that are always redacted. Use a set for O(1) lookup
@@ -51,6 +52,8 @@ SENSITIVE_FIELDS: set[str] = {
"password",
"secret",
"token",
+ "credential_recovery_token",
+ "resume_token",
"api_key",
"apikey",
"api-key",
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index b7f0c9d1c..d13de0702 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -114,6 +114,7 @@ from skyvern.forge.sdk.copilot.context import (
record_proposed_credential_in_global_llm_context,
sanitize_global_llm_context_for_prompt,
)
+from skyvern.forge.sdk.copilot.credential_pause import credential_recovery_token_digest
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
CopilotNonRetriableNavError,
@@ -683,10 +684,7 @@ def _apply_raw_secret_turn_transition(
return
if policy.raw_secret_handling == "redacted_draft" and policy.raw_secret_safety_status != "blocked":
- policy.testing_intent = "skip_test"
- policy.allow_run_blocks = False
- policy.allow_missing_credentials_in_draft = True
- policy.credential_draft_deferred_explicitly = True
+ policy.apply_raw_secret_redacted_draft()
else:
policy.raw_secret_handling = "block"
policy.user_response_policy = "ask_clarification"
@@ -5102,6 +5100,12 @@ async def _run_copilot_turn_impl(
target_block_label=getattr(chat_request, "target_block_label", None),
selected_block_label=getattr(chat_request, "selected_block_label", None),
client_supports_credential_pause=getattr(chat_request, "supports_credential_pause", False),
+ client_supports_credential_pause_recovery=chat_request.supports_credential_pause_recovery,
+ credential_recovery_token_digest=credential_recovery_token_digest(
+ chat_request.credential_recovery_token.get_secret_value()
+ if chat_request.credential_recovery_token
+ else None
+ ),
executed_block_fingerprints={
label: set(fingerprints) for label, fingerprints in (prior_executed_block_fingerprints or {}).items()
},
diff --git a/skyvern/forge/sdk/copilot/ask_user.py b/skyvern/forge/sdk/copilot/ask_user.py
index a29c0dee2..4469183e0 100644
--- a/skyvern/forge/sdk/copilot/ask_user.py
+++ b/skyvern/forge/sdk/copilot/ask_user.py
@@ -57,6 +57,10 @@ class QuestionResponse(BaseModel):
answers: list[QuestionAnswer] = Field(default_factory=list)
text: str | None = None
skipped: bool = False
+ # Server-owned result of screening the free-text answer. It is persisted
+ # with the interaction so active and reconstructed policies enforce the
+ # same trust floor without retaining the detected literal.
+ raw_secret_detected: bool = Field(default=False, exclude_if=lambda detected: not detected)
class QuestionInteraction(BaseModel):
@@ -194,6 +198,14 @@ async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_i
while True:
recorded = await repo.poll_copilot_question(ctx.organization_id, chat_id, interaction.interaction_id)
if recorded.status == "resolved":
+ if ctx.request_policy is not None:
+ ctx.request_policy.project_question_response_sites(recorded)
+ ctx.allow_untested_workflow_draft = (
+ ctx.request_policy.raw_secret_detected
+ and ctx.request_policy.raw_secret_handling == "redacted_draft"
+ )
+ if recorded.response is not None and not recorded.response.skipped:
+ ctx.credential_recovery_armed = True
await ctx.stream.send(
{
"type": "question_resolved",
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index f16e629f4..b8bec2852 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -1145,6 +1145,9 @@ class CopilotContext(AgentContext):
# is set from the chat request at construction; the rest are owned by maybe_credential_pause.
last_run_skipped_unbound_credentials: bool = False
client_supports_credential_pause: bool = False
+ client_supports_credential_pause_recovery: bool = False
+ credential_recovery_token_digest: str | None = field(default=None, repr=False)
+ credential_recovery_armed: bool = False
credential_pause_used: bool = False
# True only while a card is on screen. credential_pause_used stays true for the rest of the
# turn once one has been raised, which cannot tell a concurrent sibling ask from a later one.
diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.py
index a86d0c4d2..9aa894c6f 100644
--- a/skyvern/forge/sdk/copilot/credential_pause.py
+++ b/skyvern/forge/sdk/copilot/credential_pause.py
@@ -10,8 +10,8 @@ itself is the poller rather than a task racing the handler.
The resume path is not authorized by org auth + ``turn_id`` alone. Establishing
the pause writes an *active-pause record*, keyed by (org, chat, turn), that
-carries a one-time ``resume_token`` delivered only in the ``credential_required``
-frame. ``resolve_credential_pause`` -- the only writer of the loop-facing
+carries a one-time ``resume_token`` delivered in the ``credential_required``
+frame and authenticated chat history for clients advertising card recovery. ``resolve_credential_pause`` -- the only writer of the loop-facing
response flag -- refuses to store a decision unless the caller presents that
token against a still-pending record, and consumes the record on the first
accepted response so a leaked or replayed ``turn_id`` can't resolve the pause.
@@ -20,6 +20,7 @@ accepted response so a leaked or replayed ``turn_id`` can't resolve the pause.
from __future__ import annotations
import asyncio
+import hashlib
import json
import re
import secrets
@@ -27,9 +28,10 @@ import time
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any, Literal, Protocol
import structlog
+from pydantic import ValidationError
from skyvern.config import settings
from skyvern.forge import app
@@ -99,14 +101,25 @@ class _ActivePauseRecord:
resume_token: str
status: Literal["pending", "consumed"]
expires_at: datetime
+ card: WorkflowCopilotCredentialRequiredUpdate | None = None
+ recovery_token_digest: str | None = None
-def _encode_active_pause(resume_token: str, expires_at: datetime, *, consumed: bool = False) -> str:
+def _encode_active_pause(
+ resume_token: str,
+ expires_at: datetime,
+ *,
+ consumed: bool = False,
+ card: WorkflowCopilotCredentialRequiredUpdate | None = None,
+ recovery_token_digest: str | None = None,
+) -> str:
return json.dumps(
{
"resume_token": resume_token,
"status": "consumed" if consumed else "pending",
"expires_at": expires_at.isoformat(),
+ "card": card.model_dump(mode="json") if card is not None else None,
+ "recovery_token_digest": recovery_token_digest,
}
)
@@ -126,7 +139,18 @@ def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:
expires_at = datetime.fromisoformat(str(data.get("expires_at")))
except ValueError:
return None
- return _ActivePauseRecord(resume_token=token, status=record_status, expires_at=expires_at)
+ try:
+ card = WorkflowCopilotCredentialRequiredUpdate.model_validate(data["card"]) if data.get("card") else None
+ except ValidationError:
+ card = None
+ digest = data.get("recovery_token_digest")
+ return _ActivePauseRecord(
+ resume_token=token,
+ status=record_status,
+ expires_at=expires_at,
+ card=card,
+ recovery_token_digest=digest if isinstance(digest, str) else None,
+ )
def _validate_pending_pause(record: _ActivePauseRecord | None, resume_token: str) -> _ActivePauseRecord:
@@ -158,6 +182,86 @@ def _validate_pending_pause(record: _ActivePauseRecord | None, resume_token: str
return record
+def credential_recovery_token_digest(token: str | None) -> str | None:
+ """Hash a 32-byte browser capability without retaining the bearer value."""
+ if not isinstance(token, str) or len(token) != 64:
+ return None
+ try:
+ decoded = bytes.fromhex(token)
+ except ValueError:
+ return None
+ if len(decoded) != 32:
+ return None
+ return hashlib.sha256(token.encode()).hexdigest()
+
+
+class _CredentialRecoveryContext(Protocol):
+ credential_recovery_armed: bool
+ client_supports_credential_pause_recovery: bool
+ credential_recovery_token_digest: str | None
+
+
+def _credential_recovery_enabled(ctx: _CredentialRecoveryContext) -> bool:
+ return bool(
+ ctx.credential_recovery_armed
+ and ctx.client_supports_credential_pause_recovery
+ and ctx.credential_recovery_token_digest
+ )
+
+
+async def pending_credential_requests(
+ organization_id: str,
+ chat_id: str,
+ turn_ids: list[str],
+ recovery_token: str | None,
+) -> list[WorkflowCopilotCredentialRequiredUpdate]:
+ digest = credential_recovery_token_digest(recovery_token)
+ cache = app.CACHE
+ if cache is None or digest is None:
+ return []
+ cards: list[WorkflowCopilotCredentialRequiredUpdate] = []
+ for turn_id in turn_ids:
+ try:
+ raw = await cache.get(credential_pause_active_key(organization_id, chat_id, turn_id))
+ except Exception:
+ LOG.warning("Failed to recover Copilot credential pause", exc_info=True)
+ raise
+ record = _decode_active_pause(raw)
+ if (
+ record is None
+ or record.card is None
+ or record.recovery_token_digest is None
+ or not secrets.compare_digest(digest, record.recovery_token_digest)
+ ):
+ continue
+ try:
+ _validate_pending_pause(record, record.resume_token)
+ except CredentialPauseRejection:
+ continue
+ card = record.card
+ if (
+ card.turn_id == turn_id
+ and card.workflow_copilot_chat_id == chat_id
+ and card.resume_token == record.resume_token
+ and card.expires_at == record.expires_at
+ ):
+ cards.append(card)
+ return cards
+
+
+async def credential_pause_is_active(organization_id: str, chat_id: str, turn_id: str) -> bool | None:
+ """Report whether a live waiter owns the turn, or ``None`` when Redis is unavailable."""
+ cache = app.CACHE
+ if cache is None:
+ return False
+ try:
+ record = _decode_active_pause(await cache.get(credential_pause_active_key(organization_id, chat_id, turn_id)))
+ except Exception:
+ LOG.warning("Failed to inspect Copilot credential pause", exc_info=True)
+ return None
+ return bool(record is not None and record.status == "pending" and datetime.now(timezone.utc) < record.expires_at)
+
+
async def check_credential_pause_resumable(
cache: Any,
*,
@@ -394,6 +498,7 @@ async def _wait_for_credential_response(
stream: EventSourceStream,
timeout_seconds: int,
) -> CredentialPauseResolution | None:
+ recovery_enabled = _credential_recovery_enabled(ctx)
# Check once before the sleep loop so a card response posted in the brief
# window before the first poll doesn't cost a full extra poll interval.
first = await _try_resolve_credential_response(response_key, ctx.organization_id)
@@ -409,7 +514,7 @@ async def _wait_for_credential_response(
resolved = await _try_resolve_credential_response(response_key, ctx.organization_id)
if resolved != "pending":
return resolved
- if await stream.is_disconnected():
+ if not recovery_enabled and await stream.is_disconnected():
return None
return None
@@ -427,23 +532,20 @@ async def _run_credential_pause(
Returns the resolution, or None to let the caller proceed without one
(kill-switch off, client can't render the frame, already paused once this
- turn, no cache configured or not shared across workers, client gone, or
- the wait timed out).
+ turn, no shared cache, unrecoverable client disconnect, or timeout).
+ Recovery-capable clients restore the same active card through chat history.
"""
if not credential_pause_transport_ready(ctx, copilot_config):
return None
- # Predicted true (the sync guard chain passed) but bailing below anyway --
- # latch it now so this iteration can't loop back into the same prediction,
- # and tag the outcome "declined" (distinct from "timeout") so the caller
- # knows no frame was ever sent and can fall back to a normal nudge instead
- # of a premature finalize. credential_pause_transport_ready's own docstring
- # notes it excludes the async-only disconnect check, which is the gap here.
+ # Latch before async checks so a declined transport cannot trigger another
+ # pause. Recovery-capable clients can retrieve the card after disconnect.
ctx.credential_pause_used = True
cache = getattr(app, "CACHE", None)
if cache is None:
ctx.credential_pause_outcome = "declined"
return None
- if await stream.is_disconnected():
+ recovery_enabled = _credential_recovery_enabled(ctx)
+ if not recovery_enabled and await stream.is_disconnected():
ctx.credential_pause_outcome = "declined"
return None
policy = getattr(ctx, "request_policy", None)
@@ -465,27 +567,30 @@ async def _run_credential_pause(
# response endpoint refuses to resolve a turn that has no pending record.
# expires_at is the same deadline the frame tells the client -- the record's
# own TTL is a separate infra grace, not a resolve-after-timeout allowance.
+ card = WorkflowCopilotCredentialRequiredUpdate(
+ type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,
+ turn_id=turn_id,
+ workflow_copilot_chat_id=chat_id,
+ resume_token=resume_token,
+ reason=reason,
+ message=message,
+ login_page_urls=login_page_urls,
+ credential_refs=credential_refs,
+ timeout_seconds=timeout_seconds,
+ expires_at=expires_at,
+ timestamp=now,
+ )
await cache.set(
credential_pause_active_key(organization_id, chat_id, turn_id),
- _encode_active_pause(resume_token, expires_at),
+ _encode_active_pause(
+ resume_token,
+ expires_at,
+ card=card if recovery_enabled else None,
+ recovery_token_digest=ctx.credential_recovery_token_digest if recovery_enabled else None,
+ ),
ex=_credential_pause_record_ttl(timeout_seconds),
)
-
- await stream.send(
- WorkflowCopilotCredentialRequiredUpdate(
- type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,
- turn_id=turn_id,
- workflow_copilot_chat_id=chat_id,
- resume_token=resume_token,
- reason=reason,
- message=message,
- login_page_urls=login_page_urls,
- credential_refs=credential_refs,
- timeout_seconds=timeout_seconds,
- expires_at=expires_at,
- timestamp=now,
- )
- )
+ await stream.send(card)
async def _invalidate_active_pause_record() -> CredentialPauseResolution | None:
# The waiter can exit (disconnect, genuine timeout, cancellation, or an
@@ -510,12 +615,9 @@ async def _run_credential_pause(
response_key = credential_response_cache_key(organization_id, chat_id, turn_id)
- if await stream.is_disconnected():
- # send() can return True even when the client is already gone (its own
- # protocol contract: "queued for delivery or dropped because the client
- # is gone") -- a disconnect racing the send itself would otherwise wait
- # out the full timeout for a card nobody ever saw. Re-check right after
- # send and treat it the same as the pre-send disconnect guard above.
+ if not recovery_enabled and await stream.is_disconnected():
+ # Legacy clients cannot restore a dropped frame, so preserve their
+ # immediate decline when disconnect races the send.
await _invalidate_active_pause_record()
ctx.credential_pause_outcome = "declined"
return None
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 8b8cac3d1..80dc0bcdc 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -18,6 +18,7 @@ from skyvern.forge import app
from skyvern.forge.log_redaction import redact_sensitive_fields
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
+from skyvern.forge.sdk.copilot.ask_user import QuestionInteraction
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import ProposedCredential, StructuredContext
from skyvern.forge.sdk.copilot.credential_resolution import (
@@ -874,6 +875,27 @@ class LivePageResolutionRecord:
page_url: str = ""
+@dataclass(frozen=True)
+class UserMessageSiteURLSource:
+ message_index: int
+ kind: Literal["user_message"] = field(default="user_message", init=False)
+
+
+@dataclass(frozen=True)
+class QuestionResponseSiteURLSource:
+ interaction_id: str
+ kind: Literal["question_response"] = field(default="question_response", init=False)
+
+
+SiteURLSource = UserMessageSiteURLSource | QuestionResponseSiteURLSource
+
+
+@dataclass(frozen=True)
+class _SiteURLText:
+ text: str
+ source: SiteURLSource
+
+
@dataclass
class RequestPolicy:
testing_intent: str = "unspecified"
@@ -936,8 +958,9 @@ class RequestPolicy:
# only be released onto one of these (or a vault/tested match); a site only a model produced is
# never eligible.
user_provided_site_urls: list[str] = field(default_factory=list)
- # Which user message (1-based) each of those URLs came from, so a release records its provenance.
- user_site_url_sources: dict[str, int] = field(default_factory=dict)
+ # The ordinary user message or accepted interactive response each URL came from, so a release
+ # records truthful provenance without retaining or logging the response text.
+ user_site_url_sources: dict[str, SiteURLSource] = field(default_factory=dict)
existing_workflow_credential_ids: list[str] = field(default_factory=list)
# Read from the saved workflow row, never from the submitted YAML. The submission is the live
# canvas, which carries a copilot proposal the user has not accepted, so it cannot grant a run.
@@ -964,6 +987,18 @@ class RequestPolicy:
canonical_user_message: str = field(default="", repr=False, compare=False)
_authoring_pending: bool = field(default=False, repr=False, compare=False)
+ def project_question_response_sites(self, interaction: QuestionInteraction) -> None:
+ project_question_response_sites(self, interaction)
+
+ def apply_raw_secret_redacted_draft(self) -> None:
+ self.raw_secret_detected = True
+ self.raw_secret_handling = "redacted_draft"
+ self.raw_secret_safety_status = "detected"
+ self.testing_intent = "skip_test"
+ self.allow_run_blocks = False
+ self.allow_missing_credentials_in_draft = True
+ self.credential_draft_deferred_explicitly = True
+
def graded_completion_criteria(self) -> list[CompletionCriterion]:
return [criterion for criterion in self.completion_criteria if not criterion.method_mandated]
@@ -4115,40 +4150,87 @@ def credential_candidate_label(credential: Credential) -> str:
return f"{label} - {'; '.join(facts)}"
-def _ground_user_provided_sites(
+def _accepted_question_response_url_texts(interaction: QuestionInteraction) -> list[_SiteURLText]:
+ response = interaction.response
+ if interaction.status != "resolved" or response is None or response.skipped or response.raw_secret_detected:
+ return []
+ source = QuestionResponseSiteURLSource(interaction_id=interaction.interaction_id)
+ texts = [response.text] if response.text else []
+ texts.extend(answer.text for answer in response.answers if answer.text)
+ return [_SiteURLText(text=text, source=source) for text in texts]
+
+
+def _persisted_question_response_url_texts(raw_interaction: dict[str, Any]) -> list[_SiteURLText]:
+ try:
+ interaction = QuestionInteraction.model_validate(raw_interaction)
+ except ValidationError:
+ return []
+ return _accepted_question_response_url_texts(interaction)
+
+
+def _project_user_provided_sites(
policy: RequestPolicy,
- user_message: str,
- full_chat_history: Sequence[WorkflowCopilotChatHistoryMessage],
+ url_texts: Sequence[_SiteURLText],
+ *,
+ reset: bool,
) -> None:
- """Record every site the user themselves gave this chat, so the fill seam can release a credential
- onto one of them and never onto a site only a model produced.
-
- Linking a later "log into pathfold" back to a URL from an earlier turn is the agent's job, not
- this function's: it reads the whole conversation. What is recorded here is only the deterministic
- part — the origins the user actually wrote.
- """
- # USER only, never TURN_OPENER_SENDERS: this set releases credentials, so a site must have
- # been written by the person, not by a row the product authored on their behalf.
- user_texts = [
- message.content
- for message in full_chat_history
- if message.sender == WorkflowCopilotChatSender.USER and message.content
- ]
- user_texts.append(user_message or "")
- seen_origins: set[str] = set()
- urls: list[str] = []
- sources: dict[str, int] = {}
- for index, text in enumerate(user_texts, start=1):
- for candidate in URL_CANDIDATE_RE.findall(text):
+ """Project literal user-authored URL facts while preserving first-origin provenance."""
+ if reset:
+ policy.user_provided_site_urls = []
+ policy.user_site_url_sources = {}
+ seen_origins = {parts[2] for url in policy.user_provided_site_urls if (parts := _url_parts(url)) is not None}
+ for item in url_texts:
+ for candidate in URL_CANDIDATE_RE.findall(item.text):
cleaned = candidate.rstrip(".,;:!?")
parts = _url_parts(cleaned)
if parts is None or parts[2] in seen_origins:
continue
seen_origins.add(parts[2])
- urls.append(cleaned)
- sources[cleaned] = index
- policy.user_provided_site_urls = urls
- policy.user_site_url_sources = sources
+ policy.user_provided_site_urls.append(cleaned)
+ policy.user_site_url_sources[cleaned] = item.source
+
+
+def project_question_response_sites(policy: RequestPolicy, interaction: QuestionInteraction) -> None:
+ if interaction.response is not None and interaction.response.raw_secret_detected:
+ policy.apply_raw_secret_redacted_draft()
+ return
+ _project_user_provided_sites(policy, _accepted_question_response_url_texts(interaction), reset=False)
+
+
+def _ground_user_provided_sites(
+ policy: RequestPolicy,
+ user_message: str,
+ full_chat_history: Sequence[WorkflowCopilotChatHistoryMessage],
+) -> None:
+ """Rebuild the URL facts the person supplied in USER rows and accepted question responses.
+
+ Linking a later site name back to a URL is the agent's job. This projection only verifies URLs
+ in the two structured user-authored text surfaces; prompts, choices, rendered history, and
+ PRODUCT or AI prose never become credential-origin authority.
+ """
+ url_texts: list[_SiteURLText] = []
+ user_message_index = 0
+ for message in full_chat_history:
+ if message.sender == WorkflowCopilotChatSender.USER and message.content:
+ user_message_index += 1
+ url_texts.append(
+ _SiteURLText(
+ text=message.content,
+ source=UserMessageSiteURLSource(message_index=user_message_index),
+ )
+ )
+ if message.sender != WorkflowCopilotChatSender.AI or message.narrative_payload is None:
+ continue
+ for raw_interaction in message.narrative_payload.get("questionInteractions", []):
+ url_texts.extend(_persisted_question_response_url_texts(raw_interaction))
+ if user_message:
+ url_texts.append(
+ _SiteURLText(
+ text=user_message,
+ source=UserMessageSiteURLSource(message_index=user_message_index + 1),
+ )
+ )
+ _project_user_provided_sites(policy, url_texts, reset=True)
def _prior_approved_connection_ids(global_llm_context: str) -> set[str]:
diff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.py
index 8babc6fd2..cebe26d2d 100644
--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py
+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py
@@ -25,7 +25,10 @@ from skyvern.forge.sdk.copilot.credential_resolution import is_resolved_page_url
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
from skyvern.forge.sdk.copilot.page_identity import safe_page_origin
from skyvern.forge.sdk.copilot.request_policy import (
+ QuestionResponseSiteURLSource,
RequestPolicy,
+ SiteURLSource,
+ UserMessageSiteURLSource,
admit_credential_for_live_page,
loggable_origin,
)
@@ -343,13 +346,23 @@ def _missing_credential_origin_error(credential_id: str, page_url: str | None) -
)
-def _log_fill_grant(route: str, url: str, credential_id: str, source_message: int | None = None) -> None:
+def _log_fill_grant(
+ route: str,
+ url: str,
+ credential_id: str,
+ source: SiteURLSource | None = None,
+) -> None:
+ source_fields: dict[str, str | int] = {}
+ if isinstance(source, UserMessageSiteURLSource):
+ source_fields = {"source_kind": source.kind, "source_user_message": source.message_index}
+ elif isinstance(source, QuestionResponseSiteURLSource):
+ source_fields = {"source_kind": source.kind, "source_interaction_id": source.interaction_id}
LOG.info(
"copilot credential fill grant",
route=route,
page_origin=loggable_origin(url),
credential_id=credential_id,
- source_user_message=source_message,
+ **source_fields,
)
@@ -538,7 +551,12 @@ async def _credential_fill_origin_grant(
if settled or credential_id == await _sole_org_password_credential_id(load_once):
# An origin-only match (no registrable site) keeps the origin-scoped grant so the
# release guard can still compare it; site matches travel the whole site.
- _log_fill_grant("user_url", page_url, credential_id, policy.user_site_url_sources.get(matched_url))
+ _log_fill_grant(
+ "user_url",
+ page_url,
+ credential_id,
+ policy.user_site_url_sources.get(matched_url),
+ )
return _CredentialFillOriginGrant(page_url, whole_site=site_level), None
return None, _ambiguous_unbound_credential_steer(credential_id, page_url)
return None, _missing_credential_origin_error(credential_id, page_url or None)
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 7eb1df50a..d234ec2a4 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -14,7 +14,7 @@ from urllib.parse import urlparse
import structlog
import yaml
-from fastapi import Depends, File, Form, HTTPException, Request, UploadFile, status
+from fastapi import Depends, File, Form, Header, HTTPException, Request, UploadFile, status
from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from sse_starlette import EventSourceResponse
@@ -42,6 +42,8 @@ from skyvern.forge.sdk.copilot.context import (
from skyvern.forge.sdk.copilot.credential_pause import (
CredentialPauseRejection,
check_credential_pause_resumable,
+ credential_pause_is_active,
+ pending_credential_requests,
resolve_credential_pause,
)
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
@@ -2771,6 +2773,27 @@ async def _reconcile_interrupted_copilot_turns(chat: WorkflowCopilotChat, organi
)
if last_activity > abandoned_before:
continue
+ credential_pause_active = await credential_pause_is_active(
+ organization_id,
+ chat.workflow_copilot_chat_id,
+ entry.turn_id,
+ )
+ if credential_pause_active is True:
+ continue
+ has_recovery_handoff = any(
+ item.status == "resolved" and item.response is not None and not item.response.skipped
+ for item in entry.question_interactions
+ )
+ if (
+ credential_pause_active is None
+ and has_recovery_handoff
+ and last_activity
+ > abandoned_before - timedelta(seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS)
+ ):
+ # A transient Redis failure must not reclaim a possible second
+ # credential wait, while an unrelated stale turn must eventually
+ # reconcile even if the auxiliary cache remains unavailable.
+ continue
claimed = await app.DATABASE.workflow_params.claim_pending_copilot_turn(
organization_id=organization_id,
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
@@ -2792,6 +2815,7 @@ async def _reconcile_interrupted_copilot_turns(chat: WorkflowCopilotChat, organi
@base_router.get("/workflow/copilot/chat-history", include_in_schema=False)
async def workflow_copilot_chat_history(
+ credential_recovery_token: str | None = Header(None, alias="X-Copilot-Credential-Recovery-Token"),
workflow_permanent_id: str | None = None,
workflow_copilot_chat_id: str | None = None,
organization: Organization = Depends(org_auth_service.get_current_org),
@@ -2825,6 +2849,14 @@ async def workflow_copilot_chat_history(
else:
chat_messages = []
return WorkflowCopilotChatHistoryResponse(
+ pending_credential_requests=await pending_credential_requests(
+ organization.organization_id,
+ chat.workflow_copilot_chat_id,
+ list(chat.pending_turns),
+ credential_recovery_token,
+ )
+ if chat
+ else [],
workflow_copilot_chat_id=chat.workflow_copilot_chat_id if chat else None,
question_interactions=list(
{
@@ -2899,6 +2931,8 @@ async def workflow_copilot_question_response(
raise HTTPException(
status_code=503, detail="The safety screen is unavailable. Please retry your answer."
)
+ if safety.status == "detected":
+ response.raw_secret_detected = True
return safety.canonical_user_message
if response.text is not None:
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index c4b6c2ba6..cbb60daf2 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -3,7 +3,7 @@ from datetime import datetime
from enum import StrEnum
from typing import Annotated, Any, Literal
-from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, StringConstraints, field_validator
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, SecretStr, StringConstraints, field_validator
from skyvern.forge.sdk.copilot.ask_user import QuestionInteraction, QuestionResponse
from skyvern.forge.sdk.copilot.code_write_diff import CodeWriteDiff
@@ -250,6 +250,15 @@ class WorkflowCopilotChatRequest(BaseModel):
),
)
supports_question_tool: bool = Field(False, description="The client can display and answer ask_user requests.")
+ credential_recovery_token: SecretStr | None = Field(
+ None,
+ repr=False,
+ exclude=True,
+ description="Client-held capability for restoring this turn's pending credential card.",
+ )
+ supports_credential_pause_recovery: bool = Field(
+ False, description="The client restores pending credential cards from chat history."
+ )
supports_credential_pause: bool = Field(
False,
description=(
@@ -336,16 +345,6 @@ class WorkflowCopilotChatHistoryMessage(BaseModel):
created_at: datetime = Field(..., description="When the message was created")
-class WorkflowCopilotChatHistoryResponse(BaseModel):
- question_interactions: list[QuestionInteraction] = Field(default_factory=list)
- pending_question_cancel_token: str | None = None
- workflow_copilot_chat_id: str | None = Field(None, description="Latest chat ID for the workflow")
- chat_history: list[WorkflowCopilotChatHistoryMessage] = Field(default_factory=list, description="Chat messages")
- proposed_workflow: dict | None = Field(None, description="Latest workflow proposed by the copilot")
- auto_accept: bool | None = Field(None, description="Whether copilot auto-accepts workflow updates")
- work_plan: list[str] = Field(default_factory=list, description="Latest work plan the copilot model wrote")
-
-
class WorkflowCopilotChatSummary(BaseModel):
model_config = ConfigDict(from_attributes=True)
@@ -711,6 +710,18 @@ class WorkflowCopilotCredentialRequiredUpdate(BaseModel):
timestamp: datetime = Field(..., description="Server timestamp")
+class WorkflowCopilotChatHistoryResponse(BaseModel):
+ pending_credential_requests: list[WorkflowCopilotCredentialRequiredUpdate] = Field(default_factory=list)
+ question_interactions: list[QuestionInteraction] = Field(default_factory=list)
+ pending_question_cancel_token: str | None = None
+ workflow_copilot_chat_id: str | None = Field(None, description="Latest chat ID for the workflow")
+ chat_history: list[WorkflowCopilotChatHistoryMessage] = Field(default_factory=list, description="Chat messages")
+ proposed_workflow: dict | None = Field(None, description="Latest workflow proposed by the copilot")
+ auto_accept: bool | None = Field(None, description="Whether copilot auto-accepts workflow updates")
+
+ work_plan: list[str] = Field(default_factory=list, description="Latest work plan the copilot model wrote")
+
+
class WorkflowCopilotCodegenProgressUpdate(BaseModel):
"""Live-only drafting progress while the LLM streams authoring-tool arguments. Not persisted; the
workflow_draft frame supersedes it."""
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index bd2a985ff..af52da5f0 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -3247,7 +3247,19 @@ class ForLoopBlock(Block):
browser_session_id: str | None,
) -> BrowserState | None:
if browser_session_id:
- return await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(browser_session_id, organization_id)
+ context = skyvern_context.current()
+ expected_runnable_id = (
+ context.browser_session_runnable_id
+ if context and context.browser_session_runnable_id
+ else workflow_run_id
+ )
+ return await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
+ browser_session_id,
+ organization_id,
+ expected_runnable_id=expected_runnable_id,
+ expected_runnable_generation_id=(context.browser_session_runnable_generation_id if context else None),
+ workflow_run_id=workflow_run_id,
+ )
return app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)
async def _snapshot_loop_baseline_pages(
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 0400095ad..571b8c3ae 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -2165,6 +2165,7 @@ async def _recover_download_page(
browser_address=browser_address,
browser_profile_id=browser_state.browser_artifacts.applied_browser_profile_id,
browser_session_id=task.browser_session_id,
+ stale_context_is_unusable=True,
)
recovered_page = await browser_state.get_working_page()
if recovered_page is None:
diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.py
index 592a788e4..c709f7a99 100644
--- a/skyvern/webeye/browser_state.py
+++ b/skyvern/webeye/browser_state.py
@@ -69,6 +69,7 @@ class BrowserState(Protocol):
browser_address: str | None = None,
browser_profile_id: str | None = None,
browser_session_id: str | None = None,
+ stale_context_is_unusable: bool = False,
) -> None: ...
async def get_working_page(self) -> Page | None: ...
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index f7f7d5dd2..cb5534518 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -53,6 +53,10 @@ SETTLE_JITTER_MS = 500
RECOVERABLE_BLANK_PAGE_URLS = {":"}
+class _BrowserConnectionProbeFailed(str):
+ """Diagnostic reason whose stale driver may be stopped before replacement."""
+
+
def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
if not left or not right:
return False
@@ -758,7 +762,7 @@ class RealBrowserState(BrowserState):
try:
connected = bool(browser.is_connected())
except Exception as exc:
- return False, f"browser_connection_probe_failed:{type(exc).__name__}"
+ return False, _BrowserConnectionProbeFailed(f"browser_connection_probe_failed:{type(exc).__name__}")
return connected, None if connected else "browser_context_disconnected"
def is_connected(self) -> bool:
@@ -780,10 +784,23 @@ class RealBrowserState(BrowserState):
browser_address: str | None = None,
browser_profile_id: str | None = None,
browser_session_id: str | None = None,
+ stale_context_is_unusable: bool = False,
) -> None:
- # The old driver pipe is gone, so check_and_fix_state must not reuse self.pw; start a
- # fresh Playwright driver and reconnect to the same (still-alive) remote browser.
+ # Rebuild through a fresh Playwright driver only after the old connection is known dead or
+ # an explicitly unusable connection has completed bounded shutdown.
stale_pw = self.pw
+ stale_context = self.browser_context
+ stale_page = self.__page
+ stale_active_page = self.__active_page
+ stale_active_page_known_pages = self.__active_page_known_pages.copy()
+ stale_browser_artifacts = self.browser_artifacts
+ stale_browser_cleanup = self.browser_cleanup
+ stale_route_policy_url = self.browser_context_route_policy_url
+ stale_sessionless_init_script_registrations = self._sessionless_init_script_registrations.copy()
+ stale_disconnect_listener_browser = self._disconnect_listener_browser
+ stale_diagnostic = self._browser_state_diagnostic
+ stale_ever_connected = self._ever_connected
+ stale_close_requested = self._close_requested
# check_and_fix_state rebuilds through the factory; forward this session's download binding so the
# creator seam preserves the provider-selected destination on reconnect. The binding is carried
# forward, never overridden after the fact, so a genuine provider change is not mislabeled.
@@ -797,32 +814,93 @@ class RealBrowserState(BrowserState):
display_recording_owner_id = recorder.owner_id if isinstance(recorder, DisplayRecorder) else None
effective_route_policy_url = browser_context_route_policy_url or self.browser_context_route_policy_url
_stale_driver_connected, stale_driver_disconnect_reason = self._connection_status()
+ stale_driver_connection_probe_failed = isinstance(stale_driver_disconnect_reason, _BrowserConnectionProbeFailed)
stale_driver_is_known_disconnected = stale_driver_disconnect_reason in {
"playwright_driver_connection_closed",
"browser_context_disconnected",
}
+ stale_context_is_known_unusable = (
+ stale_driver_disconnect_reason
+ in {
+ "browser_context_missing",
+ "browser_context_close_called",
+ "browser_context_closed",
+ }
+ or stale_driver_connection_probe_failed
+ )
+ stale_driver_may_be_live = not stale_driver_is_known_disconnected
+ if stale_driver_may_be_live and not (stale_context_is_unusable or stale_context_is_known_unusable):
+ raise RuntimeError("Cannot replace a Playwright driver while its connection may still be live")
+ retain_guard_until_replacement = (
+ stale_driver_may_be_live
+ and (stale_context_is_unusable or stale_driver_connection_probe_failed)
+ and await app.AGENT_FUNCTION.has_retained_browser_egress_guard(stale_context)
+ )
+ stop_stale_before_replacement = stale_driver_may_be_live and not retain_guard_until_replacement
+ retry_stale_shutdown_after_replacement = False
+ if stop_stale_before_replacement:
+ stale_driver_pre_shutdown_error: BaseException | None = None
+
+ async def stop_stale_driver_before_replacement() -> None:
+ nonlocal stale_driver_pre_shutdown_error
+ try:
+ await stale_pw.stop()
+ except BaseException as exc:
+ stale_driver_pre_shutdown_error = exc
+ raise
+ finally:
+ # A cancellation-resistant stop may deliver the close event after
+ # reconnect has already propagated cancellation to its caller.
+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations.copy()
+
+ try:
+ stale_driver_stopped = await self._run_bounded_detachable(
+ stop_stale_driver_before_replacement(),
+ BROWSER_CLOSE_TIMEOUT,
+ "unusable stale Playwright driver shutdown before reconnect",
+ accept_failure_as_completion=True,
+ )
+ finally:
+ # The stop can synchronously deliver this context's close event before it
+ # times out, fails, or observes caller cancellation. Preserve the snapshot
+ # for this wrapper's next reconnect even when no replacement is started.
+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations.copy()
+ if stale_driver_pre_shutdown_error is not None:
+ retry_stale_shutdown_after_replacement = True
+ elif not stale_driver_stopped:
+ raise RuntimeError("Failed to stop unusable stale Playwright driver before reconnect")
+
+ def restore_stale_state() -> None:
+ self.pw = stale_pw
+ self.browser_context = stale_context
+ self.__page = stale_page
+ self.__active_page = stale_active_page
+ self.__active_page_known_pages = stale_active_page_known_pages
+ self.browser_artifacts = stale_browser_artifacts
+ self.browser_cleanup = stale_browser_cleanup
+ self.browser_context_route_policy_url = stale_route_policy_url
+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations
+ self._disconnect_listener_browser = stale_disconnect_listener_browser
+ self._browser_state_diagnostic = stale_diagnostic
+ self._ever_connected = stale_ever_connected
+ self._close_requested = stale_close_requested
+
+ try:
+ if self.engine_selection is not None:
+ fresh_pw = await self.engine_selection.start_driver()
+ else:
+ fresh_pw = await async_playwright().start()
+ close_driver_connection_on_transport_loss(fresh_pw)
+ except BaseException:
+ restore_stale_state()
+ raise
+
+ # Reconciliation installs the replacement guard before replaying idempotent registrations.
+ # A retained guard remains attached until its replacement is ready so an unusable context
+ # cannot expose scripts during recovery. Unguarded unusable drivers are stopped above.
+ self.pw = fresh_pw
self.browser_context = None
await self.set_working_page(None)
- # Detach the stale driver before the replacement connects. Playwright can otherwise replay
- # context init scripts through both CDP clients, duplicating effects such as keepalive timers.
- # A timeout or unknown liveness aborts because overlap is not safe to guess through. When the
- # transport is positively disconnected, a completed stop that raises keeps the legacy recovery
- # behavior: start a fresh driver rather than converting the transport loss into MissingBrowserState.
- stale_driver_stopped = await self._run_bounded_detachable(
- stale_pw.stop(),
- BROWSER_CLOSE_TIMEOUT,
- "stale Playwright driver shutdown before reconnect",
- accept_failure_as_completion=stale_driver_is_known_disconnected,
- )
- if not stale_driver_stopped:
- raise RuntimeError("Failed to stop stale Playwright driver before reconnect")
- # Reconnect on the SAME engine this state was pinned to at creation; never silently switch
- # engines underneath a live run. States built outside the per-run seam keep the stock driver.
- if self.engine_selection is not None:
- self.pw = await self.engine_selection.start_driver()
- else:
- self.pw = await async_playwright().start()
- close_driver_connection_on_transport_loss(self.pw)
try:
await self.check_and_fix_state(
proxy_location=proxy_location,
@@ -839,14 +917,63 @@ class RealBrowserState(BrowserState):
download_binding=prior_download_binding,
display_recording_owner_id=display_recording_owner_id,
reconcile_persistent_init_scripts=True,
- sessionless_init_script_registrations=self.sessionless_init_script_registrations,
+ sessionless_init_script_registrations=tuple(stale_sessionless_init_script_registrations),
)
- except Exception:
- # The caller abandons this state on failure, so stop the just-started driver too or it leaks.
+ except BaseException:
+ # Restore the old wrapper before any await so cancellation during replacement cleanup
+ # cannot leave the partially initialized replacement published on this state.
+ restore_stale_state()
await self._run_bounded_detachable(
- self.pw.stop(), BROWSER_CLOSE_TIMEOUT, "new Playwright driver shutdown after failed reconnect"
+ fresh_pw.stop(), BROWSER_CLOSE_TIMEOUT, "new Playwright driver shutdown after failed reconnect"
)
raise
+ # Pre-handoff shutdown can synchronously deliver the stale context's close event while it
+ # is still this state's current context, clearing the live list. The replacement replayed
+ # the snapshot above, so retain that same durable state for later reconnects.
+ self._sessionless_init_script_registrations = stale_sessionless_init_script_registrations.copy()
+ if stop_stale_before_replacement and not retry_stale_shutdown_after_replacement:
+ return
+
+ async def stop_stale_driver() -> None:
+ await stale_pw.stop()
+
+ stale_driver_shutdown_retry_scheduled = False
+
+ def schedule_stale_driver_shutdown_retry() -> None:
+ nonlocal stale_driver_shutdown_retry_scheduled
+ if stale_driver_shutdown_retry_scheduled:
+ return
+ stale_driver_shutdown_retry_scheduled = True
+ LOG.warning("Scheduling stale Playwright driver shutdown retry after reconnect")
+
+ async def retry_stale_driver_shutdown() -> None:
+ await self._run_bounded_detachable(
+ stale_pw.stop(),
+ BROWSER_CLOSE_TIMEOUT,
+ "retry stale Playwright driver shutdown after reconnect",
+ accept_failure_as_completion=True,
+ )
+
+ retry_task = asyncio.create_task(retry_stale_driver_shutdown())
+ self._own_detached_task(retry_task, "retry stale Playwright driver shutdown after reconnect")
+
+ def retry_failed_stale_driver_shutdown(finished: asyncio.Task[None]) -> None:
+ if finished.cancelled() or finished.exception() is not None:
+ # Scheduling from completion serializes the calls even when the first stop ignores
+ # cancellation, while a late successful completion needs no retry.
+ schedule_stale_driver_shutdown_retry()
+
+ stale_driver_shutdown_task = asyncio.create_task(stop_stale_driver())
+ stale_driver_shutdown_task.add_done_callback(retry_failed_stale_driver_shutdown)
+ await self._run_bounded_detachable(
+ stale_driver_shutdown_task,
+ BROWSER_CLOSE_TIMEOUT,
+ "stale Playwright driver shutdown after guarded reconnect",
+ # The replacement is already authoritative. The done callback owns any needed
+ # retry, so a stale-driver stop failure must not turn the successful handoff into
+ # a reconnect failure.
+ accept_failure_as_completion=True,
+ )
async def close_current_open_page(self) -> bool:
try:
diff --git a/tests/unit/test_copilot_ask_user.py b/tests/unit/test_copilot_ask_user.py
index 36aee0594..e5b4b81c3 100644
--- a/tests/unit/test_copilot_ask_user.py
+++ b/tests/unit/test_copilot_ask_user.py
@@ -18,6 +18,7 @@ from skyvern.forge.sdk.copilot.ask_user import (
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.human_input_wait import HumanInputWait, pause_human_input
from skyvern.forge.sdk.copilot.output_utils import summarize_tool_result
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
from skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepository
from skyvern.forge.sdk.schemas.workflow_copilot import CopilotPendingTurn
@@ -613,6 +614,55 @@ async def test_invalid_and_duplicate_replies_do_not_call_secret_provider(sqlite_
resolver.assert_not_awaited()
+async def test_question_reply_persists_raw_secret_detection(sqlite_engine, monkeypatch):
+ from skyvern.forge.sdk.routes import workflow_copilot as routes
+
+ _, client, ctx, frames = await setup_question_chat(sqlite_engine, monkeypatch)
+ ctx.request_policy = RequestPolicy()
+ monkeypatch.setattr(
+ routes,
+ "_screen_raw_secret_safety",
+ AsyncMock(
+ return_value=SimpleNamespace(
+ status="detected",
+ canonical_user_message="https://portal.example.com/login [REDACTED]",
+ )
+ ),
+ )
+
+ async with client:
+ task = asyncio.create_task(
+ ask_user(
+ ctx,
+ AskUserArguments.model_validate({"parts": [{"prompt": "Sign-in URL?"}]}),
+ "call",
+ )
+ )
+ question = (await asyncio.wait_for(frames.get(), 5))["interactions"][0]
+ result = await client.post(
+ "/reply",
+ json={
+ "workflow_copilot_chat_id": ctx.workflow_copilot_chat_id,
+ "interaction_id": question["interaction_id"],
+ "text": "https://portal.example.com/login password=hunter2",
+ },
+ )
+ await asyncio.wait_for(task, 5)
+
+ assert result.status_code == 200
+ assert result.json()["response"] == {
+ "answers": [],
+ "text": "https://portal.example.com/login [REDACTED]",
+ "skipped": False,
+ "raw_secret_detected": True,
+ }
+ assert ctx.request_policy.raw_secret_detected is True
+ assert ctx.request_policy.testing_intent == "skip_test"
+ assert ctx.request_policy.allow_run_blocks is False
+ assert ctx.allow_untested_workflow_draft is True
+
+
@pytest.mark.asyncio
async def test_client_checkins_extend_wait_and_abandoned_handler_exits(sqlite_engine, monkeypatch):
from skyvern.forge.sdk.db.repositories import workflow_parameters as records
diff --git a/tests/unit/test_copilot_credential_pause_reload.py b/tests/unit/test_copilot_credential_pause_reload.py
new file mode 100644
index 000000000..37a1e568a
--- /dev/null
+++ b/tests/unit/test_copilot_credential_pause_reload.py
@@ -0,0 +1,203 @@
+import asyncio
+from datetime import UTC
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot import credential_pause as pauses
+from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, _ground_user_provided_sites
+from skyvern.forge.sdk.copilot.tools.credential_fill import _request_credential
+from skyvern.forge.sdk.routes import workflow_copilot as routes
+from tests.unit.test_copilot_ask_user import setup_question_chat
+from tests.unit.test_copilot_credential_pause import _FakeCache, _make_credential, _make_stream
+
+
+async def test_recovery_capability_does_not_keep_a_direct_credential_ask_alive(sqlite_engine, monkeypatch):
+ _, _, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)
+ monkeypatch.setattr(app, "CACHE", _FakeCache())
+ ctx.client_supports_credential_pause = True
+ ctx.client_supports_credential_pause_recovery = True
+ ctx.credential_recovery_token_digest = pauses.credential_recovery_token_digest("a" * 64)
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+ ctx.request_policy = RequestPolicy()
+ ctx.stream = _make_stream(disconnected=True)
+
+ result = await _request_credential("https://portal.example.com/login", "Login", ctx)
+
+ assert result["status"] == "unanswered"
+ assert result["outcome"] == "declined"
+ ctx.stream.send.assert_not_awaited()
+
+
+async def test_disconnected_card_recovers_through_authenticated_history_and_resumes_same_call(
+ sqlite_engine, monkeypatch
+):
+ repo, client, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)
+ cache = _FakeCache()
+ monkeypatch.setattr(app, "CACHE", cache)
+ monkeypatch.setattr(
+ app.DATABASE,
+ "credentials",
+ SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()])),
+ )
+ monkeypatch.setattr(pauses, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+ monkeypatch.setattr(routes, "RECONCILE_ABANDON_AFTER_SECONDS", -1)
+ ctx.client_supports_credential_pause = True
+ ctx.client_supports_credential_pause_recovery = True
+ ctx.credential_recovery_armed = True
+ recovery_token = "a" * 64
+ ctx.credential_recovery_token_digest = pauses.credential_recovery_token_digest(recovery_token)
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+ ctx.request_policy = RequestPolicy()
+ _ground_user_provided_sites(ctx.request_policy, "https://portal.example.com/login", [])
+ ctx.stream = _make_stream(disconnected=True)
+ emitted = asyncio.Event()
+ ctx.stream.send.side_effect = lambda _: emitted.set()
+ client._transport.app.add_api_route(
+ "/credential-response", routes.workflow_copilot_credential_response, methods=["POST"]
+ )
+ async with client:
+ invocation = asyncio.create_task(_request_credential("https://portal.example.com/login", "Login", ctx))
+ try:
+ await asyncio.wait_for(emitted.wait(), 1)
+ await asyncio.sleep(0.03)
+ assert not invocation.done()
+ missing_proof = await client.get(
+ "/history", params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id}
+ )
+ assert missing_proof.json()["pending_credential_requests"] == []
+ wrong_proof = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},
+ headers={"X-Copilot-Credential-Recovery-Token": "b" * 64},
+ )
+ assert wrong_proof.json()["pending_credential_requests"] == []
+ history = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},
+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},
+ )
+ card = history.json()["pending_credential_requests"][0]
+ assert not invocation.done()
+ assert card == ctx.stream.send.await_args.args[0].model_dump(mode="json")
+ assert card["turn_id"] == "turn"
+ foreign = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},
+ headers={
+ "test-org": "other",
+ "X-Copilot-Credential-Recovery-Token": recovery_token,
+ },
+ )
+ assert foreign.json()["pending_credential_requests"] == []
+ other_chat = await repo.create_workflow_copilot_chat(organization_id="org", workflow_permanent_id="other")
+ other = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": other_chat.workflow_copilot_chat_id},
+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},
+ )
+ assert other.json()["pending_credential_requests"] == []
+ body = {
+ "workflow_copilot_chat_id": card["workflow_copilot_chat_id"],
+ "turn_id": card["turn_id"],
+ "resume_token": card["resume_token"],
+ "action": "connected",
+ "credential_id": "cred_1",
+ }
+ invalid = await client.post("/credential-response", json={**body, "resume_token": "invalid"})
+ assert invalid.status_code == 403
+ accepted = await client.post("/credential-response", json=body)
+ assert accepted.status_code == 200
+ result = await asyncio.wait_for(invocation, 1)
+ assert result["status"] == "connected"
+ assert result["credential_id"] == "cred_1"
+ reloaded = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},
+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},
+ )
+ assert reloaded.json()["pending_credential_requests"] == []
+ duplicate = await client.post("/credential-response", json=body)
+ assert duplicate.status_code == 409
+ finally:
+ if not invocation.done():
+ invocation.cancel()
+ await asyncio.gather(invocation, return_exceptions=True)
+
+
[email protected]("ending", ["expired", "cancelled"])
+async def test_history_hides_expired_or_cancelled_recoverable_card(sqlite_engine, monkeypatch, ending):
+ from datetime import datetime, timedelta
+
+ _, client, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)
+ cache = _FakeCache()
+ monkeypatch.setattr(app, "CACHE", cache)
+ monkeypatch.setattr(pauses, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+ ctx.client_supports_credential_pause = True
+ ctx.client_supports_credential_pause_recovery = True
+ ctx.credential_recovery_armed = True
+ recovery_token = "a" * 64
+ recovery_digest = pauses.credential_recovery_token_digest(recovery_token)
+ ctx.credential_recovery_token_digest = recovery_digest
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+ ctx.request_policy = RequestPolicy()
+ _ground_user_provided_sites(ctx.request_policy, "https://portal.example.com/login", [])
+ ctx.stream = _make_stream(disconnected=True)
+ emitted = asyncio.Event()
+ ctx.stream.send.side_effect = lambda _: emitted.set()
+ async with client:
+ invocation = asyncio.create_task(_request_credential("https://portal.example.com/login", "Login", ctx))
+ try:
+ await asyncio.wait_for(emitted.wait(), 1)
+ card = ctx.stream.send.await_args.args[0]
+ if ending == "expired":
+ expires_at = datetime.now(UTC) - timedelta(seconds=1)
+ cache.store[pauses.credential_pause_active_key("org", ctx.workflow_copilot_chat_id, "turn")] = (
+ pauses._encode_active_pause(
+ card.resume_token,
+ expires_at,
+ card=card.model_copy(update={"expires_at": expires_at}),
+ recovery_token_digest=recovery_digest,
+ )
+ )
+ else:
+ invocation.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await invocation
+ history = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},
+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},
+ )
+ assert history.json()["pending_credential_requests"] == []
+ with pytest.raises(pauses.CredentialPauseRejection) as rejection:
+ await pauses.resolve_credential_pause(
+ cache,
+ organization_id="org",
+ workflow_copilot_chat_id=ctx.workflow_copilot_chat_id,
+ turn_id="turn",
+ resume_token=card.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ assert rejection.value.status_code == (410 if ending == "expired" else 409)
+ finally:
+ if not invocation.done():
+ invocation.cancel()
+ await asyncio.gather(invocation, return_exceptions=True)
+
+
+async def test_pending_credential_requests_propagates_redis_failure(monkeypatch):
+ cache = AsyncMock()
+ cache.get.side_effect = RuntimeError("redis unavailable")
+ monkeypatch.setattr(app, "CACHE", cache)
+
+ with pytest.raises(RuntimeError, match="redis unavailable"):
+ await pauses.pending_credential_requests("org", "chat", ["turn"], "a" * 64)
diff --git a/tests/unit/test_copilot_question_url_authority.py b/tests/unit/test_copilot_question_url_authority.py
new file mode 100644
index 000000000..e71f4753f
--- /dev/null
+++ b/tests/unit/test_copilot_question_url_authority.py
@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from structlog.testing import capture_logs
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module
+from skyvern.forge.sdk.copilot.ask_user import AskUserArguments, QuestionInteraction, ask_user
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
+from skyvern.forge.sdk.copilot.request_policy import (
+ QuestionResponseSiteURLSource,
+ RequestPolicy,
+ UserMessageSiteURLSource,
+ _ground_user_provided_sites,
+)
+from skyvern.forge.sdk.copilot.tools.credential_fill import (
+ _credential_fill_origin_grant,
+ _request_credential,
+)
+from skyvern.forge.sdk.copilot.tools.discovery import _user_provided_entry_url
+from skyvern.forge.sdk.routes.workflow_copilot import _make_error_narrative_payload, _persist_turn_messages
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotChatHistoryMessage,
+ WorkflowCopilotChatSender,
+ WorkflowCopilotStreamMessageType,
+)
+from tests.unit.test_copilot_ask_user import setup_question_chat
+from tests.unit.test_copilot_credential_pause import (
+ _FakeCache,
+ _make_credential,
+ _make_stream,
+)
+
+_LOGIN_URL = "https://portal.example.com/login"
+
+
[email protected]("form", ["text", "answer_text"])
+async def test_authenticated_question_url_reaches_the_real_card_and_rebuilds_from_history(
+ sqlite_engine, monkeypatch: pytest.MonkeyPatch, form: str
+) -> None:
+ repo, client, ctx, frames = await setup_question_chat(sqlite_engine, monkeypatch)
+ ctx.request_policy = RequestPolicy()
+ _ground_user_provided_sites(ctx.request_policy, "Repair the login workflow", [])
+
+ async with client:
+ waiting = asyncio.create_task(
+ ask_user(
+ ctx,
+ AskUserArguments.model_validate({"parts": [{"prompt": "Paste the sign-in URL"}]}),
+ "url-call",
+ )
+ )
+ try:
+ required = await asyncio.wait_for(frames.get(), 5)
+ question = required["interactions"][0]
+ body = {
+ "workflow_copilot_chat_id": ctx.workflow_copilot_chat_id,
+ "interaction_id": question["interaction_id"],
+ }
+ if form == "text":
+ body["text"] = _LOGIN_URL
+ else:
+ body["answers"] = [{"part_id": question["parts"][0]["part_id"], "text": _LOGIN_URL}]
+
+ accepted = await client.post("/reply", json=body)
+ assert accepted.status_code == 200
+ duplicate = await client.post("/reply", json={**body, "text": "https://retry.example.net"})
+ assert duplicate.json() == accepted.json()
+ await asyncio.wait_for(waiting, 5)
+
+ source = ctx.request_policy.user_site_url_sources[_LOGIN_URL]
+ assert source == QuestionResponseSiteURLSource(interaction_id=question["interaction_id"])
+ assert ctx.request_policy.user_provided_site_urls == [_LOGIN_URL]
+ assert _user_provided_entry_url(ctx) == _LOGIN_URL
+
+ chat = await repo.get_workflow_copilot_chat_by_id("org", ctx.workflow_copilot_chat_id)
+ await _persist_turn_messages(
+ chat=chat,
+ turn_id="turn",
+ user_message="Repair the login workflow",
+ audio_artifact_id=None,
+ user_row_already_persisted=True,
+ sender=WorkflowCopilotChatSender.USER,
+ assistant_content="Response received",
+ global_llm_context=None,
+ turn_outcome=None,
+ narrative_payload=None,
+ )
+ saved = await client.get("/history", params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id})
+ persisted_history = [
+ WorkflowCopilotChatHistoryMessage.model_validate(message) for message in saved.json()["chat_history"]
+ ]
+ rebuilt = RequestPolicy()
+ _ground_user_provided_sites(rebuilt, "", persisted_history)
+ assert rebuilt.user_provided_site_urls == ctx.request_policy.user_provided_site_urls
+ assert rebuilt.user_site_url_sources == ctx.request_policy.user_site_url_sources
+
+ cache = _FakeCache()
+ cache.store[
+ credential_pause_module.credential_response_cache_key("org", ctx.workflow_copilot_chat_id, "turn")
+ ] = credential_pause_module.encode_credential_response("connected", "cred_1")
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+ monkeypatch.setattr(
+ app,
+ "DATABASE",
+ SimpleNamespace(
+ credentials=SimpleNamespace(get_credentials_by_ids=AsyncMock(return_value=[_make_credential()]))
+ ),
+ )
+ ctx.client_supports_credential_pause = True
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=5)
+ ctx.stream = _make_stream()
+
+ card_result = await _request_credential(_LOGIN_URL, "Login required", ctx)
+
+ assert card_result["status"] == "connected"
+ assert card_result["credential_id"] == "cred_1"
+ card = ctx.stream.send.await_args_list[0].args[0]
+ assert card.type is WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED
+ assert card.login_page_urls == [_LOGIN_URL]
+ finally:
+ if not waiting.done():
+ waiting.cancel()
+ await asyncio.gather(waiting, return_exceptions=True)
+
+
+def test_projection_preserves_first_origin_order_and_ignores_non_response_text() -> None:
+ interaction = QuestionInteraction.model_validate(
+ {
+ "interaction_id": "interaction-1",
+ "turn_id": "turn-1",
+ "tool_call_id": "call-1",
+ "parts": [
+ {
+ "part_id": "part-1",
+ "prompt": "Prompt https://prompt.example.net",
+ "choices": [{"choice_id": "choice-1", "text": "https://choice.example.net"}],
+ },
+ {"part_id": "part-2", "prompt": "Second", "choices": []},
+ ],
+ "status": "resolved",
+ "response": {
+ "text": "https://question.example.com/top",
+ "answers": [
+ {"part_id": "part-1", "choice_id": "choice-1", "text": None},
+ {"part_id": "part-2", "text": "https://answer.example.org/path"},
+ ],
+ "skipped": False,
+ },
+ "created_at": datetime.now(UTC),
+ "resolved_at": datetime.now(UTC),
+ }
+ )
+ skipped = interaction.model_copy(
+ update={
+ "interaction_id": "interaction-skipped",
+ "response": interaction.response.model_copy(
+ update={"text": "https://skipped.example.net", "skipped": True}
+ ),
+ }
+ )
+ unresolved = interaction.model_copy(
+ update={"interaction_id": "interaction-pending", "status": "pending", "response": None}
+ )
+ interrupted = interaction.model_copy(
+ update={"interaction_id": "interaction-interrupted", "status": "interrupted", "response": None}
+ )
+ redacted = interaction.model_copy(
+ update={
+ "interaction_id": "interaction-redacted",
+ "response": interaction.response.model_copy(update={"text": "[REDACTED]", "answers": []}),
+ }
+ )
+ narrative_payload = _make_error_narrative_payload("turn-1", None, "done")
+ narrative_payload["questionInteractions"] = [
+ interaction.model_dump(mode="json"),
+ skipped.model_dump(mode="json"),
+ unresolved.model_dump(mode="json"),
+ interrupted.model_dump(mode="json"),
+ redacted.model_dump(mode="json"),
+ ]
+ history = [
+ WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.USER,
+ content="Start at https://first.example.com/a",
+ created_at=datetime.now(UTC),
+ ),
+ WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.PRODUCT,
+ content="https://product.example.net",
+ created_at=datetime.now(UTC),
+ ),
+ WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.AI,
+ content="https://assistant.example.net",
+ narrative_payload=narrative_payload,
+ created_at=datetime.now(UTC),
+ ),
+ ]
+ policy = RequestPolicy()
+
+ _ground_user_provided_sites(
+ policy,
+ "Repeat https://question.example.com/other then https://last.example.edu/report",
+ history,
+ )
+
+ assert policy.user_provided_site_urls == [
+ "https://first.example.com/a",
+ "https://question.example.com/top",
+ "https://answer.example.org/path",
+ "https://last.example.edu/report",
+ ]
+ assert policy.user_site_url_sources == {
+ "https://first.example.com/a": UserMessageSiteURLSource(message_index=1),
+ "https://question.example.com/top": QuestionResponseSiteURLSource(interaction_id="interaction-1"),
+ "https://answer.example.org/path": QuestionResponseSiteURLSource(interaction_id="interaction-1"),
+ "https://last.example.edu/report": UserMessageSiteURLSource(message_index=2),
+ }
+
+
+def test_immediate_question_projection_appends_without_replacing_composer_urls() -> None:
+ policy = RequestPolicy(
+ user_provided_site_urls=["https://first.example.com/login"],
+ user_site_url_sources={
+ "https://first.example.com/login": UserMessageSiteURLSource(message_index=1),
+ },
+ )
+ interaction = QuestionInteraction.model_validate(
+ {
+ "interaction_id": "interaction-1",
+ "turn_id": "turn-1",
+ "tool_call_id": "call-1",
+ "parts": [{"part_id": "part-1", "prompt": "Backup URL?", "choices": []}],
+ "status": "resolved",
+ "response": {"text": "https://second.example.net/login", "skipped": False},
+ "created_at": datetime.now(UTC),
+ "resolved_at": datetime.now(UTC),
+ }
+ )
+
+ policy.project_question_response_sites(interaction)
+
+ assert policy.user_provided_site_urls == [
+ "https://first.example.com/login",
+ "https://second.example.net/login",
+ ]
+ assert policy.user_site_url_sources["https://first.example.com/login"] == UserMessageSiteURLSource(message_index=1)
+ assert policy.user_site_url_sources["https://second.example.net/login"] == QuestionResponseSiteURLSource(
+ interaction_id="interaction-1"
+ )
+
+
+def test_question_response_with_detected_raw_secret_grants_no_url_authority() -> None:
+ policy = RequestPolicy()
+ interaction = QuestionInteraction.model_validate(
+ {
+ "interaction_id": "interaction-1",
+ "turn_id": "turn-1",
+ "tool_call_id": "call-1",
+ "parts": [{"part_id": "part-1", "prompt": "Sign-in URL?", "choices": []}],
+ "status": "resolved",
+ "response": {
+ "text": "https://portal.example.com/login",
+ "skipped": False,
+ "raw_secret_detected": True,
+ },
+ "created_at": datetime.now(UTC),
+ "resolved_at": datetime.now(UTC),
+ }
+ )
+
+ policy.project_question_response_sites(interaction)
+
+ assert policy.user_provided_site_urls == []
+ assert policy.user_site_url_sources == {}
+ assert policy.raw_secret_detected is True
+ assert policy.raw_secret_safety_status == "detected"
+ assert policy.testing_intent == "skip_test"
+ assert policy.allow_run_blocks is False
+ assert policy.allow_missing_credentials_in_draft is True
+ assert policy.credential_draft_deferred_explicitly is True
+
+
+async def test_question_source_reaches_fill_boundary_without_weakening_origin_guard(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ source = QuestionResponseSiteURLSource(interaction_id="interaction-1")
+ policy = RequestPolicy(
+ resolved_credentials=[SimpleNamespace(credential_id="cred_1", tested_url=None)],
+ current_turn_named_credential_ids={"cred_1"},
+ user_provided_site_urls=[_LOGIN_URL],
+ user_site_url_sources={_LOGIN_URL: source},
+ )
+ ctx = SimpleNamespace(
+ organization_id="org",
+ request_policy=policy,
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ org_credentials_for_turn=None,
+ vault_login_uris_by_credential_id={},
+ persisted_workflow_yaml="",
+ )
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.tools.credential_fill._live_working_page_url",
+ AsyncMock(return_value=_LOGIN_URL),
+ ),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.credential_fill._vault_named_sites",
+ AsyncMock(return_value=[]),
+ ),
+ capture_logs() as logs,
+ ):
+ grant, error = await _credential_fill_origin_grant(ctx, "cred_1")
+
+ assert error is None
+ assert grant is not None
+ event = next(log for log in logs if log.get("event") == "copilot credential fill grant")
+ assert event["source_kind"] == "question_response"
+ assert event["source_interaction_id"] == "interaction-1"
+ assert "source_user_message" not in event
+ assert "/login" not in str(event)
+
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.credential_fill._live_working_page_url",
+ AsyncMock(return_value="https://unrelated.example.net/login"),
+ ):
+ unrelated_grant, unrelated_error = await _credential_fill_origin_grant(ctx, "cred_1")
+ assert unrelated_grant is None
+ assert unrelated_error is not None
+
+
+def test_malformed_persisted_question_record_cannot_supply_origin() -> None:
+ payload = _make_error_narrative_payload("turn", None, "done")
+ payload["questionInteractions"] = [
+ {"interaction_id": "malformed", "status": "resolved", "response": {"text": _LOGIN_URL}}
+ ]
+ history = [
+ WorkflowCopilotChatHistoryMessage(
+ sender=WorkflowCopilotChatSender.AI, content="", narrative_payload=payload, created_at=datetime.now(UTC)
+ )
+ ]
+ policy = RequestPolicy()
+ _ground_user_provided_sites(policy, "", history)
+ assert policy.user_provided_site_urls == []
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 190b17e9a..00d72bcf5 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -1471,6 +1471,7 @@ async def test_handle_action_reconnects_context_closed_during_download_wait(
browser_address=task.browser_address,
browser_profile_id=None,
browser_session_id=task.browser_session_id,
+ stale_context_is_unusable=True,
)
browser_state.navigate_to_url.assert_awaited_once_with(page=recovered_page, url=page.url)
browser_state.set_active_page.assert_awaited_once_with(recovered_page)
diff --git a/tests/unit/test_forloop_incremental_persist.py b/tests/unit/test_forloop_incremental_persist.py
index 038a6545d..4b7580206 100644
--- a/tests/unit/test_forloop_incremental_persist.py
+++ b/tests/unit/test_forloop_incremental_persist.py
@@ -420,6 +420,30 @@ class TestResetBrowserTabsBetweenIterations:
loop_blocks=[TaskBlock(label="inner", output_parameter=_make_output_param("inner"))],
)
+ @pytest.mark.asyncio
+ async def test_persistent_loop_browser_lookup_presents_workflow_owner(self) -> None:
+ loop_block = self._loop_block()
+ browser_state = MagicMock(spec=RealBrowserState)
+ context = MagicMock()
+ context.browser_session_runnable_id = "wr_owner"
+ context.browser_session_runnable_generation_id = "generation_owner"
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(block_module.skyvern_context, "current", return_value=context),
+ ):
+ mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state = AsyncMock(return_value=browser_state)
+
+ result = await loop_block._get_loop_browser_state("wr_test", "org_test", "pbs_test")
+
+ assert result is browser_state
+ mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with(
+ "pbs_test",
+ "org_test",
+ expected_runnable_id="wr_owner",
+ expected_runnable_generation_id="generation_owner",
+ workflow_run_id="wr_test",
+ )
+
@pytest.mark.asyncio
async def test_baseline_snapshot_once_and_reset_between_iterations(self) -> None:
loop_block = self._loop_block()
@@ -478,7 +502,13 @@ class TestResetBrowserTabsBetweenIterations:
mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state = AsyncMock(return_value=mock_bs)
await loop_block._reset_browser_tabs_for_iteration("wr_test", "org_test", "pbs_test", baseline)
reset_browser_tabs.assert_awaited_once_with("org_test")
- mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with("pbs_test", "org_test")
+ mock_app.PERSISTENT_SESSIONS_MANAGER.get_browser_state.assert_awaited_once_with(
+ "pbs_test",
+ "org_test",
+ expected_runnable_id="wr_test",
+ expected_runnable_generation_id=None,
+ workflow_run_id="wr_test",
+ )
mock_bs.close_pages_opened_after.assert_awaited_once_with(baseline)
@pytest.mark.asyncio
diff --git a/tests/unit/test_real_browser_state_reload.py b/tests/unit/test_real_browser_state_reload.py
index 4851d0684..eb3d018ea 100644
--- a/tests/unit/test_real_browser_state_reload.py
+++ b/tests/unit/test_real_browser_state_reload.py
@@ -207,6 +207,7 @@ async def test_scrape_with_type_normal_no_reload_call() -> None:
def _reconnect_state(binding: DownloadBinding) -> RealBrowserState:
state = RealBrowserState(pw=AsyncMock(), browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
state.engine_selection = MagicMock()
state.engine_selection.start_driver = AsyncMock(return_value=AsyncMock())
state.set_working_page = AsyncMock()
@@ -239,6 +240,7 @@ async def test_reconnect_keeps_run_dir_binding() -> None:
def _reconnect_state_capturing_check(binding: DownloadBinding, captured: dict[str, object]) -> RealBrowserState:
state = RealBrowserState(pw=AsyncMock(), browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
state.engine_selection = MagicMock()
state.engine_selection.start_driver = AsyncMock(return_value=AsyncMock())
state.set_working_page = AsyncMock()
diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.py
index 10f36f479..65dd20960 100644
--- a/tests/unit/test_request_logging.py
+++ b/tests/unit/test_request_logging.py
@@ -96,6 +96,9 @@ class TestIsSensitiveKey:
"Cookie",
"x-api-key",
"X-Api-Key",
+ "X-Copilot-Credential-Recovery-Token",
+ "credential_recovery_token",
+ "resume_token",
# Whole header dicts, masked wholesale because their inner key names are
# caller-chosen and therefore unmatchable.
"extra_http_headers",
@@ -1178,3 +1181,55 @@ class TestRawRequestLogLevel:
assert log_mock.warning.call_args.args[0] == "api.raw_request"
assert log_mock.warning.call_args.kwargs["status_code"] == status_code
log_mock.info.assert_not_called()
+
+
[email protected]("status_code", [200, 403, 500])
+def test_copilot_recovery_capabilities_never_enter_request_logs(
+ log_mock: MagicMock, monkeypatch: pytest.MonkeyPatch, status_code: int
+) -> None:
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS_SUCCESSFUL_READS", True)
+ app = _make_app()
+ recovery_proof = "a" * 64
+ selection_token = "synthetic-selection-capability"
+
+ @app.get("/v1/workflow/copilot/chat-history")
+ async def history() -> dict:
+ if status_code == 403:
+ raise HTTPException(status_code=403, detail="Forbidden")
+ if status_code == 500:
+ raise RuntimeError("Synthetic history failure")
+ return {"pending_credential_requests": [{"resume_token": selection_token, "turn_id": "turn"}]}
+
+ response = TestClient(app, raise_server_exceptions=False).get(
+ "/v1/workflow/copilot/chat-history",
+ headers={"X-Copilot-Credential-Recovery-Token": recovery_proof, "X-Request-ID": "request"},
+ )
+ assert response.status_code == status_code
+ logger = log_mock.info if status_code == 200 else log_mock.warning if status_code == 403 else log_mock.error
+ logger.assert_called_once()
+ fields = logger.call_args.kwargs
+ assert fields["headers"]["x-request-id"] == "request"
+ assert "x-copilot-credential-recovery-token" not in fields["headers"]
+ assert recovery_proof not in str(log_mock.mock_calls)
+ assert selection_token not in str(log_mock.mock_calls)
+ if status_code == 200:
+ assert response.json()["pending_credential_requests"][0]["resume_token"] == selection_token
+ assert json.loads(fields["response_body"])["pending_credential_requests"] == [
+ {"resume_token": REDACTED, "turn_id": "turn"}
+ ]
+
+
+def test_structured_recovery_capabilities_are_redacted_without_masking_turn_identity() -> None:
+ assert redact_sensitive_fields(
+ {
+ "headers": {"X-Copilot-Credential-Recovery-Token": "synthetic-proof"},
+ "credential_recovery_token": "synthetic-proof",
+ "resume_token": "synthetic-selection",
+ "turn_id": "turn",
+ }
+ ) == {
+ "headers": {"X-Copilot-Credential-Recovery-Token": REDACTED},
+ "credential_recovery_token": REDACTED,
+ "resume_token": REDACTED,
+ "turn_id": "turn",
+ }
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 97595e182..de4e07f08 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -25,6 +25,7 @@ from skyvern.forge import app
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.ask_user import QuestionInteraction, QuestionResponse
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode
from skyvern.forge.sdk.copilot.build_test_connect_failure import (
SUPERSEDED_BY_NEWER_TEST_REASON,
@@ -3492,6 +3493,77 @@ async def test_chat_history_marks_abandoned_turn_interrupted_not_cancelled(
restore_mock.assert_not_awaited()
+async def test_redis_failure_does_not_protect_an_unrelated_stale_turn(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat = _make_persisted_chat([_make_pending_turn("turn-a", RECONCILE_ABANDON_AFTER_SECONDS + 60)])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+ monkeypatch.setattr(workflow_copilot_route, "credential_pause_is_active", AsyncMock(return_value=None))
+
+ await _load_history()
+
+ assert store.claim_calls == ["turn-a"]
+ assert chat.pending_turns == {}
+
+
+async def test_redis_failure_briefly_protects_a_recent_question_handoff(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ resolved_at = datetime.now(timezone.utc) - timedelta(seconds=RECONCILE_ABANDON_AFTER_SECONDS + 60)
+ interaction = QuestionInteraction(
+ interaction_id="question-1",
+ turn_id="turn-a",
+ tool_call_id="call-1",
+ parts=[],
+ status="resolved",
+ response=QuestionResponse(text="https://portal.example.com/login"),
+ resolved_at=resolved_at,
+ )
+ entry = _make_pending_turn(
+ "turn-a",
+ RECONCILE_ABANDON_AFTER_SECONDS + 120,
+ question_interactions=[interaction],
+ )
+ chat = _make_persisted_chat([entry])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+ monkeypatch.setattr(workflow_copilot_route, "credential_pause_is_active", AsyncMock(return_value=None))
+
+ await _load_history()
+
+ assert store.claim_calls == []
+ assert list(chat.pending_turns) == ["turn-a"]
+
+
+async def test_redis_failure_eventually_releases_an_old_question_handoff(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ age = RECONCILE_ABANDON_AFTER_SECONDS + settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS + 60
+ interaction = QuestionInteraction(
+ interaction_id="question-1",
+ turn_id="turn-a",
+ tool_call_id="call-1",
+ parts=[],
+ status="resolved",
+ response=QuestionResponse(text="https://portal.example.com/login"),
+ resolved_at=datetime.now(timezone.utc) - timedelta(seconds=age),
+ )
+ entry = _make_pending_turn("turn-a", age + 60, question_interactions=[interaction])
+ chat = _make_persisted_chat([entry])
+ store, _ = _install_reconcile_store(monkeypatch, chat)
+ store.add_message(WorkflowCopilotChatSender.USER, "build me a scraper")
+ monkeypatch.setattr(workflow_copilot_route, "credential_pause_is_active", AsyncMock(return_value=None))
+
+ await _load_history()
+
+ assert store.claim_calls == ["turn-a"]
+ assert chat.pending_turns == {}
+
+
@pytest.mark.asyncio
async def test_the_recovered_row_carries_what_is_known_and_never_says_why(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/workflow/test_block_browser_state_recovery.py b/tests/unit/workflow/test_block_browser_state_recovery.py
index 8e58caad0..d7f35d777 100644
--- a/tests/unit/workflow/test_block_browser_state_recovery.py
+++ b/tests/unit/workflow/test_block_browser_state_recovery.py
@@ -425,7 +425,7 @@ async def test_is_connected_false_after_real_driver_stop(tmp_path: Path) -> None
@pytest.mark.asyncio
-async def test_reconnect_starts_fresh_driver_and_stops_stale_one(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_reconnect_restores_replacement_before_stopping_stale_driver(monkeypatch: pytest.MonkeyPatch) -> None:
events: list[str] = []
stale_pw = MagicMock()
stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))
@@ -437,15 +437,15 @@ async def test_reconnect_starts_fresh_driver_and_stops_stale_one(monkeypatch: py
return fresh_pw
monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
-
state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild and replay scripts"))
monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
await state.reconnect(browser_address="ws://remote-browser")
assert state.pw is fresh_pw
- assert events == ["stop stale driver", "start fresh driver", "rebuild and replay scripts"]
+ assert events == ["start fresh driver", "rebuild and replay scripts", "stop stale driver"]
stale_pw.stop.assert_awaited_once()
assert check_and_fix.await_args.kwargs["browser_address"] == "ws://remote-browser"
@@ -464,14 +464,154 @@ async def test_reconnect_stops_fresh_driver_when_state_rebuild_fails(monkeypatch
monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=RuntimeError("cdp handshake failed")))
with pytest.raises(RuntimeError, match="cdp handshake failed"):
await state.reconnect(browser_address="ws://remote-browser")
- # A failed rebuild must stop both drivers so it never orphans the freshly started one.
+ # The failed replacement is stopped while the still-guarded stale driver remains attached.
fresh_pw.stop.assert_awaited_once()
- stale_pw.stop.assert_awaited_once()
+ stale_pw.stop.assert_not_awaited()
+ assert state.pw is stale_pw
+
+
+async def test_reconnect_cancellation_restores_still_guarded_stale_state(monkeypatch: pytest.MonkeyPatch) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(return_value=None)
+ fresh_pw = MagicMock()
+ fresh_pw.stop = AsyncMock(return_value=None)
+
+ class _FakeAsyncPlaywright:
+ async def start(self) -> object:
+ return fresh_pw
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+
+ stale_context = MagicMock()
+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
+ stale_listener_browser = MagicMock()
+ state._disconnect_listener_browser = stale_listener_browser
+ registration = object()
+ state.add_sessionless_init_script_registration(registration)
+
+ async def cancel_after_replacement_context_was_published(**_kwargs: object) -> None:
+ replacement_context = MagicMock()
+ state.browser_context = replacement_context
+ state._disconnect_listener_browser = MagicMock()
+ state._on_browser_context_closed(replacement_context)
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(
+ state,
+ "check_and_fix_state",
+ AsyncMock(side_effect=cancel_after_replacement_context_was_published),
+ )
+
+ with pytest.raises(asyncio.CancelledError):
+ await state.reconnect(browser_address="ws://remote-browser")
+
+ assert state.pw is stale_pw
+ assert state.browser_context is stale_context
+ assert state._disconnect_listener_browser is stale_listener_browser
+ assert state.sessionless_init_script_registrations == (registration,)
+ fresh_pw.stop.assert_awaited_once()
+ stale_pw.stop.assert_not_awaited()
+
+
+async def test_successful_pre_handoff_reconnect_preserves_sessionless_registrations_for_next_reconnect(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ initial_pw = MagicMock()
+ first_replacement_pw = MagicMock()
+ second_replacement_pw = MagicMock()
+ start_fresh = AsyncMock(side_effect=[first_replacement_pw, second_replacement_pw])
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))
+
+ initial_context = MagicMock()
+ replacement_contexts = [MagicMock(), MagicMock()]
+ state = RealBrowserState(pw=initial_pw, browser_context=initial_context)
+ state._connection_status = MagicMock(return_value=(True, None))
+ registration = object()
+ state.add_sessionless_init_script_registration(registration)
+ replayed_registrations: list[tuple[object, ...]] = []
+
+ async def stop_current_driver() -> None:
+ current_context = state.browser_context
+ assert current_context is not None
+ state._on_browser_context_closed(current_context)
+
+ initial_pw.stop = AsyncMock(side_effect=stop_current_driver)
+ first_replacement_pw.stop = AsyncMock(side_effect=stop_current_driver)
+
+ async def rebuild(**kwargs: object) -> None:
+ replayed_registrations.append(kwargs["sessionless_init_script_registrations"]) # type: ignore[arg-type]
+ state.browser_context = replacement_contexts.pop(0)
+
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=rebuild))
+
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+
+ assert replayed_registrations == [(registration,), (registration,)]
+ assert state.sessionless_init_script_registrations == (registration,)
+
+
+async def test_reconnect_restores_stale_state_before_cancelled_replacement_shutdown(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(return_value=None)
+ fresh_pw = MagicMock()
+ shutdown_started = asyncio.Event()
+
+ async def wait_during_replacement_shutdown() -> None:
+ shutdown_started.set()
+ await asyncio.Event().wait()
+
+ fresh_pw.stop = AsyncMock(side_effect=wait_during_replacement_shutdown)
+
+ class _FakeAsyncPlaywright:
+ async def start(self) -> object:
+ return fresh_pw
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+
+ stale_context = MagicMock()
+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
+ replacement_context = MagicMock()
+
+ async def fail_after_replacement_context_was_published(**_kwargs: object) -> None:
+ state.browser_context = replacement_context
+ raise RuntimeError("replacement setup failed")
+
+ monkeypatch.setattr(
+ state,
+ "check_and_fix_state",
+ AsyncMock(side_effect=fail_after_replacement_context_was_published),
+ )
+
+ reconnect = asyncio.create_task(state.reconnect(browser_address="ws://remote-browser"))
+ await asyncio.wait_for(shutdown_started.wait(), timeout=1)
+ reconnect.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await reconnect
+
+ assert state.pw is stale_pw
+ assert state.browser_context is stale_context
+ fresh_pw.stop.assert_awaited_once()
+ stale_pw.stop.assert_not_awaited()
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
@pytest.mark.asyncio
@@ -497,6 +637,7 @@ async def test_reconnect_bounds_fresh_driver_shutdown_when_state_rebuild_fails(
monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)
state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=RuntimeError("cdp handshake failed")))
with pytest.raises(RuntimeError, match="cdp handshake failed"):
@@ -504,7 +645,8 @@ async def test_reconnect_bounds_fresh_driver_shutdown_when_state_rebuild_fails(
assert stop_started.is_set()
fresh_pw.stop.assert_awaited_once()
- stale_pw.stop.assert_awaited_once()
+ stale_pw.stop.assert_not_awaited()
+ assert state.pw is stale_pw
@pytest.mark.asyncio
@@ -526,17 +668,22 @@ async def test_reconnect_recovers_when_stale_driver_shutdown_has_already_failed(
monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
await state.reconnect(browser_address="ws://remote-browser")
+ await asyncio.sleep(0)
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
+ await asyncio.sleep(0)
+ assert stale_pw.stop.await_count == 2
start_fresh.assert_awaited_once()
check_and_fix.assert_awaited_once()
+ assert state._detached_teardown_tasks == set()
@pytest.mark.asyncio
-async def test_reconnect_does_not_overlap_driver_when_stale_connection_may_still_be_live(
+async def test_reconnect_refuses_to_overlap_driver_when_stale_connection_may_still_be_live(
monkeypatch: pytest.MonkeyPatch,
) -> None:
stale_pw = MagicMock()
- stale_pw.stop = AsyncMock(side_effect=RuntimeError("stale driver still attached"))
+ stale_pw.stop = AsyncMock()
start_fresh = AsyncMock()
class _FakeAsyncPlaywright:
@@ -545,28 +692,261 @@ async def test_reconnect_does_not_overlap_driver_when_stale_connection_may_still
monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
- state._connection_status = MagicMock(side_effect=[(True, None), (False, "browser_context_missing")])
+ state._connection_status = MagicMock(return_value=(True, None))
check_and_fix = AsyncMock()
monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
- with pytest.raises(RuntimeError, match="stale Playwright driver"):
+ with pytest.raises(RuntimeError, match="may still be live"):
await state.reconnect(browser_address="ws://remote-browser")
+ stale_pw.stop.assert_not_awaited()
start_fresh.assert_not_awaited()
check_and_fix.assert_not_awaited()
- # reconnect clears the context before attempting stop. A retry must treat that missing context
- # as unknown rather than proof that the first driver detached.
- with pytest.raises(RuntimeError, match="stale Playwright driver"):
- await state.reconnect(browser_address="ws://remote-browser")
+
[email protected]("disconnect_reason", ["browser_context_missing", "browser_context_closed"])
+async def test_reconnect_stops_known_unusable_context_before_starting_replacement(
+ monkeypatch: pytest.MonkeyPatch,
+ disconnect_reason: str,
+) -> None:
+ events: list[str] = []
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))
+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, disconnect_reason))
+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild"))
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser")
+
+ assert events == ["stop stale driver", "start fresh driver", "rebuild"]
+
+
+async def test_reconnect_stops_driver_after_connection_probe_failure_before_starting_replacement(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[str] = []
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))
+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))
+
+ stale_context = MagicMock()
+ stale_context._impl_obj = None
+ stale_context.browser.is_connected.side_effect = RuntimeError("connection probe failed")
+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)
+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild"))
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser")
+
+ assert events == ["stop stale driver", "start fresh driver", "rebuild"]
+
+
+async def test_reconnect_keeps_guard_through_connection_probe_failure_handoff(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[str] = []
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))
+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))
+
+ stale_context = MagicMock()
+ stale_context._impl_obj = None
+ stale_context.browser.is_connected.side_effect = RuntimeError("connection probe failed")
+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)
+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild and restore guard"))
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser")
+
+ assert events == ["start fresh driver", "rebuild and restore guard", "stop stale driver"]
+
+
+async def test_reconnect_stops_explicitly_unusable_live_context_before_starting_replacement(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[str] = []
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))
+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(True, None))
+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild"))
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+
+ assert events == ["stop stale driver", "start fresh driver", "rebuild"]
+
+
+async def test_reconnect_replaces_guarded_live_context_before_stopping_stale_driver(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[str] = []
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=lambda: events.append("stop stale driver"))
+ start_fresh = AsyncMock(side_effect=lambda: events.append("start fresh driver"))
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(True, None))
+ check_and_fix = AsyncMock(side_effect=lambda **_: events.append("rebuild and replay scripts"))
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+
+ assert events == ["start fresh driver", "rebuild and replay scripts", "stop stale driver"]
+
+
+async def test_reconnect_recovers_when_unusable_live_driver_stop_raises(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=[RuntimeError("driver still attached"), None])
+ start_fresh = AsyncMock()
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(True, None))
+ check_and_fix = AsyncMock()
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
assert stale_pw.stop.await_count == 2
+ start_fresh.assert_awaited_once()
+ check_and_fix.assert_awaited_once()
+
+
+async def test_reconnect_aborts_when_unguarded_unusable_driver_shutdown_times_out(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def hang_during_stop() -> None:
+ state._on_browser_context_closed(stale_context)
+ await asyncio.Event().wait()
+
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=hang_during_stop)
+ start_fresh = AsyncMock()
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))
+
+ stale_context = MagicMock()
+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)
+ state._connection_status = MagicMock(return_value=(True, None))
+ registration = object()
+ state.add_sessionless_init_script_registration(registration)
+ check_and_fix = AsyncMock()
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ with pytest.raises(RuntimeError, match="Failed to stop unusable stale Playwright driver"):
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
+
start_fresh.assert_not_awaited()
check_and_fix.assert_not_awaited()
+ assert state.sessionless_init_script_registrations == (registration,)
+
+
+async def test_reconnect_cancellation_during_pre_handoff_shutdown_restores_sessionless_registrations(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stop_started = asyncio.Event()
+ release_stop = asyncio.Event()
+
+ async def stop_after_context_close() -> None:
+ stop_started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ await release_stop.wait()
+ state._on_browser_context_closed(stale_context)
+
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=stop_after_context_close)
+ start_fresh = AsyncMock()
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=False))
+
+ stale_context = MagicMock()
+ state = RealBrowserState(pw=stale_pw, browser_context=stale_context)
+ state._connection_status = MagicMock(return_value=(True, None))
+ registration = object()
+ state.add_sessionless_init_script_registration(registration)
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())
+
+ reconnect = asyncio.create_task(
+ state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+ )
+ await asyncio.wait_for(stop_started.wait(), timeout=1)
+ reconnect.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await reconnect
+
+ start_fresh.assert_not_awaited()
+ assert state.sessionless_init_script_registrations == (registration,)
+
+ release_stop.set()
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
+ assert state.sessionless_init_script_registrations == (registration,)
@pytest.mark.asyncio
-async def test_reconnect_bounds_stale_driver_shutdown_before_starting_fresh_driver(
+async def test_reconnect_bounds_stale_driver_shutdown_after_guarded_replacement(
monkeypatch: pytest.MonkeyPatch,
) -> None:
stop_started = asyncio.Event()
@@ -586,15 +966,236 @@ async def test_reconnect_bounds_stale_driver_shutdown_before_starting_fresh_driv
monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)
state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
check_and_fix = AsyncMock()
monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
- with pytest.raises(RuntimeError, match="stale Playwright driver"):
- await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)
+ await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)
+ await asyncio.sleep(0)
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
assert stop_started.is_set()
- start_fresh.assert_not_awaited()
- check_and_fix.assert_not_awaited()
+ assert stale_pw.stop.await_count == 2
+ start_fresh.assert_awaited_once()
+ check_and_fix.assert_awaited_once()
+ assert state._detached_teardown_tasks == set()
+
+
+async def test_reconnect_does_not_retry_until_cancellation_resistant_stale_shutdown_completes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ release_stop = asyncio.Event()
+
+ async def cancellation_resistant_stop() -> None:
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ await release_stop.wait()
+
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=cancellation_resistant_stop)
+ start_fresh = AsyncMock()
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())
+
+ await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)
+ await asyncio.sleep(0.02)
+
+ # The first stop remains owned without racing a second stop.
+ assert stale_pw.stop.await_count == 1
+ assert len(state._detached_teardown_tasks) == 1
+
+ release_stop.set()
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
+ await asyncio.sleep(0)
+ assert state._detached_teardown_tasks == set()
+
+
+async def test_reconnect_retries_after_cancellation_resistant_stale_shutdown_eventually_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ release_stop = asyncio.Event()
+ stop_attempts = 0
+
+ async def cancellation_resistant_stop() -> None:
+ nonlocal stop_attempts
+ stop_attempts += 1
+ if stop_attempts != 1:
+ return
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ await release_stop.wait()
+ raise RuntimeError("late shutdown failure")
+
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=cancellation_resistant_stop)
+ start_fresh = AsyncMock()
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(real_browser_state_module, "BROWSER_CLOSE_TIMEOUT", 0.01)
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed"))
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())
+
+ await asyncio.wait_for(state.reconnect(browser_address="ws://remote-browser"), timeout=0.1)
+ await asyncio.sleep(0.02)
+ assert stale_pw.stop.await_count == 1
+
+ release_stop.set()
+
+ async def wait_for_retry() -> None:
+ while stale_pw.stop.await_count < 2:
+ await asyncio.sleep(0)
+
+ await asyncio.wait_for(wait_for_retry(), timeout=0.1)
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
+ await asyncio.sleep(0)
+
+ assert stale_pw.stop.await_count == 2
+ assert state._detached_teardown_tasks == set()
+
+
+async def test_reconnect_retries_post_replacement_stale_driver_stop_exception(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=[RuntimeError("driver still attached"), None])
+ fresh_pw = MagicMock()
+ start_fresh = AsyncMock(return_value=fresh_pw)
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(True, None))
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())
+
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+ await asyncio.gather(*list(state._detached_teardown_tasks))
+ await asyncio.sleep(0)
+
+ assert state.pw is fresh_pw
+ assert stale_pw.stop.await_count == 2
+ assert state._detached_teardown_tasks == set()
+
+
+async def test_reconnect_cancellation_during_post_replacement_shutdown_retains_retry(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ first_stop_started = asyncio.Event()
+ stop_attempts = 0
+
+ async def cancellation_sensitive_stop() -> None:
+ nonlocal stop_attempts
+ stop_attempts += 1
+ if stop_attempts == 1:
+ first_stop_started.set()
+ await asyncio.Event().wait()
+
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(side_effect=cancellation_sensitive_stop)
+ fresh_pw = MagicMock()
+ start_fresh = AsyncMock(return_value=fresh_pw)
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(True, None))
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())
+
+ reconnect = asyncio.create_task(
+ state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+ )
+ await asyncio.wait_for(first_stop_started.wait(), timeout=1)
+ reconnect.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await reconnect
+ await asyncio.sleep(0)
+ await asyncio.gather(*list(state._detached_teardown_tasks), return_exceptions=True)
+ await asyncio.sleep(0)
+
+ assert state.pw is fresh_pw
+ assert stale_pw.stop.await_count == 2
+ assert state._detached_teardown_tasks == set()
+
+
+async def test_reconnect_cancellation_before_stale_shutdown_starts_wakes_retry(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock()
+ fresh_pw = MagicMock()
+ start_fresh = AsyncMock(return_value=fresh_pw)
+
+ class _FakeAsyncPlaywright:
+ start = start_fresh
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+ monkeypatch.setattr(app.AGENT_FUNCTION, "has_retained_browser_egress_guard", AsyncMock(return_value=True))
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ state._connection_status = MagicMock(return_value=(True, None))
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock())
+ bounded_calls = 0
+
+ async def cancel_first_phase_before_start(
+ phase: object,
+ _timeout: float,
+ _description: str,
+ *,
+ accept_failure_as_completion: bool = False,
+ ) -> bool:
+ nonlocal bounded_calls
+ del accept_failure_as_completion
+ bounded_calls += 1
+ if bounded_calls == 1:
+ phase_task = asyncio.ensure_future(phase) # type: ignore[arg-type]
+ phase_task.cancel()
+ await asyncio.gather(phase_task, return_exceptions=True)
+ raise asyncio.CancelledError
+ await phase # type: ignore[misc]
+ return True
+
+ monkeypatch.setattr(state, "_run_bounded_detachable", cancel_first_phase_before_start)
+
+ with pytest.raises(asyncio.CancelledError):
+ await state.reconnect(browser_address="ws://remote-browser", stale_context_is_unusable=True)
+ detached = list(state._detached_teardown_tasks)
+ done, pending = await asyncio.wait(detached, timeout=0.1)
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
+ await asyncio.sleep(0)
+
+ assert state.pw is fresh_pw
+ assert len(done) == len(detached)
+ stale_pw.stop.assert_awaited_once()
+ assert state._detached_teardown_tasks == set()
@pytest.mark.asyncio