Skip to content

Automate browser based workflows with AI

Log live-stream outcomes for persistent-session and vendor live views (SKY-16903) (#8817)

Shuchang Zheng2d95a501f80d
diff --git a/alembic/versions/2026_10_05_0412-8f3cbbcbbf34_index_run_group_items_by_item_key.py b/alembic/versions/2026_10_05_0412-8f3cbbcbbf34_index_run_group_items_by_item_key.py
new file mode 100644
index 000000000..140fc0656
--- /dev/null
+++ b/alembic/versions/2026_10_05_0412-8f3cbbcbbf34_index_run_group_items_by_item_key.py
@@ -0,0 +1,29 @@
+"""index run group items by item key
+
+Revision ID: 8f3cbbcbbf34
+Revises: 798a79449b1e
+Create Date: 2026-10-05T04:12:47.344306+00:00
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "8f3cbbcbbf34"
+down_revision: Union[str, None] = "798a79449b1e"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.create_index(
+ "ix_workflow_run_group_items_item_key_created_at",
+ "workflow_run_group_items",
+ ["item_key", "created_at"],
+ )
+
+
+def downgrade() -> None:
+ op.drop_index("ix_workflow_run_group_items_item_key_created_at", table_name="workflow_run_group_items")
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index b6be207c9..cad6c05b1 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -187,6 +187,14 @@ import {
ConnectedAccountChoiceCard,
ConnectedAccountChoiceMarker,
} from "./cards/ConnectedAccountChoiceCard";
+import {
+ AccountGroupCancelReceipt,
+ AccountGroupReceiptCard,
+} from "./cards/AccountGroupReceiptCard";
+import {
+ AccountGroupCancelCard,
+ AccountGroupReviewCard,
+} from "./cards/AccountGroupReviewCard";
import { QuestionReceipt } from "./cards/QuestionReceipt";
import { QUESTION_PROMPT_ID, QuestionTray } from "./cards/QuestionTray";
import { useQuestionStepper } from "./useQuestionStepper";
@@ -7016,9 +7024,11 @@ export function WorkflowCopilotChat({
workflowCopilotChatIdRef.current !== workflowCopilotChatId
)
return true;
+ // A stream frame that resolved this question first is newer than the ack; it can carry the group id.
setQuestionInteractions((current) =>
current.map((item) =>
- item.interaction_id === interaction.interaction_id
+ item.interaction_id === interaction.interaction_id &&
+ item.status === "pending"
? accepted.data
: item,
),
@@ -7065,17 +7075,22 @@ export function WorkflowCopilotChat({
const trayQuestion = questionInteractions.find(
(item) => item.status === "pending",
);
+ // An account card is answered on the card alone, so the composer stays a message field.
+ const composerQuestion =
+ trayQuestion?.account_group_review || trayQuestion?.account_group_cancel
+ ? undefined
+ : trayQuestion;
const questionStepper = useQuestionStepper(
- trayQuestion,
+ composerQuestion,
inputValue,
setInputValue,
);
- const trayPart = trayQuestion?.parts[questionStepper.index];
+ const trayPart = composerQuestion?.parts[questionStepper.index];
// handleSend is memoised and outlives a render; the stepper is rebuilt on every keystroke.
const questionStepperRef = useRef(questionStepper);
questionStepperRef.current = questionStepper;
const [questionTrayCollapsed, setQuestionTrayCollapsed] = useState(false);
- const trayQuestionId = trayQuestion?.interaction_id ?? null;
+ const trayQuestionId = composerQuestion?.interaction_id ?? null;
const [draftQuestionId, setDraftQuestionId] = useState(trayQuestionId);
answeringQuestionRef.current = trayQuestionId !== null;
// Adjusts state during render so the composer swaps before it paints, which an effect can't
@@ -7443,7 +7458,10 @@ export function WorkflowCopilotChat({
}
const candidate = messageOverride ?? inputValue;
const pendingQuestion = questionInteractions.find(
- (item) => item.status === "pending",
+ (item) =>
+ item.status === "pending" &&
+ !item.account_group_review &&
+ !item.account_group_cancel,
);
// The composer is the text field for the question on screen, so Enter steps through the
// tray and the last step sends every part's answer together.
@@ -8676,6 +8694,7 @@ export function WorkflowCopilotChat({
),
credential_recovery_token: credentialRecoveryToken ?? undefined,
supports_question_tool: true,
+ supports_account_group_card: true,
} as WorkflowCopilotChatRequest,
(payload) => {
if (
@@ -9282,7 +9301,7 @@ export function WorkflowCopilotChat({
if (e.key === "Escape" && queuedPrompt) {
e.preventDefault();
e.stopPropagation();
- if (!trayQuestion) restoreQueuedPromptToComposer();
+ if (!composerQuestion) restoreQueuedPromptToComposer();
return;
}
if (e.key === "Enter" && !e.shiftKey) {
@@ -9945,9 +9964,7 @@ export function WorkflowCopilotChat({
: browserStatusText;
// The composer is the text field for the pending question, so while one is pending it says so
// rather than inviting a new request.
- const latestTurnIsAsk = questionInteractions.some(
- (item) => item.status === "pending",
- );
+ const latestTurnIsAsk = composerQuestion !== undefined;
// A bypassed proposal's gate stays attached to its owning turn (not
// necessarily the last message) so a chip can jump back to it.
const gateOwnerTurnId =
@@ -10061,7 +10078,7 @@ export function WorkflowCopilotChat({
!waitingOnQueueOnly;
// Attachments alone can't answer a question, so the last step needs a choice or text.
const questionAnswerMissing =
- Boolean(trayQuestion) &&
+ Boolean(composerQuestion) &&
questionStepper.isLast &&
questionStepper.answeredCount === 0 &&
(!turnObservablyRunning || hasComposerText);
@@ -10075,7 +10092,7 @@ export function WorkflowCopilotChat({
? "Starting…"
: questionAnswerMissing
? "Send disabled — answer or skip the question"
- : trayQuestion && (!turnObservablyRunning || hasComposerText)
+ : composerQuestion && (!turnObservablyRunning || hasComposerText)
? questionStepper.isLast
? "Send answer"
: "Next question"
@@ -10091,12 +10108,29 @@ export function WorkflowCopilotChat({
? "Queue for next turn"
: "Stop";
- const renderQuestionReceipt = (interaction: QuestionInteraction) => (
- <QuestionReceipt
- key={interaction.interaction_id}
- interaction={interaction}
- />
- );
+ const renderQuestionReceipt = (interaction: QuestionInteraction) =>
+ interaction.account_group_review ? (
+ <AccountGroupReceiptCard
+ key={interaction.interaction_id}
+ interaction={interaction}
+ review={interaction.account_group_review}
+ turnLive={
+ narrative.turnId === interaction.turn_id &&
+ narrative.terminal === null
+ }
+ />
+ ) : interaction.account_group_cancel ? (
+ <AccountGroupCancelReceipt
+ key={interaction.interaction_id}
+ interaction={interaction}
+ review={interaction.account_group_cancel}
+ />
+ ) : (
+ <QuestionReceipt
+ key={interaction.interaction_id}
+ interaction={interaction}
+ />
+ );
// A question renders under the step that asked it, like a plan or credential card.
const anchoredQuestion = (
interaction: QuestionInteraction,
@@ -10769,7 +10803,7 @@ export function WorkflowCopilotChat({
queuedPrompt.reason === "working"
? "Queued — sends when this turn finishes."
: "Prompt queued. Waiting for live browser...",
- onCancel: trayQuestion
+ onCancel: composerQuestion
? undefined
: () => restoreQueuedPromptToComposer(),
}
@@ -11155,7 +11189,45 @@ export function WorkflowCopilotChat({
<span className="sr-only" aria-live="polite">
{queuedPrompt && composerStatus !== "working" ? "Message queued" : ""}
</span>
- {trayQuestion ? (
+ {trayQuestion?.account_group_review ? (
+ <AccountGroupReviewCard
+ key={trayQuestion.interaction_id}
+ review={trayQuestion.account_group_review}
+ disabled={isSubmittingQuestion || acceptUnresolved}
+ lockReason={acceptUnresolved ? acceptHoldReason : null}
+ collapsed={questionTrayCollapsed}
+ onCollapsedChange={setQuestionTrayCollapsed}
+ onApprove={(credentialIds) =>
+ void handleQuestionAnswer(trayQuestion, {
+ account_group_decision: {
+ approved: true,
+ credential_ids: credentialIds,
+ },
+ })
+ }
+ onDecline={() =>
+ void handleQuestionAnswer(trayQuestion, {
+ account_group_decision: { approved: false, credential_ids: [] },
+ })
+ }
+ upNext={attentionUpNext}
+ />
+ ) : trayQuestion?.account_group_cancel ? (
+ <AccountGroupCancelCard
+ key={trayQuestion.interaction_id}
+ review={trayQuestion.account_group_cancel}
+ disabled={isSubmittingQuestion || acceptUnresolved}
+ lockReason={acceptUnresolved ? acceptHoldReason : null}
+ collapsed={questionTrayCollapsed}
+ onCollapsedChange={setQuestionTrayCollapsed}
+ onDecide={(approved) =>
+ void handleQuestionAnswer(trayQuestion, {
+ account_group_decision: { approved, credential_ids: [] },
+ })
+ }
+ upNext={attentionUpNext}
+ />
+ ) : trayQuestion ? (
<QuestionTray
interaction={trayQuestion}
stepper={questionStepper}
@@ -11218,7 +11290,7 @@ export function WorkflowCopilotChat({
text={queuedPrompt.content}
attachments={queuedPrompt.attachments ?? []}
onEdit={
- queuedPrompt.origin === "product" || trayQuestion
+ queuedPrompt.origin === "product" || composerQuestion
? undefined
: () => restoreQueuedPromptToComposer()
}
@@ -11272,9 +11344,9 @@ export function WorkflowCopilotChat({
onChange={(e) => setInputValue(e.target.value)}
onFocus={() => setQuestionTrayCollapsed(false)}
onKeyDown={handleKeyPress}
- aria-label={trayQuestion ? "Your response" : undefined}
+ aria-label={composerQuestion ? "Your response" : undefined}
aria-describedby={
- trayQuestion && !questionTrayCollapsed
+ composerQuestion && !questionTrayCollapsed
? QUESTION_PROMPT_ID
: undefined
}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReceiptCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReceiptCard.tsx
new file mode 100644
index 000000000..c4aacf453
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReceiptCard.tsx
@@ -0,0 +1,302 @@
+import { QuestionMarkCircledIcon } from "@radix-ui/react-icons";
+import { useQuery } from "@tanstack/react-query";
+import { isAxiosError } from "axios";
+import { getClient } from "@/api/AxiosClient";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { cn } from "@/util/utils";
+import type {
+ AccountGroupCancelReview,
+ AccountGroupOutcome,
+ AccountGroupReview,
+ QuestionInteraction,
+} from "../workflowCopilotTypes";
+import { AttentionMarker } from "./AttentionTray";
+import { TONE_CLASSES, type Tone } from "./receiptTone";
+
+const POLL_MS = 5000;
+
+type GroupStatus = "active" | "cancel_requested" | "finished";
+
+interface GroupItem {
+ key: string;
+ workflow_run_id: string;
+ run_status: string | null;
+ outcome: AccountGroupOutcome;
+}
+
+interface Group {
+ status: GroupStatus;
+ items: GroupItem[];
+}
+
+const OUTCOME_LABEL: Record<AccountGroupOutcome, string> = {
+ pending: "Waiting",
+ in_progress: "Running",
+ completed: "Completed",
+ failed: "Failed before acting",
+ unknown: "Unknown, may have acted",
+ canceled: "Canceled",
+};
+
+const OUTCOME_CLASS: Partial<Record<AccountGroupOutcome, string>> = {
+ failed: "text-rose-700 dark:text-rose-300",
+ unknown: "text-amber-700 dark:text-yellow-400",
+};
+
+const STATUS_LABEL: Record<GroupStatus, string> = {
+ active: "running",
+ cancel_requested: "stopping",
+ finished: "finished",
+};
+
+const COUNTED_OUTCOMES: [AccountGroupOutcome, string][] = [
+ ["completed", "completed"],
+ ["failed", "failed"],
+ ["unknown", "may have acted"],
+ ["canceled", "canceled"],
+];
+
+function plural(count: number, noun: string): string {
+ return count === 1 ? `1 ${noun}` : `${count} ${noun}s`;
+}
+
+function groupMeta(group: Group): string {
+ const counts = COUNTED_OUTCOMES.map(([outcome, label]) => {
+ const count = group.items.filter((item) => item.outcome === outcome).length;
+ return count > 0 ? `${count} ${label}` : null;
+ }).filter((part) => part !== null);
+ return [STATUS_LABEL[group.status], ...counts].join(" · ");
+}
+
+function isClientError(error: unknown): boolean {
+ const status = isAxiosError(error) ? error.response?.status : undefined;
+ return status !== undefined && status >= 400 && status < 500;
+}
+
+function useGroup(groupId: string | null) {
+ const credentialGetter = useCredentialGetter();
+ return useQuery<Group>({
+ queryKey: ["workflowRunGroup", groupId],
+ enabled: groupId !== null,
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ const response = await client.get<Group>(
+ `/workflow_run_groups/${groupId}`,
+ );
+ return response.data;
+ },
+ retry: (failureCount, error) => !isClientError(error) && failureCount < 3,
+ refetchInterval: (query) =>
+ query.state.data?.status === "finished" ||
+ isClientError(query.state.error)
+ ? false
+ : POLL_MS,
+ });
+}
+
+function Heading({
+ tone,
+ title,
+ meta,
+}: {
+ tone: Tone;
+ title: string;
+ meta?: string | null;
+}) {
+ const classes = TONE_CLASSES[tone];
+ return (
+ <p className="flex min-w-0 items-start gap-2 leading-[18px]">
+ <span
+ aria-hidden
+ className={cn("mt-[5px] size-2 shrink-0 rounded-full", classes.dot)}
+ />
+ <span className="min-w-0 break-words">
+ <span className={cn("font-semibold", classes.title)}>{title}</span>
+ {meta ? <span className="text-muted-foreground"> · {meta}</span> : null}
+ </span>
+ </p>
+ );
+}
+
+export function AccountGroupReceiptCard({
+ interaction,
+ review,
+ turnLive,
+}: {
+ interaction: QuestionInteraction;
+ review: AccountGroupReview;
+ turnLive: boolean;
+}) {
+ const query = useGroup(review.workflow_run_group_id);
+ const group = query.data;
+ const labels = new Map(
+ review.rows.map((row) => [row.credential_id, row.label]),
+ );
+ const decision = review.decision;
+
+ if (interaction.status === "pending") {
+ return (
+ <div data-interaction-id={interaction.interaction_id}>
+ <AttentionMarker
+ icon={
+ <QuestionMarkCircledIcon
+ aria-hidden
+ className="size-3.5 shrink-0 text-amber-500"
+ />
+ }
+ title={`Copilot asked you to review ${plural(review.rows.length, "account")}`}
+ hint="answer below"
+ />
+ </div>
+ );
+ }
+ if (!decision?.approved) {
+ const interrupted = interaction.status === "interrupted";
+ return (
+ <div
+ data-interaction-id={interaction.interaction_id}
+ className={cn(
+ "flex flex-col gap-1.5 rounded-lg border px-3 py-[11px] text-xs",
+ TONE_CLASSES[interrupted ? "error" : "neutral"].card,
+ )}
+ >
+ <Heading
+ tone={interrupted ? "error" : "neutral"}
+ title={
+ interaction.status === "resolved"
+ ? "You declined the account review. Nothing ran."
+ : "The account review ended. Nothing ran."
+ }
+ />
+ {interrupted ? (
+ <p className="pl-4 text-muted-foreground">
+ Send a new message to continue.
+ </p>
+ ) : null}
+ </div>
+ );
+ }
+ return (
+ <div
+ data-interaction-id={interaction.interaction_id}
+ className={cn(
+ "flex flex-col gap-1.5 rounded-lg border px-3 py-[11px] text-xs",
+ TONE_CLASSES.answered.card,
+ )}
+ >
+ <Heading
+ tone="answered"
+ title={`You approved ${plural(decision.credential_ids.length, "account run")}`}
+ meta={group ? groupMeta(group) : null}
+ />
+ {group ? (
+ <ul className="flex flex-col gap-0.5 pl-4">
+ {group.items.map((item) => (
+ <li
+ key={item.key}
+ className="grid grid-cols-[minmax(0,1fr)_auto] gap-x-2"
+ >
+ {item.run_status === null ? (
+ <span className="break-words">
+ {labels.get(item.key) ?? item.key}
+ </span>
+ ) : (
+ <a
+ className="break-words underline-offset-2 hover:underline"
+ href={`/runs/${item.workflow_run_id}`}
+ target="_blank"
+ rel="noreferrer"
+ >
+ {labels.get(item.key) ?? item.key}
+ <span className="sr-only"> (opens in new tab)</span>
+ </a>
+ )}
+ <span
+ className={cn(
+ "shrink-0",
+ OUTCOME_CLASS[item.outcome] ?? "text-muted-foreground",
+ )}
+ >
+ {OUTCOME_LABEL[item.outcome]}
+ </span>
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ {group && group.status !== "finished" ? (
+ <p className="pl-4 text-muted-foreground">
+ The chat's Stop button does not stop these runs. To stop them, ask
+ Copilot to cancel the group.
+ </p>
+ ) : null}
+ {query.isError ? (
+ <p className="pl-4 text-rose-700 dark:text-rose-300">
+ Couldn't load run status.
+ </p>
+ ) : null}
+ {review.workflow_run_group_id ? null : turnLive ? (
+ <p className="pl-4 text-muted-foreground">Starting the runs…</p>
+ ) : (
+ <p className="pl-4 text-muted-foreground">
+ No run group is linked to this approval. Copilot's reply says whether
+ the runs started.
+ </p>
+ )}
+ </div>
+ );
+}
+
+export function AccountGroupCancelReceipt({
+ interaction,
+ review,
+}: {
+ interaction: QuestionInteraction;
+ review: AccountGroupCancelReview;
+}) {
+ const count = review.unfinished_rows.length;
+ const approved = review.decision?.approved;
+ const group = useGroup(approved ? review.workflow_run_group_id : null).data;
+ if (interaction.status === "pending") {
+ return (
+ <div data-interaction-id={interaction.interaction_id}>
+ <AttentionMarker
+ icon={
+ <QuestionMarkCircledIcon
+ aria-hidden
+ className="size-3.5 shrink-0 text-amber-500"
+ />
+ }
+ title={`Copilot asked whether to stop ${plural(count, "account run")}`}
+ hint="answer below"
+ />
+ </div>
+ );
+ }
+ const interrupted = interaction.status === "interrupted";
+ return (
+ <div
+ data-interaction-id={interaction.interaction_id}
+ className={cn(
+ "flex flex-col gap-1.5 rounded-lg border px-3 py-[11px] text-xs",
+ TONE_CLASSES[interrupted ? "error" : "neutral"].card,
+ )}
+ >
+ <Heading
+ tone={interrupted ? "error" : "neutral"}
+ title={
+ approved
+ ? `You asked to stop ${plural(count, "unfinished account run")}`
+ : interaction.status === "resolved"
+ ? "You kept the account runs going. Nothing was canceled."
+ : "The cancel review ended. Nothing was canceled."
+ }
+ meta={group ? groupMeta(group) : null}
+ />
+ {interrupted ? (
+ <p className="pl-4 text-muted-foreground">
+ Send a new message to continue.
+ </p>
+ ) : null}
+ </div>
+ );
+}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReviewCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReviewCard.test.tsx
new file mode 100644
index 000000000..28902d028
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReviewCard.test.tsx
@@ -0,0 +1,100 @@
+// @vitest-environment jsdom
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { AccountGroupReview } from "../workflowCopilotTypes";
+import {
+ AccountGroupCancelCard,
+ AccountGroupReviewCard,
+} from "./AccountGroupReviewCard";
+
+afterEach(cleanup);
+
+const review: AccountGroupReview = {
+ workflow_permanent_id: "wpid_1",
+ workflow_id: "wf_1",
+ version: 3,
+ workflow_title: "Download statement",
+ credential_parameter_key: "login",
+ action_summary: "Download the latest statement",
+ common_inputs: { month: "May" },
+ rows: [
+ {
+ credential_id: "cred_a",
+ label: "Billing (al***@example.com)",
+ prior_outcome: "completed",
+ preselected: false,
+ },
+ {
+ credential_id: "cred_b",
+ label: "Payroll",
+ prior_outcome: "failed",
+ preselected: true,
+ },
+ {
+ credential_id: "cred_c",
+ label: "Ops",
+ prior_outcome: null,
+ preselected: true,
+ },
+ ],
+ workflow_run_group_id: null,
+};
+
+describe("account group review", () => {
+ it("approves exactly the accounts left checked, with repeat-risk rows starting unchecked", () => {
+ const onApprove = vi.fn();
+ render(
+ <AccountGroupReviewCard
+ review={review}
+ disabled={false}
+ lockReason={null}
+ collapsed={false}
+ onCollapsedChange={() => {}}
+ onApprove={onApprove}
+ onDecline={() => {}}
+ />,
+ );
+ expect(screen.getByText(/already completed last time/)).toBeTruthy();
+ fireEvent.click(screen.getByLabelText(/Payroll/));
+
+ fireEvent.click(screen.getByRole("button", { name: "Approve 1 run" }));
+
+ expect(onApprove).toHaveBeenCalledWith(["cred_c"]);
+ });
+
+ it("lists the runs a cancel would stop and answers stop or keep", () => {
+ const onDecide = vi.fn();
+ render(
+ <AccountGroupCancelCard
+ review={{
+ workflow_run_group_id: "wrg_1",
+ unfinished_rows: [
+ {
+ credential_id: "cred_b",
+ label: "Payroll",
+ workflow_run_id: "wr_b",
+ outcome: "in_progress",
+ },
+ {
+ credential_id: "cred_c",
+ label: "Ops",
+ workflow_run_id: "wr_c",
+ outcome: "pending",
+ },
+ ],
+ }}
+ disabled={false}
+ lockReason={null}
+ collapsed={false}
+ onCollapsedChange={() => {}}
+ onDecide={onDecide}
+ />,
+ );
+ expect(screen.getByText("Payroll")).toBeTruthy();
+
+ fireEvent.click(screen.getByRole("button", { name: "Keep running" }));
+ fireEvent.click(screen.getByRole("button", { name: "Stop 2 runs" }));
+
+ expect(onDecide.mock.calls).toEqual([[false], [true]]);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReviewCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReviewCard.tsx
new file mode 100644
index 000000000..41cfcfb08
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/AccountGroupReviewCard.tsx
@@ -0,0 +1,254 @@
+import { useId, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Checkbox } from "@/components/ui/checkbox";
+import { cn } from "@/util/utils";
+import type {
+ AccountGroupCancelReview,
+ AccountGroupOutcome,
+ AccountGroupReview,
+} from "../workflowCopilotTypes";
+import { AttentionTray } from "./AttentionTray";
+
+const PRIOR_EFFECT: Partial<Record<AccountGroupOutcome, string>> = {
+ completed: "already completed last time",
+ unknown: "may have acted last time",
+ pending: "waiting in another group",
+ in_progress: "running in another group",
+};
+
+function plural(count: number, noun: string): string {
+ return count === 1 ? `1 ${noun}` : `${count} ${noun}s`;
+}
+
+function inputText(value: unknown): string {
+ return typeof value === "string" ? value : JSON.stringify(value);
+}
+
+export function AccountGroupReviewCard({
+ review,
+ disabled,
+ lockReason,
+ collapsed,
+ onCollapsedChange,
+ onApprove,
+ onDecline,
+ upNext,
+}: {
+ review: AccountGroupReview;
+ disabled: boolean;
+ lockReason: string | null;
+ collapsed: boolean;
+ onCollapsedChange: (collapsed: boolean) => void;
+ onApprove: (credentialIds: string[]) => void;
+ onDecline: () => void;
+ upNext?: string | null;
+}) {
+ const titleId = useId();
+ const [checked, setChecked] = useState(
+ () =>
+ new Set(
+ review.rows
+ .filter((row) => row.preselected)
+ .map((row) => row.credential_id),
+ ),
+ );
+ const toggle = (credentialId: string) =>
+ setChecked((current) => {
+ const next = new Set(current);
+ if (!next.delete(credentialId)) next.add(credentialId);
+ return next;
+ });
+ const approved = review.rows
+ .map((row) => row.credential_id)
+ .filter((credentialId) => checked.has(credentialId));
+ const inputs = Object.entries(review.common_inputs);
+
+ return (
+ <AttentionTray
+ aria-label="Account review"
+ aria-describedby={titleId}
+ title="Review the accounts before anything runs"
+ titleId={titleId}
+ wrapTitle
+ collapsedTitle={`Copilot is waiting on your review of ${plural(review.rows.length, "account")}`}
+ collapsed={collapsed}
+ onCollapsedChange={onCollapsedChange}
+ minimizeLabel="Minimize account review"
+ upNext={upNext}
+ >
+ <div className="flex min-h-0 flex-col gap-2 overflow-y-auto px-3 pb-2.5 pt-1 text-xs">
+ <p className="whitespace-pre-wrap break-words text-[13.5px] font-medium leading-relaxed">
+ {review.action_summary}
+ </p>
+ <p className="text-muted-foreground">
+ {review.workflow_title} · version {review.version} · runs one account
+ at a time, each in a fresh browser, using the account's saved login as
+ it is when that run starts
+ </p>
+ {review.credential_parameter_used_by?.length === 0 ? (
+ <p className="text-amber-700 dark:text-yellow-400">
+ No step of this workflow lists {review.credential_parameter_key}, so
+ each run may not sign in as its own account.
+ </p>
+ ) : null}
+ {review.clean_group_run_id ? null : (
+ <p className="text-amber-700 dark:text-yellow-400">
+ This version has not yet completed cleanly in a group for any
+ selected account.
+ </p>
+ )}
+ {inputs.length > 0 ? (
+ <dl className="grid grid-cols-[auto,1fr] gap-x-2 gap-y-0.5">
+ {inputs.map(([key, value]) => (
+ <div key={key} className="contents">
+ <dt className="text-muted-foreground">{key}</dt>
+ <dd className="break-words">{inputText(value)}</dd>
+ </div>
+ ))}
+ </dl>
+ ) : null}
+ <ul className="flex flex-col gap-1">
+ {review.rows.map((row) => {
+ const warning = row.prior_outcome
+ ? PRIOR_EFFECT[row.prior_outcome]
+ : undefined;
+ return (
+ <li key={row.credential_id}>
+ <label
+ className={cn(
+ "flex cursor-pointer items-start gap-2",
+ disabled && "cursor-not-allowed opacity-60",
+ )}
+ >
+ <Checkbox
+ className="mt-0.5"
+ disabled={disabled}
+ checked={checked.has(row.credential_id)}
+ onCheckedChange={() => toggle(row.credential_id)}
+ />
+ <span className="min-w-0 break-words">
+ {row.label}
+ {warning ? (
+ <span className="text-amber-700 dark:text-yellow-400">
+ {" "}
+ · {warning}
+ </span>
+ ) : null}
+ </span>
+ </label>
+ </li>
+ );
+ })}
+ </ul>
+ </div>
+ <div
+ className="flex shrink-0 flex-wrap items-center gap-2 border-t border-border px-3 py-1.5"
+ title={lockReason ?? undefined}
+ >
+ <Button
+ size="sm"
+ className="h-7"
+ disabled={disabled || approved.length === 0}
+ onClick={() => onApprove(approved)}
+ >
+ Approve {plural(approved.length, "run")}
+ </Button>
+ <Button
+ size="sm"
+ variant="ghost"
+ className="h-7"
+ disabled={disabled}
+ onClick={onDecline}
+ >
+ Decline
+ </Button>
+ {lockReason ? (
+ <span className="min-w-0 truncate text-[11px] text-muted-foreground">
+ {lockReason}
+ </span>
+ ) : null}
+ </div>
+ </AttentionTray>
+ );
+}
+
+export function AccountGroupCancelCard({
+ review,
+ disabled,
+ lockReason,
+ collapsed,
+ onCollapsedChange,
+ onDecide,
+ upNext,
+}: {
+ review: AccountGroupCancelReview;
+ disabled: boolean;
+ lockReason: string | null;
+ collapsed: boolean;
+ onCollapsedChange: (collapsed: boolean) => void;
+ onDecide: (approved: boolean) => void;
+ upNext?: string | null;
+}) {
+ const titleId = useId();
+ const count = review.unfinished_rows.length;
+ return (
+ <AttentionTray
+ aria-label="Cancel review"
+ aria-describedby={titleId}
+ title={`Stop ${plural(count, "account run")} that ${count === 1 ? "has" : "have"} not finished?`}
+ titleId={titleId}
+ wrapTitle
+ collapsedTitle={`Copilot is waiting on your answer about stopping ${plural(count, "run")}`}
+ collapsed={collapsed}
+ onCollapsedChange={onCollapsedChange}
+ minimizeLabel="Minimize cancel review"
+ upNext={upNext}
+ >
+ <div className="flex min-h-0 flex-col gap-2 overflow-y-auto px-3 pb-2.5 pt-1 text-xs">
+ <p className="text-muted-foreground">
+ Stopping cancels every run listed here. A run already in progress may
+ have acted before it stops. Finished runs keep their results.
+ </p>
+ <ul className="flex flex-col gap-1">
+ {review.unfinished_rows.map((row) => (
+ <li key={row.credential_id} className="break-words">
+ {row.label}
+ <span className="text-muted-foreground">
+ {" "}
+ · {row.outcome === "in_progress" ? "running" : "waiting"}
+ </span>
+ </li>
+ ))}
+ </ul>
+ </div>
+ <div
+ className="flex shrink-0 flex-wrap items-center gap-2 border-t border-border px-3 py-1.5"
+ title={lockReason ?? undefined}
+ >
+ <Button
+ size="sm"
+ variant="destructive"
+ className="h-7"
+ disabled={disabled}
+ onClick={() => onDecide(true)}
+ >
+ Stop {plural(count, "run")}
+ </Button>
+ <Button
+ size="sm"
+ variant="ghost"
+ className="h-7"
+ disabled={disabled}
+ onClick={() => onDecide(false)}
+ >
+ Keep running
+ </Button>
+ {lockReason ? (
+ <span className="min-w-0 truncate text-[11px] text-muted-foreground">
+ {lockReason}
+ </span>
+ ) : null}
+ </div>
+ </AttentionTray>
+ );
+}
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionReceipt.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionReceipt.tsx
index ef9724871..f293b6968 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionReceipt.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/QuestionReceipt.tsx
@@ -8,27 +8,7 @@ import { parseUtcIsoMs } from "../narrativeState";
import type { QuestionInteraction } from "../workflowCopilotTypes";
import { QuestionPartsCard } from "./QuestionPartsCard";
import { answeredPartIds } from "./questionAnswers";
-
-type Tone = "answered" | "neutral" | "error";
-
-const TONE_CLASSES: Record<Tone, { card: string; dot: string; title: string }> =
- {
- answered: {
- card: "border-emerald-500/40 bg-emerald-500/[0.06]",
- dot: "bg-emerald-500 shadow-[0_0_0_3px_rgba(16,185,129,0.18)]",
- title: "text-emerald-700 dark:text-emerald-300",
- },
- neutral: {
- card: "border-slate-400/30 bg-slate-400/[0.05]",
- dot: "bg-slate-500",
- title: "text-slate-600 dark:text-slate-300",
- },
- error: {
- card: "border-rose-400/35 bg-rose-500/[0.05]",
- dot: "bg-rose-500",
- title: "text-rose-700 dark:text-rose-300",
- },
- };
+import { TONE_CLASSES, type Tone } from "./receiptTone";
function questions(count: number): string {
return count === 1 ? "1 question" : `${count} questions`;
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/receiptTone.ts b/skyvern-frontend/src/routes/workflows/copilot/cards/receiptTone.ts
new file mode 100644
index 000000000..263d417f5
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/receiptTone.ts
@@ -0,0 +1,22 @@
+export type Tone = "answered" | "neutral" | "error";
+
+export const TONE_CLASSES: Record<
+ Tone,
+ { card: string; dot: string; title: string }
+> = {
+ answered: {
+ card: "border-emerald-500/40 bg-emerald-500/[0.06]",
+ dot: "bg-emerald-500 shadow-[0_0_0_3px_rgba(16,185,129,0.18)]",
+ title: "text-emerald-700 dark:text-emerald-300",
+ },
+ neutral: {
+ card: "border-slate-400/30 bg-slate-400/[0.05]",
+ dot: "bg-slate-500",
+ title: "text-slate-600 dark:text-slate-300",
+ },
+ error: {
+ card: "border-rose-400/35 bg-rose-500/[0.05]",
+ dot: "bg-rose-500",
+ title: "text-rose-700 dark:text-rose-300",
+ },
+};
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts
index 6db40146d..cf00738bf 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotActivityLog.ts
@@ -1,5 +1,6 @@
import { CodeWriteDiff } from "./workflowCopilotTypes";
import {
+ ACCOUNT_GROUP_SUBMIT_TOOL,
ActivityEntry,
AUTHORING_TOOLS,
BlockState,
@@ -243,7 +244,12 @@ export function rowIndexOfToolCall(
// A call whose card renders after its row ends that row, so work done after the
// user answered or the plan changed never sits above the card.
-const CARD_TOOLS = new Set(["ask_user", "set_work_plan"]);
+const CARD_TOOLS = new Set([
+ "ask_user",
+ "set_work_plan",
+ ACCOUNT_GROUP_SUBMIT_TOOL,
+ "cancel_account_group",
+]);
export function deriveActivityLog(turn: TurnNarrativeState): ActivityLog {
// A cancelled or timed-out turn can terminate with a call still unmatched;
@@ -566,11 +572,28 @@ function kindPhrase(
return live ? "looking up guidance" : "looked up guidance";
case "write":
return writePhrase(entries, live);
- case "run":
- if (live) return "testing the workflow";
- return entries.length === 1
- ? "tested the workflow"
- : `tested the workflow ${entries.length} times`;
+ case "run": {
+ const groups = entries.filter(
+ (e) => e.toolName === ACCOUNT_GROUP_SUBMIT_TOOL,
+ ).length;
+ const tests = entries.length - groups;
+ return [
+ groups > 0
+ ? live
+ ? "reviewing account runs with you"
+ : "reviewed account runs with you"
+ : null,
+ tests === 0
+ ? null
+ : live
+ ? "testing the workflow"
+ : tests === 1
+ ? "tested the workflow"
+ : `tested the workflow ${tests} times`,
+ ]
+ .filter((part) => part !== null)
+ .join(", ");
+ }
case "other": {
const asks = entries.filter((e) => e.toolName === "ask_user").length;
const rest = entries.length - asks;
@@ -616,9 +639,12 @@ export function callRollup(
// block with neither, or seen after every dated call returned, goes to the row's
// last run call. Empty when the row made no run call.
export function blocksByRunCall(row: ActivityRow): Map<string, BlockState[]> {
+ // An account group's runs are separate saved-workflow runs, never blocks of this row.
const runCalls = row.entries.filter(
(entry) =>
- entry.toolName !== undefined && toolCallKind(entry.toolName) === "run",
+ entry.toolName !== undefined &&
+ entry.toolName !== ACCOUNT_GROUP_SUBMIT_TOOL &&
+ toolCallKind(entry.toolName) === "run",
);
const byCall = new Map<string, BlockState[]>();
const lastRun = runCalls[runCalls.length - 1];
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
index d2caa70b9..fc59e63ba 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
@@ -728,6 +728,8 @@ export const AUTHORING_TOOLS = new Set([
"update_and_run_blocks",
"edit_block_and_run",
]);
+export const ACCOUNT_GROUP_SUBMIT_TOOL = "run_workflow_for_accounts";
+
export const RUN_TOOLS = new Set([
"update_and_run_blocks",
"edit_block_and_run",
@@ -791,6 +793,9 @@ const ACTIVITY_TOOL_DISPLAY_LABELS: Record<string, string> = {
ask_user: "Asking you",
set_work_plan: "Updating its plan",
synthesize_demonstrated_block: "Building a block from the recorded steps",
+ [ACCOUNT_GROUP_SUBMIT_TOOL]: "Reviewing the accounts with you",
+ get_account_group_status: "Checking the account runs",
+ cancel_account_group: "Reviewing a cancel with you",
};
// What kind of work a call did, for the activity log's per-step rollup. Keyed
@@ -839,6 +844,9 @@ const TOOL_CALL_KINDS: Record<string, ToolCallKind> = {
disable_workflow_schedule: "other",
cancel_workflow_schedule: "other",
delete_workflow_schedule: "other",
+ [ACCOUNT_GROUP_SUBMIT_TOOL]: "run",
+ get_account_group_status: "other",
+ cancel_account_group: "other",
};
export function toolCallKind(toolName: string): ToolCallKind {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index e4453f9fb..82d1048e2 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -54,10 +54,59 @@ export interface QuestionAnswer {
text?: string | null;
}
+export type AccountGroupOutcome =
+ | "pending"
+ | "in_progress"
+ | "completed"
+ | "failed"
+ | "unknown"
+ | "canceled";
+
+export interface AccountGroupRow {
+ credential_id: string;
+ label: string;
+ prior_outcome: AccountGroupOutcome | null;
+ preselected: boolean;
+}
+
+export interface AccountGroupReview {
+ workflow_permanent_id: string;
+ workflow_id: string;
+ version: number;
+ workflow_title: string;
+ credential_parameter_key: string;
+ action_summary: string;
+ common_inputs: Record<string, unknown>;
+ clean_group_run_id?: string | null;
+ credential_parameter_used_by?: string[];
+ rows: AccountGroupRow[];
+ workflow_run_group_id: string | null;
+ decision?: AccountGroupDecision | null;
+}
+
+export interface AccountGroupRunRow {
+ credential_id: string;
+ label: string;
+ workflow_run_id: string;
+ outcome: AccountGroupOutcome;
+}
+
+export interface AccountGroupCancelReview {
+ workflow_run_group_id: string;
+ unfinished_rows: AccountGroupRunRow[];
+ decision?: AccountGroupDecision | null;
+}
+
+export interface AccountGroupDecision {
+ approved: boolean;
+ credential_ids: string[];
+}
+
export interface QuestionResponse {
answers?: QuestionAnswer[];
text?: string | null;
skipped?: boolean;
+ account_group_decision?: AccountGroupDecision;
}
export interface QuestionInteraction {
@@ -69,6 +118,8 @@ export interface QuestionInteraction {
response: QuestionResponse | null;
created_at: string;
resolved_at: string | null;
+ account_group_review?: AccountGroupReview;
+ account_group_cancel?: AccountGroupCancelReview;
}
export interface WorkflowCopilotQuestionRequired {
@@ -172,6 +223,7 @@ export interface WorkflowCopilotChatRequest {
credential_recovery_token?: string;
supports_credential_pause_recovery?: boolean;
supports_question_tool?: boolean;
+ supports_account_group_card?: boolean;
}
export type WorkflowCopilotCancelSource = "escape_key" | "stop_button" | "api";
diff --git a/skyvern/config.py b/skyvern/config.py
index ee6453ceb..6a39fced6 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -1006,6 +1006,8 @@ class Settings(BaseSettings):
WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED: bool = True
"""Accept new serial workflow run groups. Turning it off stops submission only; reads, cancels and
dispatch of already-submitted groups continue."""
+ COPILOT_ACCOUNT_GROUP_SUBMIT_ENABLED: bool = True
+ """Offer Copilot's run_workflow_for_accounts tool. Turning it off keeps group status, cancel and receipts."""
# OpenTelemetry Settings
OTEL_ENABLED: bool = False
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 5eba4c340..1cef2a2de 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -160,6 +160,14 @@ class WorkflowChangedSinceReview(SkyvernHTTPException):
)
+class GroupAccountsRanSinceReview(SkyvernHTTPException):
+ def __init__(self, item_keys: list[str]) -> None:
+ super().__init__(
+ f"{item_keys} started in another group after the review, so this group was not submitted.",
+ status_code=HTTPStatus.CONFLICT,
+ )
+
+
_BROWSER_CONNECTION_GUIDANCE = "Please try re-running. If this continues, contact [email protected]."
# Patterns that indicate a browser session connection failure (e.g. CDP WebSocket errors).
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 7a7c429c5..36cd620b7 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -232,18 +232,6 @@ class TOTPVerificationResponse:
headers: dict[str, str] = field(default_factory=dict)
-@dataclass(frozen=True)
-class CopilotSiteOriginAssociation:
- requested_name: str
- entity_id: str
- entity_label: str
- official_site_url: str
- origin: str
- source: str
- provider_relation_type: str
- provider_relation_text: str
-
-
@dataclass(frozen=True)
class ScriptExecutionPolicyDecision:
allowed: bool
@@ -263,13 +251,6 @@ class CopilotCandidateNetworkHop(TypedDict):
enforcement_version: str
-@dataclass(frozen=True)
-class CopilotEntrypointCandidate:
- url: str
- source_rank: int
- association: CopilotSiteOriginAssociation
-
-
@dataclass(frozen=True)
class CopilotOrganizationUsage:
"""A null field is one the deployment or account does not record; ``credits_note`` says why
@@ -2785,14 +2766,6 @@ class AgentFunction:
"""
return ""
- async def acquire_copilot_entrypoint_candidates(
- self,
- *,
- site_name: str,
- ) -> list[CopilotEntrypointCandidate]:
- del site_name
- return []
-
def copilot_candidate_network_guard(
self,
browser_context: BrowserContext,
@@ -2812,10 +2785,6 @@ class AgentFunction:
raise RuntimeError("Copilot candidate pre-connect enforcement is unavailable")
yield [] # pragma: no cover
- async def wait_for_copilot_candidate_network_idle(self, browser_context: BrowserContext) -> None:
- del browser_context
- raise RuntimeError("Copilot candidate pre-connect enforcement is unavailable")
-
def get_copilot_config(self, code_block_mode: bool | None = None) -> CopilotConfig | None:
"""Return an optional workflow copilot config override."""
del code_block_mode
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 35f520b16..2b439a292 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -39,6 +39,7 @@ from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_MARKER
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.agent_naming import schedule_agent_naming
+from skyvern.forge.sdk.copilot.ask_user import ACCOUNT_GROUP_CANCEL_TOOL_NAME, ACCOUNT_GROUP_SUBMIT_TOOL_NAME
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
@@ -637,7 +638,19 @@ def _format_chat_history(chat_history: list[WorkflowCopilotChatHistoryMessage])
questions = msg.narrative_payload.get("questionInteractions", []) if msg.narrative_payload is not None else []
for raw in questions:
interaction = QuestionInteraction.model_validate(raw)
- if interaction.status == "resolved":
+ if interaction.account_group_review is not None or interaction.account_group_cancel is not None:
+ summary = (
+ interaction.tool_result()
+ if interaction.status == "resolved"
+ else {"interaction_id": interaction.interaction_id, "status": interaction.status}
+ )
+ tool_name = (
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME
+ if interaction.account_group_review is not None
+ else ACCOUNT_GROUP_CANCEL_TOOL_NAME
+ )
+ lines.append(f"{tool_name} review: {json.dumps(summary)}")
+ elif interaction.status == "resolved":
lines.append(f"ask_user result: {json.dumps(interaction.tool_result())}")
else:
lines.append(f"ask_user request: {interaction.model_dump_json()}")
@@ -5687,6 +5700,7 @@ async def _run_copilot_turn_impl(
mode=eval_mode,
native_tools=copilot_native_tools(
supports_question_tool=chat_request.supports_question_tool,
+ supports_account_group_card=chat_request.supports_account_group_card,
browser_code_available=browser_code_mode != CopilotBrowserCodeMode.OFF
and copilot_config.authoring_capability.code_blocks,
authoring_capability=copilot_config.authoring_capability,
diff --git a/skyvern/forge/sdk/copilot/ask_user.py b/skyvern/forge/sdk/copilot/ask_user.py
index 76b1c10d0..ff932a827 100644
--- a/skyvern/forge/sdk/copilot/ask_user.py
+++ b/skyvern/forge/sdk/copilot/ask_user.py
@@ -6,11 +6,14 @@ from typing import TYPE_CHECKING, Any, Literal
from uuid import uuid4
import structlog
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, JsonValue
+from skyvern.forge import app
from skyvern.forge.sdk.copilot.human_input_wait import pause_human_input
from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_text
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.schemas.workflow_run_groups import WorkflowRunGroupItemOutcome
from skyvern.utils.contained_effects import contained_effect
if TYPE_CHECKING:
@@ -51,6 +54,71 @@ class QuestionAnswer(BaseModel):
text: str | None = None
+ACCOUNT_GROUP_SUBMIT_TOOL_NAME = "run_workflow_for_accounts"
+ACCOUNT_GROUP_STATUS_TOOL_NAME = "get_account_group_status"
+ACCOUNT_GROUP_CANCEL_TOOL_NAME = "cancel_account_group"
+
+REPEAT_RISK_OUTCOMES = (
+ WorkflowRunGroupItemOutcome.completed,
+ WorkflowRunGroupItemOutcome.unknown,
+ WorkflowRunGroupItemOutcome.pending,
+ WorkflowRunGroupItemOutcome.in_progress,
+)
+
+
+class AccountGroupDecision(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ approved: bool
+ credential_ids: list[str] = Field(default_factory=list)
+
+
+class AccountGroupRow(BaseModel):
+ credential_id: str
+ label: str
+ identity: str = ""
+ prior_outcome: WorkflowRunGroupItemOutcome | None = None
+ last_group_id: str | None = None
+ preselected: bool = True
+
+
+class AccountGroupReview(BaseModel):
+ workflow_permanent_id: str
+ workflow_id: str
+ version: int
+ workflow_title: str
+ modified_at: datetime
+ definition_hash: str
+ credential_parameter_key: str
+ action_summary: str
+ common_inputs: dict[str, JsonValue] = Field(default_factory=dict)
+ clean_group_run_id: str | None = None
+ credential_parameter_used_by: list[str] = Field(default_factory=list)
+ rows: list[AccountGroupRow]
+ workflow_run_group_id: str | None = None
+ # Stored here rather than on QuestionResponse, whose older versions forbid unknown fields.
+ decision: AccountGroupDecision | None = Field(default=None, exclude_if=lambda decision: decision is None)
+
+ def repeated_after_prior_effect(self, approved_ids: list[str]) -> list[AccountGroupRow]:
+ return [
+ row for row in self.rows if row.credential_id in approved_ids and row.prior_outcome in REPEAT_RISK_OUTCOMES
+ ]
+
+
+class AccountGroupRunRow(BaseModel):
+ credential_id: str
+ label: str
+ workflow_run_id: str
+ run_status: WorkflowRunStatus | None
+ outcome: WorkflowRunGroupItemOutcome
+
+
+class AccountGroupCancelReview(BaseModel):
+ workflow_run_group_id: str
+ unfinished_rows: list[AccountGroupRunRow]
+ decision: AccountGroupDecision | None = Field(default=None, exclude_if=lambda decision: decision is None)
+
+
class QuestionResponse(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -72,10 +140,48 @@ class QuestionInteraction(BaseModel):
response: QuestionResponse | None = None
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
resolved_at: datetime | None = None
+ account_group_review: AccountGroupReview | None = Field(default=None, exclude_if=lambda review: review is None)
+ account_group_cancel: AccountGroupCancelReview | None = Field(
+ default=None, exclude_if=lambda review: review is None
+ )
+
+ @property
+ def account_group_decision(self) -> AccountGroupDecision | None:
+ if self.account_group_review is not None:
+ return self.account_group_review.decision
+ return self.account_group_cancel.decision if self.account_group_cancel is not None else None
def tool_result(self) -> dict[str, Any]:
if self.status != "resolved" or self.response is None:
raise ValueError("Question has no accepted response")
+ if self.account_group_review is not None:
+ decision = self.account_group_decision
+ approved_ids = decision.credential_ids if decision is not None and decision.approved else []
+ group_id = self.account_group_review.workflow_run_group_id
+ # None: approved, but no group is linked, so whether it was submitted is unknown.
+ dispatched = True if group_id is not None else (None if approved_ids else False)
+ return {
+ "ok": True,
+ "interaction_id": self.interaction_id,
+ "tool_call_id": self.tool_call_id,
+ "approved": bool(approved_ids),
+ "approved_credential_ids": approved_ids,
+ "repeated_after_prior_effect": [
+ row.model_dump(mode="json", exclude={"identity"})
+ for row in self.account_group_review.repeated_after_prior_effect(approved_ids)
+ ],
+ "dispatched": dispatched,
+ "workflow_run_group_id": group_id,
+ }
+ if self.account_group_cancel is not None:
+ decision = self.account_group_decision
+ return {
+ "ok": True,
+ "interaction_id": self.interaction_id,
+ "tool_call_id": self.tool_call_id,
+ "workflow_run_group_id": self.account_group_cancel.workflow_run_group_id,
+ "cancel_approved": decision is not None and decision.approved,
+ }
answers = {answer.part_id: answer for answer in self.response.answers}
parts = []
for part in self.parts:
@@ -147,11 +253,50 @@ def create_question_interaction(
)
-def resolve_question_response(interaction: QuestionInteraction, response: QuestionResponse) -> QuestionInteraction:
+def _validate_account_group_decision(
+ interaction: QuestionInteraction, response: QuestionResponse, decision: AccountGroupDecision | None
+) -> None:
+ review = interaction.account_group_review
+ if interaction.account_group_cancel is not None:
+ if response.answers or response.text is not None:
+ raise ValueError("A cancel review takes only a decision")
+ if decision is None and not response.skipped:
+ raise ValueError("A cancel review requires a decision")
+ if decision is not None and decision.credential_ids:
+ raise ValueError("A cancel review stops the whole group, not chosen accounts")
+ return
+ if review is None:
+ if decision is not None:
+ raise ValueError("This question has no account review")
+ return
+ if response.answers or response.text is not None:
+ raise ValueError("An account review takes only a decision")
+ if decision is None:
+ if not response.skipped:
+ raise ValueError("An account review requires a decision")
+ return
+ if not decision.approved:
+ if decision.credential_ids:
+ raise ValueError("A declined review approves no accounts")
+ return
+ if not decision.credential_ids:
+ raise ValueError("Approve at least one account")
+ if len(set(decision.credential_ids)) != len(decision.credential_ids):
+ raise ValueError("An approval lists an account twice")
+ if not set(decision.credential_ids) <= {row.credential_id for row in review.rows}:
+ raise ValueError("An approval lists an account the review did not")
+
+
+def resolve_question_response(
+ interaction: QuestionInteraction,
+ response: QuestionResponse,
+ account_group_decision: AccountGroupDecision | None = None,
+) -> QuestionInteraction:
if interaction.status != "pending":
raise ValueError("Question is no longer pending")
- if response.skipped and (response.answers or response.text is not None):
+ if response.skipped and (response.answers or response.text is not None or account_group_decision):
raise ValueError("A skipped response cannot also contain answers")
+ _validate_account_group_decision(interaction, response, account_group_decision)
parts = {part.part_id: part for part in interaction.parts}
answered: set[str] = set()
for answer in response.answers:
@@ -165,24 +310,31 @@ def resolve_question_response(interaction: QuestionInteraction, response: Questi
raise ValueError("Unknown choice for this part")
if answer.choice_id is None and answer.text is None:
raise ValueError("An answer requires a choice or text")
+ decided: dict[str, Any] = {}
+ if interaction.account_group_review is not None:
+ decided["account_group_review"] = interaction.account_group_review.model_copy(
+ update={"decision": account_group_decision}
+ )
+ if interaction.account_group_cancel is not None:
+ decided["account_group_cancel"] = interaction.account_group_cancel.model_copy(
+ update={"decision": account_group_decision}
+ )
return interaction.model_copy(
update={
"status": "resolved",
"response": response.model_copy(deep=True),
"resolved_at": datetime.now(UTC),
+ **decided,
}
)
-async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_id: str) -> dict[str, Any]:
- """Display one tool request and deliver its recorded response to this same invocation."""
- from skyvern.forge import app
-
+async def wait_for_interaction(ctx: CopilotContext, interaction: QuestionInteraction) -> QuestionInteraction:
+ """Show one recorded interaction and return it once the user's response is accepted."""
if ctx.workflow_copilot_chat_id is None or ctx.stream is None:
- raise ValueError("ask_user requires an active Copilot chat")
+ raise ValueError("A Copilot question requires an active Copilot chat")
repo = app.DATABASE.workflow_params
chat_id = ctx.workflow_copilot_chat_id
- interaction = create_question_interaction(arguments, turn_id=ctx.turn_id, tool_call_id=tool_call_id, ctx=ctx)
with pause_human_input(ctx, "question"):
await repo.start_copilot_question(ctx.organization_id, chat_id, interaction)
try:
@@ -198,11 +350,6 @@ async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_i
while True:
recorded = await repo.poll_copilot_question(ctx.organization_id, chat_id, interaction.interaction_id)
if recorded.status == "resolved":
- if ctx.request_policy is not None:
- ctx.request_policy.project_question_response_sites(recorded)
- ctx.allow_untested_workflow_draft = ctx.request_policy.raw_secret_redacted_draft
- if recorded.response is not None and not recorded.response.skipped:
- ctx.credential_recovery_armed = True
await ctx.stream.send(
{
"type": "question_resolved",
@@ -210,7 +357,7 @@ async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_i
"continued": True,
}
)
- return recorded.tool_result()
+ return recorded
if recorded.status != "pending":
raise asyncio.CancelledError("The question was cancelled or interrupted")
await asyncio.sleep(QUESTION_POLL_SECONDS)
@@ -228,3 +375,15 @@ async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_i
finally:
if not cleanup.done():
cleanup.cancel()
+
+
+async def ask_user(ctx: CopilotContext, arguments: AskUserArguments, tool_call_id: str) -> dict[str, Any]:
+ """Display one tool request and deliver its recorded response to this same invocation."""
+ interaction = create_question_interaction(arguments, turn_id=ctx.turn_id, tool_call_id=tool_call_id, ctx=ctx)
+ recorded = await wait_for_interaction(ctx, interaction)
+ if ctx.request_policy is not None:
+ ctx.request_policy.project_question_response_sites(recorded)
+ ctx.allow_untested_workflow_draft = ctx.request_policy.raw_secret_redacted_draft
+ if recorded.response is not None and not recorded.response.skipped:
+ ctx.credential_recovery_armed = True
+ return recorded.tool_result()
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index ec56e1faa..01a0b03f9 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -804,6 +804,13 @@ def _summarize_tool_output(output: str) -> str:
# the user just told it. MCP results are excluded so a server cannot opt out of compaction.
if MCP_RESULT_PROVENANCE_KEY not in parsed and "interaction_id" in parsed and isinstance(parsed.get("parts"), list):
return output
+ # An account group result is the only record of which runs exist and what each did.
+ if (
+ MCP_RESULT_PROVENANCE_KEY not in parsed
+ and "workflow_run_group_id" in parsed
+ and isinstance(parsed.get("rows"), list)
+ ):
+ return output
synopsis: dict[str, Any] = {}
# Compaction must not launder untrusted MCP data into unlabelled context. The owned value is
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 90eed2480..d925ab5fd 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -13,7 +13,6 @@ from copy import deepcopy
from dataclasses import dataclass, field, replace
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Literal, Protocol, cast
-from urllib.parse import urlparse
import structlog
from agents.agent import AgentBase
@@ -34,7 +33,6 @@ from skyvern.cli.core.client import reset_api_key_override, set_api_key_override
from skyvern.cli.core.session_manager import request_session_scope
from skyvern.cli.mcp_tools.response import MCP_MAX_RESPONSE_CHARS
from skyvern.forge import app
-from skyvern.forge.agent_functions import CopilotCandidateNetworkHop
from skyvern.forge.sdk.copilot.blocker_signal import (
BROWSER_SESSION_LOST_BLOCKER_REASON_CODE,
CopilotToolBlockerSignal,
@@ -63,7 +61,6 @@ from skyvern.forge.sdk.copilot.runtime import (
mcp_browser_context,
mcp_to_copilot,
raw_secret_browser_denied,
- resolve_browser_state_for_context,
retire_browser_session_id,
sensitive_origin_page_facts_withheld,
sensitive_origin_page_is_tainted,
@@ -151,7 +148,6 @@ class _BrowserCallOutcome:
replacement_browser_session_id: str | None = None
completion_browser_session_id: str | None = None
completion_browser_session_generation: int | None = None
- evidence_drain_complete: bool | None = None
cancelled: bool = False
protocol_error_detail: str | None = None
last_run_facts: dict[str, str] = field(default_factory=dict)
@@ -365,7 +361,6 @@ def _record_browser_call_outcome(
replacement_browser_session_id=outcome.replacement_browser_session_id,
completion_browser_session_id=outcome.completion_browser_session_id,
completion_browser_session_generation=outcome.completion_browser_session_generation,
- evidence_drain_complete=outcome.evidence_drain_complete,
response_truncated=outcome.response_truncated,
payload_omitted=outcome.payload_omitted,
**_copilot_log_fields(cast("CopilotContext", ctx)),
@@ -864,9 +859,7 @@ def _scrub_tool_exception(ctx: AgentContext, tool_name: str, exception: BaseExce
return scrub_model_facing_tool_result(ctx, {"ok": False, "error": error})
-_MCPCallPhase = Literal["session_prepare", "context_enter", "dispatch", "evidence_drain", "context_exit"]
-# The segments charged to the wall clock. ``evidence_drain`` names where a call died but is never
-# charged here, so the residual stays the remainder of what the record actually reports.
+_MCPCallPhase = Literal["session_prepare", "context_enter", "dispatch", "context_exit"]
_MCP_CALL_SEGMENTS: tuple[_MCPCallPhase, ...] = ("session_prepare", "context_enter", "dispatch", "context_exit")
@@ -882,7 +875,6 @@ class _PhaseClock:
self._excluded_seconds = 0.0
self.current: _MCPCallPhase | None = None
self.elapsed: dict[_MCPCallPhase, int] = {}
- self.drain_ms: int | None = None
def _charge(self, now: float) -> None:
if self._bucket is not None:
@@ -925,12 +917,6 @@ class _PhaseClock:
self._wall = int((now - self._started - self._excluded_seconds) * 1000)
return self._wall
- def record_drain(self, started: float, *, failed: bool) -> None:
- """Settling the page runs after the wall is frozen: the caller waits for it, but it is not part of the call."""
- self.drain_ms = int((time.monotonic() - started) * 1000)
- if failed:
- self.current = "evidence_drain"
-
def settle(self) -> None:
"""The call answered, so no segment holds a failure and ``timing_phase`` would misname a returned error."""
self.current = None
@@ -982,7 +968,6 @@ def _log_mcp_timing(
phase_context_enter_ms=spent["context_enter"],
phase_dispatch_ms=dispatch_ms,
phase_context_exit_ms=spent["context_exit"],
- post_call_evidence_drain_ms=phases.drain_ms,
phase_residual_ms=wall_clock_ms - sum(spent.values()),
timing_phase=phases.current if call_status != "ok" else None,
phase_dispatch_untimed_ms=untimed_ms,
@@ -1442,17 +1427,6 @@ def _copilot_to_call_tool_result(
return CallToolResult(content=content, isError=is_error)
-def _evidence_candidate_url_origin(url: str) -> str | None:
- try:
- parsed = urlparse(url)
- if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
- return None
- port = f":{parsed.port}" if parsed.port else ""
- except ValueError:
- return None
- return f"https://{parsed.hostname.lower()}{port}"
-
-
@asynccontextmanager
async def service_worker_blocked_context(
browser_state: BrowserState,
@@ -1461,7 +1435,7 @@ async def service_worker_blocked_context(
) -> AsyncIterator[BrowserContext]:
original_context = browser_state.browser_context
if original_context is None:
- raise RuntimeError("Evidence-candidate browser does not support an isolated context")
+ raise RuntimeError("This browser does not support an isolated context")
original_page = await browser_state.get_working_page()
browser = original_context.browser
fallback_browser: Browser | None = None
@@ -1524,8 +1498,6 @@ class SkyvernOverlayMCPServer(MCPServer):
self._client: Client | None = None
self._exit_stack: AsyncExitStack | None = None
self._cached_raw_tools: list[MCPTool] | None = None
- self._evidence_candidate_origin: str | None = None
- self._evidence_candidate_guarded_hops: list[CopilotCandidateNetworkHop] | None = None
@property
def name(self) -> str:
@@ -1556,82 +1528,6 @@ class SkyvernOverlayMCPServer(MCPServer):
self._exit_stack = None
self._cached_raw_tools = None
- @asynccontextmanager
- async def evidence_candidate_navigation_guard(
- self,
- expected_origin: str,
- ) -> AsyncIterator[list[CopilotCandidateNetworkHop]]:
- if self._evidence_candidate_origin is not None:
- raise RuntimeError("Evidence-candidate navigation guard is already active")
- normalized_origin = _evidence_candidate_url_origin(expected_origin)
- if normalized_origin != expected_origin:
- raise ValueError("Evidence-candidate origin must be an exact HTTPS origin")
- ctx = self._context_provider()
- session_error = await ensure_browser_session(ctx)
- if session_error is not None:
- raise RuntimeError(str(session_error.get("error", "Evidence-candidate browser session unavailable")))
- examined_session_id = ctx.browser_session_id
- try:
- browser_state = await resolve_browser_state_for_context(ctx)
- if browser_state is None:
- await retire_browser_session_id(ctx, examined_session_id)
- raise RuntimeError("Evidence-candidate navigation guard requires a browser context")
- async with service_worker_blocked_context(
- browser_state,
- organization_id=ctx.organization_id,
- ) as browser_context:
- cookies = await browser_context.cookies()
- if (
- cookies
- or browser_context.service_workers
- or any(page.url not in {"", "about:blank"} for page in browser_context.pages)
- ):
- raise RuntimeError("Evidence-candidate navigation guard requires a pristine browser context")
- async with app.AGENT_FUNCTION.copilot_candidate_network_guard(
- browser_context, expected_origin=normalized_origin
- ) as guarded_hops:
- self._evidence_candidate_origin = normalized_origin
- self._evidence_candidate_guarded_hops = guarded_hops
- try:
- yield guarded_hops
- finally:
- await app.AGENT_FUNCTION.wait_for_copilot_candidate_network_idle(browser_context)
- finally:
- self._evidence_candidate_origin = None
- self._evidence_candidate_guarded_hops = None
-
- async def _drain_evidence_candidate_response_tasks(self) -> None:
- if self._evidence_candidate_origin is None:
- return
- browser_state = await resolve_browser_state_for_context(self._context_provider())
- browser_context = browser_state.browser_context if browser_state is not None else None
- if browser_context is None:
- raise RuntimeError("Evidence-candidate browser context became unavailable")
- await app.AGENT_FUNCTION.wait_for_copilot_candidate_network_idle(browser_context)
-
- async def evidence_candidate_browser_url(self) -> str:
- if self._evidence_candidate_origin is None:
- raise RuntimeError("Evidence-candidate navigation guard is not active")
- browser_state = await resolve_browser_state_for_context(self._context_provider())
- page = await browser_state.get_working_page() if browser_state is not None else None
- if page is None:
- raise RuntimeError("Evidence-candidate working page is unavailable")
- browser_url = page.url
- last_enforced_url = next(
- (
- hop["url"]
- for hop in reversed(self._evidence_candidate_guarded_hops or [])
- if hop["resource_type"] == "document"
- ),
- None,
- )
- if (
- _evidence_candidate_url_origin(browser_url) != self._evidence_candidate_origin
- or browser_url != last_enforced_url
- ):
- raise RuntimeError("candidate_browser_url_not_peer_verified")
- return browser_url
-
async def list_tools(
self,
run_context: RunContextWrapper[Any] | None = None,
@@ -2479,7 +2375,6 @@ class SkyvernOverlayMCPServer(MCPServer):
merged_args = {**mcp_args, "session_id": call_browser_session_id}
call_browser_session_generation = ctx.browser_session_continuity_generation
browser_outcome: _BrowserCallOutcome | None = None
- post_dispatch_status: Literal["ok", "error", "session_error"] = "ok"
try:
phases.enter("context_enter")
# Only pass the override when there is one, as the model-facing path does: an
@@ -2505,45 +2400,7 @@ class SkyvernOverlayMCPServer(MCPServer):
phases.unwind()
raise
phases.enter("context_exit")
- # The evidence drain runs outside the dispatch span this record covers.
phases.close()
- if self._evidence_candidate_origin is not None:
- drain_started = time.monotonic()
- try:
- await asyncio.sleep(0)
- await self._drain_evidence_candidate_response_tasks()
- except asyncio.CancelledError:
- phases.record_drain(drain_started, failed=True)
- if browser_outcome is not None:
- browser_outcome = replace(
- browser_outcome,
- evidence_drain_complete=False,
- cancelled=True,
- completion_browser_session_id=ctx.browser_session_id,
- completion_browser_session_generation=ctx.browser_session_continuity_generation,
- )
- _record_browser_call_outcome(ctx, browser_outcome, call_path="internal")
- raise
- except CopilotBrowserSessionUnavailable:
- phases.record_drain(drain_started, failed=True)
- if browser_outcome is None:
- raise
- post_dispatch_status = "session_error"
- if browser_outcome is not None:
- browser_outcome = replace(browser_outcome, evidence_drain_complete=False)
- LOG.warning("Internal MCP evidence drain lost its browser session", tool=mcp_tool_name)
- except Exception:
- phases.record_drain(drain_started, failed=True)
- if browser_outcome is None:
- raise
- post_dispatch_status = "error"
- if browser_outcome is not None:
- browser_outcome = replace(browser_outcome, evidence_drain_complete=False)
- LOG.warning("Internal MCP evidence drain failed", tool=mcp_tool_name)
- else:
- phases.record_drain(drain_started, failed=False)
- if browser_outcome is not None:
- browser_outcome = replace(browser_outcome, evidence_drain_complete=True)
except asyncio.CancelledError:
_log_mcp_timing(ctx, copilot_name, mcp_tool_name, phases, {}, "internal", "cancelled")
if uses_shared_browser_outcome and browser_outcome is None:
@@ -2648,9 +2505,8 @@ class SkyvernOverlayMCPServer(MCPServer):
else:
raw_mcp["error"] = raw_mcp.get("error") or "Unknown MCP error"
failed = raw.is_error or raw_mcp.get("ok", True) is not True
- if post_dispatch_status == "ok":
- phases.settle()
- call_status = "error" if failed else post_dispatch_status
+ phases.settle()
+ call_status: Literal["ok", "error"] = "error" if failed else "ok"
_log_mcp_timing(ctx, copilot_name, mcp_tool_name, phases, raw_mcp, "internal", call_status)
scrubbed = scrub_model_facing_tool_result(ctx, raw_mcp, tool_name=mcp_tool_name)
error_code = browser_outcome.error_code if browser_outcome is not None else _browser_error_code(scrubbed)
diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.py
index 7fe1511a0..0ab680a15 100644
--- a/skyvern/forge/sdk/copilot/narration.py
+++ b/skyvern/forge/sdk/copilot/narration.py
@@ -11,6 +11,11 @@ from typing import TYPE_CHECKING, Any, NamedTuple
import structlog
+from skyvern.forge.sdk.copilot.ask_user import (
+ ACCOUNT_GROUP_CANCEL_TOOL_NAME,
+ ACCOUNT_GROUP_STATUS_TOOL_NAME,
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME,
+)
from skyvern.forge.sdk.copilot.code_write_diff import CodeWriteDiff
from skyvern.forge.sdk.copilot.context import ActivityBucket, NarrativeBlockAttempt, upsert_narrative_block_attempt
from skyvern.forge.sdk.copilot.output_utils import sanitize_block_label_for_display
@@ -91,6 +96,9 @@ _TOOL_ACTIVITY_DISPLAY_LABELS = {
"delete_block": "Deleting block",
"request_credential": "Requesting a credential",
"ask_user": "Asking you",
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME: "Reviewing the accounts with you",
+ ACCOUNT_GROUP_STATUS_TOOL_NAME: "Checking the account runs",
+ ACCOUNT_GROUP_CANCEL_TOOL_NAME: "Reviewing a cancel with you",
"set_work_plan": "Updating its plan",
}
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index a15464539..1877f08e5 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -210,6 +210,24 @@ def _looks_like_envelope(parsed: dict[str, Any]) -> bool:
return isinstance(type_value, str) and type_value.upper() in COPILOT_RESPONSE_TYPES
+def _last_envelope_in(text: str) -> dict[str, Any] | None:
+ # A reply can hold a draft object before the final one, so the span from the first brace to the
+ # last is not one JSON value; each object is decoded where it starts and the last envelope wins.
+ decoder = json.JSONDecoder(strict=False)
+ found: dict[str, Any] | None = None
+ index = text.find("{")
+ while index != -1:
+ try:
+ value, end = decoder.raw_decode(text, index)
+ except json.JSONDecodeError:
+ # The next brace may open an object nested in this unparseable one, which is not a reply.
+ break
+ if isinstance(value, dict) and _looks_like_envelope(value):
+ found = value
+ index = text.find("{", end)
+ return found
+
+
def _text_looks_envelope_shaped(text: str) -> bool:
# require leading `{` so prose that merely quotes the field names (e.g.,
# "I see \"type\": \"REPLY\" but cannot find \"user_response\"") falls
@@ -245,13 +263,9 @@ def parse_final_response(text: str) -> dict[str, Any]:
return parsed
cleaned = label_stripped
- first = cleaned.find("{")
- last = cleaned.rfind("}")
- # skip when the slice equals the full string — _try_loads_dict above already tried it
- if first != -1 and last > first and not (first == 0 and last == len(cleaned) - 1):
- parsed = _try_loads_dict(cleaned[first : last + 1])
- if parsed is not None and _looks_like_envelope(parsed):
- return parsed
+ parsed = _last_envelope_in(cleaned)
+ if parsed is not None:
+ return parsed
if _text_looks_envelope_shaped(cleaned):
sniffed_type = _sniff_response_type(cleaned)
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 60dcbd1cd..d4f2ebfc9 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -15,9 +15,16 @@ import structlog
from agents import FunctionTool, function_tool
from agents.run_context import RunContextWrapper
from agents.tool_context import ToolContext
+from pydantic import JsonValue
from skyvern.forge import app as app
-from skyvern.forge.sdk.copilot.ask_user import AskUserArguments, QuestionInput
+from skyvern.forge.sdk.copilot.ask_user import (
+ ACCOUNT_GROUP_CANCEL_TOOL_NAME,
+ ACCOUNT_GROUP_STATUS_TOOL_NAME,
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME,
+ AskUserArguments,
+ QuestionInput,
+)
from skyvern.forge.sdk.copilot.browser_target import (
BROWSER_TARGET_PARAM_NAME,
BrowserTarget,
@@ -66,6 +73,15 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (
enqueue_screenshot_from_result,
)
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
+from skyvern.forge.sdk.copilot.tools.account_groups import (
+ SUBMIT_TOOL_DESCRIPTION as ACCOUNT_GROUP_SUBMIT_TOOL_DESCRIPTION,
+)
+from skyvern.forge.sdk.copilot.tools.account_groups import (
+ account_group_status,
+ account_group_submit_enabled,
+ cancel_account_group,
+ run_for_accounts,
+)
from skyvern.forge.sdk.copilot.tools.locator_inspection import TOOL_DESCRIPTION as LOCATOR_INSPECTION_TOOL_DESCRIPTION
from skyvern.forge.sdk.copilot.tools.locator_inspection import TOOL_NAME as LOCATOR_INSPECTION_TOOL_NAME
from skyvern.forge.sdk.copilot.tools.locator_inspection import TOOL_SCHEMA as LOCATOR_INSPECTION_TOOL_SCHEMA
@@ -166,7 +182,6 @@ from .discovery import _discovery_detect_anti_bot as _discovery_detect_anti_bot
from .discovery import _discovery_detect_login_wall as _discovery_detect_login_wall
from .discovery import _discovery_resolve_href as _discovery_resolve_href
from .discovery import _discovery_walk as _discovery_walk
-from .discovery import _rank_discovery_entrypoint_candidates as _rank_discovery_entrypoint_candidates
from .discovery import _resolve_discovery_entry_url as _resolve_discovery_entry_url
from .errors import copilot_tool_failure
from .frontier import _CANONICAL_WORKFLOW_SETTING_FIELDS as _CANONICAL_WORKFLOW_SETTING_FIELDS
@@ -1073,6 +1088,50 @@ async def ask_user_tool(ctx: ToolContext[CopilotContext], parts: list[QuestionIn
return json.dumps(await ask_user(ctx.context, AskUserArguments(parts=parts), ctx.tool_call_id))
+@function_tool(
+ failure_error_function=copilot_tool_failure,
+ name_override=ACCOUNT_GROUP_SUBMIT_TOOL_NAME,
+ description_override=ACCOUNT_GROUP_SUBMIT_TOOL_DESCRIPTION,
+ strict_mode=False,
+)
+async def run_workflow_for_accounts_tool(
+ ctx: ToolContext[CopilotContext],
+ credential_ids: list[str],
+ credential_parameter_key: str,
+ action_summary: str,
+ common_inputs: dict[str, JsonValue] | None = None,
+) -> str:
+ authority_error = _authority_tool_error(ctx.context, ACCOUNT_GROUP_SUBMIT_TOOL_NAME)
+ if authority_error is not None:
+ return json.dumps({"ok": False, "dispatched": False, "error": authority_error})
+ result = await run_for_accounts(
+ ctx.context,
+ tool_call_id=ctx.tool_call_id,
+ credential_ids=credential_ids,
+ credential_parameter_key=credential_parameter_key,
+ common_inputs=common_inputs or {},
+ action_summary=action_summary,
+ )
+ return result.model_dump_json(exclude_none=True, exclude={"repeated_after_prior_effect": {"__all__": {"identity"}}})
+
+
+@function_tool(failure_error_function=copilot_tool_failure, name_override=ACCOUNT_GROUP_STATUS_TOOL_NAME)
+async def get_account_group_status_tool(ctx: ToolContext[CopilotContext], workflow_run_group_id: str) -> str:
+ """Read each account's run id, status and outcome for a group started by run_workflow_for_accounts.
+ A failed outcome means the run stopped before it could have acted; unknown means it may have acted."""
+ return (await account_group_status(ctx.context, workflow_run_group_id)).model_dump_json(exclude_none=True)
+
+
+@function_tool(failure_error_function=copilot_tool_failure, name_override=ACCOUNT_GROUP_CANCEL_TOOL_NAME)
+async def cancel_account_group_tool(ctx: ToolContext[CopilotContext], workflow_run_group_id: str) -> str:
+ """Ask the user to stop every unfinished run of a group this chat started with run_workflow_for_accounts.
+ Nothing is canceled unless the user approves; returns every row's status and whether they approved."""
+ result = await cancel_account_group(
+ ctx.context, tool_call_id=ctx.tool_call_id, workflow_run_group_id=workflow_run_group_id
+ )
+ return result.model_dump_json(exclude_none=True)
+
+
# This description is measured, not prose: a storage-fidelity sentence in it took authoring from 15/20 to 3/20.
# Re-measure with the arms in cloud_docs/workflow-copilot/architecture/offline-replay.md before editing.
@function_tool(failure_error_function=copilot_tool_failure, name_override="set_work_plan")
@@ -1760,23 +1819,18 @@ async def discover_workflow_entrypoint_tool(
site_or_url: str,
intent_hint: str,
) -> str:
- """Find the page a new workflow should start at when the user named a site but not the page.
-
- Use this BEFORE writing blocks when the user named a website (with a URL,
- a bare domain, or a single brand word) but no specific page. Accepts:
- a URL with or without scheme (``example.com/login`` is fine), a bare
- domain (``example.com``), or a single brand word. A brand word is resolved
- only from an exact provider-backed official-site association that safely
- navigates over public-network HTTPS to the associated origin. Results use
- ``contract_version=discover_workflow_entrypoint_v3``. English phrases
- ("the X website") return
- ``failure_reason=could_not_resolve_site_name`` — ASK_QUESTION for a URL.
+ """Find the page a new workflow should start at on a site whose URL or domain you have.
+
+ Use this BEFORE writing blocks when you know the site but not the specific
+ page. Accepts a URL with or without scheme (``example.com/login`` is fine)
+ or a bare domain (``example.com``). A site name returns
+ ``failure_reason=could_not_resolve_site_name``: pass a URL you already have
+ for it, such as one the user gave or a credential's ``tested_url``, or find
+ it with ``search_web``.
Returns ``candidate_url`` plus a short ``evidence_trail`` and any
- ``candidate_form_fields``. Evidence-backed brand results also include
- bounded ``candidate_provenance`` and ``navigation_evidence``. Use
- ``candidate_url`` as the ``url`` value on a ``goto_url`` block. Do NOT
- paste the evidence into workflow YAML.
+ ``candidate_form_fields``. Use ``candidate_url`` as the ``url`` value on a
+ ``goto_url`` block. Do NOT paste the evidence into workflow YAML.
Discovery navigates and reads pages; it will NOT type, click form buttons,
run JavaScript, or submit forms.
@@ -1793,7 +1847,8 @@ async def search_web_tool(ctx: RunContextWrapper, query: str, max_results: int =
"""Search the web for pages matching a query, when you need candidate sites rather than one known page.
Use this while scouting -- to find companies, suppliers, listings, or
- documentation pages the user described but did not name. ``results`` holds
+ documentation pages the user described but did not name, or the URL of a
+ site the user named without giving one. ``results`` holds
up to ``max_results`` (1 to 100) entries with ``title``, ``url`` and ``snippet``, each
``url`` a direct absolute link to the result site.
@@ -2201,6 +2256,10 @@ inspect_locator_matches_tool = FunctionTool(
)
+ACCOUNT_GROUP_TOOL_NAMES = frozenset(
+ {ACCOUNT_GROUP_SUBMIT_TOOL_NAME, ACCOUNT_GROUP_STATUS_TOOL_NAME, ACCOUNT_GROUP_CANCEL_TOOL_NAME}
+)
+
NATIVE_TOOLS = [
ask_user_tool,
set_work_plan_tool,
@@ -2228,6 +2287,9 @@ NATIVE_TOOLS = [
start_fresh_browser_tool,
extend_browser_session_tool,
upload_attached_file_tool,
+ run_workflow_for_accounts_tool,
+ get_account_group_status_tool,
+ cancel_account_group_tool,
]
@@ -2244,12 +2306,20 @@ BROWSER_BOUND_TOOL_NAMES = BLOCK_RUNNING_TOOLS | frozenset(
FRESH_BROWSER_TOOL_NAME,
SESSION_EXTENSION_TOOL_NAME,
UPLOAD_TOOL_NAME,
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME,
}
)
AUTHORING_GUIDANCE_TOOL_NAMES = frozenset({"add_block", "update_workflow", "update_and_run_blocks"})
-_PAGE_STATE_TOOL_NAMES = BROWSER_BOUND_TOOL_NAMES - BLOCK_RUNNING_TOOLS - {SESSION_EXTENSION_TOOL_NAME}
+_PAGE_STATE_TOOL_NAMES = (
+ BROWSER_BOUND_TOOL_NAMES
+ - BLOCK_RUNNING_TOOLS
+ - {
+ SESSION_EXTENSION_TOOL_NAME,
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME,
+ }
+)
def _with_page_state(tool: FunctionTool) -> FunctionTool:
@@ -2315,14 +2385,18 @@ def copilot_native_tools(
supports_question_tool: bool,
browser_code_available: bool,
authoring_capability: AuthoringCapability | BlockAuthoringPolicy | str | None = None,
+ supports_account_group_card: bool = False,
) -> list[FunctionTool]:
capability = _normalized_authoring_capability(authoring_capability)
both_families = capability.code_blocks and capability.agent_blocks
appended = SCHEMA_FIRST_GUIDANCE if not both_families else f"{AUTHORING_FAMILY_GUIDANCE}\n\n{SCHEMA_FIRST_GUIDANCE}"
tools: list[FunctionTool] = []
for tool in NATIVE_TOOLS:
- if (tool.name == "ask_user" and not supports_question_tool) or (
- tool.name == BROWSER_CODE_TOOL_NAME and not browser_code_available
+ if (
+ (tool.name == "ask_user" and not supports_question_tool)
+ or (tool.name == BROWSER_CODE_TOOL_NAME and not browser_code_available)
+ or (tool.name in ACCOUNT_GROUP_TOOL_NAMES and not supports_account_group_card)
+ or (tool.name == ACCOUNT_GROUP_SUBMIT_TOOL_NAME and not account_group_submit_enabled())
):
continue
if tool.name in AUTHORING_GUIDANCE_TOOL_NAMES:
diff --git a/skyvern/forge/sdk/copilot/tools/account_groups.py b/skyvern/forge/sdk/copilot/tools/account_groups.py
new file mode 100644
index 000000000..8460c767b
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/account_groups.py
@@ -0,0 +1,623 @@
+from __future__ import annotations
+
+import asyncio
+import hmac
+import json
+from collections import Counter
+from collections.abc import Iterable
+from hashlib import sha256
+from typing import Literal
+from uuid import uuid4
+
+import structlog
+from fastapi import HTTPException
+from pydantic import BaseModel, Field, JsonValue
+
+from skyvern.config import settings
+from skyvern.exceptions import SkyvernHTTPException
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.ask_user import (
+ REPEAT_RISK_OUTCOMES,
+ AccountGroupCancelReview,
+ AccountGroupReview,
+ AccountGroupRow,
+ AccountGroupRunRow,
+ QuestionInteraction,
+ wait_for_interaction,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure, scrub_secrets_from_text
+from skyvern.forge.sdk.copilot.tools.credentials import (
+ _credential_run_approval_error,
+ _extract_credential_ids_from_tool_value,
+)
+from skyvern.forge.sdk.copilot.tools.mcp_hooks import _chat_has_pending_proposal
+from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml, workflow_blocks
+from skyvern.forge.sdk.copilot.workflow_yaml import workflow_to_copilot_yaml
+from skyvern.forge.sdk.core.permissions.permission_checker_factory import PermissionCheckerFactory
+from skyvern.forge.sdk.schemas.credentials import Credential
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
+from skyvern.forge.sdk.workflow.models.workflow import Workflow
+from skyvern.schemas.workflow_run_groups import (
+ MAX_WORKFLOW_RUN_GROUP_ITEMS,
+ WorkflowRunGroupCreateRequest,
+ WorkflowRunGroupItemOutcome,
+ WorkflowRunGroupItemRequest,
+ WorkflowRunGroupResponse,
+ WorkflowRunGroupStatus,
+)
+from skyvern.schemas.workflows import BlockStatus
+from skyvern.services import workflow_run_group_service
+
+LOG = structlog.get_logger()
+
+SUBMIT_TOOL_DESCRIPTION = (
+ "Run the saved workflow once per selected saved account, one run after another, each in a fresh "
+ "browser."
+ "\n\n"
+ "Use this when the user wants the same saved workflow run for several of their saved accounts. Pass "
+ "the credential ids the user selected (from list_credentials), the saved workflow's credential_id "
+ "parameter that takes the account, the other inputs every run shares, and a one-sentence summary of "
+ "what each run does. The user reviews the accounts on a card, can remove any, and approves or "
+ "declines; nothing runs without approval. At most 25 accounts per group. The card shows whether this "
+ "saved version has completed cleanly in an earlier group for a selected account, and starts unchecked "
+ "any account whose last group run completed, may have acted, or is still running. To retry part of an "
+ "earlier group, propose those accounts again. The result only acknowledges dispatch; use "
+ "get_account_group_status for outcomes."
+)
+
+_UNFINISHED_OUTCOMES = frozenset({WorkflowRunGroupItemOutcome.pending, WorkflowRunGroupItemOutcome.in_progress})
+
+_UNCLEAN_BLOCK_STATUSES = frozenset(
+ {BlockStatus.failed, BlockStatus.terminated, BlockStatus.timed_out, BlockStatus.canceled}
+)
+
+# Every run setting an in-place accept writes, except what the engine rewrites after runs: run_with, cache_key,
+# code_version, browser_profile_id (set on the first persisted run) and title (default-title rename).
+_VERIFIED_WORKFLOW_FIELDS = frozenset(
+ {
+ "workflow_definition",
+ "proxy_location",
+ "webhook_callback_url",
+ "totp_verification_url",
+ "totp_identifier",
+ "persist_browser_session",
+ "reuse_browser_session",
+ "mask_secrets",
+ "pin_saved_session_ip",
+ "browser_profile_key",
+ "model",
+ "max_screenshot_scrolls",
+ "max_elapsed_time_minutes",
+ "extra_http_headers",
+ "cdp_connect_headers",
+ "ai_fallback",
+ "adaptive_caching",
+ "enable_self_healing",
+ "run_sequentially",
+ "sequential_key",
+ }
+)
+
+_DEFINITION_HASH_DOMAIN = b"skyvern.copilot_account_group_definition.v1\0"
+_IDENTITY_HASH_DOMAIN = b"skyvern.copilot_account_group_identity.v1\0"
+_ACCOUNT_GROUP_KEY_ROOT = "copilot:"
+
+_PENDING_PROPOSAL_REFUSAL = (
+ "Nothing was submitted: this chat has a workflow change that is not saved yet, and a group runs the "
+ "saved version. End this turn so the change can be saved (the user accepts it, or auto-accept applies "
+ "it when the turn ends) or rejected, then propose the group again."
+)
+
+_STATUS_NOTE = (
+ "Runs execute one after another. Pending and in-progress rows have no result yet; "
+ "use get_account_group_status for outcomes."
+)
+
+
+class AccountGroupRefused(Exception):
+ pass
+
+
+class AccountGroupRefusal(BaseModel):
+ ok: Literal[False] = False
+ error: str
+ status_code: int | None = None
+ dispatched: Literal[False] | None = None
+
+
+class AccountGroupSubmission(BaseModel):
+ ok: Literal[True] = True
+ approved: bool
+ dispatched: bool
+ workflow_run_group_id: str | None = None
+ clean_group_run_id: str | None = None
+ credential_parameter_used_by: list[str] = Field(default_factory=list)
+ rows: list[AccountGroupRunRow] = Field(default_factory=list)
+ repeated_after_prior_effect: list[AccountGroupRow] = Field(default_factory=list)
+ note: str
+
+
+class AccountGroupState(BaseModel):
+ ok: Literal[True] = True
+ workflow_run_group_id: str
+ status: WorkflowRunGroupStatus
+ rows: list[AccountGroupRunRow]
+ cancel_approved: bool | None = None
+
+
+def account_group_submit_enabled() -> bool:
+ return settings.COPILOT_ACCOUNT_GROUP_SUBMIT_ENABLED and settings.WORKFLOW_RUN_GROUPS_SUBMIT_ENABLED
+
+
+def workflow_definition_hash(workflow: Workflow) -> str:
+ fields = workflow.model_dump(mode="json", include=set(_VERIFIED_WORKFLOW_FIELDS))
+ # The model serializer masks every cdp_connect_headers value, so a changed header would hash the same.
+ fields["cdp_connect_headers"] = workflow.cdp_connect_headers
+ # Keyed because the digest leaves the server on the review card and covers raw header values.
+ message = _DEFINITION_HASH_DOMAIN + json.dumps(fields, sort_keys=True).encode()
+ return hmac.new(settings.SECRET_KEY.encode(), message, sha256).hexdigest()
+
+
+def _identity_digest(credential: Credential) -> str:
+ # The masked label can hide a username change, so the review binds the exact name and username it showed.
+ message = _IDENTITY_HASH_DOMAIN + json.dumps([credential.name, credential.username]).encode()
+ return hmac.new(settings.SECRET_KEY.encode(), message, sha256).hexdigest()
+
+
+def account_group_key_prefix(chat_id: str) -> str:
+ return f"{_ACCOUNT_GROUP_KEY_ROOT}{chat_id}:"
+
+
+def account_group_submission_key(chat_id: str, interaction_id: str, definition_hash: str) -> str:
+ return f"{account_group_key_prefix(chat_id)}{interaction_id}:{definition_hash}"
+
+
+def _approved_request(
+ chat_id: str, interaction_id: str, review: AccountGroupReview, approved_ids: list[str]
+) -> WorkflowRunGroupCreateRequest:
+ return WorkflowRunGroupCreateRequest(
+ workflow_id=review.workflow_permanent_id,
+ version=review.version,
+ submission_key=account_group_submission_key(chat_id, interaction_id, review.definition_hash),
+ items=[
+ WorkflowRunGroupItemRequest(
+ key=credential_id,
+ parameters={**review.common_inputs, review.credential_parameter_key: credential_id},
+ )
+ for credential_id in approved_ids
+ ],
+ )
+
+
+async def recover_account_group_links(
+ organization_id: str, chat_id: str, interactions: Iterable[QuestionInteraction]
+) -> None:
+ pending: list[tuple[AccountGroupReview, WorkflowRunGroupCreateRequest]] = []
+ for interaction in interactions:
+ review = interaction.account_group_review
+ decision = interaction.account_group_decision
+ if review is None or review.workflow_run_group_id is not None or decision is None or not decision.approved:
+ continue
+ try:
+ request = _approved_request(chat_id, interaction.interaction_id, review, decision.credential_ids)
+ except Exception:
+ LOG.warning(
+ "Skipping an approved review whose request cannot be rebuilt",
+ interaction_id=interaction.interaction_id,
+ exc_info=True,
+ )
+ continue
+ pending.append((review, request))
+ if not pending:
+ return
+ try:
+ groups = await app.DATABASE.workflow_run_groups.get_groups_by_submission_keys(
+ organization_id, [request.submission_key for _, request in pending]
+ )
+ except Exception:
+ LOG.warning("Failed to recover the account groups of approved reviews", chat_id=chat_id, exc_info=True)
+ return
+ for review, request in pending:
+ group = groups.get(request.submission_key)
+ if group is not None and hmac.compare_digest(
+ group.input_fingerprint, workflow_run_group_service.submission_fingerprint(request)
+ ):
+ review.workflow_run_group_id = group.workflow_run_group_id
+
+
+def _mask_username(username: str) -> str:
+ local, at, domain = username.partition("@")
+ return f"{local[:2]}***{at}{domain}"
+
+
+def _label(credential: Credential) -> str:
+ return f"{credential.name} ({_mask_username(credential.username)})" if credential.username else credential.name
+
+
+async def _saved_workflow(ctx: CopilotContext) -> Workflow:
+ workflow = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id
+ )
+ if workflow is None or not workflow.workflow_definition.blocks:
+ raise AccountGroupRefused("This workflow has no saved version with blocks to run.")
+ return workflow
+
+
+async def _require_credential_parameter(workflow: Workflow, key: str) -> WorkflowParameter:
+ parameters = await app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow.workflow_id)
+ credential_parameters = {
+ p.key: p for p in parameters if p.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID
+ }
+ if key not in credential_parameters:
+ raise AccountGroupRefused(
+ f"The saved workflow has no credential_id parameter named {key!r} "
+ f"(its credential_id parameters: {list(credential_parameters) or 'none'}). Nothing was submitted. Add a "
+ "credential_id input that takes the account, use it in the blocks that sign in, and have the user "
+ "accept that change."
+ )
+ return credential_parameters[key]
+
+
+async def _clean_group_run_id(
+ ctx: CopilotContext, workflow: Workflow, parameter: WorkflowParameter, credential_ids: list[str]
+) -> str | None:
+ return await app.DATABASE.workflow_runs.get_latest_group_child_run_id(
+ organization_id=ctx.organization_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_id=workflow.workflow_id,
+ workflow_parameter_id=parameter.workflow_parameter_id,
+ values=credential_ids,
+ submission_key_suffix=f":{workflow_definition_hash(workflow)}",
+ completed_without_block_statuses=_UNCLEAN_BLOCK_STATUSES,
+ )
+
+
+def _blocks_using_parameter(workflow: Workflow, key: str) -> list[str]:
+ parsed = parse_workflow_yaml(workflow_to_copilot_yaml(workflow))
+ return [
+ str(block.get("label"))
+ for block in workflow_blocks(parsed if isinstance(parsed, dict) else {})
+ if key in (block.get("parameter_keys") or [])
+ ]
+
+
+async def _latest_outcomes(
+ ctx: CopilotContext, workflow: Workflow, credential_ids: list[str]
+) -> tuple[dict[str, str], dict[str, WorkflowRunGroupItemOutcome]]:
+ latest = await app.DATABASE.workflow_run_groups.get_latest_group_ids_by_item_key(
+ ctx.organization_id, workflow.workflow_permanent_id, _ACCOUNT_GROUP_KEY_ROOT, credential_ids
+ )
+ groups = await asyncio.gather(
+ *(
+ workflow_run_group_service.get_workflow_run_group(group_id, ctx.organization_id)
+ for group_id in set(latest.values())
+ )
+ )
+ outcomes = {
+ item.key: item.outcome
+ for group in groups
+ for item in group.items
+ if latest.get(item.key) == group.workflow_run_group_id
+ }
+ return latest, outcomes
+
+
+async def _resolve_credentials(ctx: CopilotContext, credential_ids: list[str]) -> dict[str, Credential]:
+ found = {
+ credential.credential_id: credential
+ for credential in await app.DATABASE.credentials.get_credentials_by_ids(
+ credential_ids, organization_id=ctx.organization_id
+ )
+ }
+ missing = [credential_id for credential_id in credential_ids if credential_id not in found]
+ if missing:
+ raise AccountGroupRefused(f"These credential ids are not saved accounts in this organization: {missing}.")
+ return found
+
+
+async def _organization(ctx: CopilotContext) -> Organization:
+ organization = await app.DATABASE.organizations.get_organization(ctx.organization_id)
+ if organization is None:
+ raise AccountGroupRefused("Organization not found.")
+ return organization
+
+
+def _check_common_inputs(ctx: CopilotContext, common_inputs: dict[str, JsonValue]) -> None:
+ if scrub_secrets_from_structure(ctx, common_inputs) != common_inputs:
+ raise AccountGroupRefused(
+ "A common input holds a secret value, and secrets cannot be group inputs. Nothing was submitted. "
+ "Use a saved credential for the secret instead."
+ )
+ approval_error = _credential_run_approval_error(
+ _extract_credential_ids_from_tool_value(common_inputs), ctx.request_policy
+ )
+ if approval_error is not None:
+ raise AccountGroupRefused(approval_error)
+
+
+async def build_review(
+ ctx: CopilotContext,
+ *,
+ credential_ids: list[str],
+ credential_parameter_key: str,
+ common_inputs: dict[str, JsonValue],
+ action_summary: str,
+) -> AccountGroupReview:
+ if len(credential_ids) > MAX_WORKFLOW_RUN_GROUP_ITEMS:
+ raise AccountGroupRefused(
+ f"{len(credential_ids)} accounts were selected and one group runs at most "
+ f"{MAX_WORKFLOW_RUN_GROUP_ITEMS}. Nothing was submitted. Offer the user separate reviewed groups. "
+ f"All selected ids: {credential_ids}."
+ )
+ if not credential_ids:
+ raise AccountGroupRefused("Select at least one account.")
+ duplicates = sorted(credential_id for credential_id, count in Counter(credential_ids).items() if count > 1)
+ if duplicates:
+ raise AccountGroupRefused(f"These credential ids are selected more than once: {duplicates}.")
+ if credential_parameter_key in common_inputs:
+ raise AccountGroupRefused(f"{credential_parameter_key!r} is set per account, not as a common input.")
+ _check_common_inputs(ctx, common_inputs)
+ if await _chat_has_pending_proposal(ctx):
+ raise AccountGroupRefused(_PENDING_PROPOSAL_REFUSAL)
+ workflow = await _saved_workflow(ctx)
+ if reason := workflow_run_group_service.non_fresh_browser_reason(workflow):
+ raise AccountGroupRefused(f"The saved workflow cannot run as a group because it has {reason}.")
+ credential_parameter = await _require_credential_parameter(workflow, credential_parameter_key)
+ credentials = await _resolve_credentials(ctx, credential_ids)
+ try:
+ await app.WORKFLOW_SERVICE.validate_schedule_parameters(
+ workflow,
+ await _organization(ctx),
+ {**common_inputs, credential_parameter_key: credential_ids[0]},
+ )
+ except SkyvernHTTPException as e:
+ raise AccountGroupRefused(str(e)) from e
+ last_groups, prior_outcomes = await _latest_outcomes(ctx, workflow, credential_ids)
+ return AccountGroupReview(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_id=workflow.workflow_id,
+ version=workflow.version,
+ workflow_title=workflow.title,
+ modified_at=workflow.modified_at,
+ definition_hash=workflow_definition_hash(workflow),
+ credential_parameter_key=credential_parameter_key,
+ action_summary=redact_raw_secrets_for_prompt(scrub_secrets_from_text(ctx, action_summary)),
+ common_inputs=common_inputs,
+ clean_group_run_id=await _clean_group_run_id(ctx, workflow, credential_parameter, credential_ids),
+ credential_parameter_used_by=_blocks_using_parameter(workflow, credential_parameter_key),
+ rows=[
+ AccountGroupRow(
+ credential_id=credential_id,
+ label=_label(credentials[credential_id]),
+ identity=_identity_digest(credentials[credential_id]),
+ prior_outcome=prior_outcomes.get(credential_id),
+ last_group_id=last_groups.get(credential_id),
+ preselected=prior_outcomes.get(credential_id) not in REPEAT_RISK_OUTCOMES,
+ )
+ for credential_id in credential_ids
+ ],
+ )
+
+
+async def _recheck_approval(ctx: CopilotContext, review: AccountGroupReview, approved_ids: list[str]) -> dict[str, str]:
+ if not account_group_submit_enabled():
+ raise AccountGroupRefused(
+ "Account group submission was turned off after the review, so nothing was dispatched."
+ )
+ if await _chat_has_pending_proposal(ctx):
+ raise AccountGroupRefused(_PENDING_PROPOSAL_REFUSAL)
+ workflow = await _saved_workflow(ctx)
+ if (workflow.workflow_id, workflow.version, workflow.modified_at) != (
+ review.workflow_id,
+ review.version,
+ review.modified_at,
+ ):
+ raise AccountGroupRefused(
+ "The saved workflow was updated after the review, so nothing was dispatched. "
+ "Propose the group again so the user can review the current version."
+ )
+ _check_common_inputs(ctx, review.common_inputs)
+ await _require_credential_parameter(workflow, review.credential_parameter_key)
+ credentials = await _resolve_credentials(ctx, approved_ids)
+ shown = {row.credential_id: row.identity for row in review.rows}
+ if renamed := sorted(i for i in approved_ids if _identity_digest(credentials[i]) != shown.get(i)):
+ raise AccountGroupRefused(
+ f"Nothing was dispatched: the saved account behind {renamed} changed after the review. "
+ "Propose the group again so the user can review the current accounts."
+ )
+ reviewed = {row.credential_id: row.last_group_id for row in review.rows if row.credential_id in approved_ids}
+ latest_groups = await app.DATABASE.workflow_run_groups.get_latest_group_ids_by_item_key(
+ ctx.organization_id, workflow.workflow_permanent_id, _ACCOUNT_GROUP_KEY_ROOT, approved_ids
+ )
+ changed = sorted(
+ credential_id for credential_id in approved_ids if latest_groups.get(credential_id) != reviewed[credential_id]
+ )
+ if changed:
+ raise AccountGroupRefused(
+ f"Nothing was dispatched: {changed} ran in another group after the review. "
+ "Propose the group again so the user can review their latest outcomes."
+ )
+ return {credential_id: group_id for credential_id, group_id in reviewed.items() if group_id is not None}
+
+
+def _rows(group: WorkflowRunGroupResponse, labels: dict[str, str]) -> list[AccountGroupRunRow]:
+ return [
+ AccountGroupRunRow(
+ credential_id=item.key,
+ label=labels.get(item.key, item.key),
+ workflow_run_id=item.workflow_run_id,
+ run_status=item.run_status,
+ outcome=item.outcome,
+ )
+ for item in group.items
+ ]
+
+
+async def submit_approved(
+ ctx: CopilotContext, interaction: QuestionInteraction
+) -> AccountGroupSubmission | AccountGroupRefusal:
+ review = interaction.account_group_review
+ decision = interaction.account_group_decision
+ chat_id = ctx.workflow_copilot_chat_id
+ if review is None or decision is None or not decision.approved or chat_id is None:
+ return AccountGroupSubmission(approved=False, dispatched=False, note="Nothing was dispatched.")
+ approved_ids = decision.credential_ids
+ # Held until the group exists, so a sibling tool call cannot publish a proposal after the recheck.
+ async with ctx.proposal_mutation_lock:
+ latest_groups = await _recheck_approval(ctx, review, approved_ids)
+ organization = await _organization(ctx)
+ request = _approved_request(chat_id, interaction.interaction_id, review, approved_ids)
+ try:
+ await PermissionCheckerFactory.get_instance().check(organization)
+ group = await workflow_run_group_service.submit_workflow_run_group(
+ organization,
+ request,
+ expected_workflow_modified_at=review.modified_at,
+ expected_latest_groups=(_ACCOUNT_GROUP_KEY_ROOT, latest_groups),
+ )
+ except SkyvernHTTPException as e:
+ return AccountGroupRefusal(status_code=e.status_code, error=str(e), dispatched=False)
+ except HTTPException as e:
+ return AccountGroupRefusal(status_code=e.status_code, error=str(e.detail), dispatched=False)
+ try:
+ await app.DATABASE.workflow_params.record_copilot_account_group_submission(
+ ctx.organization_id, chat_id, interaction.interaction_id, group.workflow_run_group_id
+ )
+ except Exception:
+ # Readers recover the link from the submission key, so the receipt still finds this group.
+ LOG.warning(
+ "Failed to record the account group on its review",
+ workflow_run_group_id=group.workflow_run_group_id,
+ exc_info=True,
+ )
+ submitted = interaction.model_copy(
+ update={
+ "account_group_review": review.model_copy(update={"workflow_run_group_id": group.workflow_run_group_id})
+ }
+ )
+ if ctx.stream is not None:
+ await ctx.stream.send(
+ {"type": "question_resolved", "interaction": submitted.model_dump(mode="json"), "continued": True}
+ )
+ labels = {row.credential_id: row.label for row in review.rows}
+ return AccountGroupSubmission(
+ approved=True,
+ dispatched=True,
+ workflow_run_group_id=group.workflow_run_group_id,
+ clean_group_run_id=review.clean_group_run_id,
+ credential_parameter_used_by=review.credential_parameter_used_by,
+ rows=_rows(group, labels),
+ repeated_after_prior_effect=review.repeated_after_prior_effect(approved_ids),
+ note=_STATUS_NOTE,
+ )
+
+
+async def run_for_accounts(
+ ctx: CopilotContext,
+ *,
+ tool_call_id: str,
+ credential_ids: list[str],
+ credential_parameter_key: str,
+ common_inputs: dict[str, JsonValue],
+ action_summary: str,
+) -> AccountGroupSubmission | AccountGroupRefusal:
+ try:
+ review = await build_review(
+ ctx,
+ credential_ids=credential_ids,
+ credential_parameter_key=credential_parameter_key,
+ common_inputs=common_inputs,
+ action_summary=action_summary,
+ )
+ recorded = await wait_for_interaction(
+ ctx,
+ QuestionInteraction(
+ interaction_id=uuid4().hex,
+ turn_id=ctx.turn_id,
+ tool_call_id=tool_call_id,
+ parts=[],
+ account_group_review=review,
+ ),
+ )
+ return await submit_approved(ctx, recorded)
+ except AccountGroupRefused as e:
+ return AccountGroupRefusal(error=str(e), dispatched=False)
+
+
+async def _group_for_this_workflow(ctx: CopilotContext, workflow_run_group_id: str) -> WorkflowRunGroupResponse:
+ try:
+ group = await workflow_run_group_service.get_workflow_run_group(workflow_run_group_id, ctx.organization_id)
+ except SkyvernHTTPException as e:
+ raise AccountGroupRefused(str(e)) from e
+ if group.workflow_permanent_id != ctx.workflow_permanent_id:
+ raise AccountGroupRefused(f"Group {workflow_run_group_id} runs a different workflow.")
+ return group
+
+
+async def _labels(ctx: CopilotContext, group: WorkflowRunGroupResponse) -> dict[str, str]:
+ credentials = await app.DATABASE.credentials.get_credentials_by_ids(
+ [item.key for item in group.items], organization_id=ctx.organization_id
+ )
+ return {credential.credential_id: _label(credential) for credential in credentials}
+
+
+async def _group_state(
+ ctx: CopilotContext, group: WorkflowRunGroupResponse, *, cancel_approved: bool | None = None
+) -> AccountGroupState:
+ return AccountGroupState(
+ workflow_run_group_id=group.workflow_run_group_id,
+ status=group.status,
+ rows=_rows(group, await _labels(ctx, group)),
+ cancel_approved=cancel_approved,
+ )
+
+
+async def account_group_status(
+ ctx: CopilotContext, workflow_run_group_id: str
+) -> AccountGroupState | AccountGroupRefusal:
+ try:
+ group = await _group_for_this_workflow(ctx, workflow_run_group_id)
+ except AccountGroupRefused as e:
+ return AccountGroupRefusal(error=str(e))
+ return await _group_state(ctx, group)
+
+
+async def cancel_account_group(
+ ctx: CopilotContext, *, tool_call_id: str, workflow_run_group_id: str
+) -> AccountGroupState | AccountGroupRefusal:
+ chat_id = ctx.workflow_copilot_chat_id
+ try:
+ group = await _group_for_this_workflow(ctx, workflow_run_group_id)
+ if chat_id is None or not group.submission_key.startswith(account_group_key_prefix(chat_id)):
+ raise AccountGroupRefused(
+ f"Group {workflow_run_group_id} was not started from this chat, so it cannot be canceled here."
+ )
+ state = await _group_state(ctx, group)
+ unfinished = [row for row in state.rows if row.outcome in _UNFINISHED_OUTCOMES]
+ if group.status != WorkflowRunGroupStatus.active or not unfinished:
+ raise AccountGroupRefused(f"Group {workflow_run_group_id} has no runs left to cancel.")
+ recorded = await wait_for_interaction(
+ ctx,
+ QuestionInteraction(
+ interaction_id=uuid4().hex,
+ turn_id=ctx.turn_id,
+ tool_call_id=tool_call_id,
+ parts=[],
+ account_group_cancel=AccountGroupCancelReview(
+ workflow_run_group_id=workflow_run_group_id, unfinished_rows=unfinished
+ ),
+ ),
+ )
+ decision = recorded.account_group_decision
+ if decision is None or not decision.approved:
+ group = await _group_for_this_workflow(ctx, workflow_run_group_id)
+ return await _group_state(ctx, group, cancel_approved=False)
+ group = await workflow_run_group_service.cancel_workflow_run_group(workflow_run_group_id, ctx.organization_id)
+ except AccountGroupRefused as e:
+ return AccountGroupRefusal(error=str(e))
+ except SkyvernHTTPException as e:
+ return AccountGroupRefusal(status_code=e.status_code, error=str(e))
+ return await _group_state(ctx, group, cancel_approved=True)
diff --git a/skyvern/forge/sdk/copilot/tools/discovery.py b/skyvern/forge/sdk/copilot/tools/discovery.py
index 975750f7e..bc872bf62 100644
--- a/skyvern/forge/sdk/copilot/tools/discovery.py
+++ b/skyvern/forge/sdk/copilot/tools/discovery.py
@@ -4,7 +4,6 @@ import asyncio
import re
import time
from collections.abc import Mapping
-from contextlib import AsyncExitStack
from typing import Any
from urllib.parse import urljoin, urlparse, urlsplit, urlunsplit
@@ -16,8 +15,6 @@ try:
except ImportError: # pragma: no cover — bs4 is a transitive dep but discovery degrades gracefully without it.
BeautifulSoup = None # type: ignore[assignment, misc]
-from skyvern.forge import app
-from skyvern.forge.agent_functions import CopilotCandidateNetworkHop, CopilotEntrypointCandidate
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -41,7 +38,6 @@ _DISCOVERY_EVIDENCE_TRAIL_MAX = 8
_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX = 10
_DISCOVERY_HTML_BYTES_MAX = 200_000
_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE = 0.6
-_DISCOVERY_RESULT_CONTRACT_VERSION = "discover_workflow_entrypoint_v3"
_DISCOVERY_URL_SCHEME_RE = re.compile(r"^https?://", re.IGNORECASE)
# host + optional path/query/fragment — handles `example.com/login`,
@@ -50,7 +46,6 @@ _DISCOVERY_DOMAIN_WITH_PATH_RE = re.compile(
r"^[a-z0-9-]+(\.[a-z]{2,})+([/?#][^\s]*)?$",
re.IGNORECASE,
)
-_DISCOVERY_BARE_WORD_RE = re.compile(r"^[a-z0-9-]{2,32}$", re.IGNORECASE)
_DISCOVERY_TOKEN_RE = re.compile(r"[a-z0-9]+", re.IGNORECASE)
_DISCOVERY_CANDIDATE_EVIDENCE_STOPWORDS = frozenset({"a", "an", "and", "for", "in", "of", "on", "or", "the", "to"})
_DISCOVERY_LOGIN_TITLE_RE = re.compile(r"\b(sign\s*in|log\s*in|login)\b", re.IGNORECASE)
@@ -60,72 +55,10 @@ _DISCOVERY_PASSWORD_INPUT_RE = re.compile(
)
-def _normalize_discovery_name(value: str) -> str:
- return "".join(_DISCOVERY_TOKEN_RE.findall(value.lower()))
-
-
-def _discovery_candidate_identity_bound(
- candidate: CopilotEntrypointCandidate,
- requested_name: str | None,
-) -> bool:
- association = candidate.association
- normalized_names = {
- _normalize_discovery_name(value)
- for value in (requested_name or "", association.requested_name, association.provider_relation_text)
- }
- return "" not in normalized_names and len(normalized_names) == 1
-
-
-def _user_provided_entry_url(copilot_ctx: CopilotContext) -> str | None:
- """A site the user gave this chat, when name lookup found nothing.
-
- Lookup asks the world what a name refers to; this asks the conversation. A user who pasted the
- URL has already answered, so exhausting the lookup and telling them to supply a URL they already
- supplied is the dead end this avoids. Only the sole site they gave answers; several is a question.
- """
- policy = copilot_ctx.request_policy
- urls = list(policy.user_provided_site_urls) if policy is not None else []
- return urls[0] if len(urls) == 1 else None
-
-
-def _site_name_names_host(site_name: str, url: str) -> bool:
- """Whether a hostname label opens with the requested name.
-
- A label may extend the name (`datadog` names `datadoghq`) but may not merely contain it, so
- `ups` does not name `groupsupport`.
- """
- normalized = _normalize_discovery_name(site_name or "")
- if len(normalized) < 3:
- return False
- try:
- host = urlparse(url).hostname or ""
- except ValueError:
- return False
- return any(_normalize_discovery_name(label).startswith(normalized) for label in host.split("."))
-
-
-def _credential_entry_url(copilot_ctx: CopilotContext, site_name: str) -> str | None:
- """The login page a credential resolved for this turn already signs in at.
-
- Only a credential whose host carries the requested name answers, because approvals persist
- across turns and a later request for a different site would otherwise open this one.
- """
- policy = copilot_ctx.request_policy
- credentials = list(policy.resolved_credentials) if policy is not None else []
- named: list[str] = []
- for credential in credentials:
- url = (credential.tested_url or "").strip()
- if url and url not in named and _site_name_names_host(site_name, url):
- named.append(url)
- return named[0] if len(named) == 1 else None
-
-
def _resolve_discovery_entry_url(site_or_url: str) -> tuple[str | None, str]:
- """Resolve the user-supplied site name/URL into a navigable URL.
+ """Returns ``(resolved_url, kind)`` where ``kind`` is ``url``, ``domain`` or ``unresolved``.
- Returns ``(resolved_url, kind)`` where ``kind`` is one of
- ``url`` / ``domain`` / ``bare_word`` / ``unresolved``. Bare words are
- classified here but acquired asynchronously by the AgentFunction boundary.
+ A site name is not a URL and resolves to nothing: deciding which site a name means is the model's call.
"""
token = (site_or_url or "").strip()
if not token:
@@ -134,48 +67,9 @@ def _resolve_discovery_entry_url(site_or_url: str) -> tuple[str | None, str]:
return token, "url"
if _DISCOVERY_DOMAIN_WITH_PATH_RE.match(token):
return f"https://{token}", "domain"
- if _DISCOVERY_BARE_WORD_RE.match(token):
- return None, "bare_word"
return None, "unresolved"
-def _discovery_origin(url: str) -> str | None:
- try:
- parsed = urlparse(url)
- except ValueError:
- return None
- if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
- return None
- port = f":{parsed.port}" if parsed.port else ""
- return f"https://{parsed.hostname.lower()}{port}"
-
-
-def _rank_discovery_entrypoint_candidates(
- requested_name: str,
- candidates: list[CopilotEntrypointCandidate],
-) -> list[CopilotEntrypointCandidate]:
- admitted: list[CopilotEntrypointCandidate] = []
- seen_urls: set[str] = set()
- for candidate in candidates:
- association = candidate.association
- candidate_origin = _discovery_origin(candidate.url)
- official_origin = _discovery_origin(association.official_site_url)
- if (
- not requested_name.strip()
- or requested_name.strip().casefold() != association.requested_name.strip().casefold()
- or requested_name.strip().casefold() != association.provider_relation_text.strip().casefold()
- or candidate_origin is None
- or candidate_origin != official_origin
- or candidate_origin != association.origin
- or candidate.source_rank < 1
- or candidate.url in seen_urls
- ):
- continue
- seen_urls.add(candidate.url)
- admitted.append(candidate)
- return sorted(admitted, key=lambda candidate: (candidate.source_rank, candidate.url))
-
-
def _concrete_homepage_entrypoint(entry_url: str | None, kind: str) -> str | None:
if kind not in {"domain", "url"} or not entry_url:
return None
@@ -241,8 +135,6 @@ def _discovery_build_result(
evidence_trail: list[dict[str, Any]],
confidence: float,
failure_reason: str | None,
- candidate_provenance: dict[str, str | int] | None = None,
- navigation_evidence: dict[str, str | bool | list[str] | list[CopilotCandidateNetworkHop]] | None = None,
ok: bool = True,
error: str | None = None,
) -> dict[str, Any]:
@@ -261,10 +153,6 @@ def _discovery_build_result(
"confidence": float(confidence),
"failure_reason": failure_reason,
}
- if candidate_provenance is not None:
- data["candidate_provenance"] = candidate_provenance
- if navigation_evidence is not None:
- data["navigation_evidence"] = navigation_evidence
return {"ok": ok, "data": data, "error": error}
@@ -278,18 +166,6 @@ def _redact_discovery_url_for_log(value: str | None) -> str | None:
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
-def _redact_guarded_hops_for_log(
- value: list[dict[str, str | list[str]]] | None,
-) -> list[dict[str, str | list[str] | None]] | None:
- if value is None:
- return None
- redacted: list[dict[str, str | list[str] | None]] = []
- for hop in value:
- hop_url = hop.get("url")
- redacted.append({**hop, "url": _redact_discovery_url_for_log(hop_url if isinstance(hop_url, str) else None)})
- return redacted
-
-
def _record_discovery_resolution_on_ctx(ctx: Any, result: Mapping[str, Any]) -> None:
data_payload = result.get("data")
data: Mapping[str, Any] = data_payload if isinstance(data_payload, Mapping) else {}
@@ -486,9 +362,6 @@ async def _discovery_walk(
*,
entry_url: str,
intent_hint: str,
- entrypoint_candidate: CopilotEntrypointCandidate | None = None,
- requested_name: str | None = None,
- guarded_hops: list[CopilotCandidateNetworkHop] | None = None,
) -> dict[str, Any]:
"""Deterministic anchor-scoring walker. No inner LLM call.
@@ -504,41 +377,12 @@ async def _discovery_walk(
current_page_title = ""
retried_deep_link_from_origin = False
started = time.monotonic()
- candidate_provenance: dict[str, str | int] | None = None
- navigation_evidence: dict[str, str | bool | list[str] | list[CopilotCandidateNetworkHop]] | None = None
- if entrypoint_candidate is not None:
- candidate_provenance = {
- "source": entrypoint_candidate.association.source,
- "source_rank": entrypoint_candidate.source_rank,
- "entity_id": entrypoint_candidate.association.entity_id,
- "associated_origin": entrypoint_candidate.association.origin,
- "provider_relation_type": entrypoint_candidate.association.provider_relation_type,
- "provider_relation_text": entrypoint_candidate.association.provider_relation_text,
- }
-
- def build_result(
- *,
- candidate_url: str | None,
- candidate_form_fields: list[dict[str, Any]],
- evidence_trail: list[dict[str, Any]],
- confidence: float,
- failure_reason: str | None,
- ) -> dict[str, Any]:
- return _discovery_build_result(
- candidate_url=candidate_url,
- candidate_form_fields=candidate_form_fields,
- evidence_trail=evidence_trail,
- confidence=confidence,
- failure_reason=failure_reason,
- candidate_provenance=candidate_provenance,
- navigation_evidence=navigation_evidence,
- )
for step in range(_DISCOVERY_STEP_CAP):
ctx.discovery_step_count = step + 1
elapsed = time.monotonic() - started
if elapsed > _DISCOVERY_WALL_CLOCK_SECONDS:
- return build_result(
+ return _discovery_build_result(
candidate_url=None,
candidate_form_fields=[],
evidence_trail=evidence_trail,
@@ -560,13 +404,12 @@ async def _discovery_walk(
"transition_reason": f"navigate_failed: {nav_result.get('error', 'unknown')}"[:240],
}
)
- strict_candidate = entrypoint_candidate is not None
- return build_result(
- candidate_url=None if strict_candidate else current_url,
+ return _discovery_build_result(
+ candidate_url=current_url,
candidate_form_fields=[],
evidence_trail=evidence_trail,
- confidence=0.0 if strict_candidate else _DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
- failure_reason="candidate_navigation_failed" if strict_candidate else None,
+ confidence=_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
+ failure_reason=None,
)
current_url = _discovery_extract_current_url(nav_result, current_url)
@@ -576,26 +419,6 @@ async def _discovery_walk(
# to identify a usable entrypoint. (Discovery only resolves the entrypoint, so a sliced
# tail does not matter here.)
html, _, _, _ = await _composition_get_html(ctx)
- if entrypoint_candidate is not None:
- server = ctx.discovery_mcp_server
- if server is None:
- return build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="candidate_guard_unavailable",
- )
- try:
- current_url = await server.evidence_candidate_browser_url()
- except Exception:
- return build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="candidate_browser_url_unverified",
- )
parsed_title, anchors, form_fields = _discovery_parse_html(html)
page_title = parsed_title or navigation_title
@@ -663,80 +486,19 @@ async def _discovery_walk(
current_url = origin_url
retried_deep_link_from_origin = True
continue
- return build_result(
+ return _discovery_build_result(
candidate_url=None,
candidate_form_fields=[],
evidence_trail=evidence_trail,
confidence=0.0,
failure_reason="anti_bot_wall",
)
- if login_wall_detected and login_page_is_the_only_lead and entrypoint_candidate is None:
- # A sign-in page is where a credentialed workflow starts, so a URL the user or their
- # own credential supplied is reported as the candidate rather than refused. Sign-in
- # pages routinely ship verification widgets, which the run solves; a page carrying a
- # challenge and nothing else already refused above. A machine-selected candidate falls
- # through to the binding checks below instead.
+ if login_wall_detected and login_page_is_the_only_lead:
+ # A sign-in page is where a credentialed workflow starts, so it is reported as the
+ # candidate rather than refused. Sign-in pages routinely ship verification widgets,
+ # which the run solves; a page carrying a challenge and nothing else already refused above.
evidence_trail[-1]["page_kind"] = "login"
- return build_result(
- candidate_url=current_url,
- candidate_form_fields=form_fields,
- evidence_trail=evidence_trail,
- confidence=1.0,
- failure_reason=None,
- )
-
- if entrypoint_candidate is not None:
- final_origin = _discovery_origin(current_url)
- origin_bound = final_origin == entrypoint_candidate.association.origin
- last_enforced_hop = next(
- (hop for hop in reversed(guarded_hops or []) if hop["resource_type"] == "document"),
- None,
- )
- enforcement_bound = last_enforced_hop is not None and last_enforced_hop["url"] == current_url
- identity_bound = _discovery_candidate_identity_bound(entrypoint_candidate, requested_name)
- navigation_evidence = {
- "requested_name": requested_name or "",
- "final_url": current_url,
- "final_page_title": page_title[:240],
- "https": final_origin is not None,
- "tls_valid": final_origin is not None,
- "resolved_public_ips": list(last_enforced_hop["resolved_public_ips"]) if last_enforced_hop else [],
- "associated_origin": entrypoint_candidate.association.origin,
- "origin_bound": origin_bound,
- "enforcement_bound": enforcement_bound,
- "identity_bound": identity_bound,
- "guarded_hops": list(guarded_hops or [])[:_DISCOVERY_EVIDENCE_TRAIL_MAX],
- }
- evidence_trail[-1]["candidate_source"] = entrypoint_candidate.association.source
- evidence_trail[-1]["candidate_source_rank"] = entrypoint_candidate.source_rank
- evidence_trail[-1]["https"] = final_origin is not None
- evidence_trail[-1]["tls_valid"] = final_origin is not None
- evidence_trail[-1]["origin_bound"] = origin_bound
- if not enforcement_bound:
- return build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="candidate_final_url_unenforced",
- )
- if not origin_bound:
- return build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="candidate_origin_mismatch",
- )
- if not identity_bound:
- return build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="candidate_identity_mismatch",
- )
- return build_result(
+ return _discovery_build_result(
candidate_url=current_url,
candidate_form_fields=form_fields,
evidence_trail=evidence_trail,
@@ -746,7 +508,7 @@ async def _discovery_walk(
if intent_tokens and title_score >= 2 and (form_fields or best_score <= title_score):
confidence = min(1.0, title_score / max(1, len(intent_tokens)))
- return build_result(
+ return _discovery_build_result(
candidate_url=current_url,
candidate_form_fields=form_fields,
evidence_trail=evidence_trail,
@@ -756,7 +518,7 @@ async def _discovery_walk(
if form_fields and (title_score >= 1 or step > 0):
confidence = 0.6 if title_score >= 1 else 0.4
- return build_result(
+ return _discovery_build_result(
candidate_url=current_url,
candidate_form_fields=form_fields,
evidence_trail=evidence_trail,
@@ -765,7 +527,7 @@ async def _discovery_walk(
)
if not intent_tokens:
- return build_result(
+ return _discovery_build_result(
candidate_url=current_url,
candidate_form_fields=form_fields,
evidence_trail=evidence_trail,
@@ -774,7 +536,7 @@ async def _discovery_walk(
)
if best_score == 0 or best_href is None:
- return build_result(
+ return _discovery_build_result(
candidate_url=None,
candidate_form_fields=[],
evidence_trail=evidence_trail,
@@ -801,7 +563,7 @@ async def _discovery_walk(
current_url = best_href
- return build_result(
+ return _discovery_build_result(
candidate_url=None,
candidate_form_fields=[],
evidence_trail=evidence_trail,
@@ -820,48 +582,21 @@ async def _discover_workflow_entrypoint_impl(
machinery.
"""
arguments = {"site_or_url": site_or_url, "intent_hint": intent_hint}
- input_kind: str | None = None
def finish(result: dict[str, Any], *, site_or_url_kind: str | None = None) -> dict[str, Any]:
_record_discovery_resolution_on_ctx(copilot_ctx, result)
record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
data_payload = result.get("data")
data = data_payload if isinstance(data_payload, Mapping) else {}
- if isinstance(data_payload, dict) and input_kind == "bare_word":
- data_payload.setdefault("contract_version", _DISCOVERY_RESULT_CONTRACT_VERSION)
candidate_url = data.get("candidate_url")
- candidate_provenance = data.get("candidate_provenance")
- provenance = candidate_provenance if isinstance(candidate_provenance, Mapping) else {}
- navigation_payload = data.get("navigation_evidence")
- navigation_evidence = navigation_payload if isinstance(navigation_payload, Mapping) else {}
failure_reason = data.get("failure_reason")
if not isinstance(failure_reason, str) or not failure_reason:
error = result.get("error")
failure_reason = error if isinstance(error, str) and error else None
LOG.info(
"discover_workflow_entrypoint completed",
- contract_version=data.get("contract_version"),
ok=result.get("ok"),
candidate_url=_redact_discovery_url_for_log(candidate_url if isinstance(candidate_url, str) else None),
- candidate_source=provenance.get("source"),
- candidate_source_rank=provenance.get("source_rank"),
- candidate_entity_id=provenance.get("entity_id"),
- candidate_associated_origin=provenance.get("associated_origin"),
- candidate_provider_relation_type=provenance.get("provider_relation_type"),
- candidate_provider_relation_text=provenance.get("provider_relation_text"),
- candidate_final_url=_redact_discovery_url_for_log(
- navigation_evidence.get("final_url") if isinstance(navigation_evidence.get("final_url"), str) else None
- ),
- candidate_https=navigation_evidence.get("https"),
- candidate_tls_valid=navigation_evidence.get("tls_valid"),
- candidate_resolved_public_ips=navigation_evidence.get("resolved_public_ips"),
- candidate_guarded_hops=_redact_guarded_hops_for_log(
- navigation_evidence.get("guarded_hops")
- if isinstance(navigation_evidence.get("guarded_hops"), list)
- else None
- ),
- candidate_origin_bound=navigation_evidence.get("origin_bound"),
- candidate_enforcement_bound=navigation_evidence.get("enforcement_bound"),
failure_reason=failure_reason,
site_or_url_kind=site_or_url_kind,
)
@@ -873,30 +608,6 @@ async def _discover_workflow_entrypoint_impl(
return finish(result)
entry_url, kind = _resolve_discovery_entry_url(site_or_url)
- input_kind = kind
- entrypoint_candidate: CopilotEntrypointCandidate | None = None
- if kind == "bare_word":
- try:
- candidates = await app.AGENT_FUNCTION.acquire_copilot_entrypoint_candidates(
- site_name=site_or_url.strip(),
- )
- except Exception:
- LOG.exception("Copilot entrypoint candidate acquisition raised")
- candidates = []
- ranked_candidates = _rank_discovery_entrypoint_candidates(site_or_url, candidates)
- if ranked_candidates:
- entrypoint_candidate = ranked_candidates[0]
- entry_url = entrypoint_candidate.url
- kind = "evidence_candidate"
- else:
- entry_url = _user_provided_entry_url(copilot_ctx)
- kind = "user_provided_site" if entry_url else "unresolved"
- if entry_url is None:
- # A multi-word site name is not a bare word, so this is the only point a stored login page
- # is reachable for one.
- entry_url = _credential_entry_url(copilot_ctx, site_or_url)
- if entry_url:
- kind = "credential_tested_url"
if entry_url is None:
result = _discovery_build_result(
candidate_url=None,
@@ -935,23 +646,7 @@ async def _discover_workflow_entrypoint_impl(
},
):
try:
- async with AsyncExitStack() as guard_stack:
- guarded_hops: list[CopilotCandidateNetworkHop] = []
- if entrypoint_candidate is not None:
- server = copilot_ctx.discovery_mcp_server
- if server is None:
- raise RuntimeError("discovery MCP server not attached to context")
- guarded_hops = await guard_stack.enter_async_context(
- server.evidence_candidate_navigation_guard(entrypoint_candidate.association.origin)
- )
- result = await _discovery_walk(
- copilot_ctx,
- entry_url=entry_url,
- intent_hint=intent_hint or "",
- entrypoint_candidate=entrypoint_candidate,
- requested_name=site_or_url.strip() if entrypoint_candidate is not None else None,
- guarded_hops=guarded_hops,
- )
+ result = await _discovery_walk(copilot_ctx, entry_url=entry_url, intent_hint=intent_hint or "")
except Exception as exc:
LOG.exception("discover_workflow_entrypoint walker raised")
result = {
@@ -972,7 +667,6 @@ async def _discover_workflow_entrypoint_impl(
copilot_ctx.discovery_evidence_trail = list(evidence_trail)
if (
result.get("ok")
- and entrypoint_candidate is None
and not data.get("candidate_url")
and data.get("failure_reason") in _DISCOVERY_SCORER_MISS_REASONS
):
@@ -985,12 +679,6 @@ async def _discover_workflow_entrypoint_impl(
evidence_trail=evidence_trail,
confidence=_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
failure_reason=None,
- candidate_provenance=(
- data.get("candidate_provenance") if isinstance(data.get("candidate_provenance"), dict) else None
- ),
- navigation_evidence=(
- data.get("navigation_evidence") if isinstance(data.get("navigation_evidence"), dict) else None
- ),
)
rebuilt_data = result.get("data")
data = rebuilt_data if isinstance(rebuilt_data, dict) else {}
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
index b819dad2c..595c61f53 100644
--- a/skyvern/forge/sdk/copilot/tools/guardrails.py
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -7,6 +7,7 @@ from typing import Any
import structlog
from agents import ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData
+from skyvern.forge.sdk.copilot.ask_user import ACCOUNT_GROUP_SUBMIT_TOOL_NAME
from skyvern.forge.sdk.copilot.author_time_block import CREDENTIAL_SCOUT_BLOCK_ID, AuthorTimeBlock
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_test_outcome import (
@@ -171,6 +172,7 @@ def _authority_tool_error(
"upload_attached_file",
"inspect_page_for_composition",
LOCATOR_INSPECTION_TOOL_NAME,
+ ACCOUNT_GROUP_SUBMIT_TOOL_NAME,
} and raw_secret_browser_denied(ctx):
return _emit_tool_blocker_signal(
ctx,
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 593b17742..5c83e847b 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -855,6 +855,7 @@ class WorkflowRunGroupItemModel(Base):
__table_args__ = (
UniqueConstraint("workflow_run_group_id", "item_key", name="uq_workflow_run_group_items_group_item_key"),
UniqueConstraint("workflow_run_id", name="uq_workflow_run_group_items_workflow_run_id"),
+ Index("ix_workflow_run_group_items_item_key_created_at", "item_key", "created_at"),
)
workflow_run_group_id = Column(String, primary_key=True)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index feee1b65b..35cfe12db 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -17,6 +17,7 @@ from sqlalchemy.orm import defer
from skyvern.config import settings
from skyvern.forge.sdk.copilot.ask_user import (
QUESTION_CLIENT_GRACE,
+ AccountGroupDecision,
QuestionInteraction,
QuestionResponse,
question_wait_is_live,
@@ -1294,6 +1295,7 @@ class WorkflowParametersRepository(BaseRepository):
interaction_id: str,
response: QuestionResponse,
*,
+ account_group_decision: AccountGroupDecision | None = None,
preflight_only: bool = False,
) -> QuestionInteraction:
"""Validate before external screening, then recheck and commit the screened reply."""
@@ -1327,7 +1329,7 @@ class WorkflowParametersRepository(BaseRepository):
self._store_question_turn(chat, entry)
await session.commit()
raise ValueError("The question's execution was interrupted")
- resolved = resolve_question_response(item, response)
+ resolved = resolve_question_response(item, response, account_group_decision)
if preflight_only:
entry.question_client_seen_at = datetime.now(timezone.utc)
self._store_question_turn(chat, entry)
@@ -1340,6 +1342,19 @@ class WorkflowParametersRepository(BaseRepository):
await session.commit()
return resolved
+ @db_operation("record_copilot_account_group_submission")
+ async def record_copilot_account_group_submission(
+ self, organization_id: str, chat_id: str, interaction_id: str, workflow_run_group_id: str
+ ) -> None:
+ async with self.Session() as session:
+ chat = await self._locked_question_chat(session, organization_id, chat_id)
+ entry, item = self._question_in_pending(chat, interaction_id)
+ if item.account_group_review is None:
+ raise ValueError("This question has no account review")
+ item.account_group_review.workflow_run_group_id = workflow_run_group_id
+ self._store_question_turn(chat, entry)
+ await session.commit()
+
@db_operation("interrupt_copilot_question")
async def interrupt_copilot_question(
self, organization_id: str, chat_id: str, interaction_id: str, *, stale_only: bool = False
diff --git a/skyvern/forge/sdk/db/repositories/workflow_run_groups.py b/skyvern/forge/sdk/db/repositories/workflow_run_groups.py
index 135612ec3..77ffb4898 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_run_groups.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_run_groups.py
@@ -1,18 +1,18 @@
from __future__ import annotations
import hmac
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
from datetime import datetime
from hashlib import sha256
from typing import Any, Literal
-from sqlalchemy import String, cast, func, select, update
+from sqlalchemy import String, cast, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from skyvern.config import settings
from skyvern.constants import SCRUBBED_VALUE
-from skyvern.exceptions import WorkflowChangedSinceReview
+from skyvern.exceptions import GroupAccountsRanSinceReview, WorkflowChangedSinceReview
from skyvern.forge.sdk.db._error_handling import db_operation
from skyvern.forge.sdk.db.base_repository import BaseRepository
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.db.models import (
WorkflowRunGroupItemModel,
WorkflowRunGroupModel,
)
+from skyvern.forge.sdk.workflow.models.workflow import COPILOT_TEST_WORKFLOW_CREATOR
from skyvern.schemas.workflow_run_groups import (
IN_FLIGHT_ITEM_STATES,
SCRUBBED_SUBMISSION_KEY_PREFIX,
@@ -43,6 +44,50 @@ def scrubbed_submission_key(submission_key: str) -> str:
return f"{SCRUBBED_SUBMISSION_KEY_PREFIX}{digest}"
+async def _latest_group_ids_by_item_key(
+ session: AsyncSession,
+ organization_id: str,
+ workflow_permanent_id: str,
+ submission_key_prefix: str,
+ item_keys: Sequence[str],
+) -> dict[str, str]:
+ ranked = (
+ select(
+ WorkflowRunGroupItemModel.item_key,
+ WorkflowRunGroupItemModel.workflow_run_group_id,
+ func.row_number()
+ .over(
+ partition_by=WorkflowRunGroupItemModel.item_key,
+ order_by=WorkflowRunGroupItemModel.created_at.desc(),
+ )
+ .label("recency"),
+ )
+ .join(
+ WorkflowRunGroupModel,
+ WorkflowRunGroupModel.workflow_run_group_id == WorkflowRunGroupItemModel.workflow_run_group_id,
+ )
+ .where(
+ WorkflowRunGroupModel.organization_id == organization_id,
+ WorkflowRunGroupModel.workflow_permanent_id == workflow_permanent_id,
+ WorkflowRunGroupModel.submission_key.startswith(submission_key_prefix, autoescape=True),
+ WorkflowRunGroupItemModel.item_key.in_(item_keys),
+ )
+ .subquery()
+ )
+ rows = await session.execute(select(ranked.c.item_key, ranked.c.workflow_run_group_id).where(ranked.c.recency == 1))
+ return {item_key: group_id for item_key, group_id in rows}
+
+
+async def _lock_repeat_check(session: AsyncSession, organization_id: str, workflow_permanent_id: str) -> None:
+ # The repeat check spans every version, so approvals serialize per workflow. An advisory lock rather than
+ # every version's row, which bulk writers lock in their own order and could deadlock against.
+ bind = session.get_bind()
+ if bind is not None and bind.dialect.name not in {"postgresql", "postgres"}:
+ return
+ lock_key = f"workflow_run_group_repeat:{organization_id}:{workflow_permanent_id}"
+ await session.execute(select(func.pg_advisory_xact_lock(func.hashtext(lock_key))))
+
+
class WorkflowRunGroupsRepository(BaseRepository):
# Item parameters are cleared on every transition out of pending/dispatching; the child run holds its own copy.
# Every state transition first touches the group row, so transitions of one group serialize on its row lock.
@@ -55,7 +100,10 @@ class WorkflowRunGroupsRepository(BaseRepository):
)
return WorkflowRunGroupStatus(status) if status is not None else None
- @db_operation("create_workflow_run_group", expected_errors=(IntegrityError, WorkflowChangedSinceReview))
+ @db_operation(
+ "create_workflow_run_group",
+ expected_errors=(IntegrityError, WorkflowChangedSinceReview, GroupAccountsRanSinceReview),
+ )
async def create_group(
self,
*,
@@ -67,14 +115,41 @@ class WorkflowRunGroupsRepository(BaseRepository):
input_fingerprint: str,
items: Sequence[tuple[str, dict[str, Any]]],
expected_workflow_modified_at: datetime | None = None,
+ expected_latest_groups: tuple[str, Mapping[str, str]] | None = None,
) -> WorkflowRunGroup:
async with self.Session() as session:
+ if expected_latest_groups is not None:
+ await _lock_repeat_check(session, organization_id, workflow_permanent_id)
# Locks the row in-place edits lock, so an edit either lands before this check or sees this group.
modified_at = await session.scalar(
select(WorkflowModel.modified_at).where(WorkflowModel.workflow_id == workflow_id).with_for_update()
)
if expected_workflow_modified_at is not None and modified_at != expected_workflow_modified_at:
raise WorkflowChangedSinceReview(workflow_id)
+ if expected_latest_groups is not None:
+ # A new version is a new row, so the reviewed row's lock cannot see it; recheck it is still latest.
+ latest_version_id = await session.scalar(
+ select(WorkflowModel.workflow_id)
+ .where(WorkflowModel.workflow_permanent_id == workflow_permanent_id)
+ .where(WorkflowModel.organization_id == organization_id)
+ .where(WorkflowModel.deleted_at.is_(None))
+ .where(
+ or_(
+ WorkflowModel.created_by.is_(None),
+ WorkflowModel.created_by != COPILOT_TEST_WORKFLOW_CREATOR,
+ )
+ )
+ .order_by(WorkflowModel.version.desc())
+ .limit(1)
+ )
+ if latest_version_id != workflow_id:
+ raise WorkflowChangedSinceReview(workflow_id)
+ key_prefix, expected = expected_latest_groups
+ latest = await _latest_group_ids_by_item_key(
+ session, organization_id, workflow_permanent_id, key_prefix, [key for key, _ in items]
+ )
+ if changed := sorted(key for key, group_id in latest.items() if expected.get(key) != group_id):
+ raise GroupAccountsRanSinceReview(changed)
group = WorkflowRunGroupModel(
organization_id=organization_id,
workflow_permanent_id=workflow_permanent_id,
@@ -128,6 +203,27 @@ class WorkflowRunGroupsRepository(BaseRepository):
).first()
return WorkflowRunGroup.model_validate(group) if group else None
+ @db_operation("get_latest_group_ids_by_item_key")
+ async def get_latest_group_ids_by_item_key(
+ self, organization_id: str, workflow_permanent_id: str, submission_key_prefix: str, item_keys: Sequence[str]
+ ) -> dict[str, str]:
+ async with self.Session() as session:
+ return await _latest_group_ids_by_item_key(
+ session, organization_id, workflow_permanent_id, submission_key_prefix, item_keys
+ )
+
+ @db_operation("get_workflow_run_groups_by_submission_keys")
+ async def get_groups_by_submission_keys(
+ self, organization_id: str, submission_keys: Sequence[str]
+ ) -> dict[str, WorkflowRunGroup]:
+ async with self.Session() as session:
+ groups = await session.scalars(
+ select(WorkflowRunGroupModel)
+ .where(WorkflowRunGroupModel.organization_id == organization_id)
+ .where(WorkflowRunGroupModel.submission_key.in_(submission_keys))
+ )
+ return {group.submission_key: WorkflowRunGroup.model_validate(group) for group in groups}
+
@db_operation("scrub_workflow_run_group_keys")
async def scrub_keys(self, organization_id: str, workflow_run_group_ids: Sequence[str]) -> int:
"""Replace the caller's submission and item keys; returns the groups scrubbed.
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 76012af94..4f3552a72 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import json
-from collections.abc import AsyncIterator, Callable, Sequence
+from collections.abc import AsyncIterator, Callable, Collection, Sequence
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import datetime, timedelta
@@ -70,6 +70,8 @@ from skyvern.forge.sdk.db.models import (
WorkflowRunAttemptModel,
WorkflowRunBlockModel,
WorkflowRunCredentialSelectionModel,
+ WorkflowRunGroupItemModel,
+ WorkflowRunGroupModel,
WorkflowRunModel,
WorkflowRunOutputParameterModel,
WorkflowRunParameterModel,
@@ -109,6 +111,7 @@ from skyvern.forge.sdk.workflow.status_mapping import (
)
from skyvern.schemas.run_enums import WebhookDeliveryStatus, resolve_webhook_delivery_projection
from skyvern.schemas.runs import MAX_SEARCH_FETCH_LIMIT, TERMINAL_STATUSES, ProxyLocationInput, RunType
+from skyvern.schemas.workflows import BlockStatus
LOG = structlog.get_logger()
@@ -1803,6 +1806,19 @@ class WorkflowRunsRepository(BaseRepository):
query = query.filter_by(organization_id=organization_id)
return await session.scalar(query)
+ @db_operation("get_browser_runtime", log_errors=False)
+ async def get_browser_runtime(self, workflow_run_id: str, organization_id: str) -> str | None:
+ """Where the run's browser ran (``local``, ``pbs`` or ``vendor``), once it acquired one.
+
+ Selects the column for the same reason as ``get_secure_runner_pin``: ``WorkflowRun`` is a published schema.
+ """
+ async with self.Session() as session:
+ return await session.scalar(
+ select(WorkflowRunModel.browser_runtime).filter_by(
+ workflow_run_id=workflow_run_id, organization_id=organization_id
+ )
+ )
+
@db_operation("get_workflow_run_status", log_errors=False)
async def get_workflow_run_status(
self,
@@ -2342,6 +2358,55 @@ class WorkflowRunsRepository(BaseRepository):
workflow_runs = (await session.scalars(query)).all()
return [convert_to_workflow_run(workflow_run) for workflow_run in workflow_runs]
+ @db_operation("get_latest_group_child_run_id")
+ async def get_latest_group_child_run_id(
+ self,
+ *,
+ organization_id: str,
+ workflow_permanent_id: str,
+ workflow_id: str,
+ workflow_parameter_id: str,
+ values: Sequence[str],
+ submission_key_suffix: str,
+ completed_without_block_statuses: Collection[BlockStatus] | None,
+ ) -> str | None:
+ async with self.Session() as session:
+ query = (
+ select(WorkflowRunGroupItemModel.workflow_run_id)
+ .select_from(WorkflowRunGroupModel)
+ .join(
+ WorkflowRunGroupItemModel,
+ WorkflowRunGroupItemModel.workflow_run_group_id == WorkflowRunGroupModel.workflow_run_group_id,
+ )
+ .join(
+ WorkflowRunParameterModel,
+ WorkflowRunParameterModel.workflow_run_id == WorkflowRunGroupItemModel.workflow_run_id,
+ )
+ .filter(
+ WorkflowRunGroupModel.organization_id == organization_id,
+ WorkflowRunGroupModel.workflow_permanent_id == workflow_permanent_id,
+ WorkflowRunGroupModel.workflow_id == workflow_id,
+ WorkflowRunGroupModel.submission_key.endswith(submission_key_suffix, autoescape=True),
+ WorkflowRunParameterModel.workflow_parameter_id == workflow_parameter_id,
+ WorkflowRunParameterModel.value.in_(values),
+ )
+ .order_by(WorkflowRunGroupItemModel.created_at.desc())
+ .limit(1)
+ )
+ if completed_without_block_statuses is not None:
+ query = query.join(
+ WorkflowRunModel, WorkflowRunModel.workflow_run_id == WorkflowRunGroupItemModel.workflow_run_id
+ ).filter(
+ WorkflowRunModel.organization_id == organization_id,
+ WorkflowRunModel.status == WorkflowRunStatus.completed.value,
+ ~exists().where(
+ WorkflowRunBlockModel.organization_id == organization_id,
+ WorkflowRunBlockModel.workflow_run_id == WorkflowRunGroupItemModel.workflow_run_id,
+ WorkflowRunBlockModel.status.in_([status.value for status in completed_without_block_statuses]),
+ ),
+ )
+ return await session.scalar(query)
+
@db_operation("get_last_running_workflow_run")
async def get_last_running_workflow_run(
self,
diff --git a/skyvern/forge/sdk/routes/streaming/channels/vnc.py b/skyvern/forge/sdk/routes/streaming/channels/vnc.py
index d0edb3115..fa70888b9 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/vnc.py
+++ b/skyvern/forge/sdk/routes/streaming/channels/vnc.py
@@ -33,6 +33,7 @@ from skyvern.config import settings
from skyvern.exceptions import MissingRoutedVncAddressError
from skyvern.forge.sdk.routes.streaming.auth import get_x_api_key
from skyvern.forge.sdk.routes.streaming.channels.execution import execution_channel
+from skyvern.forge.sdk.routes.streaming.run_stream_outcome import RunStreamConnection
from skyvern.forge.sdk.routes.streaming.verify import (
loop_verify_browser_session,
loop_verify_task,
@@ -175,9 +176,19 @@ class VncChannel:
remote_clipboard_synced_at: float | None = None
task: Task | None = None
workflow_run: WorkflowRun | None = None
+ live_stream: RunStreamConnection = dataclasses.field(init=False, repr=False)
def __post_init__(self, initial_interactor: Interactor) -> None:
self.interactor = initial_interactor
+ # The viewer redials a dropped VNC socket, and a run view falls back to the session's screencast.
+ self.live_stream = RunStreamConnection(
+ organization_id=self.organization_id,
+ viewer_reconnects=True,
+ transport="vnc",
+ browser_session_id=self.browser_session.persistent_browser_session_id if self.browser_session else None,
+ )
+ if self.browser_session:
+ self.live_stream.observe_session(self.browser_session)
add_vnc_channel(self)
@property
@@ -420,17 +431,21 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
continue
except WebSocketDisconnect:
+ vnc_channel.live_stream.end_first("viewer_left")
LOG.debug(f"{class_name} Frontend disconnected.", **vnc_channel.identity)
raise
except ConnectionClosedError:
+ vnc_channel.live_stream.end_first("viewer_left")
LOG.debug(f"{class_name} Frontend closed the vnc channel.", **vnc_channel.identity)
raise
except ConnectionClosedOK:
+ vnc_channel.live_stream.end_first("viewer_left")
LOG.debug(f"{class_name} Frontend closed the vnc channel cleanly.", **vnc_channel.identity)
raise
except asyncio.CancelledError:
pass
except Exception:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.exception(f"{class_name} An unexpected exception occurred.", **vnc_channel.identity)
raise
@@ -440,17 +455,21 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
try:
await novnc_ws.send(data)
except WebSocketDisconnect:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.debug(f"{class_name} Browser disconnected from vnc.", **vnc_channel.identity)
raise
except ConnectionClosedError:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.debug(f"{class_name} Browser closed vnc.", **vnc_channel.identity)
raise
except ConnectionClosedOK:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.debug(f"{class_name} Browser closed vnc cleanly.", **vnc_channel.identity)
raise
except asyncio.CancelledError:
pass
except Exception:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.exception(
f"{class_name} An unexpected exception occurred in frontend-to-browser loop.",
**vnc_channel.identity,
@@ -468,23 +487,28 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
data = await novnc_ws.recv()
except WebSocketDisconnect:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.debug(
f"{class_name} Browser disconnected from the vnc channel session.", **vnc_channel.identity
)
await vnc_channel.close(reason="browser-disconnected")
except ConnectionClosedError:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.debug(f"{class_name} Browser closed the vnc channel session.", **vnc_channel.identity)
await vnc_channel.close(reason="browser-closed")
except ConnectionClosedOK:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.debug(f"{class_name} Browser closed the vnc channel session cleanly.", **vnc_channel.identity)
await vnc_channel.close(reason="browser-closed-ok")
except asyncio.CancelledError:
pass
except Exception:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.exception(
f"{class_name} An unexpected exception occurred in browser-to-frontend loop.",
**vnc_channel.identity,
)
+ await vnc_channel.close(reason="browser-error")
raise
if not data:
@@ -494,20 +518,26 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
if vnc_channel.websocket.client_state != WebSocketState.CONNECTED:
continue
await vnc_channel.websocket.send_bytes(data)
+ # The first relayed message is the display server's handshake: the browser is on the wire.
+ vnc_channel.live_stream.frame_sent()
except WebSocketDisconnect:
+ vnc_channel.live_stream.end_first("viewer_left")
LOG.debug(
f"{class_name} Frontend disconnected from the vnc channel session.", **vnc_channel.identity
)
await vnc_channel.close(reason="frontend-disconnected")
except ConnectionClosedError:
+ vnc_channel.live_stream.end_first("viewer_left")
LOG.debug(f"{class_name} Frontend closed the vnc channel session.", **vnc_channel.identity)
await vnc_channel.close(reason="frontend-closed")
except ConnectionClosedOK:
+ vnc_channel.live_stream.end_first("viewer_left")
LOG.debug(f"{class_name} Frontend closed the vnc channel session cleanly.", **vnc_channel.identity)
await vnc_channel.close(reason="frontend-closed-ok")
except asyncio.CancelledError:
pass
except Exception:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.exception(f"{class_name} An unexpected exception occurred.", **vnc_channel.identity)
raise
@@ -523,6 +553,7 @@ async def loop_stream_vnc(vnc_channel: VncChannel) -> None:
except ConnectionClosedOK:
LOG.debug(f"{class_name} Connection closed cleanly in loop stream.", **vnc_channel.identity)
except Exception:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.exception(f"{class_name} An exception occurred in loop stream.", **vnc_channel.identity)
finally:
LOG.debug(f"{class_name} Closing the loop stream.", **vnc_channel.identity)
@@ -566,7 +597,9 @@ async def get_vnc_channel_for_browser_session(
LOG.debug("Got vnc context for browser session.", vnc_channel=vnc_channel)
loops = [
- asyncio.create_task(loop_verify_browser_session(vnc_channel)),
+ asyncio.create_task(
+ loop_verify_browser_session(vnc_channel, on_session=vnc_channel.live_stream.observe_session)
+ ),
asyncio.create_task(loop_stream_vnc(vnc_channel)),
]
diff --git a/skyvern/forge/sdk/routes/streaming/run_stream_outcome.py b/skyvern/forge/sdk/routes/streaming/run_stream_outcome.py
index 154d0f173..d764f94e0 100644
--- a/skyvern/forge/sdk/routes/streaming/run_stream_outcome.py
+++ b/skyvern/forge/sdk/routes/streaming/run_stream_outcome.py
@@ -3,10 +3,19 @@
from __future__ import annotations
import time
-from typing import Literal
+from typing import Literal, cast
import structlog
+from skyvern.forge import app
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
+ FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE,
+ PersistentBrowserSession,
+ is_final_status,
+)
+from skyvern.webeye.browser_acquisition_sample import BROWSER_RUNTIMES, BrowserRuntime
+from skyvern.webeye.persistent_sessions_manager import PBS_TASK_RUNNABLE_TYPE
+
LOG = structlog.get_logger()
RunStreamOutcome = Literal[
@@ -17,12 +26,15 @@ RunStreamOutcome = Literal[
"run_unreadable",
"stream_error",
"cancelled",
+ "session_ended",
]
RunStreamFailure = Literal["none", "recoverable", "unrecovered"]
+# run_frames is the frames a run's worker publishes; vnc and cdp_screencast stream a persistent session's browser.
+LiveStreamTransport = Literal["run_frames", "vnc", "cdp_screencast"]
# Whether the viewer gives up on this failure is the frontend's reconnect contract: a `timeout` status
# or a non-JSON message ends its stream, and a close without a status is retried only by the run page;
-# the task page never reconnects.
+# the task page never reconnects. A session that has ended leaves nothing to reconnect to.
_FAILURE_BY_OUTCOME: dict[RunStreamOutcome, RunStreamFailure] = {
"run_finished": "none",
"viewer_left": "none",
@@ -31,8 +43,20 @@ _FAILURE_BY_OUTCOME: dict[RunStreamOutcome, RunStreamFailure] = {
"run_unreadable": "unrecovered",
"stream_error": "recoverable",
"cancelled": "recoverable",
+ "session_ended": "unrecovered",
}
+# A session held under one of these is running that run's browser, so its viewer is watching that run.
+_RUN_RUNNABLE_TYPES = frozenset({"workflow_run", FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE, PBS_TASK_RUNNABLE_TYPE})
+
+
+def _run_holding(session: PersistentBrowserSession) -> tuple[str | None, str | None] | None:
+ if session.runnable_type not in _RUN_RUNNABLE_TYPES or not session.runnable_id:
+ return None
+ if session.runnable_type == PBS_TASK_RUNNABLE_TYPE:
+ return None, session.runnable_id
+ return session.runnable_id, None
+
class RunStreamConnection:
def __init__(
@@ -40,20 +64,24 @@ class RunStreamConnection:
*,
organization_id: str,
viewer_reconnects: bool,
+ transport: LiveStreamTransport = "run_frames",
workflow_run_id: str | None = None,
task_id: str | None = None,
+ browser_session_id: str | None = None,
) -> None:
self._organization_id = organization_id
self._viewer_reconnects = viewer_reconnects
+ self._transport = transport
self._workflow_run_id = workflow_run_id
self._task_id = task_id
+ self._browser_session_id = browser_session_id
self._opened_at = time.monotonic()
self._run_status: str | None = None
self._run_final = False
self._expected: bool | None = None
- self._has_browser_session = False
self._frames_sent = 0
self._outcome: RunStreamOutcome = "cancelled"
+ self._ended = False
def observe_run(
self,
@@ -67,40 +95,166 @@ class RunStreamConnection:
self._expected = not is_final
self._run_status = str(status)
self._run_final = is_final
- self._has_browser_session = browser_session_id is not None
+ self._browser_session_id = browser_session_id
if workflow_run_id is not None:
self._workflow_run_id = workflow_run_id
+ def observe_session(self, session: PersistentBrowserSession | None) -> None:
+ """A session-keyed stream is attributed to whichever run holds the session while it is watched.
+
+ The holder changes only when a run releases the session, which is that run's normal end, so the
+ released run's line is emitted then and the connection moves on to the next holder, if any.
+ """
+ if session is None or is_final_status(session.status):
+ self.end_first("session_ended")
+ if session is None:
+ return
+ holder = _run_holding(session)
+ attributed = self._attributed_run()
+ if holder == attributed:
+ return
+ if attributed is not None:
+ self._log(
+ outcome="run_finished",
+ lineage="pbs",
+ run_phase="finished",
+ run_status=None,
+ )
+ # Only the timing restarts: whether the stream ever worked belongs to the connection.
+ self._opened_at = time.monotonic()
+ self._workflow_run_id, self._task_id = holder or (None, None)
+ self._expected = True if holder else None
+
def frame_sent(self) -> None:
self._frames_sent += 1
def end(self, outcome: RunStreamOutcome) -> None:
self._outcome = outcome
+ self._ended = True
+
+ def end_first(self, outcome: RunStreamOutcome) -> None:
+ """Keep the first ending: in a relay, the side that closes first is the cause and the other side's
+ close is its echo."""
+ if not self._ended:
+ self.end(outcome)
- def _failure(self) -> RunStreamFailure:
- failure = _FAILURE_BY_OUTCOME[self._outcome]
+ def _attributed_run(self) -> tuple[str | None, str | None] | None:
+ if self._workflow_run_id is None and self._task_id is None:
+ return None
+ return self._workflow_run_id, self._task_id
+
+ def _failure(self, outcome: RunStreamOutcome) -> RunStreamFailure:
+ failure = _FAILURE_BY_OUTCOME[outcome]
if failure == "recoverable" and not self._viewer_reconnects:
return "unrecovered"
return failure
- def log(self) -> None:
- run_phase: Literal["active", "finished"] | None = None
- if self._run_status is not None:
- run_phase = "finished" if self._run_final else "active"
+ async def finish(self) -> None:
+ """Emit the connection's line. Never raises: it runs in the handler's `finally`."""
+ if self._transport == "run_frames":
+ run_phase: Literal["active", "finished"] | None = None
+ if self._run_status is not None:
+ run_phase = "finished" if self._run_final else "active"
+ self._log(
+ outcome=self._outcome,
+ lineage=await self._run_frames_lineage(),
+ run_phase=run_phase,
+ run_status=self._run_status,
+ )
+ return
+ session_gone = await self._reread_session()
+ # A session watched while no run held it is not a run's live stream.
+ if self._attributed_run() is None:
+ return
+ read = await self._read_run_status()
+ outcome = self._outcome
+ if outcome in ("stream_error", "cancelled", "session_ended") and session_gone is not None:
+ # The relay and the session poll race to see a dying browser; only the session row says it ended.
+ if session_gone:
+ outcome = "session_ended"
+ elif outcome == "session_ended":
+ outcome = "stream_error"
+ if read is not None and read[1] and outcome != "viewer_left":
+ # The run was done with the browser, so its session or stream ending is the normal teardown.
+ outcome = "run_finished"
+ self._log(
+ outcome=outcome,
+ lineage="pbs",
+ run_phase=None if read is None else ("finished" if read[1] else "active"),
+ run_status=None if read is None else read[0],
+ )
+
+ async def _run_frames_lineage(self) -> BrowserRuntime:
+ # Frames reach this route from whatever browser the run's worker holds, so only the run's own
+ # recorded runtime separates a local browser from a vendor's.
+ if self._browser_session_id is not None:
+ return "pbs"
+ runtime: str | None = None
+ if self._workflow_run_id is not None:
+ try:
+ runtime = await app.DATABASE.workflow_runs.get_browser_runtime(
+ workflow_run_id=self._workflow_run_id, organization_id=self._organization_id
+ )
+ except Exception:
+ LOG.debug("Could not read the run's browser runtime", workflow_run_id=self._workflow_run_id)
+ return cast(BrowserRuntime, runtime) if runtime in BROWSER_RUNTIMES else "local"
+
+ async def _read_run_status(self) -> tuple[str, bool] | None:
+ try:
+ if self._workflow_run_id is not None:
+ workflow_run_status = await app.DATABASE.workflow_runs.get_workflow_run_status(
+ workflow_run_id=self._workflow_run_id, organization_id=self._organization_id
+ )
+ if workflow_run_status is None:
+ return None
+ return str(workflow_run_status), workflow_run_status.is_final()
+ if self._task_id is not None:
+ task = await app.DATABASE.tasks.get_task(task_id=self._task_id, organization_id=self._organization_id)
+ return None if task is None else (str(task.status), task.status.is_final())
+ except Exception:
+ LOG.debug("Could not read the watched run's status", run_id=self._workflow_run_id or self._task_id)
+ return None
+
+ async def _reread_session(self) -> bool | None:
+ """Whether the session has ended, or None when it could not be read.
+
+ The last poll can be a holder behind, so the connection first moves to whichever run holds the session now.
+ """
+ if self._browser_session_id is None:
+ return None
+ try:
+ session = await app.PERSISTENT_SESSIONS_MANAGER.get_session(
+ session_id=self._browser_session_id, organization_id=self._organization_id
+ )
+ except Exception:
+ LOG.debug("Could not re-read the watched session", browser_session_id=self._browser_session_id)
+ return None
+ self.observe_session(session)
+ return session is None or is_final_status(session.status)
+
+ def _log(
+ self,
+ *,
+ outcome: RunStreamOutcome,
+ lineage: BrowserRuntime,
+ run_phase: Literal["active", "finished"] | None,
+ run_status: str | None,
+ ) -> None:
LOG.info(
"Run live stream connection ended",
live_stream_event="connection_ended",
run_id=self._workflow_run_id or self._task_id,
workflow_run_id=self._workflow_run_id,
task_id=self._task_id,
+ browser_session_id=self._browser_session_id,
organization_id=self._organization_id,
- live_stream_transport="run_frames",
- live_stream_lineage="session" if self._has_browser_session else "local",
- live_stream_outcome=self._outcome,
- live_stream_failure=self._failure(),
+ live_stream_transport=self._transport,
+ live_stream_lineage=lineage,
+ live_stream_outcome=outcome,
+ live_stream_failure=self._failure(outcome),
live_stream_stage="post_ready" if self._frames_sent else "pre_ready",
live_stream_expected=bool(self._expected),
run_phase=run_phase,
- run_status=self._run_status,
+ run_status=run_status,
connection_seconds=round(time.monotonic() - self._opened_at, 1),
)
diff --git a/skyvern/forge/sdk/routes/streaming/screencast.py b/skyvern/forge/sdk/routes/streaming/screencast.py
index d5a9cad9b..72129b938 100644
--- a/skyvern/forge/sdk/routes/streaming/screencast.py
+++ b/skyvern/forge/sdk/routes/streaming/screencast.py
@@ -193,6 +193,7 @@ async def start_screencast_loop(
check_finalized: Callable[[], Awaitable[bool]],
workflow_run_id: str | None = None,
organization_id: str | None = None,
+ on_frame_sent: Callable[[], None] | None = None,
) -> None:
id_key = f"{entity_type}_id"
metric_attributes = {"entity_type": entity_type}
@@ -379,6 +380,8 @@ async def start_screencast_loop(
)
except Exception:
break
+ if on_frame_sent is not None:
+ on_frame_sent()
sent_at = time.monotonic()
_frame_send_seconds.record(sent_at - dequeued_at, metric_attributes)
_frame_forward_seconds.record(sent_at - received_at, metric_attributes)
diff --git a/skyvern/forge/sdk/routes/streaming/screenshot.py b/skyvern/forge/sdk/routes/streaming/screenshot.py
index 93674c069..20efa9352 100644
--- a/skyvern/forge/sdk/routes/streaming/screenshot.py
+++ b/skyvern/forge/sdk/routes/streaming/screenshot.py
@@ -25,7 +25,7 @@ from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
from skyvern.forge.sdk.routes.streaming.client_disconnect import watch_for_client_disconnect
-from skyvern.forge.sdk.routes.streaming.run_stream_outcome import RunStreamConnection
+from skyvern.forge.sdk.routes.streaming.run_stream_outcome import RunStreamConnection, RunStreamOutcome
from skyvern.forge.sdk.routes.streaming.screencast import (
release_browser_state,
start_screencast_loop,
@@ -184,7 +184,7 @@ async def task_stream(
return
finally:
disconnected.cancel()
- connection.log()
+ await connection.finish()
LOG.info("WebSocket connection closed successfully", task_id=task_id, organization_id=organization_id)
return
@@ -360,7 +360,7 @@ async def workflow_run_streaming(
return
finally:
disconnected.cancel()
- connection.log()
+ await connection.finish()
LOG.info(
"WofklowRun Streaming: WebSocket connection closed successfully",
workflow_run_id=workflow_run_id,
@@ -444,12 +444,20 @@ async def _run_local_screencast(
# silently unreachable for whichever caller forgot to pass it.
organization_id: str,
get_workflow_run_id: Callable[[], str | None] | None = None,
+ # Only a browser session's stream reports an outcome here, so the endings below are in its terms.
+ live_stream: RunStreamConnection | None = None,
) -> None:
id_key = f"{entity_type}_id"
browser_state = None
+
+ def ended(outcome: RunStreamOutcome) -> None:
+ if live_stream is not None:
+ live_stream.end_first(outcome)
+
try:
early_exit_status = await wait_for_running()
if early_exit_status is not None:
+ ended("session_ended")
await _send_status(websocket, id_key, entity_id, early_exit_status)
return
@@ -473,6 +481,7 @@ async def _run_local_screencast(
current_status == "session_expired" or is_final_status(current_status)
):
stream_status = current_status
+ ended("no_frame_timeout" if stream_status == "timeout" else "session_ended")
await _send_status(websocket, id_key, entity_id, stream_status)
return
@@ -484,20 +493,29 @@ async def _run_local_screencast(
check_finalized=check_finalized,
workflow_run_id=workflow_run_id,
organization_id=organization_id,
+ on_frame_sent=live_stream.frame_sent if live_stream is not None else None,
)
final_status = await get_current_status()
+ if final_status is None or final_status == "session_expired" or is_final_status(final_status):
+ ended("session_ended")
if final_status is not None:
try:
await _send_status(websocket, id_key, entity_id, final_status)
except Exception:
+ ended("viewer_left")
LOG.debug("Could not send final status (WebSocket likely closed)", **{id_key: entity_id})
+ # A status that is not final is the viewer's cue to reconnect.
+ ended("stream_error")
except (WebSocketDisconnect, ConnectionClosedOK):
+ ended("viewer_left")
LOG.info("WebSocket closed during local screencast", **{id_key: entity_id})
except ConnectionClosedError:
+ ended("viewer_left")
LOG.warning("WebSocket connection error during local screencast", **{id_key: entity_id})
except Exception:
+ ended("stream_error")
LOG.warning("Error in local screencast", **{id_key: entity_id}, exc_info=True)
finally:
await release_browser_state(browser_state, entity_type, entity_id)
@@ -613,11 +631,20 @@ async def _local_screencast_for_browser_session(
browser_session_id: str,
organization_id: str,
) -> None:
+ # The viewer redials any close that carries no final status (streamLifecycle.shouldReconnectStream).
+ live_stream = RunStreamConnection(
+ organization_id=organization_id,
+ viewer_reconnects=True,
+ transport="cdp_screencast",
+ browser_session_id=browser_session_id,
+ )
+
async def wait_for_running() -> str | None:
session = await app.PERSISTENT_SESSIONS_MANAGER.get_session(
session_id=browser_session_id,
organization_id=organization_id,
)
+ live_stream.observe_session(session)
if not session:
LOG.warning(
"Browser session not found for organization",
@@ -634,6 +661,7 @@ async def _local_screencast_for_browser_session(
session_id=browser_session_id,
organization_id=organization_id,
)
+ live_stream.observe_session(s)
return s is None or is_final_status(s.status)
async def get_current_status() -> str | None:
@@ -641,14 +669,19 @@ async def _local_screencast_for_browser_session(
session_id=browser_session_id,
organization_id=organization_id,
)
+ live_stream.observe_session(s)
return _browser_session_stream_status(s.status) if s else None
- await _run_local_screencast(
- websocket=websocket,
- entity_id=browser_session_id,
- entity_type="browser_session",
- wait_for_running=wait_for_running,
- check_finalized=check_finalized,
- get_current_status=get_current_status,
- organization_id=organization_id,
- )
+ try:
+ await _run_local_screencast(
+ websocket=websocket,
+ entity_id=browser_session_id,
+ entity_type="browser_session",
+ wait_for_running=wait_for_running,
+ check_finalized=check_finalized,
+ get_current_status=get_current_status,
+ organization_id=organization_id,
+ live_stream=live_stream,
+ )
+ finally:
+ await live_stream.finish()
diff --git a/skyvern/forge/sdk/routes/streaming/verify.py b/skyvern/forge/sdk/routes/streaming/verify.py
index 75112686a..9e304a1f4 100644
--- a/skyvern/forge/sdk/routes/streaming/verify.py
+++ b/skyvern/forge/sdk/routes/streaming/verify.py
@@ -275,7 +275,10 @@ async def verify_workflow_run(
return workflow_run, addressable_browser_session
-async def loop_verify_browser_session(verifiable: MessageChannel | VncChannel) -> None:
+async def loop_verify_browser_session(
+ verifiable: MessageChannel | VncChannel,
+ on_session: t.Callable[[AddressablePersistentBrowserSession | None], None] | None = None,
+) -> None:
"""
Loop until the browser session is cleared or the websocket is closed.
"""
@@ -287,6 +290,8 @@ async def loop_verify_browser_session(verifiable: MessageChannel | VncChannel) -
)
verifiable.browser_session = browser_session
+ if on_session is not None:
+ on_session(browser_session)
await asyncio.sleep(Constants.POLL_INTERVAL_FOR_VERIFICATION_SECONDS)
diff --git a/skyvern/forge/sdk/routes/streaming/vnc.py b/skyvern/forge/sdk/routes/streaming/vnc.py
index ab2e81bb6..47fa89afd 100644
--- a/skyvern/forge/sdk/routes/streaming/vnc.py
+++ b/skyvern/forge/sdk/routes/streaming/vnc.py
@@ -172,6 +172,7 @@ async def stream(
organization_id=organization_id,
)
except Exception:
+ vnc_channel.live_stream.end_first("stream_error")
LOG.exception(
"An exception occurred in the vnc loop.",
task_id=task_id,
@@ -186,3 +187,4 @@ async def stream(
organization_id=organization_id,
)
await vnc_channel.close(reason="vnc-closed")
+ await vnc_channel.live_stream.finish()
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 8c396c0c5..158679ca4 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -24,7 +24,7 @@ from skyvern import analytics
from skyvern.config import settings
from skyvern.constants import DEFAULT_WORKFLOW_TITLES
from skyvern.exceptions import HttpException as VaultHttpException
-from skyvern.exceptions import SkyvernHTTPException
+from skyvern.exceptions import SkyvernHTTPException, WorkflowPinnedByRunGroup
from skyvern.forge import app
from skyvern.forge.agent_functions import AuditEvent
from skyvern.forge.sdk.api.files import is_uploaded_file_id
@@ -87,6 +87,7 @@ from skyvern.forge.sdk.copilot.request_policy import _screen_raw_secret_safety
from skyvern.forge.sdk.copilot.review_gate import parse_execution_receipts, serialize_execution_receipts
from skyvern.forge.sdk.copilot.runtime import close_browser_session_quietly
from skyvern.forge.sdk.copilot.steer import STEER_DOORBELL_TTL, copilot_steer_key
+from skyvern.forge.sdk.copilot.tools.account_groups import recover_account_group_links
from skyvern.forge.sdk.copilot.tools.workflow_update import (
_validated_pending_workflow_proposal,
private_workflow_settings_from_proposal,
@@ -223,6 +224,12 @@ ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {
LOG = structlog.get_logger()
_T = TypeVar("_T")
+_AUTO_ACCEPT_DEFERRED_FOR_RUN_GROUP = (
+ "I did not apply this change automatically because a group of account runs is still running the saved "
+ "version. Accept the change when you are ready; it is saved as a new version and the running group is "
+ "not affected."
+)
+
async def _resolve_copilot_agent_handler(
workflow_permanent_id: str,
@@ -1286,6 +1293,9 @@ async def _persist_turn_messages(
if pending.question_interactions:
if narrative_payload is None:
narrative_payload = _make_error_narrative_payload(turn_id, None, assistant_content)
+ await recover_account_group_links(
+ chat.organization_id, chat.workflow_copilot_chat_id, pending.question_interactions
+ )
narrative_payload["questionInteractions"] = [
item.model_dump(mode="json") for item in pending.question_interactions
]
@@ -1776,6 +1786,18 @@ async def _finalise_normal_turn(
marker = _STAGED_COMMIT_LANDED.get()
if marker is not None:
marker[0] = True
+ except WorkflowPinnedByRunGroup:
+ # Nothing landed, so there is nothing to roll back. The draft waits for a manual Accept, which
+ # saves a new version and leaves the running group's version alone.
+ LOG.info(
+ "copilot auto-accept deferred while a workflow run group runs this version",
+ auto_accept_deferred_reason="workflow_run_group_running",
+ )
+ otel_trace.get_current_span().set_attribute(
+ "copilot.auto_accept_deferred_reason", "workflow_run_group_running"
+ )
+ agent_result.proposal_disposition = "review_tested"
+ user_response = f"{user_response}\n\n{_AUTO_ACCEPT_DEFERRED_FOR_RUN_GROUP}"
except Exception:
# Undo any mid-turn degraded write so a failed commit fails the turn
# atomically instead of leaving canonical on a partial intermediate.
@@ -3972,28 +3994,33 @@ async def workflow_copilot_chat_history(
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Copilot history changed while loading"
)
+ question_interactions = list(
+ {
+ item.interaction_id: item
+ for item in (
+ [
+ QuestionInteraction.model_validate(raw)
+ for message in chat_messages
+ if message.narrative_payload is not None
+ for raw in message.narrative_payload.get("questionInteractions", [])
+ ]
+ + [
+ item
+ for entry in (chat.pending_turns.values() if chat else [])
+ for item in entry.question_interactions
+ ]
+ )
+ }.values()
+ )
+ if chat is not None:
+ await recover_account_group_links(
+ organization.organization_id, chat.workflow_copilot_chat_id, question_interactions
+ )
return WorkflowCopilotChatHistoryResponse(
pending_credential_requests=pending_credentials,
workflow_copilot_chat_id=chat.workflow_copilot_chat_id if chat else None,
request_turn_id=request_turn_id,
- question_interactions=list(
- {
- item.interaction_id: item
- for item in (
- [
- QuestionInteraction.model_validate(raw)
- for message in chat_messages
- if message.narrative_payload is not None
- for raw in message.narrative_payload.get("questionInteractions", [])
- ]
- + [
- item
- for entry in (chat.pending_turns.values() if chat else [])
- for item in entry.question_interactions
- ]
- )
- }.values()
- ),
+ question_interactions=question_interactions,
pending_question_cancel_token=next(
(
entry.cancel_token
@@ -4024,7 +4051,9 @@ async def workflow_copilot_question_response(
if chat is None:
raise HTTPException(status_code=404, detail="Unknown Copilot chat")
response = QuestionResponse(
- answers=question_response.answers, text=question_response.text, skipped=question_response.skipped
+ answers=question_response.answers,
+ text=question_response.text,
+ skipped=question_response.skipped,
)
async def resolve(*, preflight_only: bool = False) -> QuestionInteraction:
@@ -4034,6 +4063,7 @@ async def workflow_copilot_question_response(
chat.workflow_copilot_chat_id,
question_response.interaction_id,
response,
+ account_group_decision=question_response.account_group_decision,
preflight_only=preflight_only,
)
except NotFoundError as exc:
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index bbff618f9..aaf96430d 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -17,7 +17,7 @@ from pydantic import (
model_validator,
)
-from skyvern.forge.sdk.copilot.ask_user import QuestionInteraction, QuestionResponse
+from skyvern.forge.sdk.copilot.ask_user import AccountGroupDecision, QuestionInteraction, QuestionResponse
from skyvern.forge.sdk.copilot.code_write_diff import CodeWriteDiff
from skyvern.forge.sdk.copilot.context import ActivityBucket, ProposalDisposition, ResponseType, TurnNarrativePayload
from skyvern.forge.sdk.copilot.run_outcome import RunOutcomeReasonCode, RunOutcomeRole, RunOutcomeVerdict
@@ -461,6 +461,9 @@ class WorkflowCopilotChatRequest(BaseModel):
),
)
supports_question_tool: bool = Field(False, description="The client can display and answer ask_user requests.")
+ supports_account_group_card: bool = Field(
+ False, description="The client can display account group review and receipt cards."
+ )
credential_recovery_token: SecretStr | None = Field(
None,
repr=False,
@@ -551,6 +554,7 @@ class WorkflowCopilotSteerRequest(BaseModel):
class WorkflowCopilotQuestionResponseRequest(QuestionResponse):
workflow_copilot_chat_id: str
interaction_id: str
+ account_group_decision: AccountGroupDecision | None = None
class WorkflowCopilotCredentialResponseRequest(BaseModel):
diff --git a/skyvern/services/workflow_run_group_service.py b/skyvern/services/workflow_run_group_service.py
index 4ecbb271d..4fe60e8a9 100644
--- a/skyvern/services/workflow_run_group_service.py
+++ b/skyvern/services/workflow_run_group_service.py
@@ -5,7 +5,7 @@ import contextvars
import hmac
import json
import uuid
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
from datetime import datetime, timedelta
from hashlib import sha256
from http import HTTPStatus
@@ -115,6 +115,7 @@ async def submit_workflow_run_group(
request: WorkflowRunGroupCreateRequest,
*,
expected_workflow_modified_at: datetime | None = None,
+ expected_latest_groups: tuple[str, Mapping[str, str]] | None = None,
) -> WorkflowRunGroupResponse:
organization_id = organization.organization_id
fingerprint = submission_fingerprint(request)
@@ -170,6 +171,7 @@ async def submit_workflow_run_group(
items=[(item.key, item.parameters) for item in request.items],
# Without a reviewed timestamp, bind to the row validation read so an edit landing after it is refused.
expected_workflow_modified_at=expected_workflow_modified_at or workflow.modified_at,
+ expected_latest_groups=expected_latest_groups,
)
except IntegrityError:
raced = await app.DATABASE.workflow_run_groups.get_group_by_submission_key(
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index 073b633ff..b27d17686 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -10,9 +10,9 @@ import os
import shutil
import sys
import threading
-from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Iterator
+from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from types import SimpleNamespace
@@ -28,20 +28,28 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from playwright.async_api import Download
from playwright.async_api import Error as PlaywrightError
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, func, select
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
import skyvern._cli_bootstrap as cli_bootstrap
+from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api import files
from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.db.models import Base
+from skyvern.forge.sdk.db.agent_db import AgentDB
+from skyvern.forge.sdk.db.models import Base, CredentialModel, WorkflowModel
+from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
from skyvern.forge.sdk.schemas.files import FileInfo
+from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow import web_search, web_search_client
from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager
-from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
+from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, TaskBlock
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameterType
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.services import workflow_run_group_service as group_service
from skyvern.webeye.utils import page as page_module
from skyvern.webeye.utils.page import ScreenshotMode
from tests.unit._fingerprint_expectations import FINGERPRINT_TEST_SECRET_KEY
@@ -948,3 +956,108 @@ def stalled_scrolling_capture(entered: asyncio.Event, timeout_ms: float) -> Asyn
)
return AsyncMock(side_effect=_capture)
+
+
+RUN_GROUP_ORG = "o_test"
+RUN_GROUP_OTHER_ORG = "o_other"
+RUN_GROUP_WPID = "wpid_test"
+
+
+@dataclass
+class FakeExecutor:
+ database: AgentDB
+ executed: list[str] = field(default_factory=list)
+ submitted: list[str] = field(default_factory=list)
+ before_queue: Callable[[str], Awaitable[None]] | None = None
+
+ async def execute_workflow(self, *, workflow_run_id: str, **_: object) -> None:
+ self.executed.append(workflow_run_id)
+ if self.before_queue is not None:
+ await self.before_queue(workflow_run_id)
+ if await self.database.workflow_runs.update_workflow_run_if_not_final(
+ workflow_run_id, WorkflowRunStatus.queued
+ ):
+ self.submitted.append(workflow_run_id)
+
+
+@dataclass
+class RecordingRateLimiter:
+ calls: list[str] = field(default_factory=list)
+
+ async def rate_limit_submit_run(self, organization_id: str) -> None:
+ self.calls.append(organization_id)
+
+
+@dataclass
+class GroupEnv:
+ database: AgentDB
+ executor: FakeExecutor
+ organization: Organization
+ spawned: list[Coroutine[Any, Any, None]]
+ limiter: RecordingRateLimiter
+
+
+def run_group_definition(*blocks: BlockTypeVar) -> dict[str, Any]:
+ return WorkflowDefinition(parameters=[], blocks=list(blocks)).model_dump(mode="json")
+
+
+def run_group_task_block() -> TaskBlock:
+ return TaskBlock(label="login", url="https://example.com", output_parameter=make_block_output_parameter("login"))
+
+
+async def count_rows(env: GroupEnv, model: type[Base]) -> int:
+ async with env.database.Session() as session:
+ return int(await session.scalar(select(func.count()).select_from(model)) or 0)
+
+
+@pytest_asyncio.fixture
+async def run_group_env(monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine) -> AsyncIterator[GroupEnv]:
+ database = AgentDB("sqlite+aiosqlite://", db_engine=sqlite_engine)
+ organization = await database.organizations.create_organization("Test", organization_id=RUN_GROUP_ORG)
+ await database.organizations.create_organization("Other", organization_id=RUN_GROUP_OTHER_ORG)
+ async with database.Session() as session:
+ session.add(
+ WorkflowModel(
+ workflow_id="wf_1",
+ workflow_permanent_id=RUN_GROUP_WPID,
+ organization_id=RUN_GROUP_ORG,
+ title="Workflow",
+ version=1,
+ workflow_definition=run_group_definition(run_group_task_block()),
+ )
+ )
+ session.add_all(
+ CredentialModel(
+ credential_id=credential_id,
+ organization_id=org_id,
+ name="Login",
+ credential_type="password",
+ item_id=f"item_{credential_id}",
+ )
+ for credential_id, org_id in (
+ ("cred_1", RUN_GROUP_ORG),
+ ("cred_2", RUN_GROUP_ORG),
+ ("cred_foreign", RUN_GROUP_OTHER_ORG),
+ )
+ )
+ await session.commit()
+ await database.workflow_params.create_workflow_parameter(
+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID, key="login", default_value=None
+ )
+ service = WorkflowService()
+ executor = FakeExecutor(database)
+ spawned: list[Coroutine[Any, Any, None]] = []
+ limiter = RecordingRateLimiter()
+ monkeypatch.setattr(app, "DATABASE", database)
+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "RATE_LIMITER", limiter, raising=False)
+ monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)
+ monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
+ monkeypatch.setattr(app.AGENT_FUNCTION, "is_block_scoped_workflow_run", AsyncMock(return_value=False))
+ monkeypatch.setattr(AsyncExecutorFactory, "get_executor", lambda: executor)
+ monkeypatch.setattr(group_service, "_spawn", spawned.append)
+ monkeypatch.setattr(service, "_schedule_workflow_run_terminal_hooks", lambda **_: None)
+ yield GroupEnv(database, executor, organization, spawned, limiter)
+ for coroutine in spawned:
+ coroutine.close()
+ await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks, return_exceptions=True)
diff --git a/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py b/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py
index 90a86d750..bd680ffac 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_screenshot_stream_disconnect.py
@@ -2,16 +2,22 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
+from datetime import UTC, datetime
from types import SimpleNamespace
from typing import cast
from unittest.mock import AsyncMock
import pytest
-from fastapi import WebSocket
+from fastapi import WebSocket, WebSocketDisconnect
+from starlette.websockets import WebSocketState
from structlog.testing import capture_logs
+from websockets.exceptions import ConnectionClosedError
from skyvern.forge import app
-from skyvern.forge.sdk.routes.streaming import screenshot
+from skyvern.forge.sdk.routes.streaming import run_stream_outcome, screenshot, verify
+from skyvern.forge.sdk.routes.streaming import vnc as vnc_route
+from skyvern.forge.sdk.routes.streaming.channels import vnc as vnc_channels
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from tests.unit.scoped_asyncio import ScopedAsyncio
@@ -106,8 +112,16 @@ class _WatchingWebSocket:
class _ScriptedRuns:
"""Each read of a run pops its next scripted step: a status, or an exception to raise."""
- def __init__(self, script: dict[str, list[WorkflowRunStatus | Exception]]) -> None:
+ def __init__(
+ self,
+ script: dict[str, list[WorkflowRunStatus | Exception]],
+ browser_runtimes: dict[str, str] | None = None,
+ ) -> None:
self._script = script
+ self._browser_runtimes = browser_runtimes or {}
+
+ async def get_browser_runtime(self, workflow_run_id: str, organization_id: str) -> str | None:
+ return self._browser_runtimes.get(workflow_run_id)
def _next(self, workflow_run_id: str) -> WorkflowRunStatus:
step = self._script[workflow_run_id].pop(0)
@@ -189,7 +203,9 @@ async def test_run_stream_tile_counts_a_run_once_and_only_while_its_failure_is_u
WorkflowRunStatus.running,
WorkflowRunStatus.completed,
],
- }
+ },
+ # The worker publishes a vendor browser's frames on the same route as a local one.
+ browser_runtimes={"wr_blank": "vendor", "wr_recovered": "local"},
)
never_expire = 10_000
with capture_logs() as logs:
@@ -206,7 +222,10 @@ async def test_run_stream_tile_counts_a_run_once_and_only_while_its_failure_is_u
assert _scorecard_tile(logs) == ({"wr_blank"}, {"wr_blank", "wr_recovered"})
timeouts = [log for log in _connection_ended(logs) if log["live_stream_failure"] == "unrecovered"]
assert {(log["live_stream_stage"], log["run_status"], log["live_stream_lineage"]) for log in timeouts} == {
- ("pre_ready", "running", "local")
+ ("pre_ready", "running", "vendor")
+ }
+ assert {log["live_stream_lineage"] for log in _connection_ended(logs) if log["run_id"] == "wr_recovered"} == {
+ "local"
}
@@ -271,3 +290,341 @@ async def test_task_page_stream_error_counts_as_unrecovered_because_the_task_pag
("wr_run", "stream_error", "recoverable"),
}
assert _scorecard_tile(logs)[0] == {"wr_task"}
+
+
+_LOOPBACK_BROWSER = "ws://127.0.0.1:9224/devtools/browser/b-1"
+
+
+class _Sessions:
+ """Persistent sessions by id, held by the named run (or idle), changed by a scenario while watched."""
+
+ def __init__(self, **holders: str | None) -> None:
+ now = datetime.now(UTC)
+ self.rows = {
+ session_id: PersistentBrowserSession(
+ persistent_browser_session_id=session_id,
+ organization_id="o_1",
+ runnable_type="workflow_run" if holder else None,
+ runnable_id=holder,
+ status="running",
+ browser_address=_LOOPBACK_BROWSER,
+ upstream_cdp_url=_LOOPBACK_BROWSER,
+ created_at=now,
+ modified_at=now,
+ )
+ for session_id, holder in holders.items()
+ }
+ self.read_error: Exception | None = None
+ self._read = asyncio.Event()
+
+ def occupy(self, session_id: str, run_id: str) -> None:
+ self.rows[session_id] = self.rows[session_id].model_copy(
+ update={"runnable_type": "workflow_run", "runnable_id": run_id}
+ )
+
+ def release(self, session_id: str) -> None:
+ self.rows[session_id] = self.rows[session_id].model_copy(update={"runnable_type": None, "runnable_id": None})
+
+ def fail(self, session_id: str) -> None:
+ self.rows[session_id] = self.rows[session_id].model_copy(update={"status": "failed"})
+
+ async def next_read(self) -> None:
+ self._read.clear()
+ await self._read.wait()
+
+ async def get_session(self, session_id: str, organization_id: str) -> PersistentBrowserSession | None:
+ self._read.set()
+ if self.read_error is not None:
+ raise self.read_error
+ return self.rows.get(session_id)
+
+ async def release_observer_browser_state(self, session_id: str, browser_state: object) -> None:
+ return None
+
+
+def _run_statuses(**statuses: WorkflowRunStatus) -> SimpleNamespace:
+ async def get_workflow_run_status(workflow_run_id: str, organization_id: str) -> WorkflowRunStatus | None:
+ return statuses.get(workflow_run_id)
+
+ return SimpleNamespace(workflow_runs=SimpleNamespace(get_workflow_run_status=get_workflow_run_status))
+
+
+async def test_session_screencast_counts_a_run_only_while_it_holds_the_session(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ sessions = _Sessions(
+ pbs_idle=None,
+ pbs_released="wr_released",
+ pbs_torn_down="wr_done",
+ pbs_died="wr_died",
+ pbs_blank="wr_blank",
+ pbs_died_unsent="wr_died_unsent",
+ pbs_dead_on_arrival="wr_dead_on_arrival",
+ pbs_viewer_gone="wr_viewer_gone",
+ )
+ sessions.fail("pbs_dead_on_arrival")
+ running, completed = WorkflowRunStatus.running, WorkflowRunStatus.completed
+ monkeypatch.setattr(app, "PERSISTENT_SESSIONS_MANAGER", sessions)
+ monkeypatch.setattr(
+ app,
+ "DATABASE",
+ _run_statuses(
+ wr_released=completed,
+ wr_done=completed,
+ wr_died=running,
+ wr_blank=running,
+ wr_died_unsent=running,
+ wr_dead_on_arrival=running,
+ wr_viewer_gone=running,
+ ),
+ )
+ monkeypatch.setattr(screenshot, "get_current_org", AsyncMock(return_value=SimpleNamespace(organization_id="o_1")))
+
+ async def wait_for_browser_state(entity_id: str, entity_type: str, **_: object) -> object | None:
+ return None if entity_id == "pbs_blank" else object()
+
+ # A run that owns its session closes it at its end without releasing it first.
+ while_watched = {
+ "pbs_released": sessions.release,
+ "pbs_torn_down": sessions.fail,
+ "pbs_died": sessions.fail,
+ "pbs_died_unsent": sessions.fail,
+ }
+ # These viewers are gone by the time the final status is sent to them.
+ viewer_gone = {"pbs_died_unsent", "pbs_dead_on_arrival", "pbs_viewer_gone"}
+
+ async def screencast(*, entity_id: str, check_finalized: Callable[[], Awaitable[bool]], **kwargs: object) -> None:
+ cast(Callable[[], None], kwargs["on_frame_sent"])()
+ if entity_id in while_watched:
+ while_watched[entity_id](entity_id)
+ # A frame the viewer can no longer receive ends the loop as quietly as the session finishing.
+ if await check_finalized() or entity_id in viewer_gone:
+ return
+ raise WebSocketDisconnect(1001)
+
+ monkeypatch.setattr(screenshot, "wait_for_browser_state", wait_for_browser_state)
+ monkeypatch.setattr(screenshot, "start_screencast_loop", screencast)
+
+ with capture_logs() as logs:
+ for session_id in list(sessions.rows):
+ viewer = _WatchingWebSocket(WebSocketDisconnect(1006) if session_id in viewer_gone else None)
+ await asyncio.wait_for(
+ screenshot.browser_session_streaming(
+ websocket=cast(WebSocket, viewer),
+ browser_session_id=session_id,
+ apikey="key",
+ force_cdp=True,
+ ),
+ timeout=10,
+ )
+
+ ended = {log["run_id"]: log for log in _connection_ended(logs)}
+ assert {
+ run_id: (log["live_stream_outcome"], log["live_stream_failure"], log["live_stream_stage"])
+ for run_id, log in ended.items()
+ } == {
+ "wr_released": ("run_finished", "none", "post_ready"),
+ "wr_done": ("run_finished", "none", "post_ready"),
+ "wr_died": ("session_ended", "unrecovered", "post_ready"),
+ "wr_blank": ("no_frame_timeout", "unrecovered", "pre_ready"),
+ "wr_died_unsent": ("session_ended", "unrecovered", "post_ready"),
+ "wr_dead_on_arrival": ("session_ended", "unrecovered", "pre_ready"),
+ "wr_viewer_gone": ("viewer_left", "none", "post_ready"),
+ }
+ assert {(log["live_stream_transport"], log["live_stream_lineage"]) for log in ended.values()} == {
+ ("cdp_screencast", "pbs")
+ }
+ assert _scorecard_tile(logs) == (
+ {"wr_died", "wr_blank", "wr_died_unsent", "wr_dead_on_arrival"},
+ set(ended),
+ )
+
+
+class _DisplayServer:
+ """A session's VNC upstream: it sends the RFB handshake, then closes if the scenario says so."""
+
+ def __init__(self, before_close: Callable[[], Awaitable[None]] | None) -> None:
+ self._before_close = before_close
+ self._handshake_sent = False
+
+ async def __aenter__(self) -> _DisplayServer:
+ return self
+
+ async def __aexit__(self, *exc_info: object) -> None:
+ return None
+
+ async def send(self, data: bytes) -> None:
+ return None
+
+ async def recv(self) -> bytes:
+ if not self._handshake_sent:
+ self._handshake_sent = True
+ return b"RFB 003.008\n"
+ if self._before_close is None:
+ await asyncio.Event().wait()
+ else:
+ await self._before_close()
+ raise ConnectionClosedError(None, None)
+
+
+class _VncViewer:
+ def __init__(self, *, leaves: bool) -> None:
+ self.client_state = WebSocketState.CONNECTED
+ self._leaves = leaves
+ self._framed = asyncio.Event()
+ self._closed = asyncio.Event()
+
+ async def receive_bytes(self) -> bytes:
+ await (self._framed if self._leaves else self._closed).wait()
+ self.client_state = WebSocketState.DISCONNECTED
+ raise WebSocketDisconnect(1001)
+
+ async def send_bytes(self, data: bytes) -> None:
+ self._framed.set()
+
+ async def close(self, code: int = 1000, reason: str | None = None) -> None:
+ self.client_state = WebSocketState.DISCONNECTED
+ self._closed.set()
+
+
+def watch_over_vnc(monkeypatch: pytest.MonkeyPatch) -> Callable[..., Awaitable[None]]:
+ """Watch a session over the VNC route until the viewer leaves, or `before_close` runs and the display server
+ closes (or fails, when it raises)."""
+ monkeypatch.setattr(vnc_route, "auth", AsyncMock(return_value="o_1"))
+ monkeypatch.setattr(vnc_channels, "get_x_api_key", AsyncMock(return_value="sk-test"))
+
+ async def next_poll(_seconds: float) -> None:
+ await asyncio.sleep(0)
+
+ monkeypatch.setattr(verify, "asyncio", ScopedAsyncio(sleep=next_poll))
+
+ async def _watch(
+ session_id: str,
+ *,
+ before_close: Callable[[], Awaitable[None]] | None = None,
+ viewer_leaves: bool = False,
+ ) -> None:
+ display_server = _DisplayServer(before_close)
+ monkeypatch.setattr(vnc_channels.websockets, "connect", lambda url, **_: display_server)
+ await asyncio.wait_for(
+ vnc_route.browser_session_stream(
+ websocket=cast(WebSocket, _VncViewer(leaves=viewer_leaves)),
+ browser_session_id=session_id,
+ apikey="key",
+ client_id=f"client_{session_id}",
+ ),
+ timeout=10,
+ )
+
+ return _watch
+
+
+async def test_vnc_counts_a_run_whose_session_died_under_it_but_not_a_viewer_leaving_or_a_relay_hiccup(
+ monkeypatch: pytest.MonkeyPatch, watch_over_vnc: Callable[..., Awaitable[None]]
+) -> None:
+ sessions = _Sessions(
+ pbs_left="wr_left", pbs_hiccup="wr_hiccup", pbs_broken="wr_broken", pbs_debug=None, pbs_idle=None
+ )
+ running = WorkflowRunStatus.running
+ monkeypatch.setattr(app, "PERSISTENT_SESSIONS_MANAGER", sessions)
+ monkeypatch.setattr(
+ app, "DATABASE", _run_statuses(wr_left=running, wr_hiccup=running, wr_broken=running, wr_debug=running)
+ )
+
+ async def relay_hiccup() -> None:
+ return None
+
+ async def display_server_read_fails() -> None:
+ raise RuntimeError("cannot call recv while another coroutine is already running recv")
+
+ async def debug_run_starts_then_its_session_dies() -> None:
+ # The viewer opened an idle debug session; the run it then starts is what the viewer is watching.
+ sessions.occupy("pbs_debug", "wr_debug")
+ await sessions.next_read()
+ sessions.fail("pbs_debug")
+
+ with capture_logs() as logs:
+ await watch_over_vnc("pbs_left", viewer_leaves=True)
+ await watch_over_vnc("pbs_hiccup", before_close=relay_hiccup)
+ await watch_over_vnc("pbs_broken", before_close=display_server_read_fails)
+ await watch_over_vnc("pbs_debug", before_close=debug_run_starts_then_its_session_dies)
+ await watch_over_vnc("pbs_idle", viewer_leaves=True)
+
+ ended = {log["run_id"]: log for log in _connection_ended(logs)}
+ assert {run_id: (log["live_stream_outcome"], log["live_stream_failure"]) for run_id, log in ended.items()} == {
+ "wr_left": ("viewer_left", "none"),
+ "wr_hiccup": ("stream_error", "recoverable"),
+ "wr_broken": ("stream_error", "recoverable"),
+ "wr_debug": ("session_ended", "unrecovered"),
+ }
+ assert {(log["live_stream_transport"], log["live_stream_stage"]) for log in ended.values()} == {
+ ("vnc", "post_ready")
+ }
+ assert _scorecard_tile(logs) == ({"wr_debug"}, {"wr_left", "wr_hiccup", "wr_broken", "wr_debug"})
+
+
+async def test_vnc_attributes_the_stream_ending_to_the_run_holding_the_session_since_the_last_poll(
+ monkeypatch: pytest.MonkeyPatch, watch_over_vnc: Callable[..., Awaitable[None]]
+) -> None:
+ sessions = _Sessions(pbs_handoff="wr_first", pbs_idle=None)
+ running = WorkflowRunStatus.running
+ monkeypatch.setattr(app, "PERSISTENT_SESSIONS_MANAGER", sessions)
+ monkeypatch.setattr(app, "DATABASE", _run_statuses(wr_second=running, wr_late=running))
+ now = [0.0]
+ monkeypatch.setattr(run_stream_outcome, "time", SimpleNamespace(monotonic=lambda: now[0]))
+
+ # Neither of these awaits between its changes and the close, so no session poll sees them.
+ async def next_run_takes_the_session_then_it_dies() -> None:
+ now[0] += 600
+ sessions.release("pbs_handoff")
+ sessions.occupy("pbs_handoff", "wr_second")
+ sessions.fail("pbs_handoff")
+
+ async def run_takes_the_idle_session() -> None:
+ sessions.occupy("pbs_idle", "wr_late")
+
+ with capture_logs() as logs:
+ await watch_over_vnc("pbs_handoff", before_close=next_run_takes_the_session_then_it_dies)
+ await watch_over_vnc("pbs_idle", before_close=run_takes_the_idle_session)
+
+ # The stage belongs to the connection, which had a working stream before either run took the session.
+ assert {
+ log["run_id"]: (log["live_stream_outcome"], log["live_stream_failure"], log["live_stream_stage"])
+ for log in _connection_ended(logs)
+ } == {
+ "wr_first": ("run_finished", "none", "post_ready"),
+ "wr_second": ("session_ended", "unrecovered", "post_ready"),
+ "wr_late": ("stream_error", "recoverable", "post_ready"),
+ }
+ held_for = {log["run_id"]: log["connection_seconds"] for log in _connection_ended(logs)}
+ assert (held_for["wr_first"], held_for["wr_second"]) == (600.0, 0.0)
+ assert _scorecard_tile(logs) == ({"wr_second"}, {"wr_first", "wr_second", "wr_late"})
+
+
+async def test_vnc_keeps_a_session_death_it_saw_when_the_closing_session_read_fails(
+ monkeypatch: pytest.MonkeyPatch, watch_over_vnc: Callable[..., Awaitable[None]]
+) -> None:
+ sessions = _Sessions(pbs_died="wr_died")
+ monkeypatch.setattr(app, "PERSISTENT_SESSIONS_MANAGER", sessions)
+ monkeypatch.setattr(app, "DATABASE", _run_statuses(wr_died=WorkflowRunStatus.running))
+
+ async def session_dies_then_its_store_goes_down() -> None:
+ sessions.fail("pbs_died")
+ await sessions.next_read()
+ sessions.read_error = TimeoutError("QueuePool limit reached")
+
+ with capture_logs() as logs:
+ await watch_over_vnc("pbs_died", before_close=session_dies_then_its_store_goes_down)
+
+ [ended] = _connection_ended(logs)
+ assert (ended["live_stream_outcome"], ended["live_stream_failure"], ended["run_phase"]) == (
+ "session_ended",
+ "unrecovered",
+ "active",
+ )
+ assert _scorecard_tile(logs) == ({"wr_died"}, {"wr_died"})
diff --git a/tests/unit/test_copilot_account_groups.py b/tests/unit/test_copilot_account_groups.py
new file mode 100644
index 000000000..23fdd9b00
--- /dev/null
+++ b/tests/unit/test_copilot_account_groups.py
@@ -0,0 +1,698 @@
+import asyncio
+from collections import defaultdict
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock
+from uuid import uuid4
+
+import pytest
+import pytest_asyncio
+from fastapi import FastAPI
+from httpx import ASGITransport, AsyncClient
+from pydantic_core import to_json
+from sqlalchemy import delete, func, select, update
+
+from skyvern.config import settings
+from skyvern.exceptions import WorkflowPinnedByRunGroup
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.ask_user import AccountGroupReview
+from skyvern.forge.sdk.copilot.browser_ablation import resolve_copilot_tool_surface
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.tools import account_groups, copilot_native_tools
+from skyvern.forge.sdk.copilot.tools.account_groups import (
+ AccountGroupRefusal,
+ AccountGroupState,
+ AccountGroupSubmission,
+ account_group_status,
+ cancel_account_group,
+ run_for_accounts,
+)
+from skyvern.forge.sdk.db.models import (
+ CredentialModel,
+ WorkflowModel,
+ WorkflowParameterModel,
+ WorkflowRunBlockModel,
+ WorkflowRunGroupItemModel,
+ WorkflowRunGroupModel,
+ WorkflowRunModel,
+ WorkflowRunParameterModel,
+)
+from skyvern.forge.sdk.routes import workflow_copilot as routes
+from skyvern.forge.sdk.schemas.workflow_copilot import CopilotPendingTurn
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType
+from skyvern.forge.sdk.workflow.models.workflow import COPILOT_TEST_WORKFLOW_CREATOR, WorkflowRunStatus
+from skyvern.schemas.workflow_run_groups import (
+ MAX_WORKFLOW_RUN_GROUP_ITEMS,
+ WorkflowRunGroupCreateRequest,
+ WorkflowRunGroupItemOutcome,
+ WorkflowRunGroupItemRequest,
+ WorkflowRunGroupItemState,
+ WorkflowRunGroupStatus,
+)
+from skyvern.services import workflow_run_group_service as group_service
+from tests.unit.conftest import (
+ RUN_GROUP_ORG,
+ RUN_GROUP_WPID,
+ GroupEnv,
+ run_group_definition,
+ run_group_task_block,
+)
+
+SECRET = "vault-password-fixture-81723"
+
+
+@dataclass
+class Chat:
+ env: GroupEnv
+ ctx: CopilotContext
+ client: AsyncClient
+ frames: asyncio.Queue[dict[str, Any]]
+
+
+@pytest_asyncio.fixture
+async def chat(run_group_env: GroupEnv, monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[Chat]:
+ async with run_group_env.database.Session() as session:
+ session.add_all(
+ [
+ CredentialModel(
+ credential_id="cred_3",
+ organization_id=RUN_GROUP_ORG,
+ name="Billing",
+ credential_type="password",
+ username="[email protected]",
+ item_id="item_cred_3",
+ ),
+ CredentialModel(
+ credential_id="cred_revoked",
+ organization_id=RUN_GROUP_ORG,
+ name="Old",
+ credential_type="password",
+ item_id="item_cred_revoked",
+ deleted_at=datetime.now(UTC).replace(tzinfo=None),
+ ),
+ ]
+ )
+ await session.commit()
+ await run_group_env.database.workflow_params.create_workflow_parameter(
+ workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.STRING, key="note", default_value="none"
+ )
+ vault = SimpleNamespace(
+ get_credential_item=AsyncMock(return_value=SimpleNamespace(credential=SimpleNamespace(password=SECRET)))
+ )
+ monkeypatch.setattr(
+ object.__getattribute__(app, "_inst"),
+ "CREDENTIAL_VAULT_SERVICES",
+ defaultdict(lambda: vault),
+ raising=False,
+ )
+ monkeypatch.setattr(app, "CACHE", None)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.ask_user.QUESTION_POLL_SECONDS", 0.01)
+ repo = run_group_env.database.workflow_params
+ row = await repo.create_workflow_copilot_chat(organization_id=RUN_GROUP_ORG, workflow_permanent_id=RUN_GROUP_WPID)
+ await repo.start_copilot_turn(
+ organization_id=RUN_GROUP_ORG,
+ workflow_copilot_chat_id=row.workflow_copilot_chat_id,
+ pending_turn=CopilotPendingTurn(turn_id="turn", started_at=datetime.now(UTC), cancel_token="stop"),
+ user_message="Run it for my accounts",
+ )
+ api = FastAPI()
+ api.dependency_overrides[routes.org_auth_service.get_current_org] = lambda: run_group_env.organization
+ api.add_api_route("/reply", routes.workflow_copilot_question_response, methods=["POST"])
+ api.add_api_route("/history", routes.workflow_copilot_chat_history, methods=["GET"])
+ frames: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
+ ctx = CopilotContext(
+ organization_id=RUN_GROUP_ORG,
+ workflow_id="wf_1",
+ workflow_permanent_id=RUN_GROUP_WPID,
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=None,
+ api_key="",
+ turn_id="turn",
+ workflow_copilot_chat_id=row.workflow_copilot_chat_id,
+ copilot_cancel_token="stop",
+ )
+ ctx.stream = SimpleNamespace(send=frames.put)
+ await _seed_run(run_group_env)
+ async with AsyncClient(transport=ASGITransport(app=api), base_url="http://fixture") as client:
+ yield Chat(run_group_env, ctx, client, frames)
+
+
+async def _seed_run(
+ env: GroupEnv,
+ *,
+ workflow_id: str = "wf_1",
+ credential_id: str = "cred_1",
+ blocks: tuple[tuple[str, str, str | None], ...] = (("login", "completed", None),),
+ status: WorkflowRunStatus = WorkflowRunStatus.completed,
+ group: bool = True,
+) -> None:
+ workflow = await env.database.workflows.get_workflow(workflow_id)
+ parameters = await app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow_id)
+ login = next(parameter for parameter in parameters if parameter.key == "login")
+ workflow_run_id = f"wr_{uuid4().hex}"
+ block_ids = {label: f"wrb_{uuid4().hex}" for label, _, _ in blocks}
+ async with env.database.Session() as session:
+ if group:
+ group_id = f"wrg_{uuid4().hex}"
+ assert workflow is not None
+ session.add(
+ WorkflowRunGroupModel(
+ workflow_run_group_id=group_id,
+ organization_id=RUN_GROUP_ORG,
+ workflow_permanent_id=RUN_GROUP_WPID,
+ workflow_id=workflow_id,
+ submission_key=f"seed:{workflow_run_id}:{account_groups.workflow_definition_hash(workflow)}",
+ input_fingerprint="seed",
+ status=WorkflowRunGroupStatus.finished.value,
+ )
+ )
+ session.add(
+ WorkflowRunGroupItemModel(
+ workflow_run_group_id=group_id,
+ position=0,
+ item_key=credential_id,
+ parameters={"login": credential_id},
+ workflow_run_id=workflow_run_id,
+ state=WorkflowRunGroupItemState.done.value,
+ )
+ )
+ session.add(
+ WorkflowRunModel(
+ workflow_run_id=workflow_run_id,
+ workflow_id=workflow_id,
+ workflow_permanent_id=RUN_GROUP_WPID,
+ organization_id=RUN_GROUP_ORG,
+ status=status.value,
+ )
+ )
+ session.add(
+ WorkflowRunParameterModel(
+ workflow_run_id=workflow_run_id,
+ workflow_parameter_id=login.workflow_parameter_id,
+ value=credential_id,
+ )
+ )
+ session.add_all(
+ WorkflowRunBlockModel(
+ workflow_run_block_id=block_ids[label],
+ parent_workflow_run_block_id=block_ids[parent] if parent else None,
+ workflow_run_id=workflow_run_id,
+ organization_id=RUN_GROUP_ORG,
+ block_type="task",
+ status=block_status,
+ label=label,
+ )
+ for label, block_status, parent in blocks
+ )
+ await session.commit()
+
+
+def _start(
+ chat: Chat, credential_ids: list[str], **overrides: str | dict[str, str] | None
+) -> asyncio.Task[AccountGroupSubmission | AccountGroupRefusal]:
+ arguments: dict[str, Any] = {
+ "credential_parameter_key": "login",
+ "common_inputs": {},
+ "action_summary": "Download the latest statement",
+ **overrides,
+ }
+ return asyncio.create_task(
+ run_for_accounts(chat.ctx, tool_call_id=uuid4().hex, credential_ids=credential_ids, **arguments)
+ )
+
+
+async def _card(chat: Chat) -> dict[str, Any]:
+ while (frame := await asyncio.wait_for(chat.frames.get(), 5))["type"] != "question_required":
+ pass
+ return frame["interactions"][0]
+
+
+async def _decide(chat: Chat, card: dict[str, Any], decision: dict[str, Any]) -> int:
+ reply = await chat.client.post(
+ "/reply",
+ json={
+ "workflow_copilot_chat_id": chat.ctx.workflow_copilot_chat_id,
+ "interaction_id": card["interaction_id"],
+ "account_group_decision": decision,
+ },
+ )
+ return reply.status_code
+
+
+async def _groups_from_this_chat(chat: Chat) -> int:
+ prefix = account_groups.account_group_key_prefix(chat.ctx.workflow_copilot_chat_id)
+ async with chat.env.database.Session() as session:
+ query = select(func.count()).where(WorkflowRunGroupModel.submission_key.startswith(prefix))
+ return int(await session.scalar(query) or 0)
+
+
+async def _approved_group(chat: Chat, credential_ids: list[str]) -> str:
+ task = _start(chat, credential_ids)
+ card = await _card(chat)
+ assert await _decide(chat, card, {"approved": True, "credential_ids": credential_ids}) == 200
+ result = await asyncio.wait_for(task, 5)
+ assert isinstance(result, AccountGroupSubmission) and result.workflow_run_group_id is not None
+ return result.workflow_run_group_id
+
+
+async def test_http_approval_submits_exactly_the_checked_subset_without_secrets(chat: Chat) -> None:
+ task = _start(chat, ["cred_1", "cred_2", "cred_3"])
+ card = await _card(chat)
+ review = card["account_group_review"]
+ assert [row["credential_id"] for row in review["rows"]] == ["cred_1", "cred_2", "cred_3"]
+ assert review["rows"][2]["label"] == "Billing (al***@example.com)"
+ assert (review["version"], review["workflow_id"]) == (1, "wf_1")
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_3"]}) == 200
+ result = await asyncio.wait_for(task, 5)
+
+ assert isinstance(result, AccountGroupSubmission) and result.dispatched
+ assert {row.outcome for row in result.rows} <= {"pending", "in_progress"}
+ group_id = result.workflow_run_group_id
+ assert group_id is not None
+ items = await chat.env.database.workflow_run_groups.get_items(group_id)
+ assert [(item.item_key, item.parameters) for item in items] == [
+ ("cred_1", {"login": "cred_1"}),
+ ("cred_3", {"login": "cred_3"}),
+ ]
+ history = await chat.client.get("/history", params={"workflow_copilot_chat_id": chat.ctx.workflow_copilot_chat_id})
+ restored = history.json()["question_interactions"]
+ assert [interaction["account_group_review"]["workflow_run_group_id"] for interaction in restored] == [group_id]
+ # A backend rolled back to before account groups validates the stored response with extra="forbid".
+ assert set(restored[0]["response"]) <= {"answers", "text", "skipped"}
+ assert restored[0]["account_group_review"]["decision"] == {"approved": True, "credential_ids": ["cred_1", "cred_3"]}
+ for surface in (card, result, restored, [item.parameters for item in items]):
+ assert SECRET not in to_json(surface).decode()
+
+
+ "case",
+ [
+ "unknown",
+ "duplicate",
+ "other_org",
+ "revoked",
+ "over_limit",
+ ],
+)
+async def test_refused_selection_shows_no_card_and_submits_nothing(chat: Chat, case: str) -> None:
+ credential_ids = {
+ "unknown": ["cred_1", "cred_missing"],
+ "duplicate": ["cred_1", "cred_2", "cred_1"],
+ "other_org": ["cred_1", "cred_foreign"],
+ "revoked": ["cred_1", "cred_revoked"],
+ "over_limit": [f"cred_{i}" for i in range(MAX_WORKFLOW_RUN_GROUP_ITEMS + 1)],
+ }[case]
+
+ result = await asyncio.wait_for(_start(chat, credential_ids), 5)
+
+ assert isinstance(result, AccountGroupRefusal) and result.dispatched is False
+ if case == "over_limit":
+ assert all(f"'{credential_id}'" in result.error for credential_id in credential_ids)
+ assert chat.frames.empty()
+ assert await _groups_from_this_chat(chat) == 0
+
+
[email protected]("case", ["clean", "no_run", "older_version_child", "failed_branch", "other_account"])
+async def test_the_card_reports_whether_the_saved_version_ran_cleanly_without_refusing(chat: Chat, case: str) -> None:
+ async with chat.env.database.Session() as session:
+ await session.execute(delete(WorkflowRunModel))
+ await session.commit()
+ if case == "clean":
+ await _seed_run(
+ chat.env,
+ blocks=(("login", "completed", None), ("branch", "completed", None), ("unused_arm", "skipped", "branch")),
+ )
+ if case == "older_version_child":
+ await _seed_run(chat.env)
+ await _add_version(chat)
+ if case == "failed_branch":
+ await _seed_run(
+ chat.env,
+ blocks=(("login", "completed", None), ("branch", "completed", None), ("download", "failed", "branch")),
+ )
+ if case == "other_account":
+ await _seed_run(chat.env, credential_id="cred_3")
+
+ task = _start(chat, ["cred_1", "cred_2"])
+
+ card = await _card(chat)
+ review = card["account_group_review"]
+ assert (review["clean_group_run_id"] is not None) is (case == "clean")
+ assert [row["preselected"] for row in review["rows"]] == [True, True]
+ assert await _decide(chat, card, {"approved": False}) == 200
+ await asyncio.wait_for(task, 5)
+
+
+async def test_an_account_that_already_ran_starts_unchecked_without_naming_the_prior_group(chat: Chat) -> None:
+ group_id = await _approved_group(chat, ["cred_1"])
+ (item,) = await chat.env.database.workflow_run_groups.get_items(group_id)
+ await group_service.advance_workflow_run_group(group_id)
+ await chat.env.database.workflow_runs.update_workflow_run(item.workflow_run_id, status=WorkflowRunStatus.completed)
+ await group_service.advance_workflow_run_group(group_id)
+
+ task = _start(chat, ["cred_1", "cred_2"])
+
+ card = await _card(chat)
+ rows = card["account_group_review"]["rows"]
+ assert [(row["credential_id"], row["prior_outcome"], row["preselected"]) for row in rows] == [
+ ("cred_1", "completed", False),
+ ("cred_2", None, True),
+ ]
+ assert await _decide(chat, card, {"approved": False}) == 200
+ await asyncio.wait_for(task, 5)
+
+
+async def _add_version(
+ chat: Chat, workflow_id: str = "wf_2", version: int = 2, *, deleted: bool = False, created_by: str | None = None
+) -> None:
+ async with chat.env.database.Session() as session:
+ session.add(
+ WorkflowModel(
+ workflow_id=workflow_id,
+ workflow_permanent_id=RUN_GROUP_WPID,
+ organization_id=RUN_GROUP_ORG,
+ title="Workflow",
+ version=version,
+ workflow_definition=run_group_definition(run_group_task_block()),
+ deleted_at=datetime.now(UTC).replace(tzinfo=None) if deleted else None,
+ created_by=created_by,
+ )
+ )
+ await session.commit()
+ await chat.env.database.workflow_params.create_workflow_parameter(
+ workflow_id=workflow_id,
+ workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID,
+ key="login",
+ default_value=None,
+ )
+
+
+async def _edit_in_place(chat: Chat) -> None:
+ async with chat.env.database.Session() as session:
+ await session.execute(
+ update(WorkflowModel)
+ .where(WorkflowModel.workflow_id == "wf_1")
+ .values(modified_at=datetime.now(UTC).replace(tzinfo=None) + timedelta(seconds=5))
+ )
+ await session.commit()
+
+
+async def _retype_param(chat: Chat) -> None:
+ async with chat.env.database.Session() as session:
+ await session.execute(
+ update(WorkflowParameterModel)
+ .where(WorkflowParameterModel.key == "login")
+ .values(workflow_parameter_type="string")
+ )
+ await session.commit()
+
+
+ "case",
+ ["decline", "new_version", "edited_in_place", "param_retyped"],
+)
+async def test_decline_or_stale_approval_dispatches_nothing(chat: Chat, case: str) -> None:
+ task = _start(chat, ["cred_1", "cred_2"])
+ card = await _card(chat)
+ if case == "decline":
+ assert await _decide(chat, card, {"approved": False}) == 200
+ else:
+ await {"new_version": _add_version, "edited_in_place": _edit_in_place, "param_retyped": _retype_param}[case](
+ chat
+ )
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+
+ result = await asyncio.wait_for(task, 5)
+
+ if case == "decline":
+ assert isinstance(result, AccountGroupSubmission) and not result.dispatched
+ else:
+ assert isinstance(result, AccountGroupRefusal) and result.dispatched is False
+ assert await _groups_from_this_chat(chat) == 0
+
+
+async def test_copilot_restore_is_refused_while_the_group_runs(chat: Chat) -> None:
+ await _approved_group(chat, ["cred_1", "cred_2"])
+
+ saved = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(
+ workflow_permanent_id=RUN_GROUP_WPID, organization_id=RUN_GROUP_ORG
+ )
+ with pytest.raises(WorkflowPinnedByRunGroup):
+ await routes._restore_workflow_definition(saved, RUN_GROUP_ORG)
+
+
+async def _running_group_of_three(chat: Chat) -> str:
+ group_id = await _approved_group(chat, ["cred_1", "cred_2", "cred_3"])
+ await group_service.advance_workflow_run_group(group_id)
+ first, second, _ = await chat.env.database.workflow_run_groups.get_items(group_id)
+ await chat.env.database.workflow_runs.update_workflow_run(first.workflow_run_id, status=WorkflowRunStatus.completed)
+ await group_service.advance_workflow_run_group(group_id)
+ await chat.env.database.workflow_runs.update_workflow_run(second.workflow_run_id, status=WorkflowRunStatus.running)
+ while not chat.frames.empty():
+ chat.frames.get_nowait()
+ return group_id
+
+
[email protected]("answer", ["approve", "decline"])
+async def test_only_an_approved_cancel_card_cancels_the_group(chat: Chat, answer: str) -> None:
+ group_id = await _running_group_of_three(chat)
+
+ task = asyncio.create_task(
+ cancel_account_group(chat.ctx, tool_call_id="call-cancel", workflow_run_group_id=group_id)
+ )
+ card = await _card(chat)
+ unfinished = card["account_group_cancel"]["unfinished_rows"]
+ assert [(row["credential_id"], row["outcome"]) for row in unfinished] == [
+ ("cred_2", "in_progress"),
+ ("cred_3", "pending"),
+ ]
+ assert await _decide(chat, card, {"approved": answer == "approve", "credential_ids": []}) == 200
+ result = await asyncio.wait_for(task, 5)
+ assert isinstance(result, AccountGroupState) and result.cancel_approved is (answer == "approve")
+
+ group = await group_service.get_workflow_run_group(group_id, RUN_GROUP_ORG)
+ if answer == "approve":
+ assert [item.outcome for item in group.items] == [
+ WorkflowRunGroupItemOutcome.completed,
+ WorkflowRunGroupItemOutcome.unknown,
+ WorkflowRunGroupItemOutcome.canceled,
+ ]
+ else:
+ assert group.status == WorkflowRunGroupStatus.active
+ assert WorkflowRunGroupItemOutcome.canceled not in [item.outcome for item in group.items]
+
+
[email protected]("withheld_by", [None, "browser_authority", "rollback_flag"])
+def test_withholding_submission_hides_only_the_run_tool(
+ monkeypatch: pytest.MonkeyPatch, withheld_by: str | None
+) -> None:
+ if withheld_by == "rollback_flag":
+ monkeypatch.setattr(settings, "COPILOT_ACCOUNT_GROUP_SUBMIT_ENABLED", False)
+ surface = resolve_copilot_tool_surface(
+ mode=None,
+ native_tools=copilot_native_tools(
+ supports_question_tool=True, supports_account_group_card=True, browser_code_available=False
+ ),
+ alias_map={},
+ overlays={},
+ browser_tools_available=withheld_by != "browser_authority",
+ )
+
+ names = set(surface.ordered_native_names)
+ assert {"get_account_group_status", "cancel_account_group"} <= names
+ assert ("run_workflow_for_accounts" in names) is (withheld_by is None)
+
+
+async def test_retry_starts_running_and_repeat_risk_rows_unchecked(chat: Chat) -> None:
+ group_id = await _approved_group(chat, ["cred_1", "cred_2", "cred_3"])
+
+ early = _start(chat, ["cred_1", "cred_2", "cred_3"])
+ early_card = await _card(chat)
+ assert [row["preselected"] for row in early_card["account_group_review"]["rows"]] == [False] * 3
+ assert await _decide(chat, early_card, {"approved": False}) == 200
+ await asyncio.wait_for(early, 5)
+
+ items = await chat.env.database.workflow_run_groups.get_items(group_id)
+ for item, status in zip(
+ items, (WorkflowRunStatus.completed, WorkflowRunStatus.failed, WorkflowRunStatus.failed), strict=True
+ ):
+ await group_service.advance_workflow_run_group(group_id)
+ if item.item_key == "cred_3":
+ await chat.env.database.workflow_runs.update_workflow_run(item.workflow_run_id, script_id="s_1")
+ await chat.env.database.workflow_runs.update_workflow_run(
+ item.workflow_run_id, status=status, failure_reason=f"page said: wrong password for {item.item_key}"
+ )
+ await group_service.advance_workflow_run_group(group_id)
+
+ status = await account_group_status(chat.ctx, group_id)
+ assert isinstance(status, AccountGroupState) and "wrong password" not in status.model_dump_json()
+ assert [row.outcome for row in status.rows] == [
+ WorkflowRunGroupItemOutcome.completed,
+ WorkflowRunGroupItemOutcome.failed,
+ WorkflowRunGroupItemOutcome.unknown,
+ ]
+
+ retry = _start(chat, ["cred_1", "cred_2", "cred_3"])
+ card = await _card(chat)
+ rows = card["account_group_review"]["rows"]
+ assert [(row["prior_outcome"], row["preselected"]) for row in rows] == [
+ ("completed", False),
+ ("failed", True),
+ ("unknown", False),
+ ]
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+ result = await asyncio.wait_for(retry, 5)
+
+ assert isinstance(result, AccountGroupSubmission)
+ retry_id = result.workflow_run_group_id
+ assert [row.credential_id for row in result.repeated_after_prior_effect] == ["cred_1"]
+ retry_items = await chat.env.database.workflow_run_groups.get_items(retry_id)
+ assert [item.item_key for item in retry_items] == ["cred_1", "cred_2"]
+ assert not {item.workflow_run_id for item in retry_items} & {item.workflow_run_id for item in items}
+
+
+async def test_an_account_that_ran_elsewhere_after_the_review_refuses_the_approval(chat: Chat) -> None:
+ task = _start(chat, ["cred_1", "cred_2"])
+ card = await _card(chat)
+ await group_service.submit_workflow_run_group(
+ chat.env.organization,
+ WorkflowRunGroupCreateRequest(
+ workflow_id=RUN_GROUP_WPID,
+ submission_key=f"{account_groups.account_group_key_prefix('wcc_other')}interaction:hash",
+ items=[WorkflowRunGroupItemRequest(key="cred_1", parameters={"login": "cred_1"})],
+ ),
+ )
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+ result = await asyncio.wait_for(task, 5)
+
+ assert isinstance(result, AccountGroupRefusal) and result.dispatched is False
+ assert await _groups_from_this_chat(chat) == 0
+
+
+async def test_a_group_started_elsewhere_after_the_recheck_is_caught_under_the_creation_lock(
+ chat: Chat, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ recheck = account_groups._recheck_approval
+
+ async def recheck_then_race(
+ ctx: CopilotContext, review: AccountGroupReview, approved_ids: list[str]
+ ) -> dict[str, str]:
+ latest_groups = await recheck(ctx, review, approved_ids)
+ await group_service.submit_workflow_run_group(
+ chat.env.organization,
+ WorkflowRunGroupCreateRequest(
+ workflow_id=RUN_GROUP_WPID,
+ submission_key=f"{account_groups.account_group_key_prefix('wcc_other')}interaction:hash",
+ items=[WorkflowRunGroupItemRequest(key="cred_1", parameters={"login": "cred_1"})],
+ ),
+ )
+ return latest_groups
+
+ monkeypatch.setattr(account_groups, "_recheck_approval", recheck_then_race)
+ task = _start(chat, ["cred_1", "cred_2"])
+ card = await _card(chat)
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+ result = await asyncio.wait_for(task, 5)
+
+ assert isinstance(result, AccountGroupRefusal) and result.status_code == 409
+ assert await _groups_from_this_chat(chat) == 0
+
+
+async def test_an_account_replaced_under_the_same_id_after_the_review_refuses_the_approval(chat: Chat) -> None:
+ task = _start(chat, ["cred_1", "cred_3"])
+ card = await _card(chat)
+ async with chat.env.database.Session() as session:
+ await session.execute(
+ update(CredentialModel).where(CredentialModel.credential_id == "cred_3").values(username="[email protected]")
+ )
+ await session.commit()
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_3"]}) == 200
+ result = await asyncio.wait_for(task, 5)
+
+ assert isinstance(result, AccountGroupRefusal) and "cred_3" in result.error
+ assert await _groups_from_this_chat(chat) == 0
+
+
+async def test_history_does_not_link_a_group_that_claimed_the_card_key_with_other_inputs(chat: Chat) -> None:
+ task = _start(chat, ["cred_1", "cred_2"])
+ card = await _card(chat)
+ review = card["account_group_review"]
+ await group_service.submit_workflow_run_group(
+ chat.env.organization,
+ WorkflowRunGroupCreateRequest(
+ workflow_id=RUN_GROUP_WPID,
+ submission_key=account_groups.account_group_submission_key(
+ chat.ctx.workflow_copilot_chat_id, card["interaction_id"], review["definition_hash"]
+ ),
+ items=[WorkflowRunGroupItemRequest(key="cred_1", parameters={"login": "cred_1", "note": "other"})],
+ ),
+ )
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+ assert isinstance(await asyncio.wait_for(task, 5), AccountGroupRefusal)
+
+ history = await chat.client.get("/history", params={"workflow_copilot_chat_id": chat.ctx.workflow_copilot_chat_id})
+ (restored,) = history.json()["question_interactions"]
+ assert restored["account_group_review"]["workflow_run_group_id"] is None
+
+
+async def test_a_proposal_published_while_the_card_waits_refuses_the_approval(chat: Chat) -> None:
+ task = _start(chat, ["cred_1", "cred_2"])
+ card = await _card(chat)
+ chat.ctx.has_staged_proposal = True
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+ result = await asyncio.wait_for(task, 5)
+
+ assert isinstance(result, AccountGroupRefusal) and result.dispatched is False
+ assert await _groups_from_this_chat(chat) == 0
+
+
[email protected]("saved_by", [None, COPILOT_TEST_WORKFLOW_CREATOR])
+async def test_only_a_saved_version_after_the_recheck_refuses_at_group_creation(
+ chat: Chat, monkeypatch: pytest.MonkeyPatch, saved_by: str | None
+) -> None:
+ recheck = account_groups._recheck_approval
+
+ async def recheck_then_save_a_version(
+ ctx: CopilotContext, review: AccountGroupReview, approved_ids: list[str]
+ ) -> dict[str, str]:
+ latest_groups = await recheck(ctx, review, approved_ids)
+ await _add_version(chat, created_by=saved_by)
+ return latest_groups
+
+ monkeypatch.setattr(account_groups, "_recheck_approval", recheck_then_save_a_version)
+ task = _start(chat, ["cred_1", "cred_2"])
+ card = await _card(chat)
+
+ assert await _decide(chat, card, {"approved": True, "credential_ids": ["cred_1", "cred_2"]}) == 200
+ result = await asyncio.wait_for(task, 5)
+
+ if saved_by is None:
+ assert isinstance(result, AccountGroupRefusal) and result.status_code == 409
+ assert await _groups_from_this_chat(chat) == 0
+ else:
+ assert isinstance(result, AccountGroupSubmission) and result.dispatched
diff --git a/tests/unit/test_copilot_candidate_navigation_guard.py b/tests/unit/test_copilot_candidate_navigation_guard.py
index c7bb27d2c..2a0fffd7a 100644
--- a/tests/unit/test_copilot_candidate_navigation_guard.py
+++ b/tests/unit/test_copilot_candidate_navigation_guard.py
@@ -1,42 +1,15 @@
from __future__ import annotations
import asyncio
-from contextlib import asynccontextmanager
-from dataclasses import dataclass
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
-from skyvern.forge.sdk.copilot import mcp_adapter, runtime
-from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer
+from skyvern.forge.sdk.copilot import mcp_adapter
from skyvern.webeye.real_browser_state import RealBrowserState
-class _Request:
- resource_type = "document"
- url = "https://public.test/start"
-
- class _Frame:
- parent_frame = None
-
- frame = _Frame()
-
- def is_navigation_request(self) -> bool:
- return True
-
-
-class _Route:
- def __init__(self) -> None:
- self.outcome = ""
-
- async def fallback(self) -> None:
- self.outcome = "fallback"
-
- async def abort(self, reason: str) -> None:
- self.outcome = reason
-
-
class _FakePage:
def __init__(self, context: _BrowserContext, url: str = "about:blank") -> None:
self.context = context
@@ -61,8 +34,6 @@ class _FakePage:
class _BrowserContext:
def __init__(self, browser: _Browser | None = None) -> None:
- self.handler = None
- self.unrouted = False
self.pages = [_FakePage(self)]
self.service_workers = []
self.browser = browser or _Browser()
@@ -71,16 +42,6 @@ class _BrowserContext:
def on(self, _event: str, _handler) -> None:
return None
- async def route(self, _pattern: str, handler) -> None:
- self.handler = handler
-
- async def unroute(self, _pattern: str, handler) -> None:
- assert handler is self.handler
- self.unrouted = True
-
- async def cookies(self) -> list[dict[str, str]]:
- return []
-
async def new_page(self) -> _FakePage:
page = _FakePage(self)
self.pages.append(page)
@@ -92,9 +53,9 @@ class _BrowserContext:
class _Browser:
def __init__(self) -> None:
+ self.closed = False
self.new_context_kwargs: dict[str, str] | None = None
self.candidate_context: _BrowserContext | None = None
- self.closed = False
async def new_context(self, **kwargs: str) -> _BrowserContext:
self.new_context_kwargs = kwargs
@@ -108,615 +69,129 @@ class _Browser:
self.closed = True
-class _Page:
- url = "https://public.test/start"
-
-
-@dataclass
-class _BrowserState:
- browser_context: _BrowserContext
- page: SimpleNamespace | None = None
- pw: SimpleNamespace | None = None
- active_page: SimpleNamespace | None = None
- prefer_context_newest_when_unpinned: bool = False
-
- async def get_working_page(self) -> _Page | SimpleNamespace:
- if self.active_page is not None:
- if self.active_page in self.browser_context.pages:
- self.page = self.active_page
- return self.active_page
- self.active_page = None
- if self.prefer_context_newest_when_unpinned:
- self.page = self.browser_context.pages[-1]
- return self.page
- return self.page or _Page()
-
- async def set_working_page(self, page: _Page | SimpleNamespace | None) -> None:
- self.page = page
-
- async def set_active_page(self, page: SimpleNamespace) -> None:
- self.active_page = page
- self.page = page
-
-
class _AgentFunction:
- def __init__(self) -> None:
- self.active = False
- self.idle_waits = 0
-
- @asynccontextmanager
- async def copilot_candidate_network_guard(self, _browser_context, *, expected_origin: str):
- assert expected_origin == "https://public.test"
- self.active = True
- hops = [
- {
- "url": "https://public.test/start",
- "resource_type": "document",
- "resolved_public_ips": ["93.184.216.34"],
- "connected_peer_ip": "93.184.216.34",
- "enforcement_version": "copilot_candidate_preconnect_v1",
- }
- ]
- try:
- yield hops
- finally:
- self.active = False
-
- async def wait_for_copilot_candidate_network_idle(self, _browser_context) -> None:
- self.idle_waits += 1
-
async def setup_browser_context_extensions(self, _browser_context, **_kwargs) -> None:
return None
-def _server() -> SkyvernOverlayMCPServer:
- return SkyvernOverlayMCPServer(
- None,
- {},
- {},
- frozenset(),
- lambda: SimpleNamespace(browser_session_id=None, organization_id="org"),
- )
-
-
@pytest.mark.asyncio
-async def test_candidate_guard_serializes_session_provisioning_with_source_promotion(
+async def test_candidate_context_swap_arms_crash_reaping_on_the_candidate_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- recovery_lock = asyncio.Lock()
- ctx = SimpleNamespace(
- browser_session_id=None,
- browser_session_recovery_lock=recovery_lock,
- browser_session_recovery_owner=None,
- browser_session_recovery_depth=0,
- organization_id="org",
- )
- ensure_started = asyncio.Event()
- browser_context = _BrowserContext()
-
- async def provision(_ctx):
- ensure_started.set()
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- monkeypatch.setattr(runtime, "_provision_browser_session", provision)
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", runtime.ensure_browser_session)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
- server = _server()
- server._context_provider = lambda: ctx
-
- await recovery_lock.acquire()
-
- async def enter_guard() -> None:
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- pass
-
- guard_task = asyncio.create_task(enter_guard())
- await asyncio.sleep(0)
- assert not ensure_started.is_set()
-
- recovery_lock.release()
- await guard_task
- assert ensure_started.is_set()
-
-
-async def test_candidate_guard_serializes_direct_session_retirement_with_source_promotion(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- recovery_lock = asyncio.Lock()
- ctx = SimpleNamespace(
- browser_session_id="session",
- browser_session_recovery_lock=recovery_lock,
- browser_session_recovery_owner=None,
- browser_session_recovery_depth=0,
- organization_id="org",
- )
- resolve_started = asyncio.Event()
- allow_resolve = asyncio.Event()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- resolve_started.set()
- await allow_resolve.wait()
- return None
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter, "retire_browser_session_id", runtime.retire_browser_session_id)
- server = _server()
- server._context_provider = lambda: ctx
-
- async def enter_guard() -> None:
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- pass
-
- guard_task = asyncio.create_task(enter_guard())
- await resolve_started.wait()
- await recovery_lock.acquire()
- allow_resolve.set()
- await asyncio.sleep(0)
- assert ctx.browser_session_id == "session"
- assert not guard_task.done()
+ original = _BrowserContext()
+ original_page = original.pages[0]
+ state = RealBrowserState(pw=SimpleNamespace(), browser_context=original, page=original_page)
- recovery_lock.release()
- with pytest.raises(RuntimeError, match="requires a browser context"):
- await guard_task
- assert ctx.browser_session_id is None
+ async with mcp_adapter.service_worker_blocked_context(state, organization_id="org") as candidate:
+ candidate_page = candidate.pages[-1]
+ assert candidate_page in state._crash_listener_pages
+ candidate_page.crash()
+ await asyncio.gather(*list(state._detached_teardown_tasks))
+ assert candidate_page.closed is True
@pytest.mark.asyncio
-async def test_candidate_guard_uses_disposable_service_worker_blocked_context_for_attached_cdp(
+async def test_candidate_context_restore_keeps_crash_reaping_on_the_original_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- calls: list[str] = []
- browser_context = _BrowserContext()
- original_page = browser_context.pages[0]
- browser_state = _BrowserState(browser_context, page=original_page)
-
- ctx = SimpleNamespace(browser_session_id="inherited", organization_id="org")
-
- async def ensure(fresh_ctx):
- assert fresh_ctx.browser_session_id == "inherited"
- calls.append("ensure")
- return None
-
- async def resolve(_ctx):
- calls.append("resolve")
- return browser_state
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- close_session = AsyncMock()
monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
- monkeypatch.setattr(mcp_adapter.app, "PERSISTENT_SESSIONS_MANAGER", SimpleNamespace(close_session=close_session))
+ original = _BrowserContext()
+ original_page = original.pages[0]
+ state = RealBrowserState(pw=SimpleNamespace(), browser_context=original, page=original_page)
- server = _server()
- server._context_provider = lambda: ctx
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- assert browser_state.browser_context is browser_context.browser.candidate_context
- assert browser_state.page is not original_page
+ async with mcp_adapter.service_worker_blocked_context(state, organization_id="org"):
+ pass
- assert calls[:2] == ["ensure", "resolve"]
- assert ctx.browser_session_id == "inherited"
- assert browser_context.browser.new_context_kwargs == {"service_workers": "block"}
- assert browser_state.browser_context is browser_context
- assert browser_state.page is original_page
- assert browser_context.closed is False
- assert browser_context.browser.candidate_context is not None
- assert browser_context.browser.candidate_context.closed is True
- close_session.assert_not_awaited()
+ assert state.browser_context is original
+ assert original_page in state._crash_listener_pages
+ original_page.crash()
+ await asyncio.gather(*list(state._detached_teardown_tasks))
+ assert original_page.closed is True
-async def test_candidate_guard_isolates_non_pristine_attached_cdp_context(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- browser_context = _BrowserContext()
- selected_page = _FakePage(browser_context, "https://selected.test/")
- newest_page = _FakePage(browser_context, "https://newest.test/")
- browser_context.pages = [selected_page, newest_page]
- browser_state = _BrowserState(
- browser_context,
- page=selected_page,
- active_page=selected_page,
- prefer_context_newest_when_unpinned=True,
+def _persistent_state(fallback_browser: _Browser) -> tuple[RealBrowserState, _BrowserContext, _FakePage, AsyncMock]:
+ persistent = _BrowserContext()
+ persistent.browser = None
+ original_page = persistent.pages[0]
+ launch = AsyncMock(return_value=fallback_browser)
+ state = RealBrowserState(
+ pw=SimpleNamespace(chromium=SimpleNamespace(launch=launch)), browser_context=persistent, page=original_page
)
-
- async def ensure(_ctx, **_kwargs):
- return None
-
- async def resolve(_ctx):
- return browser_state
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
-
- async with _server().evidence_candidate_navigation_guard("https://public.test"):
- assert browser_state.browser_context is browser_context.browser.candidate_context
- assert browser_state.page is not selected_page
- assert browser_state.page is not newest_page
-
- assert browser_context.browser.new_context_kwargs == {"service_workers": "block"}
- assert browser_context.browser.candidate_context is not None
- assert browser_context.browser.candidate_context.closed is True
- assert browser_state.browser_context is browser_context
- assert await browser_state.get_working_page() is selected_page
+ return state, persistent, original_page, launch
@pytest.mark.asyncio
-async def test_candidate_guard_supports_persistent_context_without_browser(
+async def test_candidate_context_blocks_service_workers_and_restores_the_original(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- persistent_context = _BrowserContext()
- persistent_context.browser = None
- original_page = SimpleNamespace(url="https://persistent.test/account")
- fallback_browser = _Browser()
- chromium = SimpleNamespace(launch=AsyncMock(return_value=fallback_browser))
- browser_state = _BrowserState(
- persistent_context,
- page=original_page,
- pw=SimpleNamespace(chromium=chromium),
- )
-
- async def ensure(_ctx, **_kwargs):
- return None
-
- async def resolve(_ctx):
- return browser_state
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
+ original = _BrowserContext()
+ original_page = original.pages[0]
+ state = RealBrowserState(pw=SimpleNamespace(), browser_context=original, page=original_page)
- async with _server().evidence_candidate_navigation_guard("https://public.test"):
- assert browser_state.browser_context is fallback_browser.candidate_context
- assert browser_state.page is not original_page
+ async with mcp_adapter.service_worker_blocked_context(state, organization_id="org") as candidate:
+ assert candidate is original.browser.candidate_context
+ assert state.browser_context is candidate
+ assert await state.get_working_page() is not original_page
- chromium.launch.assert_awaited_once_with()
- assert fallback_browser.new_context_kwargs == {"service_workers": "block"}
- assert fallback_browser.closed is True
- assert persistent_context.closed is False
- assert browser_state.browser_context is persistent_context
- assert browser_state.page is original_page
+ assert original.browser.new_context_kwargs == {"service_workers": "block"}
+ assert candidate.closed is True
+ assert original.closed is False
+ assert state.browser_context is original
+ assert await state.get_working_page() is original_page
@pytest.mark.asyncio
-async def test_candidate_guard_closes_fallback_browser_when_context_creation_fails(
+async def test_candidate_context_launches_and_closes_a_fallback_browser_for_a_persistent_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- persistent_context = _BrowserContext()
- persistent_context.browser = None
- fallback_browser = _Browser()
- fallback_browser.new_context = AsyncMock(side_effect=RuntimeError("context creation failed"))
- browser_state = _BrowserState(
- persistent_context,
- pw=SimpleNamespace(chromium=SimpleNamespace(launch=AsyncMock(return_value=fallback_browser))),
- )
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return browser_state
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
+ fallback = _Browser()
+ state, persistent, original_page, launch = _persistent_state(fallback)
- with pytest.raises(RuntimeError, match="context creation failed"):
- async with _server().evidence_candidate_navigation_guard("https://public.test"):
- pass
+ async with mcp_adapter.service_worker_blocked_context(state, organization_id="org") as candidate:
+ assert candidate is fallback.candidate_context
- assert fallback_browser.closed is True
- assert browser_state.browser_context is persistent_context
+ launch.assert_awaited_once_with()
+ assert fallback.new_context_kwargs == {"service_workers": "block"}
+ assert fallback.closed is True
+ assert persistent.closed is False
+ assert state.browser_context is persistent
+ assert await state.get_working_page() is original_page
@pytest.mark.asyncio
-async def test_candidate_guard_closes_fallback_browser_when_candidate_context_cleanup_fails(
+async def test_candidate_context_closes_the_fallback_browser_when_context_creation_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- persistent_context = _BrowserContext()
- persistent_context.browser = None
- candidate_context = _BrowserContext()
- candidate_context.close = AsyncMock(side_effect=RuntimeError("context close failed"))
- fallback_browser = _Browser()
- fallback_browser.new_context = AsyncMock(return_value=candidate_context)
- browser_state = _BrowserState(
- persistent_context,
- pw=SimpleNamespace(chromium=SimpleNamespace(launch=AsyncMock(return_value=fallback_browser))),
- )
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return browser_state
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
+ fallback = _Browser()
+ fallback.new_context = AsyncMock(side_effect=RuntimeError("context creation failed"))
+ state, persistent, _, _ = _persistent_state(fallback)
- with pytest.raises(RuntimeError, match="context close failed"):
- async with _server().evidence_candidate_navigation_guard("https://public.test"):
+ with pytest.raises(RuntimeError, match="context creation failed"):
+ async with mcp_adapter.service_worker_blocked_context(state, organization_id="org"):
pass
- assert fallback_browser.closed is True
- assert browser_state.browser_context is persistent_context
+ assert fallback.closed is True
+ assert state.browser_context is persistent
@pytest.mark.asyncio
-async def test_candidate_guard_isolates_inherited_page_state(
+async def test_candidate_context_closes_the_fallback_browser_when_candidate_cleanup_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- browser_context = _BrowserContext()
- browser_context.browser = None
- browser_context.pages = [SimpleNamespace(url="https://authenticated.test/account")]
- fallback_browser = _Browser()
-
- async def ensure(_ctx):
- return None
-
- original_page = SimpleNamespace(url="https://authenticated.test/account")
- browser_state = _BrowserState(
- browser_context,
- page=original_page,
- pw=SimpleNamespace(chromium=SimpleNamespace(launch=AsyncMock(return_value=fallback_browser))),
- )
-
- async def resolve(_ctx):
- return browser_state
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
+ fallback = _Browser()
+ candidate = _BrowserContext(fallback)
+ candidate.close = AsyncMock(side_effect=RuntimeError("context close failed"))
+ fallback.new_context = AsyncMock(return_value=candidate)
+ state, persistent, _, _ = _persistent_state(fallback)
- async with _server().evidence_candidate_navigation_guard("https://public.test"):
- assert browser_state.browser_context is not browser_context
- assert all(page.url in {"", "about:blank"} for page in browser_state.browser_context.pages)
-
- assert browser_state.browser_context is browser_context
- assert browser_state.page is original_page
- assert fallback_browser.new_context_kwargs == {"service_workers": "block"}
- assert fallback_browser.closed is True
-
-
-async def test_candidate_guard_uses_terminal_agent_function_route_without_overlay_sibling(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- browser_context = _BrowserContext()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
-
- agent_function = _AgentFunction()
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", agent_function)
-
- async with _server().evidence_candidate_navigation_guard("https://public.test"):
- assert agent_function.active is True
- assert browser_context.handler is None
-
- assert agent_function.idle_waits == 1
-
-
-async def test_candidate_guard_clears_state_when_network_context_entry_fails(monkeypatch: pytest.MonkeyPatch) -> None:
- browser_context = _BrowserContext()
- server = _server()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- class FailingAgentFunction(_AgentFunction):
- @asynccontextmanager
- async def copilot_candidate_network_guard(self, _browser_context, *, expected_origin: str):
- raise RuntimeError("entry failed")
- yield []
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", FailingAgentFunction())
-
- with pytest.raises(RuntimeError, match="entry failed"):
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- pass
-
- assert server._evidence_candidate_origin is None
- assert server._evidence_candidate_guarded_hops is None
-
-
-async def test_candidate_guard_clears_state_when_idle_wait_times_out(monkeypatch: pytest.MonkeyPatch) -> None:
- browser_context = _BrowserContext()
- server = _server()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- agent_function = _AgentFunction()
-
- async def fail_idle(_browser_context) -> None:
- raise TimeoutError("idle timeout")
-
- agent_function.wait_for_copilot_candidate_network_idle = fail_idle
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", agent_function)
-
- with pytest.raises(TimeoutError, match="idle timeout"):
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- pass
-
- assert server._evidence_candidate_origin is None
- assert server._evidence_candidate_guarded_hops is None
-
-
-async def test_candidate_browser_url_uses_last_enforced_document_not_subresource(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- browser_context = _BrowserContext()
- server = _server()
- agent_function = _AgentFunction()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", agent_function)
-
- async with server.evidence_candidate_navigation_guard("https://public.test") as hops:
- hops.append(
- {
- "url": "https://public.test/favicon.ico",
- "resource_type": "image",
- "resolved_public_ips": ["93.184.216.34"],
- "connected_peer_ip": "93.184.216.34",
- "enforcement_version": "copilot_candidate_preconnect_v1",
- }
- )
- assert await server.evidence_candidate_browser_url() == "https://public.test/start"
-
-
-async def test_candidate_browser_url_rejects_fragment_not_in_enforced_document(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- browser_context = _BrowserContext()
- server = _server()
- agent_function = _AgentFunction()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context, page=SimpleNamespace(url="https://public.test/start#after"))
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", agent_function)
-
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- with pytest.raises(RuntimeError, match="candidate_browser_url_not_peer_verified"):
- await server.evidence_candidate_browser_url()
-
-
-async def test_candidate_guard_rejects_non_exact_origin_before_activation(monkeypatch: pytest.MonkeyPatch) -> None:
- browser_context = _BrowserContext()
-
- async def ensure(_ctx):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
- with pytest.raises(ValueError, match="exact HTTPS origin"):
- async with _server().evidence_candidate_navigation_guard("https://public.test/path"):
+ with pytest.raises(RuntimeError, match="context close failed"):
+ async with mcp_adapter.service_worker_blocked_context(state, organization_id="org"):
pass
-
-async def test_real_adapter_internal_call_drains_candidate_network_before_return(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- browser_context = _BrowserContext()
- agent_function = _AgentFunction()
- server = _server()
- server._context_provider = lambda: SimpleNamespace(
- browser_session_id="session",
- browser_session_continuity_generation=0,
- browser_session_recovery_lock=asyncio.Lock(),
- browser_session_recovery_owner=None,
- browser_session_recovery_depth=0,
- organization_id="org",
- turn_origin=mcp_adapter.TurnOrigin.interactive,
- request_policy=None,
- )
- server._client = SimpleNamespace(
- call_tool=AsyncMock(return_value=SimpleNamespace(structured_content={"ok": True}, is_error=False, content=[]))
- )
-
- async def ensure(_ctx, **_kwargs):
- return None
-
- async def resolve(_ctx):
- return _BrowserState(browser_context)
-
- @asynccontextmanager
- async def browser_scope(_ctx, *, session_id_override=None):
- yield
-
- monkeypatch.setattr(mcp_adapter, "ensure_browser_session", ensure)
- monkeypatch.setattr(mcp_adapter, "resolve_browser_state_for_context", resolve)
- monkeypatch.setattr(mcp_adapter, "mcp_browser_context", browser_scope)
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", agent_function)
-
- async with server.evidence_candidate_navigation_guard("https://public.test"):
- result = await server.call_internal_tool("skyvern_navigate", {"url": "https://public.test/start"})
-
- assert result["ok"] is True
- assert agent_function.idle_waits == 2
-
-
-async def test_candidate_context_swap_arms_crash_reaping_on_the_candidate_context(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
- original = _BrowserContext()
- original_page = original.pages[0]
- state = RealBrowserState(pw=SimpleNamespace(), browser_context=original, page=original_page)
-
- async with mcp_adapter.service_worker_blocked_context(state, organization_id="org") as candidate:
- candidate_page = candidate.pages[-1]
- assert candidate_page in state._crash_listener_pages
- candidate_page.crash()
- await asyncio.gather(*list(state._detached_teardown_tasks))
- assert candidate_page.closed is True
-
-
-async def test_candidate_context_restore_keeps_crash_reaping_on_the_original_context(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr(mcp_adapter.app, "AGENT_FUNCTION", _AgentFunction())
- original = _BrowserContext()
- original_page = original.pages[0]
- state = RealBrowserState(pw=SimpleNamespace(), browser_context=original, page=original_page)
-
- async with mcp_adapter.service_worker_blocked_context(state, organization_id="org"):
- pass
-
- assert state.browser_context is original
- assert original_page in state._crash_listener_pages
- original_page.crash()
- await asyncio.gather(*list(state._detached_teardown_tasks))
- assert original_page.closed is True
+ assert fallback.closed is True
+ assert state.browser_context is persistent
diff --git a/tests/unit/test_copilot_discovery.py b/tests/unit/test_copilot_discovery.py
index 23ece6be7..9f9ec0be8 100644
--- a/tests/unit/test_copilot_discovery.py
+++ b/tests/unit/test_copilot_discovery.py
@@ -12,7 +12,6 @@ from unittest.mock import AsyncMock
import pytest
import structlog.testing
-from skyvern.forge.agent_functions import CopilotEntrypointCandidate, CopilotSiteOriginAssociation
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_STRIPPED_HTML_EXPRESSION,
@@ -29,15 +28,9 @@ from skyvern.forge.sdk.copilot.runtime import (
from skyvern.forge.sdk.copilot.tools import (
_discovery_walk,
_inspect_page_for_composition_impl,
- _rank_discovery_entrypoint_candidates,
_resolve_discovery_entry_url,
)
from skyvern.forge.sdk.copilot.tools import _shared as shared_module
-from skyvern.forge.sdk.copilot.tools.discovery import (
- _credential_entry_url,
- _discovery_build_result,
- _user_provided_entry_url,
-)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatSender
@@ -441,7 +434,7 @@ class _TargetThenCurrentPageServer:
("HTTP://example.com/login", ("HTTP://example.com/login", "url")),
("example.com", ("https://example.com", "domain")),
("example.com/login?x=y", ("https://example.com/login?x=y", "domain")),
- ("example", (None, "bare_word")),
+ ("example", (None, "unresolved")),
("example search portal", (None, "unresolved")),
],
)
@@ -468,110 +461,6 @@ def test_resolve_discovery_entry_url_preserves_url_and_domain_inputs(
assert _resolve_discovery_entry_url(site_or_url) == expected
-def test_legacy_url_domain_result_envelope_has_no_bare_word_contract_version() -> None:
- result = _discovery_build_result(
- candidate_url="https://example.com/",
- candidate_form_fields=[],
- evidence_trail=[],
- confidence=0.6,
- failure_reason=None,
- )
-
- assert result == {
- "ok": True,
- "data": {
- "candidate_url": "https://example.com/",
- "candidate_form_fields": [],
- "evidence_trail": [],
- "confidence": 0.6,
- "failure_reason": None,
- },
- "error": None,
- }
-
-
-def test_rank_discovery_entrypoint_candidates_accepts_provider_association_with_different_entity_label() -> None:
- candidates = [
- CopilotEntrypointCandidate(
- url="https://irrelevant.example/news",
- source_rank=1,
- association=CopilotSiteOriginAssociation(
- requested_name="public alias",
- entity_id="Q1",
- entity_label="Unrelated result",
- official_site_url="https://irrelevant.example/news",
- origin="https://irrelevant.example",
- source="provider_official_site",
- provider_relation_type="label",
- provider_relation_text="public alias",
- ),
- ),
- CopilotEntrypointCandidate(
- url="https://attacker.example/start",
- source_rank=2,
- association=CopilotSiteOriginAssociation(
- requested_name="public alias",
- entity_id="Q2",
- entity_label="Public Alias",
- official_site_url="https://public-alias.test/start",
- origin="https://public-alias.test",
- source="provider_official_site",
- provider_relation_type="alias",
- provider_relation_text="public alias",
- ),
- ),
- CopilotEntrypointCandidate(
- url="https://public-alias.test/start",
- source_rank=3,
- association=CopilotSiteOriginAssociation(
- requested_name="public alias",
- entity_id="Q3",
- entity_label="Public Alias",
- official_site_url="https://public-alias.test/start",
- origin="https://public-alias.test",
- source="provider_official_site",
- provider_relation_type="alias",
- provider_relation_text="public alias",
- ),
- ),
- ]
-
- assert _rank_discovery_entrypoint_candidates("public alias", candidates) == [candidates[0], candidates[2]]
-
-
-def test_rank_discovery_entrypoint_candidates_keeps_unassociated_licensee_enforced() -> None:
- licensor = CopilotEntrypointCandidate(
- url="https://licensor.example/",
- source_rank=1,
- association=CopilotSiteOriginAssociation(
- requested_name="Public Alias",
- entity_id="Q1",
- entity_label="Public Alias",
- official_site_url="https://licensor.example/",
- origin="https://licensor.example",
- source="provider_official_site",
- provider_relation_type="label",
- provider_relation_text="Other Licensor",
- ),
- )
- licensee = CopilotEntrypointCandidate(
- url="https://licensee.example/",
- source_rank=2,
- association=CopilotSiteOriginAssociation(
- requested_name="Public Alias",
- entity_id="Q2",
- entity_label="Unrelated Licensee",
- official_site_url="https://licensee.example/",
- origin="https://licensee.example",
- source="provider_official_site",
- provider_relation_type="alias",
- provider_relation_text="Other Licensee",
- ),
- )
-
- assert _rank_discovery_entrypoint_candidates("public alias", [licensor, licensee]) == []
-
-
@pytest.mark.asyncio
async def test_discovery_navigation_failure_falls_back_to_entry_url() -> None:
result = await _discovery_walk(
@@ -1119,98 +1008,22 @@ async def test_capture_composition_evidence_warns_when_html_sliced_at_cap(monkey
assert "html_sliced_at_cap" in evidence["inspection_warnings"]
-class TestUserProvidedEntryUrl:
- """Name lookup asks the world; this asks the conversation, so a URL the user already pasted is
- not answered with a request for a URL."""
-
- @staticmethod
- def _ctx(user_message: str) -> SimpleNamespace:
- policy = RequestPolicy()
- _ground_user_provided_sites(policy, user_message, [])
- return SimpleNamespace(request_policy=policy)
-
- def test_a_url_only_the_product_wrote_never_grounds_a_site(self) -> None:
- """Grounding releases credentials, so it stays USER-only; widening it to every turn opener
- would let a server-authored row authorize an origin the person never typed."""
- policy = RequestPolicy()
- _ground_user_provided_sites(
- policy,
- "fix it",
- [
- SimpleNamespace(
- sender=WorkflowCopilotChatSender.PRODUCT,
- content="Diagnose run wr_1 at https://impostor.example.com and repair the workflow.",
- )
- ],
- )
-
- assert _user_provided_entry_url(SimpleNamespace(request_policy=policy)) is None
-
- def test_the_only_site_the_user_gave_is_opened(self) -> None:
- ctx = self._ctx("go to https://us.example.com/reports and pull the numbers")
-
- assert _user_provided_entry_url(ctx) == "https://us.example.com/reports"
-
- def test_several_sites_resolve_nothing(self) -> None:
- ctx = self._ctx("check https://a.example.com and https://b.example.net")
-
- assert _user_provided_entry_url(ctx) is None
-
- def test_nothing_the_user_provided_resolves_nothing(self) -> None:
- assert _user_provided_entry_url(self._ctx("no addresses here")) is None
-
-
-class TestCredentialEntryUrl:
- """The org already recorded where a credential signs in, so a site named in words whose
- credential carries a login page is opened rather than answered with a request for its URL."""
-
- @staticmethod
- def _ctx(*tested_urls: str | None) -> SimpleNamespace:
- policy = RequestPolicy()
- policy.resolved_credentials = [
- SimpleNamespace(credential_id=f"cred_{index}", name=f"credential {index}", tested_url=tested_url)
- for index, tested_url in enumerate(tested_urls)
- ]
- return SimpleNamespace(request_policy=policy)
-
- def test_the_credential_naming_the_requested_site_is_opened(self) -> None:
- ctx = self._ctx("https://apps.hydroco.example/portal/Login.jsp")
-
- assert _credential_entry_url(ctx, "hydroco") == "https://apps.hydroco.example/portal/Login.jsp"
-
- def test_a_multi_word_site_name_resolves_its_credential(self) -> None:
- ctx = self._ctx("https://apps.guelphhydro.example/portal/Login.jsp")
-
- assert _credential_entry_url(ctx, "guelph hydro") == "https://apps.guelphhydro.example/portal/Login.jsp"
-
- def test_the_credential_whose_host_names_the_site_wins_among_several(self) -> None:
- ctx = self._ctx("https://us5.other.example.net/account/login", "https://us.chosen.example/login")
-
- assert _credential_entry_url(ctx, "chosen") == "https://us.chosen.example/login"
-
- def test_a_lone_credential_for_another_site_resolves_nothing(self) -> None:
- """Approvals persist across turns, so a later request for a different site must not open this one."""
- ctx = self._ctx("https://payroll.example.net/login")
-
- assert _credential_entry_url(ctx, "zephyrmart") is None
-
- def test_a_label_extending_the_requested_name_resolves(self) -> None:
- ctx = self._ctx("https://us5.metricsdog.example/account/login")
-
- assert _credential_entry_url(ctx, "metrics") == "https://us5.metricsdog.example/account/login"
-
- def test_a_name_buried_inside_a_label_resolves_nothing(self) -> None:
- ctx = self._ctx("https://groupsupport.example.net/login")
-
- assert _credential_entry_url(ctx, "ups") is None
-
- def test_several_unrelated_credentials_resolve_nothing(self) -> None:
- ctx = self._ctx("https://a.example.net/login", "https://b.example.org/login")
-
- assert _credential_entry_url(ctx, "unrelated") is None
+def test_a_url_only_the_product_wrote_never_grounds_a_site() -> None:
+ """Grounding releases credentials, so it stays USER-only; widening it to every turn opener
+ would let a server-authored row authorize an origin the person never typed."""
+ policy = RequestPolicy()
+ _ground_user_provided_sites(
+ policy,
+ "fix it",
+ [
+ SimpleNamespace(
+ sender=WorkflowCopilotChatSender.PRODUCT,
+ content="Diagnose run wr_1 at https://impostor.example.com and repair the workflow.",
+ )
+ ],
+ )
- def test_credentials_without_a_login_page_resolve_nothing(self) -> None:
- assert _credential_entry_url(self._ctx(None), "example") is None
+ assert policy.user_provided_site_urls == []
_TUNNEL_NAVIGATION_ERROR = (
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
index 11f7245d5..d1ad8831b 100644
--- a/tests/unit/test_copilot_mcp_adapter.py
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -442,7 +442,6 @@ _PAYLOAD_KEYS_THAT_WOULD_LEAK = {"args", "merged_args", "arguments", "mcp_args",
_BROWSER_BOOT_SECONDS = 2.0
_MCP_CALL_SECONDS = 3.0
_AFTER_CALL_SECONDS = 5.0
-_AFTER_CALL_MS = 5000
_WALL_MS = 5000
_SESSION_ONLY_MS = 2000
_MCP_CALL_MS = 3000
@@ -1137,28 +1136,19 @@ class TestSharedBrowserCallOutcome:
assert second["data"]["metadata"]["width"] == 1280
@pytest.mark.asyncio
- async def test_typed_internal_accessor_preserves_legacy_dict_and_drain_incomplete(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ async def test_typed_internal_accessor_preserves_legacy_dict(self) -> None:
payload = {"ok": True, "data": {"result": {"count": 3}}}
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload=payload,
)
- server._evidence_candidate_origin = "https://public.test"
-
- async def _failed_drain() -> None:
- raise RuntimeError("drain failed")
-
- monkeypatch.setattr(server, "_drain_evidence_candidate_response_tasks", _failed_drain)
typed = await server.call_internal_browser_tool("skyvern_evaluate", {"expression": "scan()"})
legacy = await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
assert typed.result == legacy == mcp_to_copilot(payload)
assert typed.outcome.dispatched is True
- assert typed.outcome.evidence_drain_complete is False
assert typed.outcome.source_browser_session_generation == 0
@pytest.mark.asyncio
@@ -1569,16 +1559,10 @@ class TestSharedBrowserCallOutcome:
class TestMCPToolTiming:
@pytest.mark.asyncio
async def test_internal_call_logs_the_server_total_without_changing_the_result(
- self, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
+ self, _fake_clock: list[float]
) -> None:
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"sdk": 812, "total": 815}}
server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
- server._evidence_candidate_origin = "https://public.test"
-
- async def _slow_drain() -> None:
- _fake_clock[0] += _AFTER_CALL_SECONDS
-
- monkeypatch.setattr(server, "_drain_evidence_candidate_response_tasks", _slow_drain)
with capture_logs() as captured:
result = await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
@@ -2050,7 +2034,6 @@ class TestMCPToolTiming:
record = _timing_records(captured)[0]
assert record["call_status"] == "error"
assert record["timing_phase"] is None
- assert record["post_call_evidence_drain_ms"] is None
_assert_every_millisecond_is_attributed(record)
@pytest.mark.asyncio
@@ -2182,71 +2165,6 @@ class TestMCPToolTiming:
assert record["timing_server_overrun"] is True
_assert_every_millisecond_is_attributed(record)
- @pytest.mark.asyncio
- @pytest.mark.parametrize(
- ("failure", "call_status"),
- [
- (asyncio.CancelledError(), "cancelled"),
- (CopilotBrowserSessionUnavailable("pbs_1"), "session_error"),
- (RuntimeError("drain exploded"), "error"),
- ],
- ids=["cancelled", "session_error", "error"],
- )
- async def test_an_evidence_drain_that_fails_still_reports_the_budget_it_spent(
- self,
- failure: BaseException,
- call_status: str,
- monkeypatch: pytest.MonkeyPatch,
- _fake_clock: list[float],
- ) -> None:
- payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"total": 815}}
- server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
- server._evidence_candidate_origin = "https://public.test"
- server._context_provider().browser_session_replacements = {"pbs_1": "pbs_replacement"}
-
- async def _failing_drain() -> None:
- _fake_clock[0] += _AFTER_CALL_SECONDS
- raise failure
-
- monkeypatch.setattr(server, "_drain_evidence_candidate_response_tasks", _failing_drain)
-
- with capture_logs() as captured:
- if isinstance(failure, asyncio.CancelledError):
- with pytest.raises(asyncio.CancelledError):
- await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
- else:
- await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
-
- record = _timing_records(captured)[0]
- assert record["call_status"] == call_status
- assert record["wall_clock_ms"] == _WALL_MS
- assert record["post_call_evidence_drain_ms"] == _AFTER_CALL_MS
- assert record["phase_residual_ms"] == 0
- assert record["timing_phase"] == "evidence_drain"
- _assert_every_millisecond_is_attributed(record)
-
- @pytest.mark.asyncio
- async def test_an_evidence_drain_that_succeeds_reports_what_the_caller_waited_for_it(
- self, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
- ) -> None:
- payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"total": 815}}
- server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
- server._evidence_candidate_origin = "https://public.test"
-
- async def _slow_drain() -> None:
- _fake_clock[0] += _AFTER_CALL_SECONDS
-
- monkeypatch.setattr(server, "_drain_evidence_candidate_response_tasks", _slow_drain)
-
- with capture_logs() as captured:
- await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
-
- record = _timing_records(captured)[0]
- assert record["call_status"] == "ok"
- assert record["post_call_evidence_drain_ms"] == _AFTER_CALL_MS
- assert record["wall_clock_ms"] == _WALL_MS
- _assert_every_millisecond_is_attributed(record)
-
@pytest.mark.asyncio
async def test_a_timing_sink_that_fails_costs_the_record_and_not_the_tool_result(
self, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index b5ac583dd..be0736cfd 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -1844,6 +1844,19 @@ class TestParseFinalResponse:
assert parsed["type"] == "REPLY"
assert parsed["user_response"] == "ok"
+ def test_a_draft_object_before_the_final_envelope_does_not_win(self) -> None:
+ text = (
+ '{"type": "REPLY", "user_response": "draft"}\n\nOn reflection:\n'
+ '{"type": "REPLY", "user_response": "final", "global_llm_context": {"user_goal": "x"}}'
+ )
+ parsed = parse_final_response(text)
+ assert parsed["user_response"] == "final"
+ assert parsed["global_llm_context"] == {"user_goal": "x"}
+
+ def test_an_object_nested_in_a_malformed_envelope_is_not_read_as_the_reply(self) -> None:
+ text = '{"type": "REPLY", "user_response": "answer", "global_llm_context": {"user_response": "context"},}'
+ assert parse_final_response(text)["user_response"] == "answer"
+
def test_pass_b_rejects_non_envelope_dict_in_prose(self) -> None:
text = 'I cannot help with {"foo": "bar"}'
parsed = parse_final_response(text)
diff --git a/tests/unit/test_copilot_question_url_authority.py b/tests/unit/test_copilot_question_url_authority.py
index 9253d6576..2def17a6e 100644
--- a/tests/unit/test_copilot_question_url_authority.py
+++ b/tests/unit/test_copilot_question_url_authority.py
@@ -22,7 +22,6 @@ from skyvern.forge.sdk.copilot.tools.credential_fill import (
_credential_fill_origin_grant,
_request_credential,
)
-from skyvern.forge.sdk.copilot.tools.discovery import _user_provided_entry_url
from skyvern.forge.sdk.routes.workflow_copilot import _make_error_narrative_payload, _persist_turn_messages
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -84,7 +83,6 @@ async def test_authenticated_question_url_reaches_the_real_card_and_rebuilds_fro
source = ctx.request_policy.user_site_url_sources[_LOGIN_URL]
assert source == QuestionResponseSiteURLSource(interaction_id=question["interaction_id"])
assert ctx.request_policy.user_provided_site_urls == [_LOGIN_URL]
- assert _user_provided_entry_url(ctx) == _LOGIN_URL
chat = await repo.get_workflow_copilot_chat_by_id("org", ctx.workflow_copilot_chat_id)
await _persist_turn_messages(
diff --git a/tests/unit/test_streaming_screenshot_local.py b/tests/unit/test_streaming_screenshot_local.py
index abe1926da..76f84d3de 100644
--- a/tests/unit/test_streaming_screenshot_local.py
+++ b/tests/unit/test_streaming_screenshot_local.py
@@ -47,6 +47,7 @@ async def test_run_local_screencast_happy_path(monkeypatch: pytest.MonkeyPatch)
check_finalized=check_finalized,
workflow_run_id="wr_123",
organization_id="org_123",
+ on_frame_sent=None,
)
get_current_status.assert_awaited_once()
send_status_mock.assert_awaited_once_with(websocket, "task_id", "task_123", "completed")
@@ -112,7 +113,9 @@ async def test_expired_browser_session_is_distinct_from_stream_launch_timeout(
"app",
SimpleNamespace(
PERSISTENT_SESSIONS_MANAGER=SimpleNamespace(
- get_session=AsyncMock(return_value=SimpleNamespace(status="timeout"))
+ get_session=AsyncMock(
+ return_value=SimpleNamespace(status="timeout", runnable_type=None, runnable_id=None)
+ )
)
),
)
@@ -138,8 +141,8 @@ async def test_browser_session_expiring_while_waiting_for_browser_state_is_disti
websocket = object()
get_session = AsyncMock(
side_effect=[
- SimpleNamespace(status="running"),
- SimpleNamespace(status="timeout"),
+ SimpleNamespace(status="running", runnable_type=None, runnable_id=None),
+ SimpleNamespace(status="timeout", runnable_type=None, runnable_id=None),
]
)
send_status_mock = AsyncMock()
@@ -169,7 +172,7 @@ async def test_browser_state_timeout_survives_status_refresh_failure(monkeypatch
websocket = object()
get_session = AsyncMock(
side_effect=[
- SimpleNamespace(status="running"),
+ SimpleNamespace(status="running", runnable_type=None, runnable_id=None),
RuntimeError("status unavailable"),
]
)
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 9afa29f62..87638fd03 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -28,6 +28,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
from structlog.testing import capture_logs
from skyvern.config import settings
+from skyvern.exceptions import WorkflowPinnedByRunGroup
from skyvern.forge import app
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import ArtifactType
@@ -637,6 +638,60 @@ async def test_finalise_normal_turn_applies_a_verified_proposal_with_auto_accept
assert persisted_payload["proposalDisposition"] == "auto_applicable"
+async def test_auto_accept_refused_by_a_running_group_leaves_the_draft_pending(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ chat = SimpleNamespace(
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ proposed_workflow=None,
+ auto_accept=True,
+ )
+ original_workflow = fake_workflow(workflow_id="wf-canonical")
+ updated_workflow = MagicMock()
+ updated_workflow.model_dump.return_value = {"workflow_id": "wf-draft"}
+ agent_result = AgentResult(
+ user_response="done",
+ updated_workflow=updated_workflow,
+ global_llm_context=None,
+ response_type="REPLY",
+ proposal_disposition="auto_applicable",
+ narrative_payload=_narrative_payload(),
+ staged_workflow=MagicMock(),
+ has_staged_proposal=True,
+ )
+ restore, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ monkeypatch.setattr(
+ workflow_copilot_route,
+ "_commit_staged_workflow",
+ AsyncMock(side_effect=WorkflowPinnedByRunGroup("wf-canonical")),
+ )
+ stream = MagicMock(send=AsyncMock(return_value=True))
+
+ await workflow_copilot_route._finalise_normal_turn(
+ stream=stream,
+ chat=chat,
+ organization_id="org-1",
+ original_workflow=original_workflow,
+ chat_request=_make_chat_request(),
+ agent_result=agent_result,
+ )
+
+ response_frame = stream.send.await_args.args[0]
+ assert response_frame.workflow_applied is False
+ assert response_frame.proposal_disposition == "review_tested"
+ assert workflow_copilot_route._AUTO_ACCEPT_DEFERRED_FOR_RUN_GROUP in response_frame.message
+ restore.assert_not_awaited()
+ pending = [
+ call.kwargs["proposed_workflow"]
+ for call in workflow_params.update_workflow_copilot_chat.await_args_list
+ if "proposed_workflow" in call.kwargs
+ ]
+ assert pending and pending[-1] is not None
+
+
@pytest.mark.asyncio
async def test_finalise_normal_turn_is_not_applied_without_a_proposal_on_auto_accept(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_workflow_run_groups.py b/tests/unit/test_workflow_run_groups.py
index 733e6625d..2fbde2059 100644
--- a/tests/unit/test_workflow_run_groups.py
+++ b/tests/unit/test_workflow_run_groups.py
@@ -1,19 +1,24 @@
import asyncio
-from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
-from dataclasses import dataclass, field
+import contextlib
+from collections.abc import Mapping, Sequence
from datetime import timedelta
from typing import Any
from unittest.mock import AsyncMock, MagicMock
+from uuid import uuid4
import pytest
-import pytest_asyncio
from fastapi import HTTPException
from pydantic import ValidationError
-from sqlalchemy import func, select, update
-from sqlalchemy.ext.asyncio import AsyncEngine
+from sqlalchemy import select, update
+from sqlalchemy.engine import make_url
from skyvern.config import settings
-from skyvern.exceptions import SkyvernHTTPException, WorkflowChangedSinceReview, WorkflowPinnedByRunGroup
+from skyvern.exceptions import (
+ GroupAccountsRanSinceReview,
+ SkyvernHTTPException,
+ WorkflowChangedSinceReview,
+ WorkflowPinnedByRunGroup,
+)
from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.core import skyvern_context
@@ -21,21 +26,17 @@ from skyvern.forge.sdk.core.permissions.permission_checkers import PermissionChe
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
from skyvern.forge.sdk.db.models import (
- Base,
- CredentialModel,
WorkflowModel,
WorkflowRunGroupItemModel,
WorkflowRunGroupModel,
WorkflowRunModel,
)
-from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
+from skyvern.forge.sdk.db.repositories import workflow_run_groups as workflow_run_groups_repository
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.runs import Run
from skyvern.forge.sdk.workflow.models.block import (
- BlockTypeVar,
ForLoopBlock,
HttpRequestBlock,
- TaskBlock,
WorkflowTriggerBlock,
)
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType
@@ -54,53 +55,16 @@ from skyvern.schemas.workflow_run_groups import (
WorkflowRunGroupStatus,
)
from skyvern.services import workflow_run_group_service as group_service
-from tests.unit.conftest import make_block_output_parameter
-
-ORG = "o_test"
-OTHER_ORG = "o_other"
-WPID = "wpid_test"
-
-
-@dataclass
-class FakeExecutor:
- database: AgentDB
- executed: list[str] = field(default_factory=list)
- submitted: list[str] = field(default_factory=list)
- before_queue: Callable[[str], Awaitable[None]] | None = None
-
- async def execute_workflow(self, *, workflow_run_id: str, **_: object) -> None:
- self.executed.append(workflow_run_id)
- if self.before_queue is not None:
- await self.before_queue(workflow_run_id)
- if await self.database.workflow_runs.update_workflow_run_if_not_final(
- workflow_run_id, WorkflowRunStatus.queued
- ):
- self.submitted.append(workflow_run_id)
-
-
-@dataclass
-class RecordingRateLimiter:
- calls: list[str] = field(default_factory=list)
-
- async def rate_limit_submit_run(self, organization_id: str) -> None:
- self.calls.append(organization_id)
-
-
-@dataclass
-class GroupEnv:
- database: AgentDB
- executor: FakeExecutor
- organization: Organization
- spawned: list[Coroutine[Any, Any, None]]
- limiter: RecordingRateLimiter
-
-
-def _definition(*blocks: BlockTypeVar) -> dict[str, Any]:
- return WorkflowDefinition(parameters=[], blocks=list(blocks)).model_dump(mode="json")
-
-
-def _task_block() -> TaskBlock:
- return TaskBlock(label="login", url="https://example.com", output_parameter=make_block_output_parameter("login"))
+from tests.unit.conftest import RUN_GROUP_ORG as ORG
+from tests.unit.conftest import RUN_GROUP_OTHER_ORG as OTHER_ORG
+from tests.unit.conftest import RUN_GROUP_WPID as WPID
+from tests.unit.conftest import (
+ GroupEnv,
+ count_rows,
+ make_block_output_parameter,
+ run_group_definition,
+ run_group_task_block,
+)
def _looped_trigger(browser_session_id: str | None = None, wait_for_completion: bool = True) -> ForLoopBlock:
@@ -120,53 +84,9 @@ def _http_block() -> HttpRequestBlock:
)
-@pytest_asyncio.fixture
-async def env(monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine) -> AsyncIterator[GroupEnv]:
- database = AgentDB("sqlite+aiosqlite://", db_engine=sqlite_engine)
- organization = await database.organizations.create_organization("Test", organization_id=ORG)
- await database.organizations.create_organization("Other", organization_id=OTHER_ORG)
- async with database.Session() as session:
- session.add(
- WorkflowModel(
- workflow_id="wf_1",
- workflow_permanent_id=WPID,
- organization_id=ORG,
- title="Workflow",
- version=1,
- workflow_definition=_definition(_task_block()),
- )
- )
- session.add_all(
- CredentialModel(
- credential_id=credential_id,
- organization_id=org_id,
- name="Login",
- credential_type="password",
- item_id=f"item_{credential_id}",
- )
- for credential_id, org_id in (("cred_1", ORG), ("cred_2", ORG), ("cred_foreign", OTHER_ORG))
- )
- await session.commit()
- await database.workflow_params.create_workflow_parameter(
- workflow_id="wf_1", workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID, key="login", default_value=None
- )
- service = WorkflowService()
- executor = FakeExecutor(database)
- spawned: list[Coroutine[Any, Any, None]] = []
- limiter = RecordingRateLimiter()
- monkeypatch.setattr(app, "DATABASE", database)
- monkeypatch.setattr(object.__getattribute__(app, "_inst"), "RATE_LIMITER", limiter, raising=False)
- monkeypatch.setattr(app, "WORKFLOW_SERVICE", service)
- monkeypatch.setattr(service, "_resolve_managed_browser_profile_for_run_request", AsyncMock(return_value=None))
- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False))
- monkeypatch.setattr(app.AGENT_FUNCTION, "is_block_scoped_workflow_run", AsyncMock(return_value=False))
- monkeypatch.setattr(AsyncExecutorFactory, "get_executor", lambda: executor)
- monkeypatch.setattr(group_service, "_spawn", spawned.append)
- monkeypatch.setattr(service, "_schedule_workflow_run_terminal_hooks", lambda **_: None)
- yield GroupEnv(database, executor, organization, spawned, limiter)
- for coroutine in spawned:
- coroutine.close()
- await asyncio.gather(*app.WORKFLOW_SERVICE._background_tasks, return_exceptions=True)
+def env(run_group_env: GroupEnv) -> GroupEnv:
+ return run_group_env
def _request(
@@ -177,11 +97,6 @@ def _request(
return WorkflowRunGroupCreateRequest.model_validate({"workflow_id": WPID, "submission_key": key, "items": items})
-async def _count(env: GroupEnv, model: type[Base]) -> int:
- async with env.database.Session() as session:
- return int(await session.scalar(select(func.count()).select_from(model)) or 0)
-
-
async def _set_child_status(env: GroupEnv, workflow_run_id: str, status: WorkflowRunStatus) -> None:
await env.database.workflow_runs.update_workflow_run(workflow_run_id, status=status)
@@ -223,14 +138,14 @@ async def test_submit_preallocates_ids_without_child_rows_then_dispatches_one(en
run_ids = [item.workflow_run_id for item in response.items]
assert len(set(run_ids)) == 25
assert response.workflow_id == "wf_1"
- assert await _count(env, WorkflowRunModel) == 0
+ assert await count_rows(env, WorkflowRunModel) == 0
await group_service.advance_workflow_run_group(response.workflow_run_group_id)
assert env.executor.executed == [run_ids[0]]
child = await env.database.workflow_runs.get_workflow_run(run_ids[0])
assert child is not None and child.workflow_id == "wf_1" and child.start_fresh_browser is True
- assert await _count(env, WorkflowRunModel) == 1
+ assert await count_rows(env, WorkflowRunModel) == 1
group = await group_service.get_workflow_run_group(response.workflow_run_group_id, ORG)
assert [item.workflow_run_id for item in group.items] == run_ids
@@ -296,9 +211,9 @@ async def test_one_bad_credential_rejects_the_whole_group(env: GroupEnv, bad_cre
await group_service.submit_workflow_run_group(env.organization, _request(items=items))
assert 400 <= exc_info.value.status_code < 500
- assert await _count(env, WorkflowRunGroupModel) == 0
- assert await _count(env, WorkflowRunGroupItemModel) == 0
- assert await _count(env, WorkflowRunModel) == 0
+ assert await count_rows(env, WorkflowRunGroupModel) == 0
+ assert await count_rows(env, WorkflowRunGroupItemModel) == 0
+ assert await count_rows(env, WorkflowRunModel) == 0
assert env.spawned == [] and env.executor.executed == []
@@ -315,13 +230,13 @@ async def test_a_mistyped_value_in_any_item_rejects_the_whole_group(env: GroupEn
if count.isdigit():
await group_service.submit_workflow_run_group(env.organization, _request(items=items))
- assert await _count(env, WorkflowRunGroupItemModel) == 2
+ assert await count_rows(env, WorkflowRunGroupItemModel) == 2
return
with pytest.raises(SkyvernHTTPException) as exc_info:
await group_service.submit_workflow_run_group(env.organization, _request(items=items))
assert exc_info.value.status_code == 400
- assert await _count(env, WorkflowRunGroupModel) == 0
- assert await _count(env, WorkflowRunGroupItemModel) == 0
+ assert await count_rows(env, WorkflowRunGroupModel) == 0
+ assert await count_rows(env, WorkflowRunGroupItemModel) == 0
@pytest.mark.asyncio
@@ -335,7 +250,7 @@ async def test_identical_retry_after_workflow_saves_returns_the_same_group(env:
title="Workflow",
version=2,
created_by=COPILOT_TEST_WORKFLOW_CREATOR,
- workflow_definition=_definition(_task_block()),
+ workflow_definition=run_group_definition(run_group_task_block()),
)
)
await session.commit()
@@ -350,7 +265,7 @@ async def test_identical_retry_after_workflow_saves_returns_the_same_group(env:
organization_id=ORG,
title="Workflow",
version=3,
- workflow_definition=_definition(),
+ workflow_definition=run_group_definition(),
)
)
await session.commit()
@@ -359,7 +274,7 @@ async def test_identical_retry_after_workflow_saves_returns_the_same_group(env:
assert retry.workflow_run_group_id == first.workflow_run_group_id
assert retry.workflow_id == "wf_1"
assert [item.workflow_run_id for item in retry.items] == [item.workflow_run_id for item in first.items]
- assert await _count(env, WorkflowRunGroupModel) == 1
+ assert await count_rows(env, WorkflowRunGroupModel) == 1
@pytest.mark.asyncio
@@ -375,7 +290,7 @@ async def test_identical_retry_with_submission_disabled_returns_the_same_group(
assert retry.workflow_run_group_id == first.workflow_run_group_id
assert exc_info.value.status_code == 503
- assert await _count(env, WorkflowRunGroupModel) == 1
+ assert await count_rows(env, WorkflowRunGroupModel) == 1
@pytest.mark.asyncio
@@ -397,7 +312,7 @@ async def test_conflicting_input_for_a_submission_key_is_rejected(
await group_service.submit_workflow_run_group(env.organization, changed)
assert exc_info.value.status_code == 409
- assert await _count(env, WorkflowRunGroupModel) == 1
+ assert await count_rows(env, WorkflowRunGroupModel) == 1
assert await _run_ids(env, group_id) == run_ids
assert len(env.spawned) == 1
@@ -410,8 +325,8 @@ async def test_concurrent_identical_submissions_share_one_group(env: GroupEnv) -
assert len({response.workflow_run_group_id for response in responses}) == 1
assert len({tuple(item.workflow_run_id for item in response.items) for response in responses}) == 1
- assert await _count(env, WorkflowRunGroupModel) == 1
- assert await _count(env, WorkflowRunGroupItemModel) == 3
+ assert await count_rows(env, WorkflowRunGroupModel) == 1
+ assert await count_rows(env, WorkflowRunGroupItemModel) == 3
@pytest.mark.asyncio
@@ -424,7 +339,7 @@ async def test_concurrent_recoveries_of_a_stale_claim_execute_once(env: GroupEnv
await asyncio.gather(group_service.recover_workflow_run_groups(), group_service.recover_workflow_run_groups())
assert env.executor.executed == [run_ids[0]]
- assert await _count(env, WorkflowRunModel) == 1
+ assert await count_rows(env, WorkflowRunModel) == 1
@pytest.mark.asyncio
@@ -445,7 +360,7 @@ async def test_slow_owner_cannot_execute_after_recovery_reclaims_its_item(env: G
assert await group_service._dispatch_item(group, reclaimed, "new-owner") is True
assert env.executor.executed == [run_ids[0]]
- assert await _count(env, WorkflowRunModel) == 1
+ assert await count_rows(env, WorkflowRunModel) == 1
@pytest.mark.asyncio
@@ -478,7 +393,7 @@ async def test_stale_owner_interrupted_prep_leaves_the_new_owners_child_alone(
assert status_after_stale_owner == [WorkflowRunStatus.created]
assert env.executor.executed == [run_ids[0]]
assert (await _states(env, group_id))[0] == WorkflowRunGroupItemState.dispatched
- assert await _count(env, WorkflowRunModel) == 1
+ assert await count_rows(env, WorkflowRunModel) == 1
@pytest.mark.asyncio
@@ -495,7 +410,7 @@ async def test_crash_between_child_row_and_task_run_fails_that_item_and_runs_the
group = await group_service.get_workflow_run_group(group_id, ORG)
assert group.items[0].state == WorkflowRunGroupItemState.failed_to_start
assert group.items[0].outcome == WorkflowRunGroupItemOutcome.failed
- assert await _count(env, WorkflowRunModel) == 2
+ assert await count_rows(env, WorkflowRunModel) == 2
class RefusingPermissionChecker(PermissionChecker):
@@ -514,7 +429,7 @@ async def test_org_refused_after_submit_starts_no_remaining_children(
assert env.executor.executed == []
assert await _states(env, group_id) == [WorkflowRunGroupItemState.failed_to_start] * 3
- assert await _count(env, WorkflowRunModel) == 0
+ assert await count_rows(env, WorkflowRunModel) == 0
@pytest.mark.asyncio
@@ -603,7 +518,7 @@ async def test_child_cancel_between_flip_and_queued_write(env: GroupEnv) -> None
group = await group_service.get_workflow_run_group(group_id, ORG)
assert group.status == WorkflowRunGroupStatus.finished
assert [item.outcome for item in group.items] == [WorkflowRunGroupItemOutcome.canceled] * 3
- assert await _count(env, WorkflowRunModel) == 1
+ assert await count_rows(env, WorkflowRunModel) == 1
@pytest.mark.asyncio
@@ -629,7 +544,7 @@ async def test_in_place_edit_of_a_pinned_version_is_refused_until_the_group_fini
for edit in (
{"title": "Edited"},
- {"workflow_definition": WorkflowDefinition(parameters=[], blocks=[_task_block()])},
+ {"workflow_definition": WorkflowDefinition(parameters=[], blocks=[run_group_task_block()])},
):
with pytest.raises(WorkflowPinnedByRunGroup) as refused:
await app.WORKFLOW_SERVICE.update_workflow_definition(workflow_id="wf_1", organization_id=ORG, **edit)
@@ -658,7 +573,7 @@ async def test_submit_bound_to_a_reviewed_version_refuses_an_edit_that_landed_af
)
assert refused.value.status_code == 409
- assert await _count(env, WorkflowRunGroupModel) == 0
+ assert await count_rows(env, WorkflowRunGroupModel) == 0
assert env.spawned == []
current = await env.database.workflows.get_workflow(workflow_id="wf_1", organization_id=ORG)
assert current is not None
@@ -765,7 +680,9 @@ async def test_failure_after_possible_side_effects_is_unknown_and_not_replayed(e
if evidence == "http_request_block":
async with env.database.Session() as session:
await session.execute(
- update(WorkflowModel).values(workflow_definition=_definition(_task_block(), _http_block()))
+ update(WorkflowModel).values(
+ workflow_definition=run_group_definition(run_group_task_block(), _http_block())
+ )
)
await session.commit()
group_id = await _submit(env, _request(count=1))
@@ -792,7 +709,7 @@ async def test_failure_after_possible_side_effects_is_unknown_and_not_replayed(e
group = await group_service.get_workflow_run_group(group_id, ORG)
assert group.items[0].outcome == WorkflowRunGroupItemOutcome.unknown
assert env.executor.executed == run_ids
- assert await _count(env, WorkflowRunModel) == 1
+ assert await count_rows(env, WorkflowRunModel) == 1
@pytest.mark.asyncio
@@ -831,15 +748,28 @@ def test_browser_session_inputs_are_rejected() -> None:
@pytest.mark.parametrize(
"workflow_values",
[
- {"workflow_definition": {**_definition(_task_block()), "retry_policy": {"retry_on": [{"status": "failed"}]}}},
+ {
+ "workflow_definition": {
+ **run_group_definition(run_group_task_block()),
+ "retry_policy": {"retry_on": [{"status": "failed"}]},
+ }
+ },
{"persist_browser_session": True},
{"browser_profile_id": "bp_1"},
{"sequential_key": "{{ login }}"},
- {"workflow_definition": _definition()},
+ {"workflow_definition": run_group_definition()},
{"browser_profile_key": "profile-{{ login }}"},
- {"workflow_definition": _definition(_task_block(), _looped_trigger(browser_session_id="{{ session_id }}"))},
- {"workflow_definition": _definition(_task_block(), _looped_trigger(wait_for_completion=False))},
- {"workflow_definition": {**_definition(_task_block()), "finally_block_label": "login"}},
+ {
+ "workflow_definition": run_group_definition(
+ run_group_task_block(), _looped_trigger(browser_session_id="{{ session_id }}")
+ )
+ },
+ {
+ "workflow_definition": run_group_definition(
+ run_group_task_block(), _looped_trigger(wait_for_completion=False)
+ )
+ },
+ {"workflow_definition": {**run_group_definition(run_group_task_block()), "finally_block_label": "login"}},
],
)
async def test_versions_a_group_cannot_run_are_rejected(env: GroupEnv, workflow_values: dict[str, object]) -> None:
@@ -851,8 +781,8 @@ async def test_versions_a_group_cannot_run_are_rejected(env: GroupEnv, workflow_
await _submit(env)
assert exc_info.value.status_code == 400
- assert await _count(env, WorkflowRunGroupModel) == 0
- assert await _count(env, WorkflowRunGroupItemModel) == 0
+ assert await count_rows(env, WorkflowRunGroupModel) == 0
+ assert await count_rows(env, WorkflowRunGroupItemModel) == 0
@pytest.mark.asyncio
@@ -1074,7 +1004,7 @@ async def test_scrubbed_group_keeps_no_caller_keys_and_a_replay_of_its_key_confl
with pytest.raises(SkyvernHTTPException) as exc_info:
await _submit(env)
assert exc_info.value.status_code == 409
- assert await _count(env, WorkflowRunGroupModel) == 1
+ assert await count_rows(env, WorkflowRunGroupModel) == 1
with pytest.raises(ValidationError):
_request(key=stored[0])
with pytest.raises(ValidationError):
@@ -1101,7 +1031,7 @@ async def test_an_edit_landing_between_validation_and_insert_refuses_the_submit(
await _submit(env)
assert exc_info.value.status_code == 409
- assert await _count(env, WorkflowRunGroupModel) == 0
+ assert await count_rows(env, WorkflowRunGroupModel) == 0
@pytest.mark.asyncio
@@ -1198,7 +1128,7 @@ async def test_unfresh_trigger_under_a_group_child_fails_before_creating_a_run(
if supplies_session
else None
)
- runs_before = await _count(env, WorkflowRunModel)
+ runs_before = await count_rows(env, WorkflowRunModel)
block = WorkflowTriggerBlock(
label="detached",
workflow_permanent_id=WPID,
@@ -1215,4 +1145,65 @@ async def test_unfresh_trigger_under_a_group_child_fails_before_creating_a_run(
await block.execute(workflow_run_id=trigger_run_id, workflow_run_block_id="wrb_detached", organization_id=ORG)
assert result.await_args is not None and result.await_args.kwargs["success"] is False
- assert await _count(env, WorkflowRunModel) == runs_before
+ assert await count_rows(env, WorkflowRunModel) == runs_before
+
+
+async def test_two_concurrent_repeat_checked_creations_on_postgres_admit_one_group(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ if make_url(str(settings.DATABASE_STRING)).get_backend_name() != "postgresql":
+ pytest.skip("requires PostgreSQL row and advisory locks")
+ read_latest = workflow_run_groups_repository._latest_group_ids_by_item_key
+ reads: list[None] = []
+ both_read = asyncio.Event()
+
+ # Holds each creation after its history read, so without the locks both read before either inserts.
+ async def read_then_wait_for_the_other(*args: Any) -> dict[str, str]:
+ latest = await read_latest(*args)
+ reads.append(None)
+ if len(reads) >= 2:
+ both_read.set()
+ else:
+ with contextlib.suppress(TimeoutError):
+ await asyncio.wait_for(both_read.wait(), 0.5)
+ return latest
+
+ monkeypatch.setattr(workflow_run_groups_repository, "_latest_group_ids_by_item_key", read_then_wait_for_the_other)
+ database = AgentDB(str(settings.DATABASE_STRING))
+ organization = await database.organizations.create_organization(organization_name=f"race-{uuid4().hex}")
+ workflow_id, wpid = f"wf_{uuid4().hex}", f"wpid_{uuid4().hex}"
+ async with database.Session() as session:
+ session.add(
+ WorkflowModel(
+ workflow_id=workflow_id,
+ workflow_permanent_id=wpid,
+ organization_id=organization.organization_id,
+ title="Workflow",
+ version=1,
+ workflow_definition=run_group_definition(run_group_task_block()),
+ )
+ )
+ await session.commit()
+
+ async def create(label: str) -> object:
+ try:
+ return await database.workflow_run_groups.create_group(
+ organization_id=organization.organization_id,
+ workflow_permanent_id=wpid,
+ requested_version=1,
+ workflow_id=workflow_id,
+ submission_key=f"copilot:{label}:key",
+ input_fingerprint=label,
+ items=[("cred_race", {})],
+ expected_latest_groups=("copilot:", {}),
+ )
+ except GroupAccountsRanSinceReview as e:
+ return e
+
+ try:
+ results = await asyncio.gather(create("a"), create("b"))
+ finally:
+ await database.engine.dispose()
+
+ assert sum(isinstance(result, GroupAccountsRanSinceReview) for result in results) == 1