Skip to content

Automate browser based workflows with AI

refactor: split copilot tools.py (9.5k lines) into a tools/ package (#6493)

Andrew Neilsonb69930e0ed29
diff --git a/alembic/versions/2026_06_10_1800-f3a9c2b7e1d4_add_generate_browser_profile_to_persistent_browser_sessions.py b/alembic/versions/2026_06_10_1800-f3a9c2b7e1d4_add_generate_browser_profile_to_persistent_browser_sessions.py
new file mode 100644
index 000000000..c83372f53
--- /dev/null
+++ b/alembic/versions/2026_06_10_1800-f3a9c2b7e1d4_add_generate_browser_profile_to_persistent_browser_sessions.py
@@ -0,0 +1,30 @@
+"""add generate_browser_profile to persistent_browser_sessions
+
+Revision ID: f3a9c2b7e1d4
+Revises: 4dbd183edee0
+Create Date: 2026-06-10T18:00:00.000000+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "f3a9c2b7e1d4"
+down_revision: Union[str, None] = "4dbd183edee0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "persistent_browser_sessions",
+ sa.Column("generate_browser_profile", sa.Boolean(), nullable=False, server_default=sa.false()),
+ )
+
+
+def downgrade() -> None:
+ op.drop_column("persistent_browser_sessions", "generate_browser_profile")
diff --git a/docs/changelog.mdx b/docs/changelog.mdx
index 40ddb7ca8..3a7f0bd69 100644
--- a/docs/changelog.mdx
+++ b/docs/changelog.mdx
@@ -44,13 +44,14 @@ keywords:
<p className="cl-subtitle">New features, improvements, and fixes in Skyvern</p>
<a id="week-jun-8-2026"></a>
-<Update label="Week of June 8, 2026" description="Copilot modes, GCS storage, API docs split, OTP fixes">
+<Update label="Week of June 8, 2026" description="Editor onboarding, Copilot modes, GCS storage, API docs split, OTP fixes">
## New features
- **Copilot Ask/Build/Code Mode Routing** — The Copilot now routes each request based on the active mode: Ask returns answers without modifying the workflow, Build creates or updates workflows, and Code generates executable code blocks. ([#6459](https://github.com/Skyvern-AI/skyvern/pull/6459))
- **GCS Storage Backend** — Added Google Cloud Storage as an artifact storage backend for self-hosted deployments. ([#6442](https://github.com/Skyvern-AI/skyvern/pull/6442))
- **Key-Optional Labels and Flexible Tag Filters** — Tags now support key-optional labels, and tag filter queries accept more flexible formats for richer agent organization. ([#6458](https://github.com/Skyvern-AI/skyvern/pull/6458))
+- **Self-Serve Editor Onboarding** — New users in the workflow editor now see a get-started modal with intent routing and template cards, a 4-stop guided product tour, and smart empty states on the dashboard and runs pages to accelerate time-to-first-automation. ([#6475](https://github.com/Skyvern-AI/skyvern/pull/6475))
## Improvements
@@ -62,12 +63,15 @@ keywords:
- **Accessible Status Badges** — Status badges across the product have been updated with improved contrast and consistent pill styling for better accessibility. ([#6444](https://github.com/Skyvern-AI/skyvern/pull/6444))
- **Block Title Editable from Settings Sidebar** — Workflow block titles can now be edited from the right-hand settings sidebar, without needing to use the inline editor on the canvas. ([#6443](https://github.com/Skyvern-AI/skyvern/pull/6443))
- **API Reference Split into Agents and Runs Sections** — The public API reference is now organized into separate Agents and Runs sections for easier navigation. ([#6448](https://github.com/Skyvern-AI/skyvern/pull/6448))
+- **Label-First Tag Editor and Flexible Tag Filter UI** — The tag editor now accepts standalone labels or `group:label` format in a single input, and the filter bar supports filtering by label, group, or group:label for richer agent organization. ([#6470](https://github.com/Skyvern-AI/skyvern/pull/6470))
## Bug fixes
- Fixed org-verified discovered credentials not being credited in the Copilot credential gate, which could prevent runs from starting. ([#6462](https://github.com/Skyvern-AI/skyvern/pull/6462))
- Fixed OTP codes from previous workflow runs being picked up and incorrectly used in new runs. ([#6455](https://github.com/Skyvern-AI/skyvern/pull/6455))
- Fixed persistent-session screen recordings not appearing in the run view when the recording timestamp fell outside the run's time window. ([#6454](https://github.com/Skyvern-AI/skyvern/pull/6454))
+- Fixed Copilot inconsistently asking for a URL before running discovery when the user had only named a site without providing one. ([#6473](https://github.com/Skyvern-AI/skyvern/pull/6473))
+- Fixed Copilot asking for a URL after already successfully navigating to the requested site during discovery. ([#6472](https://github.com/Skyvern-AI/skyvern/pull/6472))
</Update>
diff --git a/docs/pr-screenshots/webhook-delivery-pending.png b/docs/pr-screenshots/webhook-delivery-pending.png
new file mode 100644
index 000000000..7725ea646
Binary files /dev/null and b/docs/pr-screenshots/webhook-delivery-pending.png differ
diff --git a/docs/pr-screenshots/webhook-failure-reason.png b/docs/pr-screenshots/webhook-failure-reason.png
new file mode 100644
index 000000000..d83f4108c
Binary files /dev/null and b/docs/pr-screenshots/webhook-failure-reason.png differ
diff --git a/docs/screenshots/sky-10794-2fa-login-block/00-before-always-shown.png b/docs/screenshots/sky-10794-2fa-login-block/00-before-always-shown.png
new file mode 100644
index 000000000..8298cd597
Binary files /dev/null and b/docs/screenshots/sky-10794-2fa-login-block/00-before-always-shown.png differ
diff --git a/docs/screenshots/sky-10794-2fa-login-block/01-after-collapsed.png b/docs/screenshots/sky-10794-2fa-login-block/01-after-collapsed.png
new file mode 100644
index 000000000..511358f00
Binary files /dev/null and b/docs/screenshots/sky-10794-2fa-login-block/01-after-collapsed.png differ
diff --git a/docs/screenshots/sky-10794-2fa-login-block/02-after-expanded.png b/docs/screenshots/sky-10794-2fa-login-block/02-after-expanded.png
new file mode 100644
index 000000000..7553cea46
Binary files /dev/null and b/docs/screenshots/sky-10794-2fa-login-block/02-after-expanded.png differ
diff --git a/docs/screenshots/sky-10794-2fa-login-block/03-after-saved-values.png b/docs/screenshots/sky-10794-2fa-login-block/03-after-saved-values.png
new file mode 100644
index 000000000..c908dafe5
Binary files /dev/null and b/docs/screenshots/sky-10794-2fa-login-block/03-after-saved-values.png differ
diff --git a/docs/screenshots/sky-10794-2fa-login-block/04-after-credential-summary.png b/docs/screenshots/sky-10794-2fa-login-block/04-after-credential-summary.png
new file mode 100644
index 000000000..a29e977b1
Binary files /dev/null and b/docs/screenshots/sky-10794-2fa-login-block/04-after-credential-summary.png differ
diff --git a/fern/docs/changelog.mdx b/fern/docs/changelog.mdx
index e5328b4a8..a52115aea 100644
--- a/fern/docs/changelog.mdx
+++ b/fern/docs/changelog.mdx
@@ -29,13 +29,14 @@ mode: custom
<h1 className="cl-title">Changelog</h1>
<p className="cl-subtitle">New features, improvements, and fixes in Skyvern</p>
-<Update label="Week of June 8, 2026" description="Copilot modes, GCS storage, API docs split, OTP fixes">
+<Update label="Week of June 8, 2026" description="Editor onboarding, Copilot modes, GCS storage, API docs split, OTP fixes">
## New features
- **Copilot Ask/Build/Code Mode Routing** — The Copilot now routes each request based on the active mode: Ask returns answers without modifying the workflow, Build creates or updates workflows, and Code generates executable code blocks. ([#6459](https://github.com/Skyvern-AI/skyvern/pull/6459))
- **GCS Storage Backend** — Added Google Cloud Storage as an artifact storage backend for self-hosted deployments. ([#6442](https://github.com/Skyvern-AI/skyvern/pull/6442))
- **Key-Optional Labels and Flexible Tag Filters** — Tags now support key-optional labels, and tag filter queries accept more flexible formats for richer agent organization. ([#6458](https://github.com/Skyvern-AI/skyvern/pull/6458))
+- **Self-Serve Editor Onboarding** — New users in the workflow editor now see a get-started modal with intent routing and template cards, a 4-stop guided product tour, and smart empty states on the dashboard and runs pages to accelerate time-to-first-automation. ([#6475](https://github.com/Skyvern-AI/skyvern/pull/6475))
## Improvements
@@ -47,12 +48,15 @@ mode: custom
- **Accessible Status Badges** — Status badges across the product have been updated with improved contrast and consistent pill styling for better accessibility. ([#6444](https://github.com/Skyvern-AI/skyvern/pull/6444))
- **Block Title Editable from Settings Sidebar** — Workflow block titles can now be edited from the right-hand settings sidebar, without needing to use the inline editor on the canvas. ([#6443](https://github.com/Skyvern-AI/skyvern/pull/6443))
- **API Reference Split into Agents and Runs Sections** — The public API reference is now organized into separate Agents and Runs sections for easier navigation. ([#6448](https://github.com/Skyvern-AI/skyvern/pull/6448))
+- **Label-First Tag Editor and Flexible Tag Filter UI** — The tag editor now accepts standalone labels or `group:label` format in a single input, and the filter bar supports filtering by label, group, or group:label for richer agent organization. ([#6470](https://github.com/Skyvern-AI/skyvern/pull/6470))
## Bug fixes
- Fixed org-verified discovered credentials not being credited in the Copilot credential gate, which could prevent runs from starting. ([#6462](https://github.com/Skyvern-AI/skyvern/pull/6462))
- Fixed OTP codes from previous workflow runs being picked up and incorrectly used in new runs. ([#6455](https://github.com/Skyvern-AI/skyvern/pull/6455))
- Fixed persistent-session screen recordings not appearing in the run view when the recording timestamp fell outside the run's time window. ([#6454](https://github.com/Skyvern-AI/skyvern/pull/6454))
+- Fixed Copilot inconsistently asking for a URL before running discovery when the user had only named a site without providing one. ([#6473](https://github.com/Skyvern-AI/skyvern/pull/6473))
+- Fixed Copilot asking for a URL after already successfully navigating to the requested site during discovery. ([#6472](https://github.com/Skyvern-AI/skyvern/pull/6472))
</Update>
diff --git a/pyproject.toml b/pyproject.toml
index 7186ef7c2..f81212fec 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -71,6 +71,7 @@ local = [
"pandas>=2.3.1,<4",
"azure-identity>=1.24.0,<2",
"azure-keyvault-secrets>=4.2.0,<5",
+ "google-cloud-secret-manager>=2.29,<3",
"jsonschema>=4.23.0",
"python-calamine>=0.6.1",
"python-docx>=1.1.0",
@@ -147,6 +148,7 @@ server = [
"pandas>=2.3.1,<4",
"azure-identity>=1.24.0,<2",
"azure-keyvault-secrets>=4.2.0,<5",
+ "google-cloud-secret-manager>=2.29,<3",
"jsonschema>=4.23.0",
"python-calamine>=0.6.1",
"python-docx>=1.1.0",
diff --git a/skyvern-frontend/package-lock.json b/skyvern-frontend/package-lock.json
index 926406d67..492b006eb 100644
--- a/skyvern-frontend/package-lock.json
+++ b/skyvern-frontend/package-lock.json
@@ -8,6 +8,7 @@
"name": "skyvern-frontend",
"version": "0.0.0",
"dependencies": {
+ "@clerk/clerk-react": "^5.61.8",
"@codemirror/lang-html": "^6.4.9",
"@codemirror/lang-json": "^6.0.1",
"@codemirror/lang-python": "^6.1.6",
@@ -50,6 +51,7 @@
"cron-parser": "^5.5.0",
"cronstrue": "^3.13.0",
"cross-spawn": "^7.0.6",
+ "driver.js": "^1.4.0",
"embla-carousel-react": "^8.0.0",
"express": "^4.21.2",
"fetch-to-curl": "^0.6.0",
@@ -210,6 +212,65 @@
"specificity": "bin/cli.js"
}
},
+ "node_modules/@clerk/clerk-react": {
+ "version": "5.61.8",
+ "resolved": "https://registry.npmjs.org/@clerk/clerk-react/-/clerk-react-5.61.8.tgz",
+ "integrity": "sha512-n+k3q3xeyDkIPGTVA1J4Pd0+6MbS9Ia04qNlecOztTHwFfcirO5hy4TpOXrpGnO+GzYBuUMp7pYc3//ybMdEfg==",
+ "license": "MIT",
+ "dependencies": {
+ "@clerk/shared": "^3.47.7",
+ "tslib": "2.8.1"
+ },
+ "engines": {
+ "node": ">=18.17.0"
+ },
+ "peerDependencies": {
+ "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0",
+ "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0"
+ }
+ },
+ "node_modules/@clerk/shared": {
+ "version": "3.47.7",
+ "resolved": "https://registry.npmjs.org/@clerk/shared/-/shared-3.47.7.tgz",
+ "integrity": "sha512-9Yv4MJFEaC7BzV0whxa4txQ4SoMu/3j1LBnI85EBykb5CcfXxIKvNX/9sjMUUySHlTOjsj7XZa5i3W5Dx02K/Q==",
+ "hasInstallScript": true,
+ "license": "MIT",
+ "dependencies": {
+ "csstype": "3.1.3",
+ "dequal": "2.0.3",
+ "glob-to-regexp": "0.4.1",
+ "js-cookie": "3.0.7",
+ "std-env": "^3.9.0",
+ "swr": "2.3.4"
+ },
+ "engines": {
+ "node": ">=18.17.0"
+ },
+ "peerDependencies": {
+ "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0",
+ "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0"
+ },
+ "peerDependenciesMeta": {
+ "react": {
+ "optional": true
+ },
+ "react-dom": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/@clerk/shared/node_modules/csstype": {
+ "version": "3.1.3",
+ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
+ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
+ "license": "MIT"
+ },
+ "node_modules/@clerk/shared/node_modules/std-env": {
+ "version": "3.10.0",
+ "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz",
+ "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==",
+ "license": "MIT"
+ },
"node_modules/@codemirror/autocomplete": {
"version": "6.20.1",
"resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.1.tgz",
@@ -4978,9 +5039,7 @@
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
"integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
- "dev": true,
"license": "MIT",
- "peer": true,
"engines": {
"node": ">=6"
}
@@ -5056,6 +5115,12 @@
"@types/trusted-types": "^2.0.7"
}
},
+ "node_modules/driver.js": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/driver.js/-/driver.js-1.4.0.tgz",
+ "integrity": "sha512-Gm64jm6PmcU+si21sQhBrTAM1JvUrR0QhNmjkprNLxohOBzul9+pNHXgQaT9lW84gwg9GMLB3NZGuGolsz5uew==",
+ "license": "MIT"
+ },
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -6097,6 +6162,12 @@
"node": ">=10.13.0"
}
},
+ "node_modules/glob-to-regexp": {
+ "version": "0.4.1",
+ "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
+ "license": "BSD-2-Clause"
+ },
"node_modules/glob/node_modules/brace-expansion": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz",
@@ -6521,6 +6592,15 @@
"jiti": "bin/jiti.js"
}
},
+ "node_modules/js-cookie": {
+ "version": "3.0.7",
+ "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.7.tgz",
+ "integrity": "sha512-z/wZZgDrkNV1eA0ULjM/F9/50Ya8fbzgKneSpoPsXSGd0KnpdtHfOZWK+GcwLk+EZbS4F9RBhU+K2RgzuDaItw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20"
+ }
+ },
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -8840,6 +8920,19 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/swr": {
+ "version": "2.3.4",
+ "resolved": "https://registry.npmjs.org/swr/-/swr-2.3.4.tgz",
+ "integrity": "sha512-bYd2lrhc+VarcpkgWclcUi92wYCpOgMws9Sd1hG1ntAu0NEy+14CbotuFjshBU2kt9rYj9TSmDcybpxpeTU1fg==",
+ "license": "MIT",
+ "dependencies": {
+ "dequal": "^2.0.3",
+ "use-sync-external-store": "^1.4.0"
+ },
+ "peerDependencies": {
+ "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
+ }
+ },
"node_modules/symbol-tree": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
diff --git a/skyvern-frontend/package.json b/skyvern-frontend/package.json
index e7d0d1bd8..3f194848d 100644
--- a/skyvern-frontend/package.json
+++ b/skyvern-frontend/package.json
@@ -19,6 +19,7 @@
"start": "npm run serve & npm run run-artifact-server"
},
"dependencies": {
+ "@clerk/clerk-react": "^5.61.8",
"@codemirror/lang-html": "^6.4.9",
"@codemirror/lang-json": "^6.0.1",
"@codemirror/lang-python": "^6.1.6",
@@ -61,6 +62,7 @@
"cron-parser": "^5.5.0",
"cronstrue": "^3.13.0",
"cross-spawn": "^7.0.6",
+ "driver.js": "^1.4.0",
"embla-carousel-react": "^8.0.0",
"express": "^4.21.2",
"fetch-to-curl": "^0.6.0",
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index aaef56553..640c2010d 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -164,6 +164,8 @@ export type FailureCategory = {
reasoning: string;
};
+export type WebhookDeliveryStatus = "pending" | "failed";
+
export type TaskApiResponse = {
request: CreateTaskRequest;
task_id: string;
@@ -255,6 +257,10 @@ export type CreateOnePasswordTokenResponse = {
token: OnePasswordTokenApiResponse;
};
+export type ClearOrganizationAuthTokenResponse = {
+ success: boolean;
+};
+
export interface AzureClientSecretCredential {
tenant_id: string;
client_id: string;
@@ -601,6 +607,7 @@ export type WorkflowRunStatusApiResponse = {
outputs: Record<string, unknown> | null;
failure_reason: string | null;
failure_category: Array<FailureCategory> | null;
+ webhook_delivery_status?: WebhookDeliveryStatus | null;
webhook_failure_reason: string | null;
downloaded_file_urls: Array<string> | null;
downloaded_files: Array<DownloadedFileInfo> | null;
@@ -638,6 +645,7 @@ export type WorkflowRunStatusApiResponseWithWorkflow = {
outputs: Record<string, unknown> | null;
failure_reason: string | null;
failure_category: Array<FailureCategory> | null;
+ webhook_delivery_status?: WebhookDeliveryStatus | null;
webhook_failure_reason: string | null;
downloaded_file_urls: Array<string> | null;
downloaded_files: Array<DownloadedFileInfo> | null;
diff --git a/skyvern-frontend/src/components/AzureClientSecretCredentialTokenForm.tsx b/skyvern-frontend/src/components/AzureClientSecretCredentialTokenForm.tsx
index 94e847c5d..afe3d371c 100644
--- a/skyvern-frontend/src/components/AzureClientSecretCredentialTokenForm.tsx
+++ b/skyvern-frontend/src/components/AzureClientSecretCredentialTokenForm.tsx
@@ -12,6 +12,7 @@ import {
FormLabel,
FormMessage,
} from "@/components/ui/form";
+import { ClearCredentialDialog } from "@/components/ClearCredentialDialog";
import { useAzureClientCredentialToken } from "@/hooks/useAzureClientCredentialToken";
import { EyeOpenIcon, EyeClosedIcon } from "@radix-ui/react-icons";
@@ -44,6 +45,8 @@ export function AzureClientSecretCredentialTokenForm({
isLoading,
createOrUpdateToken,
isUpdating,
+ clearCredential,
+ isClearing,
} = useAzureClientCredentialToken();
const form = useForm<FormData>({
@@ -56,6 +59,7 @@ export function AzureClientSecretCredentialTokenForm({
},
},
});
+ const isMutating = isUpdating || isClearing;
const onSubmit = (data: FormData) => {
createOrUpdateToken(data, {
@@ -68,10 +72,14 @@ export function AzureClientSecretCredentialTokenForm({
};
useEffect(() => {
- if (azureOrganizationAuthToken?.credential) {
- form.reset({ credential: azureOrganizationAuthToken.credential });
- }
- }, [azureOrganizationAuthToken, form]);
+ form.reset({
+ credential: azureOrganizationAuthToken?.credential || {
+ tenant_id: "",
+ client_id: "",
+ client_secret: "",
+ },
+ });
+ }, [azureOrganizationAuthToken?.credential, form]);
return (
<div className="space-y-4">
@@ -111,7 +119,7 @@ export function AzureClientSecretCredentialTokenForm({
{...field}
type="text"
placeholder="tenant_id"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
</div>
@@ -131,7 +139,7 @@ export function AzureClientSecretCredentialTokenForm({
{...field}
type="text"
placeholder="client_id"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
</div>
@@ -151,7 +159,7 @@ export function AzureClientSecretCredentialTokenForm({
{...field}
type={showClientSecret ? "text" : "password"}
placeholder="client_secret"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
<Button
@@ -160,7 +168,7 @@ export function AzureClientSecretCredentialTokenForm({
size="sm"
className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"
onClick={toggleClientSecretVisibility}
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
>
{showClientSecret ? (
<EyeClosedIcon className="h-4 w-4" />
@@ -175,9 +183,19 @@ export function AzureClientSecretCredentialTokenForm({
/>
<div className="flex items-center gap-4">
- <Button type="submit" disabled={isLoading || isUpdating}>
+ <Button type="submit" disabled={isLoading || isMutating}>
{isUpdating ? "Updating..." : "Update Credential"}
</Button>
+ {azureOrganizationAuthToken && (
+ <ClearCredentialDialog
+ label="Clear Credential"
+ title="Clear Azure Client Secret Credential?"
+ description="Workflows that use Azure Key Vault credentials will no longer be able to resolve them until a new Azure Client Secret Credential is added."
+ disabled={isLoading || isMutating}
+ isPending={isClearing}
+ onConfirm={() => clearCredential()}
+ />
+ )}
{azureOrganizationAuthToken && (
<div className="text-sm text-muted-foreground">
Last updated:{" "}
diff --git a/skyvern-frontend/src/components/BitwardenCredentialForm.tsx b/skyvern-frontend/src/components/BitwardenCredentialForm.tsx
index 652a7a269..d536b43ad 100644
--- a/skyvern-frontend/src/components/BitwardenCredentialForm.tsx
+++ b/skyvern-frontend/src/components/BitwardenCredentialForm.tsx
@@ -12,6 +12,7 @@ import {
FormLabel,
FormMessage,
} from "@/components/ui/form";
+import { ClearCredentialDialog } from "@/components/ClearCredentialDialog";
import { useBitwardenCredential } from "@/hooks/useBitwardenCredential";
import { EyeOpenIcon, EyeClosedIcon } from "@radix-ui/react-icons";
@@ -44,6 +45,8 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
isLoading,
createOrUpdateToken,
isUpdating,
+ clearCredential,
+ isClearing,
} = useBitwardenCredential();
const form = useForm<FormData>({
@@ -55,6 +58,7 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
},
},
});
+ const isMutating = isUpdating || isClearing;
const onSubmit = (data: FormData) => {
createOrUpdateToken(data, {
@@ -63,15 +67,13 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
};
useEffect(() => {
- if (bitwardenOrganizationAuthToken?.credential) {
- form.reset({
- credential: {
- email: bitwardenOrganizationAuthToken.credential.email,
- master_password: "",
- },
- });
- }
- }, [bitwardenOrganizationAuthToken, form]);
+ form.reset({
+ credential: {
+ email: bitwardenOrganizationAuthToken?.credential?.email || "",
+ master_password: "",
+ },
+ });
+ }, [bitwardenOrganizationAuthToken?.credential?.email, form]);
return (
<div className="space-y-4">
@@ -109,7 +111,7 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
{...field}
type="email"
placeholder="[email protected]"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
</div>
@@ -129,7 +131,7 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
{...field}
type={showMasterPassword ? "text" : "password"}
placeholder="master_password"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
<Button
@@ -138,7 +140,7 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
size="sm"
className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"
onClick={() => setShowMasterPassword((v) => !v)}
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
>
{showMasterPassword ? (
<EyeClosedIcon className="h-4 w-4" />
@@ -159,9 +161,19 @@ export function BitwardenCredentialForm({ onSuccess }: Props = {}) {
/>
<div className="flex items-center gap-4">
- <Button type="submit" disabled={isLoading || isUpdating}>
+ <Button type="submit" disabled={isLoading || isMutating}>
{isUpdating ? "Updating..." : "Update Credential"}
</Button>
+ {bitwardenOrganizationAuthToken && (
+ <ClearCredentialDialog
+ label="Clear Credential"
+ title="Clear Bitwarden credential?"
+ description="Workflows that use Bitwarden-backed credentials will no longer be able to resolve them until new Bitwarden credentials are added."
+ disabled={isLoading || isMutating}
+ isPending={isClearing}
+ onConfirm={() => clearCredential()}
+ />
+ )}
{bitwardenOrganizationAuthToken && (
<div className="text-sm text-muted-foreground">
Last updated:{" "}
diff --git a/skyvern-frontend/src/components/ClearCredentialDialog.tsx b/skyvern-frontend/src/components/ClearCredentialDialog.tsx
new file mode 100644
index 000000000..b1b4add21
--- /dev/null
+++ b/skyvern-frontend/src/components/ClearCredentialDialog.tsx
@@ -0,0 +1,67 @@
+import { useState } from "react";
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogTitle,
+ DialogTrigger,
+} from "@/components/ui/dialog";
+
+type Props = {
+ label: string;
+ title: string;
+ description: string;
+ disabled: boolean;
+ isPending: boolean;
+ onConfirm: () => void;
+};
+
+export function ClearCredentialDialog({
+ label,
+ title,
+ description,
+ disabled,
+ isPending,
+ onConfirm,
+}: Props) {
+ const [open, setOpen] = useState(false);
+
+ const handleConfirm = () => {
+ onConfirm();
+ setOpen(false);
+ };
+
+ return (
+ <Dialog open={open} onOpenChange={setOpen}>
+ <DialogTrigger asChild>
+ <Button type="button" variant="destructive" disabled={disabled}>
+ {label}
+ </Button>
+ </DialogTrigger>
+ <DialogContent onCloseAutoFocus={(e) => e.preventDefault()}>
+ <DialogTitle>{title}</DialogTitle>
+ <DialogDescription>{description}</DialogDescription>
+ <DialogFooter>
+ <Button
+ type="button"
+ variant="secondary"
+ disabled={isPending}
+ onClick={() => setOpen(false)}
+ >
+ Cancel
+ </Button>
+ <Button
+ type="button"
+ variant="destructive"
+ onClick={handleConfirm}
+ disabled={disabled}
+ >
+ {isPending ? "Clearing..." : label}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ );
+}
diff --git a/skyvern-frontend/src/components/CustomCredentialServiceConfigForm.tsx b/skyvern-frontend/src/components/CustomCredentialServiceConfigForm.tsx
index 824832648..51ca3b867 100644
--- a/skyvern-frontend/src/components/CustomCredentialServiceConfigForm.tsx
+++ b/skyvern-frontend/src/components/CustomCredentialServiceConfigForm.tsx
@@ -13,6 +13,7 @@ import {
FormMessage,
FormDescription,
} from "@/components/ui/form";
+import { ClearCredentialDialog } from "@/components/ClearCredentialDialog";
import { useCustomCredentialServiceConfig } from "@/hooks/useCustomCredentialServiceConfig";
import { EyeOpenIcon, EyeClosedIcon, GlobeIcon } from "@radix-ui/react-icons";
@@ -46,6 +47,8 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
isLoading,
createOrUpdateConfig,
isUpdating,
+ clearConfig,
+ isClearing,
testConnection,
isTesting,
} = useCustomCredentialServiceConfig();
@@ -59,6 +62,7 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
},
},
});
+ const isMutating = isUpdating || isClearing;
const onSubmit = (data: FormData) => {
createOrUpdateConfig(data, {
@@ -78,11 +82,10 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
};
useEffect(() => {
- if (parsedConfig) {
- form.reset({ config: parsedConfig });
- }
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [parsedConfig]);
+ form.reset({
+ config: parsedConfig || { api_base_url: "", api_token: "" },
+ });
+ }, [parsedConfig, form]);
return (
<div className="space-y-4">
@@ -124,7 +127,7 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
{...field}
type="url"
placeholder="https://credentials.company.com/api/v1"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
<GlobeIcon className="absolute right-3 top-3 h-4 w-4 text-muted-foreground" />
@@ -150,7 +153,7 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
{...field}
type={showApiToken ? "text" : "password"}
placeholder="your_api_token_here"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
<Button
@@ -159,7 +162,7 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
size="sm"
className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"
onClick={toggleApiTokenVisibility}
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
>
{showApiToken ? (
<EyeClosedIcon className="h-4 w-4" />
@@ -176,18 +179,28 @@ export function CustomCredentialServiceConfigForm({ onSuccess }: Props = {}) {
<div className="flex items-center gap-4">
<Button
type="submit"
- disabled={isLoading || isUpdating || isTesting}
+ disabled={isLoading || isMutating || isTesting}
>
{isUpdating ? "Updating..." : "Update Configuration"}
</Button>
<Button
type="button"
variant="outline"
- disabled={isLoading || isUpdating || isTesting}
+ disabled={isLoading || isMutating || isTesting}
onClick={onTestConnection}
>
{isTesting ? "Testing..." : "Test Connection"}
</Button>
+ {customCredentialServiceAuthToken && (
+ <ClearCredentialDialog
+ label="Clear Configuration"
+ title="Clear custom credential service configuration?"
+ description="Workflows that use the custom credential service will no longer be able to resolve credentials until a new configuration is added."
+ disabled={isLoading || isMutating || isTesting}
+ isPending={isClearing}
+ onConfirm={() => clearConfig()}
+ />
+ )}
{customCredentialServiceAuthToken && (
<div className="text-sm text-muted-foreground">
diff --git a/skyvern-frontend/src/components/OnePasswordTokenForm.tsx b/skyvern-frontend/src/components/OnePasswordTokenForm.tsx
index 9de68251f..f4f5d437a 100644
--- a/skyvern-frontend/src/components/OnePasswordTokenForm.tsx
+++ b/skyvern-frontend/src/components/OnePasswordTokenForm.tsx
@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { zodResolver } from "@hookform/resolvers/zod";
import * as z from "zod";
@@ -12,6 +12,7 @@ import {
FormLabel,
FormMessage,
} from "@/components/ui/form";
+import { ClearCredentialDialog } from "@/components/ClearCredentialDialog";
import { useOnePasswordToken } from "@/hooks/useOnePasswordToken";
import { EyeOpenIcon, EyeClosedIcon } from "@radix-ui/react-icons";
@@ -27,8 +28,14 @@ type Props = {
export function OnePasswordTokenForm({ onSuccess }: Props = {}) {
const [showToken, setShowToken] = useState(false);
- const { onePasswordToken, isLoading, createOrUpdateToken, isUpdating } =
- useOnePasswordToken();
+ const {
+ onePasswordToken,
+ isLoading,
+ createOrUpdateToken,
+ isUpdating,
+ clearToken,
+ isClearing,
+ } = useOnePasswordToken();
const form = useForm<FormData>({
resolver: zodResolver(formSchema),
@@ -36,6 +43,11 @@ export function OnePasswordTokenForm({ onSuccess }: Props = {}) {
token: onePasswordToken?.token || "",
},
});
+ const isMutating = isUpdating || isClearing;
+
+ useEffect(() => {
+ form.reset({ token: onePasswordToken?.token || "" });
+ }, [form, onePasswordToken?.token]);
const onSubmit = (data: FormData) => {
createOrUpdateToken(data, {
@@ -47,14 +59,6 @@ export function OnePasswordTokenForm({ onSuccess }: Props = {}) {
setShowToken(!showToken);
};
- // Update form when token data loads
- if (
- onePasswordToken?.token &&
- form.getValues("token") !== onePasswordToken.token
- ) {
- form.setValue("token", onePasswordToken.token);
- }
-
return (
<div className="space-y-4">
<div className="flex items-center justify-between">
@@ -93,7 +97,7 @@ export function OnePasswordTokenForm({ onSuccess }: Props = {}) {
{...field}
type={showToken ? "text" : "password"}
placeholder="op_1234567890abcdef"
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
/>
</FormControl>
<Button
@@ -102,7 +106,7 @@ export function OnePasswordTokenForm({ onSuccess }: Props = {}) {
size="sm"
className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"
onClick={toggleTokenVisibility}
- disabled={isLoading || isUpdating}
+ disabled={isLoading || isMutating}
>
{showToken ? (
<EyeClosedIcon className="h-4 w-4" />
@@ -117,9 +121,19 @@ export function OnePasswordTokenForm({ onSuccess }: Props = {}) {
/>
<div className="flex items-center gap-4">
- <Button type="submit" disabled={isLoading || isUpdating}>
+ <Button type="submit" disabled={isLoading || isMutating}>
{isUpdating ? "Updating..." : "Update Token"}
</Button>
+ {onePasswordToken && (
+ <ClearCredentialDialog
+ label="Clear Token"
+ title="Clear 1Password token?"
+ description="Workflows that use 1Password credentials will no longer be able to resolve them until a new service account token is added."
+ disabled={isLoading || isMutating}
+ isPending={isClearing}
+ onConfirm={() => clearToken()}
+ />
+ )}
{onePasswordToken && (
<div className="text-sm text-muted-foreground">
Last updated:{" "}
diff --git a/skyvern-frontend/src/components/SelectionBar.test.tsx b/skyvern-frontend/src/components/SelectionBar.test.tsx
new file mode 100644
index 000000000..750638601
--- /dev/null
+++ b/skyvern-frontend/src/components/SelectionBar.test.tsx
@@ -0,0 +1,48 @@
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { SelectionBar } from "./SelectionBar";
+
+describe("SelectionBar", () => {
+ afterEach(cleanup);
+
+ it("shows the count and clears on Escape", () => {
+ const onClear = vi.fn();
+ render(
+ <SelectionBar count={3} isOperating={false} onClear={onClear}>
+ <button>Action</button>
+ </SelectionBar>,
+ );
+ expect(screen.getByText("3 selected")).toBeTruthy();
+ document.dispatchEvent(
+ new KeyboardEvent("keydown", { key: "Escape", bubbles: true }),
+ );
+ expect(onClear).toHaveBeenCalledTimes(1);
+ });
+
+ it("ignores Escape that a Radix layer already handled", () => {
+ const onClear = vi.fn();
+ render(<SelectionBar count={1} isOperating={false} onClear={onClear} />);
+ const event = new KeyboardEvent("keydown", {
+ key: "Escape",
+ bubbles: true,
+ cancelable: true,
+ });
+ event.preventDefault();
+ document.dispatchEvent(event);
+ expect(onClear).not.toHaveBeenCalled();
+ });
+
+ it("shows Processing and ignores Escape while operating", () => {
+ const onClear = vi.fn();
+ render(<SelectionBar count={2} isOperating={true} onClear={onClear} />);
+ expect(screen.getByText("Processing…")).toBeTruthy();
+ document.dispatchEvent(
+ new KeyboardEvent("keydown", { key: "Escape", bubbles: true }),
+ );
+ expect(onClear).not.toHaveBeenCalled();
+ expect(
+ (screen.getByLabelText("Clear selection") as HTMLButtonElement).disabled,
+ ).toBe(true);
+ });
+});
diff --git a/skyvern-frontend/src/components/SelectionBar.tsx b/skyvern-frontend/src/components/SelectionBar.tsx
new file mode 100644
index 000000000..9d25c6521
--- /dev/null
+++ b/skyvern-frontend/src/components/SelectionBar.tsx
@@ -0,0 +1,53 @@
+import { Cross2Icon } from "@radix-ui/react-icons";
+import { useEffect } from "react";
+import { Button } from "@/components/ui/button";
+
+type Props = {
+ count: number; // pass selectedItems.length, never selected.size
+ isOperating: boolean;
+ onClear: () => void;
+ children?: React.ReactNode;
+};
+
+function SelectionBar({ count, isOperating, onClear, children }: Props) {
+ useEffect(() => {
+ function handleKeyDown(event: KeyboardEvent) {
+ // Radix layers (dialogs, popovers) preventDefault on the Escape that dismisses them.
+ if (event.key === "Escape" && !event.defaultPrevented && !isOperating) {
+ onClear();
+ }
+ }
+ document.addEventListener("keydown", handleKeyDown);
+ return () => document.removeEventListener("keydown", handleKeyDown);
+ }, [isOperating, onClear]);
+
+ return (
+ <div
+ role="toolbar"
+ aria-label="Bulk actions"
+ className="fixed inset-x-0 bottom-6 z-50 mx-auto flex w-fit max-w-[calc(100vw-2rem)] flex-wrap items-center gap-1.5 rounded-lg border border-border bg-background px-4 py-2.5 shadow-xl"
+ >
+ <span className="whitespace-nowrap text-sm text-muted-foreground">
+ {isOperating ? "Processing…" : `${count} selected`}
+ </span>
+ <Button
+ size="icon"
+ variant="ghost"
+ className="h-7 w-7 text-muted-foreground hover:text-foreground"
+ onClick={onClear}
+ disabled={isOperating}
+ aria-label="Clear selection"
+ >
+ <Cross2Icon className="h-4 w-4" />
+ </Button>
+ <SelectionBarDivider />
+ {children}
+ </div>
+ );
+}
+
+function SelectionBarDivider() {
+ return <div className="mx-1 h-6 w-px bg-border" />;
+}
+
+export { SelectionBar, SelectionBarDivider };
diff --git a/skyvern-frontend/src/components/SelectionCheckbox.tsx b/skyvern-frontend/src/components/SelectionCheckbox.tsx
new file mode 100644
index 000000000..9cfa77db4
--- /dev/null
+++ b/skyvern-frontend/src/components/SelectionCheckbox.tsx
@@ -0,0 +1,101 @@
+import { Checkbox } from "@/components/ui/checkbox";
+import { TableCell, TableHead } from "@/components/ui/table";
+import { cn } from "@/util/utils";
+
+type SelectionCheckboxProps = {
+ index: number; // -1 = inert (non-selectable row)
+ checked: boolean;
+ hasSelection: boolean; // any selection active keeps boxes visible
+ onSelect: (index: number, shiftKey: boolean) => void;
+ ariaLabel: string;
+};
+
+function SelectionCheckbox({
+ index,
+ checked,
+ hasSelection,
+ onSelect,
+ ariaLabel,
+}: SelectionCheckboxProps) {
+ return (
+ <div
+ className="select-none"
+ onMouseDown={(event) => {
+ if (event.shiftKey) {
+ event.preventDefault();
+ }
+ }}
+ onClick={(event) => {
+ event.preventDefault();
+ event.stopPropagation();
+ if (index >= 0) {
+ onSelect(index, event.shiftKey);
+ }
+ }}
+ >
+ <Checkbox
+ checked={checked}
+ aria-label={ariaLabel}
+ // Mouse goes through the container for shift-range; onCheckedChange covers keyboard.
+ onCheckedChange={() => {
+ if (index >= 0) {
+ onSelect(index, false);
+ }
+ }}
+ className={cn(
+ "pointer-events-none transition-opacity",
+ !hasSelection &&
+ "opacity-0 focus-visible:opacity-100 group-hover/row:opacity-100",
+ )}
+ />
+ </div>
+ );
+}
+
+function SelectionCheckboxCell(
+ props: SelectionCheckboxProps & { className?: string },
+) {
+ const { className, ...checkbox } = props;
+ return (
+ <TableCell className={className}>
+ <SelectionCheckbox {...checkbox} />
+ </TableCell>
+ );
+}
+
+function SelectionHeaderCheckboxCell({
+ allSelected,
+ someSelected,
+ hasSelection,
+ onToggleAll,
+ ariaLabel,
+ className,
+}: {
+ allSelected: boolean;
+ someSelected: boolean;
+ hasSelection: boolean;
+ onToggleAll: () => void;
+ ariaLabel: string;
+ className?: string;
+}) {
+ return (
+ <TableHead className={className}>
+ <Checkbox
+ checked={someSelected ? "indeterminate" : allSelected}
+ onCheckedChange={onToggleAll}
+ aria-label={ariaLabel}
+ className={cn(
+ "transition-opacity",
+ !hasSelection &&
+ "opacity-0 focus-visible:opacity-100 group-hover/header:opacity-100",
+ )}
+ />
+ </TableHead>
+ );
+}
+
+export {
+ SelectionCheckbox,
+ SelectionCheckboxCell,
+ SelectionHeaderCheckboxCell,
+};
diff --git a/skyvern-frontend/src/components/StatusBadge.test.tsx b/skyvern-frontend/src/components/StatusBadge.test.tsx
new file mode 100644
index 000000000..2a2870987
--- /dev/null
+++ b/skyvern-frontend/src/components/StatusBadge.test.tsx
@@ -0,0 +1,39 @@
+// @vitest-environment jsdom
+
+import { render, screen } from "@testing-library/react";
+import { describe, expect, test } from "vitest";
+
+import { Status } from "@/api/types";
+
+import { StatusBadge } from "./StatusBadge";
+
+describe("StatusBadge", () => {
+ test("always renders the status text, visually hidden below md", () => {
+ render(<StatusBadge status={Status.Completed} />);
+
+ const label = screen.getByText("completed");
+ expect(label.classList.contains("sr-only")).toBe(true);
+ expect(label.classList.contains("md:not-sr-only")).toBe(true);
+ });
+
+ test("collapses to a compact pill below md via responsive classes", () => {
+ const { container } = render(<StatusBadge status={Status.Completed} />);
+
+ const badge = container.firstElementChild;
+ expect(badge?.classList.contains("w-28")).toBe(false);
+ expect(badge?.classList.contains("md:w-28")).toBe(true);
+ });
+
+ test("exposes the status text as a tooltip", () => {
+ render(<StatusBadge status={Status.Running} />);
+
+ expect(screen.getByTitle("running")).not.toBeNull();
+ });
+
+ test("humanizes timed_out", () => {
+ render(<StatusBadge status={Status.TimedOut} />);
+
+ expect(screen.getByText("timed out")).not.toBeNull();
+ expect(screen.getByTitle("timed out")).not.toBeNull();
+ });
+});
diff --git a/skyvern-frontend/src/components/StatusBadge.tsx b/skyvern-frontend/src/components/StatusBadge.tsx
index bb21866c7..eaaa5b4d7 100644
--- a/skyvern-frontend/src/components/StatusBadge.tsx
+++ b/skyvern-frontend/src/components/StatusBadge.tsx
@@ -115,10 +115,14 @@ function StatusBadge({ className, status }: Props) {
return (
<Badge
variant={variantForStatus(status)}
- className={cn("w-28 justify-start gap-1.5 capitalize", className)}
+ className={cn(
+ "justify-center gap-1.5 px-1.5 capitalize md:w-28 md:justify-start md:px-2.5",
+ className,
+ )}
+ title={statusText}
>
{iconForStatus(status)}
- {statusText}
+ <span className="sr-only md:not-sr-only">{statusText}</span>
</Badge>
);
}
diff --git a/skyvern-frontend/src/components/onboarding/ContextualHints.tsx b/skyvern-frontend/src/components/onboarding/ContextualHints.tsx
new file mode 100644
index 000000000..d414afb95
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/ContextualHints.tsx
@@ -0,0 +1,9 @@
+import { useContextualHints } from "@/hooks/useContextualHints";
+import "@/util/onboarding/product-tour.css";
+
+function ContextualHints() {
+ useContextualHints();
+ return null;
+}
+
+export { ContextualHints };
diff --git a/skyvern-frontend/src/components/onboarding/CopilotCTAStep.test.tsx b/skyvern-frontend/src/components/onboarding/CopilotCTAStep.test.tsx
new file mode 100644
index 000000000..4c7775c6b
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/CopilotCTAStep.test.tsx
@@ -0,0 +1,148 @@
+// @vitest-environment jsdom
+import {
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { MemoryRouter } from "react-router-dom";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const { mockPost, mockNavigate } = vi.hoisted(() => ({
+ mockPost: vi.fn(),
+ mockNavigate: vi.fn(),
+}));
+
+vi.mock("react-router-dom", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("react-router-dom")>();
+ return {
+ ...actual,
+ useNavigate: () => mockNavigate,
+ };
+});
+
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => () => Promise.resolve("test-token"),
+}));
+
+vi.mock("@/api/AxiosClient", () => ({
+ getClient: () => Promise.resolve({ post: mockPost }),
+}));
+
+vi.mock("@/routes/workflows/hooks/useGlobalWorkflowsQuery", () => ({
+ useGlobalWorkflowsQuery: () => ({ data: [], isLoading: false }),
+}));
+
+vi.mock("@/util/onboarding/OnboardingTelemetry", () => ({
+ OnboardingTelemetry: {
+ flowCompleted: vi.fn(),
+ modalCopilotClicked: vi.fn(),
+ modalTemplateSelected: vi.fn(),
+ },
+}));
+
+import { Dialog, DialogContent } from "@/components/ui/dialog";
+import { CopilotCTAStep } from "./CopilotCTAStep";
+
+function setup(onBusyChange?: (busy: boolean) => void) {
+ const queryClient = new QueryClient({
+ defaultOptions: {
+ queries: { retry: false },
+ mutations: { retry: false },
+ },
+ });
+ const invalidateSpy = vi.spyOn(queryClient, "invalidateQueries");
+ render(
+ <QueryClientProvider client={queryClient}>
+ <MemoryRouter>
+ <Dialog defaultOpen>
+ <DialogContent>
+ <CopilotCTAStep
+ selectedIntent="fill_forms"
+ onBack={vi.fn()}
+ onSkip={vi.fn()}
+ onDismiss={vi.fn()}
+ onBusyChange={onBusyChange}
+ />
+ </DialogContent>
+ </Dialog>
+ </MemoryRouter>
+ </QueryClientProvider>,
+ );
+ return { invalidateSpy };
+}
+
+afterEach(() => {
+ cleanup();
+ vi.clearAllMocks();
+});
+
+describe("CopilotCTAStep", () => {
+ it("invalidates userOnboarding after the copilot handoff creates a workflow", async () => {
+ mockPost.mockResolvedValue({
+ data: { workflow_permanent_id: "wpid_x" },
+ });
+ const { invalidateSpy } = setup();
+
+ fireEvent.change(screen.getByRole("textbox"), {
+ target: { value: "fill out my form" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Create with AI" }));
+
+ await waitFor(() =>
+ expect(invalidateSpy).toHaveBeenCalledWith({
+ queryKey: ["userOnboarding"],
+ }),
+ );
+ });
+
+ it("hands the prompt off to /build with it in location state", async () => {
+ mockPost.mockResolvedValue({
+ data: { workflow_permanent_id: "wpid_x" },
+ });
+ setup();
+
+ fireEvent.change(screen.getByRole("textbox"), {
+ target: { value: "fill out my form" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Create with AI" }));
+
+ await waitFor(() =>
+ expect(mockNavigate).toHaveBeenCalledWith("/workflows/wpid_x/build", {
+ state: { copilotMessage: "fill out my form" },
+ }),
+ );
+ });
+
+ it("reports busy to the parent while the handoff is in flight", async () => {
+ mockPost.mockReturnValue(new Promise(() => {}));
+ const onBusyChange = vi.fn();
+ setup(onBusyChange);
+
+ fireEvent.change(screen.getByRole("textbox"), {
+ target: { value: "fill out my form" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Create with AI" }));
+
+ await waitFor(() => expect(onBusyChange).toHaveBeenCalledWith(true));
+ });
+
+ it("disables Skip while the copilot handoff is in flight", async () => {
+ // Never resolves: keep the handoff mutation pending so we can assert the
+ // Skip control is disabled (otherwise a skip mid-create still navigates).
+ mockPost.mockReturnValue(new Promise(() => {}));
+ setup();
+
+ fireEvent.change(screen.getByRole("textbox"), {
+ target: { value: "fill out my form" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Create with AI" }));
+
+ await waitFor(() => {
+ const skip = screen.getByRole("button", { name: "Skip" });
+ expect((skip as HTMLButtonElement).disabled).toBe(true);
+ });
+ });
+});
diff --git a/skyvern-frontend/src/components/onboarding/CopilotCTAStep.tsx b/skyvern-frontend/src/components/onboarding/CopilotCTAStep.tsx
new file mode 100644
index 000000000..279b6876b
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/CopilotCTAStep.tsx
@@ -0,0 +1,301 @@
+import { useEffect, useRef, useState } from "react";
+import { useNavigate } from "react-router-dom";
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { stringify as convertToYAML } from "yaml";
+import {
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { Button } from "@/components/ui/button";
+import { Textarea } from "@/components/ui/textarea";
+import { Skeleton } from "@/components/ui/skeleton";
+import { cn } from "@/util/utils";
+import { getClient } from "@/api/AxiosClient";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
+import { useGlobalWorkflowsQuery } from "@/routes/workflows/hooks/useGlobalWorkflowsQuery";
+import { useCreateWorkflowMutation } from "@/routes/workflows/hooks/useCreateWorkflowMutation";
+import { convert } from "@/routes/workflows/editor/workflowEditorUtils";
+import type { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";
+import type { WorkflowCreateYAMLRequest } from "@/routes/workflows/types/workflowYamlTypes";
+import {
+ getTemplatesForIntent,
+ getTemplateIcon,
+ getSetupTime,
+} from "./templateUtils";
+
+const SURFACE = "dashboard" as const;
+
+const INTENT_PLACEHOLDERS: Record<string, string> = {
+ fill_forms: "Describe the form you want to fill out...",
+ extract_data: "What data do you want to extract and from where?",
+ monitor_website: "What website changes do you want to monitor?",
+ something_else: "Describe what you want to automate...",
+};
+
+const HANDOFF_TITLE_MAX_LEN = 80;
+
+function deriveTitle(prompt: string): string {
+ const collapsed = prompt.replace(/\s+/g, " ").trim();
+ if (!collapsed) return "New Workflow";
+ if (collapsed.length <= HANDOFF_TITLE_MAX_LEN) return collapsed;
+ return `${collapsed.slice(0, HANDOFF_TITLE_MAX_LEN - 1).trimEnd()}...`;
+}
+
+type CopilotCTAStepProps = {
+ selectedIntent: string;
+ onBack: () => void;
+ onSkip: () => void;
+ onDismiss: () => void;
+ /** Reports in-flight creations so the parent can ignore its dialog-level
+ * close path (Escape / X), which cannot see this step's mutations. */
+ onBusyChange?: (busy: boolean) => void;
+};
+
+function CopilotCTAStep({
+ selectedIntent,
+ onBack,
+ onSkip,
+ onDismiss,
+ onBusyChange,
+}: Readonly<CopilotCTAStepProps>) {
+ const navigate = useNavigate();
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+
+ const [promptText, setPromptText] = useState("");
+ const [showFallback, setShowFallback] = useState(false);
+ const submitRef = useRef(false);
+
+ const { data: globalTemplates = [], isLoading: templatesLoading } =
+ useGlobalWorkflowsQuery();
+ const createWorkflowMutation = useCreateWorkflowMutation();
+
+ const handoffMutation = useMutation({
+ mutationFn: async (prompt: string) => {
+ const client = await getClient(credentialGetter);
+ const request: WorkflowCreateYAMLRequest = {
+ title: deriveTitle(prompt),
+ description: "",
+ ai_fallback: true,
+ code_version: 2,
+ run_with: "agent",
+ workflow_definition: {
+ version: 2,
+ blocks: [],
+ parameters: [],
+ },
+ };
+ const yaml = convertToYAML(request);
+ const result = await client.post<WorkflowApiResponse>(
+ "/workflows",
+ yaml,
+ { headers: { "Content-Type": "text/plain" } },
+ );
+ return { workflow: result.data, prompt };
+ },
+ onSuccess: ({ workflow, prompt }) => {
+ OnboardingTelemetry.flowCompleted(SURFACE);
+ queryClient.invalidateQueries({ queryKey: ["workflows"] });
+ queryClient.invalidateQueries({ queryKey: ["folders"] });
+ // Let the provider observe the backend first_save_at milestone so the
+ // first_workflow_created funnel event fires for copilot-first users.
+ queryClient.invalidateQueries({ queryKey: ["userOnboarding"] });
+ onDismiss();
+ navigate(`/workflows/${workflow.workflow_permanent_id}/build`, {
+ state: { copilotMessage: prompt },
+ });
+ },
+ onError: () => {
+ setShowFallback(true);
+ },
+ onSettled: () => {
+ submitRef.current = false;
+ },
+ });
+
+ const busy = handoffMutation.isPending || createWorkflowMutation.isPending;
+ useEffect(() => {
+ onBusyChange?.(busy);
+ }, [busy, onBusyChange]);
+ useEffect(() => {
+ return () => {
+ onBusyChange?.(false);
+ };
+ }, [onBusyChange]);
+
+ function handleSubmit() {
+ const trimmed = promptText.trim();
+ if (!trimmed || submitRef.current || handoffMutation.isPending) return;
+ submitRef.current = true;
+ OnboardingTelemetry.modalCopilotClicked(SURFACE, selectedIntent, trimmed);
+ handoffMutation.mutate(trimmed);
+ }
+
+ function handleKeyDown(e: React.KeyboardEvent<HTMLTextAreaElement>) {
+ if (e.key === "Enter" && !e.shiftKey) {
+ e.preventDefault();
+ handleSubmit();
+ }
+ }
+
+ function handleTemplateSelect(template: WorkflowApiResponse) {
+ if (createWorkflowMutation.isPending) return;
+ OnboardingTelemetry.modalTemplateSelected(
+ SURFACE,
+ template.workflow_permanent_id,
+ selectedIntent,
+ );
+ const cloned = convert({
+ ...template,
+ title: `${template.title} (copy)`,
+ });
+ // flow_completed fires from useCreateWorkflowMutation (it owns the navigation
+ // that unmounts this modal); the navigation also dismisses the modal.
+ createWorkflowMutation.mutate({ ...cloned, _via: "onboarding_template" });
+ }
+
+ const filteredTemplates =
+ globalTemplates.length > 0
+ ? getTemplatesForIntent(globalTemplates, selectedIntent)
+ : [];
+
+ if (showFallback) {
+ return (
+ <>
+ <DialogHeader>
+ <DialogTitle className="text-xl">Or pick a template</DialogTitle>
+ <DialogDescription>
+ Choose a pre-built workflow and customize it in the editor.
+ </DialogDescription>
+ </DialogHeader>
+ <div className="grid grid-cols-2 gap-3 py-2">
+ {templatesLoading ? (
+ Array.from({ length: 4 }).map((_, i) => (
+ <Skeleton key={i} className="h-28 rounded-lg" />
+ ))
+ ) : filteredTemplates.length > 0 ? (
+ filteredTemplates.map((template) => {
+ const Icon = getTemplateIcon(template);
+ return (
+ <button
+ key={template.workflow_permanent_id}
+ type="button"
+ disabled={createWorkflowMutation.isPending}
+ onClick={() => handleTemplateSelect(template)}
+ className={cn(
+ "flex flex-col items-start gap-2 rounded-lg border border-border p-4 text-left transition-colors",
+ "hover:border-primary hover:bg-primary/5",
+ createWorkflowMutation.isPending &&
+ "pointer-events-none opacity-50",
+ )}
+ >
+ <Icon className="h-6 w-6 text-primary" />
+ <div className="min-w-0 self-stretch">
+ <p className="truncate text-sm font-medium">
+ {template.title}
+ </p>
+ {template.description && (
+ <p className="line-clamp-2 text-xs text-muted-foreground">
+ {template.description}
+ </p>
+ )}
+ </div>
+ <span className="text-xs text-muted-foreground">
+ ~{getSetupTime(template)} setup
+ </span>
+ </button>
+ );
+ })
+ ) : (
+ <p className="col-span-2 py-8 text-center text-sm text-muted-foreground">
+ No templates available yet.
+ </p>
+ )}
+ </div>
+ <DialogFooter className="gap-2 sm:gap-0">
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={() => setShowFallback(false)}
+ disabled={createWorkflowMutation.isPending}
+ >
+ Back
+ </Button>
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={onSkip}
+ disabled={createWorkflowMutation.isPending}
+ >
+ Skip
+ </Button>
+ </DialogFooter>
+ </>
+ );
+ }
+
+ return (
+ <>
+ <DialogHeader>
+ <DialogTitle className="text-xl">
+ Describe what you want to automate
+ </DialogTitle>
+ <DialogDescription>
+ Tell us in plain language and our AI copilot will build a workflow for
+ you.
+ </DialogDescription>
+ </DialogHeader>
+ <div className="py-2">
+ <Textarea
+ value={promptText}
+ onChange={(e) => setPromptText(e.target.value)}
+ onKeyDown={handleKeyDown}
+ placeholder={
+ INTENT_PLACEHOLDERS[selectedIntent] ??
+ INTENT_PLACEHOLDERS.something_else
+ }
+ className="min-h-[100px] resize-none"
+ disabled={handoffMutation.isPending}
+ autoFocus
+ />
+ {handoffMutation.isPending && (
+ <div className="mt-3 flex items-center gap-2 text-sm text-muted-foreground">
+ <span className="inline-block h-4 w-4 animate-spin rounded-full border-2 border-current border-t-transparent" />
+ Setting up your workflow...
+ </div>
+ )}
+ </div>
+ <DialogFooter className="gap-2 sm:gap-0">
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={onBack}
+ disabled={handoffMutation.isPending}
+ >
+ Back
+ </Button>
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={onSkip}
+ disabled={handoffMutation.isPending}
+ >
+ Skip
+ </Button>
+ <Button
+ size="sm"
+ disabled={!promptText.trim() || handoffMutation.isPending}
+ onClick={handleSubmit}
+ >
+ {handoffMutation.isPending ? "Creating..." : "Create with AI"}
+ </Button>
+ </DialogFooter>
+ </>
+ );
+}
+
+export { CopilotCTAStep };
+export type { CopilotCTAStepProps };
diff --git a/skyvern-frontend/src/components/onboarding/CredentialSetupPrompt.test.tsx b/skyvern-frontend/src/components/onboarding/CredentialSetupPrompt.test.tsx
new file mode 100644
index 000000000..be71fd976
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/CredentialSetupPrompt.test.tsx
@@ -0,0 +1,100 @@
+// @vitest-environment jsdom
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { MemoryRouter } from "react-router-dom";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("@/util/onboarding/credentialSetupTelemetry", () => ({
+ CredentialSetupTelemetry: {
+ credentialSetupShown: vi.fn(),
+ credentialSetupCtaClicked: vi.fn(),
+ },
+}));
+
+import { CredentialSetupTelemetry } from "@/util/onboarding/credentialSetupTelemetry";
+import { CredentialSetupPrompt } from "./CredentialSetupPrompt";
+
+function renderPrompt(blocks: Array<{ label: string }>) {
+ return render(
+ <MemoryRouter>
+ <CredentialSetupPrompt
+ workflowPermanentId="wpid_123"
+ blocksMissingCredentials={blocks}
+ />
+ </MemoryRouter>,
+ );
+}
+
+afterEach(() => {
+ cleanup();
+ vi.clearAllMocks();
+});
+
+describe("CredentialSetupPrompt", () => {
+ it("lists each login block missing a credential and routes the CTA into the editor", () => {
+ renderPrompt([{ label: "Login" }, { label: "Member sign in" }]);
+
+ // getByText throws when absent, so these assert presence on their own.
+ screen.getByText("Login");
+ screen.getByText("Member sign in");
+ expect(CredentialSetupTelemetry.credentialSetupShown).toHaveBeenCalledWith(
+ "run_parameters",
+ 2,
+ );
+
+ const cta = screen.getByRole("link", {
+ name: /set up credentials in the editor/i,
+ });
+ expect(cta.getAttribute("href")).toBe("/workflows/wpid_123/build");
+ });
+
+ it("fires credentialSetupShown once on mount with the block count", () => {
+ renderPrompt([{ label: "Login" }]);
+
+ expect(CredentialSetupTelemetry.credentialSetupShown).toHaveBeenCalledTimes(
+ 1,
+ );
+ expect(CredentialSetupTelemetry.credentialSetupShown).toHaveBeenCalledWith(
+ "run_parameters",
+ 1,
+ );
+ });
+
+ it("does not re-fire credentialSetupShown on re-render", () => {
+ const { rerender } = renderPrompt([{ label: "Login" }]);
+
+ rerender(
+ <MemoryRouter>
+ <CredentialSetupPrompt
+ workflowPermanentId="wpid_123"
+ blocksMissingCredentials={[{ label: "Login" }]}
+ />
+ </MemoryRouter>,
+ );
+
+ expect(CredentialSetupTelemetry.credentialSetupShown).toHaveBeenCalledTimes(
+ 1,
+ );
+ });
+
+ it("fires credentialSetupCtaClicked when the CTA is clicked", () => {
+ renderPrompt([{ label: "Login" }]);
+
+ fireEvent.click(
+ screen.getByRole("link", { name: /set up credentials in the editor/i }),
+ );
+
+ expect(
+ CredentialSetupTelemetry.credentialSetupCtaClicked,
+ ).toHaveBeenCalledWith("run_parameters", 1);
+ });
+
+ it("renders nothing and fires no telemetry when no blocks are missing credentials", () => {
+ const { container } = renderPrompt([]);
+
+ expect(container.firstChild).toBeNull();
+ expect(screen.queryByRole("alert")).toBeNull();
+ expect(
+ CredentialSetupTelemetry.credentialSetupShown,
+ ).not.toHaveBeenCalled();
+ });
+});
diff --git a/skyvern-frontend/src/components/onboarding/CredentialSetupPrompt.tsx b/skyvern-frontend/src/components/onboarding/CredentialSetupPrompt.tsx
new file mode 100644
index 000000000..d271f28b7
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/CredentialSetupPrompt.tsx
@@ -0,0 +1,71 @@
+import { useEffect, useRef } from "react";
+import { Link } from "react-router-dom";
+import { ExclamationTriangleIcon } from "@radix-ui/react-icons";
+
+import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+import { Button } from "@/components/ui/button";
+import { CredentialSetupTelemetry } from "@/util/onboarding/credentialSetupTelemetry";
+
+type CredentialSetupPromptProps = {
+ workflowPermanentId: string;
+ blocksMissingCredentials: Array<{ label: string }>;
+};
+
+/**
+ * Activation-surface credential prompt shown on the run-parameters page when a
+ * workflow's login block(s) still need a credential. The CTA routes into the
+ * in-editor login-node credential experience (the only surface that binds a
+ * credential to the block); opening the credential modal here instead would
+ * orphan the new credential, since binding writes to the editor-only store.
+ */
+function CredentialSetupPrompt({
+ workflowPermanentId,
+ blocksMissingCredentials,
+}: Readonly<CredentialSetupPromptProps>) {
+ const blockCount = blocksMissingCredentials.length;
+ const shownRef = useRef(false);
+
+ useEffect(() => {
+ if (blockCount === 0 || shownRef.current) return;
+ shownRef.current = true;
+ CredentialSetupTelemetry.credentialSetupShown("run_parameters", blockCount);
+ }, [blockCount]);
+
+ if (blockCount === 0) {
+ return null;
+ }
+
+ return (
+ <Alert variant="warning" data-testid="credential-setup-prompt">
+ <ExclamationTriangleIcon className="h-4 w-4" />
+ <AlertTitle>Set up login credentials to run this agent</AlertTitle>
+ <AlertDescription>
+ <p>
+ This agent signs in to a site, so it needs a saved credential before
+ its first run. These login step(s) still need one:
+ </p>
+ <ul className="mt-2 list-inside list-disc">
+ {blocksMissingCredentials.map((block, index) => (
+ <li key={index}>{block.label}</li>
+ ))}
+ </ul>
+ <Button asChild size="sm" className="mt-3">
+ <Link
+ to={`/workflows/${workflowPermanentId}/build`}
+ onClick={() =>
+ CredentialSetupTelemetry.credentialSetupCtaClicked(
+ "run_parameters",
+ blockCount,
+ )
+ }
+ >
+ Set up credentials in the editor
+ </Link>
+ </Button>
+ </AlertDescription>
+ </Alert>
+ );
+}
+
+export { CredentialSetupPrompt };
+export type { CredentialSetupPromptProps };
diff --git a/skyvern-frontend/src/components/onboarding/FirstRunRecoveryGuidance.test.tsx b/skyvern-frontend/src/components/onboarding/FirstRunRecoveryGuidance.test.tsx
new file mode 100644
index 000000000..f453cf6b6
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/FirstRunRecoveryGuidance.test.tsx
@@ -0,0 +1,180 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { render, fireEvent, cleanup, waitFor } from "@testing-library/react";
+import posthog from "posthog-js";
+import { FirstRunRecoveryGuidance } from "./FirstRunRecoveryGuidance";
+import { getRecoveryPaths } from "./recoveryPaths";
+
+const navigateMock = vi.fn();
+vi.mock("react-router-dom", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("react-router-dom")>();
+ return { ...actual, useNavigate: () => navigateMock };
+});
+
+vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));
+
+describe("getRecoveryPaths", () => {
+ it("always returns at least two paths (AC1)", () => {
+ for (const category of [
+ null,
+ "",
+ "totally_unknown_thing",
+ "invalid_credentials",
+ "element_not_found",
+ "network_error",
+ ]) {
+ expect(getRecoveryPaths(category).length).toBeGreaterThanOrEqual(2);
+ }
+ });
+
+ it("selects credential recovery for auth failures (AC2)", () => {
+ const ids = getRecoveryPaths("invalid_credentials").map((p) => p.id);
+ expect(ids).toContain("update_credentials");
+ expect(ids).toContain("retry");
+ });
+
+ it("selects workflow editing for element/selector failures (AC2)", () => {
+ const ids = getRecoveryPaths("element_not_found").map((p) => p.id);
+ expect(ids).toContain("edit_workflow");
+ expect(ids).toContain("retry");
+ });
+
+ it("selects retry + docs for network failures (AC2)", () => {
+ const ids = getRecoveryPaths("network_error").map((p) => p.id);
+ expect(ids).toContain("retry");
+ expect(ids).toContain("view_docs");
+ });
+
+ it("falls back to a generic catalog for unknown categories (AC2 default)", () => {
+ const ids = getRecoveryPaths(null).map((p) => p.id);
+ expect(ids).toContain("retry");
+ expect(ids).toContain("contact_support");
+ });
+});
+
+describe("FirstRunRecoveryGuidance", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+ afterEach(() => {
+ cleanup();
+ });
+
+ it("renders one button per recovery path with at least two actions (AC1)", () => {
+ const { getByTestId, container } = render(
+ <FirstRunRecoveryGuidance surface="runs" failureCategory={null} />,
+ );
+ expect(getByTestId("first-run-recovery-guidance")).toBeTruthy();
+ const buttons = container.querySelectorAll("button");
+ expect(buttons.length).toBeGreaterThanOrEqual(2);
+ });
+
+ it("emits the shown event once on mount with the path count (AC3)", () => {
+ render(
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory="invalid_credentials"
+ />,
+ );
+ const shownCalls = vi
+ .mocked(posthog.capture)
+ .mock.calls.filter((c) => c[0] === "onboarding.recovery_guidance_shown");
+ expect(shownCalls).toHaveLength(1);
+ expect(shownCalls[0]?.[1]).toMatchObject({
+ surface: "runs",
+ failure_category: "invalid_credentials",
+ path_count: 2,
+ });
+ });
+
+ it("records chosen + navigated outcome for a navigate path (AC3)", () => {
+ const { getByTestId } = render(
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory="invalid_credentials"
+ workflowPermanentId="wpid_123"
+ />,
+ );
+ fireEvent.click(getByTestId("recovery-path-update_credentials"));
+ expect(navigateMock).toHaveBeenCalledWith("/credentials");
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_path_chosen",
+ expect.objectContaining({ path_id: "update_credentials" }),
+ );
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_outcome",
+ expect.objectContaining({
+ path_id: "update_credentials",
+ outcome: "navigated",
+ }),
+ );
+ });
+
+ it("navigates to the workflow editor with the permanent id (AC2 wiring)", () => {
+ const { getByTestId } = render(
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory="element_not_found"
+ workflowPermanentId="wpid_123"
+ />,
+ );
+ fireEvent.click(getByTestId("recovery-path-edit_workflow"));
+ expect(navigateMock).toHaveBeenCalledWith("/workflows/wpid_123/build");
+ });
+
+ it("records chosen + opened outcome for an external path (AC3)", () => {
+ const openSpy = vi.spyOn(window, "open").mockImplementation(() => null);
+ const { getByTestId } = render(
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory="network_error"
+ />,
+ );
+ fireEvent.click(getByTestId("recovery-path-view_docs"));
+ expect(openSpy).toHaveBeenCalled();
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_outcome",
+ expect.objectContaining({ path_id: "view_docs", outcome: "opened" }),
+ );
+ openSpy.mockRestore();
+ });
+
+ it("records retry_started when the retry handler resolves (AC3)", async () => {
+ const onRetry = vi.fn().mockResolvedValue(undefined);
+ const { getByTestId } = render(
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory="network_error"
+ onRetry={onRetry}
+ />,
+ );
+ fireEvent.click(getByTestId("recovery-path-retry"));
+ expect(onRetry).toHaveBeenCalled();
+ await waitFor(() => {
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_outcome",
+ expect.objectContaining({ path_id: "retry", outcome: "retry_started" }),
+ );
+ });
+ });
+
+ it("records retry_failed_to_start when the retry handler rejects (AC3)", async () => {
+ const onRetry = vi.fn().mockRejectedValue(new Error("boom"));
+ const { getByTestId } = render(
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory="network_error"
+ onRetry={onRetry}
+ />,
+ );
+ fireEvent.click(getByTestId("recovery-path-retry"));
+ await waitFor(() => {
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_outcome",
+ expect.objectContaining({
+ path_id: "retry",
+ outcome: "retry_failed_to_start",
+ }),
+ );
+ });
+ });
+});
diff --git a/skyvern-frontend/src/components/onboarding/FirstRunRecoveryGuidance.tsx b/skyvern-frontend/src/components/onboarding/FirstRunRecoveryGuidance.tsx
new file mode 100644
index 000000000..7f8557b36
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/FirstRunRecoveryGuidance.tsx
@@ -0,0 +1,140 @@
+import { useEffect, useRef } from "react";
+import { useNavigate } from "react-router-dom";
+import { Button } from "@/components/ui/button";
+import {
+ RecoveryGuidanceTelemetry,
+ type Surface,
+} from "@/util/onboarding/recoveryGuidanceTelemetry";
+import {
+ getRecoveryPaths,
+ type RecoveryPath,
+ type RecoveryPathId,
+} from "./recoveryPaths";
+
+const DOCS_URL = "https://docs.skyvern.com";
+const SUPPORT_URL = "mailto:[email protected]";
+
+type Props = Readonly<{
+ surface: Surface;
+ failureCategory: string | null;
+ workflowPermanentId?: string | null;
+ onRetry?: () => Promise<void> | void;
+}>;
+
+function FirstRunRecoveryGuidance({
+ surface,
+ failureCategory,
+ workflowPermanentId,
+ onRetry,
+}: Props) {
+ const navigate = useNavigate();
+ const paths = getRecoveryPaths(failureCategory);
+ const shownRef = useRef(false);
+
+ useEffect(() => {
+ if (shownRef.current) {
+ return;
+ }
+ shownRef.current = true;
+ RecoveryGuidanceTelemetry.recoveryGuidanceShown(
+ surface,
+ failureCategory,
+ paths.length,
+ );
+ }, [surface, failureCategory, paths.length]);
+
+ function externalUrlFor(id: RecoveryPathId): string {
+ return id === "contact_support" ? SUPPORT_URL : DOCS_URL;
+ }
+
+ function routeFor(id: RecoveryPathId): string {
+ if (id === "edit_workflow") {
+ return workflowPermanentId
+ ? `/workflows/${workflowPermanentId}/build`
+ : "/workflows";
+ }
+ return "/credentials";
+ }
+
+ async function handlePathClick(path: RecoveryPath): Promise<void> {
+ RecoveryGuidanceTelemetry.recoveryPathChosen(
+ surface,
+ failureCategory,
+ path.id,
+ );
+
+ if (path.kind === "retry") {
+ try {
+ await onRetry?.();
+ RecoveryGuidanceTelemetry.recoveryOutcome(
+ surface,
+ failureCategory,
+ path.id,
+ "retry_started",
+ );
+ } catch {
+ RecoveryGuidanceTelemetry.recoveryOutcome(
+ surface,
+ failureCategory,
+ path.id,
+ "retry_failed_to_start",
+ );
+ }
+ return;
+ }
+
+ if (path.kind === "external") {
+ const url = externalUrlFor(path.id);
+ // Capture before navigating: a mailto hand-off can pause JS and drop a
+ // post-navigation capture.
+ RecoveryGuidanceTelemetry.recoveryOutcome(
+ surface,
+ failureCategory,
+ path.id,
+ "opened",
+ );
+ if (url.startsWith("mailto:")) {
+ window.location.href = url;
+ } else {
+ window.open(url, "_blank", "noopener,noreferrer");
+ }
+ return;
+ }
+
+ navigate(routeFor(path.id));
+ RecoveryGuidanceTelemetry.recoveryOutcome(
+ surface,
+ failureCategory,
+ path.id,
+ "navigated",
+ );
+ }
+
+ return (
+ <div
+ data-testid="first-run-recovery-guidance"
+ className="space-y-2 border-t border-red-600/40 pt-3"
+ >
+ <div className="text-sm font-medium">
+ Not sure what to do next? Try one of these:
+ </div>
+ <div className="flex flex-wrap gap-2">
+ {paths.map((path) => (
+ <Button
+ key={path.id}
+ size="sm"
+ variant="secondary"
+ data-testid={`recovery-path-${path.id}`}
+ onClick={() => {
+ void handlePathClick(path);
+ }}
+ >
+ {path.label}
+ </Button>
+ ))}
+ </div>
+ </div>
+ );
+}
+
+export { FirstRunRecoveryGuidance };
diff --git a/skyvern-frontend/src/components/onboarding/GetStartedModal.test.tsx b/skyvern-frontend/src/components/onboarding/GetStartedModal.test.tsx
new file mode 100644
index 000000000..06f73cd2b
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/GetStartedModal.test.tsx
@@ -0,0 +1,127 @@
+// @vitest-environment jsdom
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { OnboardingState } from "@/store/onboarding/types";
+
+const { mockUpdateState, mockMutate, mockTelemetry, mutationState, flagState } =
+ vi.hoisted(() => ({
+ mockUpdateState: vi.fn(),
+ mockMutate: vi.fn(),
+ mockTelemetry: {
+ registerVariant: vi.fn(),
+ flowStarted: vi.fn(),
+ modalOpened: vi.fn(),
+ abVariantAssigned: vi.fn(),
+ modalSkipped: vi.fn(),
+ modalTemplateSelected: vi.fn(),
+ },
+ mutationState: { isPending: false },
+ flagState: { variant: "template-first" },
+ }));
+
+const baseState: OnboardingState = {
+ tour_completed_at: null,
+ modal_dismissed_at: null,
+ first_save_at: null,
+ first_run_at: null,
+ ab_variant: "template-first",
+ user_intent: "fill_forms",
+ seen_canvas: null,
+ seen_node_adder: null,
+ seen_sidebar: null,
+ seen_save_run: null,
+};
+
+vi.mock("posthog-js/react", () => ({
+ useFeatureFlagVariantKey: () => flagState.variant,
+}));
+
+vi.mock("@/store/onboarding/useOnboardingState", () => ({
+ useOnboardingState: () => ({
+ state: { ...baseState, ab_variant: flagState.variant },
+ isLoading: false,
+ updateState: mockUpdateState,
+ isNewUser: true,
+ abVariant: flagState.variant,
+ }),
+}));
+
+vi.mock("./CopilotCTAStep", async () => {
+ const { useEffect } = await import("react");
+ return {
+ CopilotCTAStep: (props: {
+ onSkip: () => void;
+ onBusyChange?: (busy: boolean) => void;
+ }) => {
+ useEffect(() => {
+ props.onBusyChange?.(true);
+ }, [props]);
+ return (
+ <button type="button" onClick={props.onSkip}>
+ child-skip
+ </button>
+ );
+ },
+ };
+});
+
+vi.mock("@/routes/workflows/hooks/useGlobalWorkflowsQuery", () => ({
+ useGlobalWorkflowsQuery: () => ({ data: [], isLoading: false }),
+}));
+
+vi.mock("@/routes/workflows/hooks/useCreateWorkflowMutation", () => ({
+ useCreateWorkflowMutation: () => ({
+ mutate: mockMutate,
+ isPending: mutationState.isPending,
+ }),
+}));
+
+vi.mock("@/util/onboarding/OnboardingTelemetry", () => ({
+ OnboardingTelemetry: mockTelemetry,
+}));
+
+import { GetStartedModal } from "./GetStartedModal";
+
+afterEach(() => {
+ cleanup();
+ vi.clearAllMocks();
+ mutationState.isPending = false;
+ flagState.variant = "template-first";
+});
+
+describe("GetStartedModal skip-while-creating guard", () => {
+ it("blocks Skip while a template workflow creation is in flight", () => {
+ mutationState.isPending = true;
+ render(<GetStartedModal hasWorkflows={false} isLoading={false} />);
+ const skip = screen.getByRole("button", { name: "Skip" });
+ expect((skip as HTMLButtonElement).disabled).toBe(true);
+ fireEvent.click(skip);
+ expect(mockTelemetry.modalSkipped).not.toHaveBeenCalled();
+ expect(mockUpdateState).not.toHaveBeenCalledWith(
+ expect.objectContaining({ modal_dismissed_at: expect.any(String) }),
+ );
+ });
+
+ it("dismisses on Skip when no creation is pending", () => {
+ mutationState.isPending = false;
+ render(<GetStartedModal hasWorkflows={false} isLoading={false} />);
+ fireEvent.click(screen.getByRole("button", { name: "Skip" }));
+ expect(mockTelemetry.modalSkipped).toHaveBeenCalledTimes(1);
+ expect(mockUpdateState).toHaveBeenCalledWith(
+ expect.objectContaining({ modal_dismissed_at: expect.any(String) }),
+ );
+ });
+
+ it("ignores skip and Escape-close while the copilot step reports busy", () => {
+ flagState.variant = "copilot-first";
+ render(<GetStartedModal hasWorkflows={false} isLoading={false} />);
+
+ fireEvent.click(screen.getByRole("button", { name: "child-skip" }));
+ fireEvent.keyDown(document, { key: "Escape" });
+
+ expect(mockTelemetry.modalSkipped).not.toHaveBeenCalled();
+ expect(mockUpdateState).not.toHaveBeenCalledWith(
+ expect.objectContaining({ modal_dismissed_at: expect.any(String) }),
+ );
+ });
+});
diff --git a/skyvern-frontend/src/components/onboarding/GetStartedModal.tsx b/skyvern-frontend/src/components/onboarding/GetStartedModal.tsx
new file mode 100644
index 000000000..dfb6cbab3
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/GetStartedModal.tsx
@@ -0,0 +1,332 @@
+import { useEffect, useRef, useState } from "react";
+import { useFeatureFlagVariantKey } from "posthog-js/react";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { Button } from "@/components/ui/button";
+import { Skeleton } from "@/components/ui/skeleton";
+import { useOnboardingState } from "@/store/onboarding/useOnboardingState";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
+import { ClickIcon } from "@/components/icons/ClickIcon";
+import { ExtractIcon } from "@/components/icons/ExtractIcon";
+import { SearchIcon } from "@/components/icons/SearchIcon";
+import { CompassIcon } from "@/components/icons/CompassIcon";
+import { cn } from "@/util/utils";
+import { useGlobalWorkflowsQuery } from "@/routes/workflows/hooks/useGlobalWorkflowsQuery";
+import { useCreateWorkflowMutation } from "@/routes/workflows/hooks/useCreateWorkflowMutation";
+import { convert } from "@/routes/workflows/editor/workflowEditorUtils";
+import type { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";
+import {
+ getTemplatesForIntent,
+ getTemplateIcon,
+ getSetupTime,
+} from "./templateUtils";
+import { CopilotCTAStep } from "./CopilotCTAStep";
+import {
+ type ABVariant,
+ DEFAULT_VARIANT,
+ VARIANTS,
+ isABVariant,
+ EXPERIMENT,
+} from "@/util/onboarding/experimentConfig";
+
+const SURFACE = "dashboard" as const;
+
+type Step = "intent" | "templates";
+
+type IntentOption = {
+ id: string;
+ label: string;
+ description: string;
+ icon: React.FC<{ className?: string }>;
+};
+
+const intentOptions: IntentOption[] = [
+ {
+ id: "fill_forms",
+ label: "Fill out forms",
+ description: "Automate form submissions across websites",
+ icon: ClickIcon,
+ },
+ {
+ id: "extract_data",
+ label: "Extract data from websites",
+ description: "Scrape and collect data at scale",
+ icon: ExtractIcon,
+ },
+ {
+ id: "monitor_website",
+ label: "Monitor a website for changes",
+ description: "Track updates and get notified",
+ icon: SearchIcon,
+ },
+ {
+ id: "something_else",
+ label: "Something else",
+ description: "Other browser automation tasks",
+ icon: CompassIcon,
+ },
+];
+
+type Props = {
+ hasWorkflows: boolean;
+ isLoading: boolean;
+};
+
+function resolveVariant(
+ stateVariant: string | null | undefined,
+ flagVariant: string | boolean | undefined,
+): ABVariant {
+ if (isABVariant(stateVariant)) {
+ return stateVariant;
+ }
+ return flagVariant === VARIANTS.COPILOT_FIRST
+ ? VARIANTS.COPILOT_FIRST
+ : DEFAULT_VARIANT;
+}
+
+function GetStartedModal({ hasWorkflows, isLoading }: Readonly<Props>) {
+ const { state, updateState } = useOnboardingState();
+ const flagVariant = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
+ const [step, setStep] = useState<Step>("intent");
+ const [selectedIntent, setSelectedIntent] = useState<string | null>(null);
+ const [copilotStepBusy, setCopilotStepBusy] = useState(false);
+ const openedRef = useRef(false);
+ const variantAssignedRef = useRef(false);
+
+ const variant = resolveVariant(state?.ab_variant, flagVariant);
+
+ const { data: globalTemplates = [], isLoading: templatesLoading } =
+ useGlobalWorkflowsQuery();
+ const createWorkflowMutation = useCreateWorkflowMutation();
+
+ const onboardingEnabled = isABVariant(flagVariant);
+
+ const isOpen =
+ onboardingEnabled &&
+ !isLoading &&
+ !hasWorkflows &&
+ state !== null &&
+ state.modal_dismissed_at === null &&
+ state.first_save_at === null;
+
+ useEffect(() => {
+ if (state?.user_intent && state.modal_dismissed_at === null) {
+ setSelectedIntent(state.user_intent);
+ setStep("templates");
+ }
+ }, [state?.user_intent, state?.modal_dismissed_at]);
+
+ useEffect(() => {
+ if (isOpen && !openedRef.current) {
+ openedRef.current = true;
+ OnboardingTelemetry.registerVariant(variant);
+ OnboardingTelemetry.flowStarted(SURFACE);
+ OnboardingTelemetry.modalOpened(SURFACE);
+ }
+ }, [isOpen, variant]);
+
+ useEffect(() => {
+ if (!state || state.ab_variant !== null || variantAssignedRef.current)
+ return;
+ // only persist a variant once the flag resolves to a real arm; an unloaded/disabled flag (pre-load, 0% rollout, rollback) must not bias the split
+ if (!isABVariant(flagVariant)) return;
+ variantAssignedRef.current = true;
+ updateState({ ab_variant: variant });
+ OnboardingTelemetry.abVariantAssigned(SURFACE, variant);
+ }, [state, variant, flagVariant, updateState]);
+
+ function handleSelectIntent(intentId: string) {
+ setSelectedIntent(intentId);
+ }
+
+ function handleContinue() {
+ if (!selectedIntent) return;
+ updateState({ user_intent: selectedIntent });
+ setStep("templates");
+ }
+
+ function handleBack() {
+ setStep("intent");
+ }
+
+ function handleSkip() {
+ if (createWorkflowMutation.isPending || copilotStepBusy) return;
+ OnboardingTelemetry.modalSkipped(SURFACE);
+ updateState({ modal_dismissed_at: new Date().toISOString() });
+ }
+
+ function handleTemplateSelect(template: WorkflowApiResponse) {
+ if (createWorkflowMutation.isPending) return;
+ OnboardingTelemetry.modalTemplateSelected(
+ SURFACE,
+ template.workflow_permanent_id,
+ selectedIntent!,
+ );
+ const cloned = convert({
+ ...template,
+ title: `${template.title} (copy)`,
+ });
+ // Completion telemetry fires from useCreateWorkflowMutation (it owns the
+ // navigation that unmounts this modal); modal re-display is gated by
+ // first_save_at + hasWorkflows, so no dismiss write is needed here.
+ createWorkflowMutation.mutate({ ...cloned, _via: "onboarding_template" });
+ }
+
+ const filteredTemplates =
+ selectedIntent && globalTemplates.length > 0
+ ? getTemplatesForIntent(globalTemplates, selectedIntent)
+ : [];
+
+ return (
+ <Dialog open={isOpen} onOpenChange={() => handleSkip()}>
+ <DialogContent
+ className="max-w-xl"
+ onPointerDownOutside={(e) => e.preventDefault()}
+ >
+ {step === "intent" ? (
+ <>
+ <DialogHeader>
+ <DialogTitle className="text-xl">
+ What do you want to automate?
+ </DialogTitle>
+ <DialogDescription>
+ Pick the option that best describes your goal. You can always
+ change this later.
+ </DialogDescription>
+ </DialogHeader>
+ <div className="grid grid-cols-2 gap-3 py-2">
+ {intentOptions.map((option) => {
+ const Icon = option.icon;
+ const isSelected = selectedIntent === option.id;
+ return (
+ <button
+ key={option.id}
+ type="button"
+ onClick={() => handleSelectIntent(option.id)}
+ className={cn(
+ "flex flex-col items-start gap-2 rounded-lg border p-4 text-left transition-colors hover:bg-muted/50",
+ isSelected
+ ? "border-primary bg-primary/5"
+ : "border-border",
+ )}
+ >
+ <Icon className="h-6 w-6 text-primary" />
+ <div>
+ <p className="text-sm font-medium">{option.label}</p>
+ <p className="text-xs text-muted-foreground">
+ {option.description}
+ </p>
+ </div>
+ </button>
+ );
+ })}
+ </div>
+ <DialogFooter className="gap-2 sm:gap-0">
+ <Button variant="ghost" size="sm" onClick={handleSkip}>
+ Skip
+ </Button>
+ <Button
+ size="sm"
+ disabled={!selectedIntent}
+ onClick={handleContinue}
+ >
+ Continue
+ </Button>
+ </DialogFooter>
+ </>
+ ) : variant === VARIANTS.COPILOT_FIRST ? (
+ <CopilotCTAStep
+ selectedIntent={selectedIntent!}
+ onBack={handleBack}
+ onSkip={handleSkip}
+ onDismiss={() =>
+ updateState({ modal_dismissed_at: new Date().toISOString() })
+ }
+ onBusyChange={setCopilotStepBusy}
+ />
+ ) : (
+ <>
+ <DialogHeader>
+ <DialogTitle className="text-xl">
+ Pick a template to start
+ </DialogTitle>
+ <DialogDescription>
+ Choose a pre-built workflow and customize it in the editor.
+ </DialogDescription>
+ </DialogHeader>
+ <div className="grid grid-cols-2 gap-3 py-2">
+ {templatesLoading ? (
+ Array.from({ length: 4 }).map((_, i) => (
+ <Skeleton key={i} className="h-28 rounded-lg" />
+ ))
+ ) : filteredTemplates.length > 0 ? (
+ filteredTemplates.map((template) => {
+ const Icon = getTemplateIcon(template);
+ return (
+ <button
+ key={template.workflow_permanent_id}
+ type="button"
+ disabled={createWorkflowMutation.isPending}
+ onClick={() => handleTemplateSelect(template)}
+ className={cn(
+ "flex flex-col items-start gap-2 rounded-lg border border-border p-4 text-left transition-colors",
+ "hover:border-primary hover:bg-primary/5",
+ createWorkflowMutation.isPending &&
+ "pointer-events-none opacity-50",
+ )}
+ >
+ <Icon className="h-6 w-6 text-primary" />
+ <div className="min-w-0 self-stretch">
+ <p className="truncate text-sm font-medium">
+ {template.title}
+ </p>
+ {template.description && (
+ <p className="line-clamp-2 text-xs text-muted-foreground">
+ {template.description}
+ </p>
+ )}
+ </div>
+ <span className="text-xs text-muted-foreground">
+ ~{getSetupTime(template)} setup
+ </span>
+ </button>
+ );
+ })
+ ) : (
+ <p className="col-span-2 py-8 text-center text-sm text-muted-foreground">
+ No templates available yet.
+ </p>
+ )}
+ </div>
+ <DialogFooter className="gap-2 sm:gap-0">
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={handleBack}
+ disabled={createWorkflowMutation.isPending}
+ >
+ Back
+ </Button>
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={handleSkip}
+ disabled={createWorkflowMutation.isPending}
+ >
+ Skip
+ </Button>
+ </DialogFooter>
+ </>
+ )}
+ </DialogContent>
+ </Dialog>
+ );
+}
+
+export { GetStartedModal };
diff --git a/skyvern-frontend/src/components/onboarding/OnboardingEmptyState.test.tsx b/skyvern-frontend/src/components/onboarding/OnboardingEmptyState.test.tsx
new file mode 100644
index 000000000..ba6642760
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/OnboardingEmptyState.test.tsx
@@ -0,0 +1,54 @@
+import { describe, expect, it, vi } from "vitest";
+import { renderToStaticMarkup } from "react-dom/server";
+import { OnboardingEmptyState } from "./OnboardingEmptyState";
+
+vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));
+
+describe("OnboardingEmptyState", () => {
+ it("renders the title, description, and primary action", () => {
+ const html = renderToStaticMarkup(
+ <OnboardingEmptyState
+ surface="runs"
+ icon={<span data-testid="icon">icon</span>}
+ title="Your run history will appear here"
+ description="Create a workflow to get started."
+ primaryAction={{ label: "Create workflow", onClick: vi.fn() }}
+ />,
+ );
+ expect(html).toContain('data-testid="onboarding-empty-state-runs"');
+ expect(html).toContain("Your run history will appear here");
+ expect(html).toContain("Create a workflow to get started.");
+ expect(html).toContain("Create workflow");
+ expect(html).toContain('data-testid="icon"');
+ });
+
+ it("renders the secondary action when provided", () => {
+ const html = renderToStaticMarkup(
+ <OnboardingEmptyState
+ surface="dashboard"
+ icon={<span>icon</span>}
+ title="Title"
+ description="Desc"
+ primaryAction={{ label: "Primary", onClick: vi.fn() }}
+ secondaryAction={{ label: "Secondary", onClick: vi.fn() }}
+ />,
+ );
+ expect(html).toContain("Primary");
+ expect(html).toContain("Secondary");
+ });
+
+ it("omits the secondary action when not provided", () => {
+ const html = renderToStaticMarkup(
+ <OnboardingEmptyState
+ surface="runs"
+ icon={<span>icon</span>}
+ title="Title"
+ description="Desc"
+ primaryAction={{ label: "Primary", onClick: vi.fn() }}
+ />,
+ );
+ expect(html).toContain("Primary");
+ const buttonCount = (html.match(/<button/g) ?? []).length;
+ expect(buttonCount).toBe(1);
+ });
+});
diff --git a/skyvern-frontend/src/components/onboarding/OnboardingEmptyState.tsx b/skyvern-frontend/src/components/onboarding/OnboardingEmptyState.tsx
new file mode 100644
index 000000000..ab6f9b459
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/OnboardingEmptyState.tsx
@@ -0,0 +1,83 @@
+import { useEffect, useRef } from "react";
+import { Button } from "@/components/ui/button";
+import {
+ OnboardingTelemetry,
+ type Surface,
+} from "@/util/onboarding/OnboardingTelemetry";
+
+type Action = {
+ label: string;
+ onClick: () => void;
+};
+
+type Props = {
+ surface: Surface;
+ icon: React.ReactNode;
+ title: string;
+ description: string;
+ primaryAction: Action;
+ secondaryAction?: Action;
+};
+
+function OnboardingEmptyState({
+ surface,
+ icon,
+ title,
+ description,
+ primaryAction,
+ secondaryAction,
+}: Readonly<Props>) {
+ const viewedRef = useRef(false);
+
+ useEffect(() => {
+ if (viewedRef.current) return;
+ viewedRef.current = true;
+ OnboardingTelemetry.emptyStateViewed(surface);
+ }, [surface]);
+
+ return (
+ <div
+ data-testid={`onboarding-empty-state-${surface}`}
+ className="flex w-full flex-col items-center justify-center gap-4 py-16 text-center"
+ >
+ <div className="flex h-12 w-12 items-center justify-center rounded-full bg-primary/10 text-primary">
+ {icon}
+ </div>
+ <div className="max-w-sm space-y-2">
+ <h3 className="text-lg font-medium">{title}</h3>
+ <p className="text-sm text-muted-foreground">{description}</p>
+ </div>
+ <div className="flex items-center gap-3 pt-2">
+ <Button
+ size="sm"
+ onClick={() => {
+ OnboardingTelemetry.emptyStateCTAClicked(
+ surface,
+ primaryAction.label,
+ );
+ primaryAction.onClick();
+ }}
+ >
+ {primaryAction.label}
+ </Button>
+ {secondaryAction && (
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={() => {
+ OnboardingTelemetry.emptyStateCTAClicked(
+ surface,
+ secondaryAction.label,
+ );
+ secondaryAction.onClick();
+ }}
+ >
+ {secondaryAction.label}
+ </Button>
+ )}
+ </div>
+ </div>
+ );
+}
+
+export { OnboardingEmptyState };
diff --git a/skyvern-frontend/src/components/onboarding/OnboardingErrorBoundary.tsx b/skyvern-frontend/src/components/onboarding/OnboardingErrorBoundary.tsx
new file mode 100644
index 000000000..4bbe7243a
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/OnboardingErrorBoundary.tsx
@@ -0,0 +1,26 @@
+import { Component, type ReactNode } from "react";
+
+type Props = {
+ onError: () => void;
+ children: ReactNode;
+};
+
+type State = { hasError: boolean };
+
+class OnboardingErrorBoundary extends Component<Props, State> {
+ state: State = { hasError: false };
+
+ static getDerivedStateFromError(): State {
+ return { hasError: true };
+ }
+
+ componentDidCatch(): void {
+ this.props.onError();
+ }
+
+ render(): ReactNode {
+ return this.state.hasError ? null : this.props.children;
+ }
+}
+
+export { OnboardingErrorBoundary };
diff --git a/skyvern-frontend/src/components/onboarding/ProductTour.tsx b/skyvern-frontend/src/components/onboarding/ProductTour.tsx
new file mode 100644
index 000000000..15072b971
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/ProductTour.tsx
@@ -0,0 +1,48 @@
+import { useEditorOnboardingTour } from "@/hooks/useEditorOnboardingTour";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { Button } from "@/components/ui/button";
+import "@/util/onboarding/product-tour.css";
+
+function ProductTour() {
+ const { showExitDialog, onExitConfirm, onExitCancel } =
+ useEditorOnboardingTour();
+
+ return (
+ <Dialog
+ open={showExitDialog}
+ onOpenChange={(open) => {
+ if (!open) onExitCancel();
+ }}
+ >
+ <DialogContent
+ className="skyvern-tour-exit"
+ onPointerDownOutside={(e) => e.preventDefault()}
+ onEscapeKeyDown={(e) => e.preventDefault()}
+ >
+ <DialogHeader>
+ <DialogTitle>End the tour?</DialogTitle>
+ <DialogDescription>
+ You can restart anytime with Shift+? or from the overflow menu.
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <Button variant="secondary" onClick={onExitCancel}>
+ Continue tour
+ </Button>
+ <Button variant="default" onClick={onExitConfirm}>
+ End tour
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ );
+}
+
+export { ProductTour };
diff --git a/skyvern-frontend/src/components/onboarding/recoveryPaths.ts b/skyvern-frontend/src/components/onboarding/recoveryPaths.ts
new file mode 100644
index 000000000..ce2baa5ff
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/recoveryPaths.ts
@@ -0,0 +1,55 @@
+type RecoveryPathKind = "retry" | "navigate" | "external";
+
+type RecoveryPathId =
+ | "retry"
+ | "edit_workflow"
+ | "update_credentials"
+ | "view_docs"
+ | "contact_support";
+
+type RecoveryPath = {
+ id: RecoveryPathId;
+ label: string;
+ kind: RecoveryPathKind;
+};
+
+const RETRY: RecoveryPath = { id: "retry", label: "Retry run", kind: "retry" };
+const EDIT_WORKFLOW: RecoveryPath = {
+ id: "edit_workflow",
+ label: "Edit workflow",
+ kind: "navigate",
+};
+const UPDATE_CREDENTIALS: RecoveryPath = {
+ id: "update_credentials",
+ label: "Update credentials",
+ kind: "navigate",
+};
+const VIEW_DOCS: RecoveryPath = {
+ id: "view_docs",
+ label: "View troubleshooting docs",
+ kind: "external",
+};
+const CONTACT_SUPPORT: RecoveryPath = {
+ id: "contact_support",
+ label: "Contact support",
+ kind: "external",
+};
+
+// Maps a backend failure_category (snake_case, may be null) to recovery paths.
+// Always returns at least two paths so the failure state offers real choices.
+function getRecoveryPaths(failureCategory: string | null): RecoveryPath[] {
+ const category = failureCategory?.toLowerCase() ?? "";
+ if (/credential|auth|login|password|2fa|verification/.test(category)) {
+ return [UPDATE_CREDENTIALS, RETRY];
+ }
+ if (/element|selector|navigation|not_found|locator|interact/.test(category)) {
+ return [EDIT_WORKFLOW, RETRY];
+ }
+ if (/network|timeout|proxy|rate_limit|connection|throttle/.test(category)) {
+ return [RETRY, VIEW_DOCS];
+ }
+ return [RETRY, EDIT_WORKFLOW, CONTACT_SUPPORT];
+}
+
+export { getRecoveryPaths };
+export type { RecoveryPath, RecoveryPathId, RecoveryPathKind };
diff --git a/skyvern-frontend/src/components/onboarding/templateUtils.ts b/skyvern-frontend/src/components/onboarding/templateUtils.ts
new file mode 100644
index 000000000..0cedef97d
--- /dev/null
+++ b/skyvern-frontend/src/components/onboarding/templateUtils.ts
@@ -0,0 +1,66 @@
+import { ClickIcon } from "@/components/icons/ClickIcon";
+import { ExtractIcon } from "@/components/icons/ExtractIcon";
+import { SearchIcon } from "@/components/icons/SearchIcon";
+import { GovernmentIcon } from "@/components/icons/GovernmentIcon";
+import { BagIcon } from "@/components/icons/BagIcon";
+import { ReceiptIcon } from "@/components/icons/ReceiptIcon";
+import { DocumentIcon } from "@/components/icons/DocumentIcon";
+import type { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";
+
+const INTENT_KEYWORDS: Record<string, RegExp> = {
+ fill_forms:
+ /form|application|submit|apply|registration|contact|intake|questionnaire|filing/i,
+ extract_data:
+ /extract|scrape|data|lookup|search|download|invoice|collect|report/i,
+ monitor_website:
+ /monitor|track|watch|check|alert|notification|change|verify/i,
+};
+
+function getTemplatesForIntent(
+ templates: WorkflowApiResponse[],
+ intent: string,
+): WorkflowApiResponse[] {
+ const pattern = INTENT_KEYWORDS[intent];
+ if (!pattern) return templates.slice(0, 4);
+
+ const matches = templates.filter((t) => {
+ const text = `${t.title} ${t.description}`;
+ return pattern.test(text);
+ });
+
+ if (matches.length >= 4) return matches.slice(0, 4);
+
+ const matchedIds = new Set(matches.map((m) => m.workflow_permanent_id));
+ const remaining = templates.filter(
+ (t) => !matchedIds.has(t.workflow_permanent_id),
+ );
+ return [...matches, ...remaining].slice(0, 4);
+}
+
+function getTemplateIcon(
+ template: WorkflowApiResponse,
+): React.FC<{ className?: string }> {
+ const text = `${template.title} ${template.description}`.toLowerCase();
+ if (/form|application|submit|registration|contact/.test(text))
+ return ClickIcon;
+ if (/invoice|receipt|billing|payment/.test(text)) return ReceiptIcon;
+ if (/extract|scrape|data|download/.test(text)) return ExtractIcon;
+ if (/monitor|track|watch|check|alert/.test(text)) return SearchIcon;
+ if (/government|entity|ein|sam|irs|federal/.test(text)) return GovernmentIcon;
+ if (/job|career|hiring|recruit|employ/.test(text)) return BagIcon;
+ return DocumentIcon;
+}
+
+function getSetupTime(template: WorkflowApiResponse): string {
+ const blocks = template.workflow_definition.blocks.length;
+ if (blocks <= 3) return "2 min";
+ if (blocks <= 7) return "5 min";
+ return "10 min";
+}
+
+export {
+ INTENT_KEYWORDS,
+ getTemplatesForIntent,
+ getTemplateIcon,
+ getSetupTime,
+};
diff --git a/skyvern-frontend/src/components/ui/checkbox.tsx b/skyvern-frontend/src/components/ui/checkbox.tsx
index 663242c8c..556ba4743 100644
--- a/skyvern-frontend/src/components/ui/checkbox.tsx
+++ b/skyvern-frontend/src/components/ui/checkbox.tsx
@@ -1,17 +1,18 @@
import * as React from "react";
import * as CheckboxPrimitive from "@radix-ui/react-checkbox";
-import { CheckIcon } from "@radix-ui/react-icons";
+import { CheckIcon, DashIcon } from "@radix-ui/react-icons";
import { cn } from "@/util/utils";
const Checkbox = React.forwardRef<
React.ElementRef<typeof CheckboxPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof CheckboxPrimitive.Root>
->(({ className, ...props }, ref) => (
+>(({ className, checked, ...props }, ref) => (
<CheckboxPrimitive.Root
ref={ref}
+ checked={checked}
className={cn(
- "peer h-4 w-4 shrink-0 rounded-sm border border-primary shadow focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground",
+ "peer h-4 w-4 shrink-0 rounded-sm border border-primary shadow focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=indeterminate]:bg-primary data-[state=checked]:text-primary-foreground data-[state=indeterminate]:text-primary-foreground",
className,
)}
{...props}
@@ -19,7 +20,11 @@ const Checkbox = React.forwardRef<
<CheckboxPrimitive.Indicator
className={cn("flex items-center justify-center text-current")}
>
- <CheckIcon className="h-4 w-4" />
+ {checked === "indeterminate" ? (
+ <DashIcon className="h-4 w-4" />
+ ) : (
+ <CheckIcon className="h-4 w-4" />
+ )}
</CheckboxPrimitive.Indicator>
</CheckboxPrimitive.Root>
));
diff --git a/skyvern-frontend/src/hooks/useAzureClientCredentialToken.ts b/skyvern-frontend/src/hooks/useAzureClientCredentialToken.ts
index 0f0974519..b2ed1b423 100644
--- a/skyvern-frontend/src/hooks/useAzureClientCredentialToken.ts
+++ b/skyvern-frontend/src/hooks/useAzureClientCredentialToken.ts
@@ -7,6 +7,7 @@ import {
CreateAzureClientSecretCredentialRequest,
} from "@/api/types";
import { useToast } from "@/components/ui/use-toast";
+import { useClearOrganizationAuthToken } from "./useClearOrganizationAuthToken";
export function useAzureClientCredentialToken() {
const credentialGetter = useCredentialGetter();
@@ -14,7 +15,7 @@ export function useAzureClientCredentialToken() {
const { toast } = useToast();
const { data: azureOrganizationAuthToken, isLoading } =
- useQuery<AzureOrganizationAuthToken>({
+ useQuery<AzureOrganizationAuthToken | null>({
queryKey: ["azureOrganizationAuthToken"],
queryFn: async () => {
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -57,10 +58,19 @@ export function useAzureClientCredentialToken() {
},
});
+ const clearCredentialMutation = useClearOrganizationAuthToken({
+ providerPath: "azure_credential",
+ queryKey: "azureOrganizationAuthToken",
+ successDescription: "Azure Client Secret Credential cleared successfully",
+ errorDescription: "Failed to clear Azure Client Secret Credential",
+ });
+
return {
azureOrganizationAuthToken,
isLoading,
createOrUpdateToken: createOrUpdateTokenMutation.mutate,
isUpdating: createOrUpdateTokenMutation.isPending,
+ clearCredential: clearCredentialMutation.mutate,
+ isClearing: clearCredentialMutation.isPending,
};
}
diff --git a/skyvern-frontend/src/hooks/useBitwardenCredential.ts b/skyvern-frontend/src/hooks/useBitwardenCredential.ts
index 4aeb0da67..e645cc267 100644
--- a/skyvern-frontend/src/hooks/useBitwardenCredential.ts
+++ b/skyvern-frontend/src/hooks/useBitwardenCredential.ts
@@ -7,6 +7,7 @@ import {
CreateBitwardenCredentialRequest,
} from "@/api/types";
import { useToast } from "@/components/ui/use-toast";
+import { useClearOrganizationAuthToken } from "./useClearOrganizationAuthToken";
export function useBitwardenCredential() {
const credentialGetter = useCredentialGetter();
@@ -14,7 +15,7 @@ export function useBitwardenCredential() {
const { toast } = useToast();
const { data: bitwardenOrganizationAuthToken, isLoading } =
- useQuery<BitwardenOrganizationAuthToken>({
+ useQuery<BitwardenOrganizationAuthToken | null>({
queryKey: ["bitwardenOrganizationAuthToken"],
queryFn: async () => {
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -60,10 +61,19 @@ export function useBitwardenCredential() {
},
});
+ const clearCredentialMutation = useClearOrganizationAuthToken({
+ providerPath: "bitwarden",
+ queryKey: "bitwardenOrganizationAuthToken",
+ successDescription: "Bitwarden credential cleared successfully",
+ errorDescription: "Failed to clear Bitwarden credential",
+ });
+
return {
bitwardenOrganizationAuthToken,
isLoading,
createOrUpdateToken: createOrUpdateTokenMutation.mutate,
isUpdating: createOrUpdateTokenMutation.isPending,
+ clearCredential: clearCredentialMutation.mutate,
+ isClearing: clearCredentialMutation.isPending,
};
}
diff --git a/skyvern-frontend/src/hooks/useClearOrganizationAuthToken.ts b/skyvern-frontend/src/hooks/useClearOrganizationAuthToken.ts
new file mode 100644
index 000000000..13ac66d0b
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useClearOrganizationAuthToken.ts
@@ -0,0 +1,54 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+import { ClearOrganizationAuthTokenResponse } from "@/api/types";
+import { getClient } from "@/api/AxiosClient";
+import { useToast } from "@/components/ui/use-toast";
+import { useCredentialGetter } from "./useCredentialGetter";
+
+type ClearOrganizationAuthTokenOptions = {
+ providerPath: string;
+ queryKey: string;
+ successDescription: string;
+ errorDescription: string;
+};
+
+export function useClearOrganizationAuthToken({
+ providerPath,
+ queryKey,
+ successDescription,
+ errorDescription,
+}: ClearOrganizationAuthTokenOptions) {
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const { toast } = useToast();
+
+ return useMutation({
+ mutationFn: async () => {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return await client
+ .delete(`/credentials/${providerPath}`)
+ .then(
+ (response) => response.data as ClearOrganizationAuthTokenResponse,
+ );
+ },
+ onSuccess: () => {
+ queryClient.setQueryData([queryKey], null);
+ queryClient.invalidateQueries({ queryKey: [queryKey] });
+ toast({
+ title: "Success",
+ description: successDescription,
+ });
+ },
+ onError: (error: unknown) => {
+ const message =
+ (error as { response?: { data?: { detail?: string } } })?.response?.data
+ ?.detail ||
+ (error as Error)?.message ||
+ errorDescription;
+ toast({
+ title: "Error",
+ description: message,
+ variant: "destructive",
+ });
+ },
+ });
+}
diff --git a/skyvern-frontend/src/hooks/useContextualHints.test.tsx b/skyvern-frontend/src/hooks/useContextualHints.test.tsx
new file mode 100644
index 000000000..b4642c09c
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useContextualHints.test.tsx
@@ -0,0 +1,171 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { render } from "@testing-library/react";
+import { MemoryRouter } from "react-router-dom";
+import type { ReactNode } from "react";
+import { driver } from "driver.js";
+import posthog from "posthog-js";
+import { useContextualHints } from "./useContextualHints";
+import {
+ OnboardingContext,
+ type OnboardingContextValue,
+} from "@/store/onboarding/useOnboardingState";
+import type { OnboardingState } from "@/store/onboarding/types";
+
+type DriverCloseCb = () => void;
+
+vi.mock("driver.js", () => ({
+ driver: vi.fn(() => ({ highlight: vi.fn(), destroy: vi.fn() })),
+}));
+
+vi.mock("posthog-js", () => ({
+ default: { capture: vi.fn(), register: vi.fn() },
+}));
+
+const flagState = vi.hoisted(() => ({
+ variant: "template-first" as string | boolean | undefined,
+}));
+vi.mock("posthog-js/react", () => ({
+ useFeatureFlagVariantKey: () => flagState.variant,
+}));
+
+const COMPLETED_STATE: OnboardingState = {
+ tour_completed_at: "2026-06-01T00:00:00Z",
+ modal_dismissed_at: null,
+ first_save_at: null,
+ first_run_at: null,
+ ab_variant: "template-first",
+ user_intent: null,
+ seen_canvas: null,
+ seen_node_adder: null,
+ seen_sidebar: null,
+ seen_save_run: null,
+};
+
+function makeCtx(
+ overrides: Partial<OnboardingContextValue> = {},
+): OnboardingContextValue {
+ return {
+ state: COMPLETED_STATE,
+ isLoading: false,
+ updateState: vi.fn(),
+ isNewUser: false,
+ abVariant: "template-first",
+ ...overrides,
+ };
+}
+
+function Harness() {
+ useContextualHints();
+ return null;
+}
+
+function renderAt(path: string, ctx: OnboardingContextValue) {
+ function Wrapper({ children }: { children: ReactNode }) {
+ return (
+ <MemoryRouter initialEntries={[path]}>
+ <OnboardingContext.Provider value={ctx}>
+ {children}
+ </OnboardingContext.Provider>
+ </MemoryRouter>
+ );
+ }
+ return render(<Harness />, { wrapper: Wrapper });
+}
+
+function addAnchor(attr: string, value: string) {
+ const el = document.createElement("div");
+ el.setAttribute(attr, value);
+ document.body.appendChild(el);
+}
+
+const DRIVE_AND_SHOW_MS = 500 + 800;
+
+describe("useContextualHints", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.clearAllMocks();
+ flagState.variant = "template-first";
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ document.body.innerHTML = "";
+ });
+
+ it("shows the block hint on the editor route once the anchor is present", () => {
+ addAnchor("data-tour", "node-adder");
+ const ctx = makeCtx();
+ renderAt("/workflows/wpid_1/build", ctx);
+
+ vi.advanceTimersByTime(DRIVE_AND_SHOW_MS);
+
+ expect(vi.mocked(driver)).toHaveBeenCalledOnce();
+ const instance = vi.mocked(driver).mock.results[0]!.value;
+ expect(instance.highlight).toHaveBeenCalledWith(
+ expect.objectContaining({ element: "[data-tour='node-adder']" }),
+ );
+ expect(ctx.updateState).toHaveBeenCalledWith({ seen_hint_block: true });
+ expect(posthog.capture).toHaveBeenCalledWith("onboarding.hint_shown", {
+ surface: "editor",
+ hint_id: "add-another-block",
+ layer: 2,
+ });
+ });
+
+ it("does not show when the Layer-1 tour is not completed", () => {
+ addAnchor("data-tour", "node-adder");
+ const ctx = makeCtx({
+ state: { ...COMPLETED_STATE, tour_completed_at: null },
+ });
+ renderAt("/workflows/wpid_1/build", ctx);
+
+ vi.advanceTimersByTime(6000);
+
+ expect(vi.mocked(driver)).not.toHaveBeenCalled();
+ });
+
+ it("does not show when the experiment flag is not an arm (kill-switch)", () => {
+ addAnchor("data-tour", "node-adder");
+ flagState.variant = false;
+ renderAt("/workflows/wpid_1/build", makeCtx());
+
+ vi.advanceTimersByTime(6000);
+
+ expect(vi.mocked(driver)).not.toHaveBeenCalled();
+ });
+
+ it("does not re-show a hint already marked seen", () => {
+ addAnchor("data-tour", "node-adder");
+ const ctx = makeCtx({
+ state: { ...COMPLETED_STATE, seen_hint_block: true },
+ });
+ renderAt("/workflows/wpid_1/build", ctx);
+
+ vi.advanceTimersByTime(6000);
+
+ expect(vi.mocked(driver)).not.toHaveBeenCalled();
+ });
+
+ it("does not show the run hint when its anchor is absent", () => {
+ renderAt("/runs", makeCtx());
+
+ vi.advanceTimersByTime(8000);
+
+ expect(vi.mocked(driver)).not.toHaveBeenCalled();
+ });
+
+ it("emits hint_dismissed when the user closes the hint", () => {
+ addAnchor("data-tour", "node-adder");
+ renderAt("/workflows/wpid_1/build", makeCtx());
+ vi.advanceTimersByTime(DRIVE_AND_SHOW_MS);
+
+ const config = vi.mocked(driver).mock.calls[0]![0]!;
+ (config.onCloseClick as DriverCloseCb)();
+
+ expect(posthog.capture).toHaveBeenCalledWith("onboarding.hint_dismissed", {
+ surface: "editor",
+ hint_id: "add-another-block",
+ layer: 2,
+ });
+ });
+});
diff --git a/skyvern-frontend/src/hooks/useContextualHints.ts b/skyvern-frontend/src/hooks/useContextualHints.ts
new file mode 100644
index 000000000..c5adf382b
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useContextualHints.ts
@@ -0,0 +1,142 @@
+import { useCallback, useEffect, useRef } from "react";
+import { driver, type Config } from "driver.js";
+import { useLocation } from "react-router-dom";
+import { useFeatureFlagVariantKey } from "posthog-js/react";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
+import { EXPERIMENT, isABVariant } from "@/util/onboarding/experimentConfig";
+import {
+ HINT_REGISTRY,
+ type ContextualHint,
+} from "@/util/onboarding/contextualHints";
+import type {
+ OnboardingState,
+ OnboardingStatePatch,
+} from "@/store/onboarding/types";
+
+const SHOW_DELAY_MS = 800;
+const POLL_INTERVAL_MS = 500;
+const MAX_POLLS = 10;
+
+// Inlined rather than imported from cloud/ so this hook stays in the OSS-synced src/ tree.
+const BASE_DRIVER_CONFIG: Config = {
+ popoverClass: "skyvern-onboarding",
+ overlayOpacity: 0.5,
+ animate: true,
+ smoothScroll: true,
+ allowClose: true,
+ allowKeyboardControl: true,
+ stagePadding: 8,
+ stageRadius: 8,
+};
+
+function selectHint(
+ pathname: string,
+ state: OnboardingState,
+): ContextualHint | null {
+ for (const hint of HINT_REGISTRY) {
+ if (!hint.matchRoute(pathname)) continue;
+ if (state[hint.seenKey]) continue;
+ if (!hint.prerequisite(state)) continue;
+ return hint;
+ }
+ return null;
+}
+
+function useContextualHints(): void {
+ const onboarding = useOnboardingStateOptional();
+ const { pathname } = useLocation();
+ const flagVariant = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
+ const driverRef = useRef<ReturnType<typeof driver> | null>(null);
+ const activeHintRef = useRef<string | null>(null);
+
+ const showHint = useCallback(
+ (hint: ContextualHint) => {
+ if (activeHintRef.current) return;
+ if (document.querySelector(hint.anchor) === null) return;
+ activeHintRef.current = hint.id;
+ try {
+ const instance = driver({
+ ...BASE_DRIVER_CONFIG,
+ overlayOpacity: 0,
+ allowClose: true,
+ onCloseClick: () => {
+ OnboardingTelemetry.hintDismissed(hint.surface, hint.id);
+ driverRef.current?.destroy();
+ },
+ onDestroyed: () => {
+ activeHintRef.current = null;
+ driverRef.current = null;
+ },
+ });
+ instance.highlight({
+ element: hint.anchor,
+ popover: {
+ title: hint.popover.title,
+ description: hint.popover.description,
+ side: hint.popover.side,
+ align: hint.popover.align,
+ },
+ });
+ driverRef.current = instance;
+ // Persist "seen" at show time so a flaky close handler can never
+ // resurrect a dismissed hint (show-once frequency cap).
+ onboarding?.updateState({
+ [hint.seenKey]: true,
+ } as OnboardingStatePatch);
+ OnboardingTelemetry.hintShown(hint.surface, hint.id);
+ } catch {
+ activeHintRef.current = null;
+ }
+ },
+ [onboarding],
+ );
+
+ const state = onboarding?.state ?? null;
+ const eligible =
+ onboarding !== null &&
+ !onboarding.isLoading &&
+ state !== null &&
+ state.tour_completed_at !== null &&
+ isABVariant(flagVariant);
+
+ useEffect(() => {
+ if (!eligible || state === null) return;
+ if (activeHintRef.current) return;
+ const hint = selectHint(pathname, state);
+ if (!hint) return;
+
+ let polls = 0;
+ let showTimeout: ReturnType<typeof setTimeout> | null = null;
+ const interval = setInterval(() => {
+ polls += 1;
+ if (document.querySelector(hint.anchor) !== null) {
+ clearInterval(interval);
+ showTimeout = setTimeout(() => showHint(hint), SHOW_DELAY_MS);
+ } else if (polls >= MAX_POLLS) {
+ clearInterval(interval);
+ }
+ }, POLL_INTERVAL_MS);
+
+ return () => {
+ clearInterval(interval);
+ if (showTimeout) clearTimeout(showTimeout);
+ };
+ }, [eligible, pathname, state, showHint]);
+
+ // Tear down an active hint when navigating away from its route.
+ useEffect(() => {
+ return () => {
+ driverRef.current?.destroy();
+ };
+ }, [pathname]);
+
+ // Final unmount cleanup.
+ useEffect(() => {
+ return () => {
+ driverRef.current?.destroy();
+ };
+ }, []);
+}
+
+export { useContextualHints, selectHint };
diff --git a/skyvern-frontend/src/hooks/useCustomCredentialServiceConfig.ts b/skyvern-frontend/src/hooks/useCustomCredentialServiceConfig.ts
index 6f7e356f7..19e328d9d 100644
--- a/skyvern-frontend/src/hooks/useCustomCredentialServiceConfig.ts
+++ b/skyvern-frontend/src/hooks/useCustomCredentialServiceConfig.ts
@@ -9,6 +9,7 @@ import {
CustomCredentialServiceConfig,
} from "@/api/types";
import { useToast } from "@/components/ui/use-toast";
+import { useClearOrganizationAuthToken } from "./useClearOrganizationAuthToken";
export function useCustomCredentialServiceConfig() {
const credentialGetter = useCredentialGetter();
@@ -16,7 +17,7 @@ export function useCustomCredentialServiceConfig() {
const { toast } = useToast();
const { data: customCredentialServiceAuthToken, isLoading } =
- useQuery<CustomCredentialServiceOrganizationAuthToken>({
+ useQuery<CustomCredentialServiceOrganizationAuthToken | null>({
queryKey: ["customCredentialServiceAuthToken"],
queryFn: async () => {
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -82,6 +83,14 @@ export function useCustomCredentialServiceConfig() {
},
});
+ const clearConfigMutation = useClearOrganizationAuthToken({
+ providerPath: "custom_credential",
+ queryKey: "customCredentialServiceAuthToken",
+ successDescription:
+ "Custom credential service configuration cleared successfully",
+ errorDescription: "Failed to clear custom credential service configuration",
+ });
+
const testConnectionMutation = useMutation({
mutationFn: async (data: CreateCustomCredentialServiceConfigRequest) => {
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -122,6 +131,8 @@ export function useCustomCredentialServiceConfig() {
isLoading,
createOrUpdateConfig: createOrUpdateConfigMutation.mutate,
isUpdating: createOrUpdateConfigMutation.isPending,
+ clearConfig: clearConfigMutation.mutate,
+ isClearing: clearConfigMutation.isPending,
testConnection: testConnectionMutation.mutate,
isTesting: testConnectionMutation.isPending,
};
diff --git a/skyvern-frontend/src/hooks/useEditorOnboardingTour.test.tsx b/skyvern-frontend/src/hooks/useEditorOnboardingTour.test.tsx
new file mode 100644
index 000000000..5ecd6b8f2
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useEditorOnboardingTour.test.tsx
@@ -0,0 +1,864 @@
+import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
+import { cleanup, renderHook, act } from "@testing-library/react";
+import type { ReactNode } from "react";
+import posthog from "posthog-js";
+import { driver, type Config, type DriveStep } from "driver.js";
+import { useEditorOnboardingTour } from "./useEditorOnboardingTour";
+import {
+ OnboardingContext,
+ type OnboardingContextValue,
+} from "@/store/onboarding/useOnboardingState";
+import type { OnboardingState } from "@/store/onboarding/types";
+import { useProductTourStore } from "@/store/ProductTourStore";
+import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
+
+vi.mock("driver.js", () => ({
+ driver: vi.fn(() => ({
+ drive: vi.fn(),
+ destroy: vi.fn(),
+ moveNext: vi.fn(),
+ })),
+}));
+
+vi.mock("posthog-js", () => ({
+ default: { capture: vi.fn(), register: vi.fn() },
+}));
+
+function captureCount(event: string): number {
+ return vi
+ .mocked(posthog.capture)
+ .mock.calls.filter((call) => call[0] === event).length;
+}
+
+vi.mock("@/store/WorkflowPanelStore", async () => {
+ const { create } = await import("zustand");
+ const useWorkflowPanelStore = create(() => ({
+ workflowPanelState: { active: false, content: "parameters" },
+ }));
+ return { useWorkflowPanelStore };
+});
+
+const flagState = vi.hoisted(() => ({
+ variant: "template-first" as string | boolean | undefined,
+}));
+vi.mock("posthog-js/react", () => ({
+ useFeatureFlagVariantKey: () => flagState.variant,
+}));
+
+type DriverStepCb = (el: Element | undefined, step: DriveStep) => void;
+type DriverCloseCb = () => void;
+
+const ANCHOR_ATTRS = [
+ "editor-canvas",
+ "node-adder",
+ "sidebar-region",
+ "editor-actions",
+];
+
+function addAnchors() {
+ for (const attr of ANCHOR_ATTRS) {
+ const el = document.createElement("div");
+ el.setAttribute("data-tour", attr);
+ document.body.appendChild(el);
+ }
+}
+
+function removeAnchors() {
+ document.querySelectorAll("[data-tour]").forEach((el) => el.remove());
+}
+
+const DEFAULT_STATE: OnboardingState = {
+ tour_completed_at: null,
+ modal_dismissed_at: null,
+ first_save_at: null,
+ first_run_at: null,
+ ab_variant: null,
+ user_intent: null,
+ seen_canvas: null,
+ seen_node_adder: null,
+ seen_sidebar: null,
+ seen_save_run: null,
+};
+
+function makeContext(
+ overrides: Partial<OnboardingContextValue> = {},
+): OnboardingContextValue {
+ return {
+ state: DEFAULT_STATE,
+ isLoading: false,
+ updateState: vi.fn(),
+ isNewUser: false,
+ abVariant: null,
+ ...overrides,
+ };
+}
+
+function wrapWith(ctx: OnboardingContextValue) {
+ return function Wrapper({ children }: { children: ReactNode }) {
+ return (
+ <OnboardingContext.Provider value={ctx}>
+ {children}
+ </OnboardingContext.Provider>
+ );
+ };
+}
+
+function driverMock() {
+ return vi.mocked(driver);
+}
+
+function lastConfig(): Config {
+ const calls = driverMock().mock.calls;
+ return calls[calls.length - 1]![0]!;
+}
+
+function lastInstance() {
+ const results = driverMock().mock.results;
+ return results[results.length - 1]!.value as ReturnType<typeof driver>;
+}
+
+function nodeAdderStep(config: Config): DriveStep {
+ return config.steps!.find(
+ (step) => step.element === "[data-tour='node-adder']",
+ )!;
+}
+
+function highlight(config: Config, step: DriveStep) {
+ act(() => {
+ (config.onHighlightStarted as DriverStepCb)(undefined, step);
+ });
+}
+
+function openNodeLibrary(connectingEdgeType = "default") {
+ act(() => {
+ useWorkflowPanelStore.setState({
+ workflowPanelState: {
+ active: true,
+ content: "nodeLibrary",
+ data: { connectingEdgeType },
+ },
+ });
+ });
+}
+
+function closePanel() {
+ act(() => {
+ useWorkflowPanelStore.setState({
+ workflowPanelState: { active: false, content: "parameters" },
+ });
+ });
+}
+
+describe("useEditorOnboardingTour", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.clearAllMocks();
+ flagState.variant = "template-first";
+ useProductTourStore.setState({ requestedAt: null });
+ useWorkflowPanelStore.setState({
+ workflowPanelState: { active: false, content: "parameters" },
+ });
+ addAnchors();
+ });
+
+ afterEach(() => {
+ cleanup();
+ vi.useRealTimers();
+ removeAnchors();
+ });
+
+ describe("auto-trigger logic", () => {
+ it("auto-starts for existing users who have not completed the tour", () => {
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).toHaveBeenCalledOnce();
+ expect(lastInstance().drive).toHaveBeenCalledOnce();
+ });
+
+ it("does not auto-start for new users", () => {
+ const ctx = makeContext({ isNewUser: true });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+
+ it("does not auto-start when tour is already completed", () => {
+ const ctx = makeContext({
+ isNewUser: false,
+ state: { ...DEFAULT_STATE, tour_completed_at: "2024-01-01T00:00:00Z" },
+ });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+
+ it("does not auto-start when the experiment flag is disabled", () => {
+ flagState.variant = false;
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+
+ it("does not auto-start while onboarding state is loading", () => {
+ const ctx = makeContext({ isLoading: true });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+ });
+
+ describe("completion tracking", () => {
+ it("persists seen patches on each intermediate step", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ const steps = config.steps!;
+
+ act(() => {
+ (config.onNextClick as DriverStepCb)(undefined, steps[0]!);
+ });
+ expect(updateState).toHaveBeenCalledWith({ seen_canvas: true });
+
+ highlight(config, steps[1]!);
+ openNodeLibrary("default");
+ expect(updateState).toHaveBeenCalledWith({ seen_node_adder: true });
+
+ act(() => {
+ (config.onNextClick as DriverStepCb)(undefined, steps[2]!);
+ });
+ expect(updateState).toHaveBeenCalledWith({ seen_sidebar: true });
+ });
+
+ it("persists tour_completed_at on the final step", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ const steps = config.steps!;
+
+ for (let i = 0; i < steps.length - 1; i++) {
+ act(() => {
+ (config.onNextClick as DriverStepCb)(undefined, steps[i]!);
+ });
+ }
+
+ updateState.mockClear();
+
+ act(() => {
+ (config.onNextClick as DriverStepCb)(
+ undefined,
+ steps[steps.length - 1]!,
+ );
+ });
+
+ expect(updateState).toHaveBeenCalledWith(
+ expect.objectContaining({
+ seen_save_run: true,
+ tour_completed_at: expect.any(String),
+ }),
+ );
+ expect(lastInstance().destroy).toHaveBeenCalled();
+ });
+
+ it("persists patches when driver passes cloned steps, as driver.js drive() does", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ const steps = config.steps!;
+ const clone = (step: DriveStep): DriveStep => ({
+ ...step,
+ popover: { ...step.popover },
+ });
+
+ act(() => {
+ (config.onNextClick as DriverStepCb)(undefined, clone(steps[0]!));
+ });
+ expect(updateState).toHaveBeenCalledWith({ seen_canvas: true });
+
+ updateState.mockClear();
+ act(() => {
+ (config.onNextClick as DriverStepCb)(
+ undefined,
+ clone(steps[steps.length - 1]!),
+ );
+ });
+ expect(updateState).toHaveBeenCalledWith(
+ expect.objectContaining({
+ tour_completed_at: expect.any(String),
+ }),
+ );
+ expect(lastInstance().destroy).toHaveBeenCalled();
+ });
+ });
+
+ describe("re-trigger", () => {
+ it("starts tour when requestTour is called via store", () => {
+ const ctx = makeContext({
+ isNewUser: false,
+ state: {
+ ...DEFAULT_STATE,
+ tour_completed_at: "2024-01-01T00:00:00Z",
+ },
+ });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+ expect(driverMock()).not.toHaveBeenCalled();
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+
+ expect(driverMock()).toHaveBeenCalledOnce();
+ expect(lastInstance().drive).toHaveBeenCalledOnce();
+ });
+
+ it("ignores manual re-trigger when the experiment flag is disabled", () => {
+ flagState.variant = false;
+ const ctx = makeContext({
+ isNewUser: false,
+ state: {
+ ...DEFAULT_STATE,
+ tour_completed_at: "2024-01-01T00:00:00Z",
+ },
+ });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+
+ it("destroys existing tour before restarting", () => {
+ const ctx = makeContext({
+ isNewUser: false,
+ state: {
+ ...DEFAULT_STATE,
+ tour_completed_at: "2024-01-01T00:00:00Z",
+ },
+ });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+
+ expect(driverMock()).toHaveBeenCalledTimes(1);
+ const firstDestroy = driverMock().mock.results[0]!.value.destroy;
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+
+ expect(firstDestroy).toHaveBeenCalled();
+ expect(driverMock()).toHaveBeenCalledTimes(2);
+ });
+ });
+
+ describe("exit dialog", () => {
+ it("shows exit dialog on close click", () => {
+ const ctx = makeContext({ isNewUser: false });
+ const { result } = renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(result.current.showExitDialog).toBe(false);
+
+ act(() => {
+ (lastConfig().onCloseClick as DriverCloseCb)();
+ });
+
+ expect(result.current.showExitDialog).toBe(true);
+ });
+
+ it("hides exit dialog on cancel", () => {
+ const ctx = makeContext({ isNewUser: false });
+ const { result } = renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ act(() => {
+ (lastConfig().onCloseClick as DriverCloseCb)();
+ });
+ expect(result.current.showExitDialog).toBe(true);
+
+ act(() => {
+ result.current.onExitCancel();
+ });
+ expect(result.current.showExitDialog).toBe(false);
+ });
+
+ it("ends tour and persists completion on confirm", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ const { result } = renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ act(() => {
+ (lastConfig().onCloseClick as DriverCloseCb)();
+ });
+
+ act(() => {
+ result.current.onExitConfirm();
+ });
+
+ expect(result.current.showExitDialog).toBe(false);
+ expect(updateState).toHaveBeenCalledWith(
+ expect.objectContaining({ tour_completed_at: expect.any(String) }),
+ );
+ expect(lastInstance().destroy).toHaveBeenCalled();
+ });
+
+ it("pauses the driver layer while the exit dialog is open", () => {
+ const ctx = makeContext({ isNewUser: false });
+ const { result } = renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+ expect(document.documentElement.classList.contains("tour-paused")).toBe(
+ false,
+ );
+
+ act(() => {
+ (lastConfig().onCloseClick as DriverCloseCb)();
+ });
+ expect(document.documentElement.classList.contains("tour-paused")).toBe(
+ true,
+ );
+
+ act(() => {
+ result.current.onExitCancel();
+ });
+ expect(document.documentElement.classList.contains("tour-paused")).toBe(
+ false,
+ );
+ });
+
+ it("unpauses the driver layer when the tour ends", () => {
+ const ctx = makeContext({ isNewUser: false });
+ const { result } = renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+ act(() => {
+ (lastConfig().onCloseClick as DriverCloseCb)();
+ });
+ expect(document.documentElement.classList.contains("tour-paused")).toBe(
+ true,
+ );
+
+ act(() => {
+ result.current.onExitConfirm();
+ });
+ expect(document.documentElement.classList.contains("tour-paused")).toBe(
+ false,
+ );
+ });
+ });
+
+ describe("anchor-missing graceful skip", () => {
+ it("does not start when no anchor elements exist", () => {
+ removeAnchors();
+
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+
+ it("starts with only available steps when some anchors are missing", () => {
+ document.querySelector("[data-tour='node-adder']")?.remove();
+ document.querySelector("[data-tour='sidebar-region']")?.remove();
+
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(driverMock()).toHaveBeenCalledOnce();
+ const config = lastConfig();
+ expect(config.steps).toHaveLength(2);
+ });
+
+ it("does not start on re-trigger when no anchors exist", () => {
+ const ctx = makeContext({
+ isNewUser: false,
+ state: {
+ ...DEFAULT_STATE,
+ tour_completed_at: "2024-01-01T00:00:00Z",
+ },
+ });
+ renderHook(() => useEditorOnboardingTour(), {
+ wrapper: wrapWith(ctx),
+ });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ removeAnchors();
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+ });
+
+ describe("interactive node-adder gating", () => {
+ it("hides the Next button on the node-adder step until the gate is satisfied", () => {
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(nodeAdderStep(lastConfig()).popover!.showButtons).not.toContain(
+ "next",
+ );
+ });
+
+ it("shows the Next button on the node-adder step once the gate is satisfied", () => {
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ highlight(config, nodeAdderStep(config));
+ openNodeLibrary("default");
+ expect(lastInstance().moveNext).toHaveBeenCalledOnce();
+
+ expect(nodeAdderStep(config).popover!.showButtons).toContain("next");
+
+ vi.mocked(lastInstance().moveNext).mockClear();
+ act(() => {
+ (config.onNextClick as DriverStepCb)(undefined, nodeAdderStep(config));
+ });
+ expect(lastInstance().moveNext).toHaveBeenCalledOnce();
+ });
+
+ it("advances past step 2 when the + opens the node library", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ highlight(config, nodeAdderStep(config));
+ expect(lastInstance().moveNext).not.toHaveBeenCalled();
+
+ openNodeLibrary("default");
+
+ expect(lastInstance().moveNext).toHaveBeenCalledOnce();
+ expect(updateState).toHaveBeenCalledWith({ seen_node_adder: true });
+ });
+
+ it("ignores keyboard/Next advance on the node-adder step", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ highlight(config, nodeAdderStep(config));
+
+ act(() => {
+ (config.onNextClick as DriverStepCb)(undefined, nodeAdderStep(config));
+ });
+
+ expect(lastInstance().moveNext).not.toHaveBeenCalled();
+ expect(updateState).not.toHaveBeenCalledWith({ seen_node_adder: true });
+ });
+
+ it("advances on the node-adder step when the library opens via any +", () => {
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ highlight(config, nodeAdderStep(config));
+ openNodeLibrary("edgeWithAddButton");
+
+ expect(lastInstance().moveNext).toHaveBeenCalledOnce();
+ });
+
+ it("does not advance when the user is not on the node-adder step", () => {
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ highlight(config, config.steps![0]!);
+ openNodeLibrary("default");
+
+ expect(lastInstance().moveNext).not.toHaveBeenCalled();
+ });
+
+ it("completes the tour when the node-adder is the last available step", () => {
+ document.querySelector("[data-tour='sidebar-region']")?.remove();
+ document.querySelector("[data-tour='editor-actions']")?.remove();
+
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ expect(config.steps).toHaveLength(2);
+
+ highlight(config, nodeAdderStep(config));
+ openNodeLibrary("default");
+
+ expect(lastInstance().moveNext).not.toHaveBeenCalled();
+ expect(lastInstance().destroy).toHaveBeenCalled();
+ expect(updateState).toHaveBeenCalledWith(
+ expect.objectContaining({
+ seen_node_adder: true,
+ tour_completed_at: expect.any(String),
+ }),
+ );
+ });
+
+ it("re-arms the gate after a re-trigger", () => {
+ const ctx = makeContext({
+ isNewUser: false,
+ state: { ...DEFAULT_STATE, tour_completed_at: "2024-01-01T00:00:00Z" },
+ });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+ const firstConfig = lastConfig();
+ highlight(firstConfig, nodeAdderStep(firstConfig));
+ openNodeLibrary("default");
+ expect(lastInstance().moveNext).toHaveBeenCalledOnce();
+
+ closePanel();
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+ const secondConfig = lastConfig();
+ highlight(secondConfig, nodeAdderStep(secondConfig));
+ openNodeLibrary("default");
+ expect(lastInstance().moveNext).toHaveBeenCalledOnce();
+ });
+ });
+
+ describe("tour_started emission", () => {
+ it("emits tour_started once even when the user returns to the first step", () => {
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ const config = lastConfig();
+ const steps = config.steps!;
+ highlight(config, steps[0]!);
+ highlight(config, steps[1]!);
+ highlight(config, steps[0]!);
+
+ expect(captureCount("onboarding.tour_started")).toBe(1);
+ });
+
+ it("re-emits tour_started on a fresh tour run", () => {
+ const ctx = makeContext({
+ isNewUser: false,
+ state: { ...DEFAULT_STATE, tour_completed_at: "2024-01-01T00:00:00Z" },
+ });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+ highlight(lastConfig(), lastConfig().steps![0]!);
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+ highlight(lastConfig(), lastConfig().steps![0]!);
+
+ expect(captureCount("onboarding.tour_started")).toBe(2);
+ });
+ });
+
+ describe("experiment arm registration", () => {
+ it("registers and persists the arm for existing users before tour telemetry", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({ isNewUser: false, updateState });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(vi.mocked(posthog.register)).toHaveBeenCalledWith({
+ variant: "template-first",
+ });
+ expect(updateState).toHaveBeenCalledWith({
+ ab_variant: "template-first",
+ });
+ });
+
+ it("does not reassign the arm when one is already persisted", () => {
+ const updateState = vi.fn();
+ const ctx = makeContext({
+ isNewUser: false,
+ updateState,
+ state: { ...DEFAULT_STATE, ab_variant: "copilot-first" },
+ });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(updateState).not.toHaveBeenCalledWith(
+ expect.objectContaining({ ab_variant: expect.anything() }),
+ );
+ });
+
+ it("does not register an arm when the experiment flag is disabled", () => {
+ flagState.variant = false;
+ const ctx = makeContext({ isNewUser: false });
+ renderHook(() => useEditorOnboardingTour(), { wrapper: wrapWith(ctx) });
+
+ act(() => {
+ vi.runAllTimers();
+ });
+
+ expect(vi.mocked(posthog.register)).not.toHaveBeenCalled();
+ });
+ });
+
+ describe("manual start without a provider", () => {
+ it("does not start a manual tour when no onboarding provider is mounted", () => {
+ renderHook(() => useEditorOnboardingTour());
+
+ act(() => {
+ useProductTourStore.getState().requestTour();
+ });
+
+ expect(driverMock()).not.toHaveBeenCalled();
+ });
+ });
+});
diff --git a/skyvern-frontend/src/hooks/useEditorOnboardingTour.ts b/skyvern-frontend/src/hooks/useEditorOnboardingTour.ts
new file mode 100644
index 000000000..06222c2bc
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useEditorOnboardingTour.ts
@@ -0,0 +1,346 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { driver, type DriveStep, type Config } from "driver.js";
+import { useFeatureFlagVariantKey } from "posthog-js/react";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
+import { useProductTourStore } from "@/store/ProductTourStore";
+import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
+import { EXPERIMENT, isABVariant } from "@/util/onboarding/experimentConfig";
+import type { OnboardingStatePatch } from "@/store/onboarding/types";
+
+const SURFACE = "editor" as const;
+const LAYER = 1 as const;
+const AUTO_START_DELAY_MS = 1500;
+
+const BASE_DRIVER_CONFIG: Config = {
+ popoverClass: "skyvern-onboarding",
+ overlayOpacity: 0.5,
+ animate: true,
+ smoothScroll: true,
+ allowClose: true,
+ allowKeyboardControl: true,
+ stagePadding: 8,
+ stageRadius: 8,
+};
+
+const TOUR_STEPS: DriveStep[] = [
+ {
+ element: "[data-tour='editor-canvas']",
+ popover: {
+ title: "Your workflow canvas",
+ description: "Drag blocks here to compose your automation.",
+ side: "left",
+ align: "center",
+ },
+ },
+ {
+ element: "[data-tour='node-adder']",
+ popover: {
+ title: "Add blocks",
+ description: "Click the + button to add your first block.",
+ side: "left",
+ align: "center",
+ showButtons: ["previous", "close"],
+ },
+ },
+ {
+ element: "[data-tour='sidebar-region']",
+ popover: {
+ title: "Block settings",
+ description:
+ "Click any block to configure its settings in this side panel.",
+ side: "left",
+ align: "center",
+ },
+ },
+ {
+ element: "[data-tour='editor-actions']",
+ popover: {
+ title: "Save and run",
+ description: "Save your work and run it to see results.",
+ side: "bottom",
+ align: "end",
+ },
+ },
+];
+
+const STEP_NAMES = ["canvas", "node_adder", "sidebar", "save_run"] as const;
+
+const NODE_ADDER_INDEX = STEP_NAMES.indexOf("node_adder");
+
+const NODE_ADDER_SELECTOR = "[data-tour='node-adder']";
+
+// driver.js reads a step's showButtons fresh on each render, so toggling this
+// reflects on the next time step 2 is shown (e.g. Previous back to it).
+function setNodeAdderNextVisible(visible: boolean): void {
+ const popover = TOUR_STEPS[NODE_ADDER_INDEX]?.popover;
+ if (popover) {
+ popover.showButtons = visible
+ ? ["next", "previous", "close"]
+ : ["previous", "close"];
+ }
+}
+
+const SEEN_PATCHES: Record<(typeof STEP_NAMES)[number], OnboardingStatePatch> =
+ {
+ canvas: { seen_canvas: true },
+ node_adder: { seen_node_adder: true },
+ sidebar: { seen_sidebar: true },
+ save_run: { seen_save_run: true },
+ };
+
+// driver.js hands callbacks a shallow clone of the active step, so match by element selector, never by object identity.
+function indexOfStep(steps: DriveStep[], step: DriveStep): number {
+ if (step.element == null) return -1;
+ return steps.findIndex((s) => s.element === step.element);
+}
+
+function filterAvailableSteps(steps: DriveStep[]): DriveStep[] {
+ return steps.filter((step) => {
+ if (typeof step.element !== "string") return true;
+ return document.querySelector(step.element) !== null;
+ });
+}
+
+type UseEditorOnboardingTourReturn = {
+ showExitDialog: boolean;
+ onExitConfirm: () => void;
+ onExitCancel: () => void;
+};
+
+function useEditorOnboardingTour(): UseEditorOnboardingTourReturn {
+ const onboarding = useOnboardingStateOptional();
+ const flagVariant = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
+ const [showExitDialog, setShowExitDialog] = useState(false);
+ const [tourActive, setTourActive] = useState(false);
+ const driverRef = useRef<ReturnType<typeof driver> | null>(null);
+ const currentStepRef = useRef(0);
+ const tourActiveRef = useRef(false);
+ const requestedAt = useProductTourStore((s) => s.requestedAt);
+ const clearRequest = useProductTourStore((s) => s.clearRequest);
+ const panelActive = useWorkflowPanelStore((s) => s.workflowPanelState.active);
+ const panelContent = useWorkflowPanelStore(
+ (s) => s.workflowPanelState.content,
+ );
+ const availableStepsRef = useRef<DriveStep[]>([]);
+ const nodeAdderGateSatisfiedRef = useRef(false);
+ const tourStartedEmittedRef = useRef(false);
+
+ const shouldAutoStart =
+ onboarding !== null &&
+ !onboarding.isLoading &&
+ onboarding.state !== null &&
+ !onboarding.isNewUser &&
+ onboarding.state.tour_completed_at === null &&
+ isABVariant(flagVariant);
+
+ const advanceFromStep = useCallback(
+ (step: DriveStep) => {
+ const availableSteps = availableStepsRef.current;
+ const stepIndex = indexOfStep(TOUR_STEPS, step);
+ const stepName = STEP_NAMES[stepIndex];
+ const isLastStep =
+ indexOfStep(availableSteps, step) === availableSteps.length - 1;
+
+ if (stepName) {
+ OnboardingTelemetry.stepCompleted(SURFACE, stepName);
+ }
+
+ if (isLastStep) {
+ OnboardingTelemetry.tourCompleted(SURFACE);
+ onboarding?.updateState({
+ ...(stepName ? SEEN_PATCHES[stepName] : {}),
+ tour_completed_at: new Date().toISOString(),
+ });
+ tourActiveRef.current = false;
+ setTourActive(false);
+ driverRef.current?.destroy();
+ } else {
+ if (stepName) {
+ onboarding?.updateState(SEEN_PATCHES[stepName]);
+ }
+ driverRef.current?.moveNext();
+ }
+ },
+ [onboarding],
+ );
+
+ const startDriver = useCallback(() => {
+ if (tourActiveRef.current) return;
+
+ const availableSteps = filterAvailableSteps(TOUR_STEPS);
+ if (availableSteps.length === 0) return;
+
+ tourActiveRef.current = true;
+ setTourActive(true);
+ availableStepsRef.current = availableSteps;
+ nodeAdderGateSatisfiedRef.current = false;
+ tourStartedEmittedRef.current = false;
+ setNodeAdderNextVisible(false);
+
+ // Existing users reach the tour without the get-started modal, where the arm
+ // is normally registered/persisted, so set it here before any tour telemetry
+ // so those events carry the experiment arm.
+ if (
+ onboarding?.state &&
+ onboarding.state.ab_variant === null &&
+ isABVariant(flagVariant)
+ ) {
+ OnboardingTelemetry.registerVariant(flagVariant);
+ OnboardingTelemetry.abVariantAssigned(SURFACE, flagVariant);
+ onboarding.updateState({ ab_variant: flagVariant });
+ }
+
+ const config: Config = {
+ ...BASE_DRIVER_CONFIG,
+ showProgress: true,
+ progressText: "Step {{current}} of {{total}}",
+ allowClose: false,
+ steps: availableSteps,
+ onHighlightStarted: (_element, step) => {
+ const stepIndex = indexOfStep(TOUR_STEPS, step);
+ if (stepIndex >= 0) {
+ currentStepRef.current = stepIndex;
+ if (
+ indexOfStep(availableSteps, step) === 0 &&
+ !tourStartedEmittedRef.current
+ ) {
+ tourStartedEmittedRef.current = true;
+ OnboardingTelemetry.tourStarted(SURFACE);
+ }
+ OnboardingTelemetry.tourStepViewed(
+ SURFACE,
+ STEP_NAMES[stepIndex] ?? "unknown",
+ stepIndex,
+ LAYER,
+ );
+ }
+ },
+ onCloseClick: () => {
+ setShowExitDialog(true);
+ },
+ onNextClick: (_element, step) => {
+ // Until the "+" gate is satisfied, ignore Next/ArrowRight on step 2 (driver
+ // routes ArrowRight here too) so it only advances via the effect below.
+ if (
+ indexOfStep(TOUR_STEPS, step) === NODE_ADDER_INDEX &&
+ !nodeAdderGateSatisfiedRef.current
+ ) {
+ return;
+ }
+ advanceFromStep(step);
+ },
+ };
+
+ try {
+ const driverInstance = driver(config);
+ driverInstance.drive();
+ driverRef.current = driverInstance;
+ } catch {
+ OnboardingTelemetry.tourError(SURFACE);
+ tourActiveRef.current = false;
+ setTourActive(false);
+ }
+ }, [advanceFromStep, onboarding, flagVariant]);
+
+ useEffect(() => {
+ if (!shouldAutoStart || tourActiveRef.current) return;
+
+ const timeout = setTimeout(() => {
+ startDriver();
+ }, AUTO_START_DELAY_MS);
+
+ return () => {
+ clearTimeout(timeout);
+ };
+ }, [shouldAutoStart, startDriver]);
+
+ useEffect(() => {
+ if (requestedAt === null) return;
+ clearRequest();
+ // No provider (non-cloud app) means dismissals/completions cannot persist, so
+ // never start an untracked tour - mirrors the auto-start guard.
+ if (onboarding === null) return;
+ if (!isABVariant(flagVariant)) return;
+ if (tourActiveRef.current) {
+ driverRef.current?.destroy();
+ tourActiveRef.current = false;
+ setTourActive(false);
+ }
+ startDriver();
+ }, [requestedAt, clearRequest, startDriver, flagVariant, onboarding]);
+
+ // Step 2 ("Add blocks") is gated: opening the block library while it is the
+ // active step satisfies the gate once and advances. Once satisfied the Next
+ // button is restored so returning to step 2 (via Previous) isn't a dead end.
+ useEffect(() => {
+ const libraryOpen = panelActive && panelContent === "nodeLibrary";
+ const onAdderStep =
+ currentStepRef.current === NODE_ADDER_INDEX ||
+ driverRef.current?.getActiveStep?.()?.element === NODE_ADDER_SELECTOR;
+
+ if (
+ !tourActiveRef.current ||
+ nodeAdderGateSatisfiedRef.current ||
+ !libraryOpen ||
+ !onAdderStep
+ ) {
+ return;
+ }
+ nodeAdderGateSatisfiedRef.current = true;
+ setNodeAdderNextVisible(true);
+ advanceFromStep(TOUR_STEPS[NODE_ADDER_INDEX]!);
+ }, [panelActive, panelContent, advanceFromStep]);
+
+ useEffect(() => {
+ return () => {
+ driverRef.current?.destroy();
+ };
+ }, []);
+
+ // Hide the driver.js layer while the exit dialog is open (see product-tour.css for why).
+ useEffect(() => {
+ if (!showExitDialog) return;
+ const root = document.documentElement;
+ root.classList.add("tour-paused");
+ return () => {
+ root.classList.remove("tour-paused");
+ };
+ }, [showExitDialog]);
+
+ useEffect(() => {
+ if (!tourActive) return;
+
+ const handleKeyDown = (e: KeyboardEvent) => {
+ if (e.key === "Escape") {
+ e.preventDefault();
+ e.stopPropagation();
+ setShowExitDialog(true);
+ }
+ };
+
+ document.addEventListener("keydown", handleKeyDown, true);
+ return () => {
+ document.removeEventListener("keydown", handleKeyDown, true);
+ };
+ }, [tourActive]);
+
+ const onExitConfirm = useCallback(() => {
+ const lastStep = STEP_NAMES[currentStepRef.current] ?? "unknown";
+ OnboardingTelemetry.tourDismissed(SURFACE, lastStep);
+ onboarding?.updateState({ tour_completed_at: new Date().toISOString() });
+ tourActiveRef.current = false;
+ setTourActive(false);
+ setShowExitDialog(false);
+ driverRef.current?.destroy();
+ }, [onboarding]);
+
+ const onExitCancel = useCallback(() => {
+ setShowExitDialog(false);
+ }, []);
+
+ return { showExitDialog, onExitConfirm, onExitCancel };
+}
+
+export { useEditorOnboardingTour };
+export type { UseEditorOnboardingTourReturn };
diff --git a/skyvern-frontend/src/hooks/useOnePasswordToken.ts b/skyvern-frontend/src/hooks/useOnePasswordToken.ts
index da1a0ea4c..aa288a399 100644
--- a/skyvern-frontend/src/hooks/useOnePasswordToken.ts
+++ b/skyvern-frontend/src/hooks/useOnePasswordToken.ts
@@ -7,6 +7,7 @@ import {
OnePasswordTokenApiResponse,
} from "@/api/types";
import { useToast } from "@/components/ui/use-toast";
+import { useClearOrganizationAuthToken } from "./useClearOrganizationAuthToken";
export function useOnePasswordToken() {
const credentialGetter = useCredentialGetter();
@@ -14,7 +15,7 @@ export function useOnePasswordToken() {
const { toast } = useToast();
const { data: onePasswordToken, isLoading } =
- useQuery<OnePasswordTokenApiResponse>({
+ useQuery<OnePasswordTokenApiResponse | null>({
queryKey: ["onePasswordToken"],
queryFn: async () => {
const client = await getClient(credentialGetter, "sans-api-v1");
@@ -33,7 +34,7 @@ export function useOnePasswordToken() {
.then((response) => response.data as CreateOnePasswordTokenResponse);
},
onSuccess: () => {
- queryClient.invalidateQueries({ queryKey: ["onePasswordToken"] });
+ void queryClient.invalidateQueries({ queryKey: ["onePasswordToken"] });
toast({
title: "Success",
description: "1Password service account token updated successfully",
@@ -53,10 +54,19 @@ export function useOnePasswordToken() {
},
});
+ const clearTokenMutation = useClearOrganizationAuthToken({
+ providerPath: "onepassword",
+ queryKey: "onePasswordToken",
+ successDescription: "1Password service account token cleared successfully",
+ errorDescription: "Failed to clear 1Password token",
+ });
+
return {
onePasswordToken,
isLoading,
createOrUpdateToken: createOrUpdateTokenMutation.mutate,
isUpdating: createOrUpdateTokenMutation.isPending,
+ clearToken: clearTokenMutation.mutate,
+ isClearing: clearTokenMutation.isPending,
};
}
diff --git a/skyvern-frontend/src/hooks/useProductTourShortcut.ts b/skyvern-frontend/src/hooks/useProductTourShortcut.ts
new file mode 100644
index 000000000..8da647c9f
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useProductTourShortcut.ts
@@ -0,0 +1,41 @@
+import { useEffect } from "react";
+import { useProductTourStore } from "@/store/ProductTourStore";
+
+function isEditableTarget(target: EventTarget | null): boolean {
+ if (!(target instanceof HTMLElement)) return false;
+ if (
+ target instanceof HTMLInputElement ||
+ target instanceof HTMLTextAreaElement
+ ) {
+ return true;
+ }
+ if (target instanceof HTMLSelectElement) return true;
+ return target.isContentEditable;
+}
+
+function useProductTourShortcut() {
+ const requestTour = useProductTourStore((s) => s.requestTour);
+
+ useEffect(() => {
+ const handleKeyDown = (e: KeyboardEvent) => {
+ if (
+ e.key === "?" &&
+ e.shiftKey &&
+ !e.ctrlKey &&
+ !e.metaKey &&
+ !e.altKey
+ ) {
+ if (isEditableTarget(e.target)) return;
+ e.preventDefault();
+ requestTour();
+ }
+ };
+
+ document.addEventListener("keydown", handleKeyDown);
+ return () => {
+ document.removeEventListener("keydown", handleKeyDown);
+ };
+ }, [requestTour]);
+}
+
+export { useProductTourShortcut };
diff --git a/skyvern-frontend/src/hooks/useRowSelection.test.tsx b/skyvern-frontend/src/hooks/useRowSelection.test.tsx
new file mode 100644
index 000000000..a5fb80276
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useRowSelection.test.tsx
@@ -0,0 +1,130 @@
+import { act, renderHook } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+
+import { useRowSelection } from "./useRowSelection";
+
+type Row = { id: string };
+
+const rows = (...ids: string[]): Row[] => ids.map((id) => ({ id }));
+
+function setup(items: Row[], resetKey?: unknown, anchorResetKey?: unknown) {
+ return renderHook(
+ ({
+ items,
+ resetKey,
+ anchorResetKey,
+ }: {
+ items: Row[];
+ resetKey?: unknown;
+ anchorResetKey?: unknown;
+ }) =>
+ useRowSelection({
+ items,
+ getId: (row: Row) => row.id,
+ resetKey,
+ anchorResetKey,
+ }),
+ { initialProps: { items, resetKey, anchorResetKey } },
+ );
+}
+
+describe("useRowSelection", () => {
+ it("toggles a single row and re-anchors", () => {
+ const { result } = setup(rows("a", "b", "c"));
+ act(() => result.current.handleSelect(1, false));
+ expect(result.current.isSelected("b")).toBe(true);
+ act(() => result.current.handleSelect(1, false));
+ expect(result.current.isSelected("b")).toBe(false);
+ });
+
+ it("shift-click selects the range from the anchor in both directions", () => {
+ const { result } = setup(rows("a", "b", "c", "d"));
+ act(() => result.current.handleSelect(2, false));
+ act(() => result.current.handleSelect(0, true));
+ expect([...result.current.selected].sort()).toEqual(["a", "b", "c"]);
+ });
+
+ it("shift-click over a fully selected range deselects it", () => {
+ const { result } = setup(rows("a", "b", "c"));
+ act(() => result.current.handleSelect(0, false));
+ act(() => result.current.handleSelect(2, true));
+ act(() => result.current.handleSelect(2, true));
+ expect(result.current.selected.size).toBe(0);
+ });
+
+ it("shift-click with no anchor falls back to a single toggle", () => {
+ const { result } = setup(rows("a", "b", "c"));
+ act(() => result.current.handleSelect(2, true));
+ expect([...result.current.selected]).toEqual(["c"]);
+ });
+
+ it("toggleSelectAll selects everything, then clears; flags are correct midway", () => {
+ const { result } = setup(rows("a", "b"));
+ act(() => result.current.handleSelect(0, false));
+ expect(result.current.someSelected).toBe(true);
+ expect(result.current.allSelected).toBe(false);
+ act(() => result.current.toggleSelectAll());
+ expect(result.current.allSelected).toBe(true);
+ expect(result.current.someSelected).toBe(false);
+ act(() => result.current.toggleSelectAll());
+ expect(result.current.selected.size).toBe(0);
+ });
+
+ it("resetKey change clears selection and anchor", () => {
+ const { result, rerender } = setup(rows("a", "b", "c"), "k1");
+ act(() => result.current.handleSelect(0, false));
+ rerender({
+ items: rows("a", "b", "c"),
+ resetKey: "k2",
+ anchorResetKey: undefined,
+ });
+ expect(result.current.selected.size).toBe(0);
+ act(() => result.current.handleSelect(2, true));
+ expect([...result.current.selected]).toEqual(["c"]);
+ });
+
+ it("anchorResetKey change clears only the anchor", () => {
+ const { result, rerender } = setup(rows("a", "b", "c"), "k", "p1");
+ act(() => result.current.handleSelect(0, false));
+ rerender({
+ items: rows("a", "b", "c"),
+ resetKey: "k",
+ anchorResetKey: "p2",
+ });
+ expect(result.current.isSelected("a")).toBe(true);
+ act(() => result.current.handleSelect(2, true));
+ expect([...result.current.selected].sort()).toEqual(["a", "c"]);
+ });
+
+ it("replaceSelection sets exactly the given ids and nulls the anchor", () => {
+ const { result } = setup(rows("a", "b", "c"));
+ act(() => result.current.handleSelect(0, false));
+ act(() => result.current.replaceSelection(["b", "c"]));
+ expect([...result.current.selected].sort()).toEqual(["b", "c"]);
+ act(() => result.current.handleSelect(0, true));
+ expect(result.current.isSelected("a")).toBe(true);
+ });
+
+ it("selectedItems keeps item order and drops stale ids", () => {
+ const { result, rerender } = setup(rows("a", "b", "c"));
+ act(() => result.current.toggleSelectAll());
+ rerender({
+ items: rows("c", "a"),
+ resetKey: undefined,
+ anchorResetKey: undefined,
+ });
+ expect(result.current.selectedItems.map((r) => r.id)).toEqual(["c", "a"]);
+ });
+
+ it("stale anchor beyond a shrunken list clamps instead of throwing", () => {
+ const { result, rerender } = setup(rows("a", "b", "c", "d"));
+ act(() => result.current.handleSelect(3, false));
+ rerender({
+ items: rows("a", "b"),
+ resetKey: undefined,
+ anchorResetKey: undefined,
+ });
+ act(() => result.current.handleSelect(0, true));
+ expect(result.current.isSelected("a")).toBe(true);
+ });
+});
diff --git a/skyvern-frontend/src/hooks/useRowSelection.ts b/skyvern-frontend/src/hooks/useRowSelection.ts
new file mode 100644
index 000000000..108ed81a6
--- /dev/null
+++ b/skyvern-frontend/src/hooks/useRowSelection.ts
@@ -0,0 +1,114 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+
+type UseRowSelectionOptions<T> = {
+ items: T[];
+ getId: (item: T) => string;
+ // Selection and anchor clear whenever this changes; compose tuples with JSON.stringify.
+ resetKey?: unknown;
+ // Anchor-only reset (selection survives), e.g. page-size changes shift row indices.
+ anchorResetKey?: unknown;
+};
+
+function useRowSelection<T>({
+ items,
+ getId,
+ resetKey,
+ anchorResetKey,
+}: UseRowSelectionOptions<T>) {
+ const [selected, setSelected] = useState<Set<string>>(new Set());
+ const anchorIndex = useRef<number | null>(null);
+
+ useEffect(() => {
+ setSelected(new Set());
+ anchorIndex.current = null;
+ }, [resetKey]);
+
+ useEffect(() => {
+ anchorIndex.current = null;
+ }, [anchorResetKey]);
+
+ const selectedItems = items.filter((item) => selected.has(getId(item)));
+ const allSelected =
+ items.length > 0 && items.every((item) => selected.has(getId(item)));
+ const someSelected = selected.size > 0 && !allSelected;
+ const indexById = new Map(items.map((item, index) => [getId(item), index]));
+
+ function isSelected(id: string) {
+ return selected.has(id);
+ }
+
+ function handleSelect(index: number, shiftKey: boolean) {
+ const item = items[index];
+ if (!item) {
+ return;
+ }
+ const id = getId(item);
+ if (shiftKey && anchorIndex.current !== null) {
+ const anchor = Math.min(anchorIndex.current, items.length - 1);
+ const start = Math.min(anchor, index);
+ const end = Math.max(anchor, index);
+ setSelected((prev) => {
+ const rangeIds: string[] = [];
+ for (let i = start; i <= end; i++) {
+ rangeIds.push(getId(items[i]!));
+ }
+ const allInRangeSelected = rangeIds.every((rangeId) =>
+ prev.has(rangeId),
+ );
+ const next = new Set(prev);
+ for (const rangeId of rangeIds) {
+ if (allInRangeSelected) {
+ next.delete(rangeId);
+ } else {
+ next.add(rangeId);
+ }
+ }
+ return next;
+ });
+ return;
+ }
+ setSelected((prev) => {
+ const next = new Set(prev);
+ if (next.has(id)) {
+ next.delete(id);
+ } else {
+ next.add(id);
+ }
+ return next;
+ });
+ anchorIndex.current = index;
+ }
+
+ function toggleSelectAll() {
+ if (allSelected) {
+ setSelected(new Set());
+ } else {
+ setSelected(new Set(items.map(getId)));
+ }
+ }
+
+ const clearSelection = useCallback(() => {
+ setSelected(new Set());
+ anchorIndex.current = null;
+ }, []);
+
+ const replaceSelection = useCallback((ids: Iterable<string>) => {
+ setSelected(new Set(ids));
+ anchorIndex.current = null;
+ }, []);
+
+ return {
+ selected: selected as ReadonlySet<string>,
+ selectedItems,
+ isSelected,
+ allSelected,
+ someSelected,
+ indexById: indexById as ReadonlyMap<string, number>,
+ handleSelect,
+ toggleSelectAll,
+ clearSelection,
+ replaceSelection,
+ };
+}
+
+export { useRowSelection };
diff --git a/skyvern-frontend/src/routes/browserProfiles/BrowserProfileItem.tsx b/skyvern-frontend/src/routes/browserProfiles/BrowserProfileItem.tsx
index a34734b72..4cb7306b7 100644
--- a/skyvern-frontend/src/routes/browserProfiles/BrowserProfileItem.tsx
+++ b/skyvern-frontend/src/routes/browserProfiles/BrowserProfileItem.tsx
@@ -3,6 +3,7 @@ import { Pencil1Icon } from "@radix-ui/react-icons";
import { useNavigate } from "react-router-dom";
import { BrowserProfileApiResponse } from "@/api/types";
+import { SelectionCheckboxCell } from "@/components/SelectionCheckbox";
import { Button } from "@/components/ui/button";
import { TableCell, TableRow } from "@/components/ui/table";
import {
@@ -18,9 +19,19 @@ import { RenameBrowserProfileDialog } from "./RenameBrowserProfileDialog";
type Props = {
profile: BrowserProfileApiResponse;
+ index?: number;
+ selected?: boolean;
+ hasSelection?: boolean;
+ onSelect?: (index: number, shiftKey: boolean) => void;
};
-function BrowserProfileItem({ profile }: Props) {
+function BrowserProfileItem({
+ profile,
+ index = -1,
+ selected = false,
+ hasSelection = false,
+ onSelect,
+}: Props) {
const navigate = useNavigate();
const [renameOpen, setRenameOpen] = useState(false);
@@ -41,7 +52,20 @@ function BrowserProfileItem({ profile }: Props) {
};
return (
- <TableRow className="cursor-pointer" onClick={handleRowClick}>
+ <TableRow
+ className="group/row cursor-pointer"
+ data-state={selected ? "selected" : undefined}
+ onClick={handleRowClick}
+ >
+ {onSelect && (
+ <SelectionCheckboxCell
+ index={index}
+ checked={selected}
+ hasSelection={hasSelection}
+ onSelect={onSelect}
+ ariaLabel={`Select ${profile.name}`}
+ />
+ )}
<TableCell className="truncate">
<span title={profile.name}>{profile.name}</span>
</TableCell>
diff --git a/skyvern-frontend/src/routes/browserProfiles/BrowserProfilesList.tsx b/skyvern-frontend/src/routes/browserProfiles/BrowserProfilesList.tsx
index bc5bc8e31..e023eb31c 100644
--- a/skyvern-frontend/src/routes/browserProfiles/BrowserProfilesList.tsx
+++ b/skyvern-frontend/src/routes/browserProfiles/BrowserProfilesList.tsx
@@ -1,8 +1,29 @@
import { ReloadIcon } from "@radix-ui/react-icons";
+import { useQueryClient } from "@tanstack/react-query";
+import { useState } from "react";
import { useSearchParams } from "react-router-dom";
+import { getClient } from "@/api/AxiosClient";
import { BrowserIcon } from "@/components/icons/BrowserIcon";
+import { GarbageIcon } from "@/components/icons/GarbageIcon";
+import { SelectionBar } from "@/components/SelectionBar";
+import { SelectionHeaderCheckboxCell } from "@/components/SelectionCheckbox";
import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useRowSelection } from "@/hooks/useRowSelection";
+import { bulkResultToast } from "@/util/bulkResultToast";
+import {
+ BULK_CONCURRENCY_LIMIT,
+ runWithConcurrency,
+} from "@/util/runWithConcurrency";
import {
Pagination,
PaginationContent,
@@ -79,6 +100,82 @@ function BrowserProfilesList({ searchKey }: Props = {}) {
const hasSearch = Boolean(searchKey && searchKey.length > 0);
const showPlaceholderRow = Boolean(activeCreate && page === 1 && !hasSearch);
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const [isBulkOperating, setIsBulkOperating] = useState(false);
+ const [deleteDialog, setDeleteDialog] = useState<{
+ open: boolean;
+ targets: string[];
+ }>({ open: false, targets: [] });
+ const {
+ selected,
+ selectedItems: selectedProfiles,
+ isSelected,
+ allSelected,
+ someSelected,
+ handleSelect,
+ toggleSelectAll,
+ clearSelection,
+ replaceSelection,
+ } = useRowSelection({
+ items: profiles ?? [],
+ getId: (profile) => profile.browser_profile_id,
+ resetKey: JSON.stringify([page, itemsPerPage, searchKey ?? ""]),
+ });
+
+ async function handleBulkDeleteConfirm() {
+ const targets = deleteDialog.targets;
+ if (targets.length === 0) {
+ return;
+ }
+ setIsBulkOperating(true);
+ try {
+ // Browser-profile endpoints live on /v1 (no /api prefix).
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ const results = await runWithConcurrency(
+ targets.map(
+ (profileId) => () => client.delete(`/browser_profiles/${profileId}`),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const failedIds = new Set<string>();
+ const succeededIds: string[] = [];
+ results.forEach((result, index) => {
+ if (result.status === "fulfilled") {
+ succeededIds.push(targets[index]!);
+ } else {
+ failedIds.add(targets[index]!);
+ }
+ });
+ bulkResultToast({
+ succeeded: succeededIds.length,
+ total: targets.length,
+ results,
+ successTitle: (n) => `Deleted ${n} profile${n !== 1 ? "s" : ""}.`,
+ failureTitle: (n) =>
+ `Failed to delete ${n} profile${n !== 1 ? "s" : ""}.`,
+ partialTitle: (successCount, failedCount) =>
+ `Deleted ${successCount} profile${successCount !== 1 ? "s" : ""}. ${failedCount} failed.`,
+ });
+ if (failedIds.size === 0) {
+ clearSelection();
+ } else {
+ replaceSelection(failedIds);
+ }
+ if (succeededIds.length > 0) {
+ queryClient.invalidateQueries({ queryKey: ["browserProfiles"] });
+ succeededIds.forEach((profileId) => {
+ queryClient.invalidateQueries({
+ queryKey: ["browserProfile", profileId],
+ });
+ });
+ }
+ } finally {
+ setIsBulkOperating(false);
+ setDeleteDialog({ open: false, targets: [] });
+ }
+ }
+
if (isLoading) {
return (
<div className="space-y-3">
@@ -135,9 +232,17 @@ function BrowserProfilesList({ searchKey }: Props = {}) {
<div className="overflow-hidden rounded-lg border border-border">
<Table className="w-full table-fixed">
<TableHeader>
- <TableRow>
- <TableHead className="w-[22%] truncate">Name</TableHead>
- <TableHead className="w-[40%] truncate">Description</TableHead>
+ <TableRow className="group/header">
+ <SelectionHeaderCheckboxCell
+ className="w-10"
+ allSelected={allSelected}
+ someSelected={someSelected}
+ hasSelection={selected.size > 0}
+ onToggleAll={toggleSelectAll}
+ ariaLabel="Select all browser profiles"
+ />
+ <TableHead className="w-[20%] truncate">Name</TableHead>
+ <TableHead className="w-[37%] truncate">Description</TableHead>
<TableHead className="w-[15%] truncate">Source Browser</TableHead>
<TableHead className="w-[15%] truncate">Created</TableHead>
<TableHead className="w-32 text-right">Actions</TableHead>
@@ -146,6 +251,7 @@ function BrowserProfilesList({ searchKey }: Props = {}) {
<TableBody>
{showPlaceholderRow && activeCreate ? (
<TableRow className="opacity-70">
+ <TableCell />
<TableCell className="truncate">
<div className="flex min-w-0 items-center gap-2">
<ReloadIcon className="h-4 w-4 shrink-0 animate-spin text-blue-400" />
@@ -168,10 +274,14 @@ function BrowserProfilesList({ searchKey }: Props = {}) {
</TableCell>
</TableRow>
) : null}
- {pageItems.map((profile) => (
+ {pageItems.map((profile, index) => (
<BrowserProfileItem
key={profile.browser_profile_id}
profile={profile}
+ index={index}
+ selected={isSelected(profile.browser_profile_id)}
+ hasSelection={selected.size > 0}
+ onSelect={handleSelect}
/>
))}
</TableBody>
@@ -223,6 +333,73 @@ function BrowserProfilesList({ searchKey }: Props = {}) {
</Pagination>
</div>
</div>
+ {selectedProfiles.length > 0 && (
+ <SelectionBar
+ count={selectedProfiles.length}
+ isOperating={isBulkOperating}
+ onClear={clearSelection}
+ >
+ <Button
+ size="sm"
+ variant="ghost"
+ className="text-destructive hover:bg-destructive/10 hover:text-destructive"
+ disabled={isBulkOperating}
+ onClick={() =>
+ setDeleteDialog({
+ open: true,
+ targets: selectedProfiles.map(
+ (profile) => profile.browser_profile_id,
+ ),
+ })
+ }
+ >
+ <GarbageIcon className="mr-1.5 h-4 w-4" />
+ Delete
+ </Button>
+ </SelectionBar>
+ )}
+ <Dialog
+ open={deleteDialog.open}
+ onOpenChange={(open) => {
+ if (!open && !isBulkOperating) {
+ setDeleteDialog({ open: false, targets: [] });
+ }
+ }}
+ >
+ <DialogContent>
+ <DialogHeader>
+ <DialogTitle>
+ Delete {deleteDialog.targets.length} Browser Profile
+ {deleteDialog.targets.length === 1 ? "" : "s"}
+ </DialogTitle>
+ <DialogDescription>
+ Are you sure you want to delete {deleteDialog.targets.length}{" "}
+ {deleteDialog.targets.length === 1
+ ? "browser profile"
+ : "browser profiles"}
+ ? This action cannot be undone.
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <Button
+ variant="secondary"
+ disabled={isBulkOperating}
+ onClick={() => setDeleteDialog({ open: false, targets: [] })}
+ >
+ Cancel
+ </Button>
+ <Button
+ variant="destructive"
+ disabled={isBulkOperating}
+ onClick={() => {
+ void handleBulkDeleteConfirm();
+ }}
+ >
+ {isBulkOperating ? "Deleting..." : "Delete"}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
</div>
);
}
diff --git a/skyvern-frontend/src/routes/browserProfiles/CreateBrowserProfileButton.tsx b/skyvern-frontend/src/routes/browserProfiles/CreateBrowserProfileButton.tsx
index 740076d59..6a111b76e 100644
--- a/skyvern-frontend/src/routes/browserProfiles/CreateBrowserProfileButton.tsx
+++ b/skyvern-frontend/src/routes/browserProfiles/CreateBrowserProfileButton.tsx
@@ -34,6 +34,7 @@ function CreateBrowserProfileButton({
createBrowserSessionMutation.mutate({
proxyLocation: null,
timeout: null,
+ generateBrowserProfile: true,
});
}}
>
diff --git a/skyvern-frontend/src/routes/browserProfiles/hooks/useBackgroundBrowserProfileCreate.tsx b/skyvern-frontend/src/routes/browserProfiles/hooks/useBackgroundBrowserProfileCreate.tsx
index 4a4ecc6a3..ee4b07fd4 100644
--- a/skyvern-frontend/src/routes/browserProfiles/hooks/useBackgroundBrowserProfileCreate.tsx
+++ b/skyvern-frontend/src/routes/browserProfiles/hooks/useBackgroundBrowserProfileCreate.tsx
@@ -20,6 +20,7 @@ const CREATE_RETRY_INTERVAL_MS = 10000;
const MAX_CONSECUTIVE_ERRORS = 10;
const ARCHIVE_NOT_READY_HINT = "persisted profile archive";
+const NOT_OPTED_IN_HINT = "was not configured to generate a browser profile";
type ActiveRefState = ActiveBrowserProfileCreate & {
timeoutId: ReturnType<typeof setTimeout> | null;
@@ -133,6 +134,21 @@ function useBackgroundBrowserProfileCreate() {
const axiosError = error as AxiosError;
const detail = getErrorDetail(axiosError);
+ // An opted-out session never uploads an archive, so retrying can't help — fail fast.
+ if (
+ axiosError.response?.status === 400 &&
+ typeof detail === "string" &&
+ detail.includes(NOT_OPTED_IN_HINT)
+ ) {
+ cleanup();
+ toast({
+ title: "Profile generation not enabled for this session",
+ description: detail,
+ variant: "destructive",
+ });
+ return;
+ }
+
if (
axiosError.response?.status === 400 &&
typeof detail === "string" &&
@@ -315,6 +331,11 @@ function useBackgroundBrowserProfileCreate() {
try {
const client = await getClient(credentialGetter, "sans-api-v1");
+ // Opt the running session into profile generation before closing so teardown uploads its
+ // profile — sessions default to not persisting one.
+ await client.patch(`/browser_sessions/${browserSessionId}`, {
+ generate_browser_profile: true,
+ });
await client.post(`/browser_sessions/${browserSessionId}/close`);
queryClient.invalidateQueries({
queryKey: ["browserSession", browserSessionId],
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessions.test.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessions.test.tsx
index fa05f2fe0..b5b4cf289 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessions.test.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessions.test.tsx
@@ -1,6 +1,6 @@
// @vitest-environment jsdom
-import { cleanup, render, screen } from "@testing-library/react";
+import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { MemoryRouter } from "react-router-dom";
@@ -21,6 +21,8 @@ const openUnoccupiedSession: BrowserSession = {
vnc_streaming_supported: true,
};
+const createBrowserSessionMock = vi.fn();
+
vi.mock("@/routes/browserSessions/hooks/useBrowserSessionsQuery", () => ({
useBrowserSessionsQuery: vi.fn((page: number) => ({
data: page === 1 ? [openUnoccupiedSession] : [],
@@ -33,13 +35,14 @@ vi.mock(
() => ({
useCreateBrowserSessionMutation: vi.fn(() => ({
isPending: false,
- mutate: vi.fn(),
+ mutate: createBrowserSessionMock,
})),
}),
);
afterEach(() => {
cleanup();
+ createBrowserSessionMock.mockClear();
});
function renderPage() {
@@ -67,4 +70,25 @@ describe("BrowserSessions", () => {
// runnable_id === null => not occupied
expect(screen.getAllByText("No")).toHaveLength(1);
});
+
+ it("marks Captcha Solver as enterprise without changing the extension value", () => {
+ renderPage();
+ fireEvent.click(screen.getByRole("button", { name: "Create" }));
+
+ const captchaSolverLabel = screen.getByText("Captcha Solver");
+ expect(captchaSolverLabel.parentElement?.textContent).toContain(
+ "Enterprise",
+ );
+
+ fireEvent.click(screen.getByLabelText(/Captcha Solver/));
+ const createButtons = screen.getAllByRole("button", { name: "Create" });
+ fireEvent.click(createButtons[createButtons.length - 1]!);
+
+ expect(createBrowserSessionMock).toHaveBeenCalledWith({
+ proxyLocation: "RESIDENTIAL",
+ timeout: 60,
+ browserType: null,
+ extensions: ["captcha-solver"],
+ });
+ });
});
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx
index 804ba4c24..d80d030fc 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSessions.tsx
@@ -76,6 +76,7 @@ const EXTENSION_OPTIONS: Array<{
value: BrowserSessionExtension;
label: string;
description: string;
+ enterprise?: boolean;
}> = [
{
value: "ad-blocker",
@@ -86,6 +87,7 @@ const EXTENSION_OPTIONS: Array<{
value: "captcha-solver",
label: "Captcha Solver",
description: "Enables automated captcha solving when available.",
+ enterprise: true,
},
];
@@ -466,7 +468,14 @@ function BrowserSessions() {
htmlFor={`extension-${extension.value}`}
className="font-medium"
>
- {extension.label}
+ <span className="inline-flex items-center gap-2">
+ <span>{extension.label}</span>
+ {extension.enterprise ? (
+ <span className="rounded bg-amber-500/20 px-1.5 py-0.5 text-[10px] font-medium text-amber-400">
+ Enterprise
+ </span>
+ ) : null}
+ </span>
</Label>
<p className="text-xs text-muted-foreground">
{extension.description}
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts
index 07862cf44..dc1526ee6 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useCreateBrowserSessionMutation.ts
@@ -22,11 +22,13 @@ function useCreateBrowserSessionMutation() {
timeout = null,
extensions = [],
browserType = null,
+ generateBrowserProfile = false,
}: {
proxyLocation: ProxyLocation | null;
timeout: number | null;
extensions?: BrowserSessionExtension[];
browserType?: BrowserSessionType | null;
+ generateBrowserProfile?: boolean;
}) => {
const client = await getClient(credentialGetter, "sans-api-v1");
return client.post<string, { data: BrowserSession }>(
@@ -36,6 +38,7 @@ function useCreateBrowserSessionMutation() {
timeout,
extensions,
browser_type: browserType,
+ generate_browser_profile: generateBrowserProfile,
},
);
},
diff --git a/skyvern-frontend/src/routes/credentials/CredentialFolderSelector.tsx b/skyvern-frontend/src/routes/credentials/CredentialFolderSelector.tsx
index e1eec459a..6170bad98 100644
--- a/skyvern-frontend/src/routes/credentials/CredentialFolderSelector.tsx
+++ b/skyvern-frontend/src/routes/credentials/CredentialFolderSelector.tsx
@@ -20,18 +20,29 @@ import { useDebounce } from "use-debounce";
import { Skeleton } from "@/components/ui/skeleton";
interface CredentialFolderSelectorProps {
- credentialId: string;
+ credentialId?: string;
currentFolderId: string | null;
+ bulkCount?: number;
+ onBulkFolderSelect?: (folderId: string | null) => Promise<void>;
+ bulkHasFolders?: boolean;
+ disabled?: boolean;
+ trigger?: React.ReactNode;
}
function CredentialFolderSelector({
credentialId,
currentFolderId,
+ bulkCount,
+ onBulkFolderSelect,
+ bulkHasFolders = true,
+ disabled = false,
+ trigger,
}: CredentialFolderSelectorProps) {
const [open, setOpen] = useState(false);
const [search, setSearch] = useState("");
const [debouncedSearch] = useDebounce(search, 500);
const isTyping = search !== debouncedSearch;
+ const isBulkMode = (bulkCount ?? 0) > 0 && Boolean(onBulkFolderSelect);
const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isFetching } =
useInfiniteCredentialFoldersQuery({
@@ -46,32 +57,49 @@ function CredentialFolderSelector({
const updateFolderMutation = useUpdateCredentialFolderAssignmentMutation();
const handleFolderSelect = async (folderId: string | null) => {
- try {
- await updateFolderMutation.mutateAsync({
- credentialId,
- data: { folder_id: folderId },
- });
- setOpen(false);
- setSearch("");
- } catch {
- // onError toast already surfaces the failure
+ // Close before the request; progress feedback lives in the caller's toast
+ // or the bulk bar's Processing label, not in a hanging popover.
+ setOpen(false);
+ setSearch("");
+ if (isBulkMode && onBulkFolderSelect) {
+ await onBulkFolderSelect(folderId);
+ return;
+ }
+ if (credentialId) {
+ try {
+ await updateFolderMutation.mutateAsync({
+ credentialId,
+ data: { folder_id: folderId },
+ });
+ } catch {
+ // onError toast already surfaces the failure
+ }
+ return;
+ }
+ if (import.meta.env.DEV) {
+ console.warn(
+ "CredentialFolderSelector needs credentialId (single mode) or onBulkFolderSelect (bulk mode).",
+ );
}
};
return (
<Popover open={open} onOpenChange={setOpen}>
<PopoverTrigger asChild>
- <Button
- variant="ghost"
- size="icon"
- aria-label="Assign to folder"
- className={cn(
- "h-8 w-8",
- currentFolderId ? "text-blue-400" : "text-slate-400",
- )}
- >
- <FolderIcon className="h-4 w-4" />
- </Button>
+ {trigger ?? (
+ <Button
+ variant="ghost"
+ size="icon"
+ aria-label="Assign to folder"
+ disabled={disabled}
+ className={cn(
+ "h-8 w-8",
+ currentFolderId ? "text-blue-400" : "text-slate-400",
+ )}
+ >
+ <FolderIcon className="h-4 w-4" />
+ </Button>
+ )}
</PopoverTrigger>
<PopoverContent
className="w-80 p-0"
@@ -79,7 +107,11 @@ function CredentialFolderSelector({
onCloseAutoFocus={(e) => e.preventDefault()}
>
<div className="border-b p-3">
- <h4 className="mb-2 text-sm font-medium">Move to folder</h4>
+ <h4 className="mb-2 text-sm font-medium">
+ {isBulkMode
+ ? `Move ${bulkCount} credential${bulkCount === 1 ? "" : "s"} to folder`
+ : "Move to folder"}
+ </h4>
<div className="relative">
<MagnifyingGlassIcon className="absolute left-2 top-1/2 h-4 w-4 -translate-y-1/2 text-slate-400" />
<Input
@@ -102,10 +134,10 @@ function CredentialFolderSelector({
)
}
>
- {currentFolderId && (
+ {(isBulkMode ? bulkHasFolders : Boolean(currentFolderId)) && (
<button
onClick={() => handleFolderSelect(null)}
- disabled={updateFolderMutation.isPending}
+ disabled={disabled || updateFolderMutation.isPending}
className="flex w-full items-center justify-between border-b px-3 py-2 text-left text-sm transition-colors hover:bg-slate-50 disabled:opacity-50 dark:hover:bg-slate-800"
>
<div className="flex items-center gap-2">
@@ -137,12 +169,17 @@ function CredentialFolderSelector({
) : (
<>
{folders.map((folder) => {
- const isCurrentFolder = currentFolderId === folder.folder_id;
+ const isCurrentFolder =
+ !isBulkMode && currentFolderId === folder.folder_id;
return (
<button
key={folder.folder_id}
onClick={() => handleFolderSelect(folder.folder_id)}
- disabled={isCurrentFolder || updateFolderMutation.isPending}
+ disabled={
+ disabled ||
+ isCurrentFolder ||
+ updateFolderMutation.isPending
+ }
className="flex w-full items-center justify-between px-3 py-2 text-left text-sm transition-colors hover:bg-slate-50 disabled:opacity-50 dark:hover:bg-slate-800"
>
<div className="flex items-center gap-2">
diff --git a/skyvern-frontend/src/routes/credentials/CredentialItem.tsx b/skyvern-frontend/src/routes/credentials/CredentialItem.tsx
index 8d5de44d2..b63c24a5a 100644
--- a/skyvern-frontend/src/routes/credentials/CredentialItem.tsx
+++ b/skyvern-frontend/src/routes/credentials/CredentialItem.tsx
@@ -4,6 +4,7 @@ import {
isPasswordCredential,
isSecretCredential,
} from "@/api/types";
+import { SelectionCheckbox } from "@/components/SelectionCheckbox";
import { useState } from "react";
import {
ExclamationTriangleIcon,
@@ -33,9 +34,20 @@ type Props = {
url: string,
userContext?: string,
) => void;
+ index?: number;
+ selected?: boolean;
+ hasSelection?: boolean;
+ onSelect?: (index: number, shiftKey: boolean) => void;
};
-function CredentialItem({ credential, onStartBackgroundTest }: Props) {
+function CredentialItem({
+ credential,
+ onStartBackgroundTest,
+ index = -1,
+ selected = false,
+ hasSelection = false,
+ onSelect,
+}: Props) {
const [editModalOpen, setEditModalOpen] = useState(false);
const activeTest = useCredentialTestStore((s) =>
s.activeTest?.credentialId === credential.credential_id
@@ -151,7 +163,21 @@ function CredentialItem({ credential, onStartBackgroundTest }: Props) {
}
return (
- <div className="flex gap-5 rounded-lg bg-slate-elevation2 p-4">
+ <div
+ className="group/row flex gap-5 rounded-lg bg-slate-elevation2 p-4 data-[state=selected]:ring-1 data-[state=selected]:ring-primary"
+ data-state={selected ? "selected" : undefined}
+ >
+ {onSelect && (
+ <div className="self-start pt-1">
+ <SelectionCheckbox
+ index={index}
+ checked={selected}
+ hasSelection={hasSelection}
+ onSelect={onSelect}
+ ariaLabel={`Select ${credential.name}`}
+ />
+ </div>
+ )}
<div className="w-48 space-y-2">
<p className="w-full truncate" title={credential.name}>
{credential.name}
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsBulkBar.tsx b/skyvern-frontend/src/routes/credentials/CredentialsBulkBar.tsx
new file mode 100644
index 000000000..ce6ece1a8
--- /dev/null
+++ b/skyvern-frontend/src/routes/credentials/CredentialsBulkBar.tsx
@@ -0,0 +1,218 @@
+import { getClient } from "@/api/AxiosClient";
+import { CredentialApiResponse } from "@/api/types";
+import { FolderIcon } from "@/components/icons/FolderIcon";
+import { GarbageIcon } from "@/components/icons/GarbageIcon";
+import { SelectionBar, SelectionBarDivider } from "@/components/SelectionBar";
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { bulkResultToast } from "@/util/bulkResultToast";
+import {
+ BULK_CONCURRENCY_LIMIT,
+ runWithConcurrency,
+} from "@/util/runWithConcurrency";
+import { useQueryClient } from "@tanstack/react-query";
+import { useState } from "react";
+import { CredentialFolderSelector } from "./CredentialFolderSelector";
+
+type Props = {
+ selectedCredentials: CredentialApiResponse[];
+ isOperating: boolean;
+ onOperatingChange: (operating: boolean) => void;
+ onClear: () => void;
+ onReplaceSelection: (ids: Iterable<string>) => void;
+};
+
+function CredentialsBulkBar({
+ selectedCredentials,
+ isOperating,
+ onOperatingChange,
+ onClear,
+ onReplaceSelection,
+}: Props) {
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const [deleteDialog, setDeleteDialog] = useState<{
+ open: boolean;
+ targets: string[];
+ }>({ open: false, targets: [] });
+ const count = selectedCredentials.length;
+
+ async function handleBulkMoveToFolder(folderId: string | null) {
+ onOperatingChange(true);
+ try {
+ const client = await getClient(credentialGetter);
+ const results = await runWithConcurrency(
+ selectedCredentials.map(
+ (credential) => () =>
+ client.put(`/credentials/${credential.credential_id}/folder`, {
+ folder_id: folderId,
+ }),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const succeeded = results.filter((r) => r.status === "fulfilled").length;
+ bulkResultToast({
+ succeeded,
+ total: count,
+ results,
+ successTitle: (n) =>
+ folderId
+ ? `Moved ${n} credential${n !== 1 ? "s" : ""} to folder.`
+ : `Removed ${n} credential${n !== 1 ? "s" : ""} from folder.`,
+ failureTitle: (n) =>
+ folderId
+ ? `Failed to move ${n} credential${n !== 1 ? "s" : ""} to folder.`
+ : `Failed to remove ${n} credential${n !== 1 ? "s" : ""} from folder.`,
+ partialTitle: (successCount, failedCount) =>
+ folderId
+ ? `Moved ${successCount} credential${successCount !== 1 ? "s" : ""} to folder. ${failedCount} failed.`
+ : `Removed ${successCount} credential${successCount !== 1 ? "s" : ""} from folder. ${failedCount} failed.`,
+ });
+ if (succeeded === count) {
+ onClear();
+ }
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["credentials"] });
+ queryClient.invalidateQueries({ queryKey: ["credential-folders"] });
+ }
+ } finally {
+ onOperatingChange(false);
+ }
+ }
+
+ async function handleBulkDeleteConfirm() {
+ const targets = deleteDialog.targets;
+ if (targets.length === 0) {
+ return;
+ }
+ onOperatingChange(true);
+ try {
+ const client = await getClient(credentialGetter);
+ const results = await runWithConcurrency(
+ targets.map(
+ (credentialId) => () => client.delete(`/credentials/${credentialId}`),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const failedIds = new Set<string>();
+ results.forEach((result, index) => {
+ if (result.status === "rejected") {
+ failedIds.add(targets[index]!);
+ }
+ });
+ const succeeded = targets.length - failedIds.size;
+ bulkResultToast({
+ succeeded,
+ total: targets.length,
+ results,
+ successTitle: (n) => `Deleted ${n} credential${n !== 1 ? "s" : ""}.`,
+ failureTitle: (n) =>
+ `Failed to delete ${n} credential${n !== 1 ? "s" : ""}.`,
+ partialTitle: (successCount, failedCount) =>
+ `Deleted ${successCount} credential${successCount !== 1 ? "s" : ""}. ${failedCount} failed.`,
+ });
+ if (failedIds.size === 0) {
+ onClear();
+ } else {
+ onReplaceSelection(failedIds);
+ }
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["credentials"] });
+ queryClient.invalidateQueries({ queryKey: ["credential-folders"] });
+ }
+ } finally {
+ onOperatingChange(false);
+ setDeleteDialog({ open: false, targets: [] });
+ }
+ }
+
+ return (
+ <>
+ <SelectionBar count={count} isOperating={isOperating} onClear={onClear}>
+ <CredentialFolderSelector
+ currentFolderId={null}
+ bulkCount={count}
+ onBulkFolderSelect={handleBulkMoveToFolder}
+ bulkHasFolders={selectedCredentials.some(
+ (credential) => (credential.folder_id ?? null) !== null,
+ )}
+ disabled={isOperating}
+ trigger={
+ <Button size="sm" variant="ghost" disabled={isOperating}>
+ <FolderIcon className="mr-1.5 h-4 w-4" />
+ Move to folder
+ </Button>
+ }
+ />
+ <SelectionBarDivider />
+ <Button
+ size="sm"
+ variant="ghost"
+ className="text-destructive hover:bg-destructive/10 hover:text-destructive"
+ disabled={isOperating}
+ onClick={() =>
+ setDeleteDialog({
+ open: true,
+ targets: selectedCredentials.map(
+ (credential) => credential.credential_id,
+ ),
+ })
+ }
+ >
+ <GarbageIcon className="mr-1.5 h-4 w-4" />
+ Delete
+ </Button>
+ </SelectionBar>
+ <Dialog
+ open={deleteDialog.open}
+ onOpenChange={(open) => {
+ if (!open && !isOperating) {
+ setDeleteDialog({ open: false, targets: [] });
+ }
+ }}
+ >
+ <DialogContent>
+ <DialogHeader>
+ <DialogTitle>
+ Delete {deleteDialog.targets.length} Credential
+ {deleteDialog.targets.length === 1 ? "" : "s"}
+ </DialogTitle>
+ <DialogDescription>
+ Are you sure you want to delete {deleteDialog.targets.length}{" "}
+ {deleteDialog.targets.length === 1 ? "credential" : "credentials"}
+ ? This action cannot be undone.
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <Button
+ variant="secondary"
+ disabled={isOperating}
+ onClick={() => setDeleteDialog({ open: false, targets: [] })}
+ >
+ Cancel
+ </Button>
+ <Button
+ variant="destructive"
+ disabled={isOperating}
+ onClick={() => {
+ void handleBulkDeleteConfirm();
+ }}
+ >
+ {isOperating ? "Deleting..." : "Delete"}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ </>
+ );
+}
+
+export { CredentialsBulkBar };
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsList.tsx b/skyvern-frontend/src/routes/credentials/CredentialsList.tsx
index 842c16e0b..276e17b5a 100644
--- a/skyvern-frontend/src/routes/credentials/CredentialsList.tsx
+++ b/skyvern-frontend/src/routes/credentials/CredentialsList.tsx
@@ -1,5 +1,7 @@
import { Skeleton } from "@/components/ui/skeleton";
+import { useRowSelection } from "@/hooks/useRowSelection";
import { CredentialItem } from "./CredentialItem";
+import { CredentialsBulkBar } from "./CredentialsBulkBar";
import { useCredentialsQuery } from "@/routes/workflows/hooks/useCredentialsQuery";
import {
Pagination,
@@ -65,6 +67,20 @@ function CredentialsList({
enabled: !isResolvingFolder,
});
+ const [isBulkOperating, setIsBulkOperating] = useState(false);
+ const {
+ selected,
+ selectedItems: selectedCredentials,
+ isSelected,
+ handleSelect,
+ clearSelection,
+ replaceSelection,
+ } = useRowSelection({
+ items: credentials ?? [],
+ getId: (credential) => credential.credential_id,
+ resetKey: JSON.stringify([page, trimmedSearch, folderId ?? null]),
+ });
+
if (isResolvingFolder || isLoading) {
return (
<div className="space-y-5">
@@ -96,11 +112,15 @@ function CredentialsList({
return (
<div className="space-y-5">
<div className="space-y-5">
- {credentials.map((credential) => (
+ {credentials.map((credential, index) => (
<CredentialItem
key={credential.credential_id}
credential={credential}
onStartBackgroundTest={onStartBackgroundTest}
+ index={index}
+ selected={isSelected(credential.credential_id)}
+ hasSelection={selected.size > 0}
+ onSelect={handleSelect}
/>
))}
</div>
@@ -133,6 +153,15 @@ function CredentialsList({
</PaginationContent>
</Pagination>
)}
+ {selectedCredentials.length > 0 && (
+ <CredentialsBulkBar
+ selectedCredentials={selectedCredentials}
+ isOperating={isBulkOperating}
+ onOperatingChange={setIsBulkOperating}
+ onClear={clearSelection}
+ onReplaceSelection={replaceSelection}
+ />
+ )}
</div>
);
}
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsTotpTab.tsx b/skyvern-frontend/src/routes/credentials/CredentialsTotpTab.tsx
index b7863891e..1d65a6ff9 100644
--- a/skyvern-frontend/src/routes/credentials/CredentialsTotpTab.tsx
+++ b/skyvern-frontend/src/routes/credentials/CredentialsTotpTab.tsx
@@ -75,6 +75,18 @@ function CredentialsTotpTab() {
Paste the verification message you received. Skyvern extracts the code
and attaches it to the relevant run.
</p>
+ <p className="mt-2 text-sm text-neutral-600 dark:text-slate-400">
+ Prefer to send codes programmatically? See the{" "}
+ <a
+ href="https://docs.skyvern.com/api-reference/credentials/send-totp-code"
+ target="_blank"
+ rel="noopener noreferrer"
+ className="text-blue-400 underline hover:text-blue-300"
+ >
+ API documentation
+ </a>
+ .
+ </p>
<PushTotpCodeForm
className="mt-4"
showAdvancedFields
diff --git a/skyvern-frontend/src/routes/history/RunHistory.tsx b/skyvern-frontend/src/routes/history/RunHistory.tsx
index cc62caa96..3a8dfbb8c 100644
--- a/skyvern-frontend/src/routes/history/RunHistory.tsx
+++ b/skyvern-frontend/src/routes/history/RunHistory.tsx
@@ -2,6 +2,7 @@ import {
ExclamationTriangleIcon,
LightningBoltIcon,
MixerHorizontalIcon,
+ RocketIcon,
} from "@radix-ui/react-icons";
import {
@@ -66,6 +67,10 @@ import { useKeywordSearch } from "@/routes/workflows/hooks/useKeywordSearch";
import { useParameterExpansion } from "@/routes/workflows/hooks/useParameterExpansion";
import { ParameterDisplayInline } from "@/routes/workflows/components/ParameterDisplayInline";
import { HighlightText } from "@/routes/workflows/components/HighlightText";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
+import { OnboardingEmptyState } from "@/components/onboarding/OnboardingEmptyState";
+import { useFeatureFlagVariantKey } from "posthog-js/react";
+import { EXPERIMENT, isABVariant } from "@/util/onboarding/experimentConfig";
const statusValues = new Set<string>(Object.values(Status));
function isKnownStatus(value: string): value is Status {
@@ -116,6 +121,10 @@ function getRunNavigationPath(run: TaskRunListItem): string {
}
function RunHistory() {
+ const onboarding = useOnboardingStateOptional();
+ const isNewUser = onboarding?.isNewUser ?? false;
+ const onboardingState = onboarding?.state ?? null;
+ const onboardingFlag = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
const [searchParams, setSearchParams] = useSearchParams();
const page = searchParams.get("page") ? Number(searchParams.get("page")) : 1;
const itemsPerPage = searchParams.get("page_size")
@@ -242,7 +251,7 @@ function RunHistory() {
return <TableMessageRow colSpan={6}>No runs found</TableMessageRow>;
}
- return runs?.map((run) => {
+ return runs?.map((run, index) => {
const executionTime = formatExecutionTime(
run.started_at ?? run.created_at,
run.finished_at,
@@ -283,6 +292,7 @@ function RunHistory() {
<React.Fragment key={run.task_run_id}>
<TableRow
className="cursor-pointer"
+ data-hint={index === 0 ? "run-recording" : undefined}
onClick={(event) => {
handleNavigate(event, navPath);
}}
@@ -372,125 +382,158 @@ function RunHistory() {
setSearchParams(params, { replace: true });
}
+ const hasActiveFilters =
+ statusFilters.length > 0 ||
+ !!debouncedSearch ||
+ !!workflowPermanentIdFilter;
+ const showOnboardingEmpty =
+ !isFetching &&
+ runs?.length === 0 &&
+ !hasActiveFilters &&
+ isNewUser &&
+ onboardingState?.first_run_at === null &&
+ isABVariant(onboardingFlag);
+
return (
<div className="space-y-4">
<header>
<h1 className="text-2xl">Run History</h1>
</header>
- {workflowPermanentIdFilter ? (
- <div
- className="flex items-center justify-between gap-2 rounded-md border border-dashed bg-muted/30 px-3 py-2 text-xs"
- data-testid="workflow-filter-banner"
- >
- <span className="truncate">
- Filtering runs for agent{" "}
- <span className="font-mono">{workflowPermanentIdFilter}</span>
- </span>
- <Button
- variant="ghost"
- size="sm"
- onClick={clearWorkflowFilter}
- className="h-auto py-1 text-xs"
- >
- Clear
- </Button>
+ {showOnboardingEmpty ? (
+ <div className="rounded-lg border">
+ <OnboardingEmptyState
+ surface="runs"
+ icon={<RocketIcon className="h-6 w-6" />}
+ title="Your run history will appear here"
+ description="Every time you run a workflow, the result shows up on this page. Create your first workflow to get started."
+ primaryAction={{
+ label: "Create your first workflow",
+ onClick: () => navigate("/workflows"),
+ }}
+ secondaryAction={{
+ label: "Browse templates",
+ onClick: () => navigate("/workflows"),
+ }}
+ />
</div>
- ) : null}
- <div className="flex items-center justify-between gap-4">
- <TableSearchInput
- value={search}
- onChange={(value) => {
- setSearch(value);
- const params = new URLSearchParams(searchParams);
- params.set("page", "1");
- setSearchParams(params, { replace: true });
- }}
- placeholder={
- workflowPermanentIdFilter
- ? "Clear the agent filter above to search"
- : "Search by run ID or input..."
- }
- disabled={!!workflowPermanentIdFilter}
- className="w-48 lg:w-72"
- />
- <StatusFilterDropdown
- values={statusFilters}
- onChange={(filters) => {
- const params = new URLSearchParams(searchParams);
- if (filters.length === 0) {
- params.delete("status");
- } else {
- params.set("status", filters.join(","));
- }
- params.set("page", "1");
- setSearchParams(params, { replace: true });
- }}
- />
- </div>
- <div className="overflow-hidden rounded-lg border border-border">
- <Table className="sm:table-fixed">
- <TableHeader>
- <TableRow>
- <TableHead className="w-[20%]">Run ID</TableHead>
- <TableHead className="w-[20%]">Detail</TableHead>
- <TableHead className="w-[16%]">Status</TableHead>
- <TableHead className="w-[27%]">Created At</TableHead>
- <TableHead className="w-[8%]">Duration</TableHead>
- <TableHead className="w-[8%]"></TableHead>
- </TableRow>
- </TableHeader>
- <TableBody>{displayTableBody()}</TableBody>
- </Table>
- <div className="relative px-3 py-3">
- <div className="absolute left-3 top-1/2 flex -translate-y-1/2 items-center gap-2 text-sm">
- <span className="text-neutral-600 dark:text-slate-400">
- Items per page
- </span>
- <Select
- value={String(itemsPerPage)}
- onValueChange={(size) => {
+ ) : (
+ <>
+ {workflowPermanentIdFilter ? (
+ <div
+ className="flex items-center justify-between gap-2 rounded-md border border-dashed bg-muted/30 px-3 py-2 text-xs"
+ data-testid="workflow-filter-banner"
+ >
+ <span className="truncate">
+ Filtering runs for workflow{" "}
+ <span className="font-mono">{workflowPermanentIdFilter}</span>
+ </span>
+ <Button
+ variant="ghost"
+ size="sm"
+ onClick={clearWorkflowFilter}
+ className="h-auto py-1 text-xs"
+ >
+ Clear
+ </Button>
+ </div>
+ ) : null}
+ <div className="flex items-center justify-between gap-4">
+ <TableSearchInput
+ value={search}
+ onChange={(value) => {
+ setSearch(value);
const params = new URLSearchParams(searchParams);
- params.set("page_size", size);
params.set("page", "1");
setSearchParams(params, { replace: true });
}}
- >
- <SelectTrigger className="w-[65px]">
- <SelectValue />
- </SelectTrigger>
- <SelectContent>
- <SelectItem value="5">5</SelectItem>
- <SelectItem value="10">10</SelectItem>
- <SelectItem value="20">20</SelectItem>
- <SelectItem value="50">50</SelectItem>
- </SelectContent>
- </Select>
+ placeholder={
+ workflowPermanentIdFilter
+ ? "Clear the agent filter above to search"
+ : "Search by run ID or input..."
+ }
+ disabled={!!workflowPermanentIdFilter}
+ className="w-48 lg:w-72"
+ />
+ <StatusFilterDropdown
+ values={statusFilters}
+ onChange={(filters) => {
+ const params = new URLSearchParams(searchParams);
+ if (filters.length === 0) {
+ params.delete("status");
+ } else {
+ params.set("status", filters.join(","));
+ }
+ params.set("page", "1");
+ setSearchParams(params, { replace: true });
+ }}
+ />
</div>
- <Pagination className="pt-0">
- <PaginationContent>
- <PaginationItem>
- <PaginationPrevious
- className={cn({
- "cursor-not-allowed opacity-50": page === 1,
- })}
- onClick={handlePreviousPage}
- />
- </PaginationItem>
- <PaginationItem>
- <PaginationLink>{page}</PaginationLink>
- </PaginationItem>
- <PaginationItem>
- <PaginationNext
- className={cn({
- "cursor-not-allowed opacity-50": isNextDisabled,
- })}
- onClick={handleNextPage}
- />
- </PaginationItem>
- </PaginationContent>
- </Pagination>
- </div>
- </div>
+ <div className="overflow-hidden rounded-lg border border-border">
+ <Table className="sm:table-fixed">
+ <TableHeader>
+ <TableRow>
+ <TableHead className="w-[20%]">Run ID</TableHead>
+ <TableHead className="w-[20%]">Detail</TableHead>
+ <TableHead className="w-[16%]">Status</TableHead>
+ <TableHead className="w-[27%]">Created At</TableHead>
+ <TableHead className="w-[8%]">Duration</TableHead>
+ <TableHead className="w-[8%]"></TableHead>
+ </TableRow>
+ </TableHeader>
+ <TableBody>{displayTableBody()}</TableBody>
+ </Table>
+ <div className="relative px-3 py-3">
+ <div className="absolute left-3 top-1/2 flex -translate-y-1/2 items-center gap-2 text-sm">
+ <span className="text-neutral-600 dark:text-slate-400">
+ Items per page
+ </span>
+ <Select
+ value={String(itemsPerPage)}
+ onValueChange={(size) => {
+ const params = new URLSearchParams(searchParams);
+ params.set("page_size", size);
+ params.set("page", "1");
+ setSearchParams(params, { replace: true });
+ }}
+ >
+ <SelectTrigger className="w-[65px]">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="5">5</SelectItem>
+ <SelectItem value="10">10</SelectItem>
+ <SelectItem value="20">20</SelectItem>
+ <SelectItem value="50">50</SelectItem>
+ </SelectContent>
+ </Select>
+ </div>
+ <Pagination className="pt-0">
+ <PaginationContent>
+ <PaginationItem>
+ <PaginationPrevious
+ className={cn({
+ "cursor-not-allowed opacity-50": page === 1,
+ })}
+ onClick={handlePreviousPage}
+ />
+ </PaginationItem>
+ <PaginationItem>
+ <PaginationLink>{page}</PaginationLink>
+ </PaginationItem>
+ <PaginationItem>
+ <PaginationNext
+ className={cn({
+ "cursor-not-allowed opacity-50": isNextDisabled,
+ })}
+ onClick={handleNextPage}
+ />
+ </PaginationItem>
+ </PaginationContent>
+ </Pagination>
+ </div>
+ </div>
+ </>
+ )}
</div>
);
}
diff --git a/skyvern-frontend/src/routes/schedules/SchedulesPage.tsx b/skyvern-frontend/src/routes/schedules/SchedulesPage.tsx
index b82e032cb..c45f5a728 100644
--- a/skyvern-frontend/src/routes/schedules/SchedulesPage.tsx
+++ b/skyvern-frontend/src/routes/schedules/SchedulesPage.tsx
@@ -1,4 +1,4 @@
-import { useRef, useState } from "react";
+import { useState } from "react";
import { useNavigate, useSearchParams } from "react-router-dom";
import { useDebounce } from "use-debounce";
import {
@@ -51,11 +51,21 @@ import {
import { useQueryClient } from "@tanstack/react-query";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
+import { SelectionBar } from "@/components/SelectionBar";
+import {
+ SelectionCheckboxCell,
+ SelectionHeaderCheckboxCell,
+} from "@/components/SelectionCheckbox";
+import { useRowSelection } from "@/hooks/useRowSelection";
+import { bulkResultToast } from "@/util/bulkResultToast";
+import {
+ BULK_CONCURRENCY_LIMIT,
+ runWithConcurrency,
+} from "@/util/runWithConcurrency";
import { TableSearchInput } from "@/components/TableSearchInput";
import { Pill } from "@/components/StatusBadge";
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
-import { toast } from "@/components/ui/use-toast";
import { useOrganizationSchedulesQuery } from "./useOrganizationSchedulesQuery";
import {
useDeleteOrgScheduleMutation,
@@ -68,32 +78,6 @@ import { basicLocalTimeFormat, basicTimeFormat } from "@/util/timeFormat";
import type { OrganizationScheduleItem } from "@/routes/workflows/types/scheduleTypes";
import { CreateOrgScheduleDialog } from "./CreateOrgScheduleDialog";
-const BULK_CONCURRENCY_LIMIT = 5;
-
-async function runWithConcurrency<T>(
- tasks: Array<() => Promise<T>>,
- limit: number,
-): Promise<PromiseSettledResult<T>[]> {
- const results: Array<PromiseSettledResult<T>> = new Array(tasks.length);
- let nextIndex = 0;
-
- async function worker() {
- while (nextIndex < tasks.length) {
- const index = nextIndex++;
- try {
- results[index] = { status: "fulfilled", value: await tasks[index]!() };
- } catch (reason) {
- results[index] = { status: "rejected", reason };
- }
- }
- }
-
- await Promise.all(
- Array.from({ length: Math.min(limit, tasks.length) }, () => worker()),
- );
- return results;
-}
-
type ScheduleStatus = "active" | "paused";
const STATUS_OPTIONS: Array<{ label: string; value: ScheduleStatus }> = [
@@ -119,8 +103,6 @@ function SchedulesPage() {
const [search, setSearch] = useState("");
const [debouncedSearch] = useDebounce(search, 500);
const [statusFilters, setStatusFilters] = useState<ScheduleStatus[]>([]);
- const [selected, setSelected] = useState<Set<string>>(new Set());
- const lastSelectedIndex = useRef<number | null>(null);
const [isBulkOperating, setIsBulkOperating] = useState(false);
const [deleteDialog, setDeleteDialog] = useState<{
open: boolean;
@@ -154,49 +136,26 @@ function SchedulesPage() {
const totalCount = data?.total_count ?? 0;
const totalPages = Math.max(1, Math.ceil(totalCount / pageSize));
- const allSelected =
- schedules.length > 0 &&
- schedules.every((s) => selected.has(s.workflow_schedule_id));
-
- function toggleSelectAll() {
- if (allSelected) {
- setSelected(new Set());
- } else {
- setSelected(new Set(schedules.map((s) => s.workflow_schedule_id)));
- }
- }
-
- function handleSelect(index: number, shiftKey: boolean) {
- const id = schedules[index]!.workflow_schedule_id;
- if (shiftKey && lastSelectedIndex.current !== null) {
- const start = Math.min(lastSelectedIndex.current, index);
- const end = Math.max(lastSelectedIndex.current, index);
- setSelected((prev) => {
- const next = new Set(prev);
- for (let i = start; i <= end; i++) {
- next.add(schedules[i]!.workflow_schedule_id);
- }
- return next;
- });
- } else {
- setSelected((prev) => {
- const next = new Set(prev);
- if (next.has(id)) {
- next.delete(id);
- } else {
- next.add(id);
- }
- return next;
- });
- }
- lastSelectedIndex.current = index;
- }
+ const {
+ selected,
+ selectedItems: selectedSchedules,
+ isSelected,
+ allSelected,
+ someSelected,
+ handleSelect,
+ toggleSelectAll,
+ clearSelection,
+ replaceSelection,
+ } = useRowSelection({
+ items: schedules,
+ getId: (s) => s.workflow_schedule_id,
+ resetKey: JSON.stringify([page, pageSize, statusFilters, debouncedSearch]),
+ });
function setPage(p: number) {
const params = new URLSearchParams(searchParams);
params.set("page", String(p));
setSearchParams(params);
- setSelected(new Set());
}
function setPageSize(size: string) {
@@ -204,7 +163,6 @@ function SchedulesPage() {
params.set("page_size", size);
params.set("page", "1");
setSearchParams(params);
- setSelected(new Set());
}
function handleDeleteConfirm() {
@@ -212,19 +170,10 @@ function SchedulesPage() {
deleteMutation.mutate(deleteDialog.schedule, {
onSettled: () => {
setDeleteDialog({ open: false, schedule: null });
- setSelected((prev) => {
- const next = new Set(prev);
- next.delete(deleteDialog.schedule!.workflow_schedule_id);
- return next;
- });
},
});
}
- const selectedSchedules = schedules.filter((s) =>
- selected.has(s.workflow_schedule_id),
- );
-
async function runBulkOperation(
items: OrganizationScheduleItem[],
makeTask: (
@@ -243,23 +192,20 @@ function SchedulesPage() {
BULK_CONCURRENCY_LIMIT,
);
const succeeded = results.filter((r) => r.status === "fulfilled").length;
- const failed = results.filter((r) => r.status === "rejected").length;
- if (failed === 0) {
- toast({
- title: `${succeeded} schedule${succeeded !== 1 ? "s" : ""} ${successLabel} successfully.`,
- variant: "success",
- });
- setSelected(new Set());
- } else if (succeeded === 0) {
- toast({
- title: `Failed to ${failureLabel} ${failed} schedule${failed !== 1 ? "s" : ""}.`,
- variant: "destructive",
- });
- } else {
- toast({
- title: `${succeeded}/${items.length} schedules ${successLabel} successfully. ${failed} failed.`,
- variant: "destructive",
- });
+ bulkResultToast({
+ succeeded,
+ total: items.length,
+ results,
+ successTitle: (n) =>
+ `${n} schedule${n !== 1 ? "s" : ""} ${successLabel} successfully.`,
+ failureTitle: (n) =>
+ `Failed to ${failureLabel} ${n} schedule${n !== 1 ? "s" : ""}.`,
+ partialTitle: (successCount, failedCount) =>
+ `${successCount} schedule${successCount !== 1 ? "s" : ""} ${successLabel} successfully. ${failedCount} failed.`,
+ });
+ if (succeeded === items.length) {
+ clearSelection();
+ } else if (succeeded > 0) {
// Keep only failed items selected so the user can retry
const failedIds = new Set<string>();
results.forEach((result, i) => {
@@ -267,7 +213,7 @@ function SchedulesPage() {
failedIds.add(items[i]!.workflow_schedule_id);
}
});
- setSelected(failedIds);
+ replaceSelection(failedIds);
}
queryClient.invalidateQueries({ queryKey: ["organizationSchedules"] });
queryClient.invalidateQueries({ queryKey: ["scheduleDetail"] });
@@ -336,7 +282,7 @@ function SchedulesPage() {
setBulkDeleteDialog({ open: false, count: 0 });
}
- const showCheckbox = schedules.length > 1;
+ const showCheckbox = schedules.length > 0;
const columnCount = showCheckbox ? 7 : 6;
return (
@@ -411,14 +357,16 @@ function SchedulesPage() {
<div className="overflow-hidden rounded-lg border border-border">
<Table className="table-fixed">
<TableHeader>
- <TableRow>
+ <TableRow className="group/header">
{showCheckbox && (
- <TableHead className="w-[3%]">
- <Checkbox
- checked={allSelected}
- onCheckedChange={toggleSelectAll}
- />
- </TableHead>
+ <SelectionHeaderCheckboxCell
+ className="w-[3%]"
+ allSelected={allSelected}
+ someSelected={someSelected}
+ hasSelection={selected.size > 0}
+ onToggleAll={toggleSelectAll}
+ ariaLabel="Select all schedules"
+ />
)}
<TableHead className={showCheckbox ? "w-[28%]" : "w-[31%]"}>
Agent
@@ -468,7 +416,12 @@ function SchedulesPage() {
key={schedule.workflow_schedule_id}
tabIndex={0}
aria-label={`Open schedule ${schedule.name ?? schedule.workflow_title}`}
- className="cursor-pointer select-none hover:bg-muted/50 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-[-2px] focus-visible:outline-primary"
+ data-state={
+ isSelected(schedule.workflow_schedule_id)
+ ? "selected"
+ : undefined
+ }
+ className="group/row cursor-pointer select-none hover:bg-muted/50 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-[-2px] focus-visible:outline-primary"
onClick={() =>
navigate(
`/schedules/${schedule.workflow_permanent_id}/${schedule.workflow_schedule_id}`,
@@ -487,17 +440,13 @@ function SchedulesPage() {
}}
>
{showCheckbox && (
- <TableCell
- onClick={(e) => {
- e.stopPropagation();
- handleSelect(index, e.shiftKey);
- }}
- >
- <Checkbox
- checked={selected.has(schedule.workflow_schedule_id)}
- className="pointer-events-none"
- />
- </TableCell>
+ <SelectionCheckboxCell
+ index={index}
+ checked={isSelected(schedule.workflow_schedule_id)}
+ hasSelection={selected.size > 0}
+ onSelect={handleSelect}
+ ariaLabel={`Select schedule ${schedule.name ?? schedule.workflow_title}`}
+ />
)}
<TableCell className="truncate font-medium">
{schedule.workflow_title}
@@ -521,49 +470,52 @@ function SchedulesPage() {
<StatusDisplay enabled={schedule.enabled} />
</TableCell>
<TableCell onClick={(e) => e.stopPropagation()}>
- <DropdownMenu modal={false}>
- <DropdownMenuTrigger asChild>
- <Button
- variant="ghost"
- size="icon"
- className="size-8 text-muted-foreground hover:text-foreground"
- >
- <DotsHorizontalIcon className="size-4" />
- </Button>
- </DropdownMenuTrigger>
- <DropdownMenuContent align="end">
- {schedule.enabled ? (
+ {/* Row menus yield to the bulk bar while any selection is active. */}
+ {selected.size === 0 && (
+ <DropdownMenu modal={false}>
+ <DropdownMenuTrigger asChild>
+ <Button
+ variant="ghost"
+ size="icon"
+ className="size-8 text-muted-foreground hover:text-foreground"
+ >
+ <DotsHorizontalIcon className="size-4" />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="end">
+ {schedule.enabled ? (
+ <DropdownMenuItem
+ onSelect={() => disableMutation.mutate(schedule)}
+ >
+ <PauseIcon className="mr-2 size-4" />
+ Pause
+ </DropdownMenuItem>
+ ) : (
+ <DropdownMenuItem
+ onSelect={() => enableMutation.mutate(schedule)}
+ >
+ <PlayIcon className="mr-2 size-4" />
+ Activate
+ </DropdownMenuItem>
+ )}
<DropdownMenuItem
- onSelect={() => disableMutation.mutate(schedule)}
+ onSelect={() => duplicateMutation.mutate(schedule)}
>
- <PauseIcon className="mr-2 size-4" />
- Pause
+ <CopyIcon className="mr-2 size-4" />
+ Duplicate
</DropdownMenuItem>
- ) : (
<DropdownMenuItem
- onSelect={() => enableMutation.mutate(schedule)}
+ onSelect={() =>
+ setDeleteDialog({ open: true, schedule })
+ }
+ className="text-destructive"
>
- <PlayIcon className="mr-2 size-4" />
- Activate
+ <TrashIcon className="mr-2 size-4" />
+ Delete
</DropdownMenuItem>
- )}
- <DropdownMenuItem
- onSelect={() => duplicateMutation.mutate(schedule)}
- >
- <CopyIcon className="mr-2 size-4" />
- Duplicate
- </DropdownMenuItem>
- <DropdownMenuItem
- onSelect={() =>
- setDeleteDialog({ open: true, schedule })
- }
- className="text-destructive"
- >
- <TrashIcon className="mr-2 size-4" />
- Delete
- </DropdownMenuItem>
- </DropdownMenuContent>
- </DropdownMenu>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ )}
</TableCell>
</TableRow>
))}
@@ -634,15 +586,15 @@ function SchedulesPage() {
</div>
{/* Multi-select bulk action bar */}
- {selected.size > 0 && (
- <div className="fixed inset-x-0 bottom-6 mx-auto flex w-fit items-center gap-3 rounded-lg border border-slate-700 bg-slate-900 px-6 py-3 shadow-xl">
- <span className="text-sm text-slate-300">
- {isBulkOperating ? "Processing…" : `${selected.size} selected`}
- </span>
- <div className="h-6 w-px bg-slate-700" />
+ {selectedSchedules.length > 0 && (
+ <SelectionBar
+ count={selectedSchedules.length}
+ isOperating={isBulkOperating}
+ onClear={clearSelection}
+ >
<Button
size="sm"
- className="bg-green-900 text-green-50 hover:bg-green-800"
+ variant="ghost"
onClick={handleBulkActivate}
disabled={isBulkOperating}
>
@@ -651,7 +603,7 @@ function SchedulesPage() {
</Button>
<Button
size="sm"
- className="bg-amber-800 text-amber-50 hover:bg-amber-700"
+ variant="ghost"
onClick={handleBulkPause}
disabled={isBulkOperating}
>
@@ -660,7 +612,7 @@ function SchedulesPage() {
</Button>
<Button
size="sm"
- className="bg-blue-800 text-blue-50 hover:bg-blue-700"
+ variant="ghost"
onClick={handleBulkDuplicate}
disabled={isBulkOperating}
>
@@ -669,14 +621,15 @@ function SchedulesPage() {
</Button>
<Button
size="sm"
- className="bg-red-900 text-red-50 hover:bg-red-800"
+ variant="ghost"
+ className="text-destructive hover:bg-destructive/10 hover:text-destructive"
onClick={handleBulkDelete}
disabled={isBulkOperating}
>
<TrashIcon className="mr-1.5 size-3.5" />
Delete
</Button>
- </div>
+ </SelectionBar>
)}
{/* Bulk delete confirmation dialog */}
diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
index ba768714d..02d1d6f9a 100644
--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
@@ -46,6 +46,11 @@ import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";
import { constructCacheKeyValueFromParameters } from "@/routes/workflows/editor/utils";
import { useWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowQuery";
+import { CredentialSetupPrompt } from "@/components/onboarding/CredentialSetupPrompt";
+import { useFeatureFlagVariantKey } from "posthog-js/react";
+import { EXPERIMENT } from "@/util/onboarding/experimentConfig";
+import { isActivationRun } from "@/util/onboarding/rolloutGating";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
import { type ApiCommandOptions } from "@/util/apiCommands";
import { parseHeaderJson } from "@/util/secretHeaders";
@@ -328,6 +333,14 @@ function RunWorkflowForm({
[workflow],
);
const hasLoginBlockValidationError = loginBlocksWithoutCredentials.length > 0;
+ const onboarding = useOnboardingStateOptional();
+ const onboardingFlagVariant = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
+ const onboardingLoading = onboarding != null && onboarding.isLoading;
+ // Gate on the rollout arm so a 0% rollout / rollback restores the
+ // pre-onboarding login-block alert instead of the credential prompt.
+ const isActivation = isActivationRun(onboardingFlagVariant, onboarding);
+ const showDestructiveLoginAlert =
+ hasLoginBlockValidationError && !isActivation && !onboardingLoading;
const blockingParameterTypes = new Set([
"boolean",
@@ -623,7 +636,14 @@ function RunWorkflowForm({
</div>
</header>
- {hasLoginBlockValidationError && (
+ {hasLoginBlockValidationError && isActivation && (
+ <CredentialSetupPrompt
+ workflowPermanentId={workflowPermanentId}
+ blocksMissingCredentials={loginBlocksWithoutCredentials}
+ />
+ )}
+
+ {showDestructiveLoginAlert && (
<Alert variant="destructive">
<ExclamationTriangleIcon className="h-4 w-4" />
<AlertTitle>Cannot run agent</AlertTitle>
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx
index 557b272ec..2fea58a41 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx
@@ -12,6 +12,7 @@ const { mockFeatureFlagEnabled } = vi.hoisted(() => ({
vi.mock("posthog-js/react", () => ({
useFeatureFlagEnabled: (flag: string) => mockFeatureFlagEnabled(flag),
+ useFeatureFlagVariantKey: () => undefined,
}));
vi.mock("use-debounce", () => ({
@@ -86,6 +87,23 @@ vi.mock("./hooks/useParameterExpansion", () => ({
}),
}));
+vi.mock("@/store/onboarding/useOnboardingState", () => ({
+ useOnboardingState: () => ({
+ state: null,
+ isLoading: false,
+ updateState: vi.fn(),
+ isNewUser: false,
+ abVariant: null,
+ }),
+ useOnboardingStateOptional: () => ({
+ state: null,
+ isLoading: false,
+ updateState: vi.fn(),
+ isNewUser: false,
+ abVariant: null,
+ }),
+}));
+
afterEach(() => {
cleanup();
vi.clearAllMocks();
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx
index e95a37bee..ee4cf90cb 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx
@@ -76,12 +76,22 @@ import { useParameterExpansion } from "./hooks/useParameterExpansion";
import { ParameterDisplayInline } from "./components/ParameterDisplayInline";
import { getOrderedRunParameters } from "./utils";
import { buildWorkflowAnalyticsPath } from "./workflowAnalyticsPath";
-import { useFeatureFlagEnabled } from "posthog-js/react";
+import {
+ useFeatureFlagEnabled,
+ useFeatureFlagVariantKey,
+} from "posthog-js/react";
+import { EXPERIMENT, isABVariant } from "@/util/onboarding/experimentConfig";
import { ANALYTICS_DASHBOARD_FLAG } from "@/util/featureFlags";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
+import { OnboardingEmptyState } from "@/components/onboarding/OnboardingEmptyState";
function WorkflowPage() {
const { workflowPermanentId } = useParams();
const isCloud = useContext(CloudContext);
+ const onboarding = useOnboardingStateOptional();
+ const isNewUser = onboarding?.isNewUser ?? false;
+ const onboardingState = onboarding?.state ?? null;
+ const onboardingFlag = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
const analyticsEnabled =
useFeatureFlagEnabled(ANALYTICS_DASHBOARD_FLAG) === true;
const [searchParams, setSearchParams] = useSearchParams();
@@ -226,224 +236,264 @@ function WorkflowPage() {
<header>
<h1 className="text-2xl">Past Runs</h1>
</header>
- <div className="flex items-center justify-between gap-4">
- <TableSearchInput
- value={search}
- onChange={(value) => {
- setSearch(value);
- const params = new URLSearchParams(searchParams);
- params.set("page", "1");
- setSearchParams(params, { replace: true });
- }}
- placeholder="Search runs by input..."
- className="w-48 lg:w-72"
- />
- <StatusFilterDropdown
- values={statusFilters}
- onChange={setStatusFilters}
- />
- </div>
- <div className="overflow-hidden rounded-lg border border-border">
- <Table>
- <TableHeader>
- <TableRow>
- <TableHead className="w-[20%]">ID</TableHead>
- <TableHead className="w-[20%]">Status</TableHead>
- <TableHead className="w-[20%]">Created At</TableHead>
- <TableHead className="w-[20%]">Duration</TableHead>
- <TableHead className="w-[20%] text-right">Actions</TableHead>
- </TableRow>
- </TableHeader>
- <TableBody>
- {isLoading ? (
- <TableMessageRow colSpan={5}>Loading runs…</TableMessageRow>
- ) : workflowRuns?.length === 0 ? (
- <TableMessageRow colSpan={5}>
- No agent runs found
- </TableMessageRow>
- ) : (
- workflowRuns?.map((workflowRun) => {
- const workflowRunId =
- workflowRun.script_run === true ? (
- <div className="flex items-center gap-2">
- <Tip content="Ran with code">
- <LightningBoltIcon className="text-[gold]" />
- </Tip>
- <span>{workflowRun.workflow_run_id ?? ""}</span>
- </div>
- ) : (
- (workflowRun.workflow_run_id ?? "")
- );
-
- const isExpanded = expandedRows.has(
- workflowRun.workflow_run_id,
- );
-
- return (
- <React.Fragment key={workflowRun.workflow_run_id}>
- {/* Main run row */}
- <TableRow
- onClick={(event) => {
- const url = env.useNewRunsUrl
- ? `/runs/${workflowRun.workflow_run_id}`
- : `/workflows/${workflowPermanentId}/${workflowRun.workflow_run_id}/overview`;
-
- if (event.ctrlKey || event.metaKey) {
- window.open(
- window.location.origin + url,
- "_blank",
- "noopener,noreferrer",
- );
- return;
- }
- navigate(url);
- }}
- className="cursor-pointer"
- >
- <TableCell className="font-mono text-xs text-muted-foreground">
- {workflowRunId}
- </TableCell>
- <TableCell>
- <StatusBadge status={workflowRun.status} />
- </TableCell>
- <TableCell
- className="text-muted-foreground"
- title={basicTimeFormat(workflowRun.created_at)}
- >
- {compactLocalDateTime(workflowRun.created_at)}
- </TableCell>
- <TableCell className="tabular-nums text-muted-foreground">
- {formatExecutionTime(
- workflowRun.started_at ?? workflowRun.created_at,
- workflowRun.finished_at,
- ) ?? "-"}
- </TableCell>
- <TableCell>
- <div className="flex justify-end gap-2">
- <TooltipProvider>
- <Tooltip>
- <TooltipTrigger asChild>
- <Button
- size="icon"
- variant="ghost"
- onClick={(event) => {
- event.stopPropagation();
- toggleParametersExpanded(
- workflowRun.workflow_run_id,
- );
- }}
- className={cn(
- isExpanded
- ? "text-blue-400"
- : "text-muted-foreground hover:text-foreground",
- )}
- >
- <MixerHorizontalIcon className="h-4 w-4" />
- </Button>
- </TooltipTrigger>
- <TooltipContent>
- {isExpanded ? "Hide Inputs" : "Show Inputs"}
- </TooltipContent>
- </Tooltip>
- </TooltipProvider>
- </div>
- </TableCell>
- </TableRow>
-
- {/* Expanded parameters section */}
- {isExpanded && (
- <TableRow key={`${workflowRun.workflow_run_id}-params`}>
- <TableCell
- colSpan={5}
- className="bg-slate-50 dark:bg-slate-900/50"
- >
- <WorkflowRunParameters
- workflowPermanentId={workflowPermanentId}
- workflowRunId={workflowRun.workflow_run_id}
- workflow={workflow}
- searchQuery={debouncedSearch}
- keywordMatchesParameter={matchesParameter}
- />
- </TableCell>
- </TableRow>
- )}
- </React.Fragment>
- );
- })
- )}
- </TableBody>
- </Table>
- <RunParametersDialog
- open={openRunParams !== null}
- onOpenChange={(open) => {
- if (!open) setOpenRunParams(null);
- }}
- workflowPermanentId={workflowPermanentId}
- workflowRunId={openRunParams}
- />
- <div className="relative px-3 py-3">
- <div className="absolute left-3 top-1/2 flex -translate-y-1/2 items-center gap-2 text-sm">
- <span className="text-slate-400">Items per page</span>
- <Select
- value={String(pageSize)}
- onValueChange={(size) => {
+ {!isLoading &&
+ workflowRuns?.length === 0 &&
+ statusFilters.length === 0 &&
+ !debouncedSearch &&
+ isNewUser &&
+ onboardingState?.first_run_at === null &&
+ isABVariant(onboardingFlag) ? (
+ <div className="rounded-md border">
+ <OnboardingEmptyState
+ surface="runs"
+ icon={<PlayIcon className="h-6 w-6" />}
+ title="No runs yet for this workflow"
+ description="Run this workflow to see results here. Each run tracks status, parameters, and duration."
+ primaryAction={{
+ label: "Run this workflow",
+ onClick: () =>
+ navigate(`/workflows/${workflowPermanentId}/run`),
+ }}
+ secondaryAction={{
+ label: "View documentation",
+ onClick: () =>
+ window.open(
+ "https://docs.skyvern.com",
+ "_blank",
+ "noopener,noreferrer",
+ ),
+ }}
+ />
+ </div>
+ ) : (
+ <>
+ <div className="flex items-center justify-between gap-4">
+ <TableSearchInput
+ value={search}
+ onChange={(value) => {
+ setSearch(value);
const params = new URLSearchParams(searchParams);
- params.set("page_size", size);
params.set("page", "1");
setSearchParams(params, { replace: true });
}}
- >
- <SelectTrigger className="w-[65px]">
- <SelectValue />
- </SelectTrigger>
- <SelectContent>
- {PAGE_SIZE_OPTIONS.map((size) => (
- <SelectItem key={size} value={size}>
- {size}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
+ placeholder="Search runs by input..."
+ className="w-48 lg:w-72"
+ />
+ <StatusFilterDropdown
+ values={statusFilters}
+ onChange={setStatusFilters}
+ />
</div>
- <Pagination className="pt-0">
- <PaginationContent>
- <PaginationItem>
- <PaginationPrevious
- className={cn({ "cursor-not-allowed": page === 1 })}
- onClick={() => {
- if (page === 1) {
- return;
- }
- const params = new URLSearchParams(searchParams);
- params.set("page", String(Math.max(1, page - 1)));
- setSearchParams(params, { replace: true });
- }}
- />
- </PaginationItem>
- <PaginationItem>
- <PaginationLink>{page}</PaginationLink>
- </PaginationItem>
- <PaginationItem>
- <PaginationNext
- className={cn({
- "cursor-not-allowed":
- workflowRuns !== undefined &&
- workflowRuns.length < pageSize,
- })}
- onClick={() => {
- if (workflowRuns && workflowRuns.length < pageSize) {
- return;
- }
+ <div className="overflow-hidden rounded-lg border border-border">
+ <Table>
+ <TableHeader>
+ <TableRow>
+ <TableHead className="w-[20%]">ID</TableHead>
+ <TableHead className="w-[20%]">Status</TableHead>
+ <TableHead className="w-[20%]">Created At</TableHead>
+ <TableHead className="w-[20%]">Duration</TableHead>
+ <TableHead className="w-[20%] text-right">
+ Actions
+ </TableHead>
+ </TableRow>
+ </TableHeader>
+ <TableBody>
+ {isLoading ? (
+ <TableMessageRow colSpan={5}>Loading runs…</TableMessageRow>
+ ) : workflowRuns?.length === 0 ? (
+ <TableMessageRow colSpan={5}>
+ No agent runs found
+ </TableMessageRow>
+ ) : (
+ workflowRuns?.map((workflowRun) => {
+ const workflowRunId =
+ workflowRun.script_run === true ? (
+ <div className="flex items-center gap-2">
+ <Tip content="Ran with code">
+ <LightningBoltIcon className="text-[gold]" />
+ </Tip>
+ <span>{workflowRun.workflow_run_id ?? ""}</span>
+ </div>
+ ) : (
+ (workflowRun.workflow_run_id ?? "")
+ );
+
+ const isExpanded = expandedRows.has(
+ workflowRun.workflow_run_id,
+ );
+
+ return (
+ <React.Fragment key={workflowRun.workflow_run_id}>
+ {/* Main run row */}
+ <TableRow
+ onClick={(event) => {
+ const url = env.useNewRunsUrl
+ ? `/runs/${workflowRun.workflow_run_id}`
+ : `/workflows/${workflowPermanentId}/${workflowRun.workflow_run_id}/overview`;
+
+ if (event.ctrlKey || event.metaKey) {
+ window.open(
+ window.location.origin + url,
+ "_blank",
+ "noopener,noreferrer",
+ );
+ return;
+ }
+ navigate(url);
+ }}
+ className="cursor-pointer"
+ >
+ <TableCell className="font-mono text-xs text-muted-foreground">
+ {workflowRunId}
+ </TableCell>
+ <TableCell>
+ <StatusBadge status={workflowRun.status} />
+ </TableCell>
+ <TableCell
+ className="text-muted-foreground"
+ title={basicTimeFormat(workflowRun.created_at)}
+ >
+ {compactLocalDateTime(workflowRun.created_at)}
+ </TableCell>
+ <TableCell className="tabular-nums text-muted-foreground">
+ {formatExecutionTime(
+ workflowRun.started_at ??
+ workflowRun.created_at,
+ workflowRun.finished_at,
+ ) ?? "-"}
+ </TableCell>
+ <TableCell>
+ <div className="flex justify-end gap-2">
+ <TooltipProvider>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <Button
+ size="icon"
+ variant="ghost"
+ onClick={(event) => {
+ event.stopPropagation();
+ toggleParametersExpanded(
+ workflowRun.workflow_run_id,
+ );
+ }}
+ className={cn(
+ isExpanded
+ ? "text-blue-400"
+ : "text-muted-foreground hover:text-foreground",
+ )}
+ >
+ <MixerHorizontalIcon className="h-4 w-4" />
+ </Button>
+ </TooltipTrigger>
+ <TooltipContent>
+ {isExpanded
+ ? "Hide Inputs"
+ : "Show Inputs"}
+ </TooltipContent>
+ </Tooltip>
+ </TooltipProvider>
+ </div>
+ </TableCell>
+ </TableRow>
+
+ {/* Expanded parameters section */}
+ {isExpanded && (
+ <TableRow
+ key={`${workflowRun.workflow_run_id}-params`}
+ >
+ <TableCell
+ colSpan={5}
+ className="bg-slate-50 dark:bg-slate-900/50"
+ >
+ <WorkflowRunParameters
+ workflowPermanentId={workflowPermanentId}
+ workflowRunId={workflowRun.workflow_run_id}
+ workflow={workflow}
+ searchQuery={debouncedSearch}
+ keywordMatchesParameter={matchesParameter}
+ />
+ </TableCell>
+ </TableRow>
+ )}
+ </React.Fragment>
+ );
+ })
+ )}
+ </TableBody>
+ </Table>
+ <RunParametersDialog
+ open={openRunParams !== null}
+ onOpenChange={(open) => {
+ if (!open) setOpenRunParams(null);
+ }}
+ workflowPermanentId={workflowPermanentId}
+ workflowRunId={openRunParams}
+ />
+ <div className="relative px-3 py-3">
+ <div className="absolute left-3 top-1/2 flex -translate-y-1/2 items-center gap-2 text-sm">
+ <span className="text-slate-400">Items per page</span>
+ <Select
+ value={String(pageSize)}
+ onValueChange={(size) => {
const params = new URLSearchParams(searchParams);
- params.set("page", String(page + 1));
+ params.set("page_size", size);
+ params.set("page", "1");
setSearchParams(params, { replace: true });
}}
- />
- </PaginationItem>
- </PaginationContent>
- </Pagination>
- </div>
- </div>
+ >
+ <SelectTrigger className="w-[65px]">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {PAGE_SIZE_OPTIONS.map((size) => (
+ <SelectItem key={size} value={size}>
+ {size}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ <Pagination className="pt-0">
+ <PaginationContent>
+ <PaginationItem>
+ <PaginationPrevious
+ className={cn({ "cursor-not-allowed": page === 1 })}
+ onClick={() => {
+ if (page === 1) {
+ return;
+ }
+ const params = new URLSearchParams(searchParams);
+ params.set("page", String(Math.max(1, page - 1)));
+ setSearchParams(params, { replace: true });
+ }}
+ />
+ </PaginationItem>
+ <PaginationItem>
+ <PaginationLink>{page}</PaginationLink>
+ </PaginationItem>
+ <PaginationItem>
+ <PaginationNext
+ className={cn({
+ "cursor-not-allowed":
+ workflowRuns !== undefined &&
+ workflowRuns.length < pageSize,
+ })}
+ onClick={() => {
+ if (workflowRuns && workflowRuns.length < pageSize) {
+ return;
+ }
+ const params = new URLSearchParams(searchParams);
+ params.set("page", String(page + 1));
+ setSearchParams(params, { replace: true });
+ }}
+ />
+ </PaginationItem>
+ </PaginationContent>
+ </Pagination>
+ </div>
+ </div>
+ </>
+ )}
</div>
</div>
);
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
index 60a219222..46ad174f4 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
@@ -34,9 +34,14 @@ import {
ReloadIcon,
} from "@radix-ui/react-icons";
import { useMutation, useQueryClient } from "@tanstack/react-query";
-import { Link, Outlet, useSearchParams } from "react-router-dom";
-import { statusIsCancellable, statusIsFinalized } from "../tasks/types";
+import { Link, Outlet, useNavigate, useSearchParams } from "react-router-dom";
+import {
+ statusIsAFailureType,
+ statusIsCancellable,
+ statusIsFinalized,
+} from "../tasks/types";
import { useWorkflowRunWithWorkflowQuery } from "./hooks/useWorkflowRunWithWorkflowQuery";
+import { useRefreshOnboardingOnRunCompletion } from "./hooks/useRefreshOnboardingOnRunCompletion";
import { WorkflowRunBlockDetail } from "./workflowRun/WorkflowRunBlockDetail";
import { WorkflowRunTimeline } from "./workflowRun/WorkflowRunTimeline";
import { useWorkflowRunTimelineQuery } from "./hooks/useWorkflowRunTimelineQuery";
@@ -61,6 +66,13 @@ import { WorkflowRunStatusAlert } from "@/routes/workflows/workflowRun/WorkflowR
import { WorkflowRunVerificationCodeForm } from "@/routes/workflows/workflowRun/WorkflowRunVerificationCodeForm";
import { ScriptUpdateCard } from "@/routes/workflows/workflowRun/ScriptUpdateCard";
import { useFallbackEpisodesQuery } from "@/routes/workflows/hooks/useFallbackEpisodesQuery";
+import { WebhookDeliveryStatus } from "@/routes/workflows/workflowRun/WebhookDeliveryStatus";
+import { useRunsQuery } from "@/hooks/useRunsQuery";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
+import { FirstRunRecoveryGuidance } from "@/components/onboarding/FirstRunRecoveryGuidance";
+import { useFeatureFlagVariantKey } from "posthog-js/react";
+import { EXPERIMENT } from "@/util/onboarding/experimentConfig";
+import { isFirstFailedRunRecoveryEligible } from "@/util/onboarding/rolloutGating";
function WorkflowRunRightColumn({
activeItem,
@@ -135,6 +147,8 @@ function WorkflowRun() {
const credentialGetter = useCredentialGetter();
const apiCredential = useApiCredential();
const queryClient = useQueryClient();
+ const navigate = useNavigate();
+ const onboarding = useOnboardingStateOptional();
const {
data: workflowRun,
@@ -143,6 +157,8 @@ function WorkflowRun() {
error,
} = useWorkflowRunWithWorkflowQuery();
+ useRefreshOnboardingOnRunCompletion(workflowRun);
+
const status = (error as AxiosError | undefined)?.response?.status;
const workflow = workflowRun?.workflow;
const workflowPermanentId =
@@ -301,6 +317,52 @@ function WorkflowRun() {
finallyBlockLabel &&
finallyBlockInTimeline;
+ // Gate the first-failed-run guidance. first_run_at is stamped on any first
+ // terminal run, so derive "first run" from the runs list and assert this run
+ // is that single run.
+ const isFailureRun = workflowRun
+ ? statusIsAFailureType({ status: workflowRun.status })
+ : false;
+ const onboardingFlagVariant = useFeatureFlagVariantKey(EXPERIMENT.flagKey);
+ // Gate on the rollout arm so a 0% rollout / rollback hides the recovery surface.
+ const firstFailedRunGateEnabled = isFirstFailedRunRecoveryEligible({
+ flagVariant: onboardingFlagVariant,
+ isNewUser: onboarding?.isNewUser === true,
+ isFailureRun,
+ hasFailureReason: Boolean(workflowRun?.failure_reason),
+ });
+ const { data: recentRuns } = useRunsQuery({
+ page: 1,
+ pageSize: 2,
+ enabled: firstFailedRunGateEnabled,
+ });
+ const showFirstFailedRunRecovery =
+ firstFailedRunGateEnabled &&
+ recentRuns?.length === 1 &&
+ recentRuns[0]?.run_id === workflowRun?.workflow_run_id;
+
+ const handleFirstFailedRunRetry = useCallback(() => {
+ navigate(`/workflows/${workflowPermanentId}/run`, {
+ state: {
+ data: workflowRun?.parameters ?? {},
+ proxyLocation,
+ webhookCallbackUrl: workflowRun?.webhook_callback_url ?? "",
+ maxScreenshotScrolls,
+ runWith: workflowRun?.run_with ?? "agent",
+ browserProfileId: workflowRun?.browser_profile_id ?? null,
+ },
+ });
+ }, [
+ navigate,
+ workflowPermanentId,
+ proxyLocation,
+ maxScreenshotScrolls,
+ workflowRun?.parameters,
+ workflowRun?.webhook_callback_url,
+ workflowRun?.run_with,
+ workflowRun?.browser_profile_id,
+ ]);
+
const workflowFailureReason = workflowRun?.failure_reason ? (
<div className="space-y-2 rounded-md border border-red-600 bg-error-light p-4">
<div className="flex items-center gap-2">
@@ -309,6 +371,14 @@ function WorkflowRun() {
</div>
<div className="text-sm">{workflowRun.failure_reason}</div>
{matchedTips}
+ {showFirstFailedRunRecovery && (
+ <FirstRunRecoveryGuidance
+ surface="runs"
+ failureCategory={workflowRun.failure_category?.[0]?.category ?? null}
+ workflowPermanentId={workflowPermanentId}
+ onRetry={handleFirstFailedRunRetry}
+ />
+ )}
{shouldShowFinallyNote && (
<div className="mt-2 flex items-center gap-2 rounded bg-amber-500/20 px-3 py-2 text-sm text-amber-200">
<span className="font-medium">Note:</span>
@@ -360,18 +430,25 @@ function WorkflowRun() {
const isTaskv2Run = workflowRun && workflowRun.task_v2 !== null;
+ const taskWebhookFailureReason =
+ workflowRun?.task_v2?.webhook_failure_reason ?? null;
+ const workflowWebhookFailureReason =
+ workflowRun?.webhook_failure_reason ?? null;
const webhookFailureReasonData =
- workflowRun?.task_v2?.webhook_failure_reason ??
- workflowRun?.webhook_failure_reason;
-
- const webhookFailureReason = webhookFailureReasonData ? (
- <div className="space-y-4">
- <Label>Webhook Failure Reason</Label>
- <div className="rounded-md border border-yellow-600 p-4 text-sm">
- {webhookFailureReasonData}
- </div>
- </div>
- ) : null;
+ taskWebhookFailureReason ?? workflowWebhookFailureReason;
+ // TaskV2 webhook state is legacy failure-only data; workflow-level pending
+ // delivery status is surfaced separately by webhook_delivery_status.
+ const webhookDeliveryStatusData = taskWebhookFailureReason
+ ? "failed"
+ : (workflowRun?.webhook_delivery_status ??
+ (workflowWebhookFailureReason ? "failed" : null));
+
+ const webhookDeliveryStatus = (
+ <WebhookDeliveryStatus
+ webhookDeliveryStatus={webhookDeliveryStatusData}
+ webhookFailureReason={webhookFailureReasonData}
+ />
+ );
const outputs = workflowRun?.outputs;
const extractedInformation =
@@ -674,9 +751,9 @@ function WorkflowRun() {
</ScrollArea>
</div>
)}
- {webhookFailureReason}
</div>
)}
+ {webhookDeliveryStatus}
{workflowFailureReason}
{fallbackEpisodes && fallbackEpisodes.episodes.length > 0 && (
<ScriptUpdateCard
diff --git a/skyvern-frontend/src/routes/workflows/Workflows.tsx b/skyvern-frontend/src/routes/workflows/Workflows.tsx
index db6233b0f..62d2a6d79 100644
--- a/skyvern-frontend/src/routes/workflows/Workflows.tsx
+++ b/skyvern-frontend/src/routes/workflows/Workflows.tsx
@@ -1,5 +1,22 @@
import { getClient } from "@/api/AxiosClient";
+import { GetStartedModal } from "@/components/onboarding/GetStartedModal";
+import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";
+import { OnboardingErrorBoundary } from "@/components/onboarding/OnboardingErrorBoundary";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
import { Button } from "@/components/ui/button";
+import {
+ SelectionCheckboxCell,
+ SelectionHeaderCheckboxCell,
+} from "@/components/SelectionCheckbox";
+import { useRowSelection } from "@/hooks/useRowSelection";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
import {
Pagination,
PaginationContent,
@@ -25,6 +42,11 @@ import {
} from "@/components/ui/tooltip";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { basicTimeFormat, compactLocalDateTime } from "@/util/timeFormat";
+import {
+ BULK_CONCURRENCY_LIMIT,
+ runWithConcurrency,
+} from "@/util/runWithConcurrency";
+import { bulkResultToast } from "@/util/bulkResultToast";
import { cn } from "@/util/utils";
import {
BookmarkFilledIcon,
@@ -37,8 +59,14 @@ import {
ReloadIcon,
} from "@radix-ui/react-icons";
import { FolderIcon } from "@/components/icons/FolderIcon";
-import { useQuery } from "@tanstack/react-query";
-import React, { useCallback, useEffect, useMemo, useState } from "react";
+import { useQuery, useQueryClient } from "@tanstack/react-query";
+import React, {
+ useCallback,
+ useEffect,
+ useMemo,
+ useRef,
+ useState,
+} from "react";
import { Link, useNavigate, useSearchParams } from "react-router-dom";
import { useDebounce } from "use-debounce";
import {
@@ -48,6 +76,7 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { NarrativeCard } from "./components/header/NarrativeCard";
+import { BulkActionBar } from "./components/BulkActionBar";
import { FolderCard } from "./components/FolderCard";
import { CreateFolderDialog } from "./components/CreateFolderDialog";
import { CreateFromTemplateDialog } from "./components/CreateFromTemplateDialog";
@@ -68,6 +97,7 @@ import {
type TagFilterTerm,
} from "./types/tagTypes";
import { ImportWorkflowButton } from "./ImportWorkflowButton";
+import { useNodeCollapseStore } from "./editor/collapse/useNodeCollapseStore";
import { convert } from "./editor/workflowEditorUtils";
import { WorkflowApiResponse } from "./types/workflowTypes";
import { WorkflowActions } from "./WorkflowActions";
@@ -91,6 +121,8 @@ function slugifyFolderName(name: string): string {
.replace(/^-|-$/g, ""); // Remove leading/trailing hyphens
}
+const FOLDERS_PAGE_SIZE = 10;
+
// Generate a unique slug for a folder, appending a number suffix only if there's a collision
function getUniqueSlugForFolder(folder: Folder, allFolders: Folder[]): string {
const baseSlug = slugifyFolderName(folder.title);
@@ -120,11 +152,17 @@ function getUniqueSlugForFolder(folder: Folder, allFolders: Folder[]): string {
}
function Workflows() {
const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
const navigate = useNavigate();
const createWorkflowMutation = useCreateWorkflowMutation();
const [searchParams, setSearchParams] = useSearchParams();
const [search, setSearch] = useState("");
const [debouncedSearch] = useDebounce(search, 250);
+ const [isBulkOperating, setIsBulkOperating] = useState(false);
+ const [bulkDeleteDialog, setBulkDeleteDialog] = useState<{
+ open: boolean;
+ targets: string[];
+ }>({ open: false, targets: [] });
const page = searchParams.get("page") ? Number(searchParams.get("page")) : 1;
const itemsPerPage = searchParams.get("page_size")
? Number(searchParams.get("page_size"))
@@ -171,7 +209,7 @@ function Workflows() {
// Fetch folders
const { data: allFolders = [], isLoading: isFoldersLoading } =
- useFoldersQuery({ page_size: 10 });
+ useFoldersQuery({ page_size: FOLDERS_PAGE_SIZE });
// Create a memoized map of slugs to folders to avoid O(n²) lookups
const slugToFolderMap = useMemo(() => {
@@ -185,11 +223,26 @@ function Workflows() {
// Folder state - derived from URL query param
// Look up the folder by matching the slug (handles collision suffixes like my-folder-2)
+ const resolvedFolderIdBySlug = useRef<Map<string, string>>(new Map());
+ // A short list means we saw every folder, so a lookup miss is a real
+ // deletion/rename rather than pagination dropping the active folder.
+ const foldersListComplete =
+ !isFoldersLoading && allFolders.length < FOLDERS_PAGE_SIZE;
const selectedFolderId = useMemo(() => {
- if (!folderSlug || allFolders.length === 0) return null;
+ if (!folderSlug) return null;
const matchingFolder = slugToFolderMap.get(folderSlug);
- return matchingFolder?.folder_id ?? null;
- }, [folderSlug, allFolders.length, slugToFolderMap]);
+ if (matchingFolder) {
+ resolvedFolderIdBySlug.current.set(folderSlug, matchingFolder.folder_id);
+ return matchingFolder.folder_id;
+ }
+ if (foldersListComplete) {
+ resolvedFolderIdBySlug.current.delete(folderSlug);
+ return null;
+ }
+ // The folders list is paginated, so a refetch can transiently drop the
+ // active folder; reuse the last resolution instead of flapping to null.
+ return resolvedFolderIdBySlug.current.get(folderSlug) ?? null;
+ }, [folderSlug, slugToFolderMap, foldersListComplete]);
// Clear folder param if the folder slug is invalid (folder deleted/renamed)
// Only validate after folders have finished loading to avoid race conditions
@@ -284,6 +337,24 @@ function Workflows() {
placeholderData: (previousData) => previousData,
});
+ // unfiltered "owns any workflow" check; the filtered/paginated list above can read empty for a user who has workflows
+ const { data: ownedWorkflows = [], isLoading: ownedWorkflowsLoading } =
+ useQuery<Array<WorkflowApiResponse>>({
+ queryKey: ["workflows", "exists"],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ const params = new URLSearchParams();
+ params.append("page", "1");
+ params.append("page_size", "1");
+ params.append("only_workflows", "true");
+ return client
+ .get(`/workflows`, { params })
+ .then((response) => response.data);
+ },
+ });
+
+ const onboarding = useOnboardingStateOptional();
+
const { data: nextPageWorkflows } = useQuery<Array<WorkflowApiResponse>>({
queryKey: [
"workflows",
@@ -468,6 +539,143 @@ function Workflows() {
return workflows;
}, [activeImports, workflows, page]);
+ const selectableWorkflows = useMemo(
+ () =>
+ displayWorkflows.filter((workflow) => workflow.status !== "importing"),
+ [displayWorkflows],
+ );
+
+ const showCheckbox = selectableWorkflows.length > 0;
+ const columnCount = showCheckbox ? 6 : 5;
+
+ const {
+ selected,
+ selectedItems: selectedWorkflows,
+ isSelected,
+ allSelected,
+ someSelected,
+ indexById: selectableIndexById,
+ handleSelect,
+ toggleSelectAll,
+ clearSelection,
+ replaceSelection,
+ } = useRowSelection({
+ items: selectableWorkflows,
+ getId: (workflow) => workflow.workflow_permanent_id,
+ // Keyed on the folder SLUG (user navigation), not the derived id, so folder
+ // list refetches can never wipe an in-progress selection.
+ resetKey: JSON.stringify([page, folderSlug, debouncedSearch]),
+ // Page-size changes keep the selection; only the shift anchor resets.
+ anchorResetKey: itemsPerPage,
+ });
+
+ async function handleBulkMoveToFolder(folderId: string | null) {
+ if (selectedWorkflows.length === 0) {
+ return;
+ }
+
+ setIsBulkOperating(true);
+ try {
+ const client = await getClient(credentialGetter);
+ const results = await runWithConcurrency(
+ selectedWorkflows.map(
+ (workflow) => () =>
+ client.put(`/workflows/${workflow.workflow_permanent_id}/folder`, {
+ folder_id: folderId,
+ }),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const succeeded = results.filter((r) => r.status === "fulfilled").length;
+ bulkResultToast({
+ succeeded,
+ total: selectedWorkflows.length,
+ results,
+ successTitle: (count) =>
+ folderId
+ ? `Moved ${count} agent${count !== 1 ? "s" : ""} to folder.`
+ : `Removed ${count} agent${count !== 1 ? "s" : ""} from folder.`,
+ failureTitle: (count) =>
+ folderId
+ ? `Failed to move ${count} agent${count !== 1 ? "s" : ""} to folder.`
+ : `Failed to remove ${count} agent${count !== 1 ? "s" : ""} from folder.`,
+ partialTitle: (successCount, failedCount) =>
+ folderId
+ ? `Moved ${successCount} agent${successCount !== 1 ? "s" : ""} to folder. ${failedCount} failed.`
+ : `Removed ${successCount} agent${successCount !== 1 ? "s" : ""} from folder. ${failedCount} failed.`,
+ });
+ if (succeeded === selectedWorkflows.length) {
+ clearSelection();
+ }
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["workflows"] });
+ queryClient.invalidateQueries({ queryKey: ["folders"] });
+ }
+ } finally {
+ setIsBulkOperating(false);
+ }
+ }
+
+ async function handleBulkDeleteConfirm() {
+ // Snapshot from dialog-open time; the live selection may have changed since.
+ const targets = bulkDeleteDialog.targets;
+ if (targets.length === 0) {
+ return;
+ }
+
+ setIsBulkOperating(true);
+ try {
+ const client = await getClient(credentialGetter);
+ const results = await runWithConcurrency(
+ targets.map(
+ (workflowId) => () => client.delete(`/workflows/${workflowId}`),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+
+ const succeededIds: string[] = [];
+ const failedIds = new Set<string>();
+ results.forEach((result, index) => {
+ const workflowId = targets[index]!;
+ if (result.status === "fulfilled") {
+ succeededIds.push(workflowId);
+ useNodeCollapseStore.getState().pruneWorkflow(workflowId);
+ } else {
+ failedIds.add(workflowId);
+ }
+ });
+
+ const succeeded = succeededIds.length;
+ const failed = failedIds.size;
+
+ bulkResultToast({
+ succeeded,
+ total: targets.length,
+ results,
+ successTitle: (count) =>
+ `${count} agent${count !== 1 ? "s" : ""} deleted successfully.`,
+ failureTitle: (count) =>
+ `Failed to delete ${count} agent${count !== 1 ? "s" : ""}.`,
+ partialTitle: (successCount, failedCount) =>
+ `Deleted ${successCount} agent${successCount !== 1 ? "s" : ""}. ${failedCount} failed.`,
+ });
+ // Deletions shift row indices; replace/clear also resets the shift anchor.
+ if (failed === 0) {
+ clearSelection();
+ } else {
+ replaceSelection(failedIds);
+ }
+
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["workflows"] });
+ queryClient.invalidateQueries({ queryKey: ["folders"] });
+ }
+ } finally {
+ setIsBulkOperating(false);
+ setBulkDeleteDialog({ open: false, targets: [] });
+ }
+ }
+
return (
<div className="space-y-10">
<div className="flex h-32 justify-between gap-6">
@@ -648,9 +856,23 @@ function Workflows() {
<div className="overflow-hidden rounded-lg border border-border">
<Table className="table-fixed">
<TableHeader>
- <TableRow>
- <TableHead className="w-[25%]">ID</TableHead>
- <TableHead className="w-[30%]">Title</TableHead>
+ <TableRow className="group/header">
+ {showCheckbox && (
+ <SelectionHeaderCheckboxCell
+ className="w-[3%]"
+ allSelected={allSelected}
+ someSelected={someSelected}
+ hasSelection={selected.size > 0}
+ onToggleAll={toggleSelectAll}
+ ariaLabel="Select all agents"
+ />
+ )}
+ <TableHead className={showCheckbox ? "w-[22%]" : "w-[25%]"}>
+ ID
+ </TableHead>
+ <TableHead className={showCheckbox ? "w-[27%]" : "w-[30%]"}>
+ Title
+ </TableHead>
<TableHead className="w-[15%]">Folder</TableHead>
<TableHead className="w-[15%]">Created At</TableHead>
<TableHead className="w-[15%] text-right">Actions</TableHead>
@@ -663,6 +885,11 @@ function Workflows() {
// Show skeleton rows only on initial load (not during search refinement)
Array.from({ length: 10 }).map((_, index) => (
<TableRow key={`skeleton-${index}`}>
+ {showCheckbox && (
+ <TableCell>
+ <Skeleton className="h-4 w-4" />
+ </TableCell>
+ )}
<TableCell>
<Skeleton className="h-5 w-full" />
</TableCell>
@@ -686,7 +913,9 @@ function Workflows() {
</TableRow>
))
) : displayWorkflows?.length === 0 ? (
- <TableMessageRow colSpan={5}>No agents found</TableMessageRow>
+ <TableMessageRow colSpan={columnCount}>
+ No agents found
+ </TableMessageRow>
) : (
displayWorkflows?.map((workflow) => {
const parameterItems = workflow.workflow_definition.parameters
@@ -703,16 +932,25 @@ function Workflows() {
const isExpanded = expandedRows.has(
workflow.workflow_permanent_id,
);
+ const isRowSelected = isSelected(
+ workflow.workflow_permanent_id,
+ );
// Check if this is an importing agent
const isUploading = workflow.status === "importing";
const workflowTags =
workflowTagsMap[workflow.workflow_permanent_id];
+ const selectableIndex = isUploading
+ ? -1
+ : (selectableIndexById.get(
+ workflow.workflow_permanent_id,
+ ) ?? -1);
return (
<React.Fragment key={workflow.workflow_permanent_id}>
{/* Main workflow row */}
{isUploading ? (
<TableRow className="opacity-70">
+ {showCheckbox && <TableCell />}
<TableCell colSpan={2}>
<div className="flex min-w-0 items-center gap-2">
<ReloadIcon className="h-4 w-4 shrink-0 animate-spin text-blue-400" />
@@ -729,23 +967,40 @@ function Workflows() {
</TableCell>
<TableCell>
<div className="flex justify-end gap-0.5">
- <Button size="icon" variant="ghost" disabled>
- <FolderIcon className="h-4 w-4" />
- </Button>
+ {selected.size === 0 && (
+ <Button size="icon" variant="ghost" disabled>
+ <FolderIcon className="h-4 w-4" />
+ </Button>
+ )}
<Button size="icon" variant="ghost" disabled>
<Pencil2Icon className="h-4 w-4" />
</Button>
<Button size="icon" variant="ghost" disabled>
<PlayIcon className="h-4 w-4" />
</Button>
- <Button size="icon" variant="ghost" disabled>
- <DotsHorizontalIcon className="h-4 w-4" />
- </Button>
+ {selected.size === 0 && (
+ <Button size="icon" variant="ghost" disabled>
+ <DotsHorizontalIcon className="h-4 w-4" />
+ </Button>
+ )}
</div>
</TableCell>
</TableRow>
) : (
- <TableRow className="cursor-pointer">
+ <TableRow
+ className="group/row cursor-pointer select-none"
+ data-state={isRowSelected ? "selected" : undefined}
+ >
+ {showCheckbox && (
+ <SelectionCheckboxCell
+ className="select-none"
+ index={selectableIndex}
+ checked={isRowSelected}
+ hasSelection={selected.size > 0}
+ onSelect={handleSelect}
+ ariaLabel={`Select ${workflow.title}`}
+ />
+ )}
<TableCell
onClick={(event) => {
handleRowClick(
@@ -847,32 +1102,40 @@ function Workflows() {
</TableCell>
<TableCell>
<div className="flex justify-end gap-0.5">
- <TooltipProvider>
- <Tooltip>
- <TooltipTrigger asChild>
- <div>
- <WorkflowFolderSelector
- workflowPermanentId={
- workflow.workflow_permanent_id
- }
- currentFolderId={workflow.folder_id}
- />
- </div>
- </TooltipTrigger>
- <TooltipContent>
- Assign to Folder
- </TooltipContent>
- </Tooltip>
- </TooltipProvider>
- <WorkflowTagEditor
- workflowPermanentId={
- workflow.workflow_permanent_id
- }
- tags={workflowTags ?? []}
- tagKeys={tagKeys}
- labelSuggestions={labelSuggestions}
- valueSuggestionsByKey={valueSuggestionsByKey}
- />
+ {/* Folder, tags, and the row menu yield to the bulk bar while any selection is active; Edit and Run stay. */}
+ {selected.size === 0 && (
+ <>
+ <TooltipProvider>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <div>
+ <WorkflowFolderSelector
+ workflowPermanentId={
+ workflow.workflow_permanent_id
+ }
+ currentFolderId={workflow.folder_id}
+ disabled={isBulkOperating}
+ />
+ </div>
+ </TooltipTrigger>
+ <TooltipContent>
+ Assign to Folder
+ </TooltipContent>
+ </Tooltip>
+ </TooltipProvider>
+ <WorkflowTagEditor
+ workflowPermanentId={
+ workflow.workflow_permanent_id
+ }
+ tags={workflowTags ?? []}
+ tagKeys={tagKeys}
+ labelSuggestions={labelSuggestions}
+ valueSuggestionsByKey={
+ valueSuggestionsByKey
+ }
+ />
+ </>
+ )}
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
@@ -917,16 +1180,18 @@ function Workflows() {
</TooltipContent>
</Tooltip>
</TooltipProvider>
- <WorkflowActions
- workflow={workflow}
- hasParameters={hasParameters}
- parametersExpanded={isExpanded}
- onToggleParameters={() =>
- toggleParametersExpanded(
- workflow.workflow_permanent_id,
- )
- }
- />
+ {selected.size === 0 && (
+ <WorkflowActions
+ workflow={workflow}
+ hasParameters={hasParameters}
+ parametersExpanded={isExpanded}
+ onToggleParameters={() =>
+ toggleParametersExpanded(
+ workflow.workflow_permanent_id,
+ )
+ }
+ />
+ )}
</div>
</TableCell>
</TableRow>
@@ -938,7 +1203,7 @@ function Workflows() {
key={`${workflow.workflow_permanent_id}-params`}
>
<TableCell
- colSpan={5}
+ colSpan={columnCount}
className="bg-slate-50 dark:bg-slate-900/50"
>
<ParameterDisplayInline
@@ -1001,6 +1266,71 @@ function Workflows() {
</div>
</div>
+ {selectedWorkflows.length > 0 && (
+ <BulkActionBar
+ selectedWorkflows={selectedWorkflows}
+ isOperating={isBulkOperating}
+ onOperatingChange={setIsBulkOperating}
+ onClearSelection={clearSelection}
+ onDeleteRequest={() =>
+ setBulkDeleteDialog({
+ open: true,
+ targets: selectedWorkflows.map(
+ (workflow) => workflow.workflow_permanent_id,
+ ),
+ })
+ }
+ onMoveToFolder={handleBulkMoveToFolder}
+ tagKeys={tagKeys}
+ labelSuggestions={labelSuggestions}
+ valueSuggestionsByKey={valueSuggestionsByKey}
+ />
+ )}
+
+ <Dialog
+ open={bulkDeleteDialog.open}
+ onOpenChange={(open) => {
+ if (!open && !isBulkOperating) {
+ setBulkDeleteDialog({ open: false, targets: [] });
+ }
+ }}
+ >
+ <DialogContent>
+ <DialogHeader>
+ <DialogTitle>
+ Delete {bulkDeleteDialog.targets.length} Agent
+ {bulkDeleteDialog.targets.length === 1 ? "" : "s"}
+ </DialogTitle>
+ <DialogDescription>
+ Are you sure you want to delete{" "}
+ {bulkDeleteDialog.targets.length}{" "}
+ {bulkDeleteDialog.targets.length === 1 ? "agent" : "agents"}?
+ This action cannot be undone.
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <Button
+ variant="secondary"
+ disabled={isBulkOperating}
+ onClick={() =>
+ setBulkDeleteDialog({ open: false, targets: [] })
+ }
+ >
+ Cancel
+ </Button>
+ <Button
+ variant="destructive"
+ disabled={isBulkOperating}
+ onClick={() => {
+ void handleBulkDeleteConfirm();
+ }}
+ >
+ {isBulkOperating ? "Deleting..." : "Delete"}
+ </Button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+
{/* Folder Dialogs */}
<CreateFolderDialog
open={isCreateFolderOpen}
@@ -1029,7 +1359,20 @@ function Workflows() {
}}
/>
- <WorkflowTemplates />
+ <div data-hint="start-template">
+ <WorkflowTemplates />
+ </div>
+
+ {onboarding ? (
+ <OnboardingErrorBoundary
+ onError={() => OnboardingTelemetry.modalRenderError("dashboard")}
+ >
+ <GetStartedModal
+ hasWorkflows={ownedWorkflows.length > 0}
+ isLoading={ownedWorkflowsLoading}
+ />
+ </OnboardingErrorBoundary>
+ ) : null}
</div>
</div>
);
diff --git a/skyvern-frontend/src/routes/workflows/components/BulkActionBar.tsx b/skyvern-frontend/src/routes/workflows/components/BulkActionBar.tsx
new file mode 100644
index 000000000..3035ea49e
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/components/BulkActionBar.tsx
@@ -0,0 +1,340 @@
+import { getClient } from "@/api/AxiosClient";
+import { FolderIcon } from "@/components/icons/FolderIcon";
+import { GarbageIcon } from "@/components/icons/GarbageIcon";
+import { SelectionBar, SelectionBarDivider } from "@/components/SelectionBar";
+import { Button } from "@/components/ui/button";
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuPortal,
+ DropdownMenuSub,
+ DropdownMenuSubContent,
+ DropdownMenuSubTrigger,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { bulkResultToast } from "@/util/bulkResultToast";
+import {
+ BULK_CONCURRENCY_LIMIT,
+ runWithConcurrency,
+} from "@/util/runWithConcurrency";
+import {
+ BookmarkFilledIcon,
+ BookmarkIcon,
+ CopyIcon,
+ DotsHorizontalIcon,
+ DownloadIcon,
+} from "@radix-ui/react-icons";
+import { useQueryClient } from "@tanstack/react-query";
+import { stringify as convertToYAML } from "yaml";
+import { convert } from "../editor/workflowEditorUtils";
+import { Tag, TagKey } from "../types/tagTypes";
+import { WorkflowApiResponse } from "../types/workflowTypes";
+import { BulkTagPicker } from "./tagging/BulkTagPicker";
+import { WorkflowFolderSelector } from "./WorkflowFolderSelector";
+
+type Props = {
+ selectedWorkflows: WorkflowApiResponse[];
+ isOperating: boolean;
+ onOperatingChange: (operating: boolean) => void;
+ onClearSelection: () => void;
+ onDeleteRequest: () => void;
+ onMoveToFolder: (folderId: string | null) => Promise<void>;
+ tagKeys: Array<TagKey>;
+ labelSuggestions: Array<string>;
+ valueSuggestionsByKey?: Map<string, Array<string>>;
+};
+
+// Blob URLs avoid the ~2MB data-URI cap that truncates large workflow exports.
+function downloadFile(fileName: string, contents: string) {
+ const blob = new Blob([contents], { type: "text/plain;charset=utf-8" });
+ const url = URL.createObjectURL(blob);
+ const element = document.createElement("a");
+ element.href = url;
+ element.download = fileName;
+ element.style.display = "none";
+ document.body.appendChild(element);
+ element.click();
+ document.body.removeChild(element);
+ // Deferred so a still-starting download cannot lose its blob.
+ setTimeout(() => URL.revokeObjectURL(url), 60_000);
+}
+
+function BulkActionBar({
+ selectedWorkflows,
+ isOperating,
+ onOperatingChange,
+ onClearSelection,
+ onDeleteRequest,
+ onMoveToFolder,
+ tagKeys,
+ labelSuggestions,
+ valueSuggestionsByKey,
+}: Props) {
+ const credentialGetter = useCredentialGetter();
+ const queryClient = useQueryClient();
+ const count = selectedWorkflows.length;
+ const nonTemplates = selectedWorkflows.filter(
+ (workflow) => !workflow.is_template,
+ );
+ const templates = selectedWorkflows.filter(
+ (workflow) => workflow.is_template,
+ );
+
+ async function handleBulkClone() {
+ onOperatingChange(true);
+ try {
+ const client = await getClient(credentialGetter);
+ const results = await runWithConcurrency(
+ selectedWorkflows.map((workflow) => () => {
+ const yaml = convertToYAML(
+ convert({
+ ...workflow,
+ title: `Copy of ${workflow.title}`,
+ }),
+ );
+ return client.post("/workflows", yaml, {
+ headers: { "Content-Type": "text/plain" },
+ });
+ }),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const succeeded = results.filter((r) => r.status === "fulfilled").length;
+ bulkResultToast({
+ succeeded,
+ total: count,
+ results,
+ successTitle: (n) => `Cloned ${n} agent${n !== 1 ? "s" : ""}.`,
+ failureTitle: (n) => `Failed to clone ${n} agent${n !== 1 ? "s" : ""}.`,
+ partialTitle: (successCount, failedCount) =>
+ `Cloned ${successCount} agent${successCount !== 1 ? "s" : ""}. ${failedCount} failed.`,
+ });
+ if (succeeded === count) {
+ onClearSelection();
+ }
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["workflows"] });
+ queryClient.invalidateQueries({ queryKey: ["folders"] });
+ }
+ } finally {
+ onOperatingChange(false);
+ }
+ }
+
+ async function handleBulkTemplateUpdate(
+ workflows: WorkflowApiResponse[],
+ isTemplate: boolean,
+ ) {
+ if (workflows.length === 0) {
+ return;
+ }
+ onOperatingChange(true);
+ try {
+ // Template endpoint only exists on /v1 (no /api prefix)
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ const results = await runWithConcurrency(
+ workflows.map(
+ (workflow) => () =>
+ client.put(
+ `/workflows/${workflow.workflow_permanent_id}/template?is_template=${isTemplate}`,
+ ),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const succeeded = results.filter((r) => r.status === "fulfilled").length;
+ bulkResultToast({
+ succeeded,
+ total: workflows.length,
+ results,
+ successTitle: (n) =>
+ isTemplate
+ ? `Saved ${n} agent${n !== 1 ? "s" : ""} as templates.`
+ : `Removed ${n} agent${n !== 1 ? "s" : ""} from templates.`,
+ failureTitle: (n) =>
+ isTemplate
+ ? `Failed to save ${n} agent${n !== 1 ? "s" : ""} as templates.`
+ : `Failed to remove ${n} agent${n !== 1 ? "s" : ""} from templates.`,
+ partialTitle: (successCount, failedCount) =>
+ isTemplate
+ ? `Saved ${successCount} agent${successCount !== 1 ? "s" : ""} as templates. ${failedCount} failed.`
+ : `Removed ${successCount} agent${successCount !== 1 ? "s" : ""} from templates. ${failedCount} failed.`,
+ });
+ if (succeeded === workflows.length) {
+ onClearSelection();
+ }
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["workflows"] });
+ queryClient.invalidateQueries({ queryKey: ["orgTemplates"] });
+ }
+ } finally {
+ onOperatingChange(false);
+ }
+ }
+
+ async function handleBulkTagApply(tag: Tag) {
+ onOperatingChange(true);
+ try {
+ const client = await getClient(credentialGetter);
+ const results = await runWithConcurrency(
+ selectedWorkflows.map(
+ (workflow) => () =>
+ client.post(`/workflows/${workflow.workflow_permanent_id}/tags`, {
+ tags: [tag],
+ }),
+ ),
+ BULK_CONCURRENCY_LIMIT,
+ );
+ const succeeded = results.filter((r) => r.status === "fulfilled").length;
+ const tagLabel =
+ tag.key !== null ? `${tag.key}: ${tag.value}` : tag.value;
+ bulkResultToast({
+ succeeded,
+ total: count,
+ results,
+ successTitle: (n) =>
+ `Tagged ${n} agent${n !== 1 ? "s" : ""} with ${tagLabel}.`,
+ failureTitle: (n) => `Failed to tag ${n} agent${n !== 1 ? "s" : ""}.`,
+ partialTitle: (successCount, failedCount) =>
+ `Tagged ${successCount} agent${successCount !== 1 ? "s" : ""}. ${failedCount} failed.`,
+ });
+ // Tagging is additive; the selection stays for more tags or follow-up actions.
+ if (succeeded > 0) {
+ queryClient.invalidateQueries({ queryKey: ["workflow-tags"] });
+ queryClient.invalidateQueries({ queryKey: ["tag-keys"] });
+ queryClient.invalidateQueries({ queryKey: ["workflows"] });
+ }
+ } finally {
+ onOperatingChange(false);
+ }
+ }
+
+ function handleBulkExport(type: "json" | "yaml") {
+ if (selectedWorkflows.length === 1) {
+ const workflow = selectedWorkflows[0]!;
+ const safeTitle = workflow.title.replace(/[/\\:*?"<>|]/g, "_");
+ const contents =
+ type === "json"
+ ? JSON.stringify(convert(workflow), null, 2)
+ : convertToYAML(convert(workflow));
+ downloadFile(`${safeTitle}.${type}`, contents);
+ return;
+ }
+ // One bundled file; per-agent downloads trip the browser's
+ // multiple-downloads permission prompt.
+ const converted = selectedWorkflows.map((workflow) => convert(workflow));
+ const contents =
+ type === "json"
+ ? JSON.stringify(converted, null, 2)
+ : converted
+ .map((definition) => convertToYAML(definition))
+ .join("---\n");
+ downloadFile(`agents-export-${selectedWorkflows.length}.${type}`, contents);
+ }
+
+ return (
+ <SelectionBar
+ count={count}
+ isOperating={isOperating}
+ onClear={onClearSelection}
+ >
+ <WorkflowFolderSelector
+ currentFolderId={null}
+ bulkCount={count}
+ onBulkFolderSelect={onMoveToFolder}
+ bulkHasFolders={selectedWorkflows.some(
+ (workflow) => workflow.folder_id !== null,
+ )}
+ disabled={isOperating}
+ trigger={
+ <Button size="sm" variant="ghost" disabled={isOperating}>
+ <FolderIcon className="mr-1.5 h-4 w-4" />
+ Move to folder
+ </Button>
+ }
+ />
+ <BulkTagPicker
+ bulkCount={count}
+ tagKeys={tagKeys}
+ labelSuggestions={labelSuggestions}
+ valueSuggestionsByKey={valueSuggestionsByKey}
+ disabled={isOperating}
+ onApplyTag={handleBulkTagApply}
+ />
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <Button size="sm" variant="ghost" disabled={isOperating}>
+ <DotsHorizontalIcon className="mr-1.5 h-4 w-4" />
+ More
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="end" side="top">
+ <DropdownMenuItem
+ className="p-2"
+ onSelect={() => void handleBulkClone()}
+ >
+ <CopyIcon className="mr-2 h-4 w-4" />
+ Clone
+ </DropdownMenuItem>
+ <DropdownMenuSub>
+ <DropdownMenuSubTrigger>
+ <BookmarkIcon className="mr-2 h-4 w-4" />
+ Template
+ </DropdownMenuSubTrigger>
+ <DropdownMenuPortal>
+ <DropdownMenuSubContent>
+ <DropdownMenuItem
+ disabled={nonTemplates.length === 0}
+ onSelect={() =>
+ void handleBulkTemplateUpdate(nonTemplates, true)
+ }
+ >
+ <BookmarkIcon className="mr-2 h-4 w-4" />
+ Save as Templates ({nonTemplates.length})
+ </DropdownMenuItem>
+ <DropdownMenuItem
+ disabled={templates.length === 0}
+ onSelect={() =>
+ void handleBulkTemplateUpdate(templates, false)
+ }
+ >
+ <BookmarkFilledIcon className="mr-2 h-4 w-4" />
+ Remove from Templates ({templates.length})
+ </DropdownMenuItem>
+ </DropdownMenuSubContent>
+ </DropdownMenuPortal>
+ </DropdownMenuSub>
+ <DropdownMenuSub>
+ <DropdownMenuSubTrigger>
+ <DownloadIcon className="mr-2 h-4 w-4" />
+ Export as...
+ </DropdownMenuSubTrigger>
+ <DropdownMenuPortal>
+ <DropdownMenuSubContent>
+ <DropdownMenuItem onSelect={() => handleBulkExport("yaml")}>
+ YAML
+ </DropdownMenuItem>
+ <DropdownMenuItem onSelect={() => handleBulkExport("json")}>
+ JSON
+ </DropdownMenuItem>
+ </DropdownMenuSubContent>
+ </DropdownMenuPortal>
+ </DropdownMenuSub>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ <SelectionBarDivider />
+ <Button
+ size="sm"
+ variant="ghost"
+ className="text-destructive hover:bg-destructive/10 hover:text-destructive"
+ onClick={onDeleteRequest}
+ disabled={isOperating}
+ >
+ <GarbageIcon className="mr-1.5 h-4 w-4" />
+ Delete
+ </Button>
+ </SelectionBar>
+ );
+}
+
+export { BulkActionBar };
diff --git a/skyvern-frontend/src/routes/workflows/components/WorkflowFolderSelector.tsx b/skyvern-frontend/src/routes/workflows/components/WorkflowFolderSelector.tsx
index addfe3add..1ce44df9d 100644
--- a/skyvern-frontend/src/routes/workflows/components/WorkflowFolderSelector.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/WorkflowFolderSelector.tsx
@@ -1,4 +1,4 @@
-import { useState, useMemo } from "react";
+import { useMemo, useState } from "react";
import {
CheckIcon,
Cross2Icon,
@@ -18,20 +18,32 @@ import { useInfiniteFoldersQuery } from "../hooks/useInfiniteFoldersQuery";
import { useUpdateWorkflowFolderMutation } from "../hooks/useFolderMutations";
import { useDebounce } from "use-debounce";
import { Skeleton } from "@/components/ui/skeleton";
+import { toast } from "@/components/ui/use-toast";
interface WorkflowFolderSelectorProps {
- workflowPermanentId: string;
+ workflowPermanentId?: string;
currentFolderId: string | null;
+ bulkCount?: number;
+ onBulkFolderSelect?: (folderId: string | null) => Promise<void>;
+ bulkHasFolders?: boolean;
+ disabled?: boolean;
+ trigger?: React.ReactNode;
}
function WorkflowFolderSelector({
workflowPermanentId,
currentFolderId,
+ bulkCount,
+ onBulkFolderSelect,
+ bulkHasFolders = true,
+ disabled = false,
+ trigger,
}: WorkflowFolderSelectorProps) {
const [open, setOpen] = useState(false);
const [search, setSearch] = useState("");
const [debouncedSearch] = useDebounce(search, 500);
const isTyping = search !== debouncedSearch;
+ const isBulkMode = (bulkCount ?? 0) > 0 && Boolean(onBulkFolderSelect);
const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isFetching } =
useInfiniteFoldersQuery({
@@ -39,35 +51,61 @@ function WorkflowFolderSelector({
page_size: 20,
});
- // Flatten pages into a single array
const folders = useMemo(() => {
return data?.pages.flatMap((page) => page) ?? [];
}, [data]);
const updateFolderMutation = useUpdateWorkflowFolderMutation();
- const handleFolderSelect = async (folderId: string | null) => {
- await updateFolderMutation.mutateAsync({
- workflowPermanentId,
- data: { folder_id: folderId },
- });
+ async function handleFolderSelect(folderId: string | null) {
+ // Close before the request; progress feedback lives in the caller (toast
+ // or the bulk bar's Processing label), not in a hanging popover.
setOpen(false);
setSearch("");
- };
+ if (isBulkMode && onBulkFolderSelect) {
+ await onBulkFolderSelect(folderId);
+ } else if (workflowPermanentId) {
+ try {
+ await updateFolderMutation.mutateAsync({
+ workflowPermanentId,
+ data: { folder_id: folderId },
+ });
+ } catch (error) {
+ toast({
+ variant: "destructive",
+ title: folderId
+ ? "Failed to move agent to folder"
+ : "Failed to remove agent from folder",
+ description: error instanceof Error ? error.message : undefined,
+ });
+ }
+ } else if (import.meta.env.DEV) {
+ console.warn(
+ "WorkflowFolderSelector needs workflowPermanentId (single mode) or onBulkFolderSelect (bulk mode).",
+ );
+ }
+ }
+
+ const showRemoveFromFolder = isBulkMode
+ ? bulkHasFolders
+ : Boolean(currentFolderId);
return (
<Popover open={open} onOpenChange={setOpen}>
<PopoverTrigger asChild>
- <Button
- variant="ghost"
- size="icon"
- className={cn(
- "text-muted-foreground hover:text-foreground",
- currentFolderId && "text-blue-400 hover:text-blue-400",
- )}
- >
- <FolderIcon className="h-4 w-4" />
- </Button>
+ {trigger ?? (
+ <Button
+ variant="ghost"
+ size="icon"
+ disabled={disabled}
+ className={cn(
+ "text-muted-foreground hover:text-foreground",
+ currentFolderId && "text-blue-400 hover:text-blue-400",
+ )}
+ >
+ <FolderIcon className="h-4 w-4" />
+ </Button>
+ )}
</PopoverTrigger>
<PopoverContent
className="w-80 p-0"
@@ -75,7 +113,11 @@ function WorkflowFolderSelector({
onCloseAutoFocus={(e) => e.preventDefault()}
>
<div className="border-b p-3">
- <h4 className="mb-2 text-sm font-medium">Move to folder</h4>
+ <h4 className="mb-2 text-sm font-medium">
+ {isBulkMode
+ ? `Move ${bulkCount} agent${bulkCount === 1 ? "" : "s"} to folder`
+ : "Move to folder"}
+ </h4>
<div className="relative">
<MagnifyingGlassIcon className="absolute left-2 top-1/2 h-4 w-4 -translate-y-1/2 text-slate-400" />
<Input
@@ -88,7 +130,7 @@ function WorkflowFolderSelector({
</div>
</div>
<div
- className="max-h-[300px] overflow-y-auto [&::-webkit-scrollbar-thumb]:rounded-full [&::-webkit-scrollbar-thumb]:border-2 [&::-webkit-scrollbar-thumb]:border-slate-100 [&::-webkit-scrollbar-thumb]:bg-slate-300 dark:[&::-webkit-scrollbar-thumb]:border-slate-800 dark:[&::-webkit-scrollbar-thumb]:bg-slate-600 [&::-webkit-scrollbar-track]:bg-slate-100 dark:[&::-webkit-scrollbar-track]:bg-slate-800 [&::-webkit-scrollbar]:w-2"
+ className="max-h-[300px] overflow-y-auto overflow-x-hidden [&::-webkit-scrollbar-thumb]:rounded-full [&::-webkit-scrollbar-thumb]:border-2 [&::-webkit-scrollbar-thumb]:border-slate-100 [&::-webkit-scrollbar-thumb]:bg-slate-300 dark:[&::-webkit-scrollbar-thumb]:border-slate-800 dark:[&::-webkit-scrollbar-thumb]:bg-slate-600 [&::-webkit-scrollbar-track]:bg-slate-100 dark:[&::-webkit-scrollbar-track]:bg-slate-800 [&::-webkit-scrollbar]:w-2"
onScroll={(e) =>
handleInfiniteScroll(
e,
@@ -98,10 +140,14 @@ function WorkflowFolderSelector({
)
}
>
- {currentFolderId && (
+ {showRemoveFromFolder && (
<button
- onClick={() => handleFolderSelect(null)}
- className="flex w-full items-center justify-between border-b px-3 py-2 text-left text-sm transition-colors hover:bg-slate-50 dark:hover:bg-slate-800"
+ type="button"
+ onClick={() => {
+ void handleFolderSelect(null);
+ }}
+ disabled={disabled || updateFolderMutation.isPending}
+ className="flex w-full items-center justify-between border-b px-3 py-2 text-left text-sm transition-colors hover:bg-slate-50 disabled:opacity-50 dark:hover:bg-slate-800"
>
<div className="flex items-center gap-2">
<Cross2Icon className="h-4 w-4 text-red-400" />
@@ -111,7 +157,6 @@ function WorkflowFolderSelector({
)}
{(isFetching || isTyping) && folders.length === 0 ? (
- // Show 8 skeleton rows while typing or fetching
<>
{Array.from({ length: 8 }).map((_, index) => (
<div
@@ -133,20 +178,28 @@ function WorkflowFolderSelector({
) : (
<>
{folders.map((folder) => {
- const isCurrentFolder = currentFolderId === folder.folder_id;
+ const isCurrentFolder =
+ !isBulkMode && currentFolderId === folder.folder_id;
return (
<button
key={folder.folder_id}
- onClick={() => handleFolderSelect(folder.folder_id)}
- disabled={isCurrentFolder}
+ type="button"
+ onClick={() => {
+ void handleFolderSelect(folder.folder_id);
+ }}
+ disabled={
+ disabled ||
+ updateFolderMutation.isPending ||
+ isCurrentFolder
+ }
className="flex w-full items-center justify-between px-3 py-2 text-left text-sm transition-colors hover:bg-slate-50 disabled:opacity-50 dark:hover:bg-slate-800"
>
- <div className="flex items-center gap-2">
- <FolderIcon className="h-4 w-4 text-blue-400" />
- <div className="flex flex-col">
- <span>{folder.title}</span>
+ <div className="flex min-w-0 items-center gap-2">
+ <FolderIcon className="h-4 w-4 shrink-0 text-blue-400" />
+ <div className="flex min-w-0 flex-col">
+ <span className="break-words">{folder.title}</span>
{folder.description && (
- <span className="text-xs text-slate-400">
+ <span className="line-clamp-2 break-words text-xs text-slate-400 [overflow-wrap:anywhere]">
{folder.description}
</span>
)}
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/BulkTagPicker.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/BulkTagPicker.tsx
new file mode 100644
index 000000000..ad7fbddc9
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/components/tagging/BulkTagPicker.tsx
@@ -0,0 +1,183 @@
+import * as React from "react";
+import { PlusIcon, TokensIcon } from "@radix-ui/react-icons";
+import { Button } from "@/components/ui/button";
+import {
+ Command,
+ CommandEmpty,
+ CommandGroup,
+ CommandInput,
+ CommandItem,
+ CommandList,
+} from "@/components/ui/command";
+import {
+ Popover,
+ PopoverContent,
+ PopoverTrigger,
+} from "@/components/ui/popover";
+import {
+ MAX_AUTOCOMPLETE_SUGGESTIONS,
+ Tag,
+ TagKey,
+ parseTagInput,
+ parseTypedTagQuery,
+ validateTag,
+} from "../../types/tagTypes";
+
+type Props = {
+ bulkCount: number;
+ tagKeys: Array<TagKey>;
+ labelSuggestions: Array<string>;
+ valueSuggestionsByKey?: Map<string, Array<string>>;
+ disabled?: boolean;
+ onApplyTag: (tag: Tag) => Promise<void>;
+};
+
+function BulkTagPicker({
+ bulkCount,
+ tagKeys,
+ labelSuggestions,
+ valueSuggestionsByKey,
+ disabled = false,
+ onApplyTag,
+}: Props) {
+ const [open, setOpen] = React.useState(false);
+ const [query, setQuery] = React.useState("");
+ const [error, setError] = React.useState<string | null>(null);
+
+ React.useEffect(() => {
+ if (!open) {
+ setQuery("");
+ setError(null);
+ }
+ }, [open]);
+
+ function applyTag(tag: Tag) {
+ const validationError = validateTag(tag);
+ if (validationError) {
+ setError(validationError);
+ return;
+ }
+ setOpen(false);
+ void onApplyTag(tag);
+ }
+
+ const trimmedQuery = query.trim();
+ const normalizedQuery = trimmedQuery.toLowerCase();
+ const candidate = parseTagInput(query);
+ const { typedKey, typedValuePartial } = parseTypedTagQuery(trimmedQuery);
+
+ const groupSuggestions =
+ typedKey === null
+ ? tagKeys
+ .filter((tk) => tk.key.toLowerCase().includes(normalizedQuery))
+ .slice(0, MAX_AUTOCOMPLETE_SUGGESTIONS)
+ : [];
+ const labelMatches =
+ typedKey === null
+ ? labelSuggestions
+ .filter((value) => value.toLowerCase().includes(normalizedQuery))
+ .slice(0, MAX_AUTOCOMPLETE_SUGGESTIONS)
+ : [];
+ const groupedValueMatches =
+ typedKey !== null
+ ? (valueSuggestionsByKey?.get(typedKey) ?? [])
+ .filter((value) => value.toLowerCase().includes(typedValuePartial))
+ .slice(0, MAX_AUTOCOMPLETE_SUGGESTIONS)
+ : [];
+ const hasItems =
+ candidate !== null ||
+ groupSuggestions.length > 0 ||
+ labelMatches.length > 0 ||
+ groupedValueMatches.length > 0;
+
+ return (
+ <Popover open={open} onOpenChange={setOpen}>
+ <PopoverTrigger asChild>
+ <Button size="sm" variant="ghost" disabled={disabled}>
+ <TokensIcon className="mr-1.5 h-4 w-4" />
+ Tags
+ </Button>
+ </PopoverTrigger>
+ <PopoverContent className="w-80 p-0" align="start">
+ <div className="border-b p-3">
+ <h4 className="text-sm font-medium">
+ Add tag to {bulkCount} agent{bulkCount === 1 ? "" : "s"}
+ </h4>
+ {error ? (
+ <div className="mt-1 text-xs text-destructive">{error}</div>
+ ) : null}
+ </div>
+ <Command shouldFilter={false}>
+ <CommandInput
+ placeholder="Label or group:label…"
+ value={query}
+ onValueChange={(value) => {
+ setQuery(value);
+ setError(null);
+ }}
+ />
+ <CommandList>
+ {candidate !== null && (
+ <CommandGroup>
+ <CommandItem
+ value={`add-${trimmedQuery}`}
+ onSelect={() => applyTag(candidate)}
+ >
+ <PlusIcon className="mr-2 h-4 w-4" />
+ Add{" "}
+ {candidate.key !== null
+ ? `${candidate.key}: ${candidate.value}`
+ : candidate.value}
+ </CommandItem>
+ </CommandGroup>
+ )}
+ {typedKey !== null && groupedValueMatches.length > 0 && (
+ <CommandGroup heading={`${typedKey}:`}>
+ {groupedValueMatches.map((value) => (
+ <CommandItem
+ key={value}
+ value={`${typedKey}:${value}`}
+ onSelect={() => applyTag({ key: typedKey, value })}
+ >
+ {value}
+ </CommandItem>
+ ))}
+ </CommandGroup>
+ )}
+ {groupSuggestions.length > 0 && (
+ <CommandGroup heading="Groups">
+ {groupSuggestions.map((tk) => (
+ <CommandItem
+ key={tk.key}
+ value={`group-${tk.key}`}
+ onSelect={() => setQuery(`${tk.key}: `)}
+ >
+ {tk.key}:
+ </CommandItem>
+ ))}
+ </CommandGroup>
+ )}
+ {labelMatches.length > 0 && (
+ <CommandGroup heading="Labels">
+ {labelMatches.map((value) => (
+ <CommandItem
+ key={value}
+ value={`label-${value}`}
+ onSelect={() => applyTag({ key: null, value })}
+ >
+ {value}
+ </CommandItem>
+ ))}
+ </CommandGroup>
+ )}
+ {!hasItems && (
+ <CommandEmpty>Type a label or group:label to add.</CommandEmpty>
+ )}
+ </CommandList>
+ </Command>
+ </PopoverContent>
+ </Popover>
+ );
+}
+
+export { BulkTagPicker };
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagChip.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagChip.tsx
index 59fa1fad2..1cdb4d94d 100644
--- a/skyvern-frontend/src/routes/workflows/components/tagging/TagChip.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagChip.tsx
@@ -20,6 +20,15 @@ type Props = {
// Generic single-tag chip: a styled span (not Badge div) so it can be a Radix
// TooltipTrigger `asChild` target. Grouped shows `key: value`, standalone the value.
function TagChip({ tagKey, value, description, onRemove, className }: Props) {
+ // Last line of defense against payload shape skew: a non-string child here
+ // is React error #31, which unmounts the route through the error boundary.
+ if (
+ typeof value !== "string" ||
+ (tagKey !== null && typeof tagKey !== "string")
+ ) {
+ console.warn("[tags] skipping tag chip with a non-string key or value");
+ return null;
+ }
const label = tagKey === null ? value : `${tagKey}: ${value}`;
const chip = (
<span
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.shape.test.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.shape.test.tsx
new file mode 100644
index 000000000..045abe872
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.shape.test.tsx
@@ -0,0 +1,46 @@
+// @vitest-environment jsdom
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { Tag } from "../../types/tagTypes";
+import { TagChipList } from "./TagChipList";
+import { TagChip } from "./TagChip";
+
+// Regression for the SKY-10683 backend/frontend shape split: a skewed tags
+// payload reached the chips untyped, rendered a raw object child (React #31),
+// and the route error boundary took down the whole /workflows page.
+
+afterEach(() => {
+ cleanup();
+ vi.restoreAllMocks();
+});
+
+describe("TagChipList shape tolerance", () => {
+ it("renders the valid subset when list entries are malformed", () => {
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ const skewed = [
+ { key: "env", value: "prod" },
+ { key: "junk", value: { key: "nested", value: "object" } },
+ null,
+ ] as unknown as Array<Tag>;
+ expect(() => render(<TagChipList tags={skewed} />)).not.toThrow();
+ expect(screen.getByText("prod")).toBeTruthy();
+ expect(screen.queryByText("junk")).toBeNull();
+ });
+
+ it("converts a legacy record payload instead of throwing", () => {
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ const legacy = { env: "prod" } as unknown as Array<Tag>;
+ expect(() => render(<TagChipList tags={legacy} />)).not.toThrow();
+ expect(screen.getByText("prod")).toBeTruthy();
+ });
+});
+
+describe("TagChip shape tolerance", () => {
+ it("renders nothing and warns when value is not a string", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const objectValue = { key: "k", value: "v" } as unknown as string;
+ const { container } = render(<TagChip tagKey="env" value={objectValue} />);
+ expect(container.textContent).toBe("");
+ expect(warn).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.tsx b/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.tsx
index 51279d5c5..643408f53 100644
--- a/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/tagging/TagChipList.tsx
@@ -6,7 +6,12 @@ import {
} from "@/components/ui/tooltip";
import { badgeVariants } from "@/components/ui/badge-variants";
import { cn } from "@/util/utils";
-import { sortTags, tagElementKey, type Tag } from "../../types/tagTypes";
+import {
+ normalizeWorkflowTags,
+ sortTags,
+ tagElementKey,
+ type Tag,
+} from "../../types/tagTypes";
import { TagChip } from "./TagChip";
type Props = {
@@ -21,10 +26,13 @@ type Props = {
// Generic list of tag chips with a "+N" overflow affordance. Standalone labels
// sort first, then grouped by key, for stable ordering across renders.
function TagChipList({ tags, descriptions, maxVisible = 3, className }: Props) {
- if (tags.length === 0) {
+ // Re-validate at render time: callers feed this straight from API payloads,
+ // and a shape skew here previously killed the whole route via React #31.
+ const safeTags = normalizeWorkflowTags(tags);
+ if (safeTags.length === 0) {
return null;
}
- const sorted = sortTags(tags);
+ const sorted = sortTags(safeTags);
const visible = sorted.slice(0, maxVisible);
const hidden = sorted.slice(maxVisible);
diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
index c414f5172..8dca148f3 100644
--- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
@@ -152,7 +152,6 @@ import { PoliteDndLiveRegionPolicy } from "./sortable/dragLiveRegionPolicy";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useIsCanvasLocked } from "./controls/useIsCanvasLocked";
import { BlockConfigSidebar } from "./panels/BlockConfigSidebar";
-import { BlockSidebarMigrationPopover } from "./panels/BlockSidebarMigrationPopover";
// Grace period after nodesInitialized before we start tracking changes.
// Allows mount-time effects (ResizeObserver, visibility toggling) to settle.
@@ -1570,6 +1569,7 @@ function FlowRenderer({
return (
<WorkflowScopeContext.Provider value={workflowScopeValue}>
<div
+ data-tour="editor-canvas"
className={cn("workflow-editor-shell relative h-full w-full", {
"react-flow--pre-layout": layoutPhase === "pre-layout",
"react-flow--initial-load":
@@ -1861,7 +1861,13 @@ function FlowRenderer({
expose the editable block form.
*/}
{!readOnly && <BlockConfigSidebar onAddNode={onAddNode} />}
- {!readOnly && <BlockSidebarMigrationPopover />}
+ {!readOnly && (
+ <div
+ data-tour="sidebar-region"
+ className="pointer-events-none absolute bottom-6 right-6 top-8"
+ style={{ width: 320 }}
+ />
+ )}
<DragOverlay dropAnimation={null}>
{(() => {
if (!activeDragId) return null;
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
index 2639d73b3..82914b89d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx
@@ -14,6 +14,8 @@ import { useBlockOutputStore } from "@/store/BlockOutputStore";
import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";
import { getInitialParameters } from "./utils";
import { Workspace } from "./Workspace";
+import { ProductTour } from "@/components/onboarding/ProductTour";
+import { useProductTourShortcut } from "@/hooks/useProductTourShortcut";
import { useMountEffect } from "@/hooks/useMountEffect";
function WorkflowEditor() {
@@ -33,6 +35,8 @@ function WorkflowEditor() {
const blockOutputStore = useBlockOutputStore();
+ useProductTourShortcut();
+
useMountEffect(() => blockOutputStore.reset());
useMountEffect(() => {
@@ -128,6 +132,7 @@ function WorkflowEditor() {
/>
</ReactFlowProvider>
</div>
+ <ProductTour />
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
index faf81406f..d9f448266 100644
--- a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
@@ -256,7 +256,7 @@ function RunButton() {
function EditorActionToolbar() {
return (
- <div className="flex items-center gap-2">
+ <div data-tour="editor-actions" className="flex items-center gap-2">
<BrowserModeButton />
<SaveButton />
<PanelToggleButton
diff --git a/skyvern-frontend/src/routes/workflows/editor/header/EditorOverflowMenu.tsx b/skyvern-frontend/src/routes/workflows/editor/header/EditorOverflowMenu.tsx
index 2775b4b34..578d32ba5 100644
--- a/skyvern-frontend/src/routes/workflows/editor/header/EditorOverflowMenu.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/header/EditorOverflowMenu.tsx
@@ -26,6 +26,7 @@ import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowQuery";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
+import { useProductTourStore } from "@/store/ProductTourStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
@@ -40,6 +41,7 @@ export function EditorOverflowMenu() {
const isTemplate = workflow?.is_template ?? false;
const saving = useWorkflowHasChangesStore((s) => s.saveIsPending);
const isRecording = useRecordingStore((s) => s.isRecording);
+ const requestTour = useProductTourStore((s) => s.requestTour);
const onSave = useSaveWorkflow();
const credentialGetter = useCredentialGetter();
const queryClient = useQueryClient();
@@ -131,6 +133,11 @@ export function EditorOverflowMenu() {
>
{isTemplate ? "Remove from Templates" : "Save as Template"}
</DropdownMenuItem>
+ <DropdownMenuSeparator />
+ <DropdownMenuItem onSelect={requestTour}>
+ <span className="flex-1">Take a tour</span>
+ <kbd className="ml-4 text-xs text-slate-400">Shift+?</kbd>
+ </DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
);
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx
index 199dc8e3b..8fbddfeef 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx
@@ -1,3 +1,6 @@
+import { useState } from "react";
+
+import { ArrowTopRightIcon, PlusIcon } from "@radix-ui/react-icons";
import { useEdges, useNodes, useNodesData } from "@xyflow/react";
import { HelpTooltip } from "@/components/HelpTooltip";
@@ -7,6 +10,7 @@ import {
AccordionItem,
AccordionTrigger,
} from "@/components/ui/accordion";
+import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Separator } from "@/components/ui/separator";
@@ -44,7 +48,11 @@ function LoginEditor({ blockId }: { blockId: string }) {
if (!nodeSlice || nodeSlice.type !== "login") {
return null;
}
- return <LoginEditorBody blockId={blockId} data={nodeSlice.data} />;
+ // Key by blockId: the sidebar reuses one LoginEditorBody instance across
+ // same-type blocks; remounting resets local disclosure state.
+ return (
+ <LoginEditorBody key={blockId} blockId={blockId} data={nodeSlice.data} />
+ );
}
function LoginEditorBody({
@@ -69,6 +77,15 @@ function LoginEditorBody({
const credentialTotpIdentifier = useSelectedCredentialTotpIdentifier(
data.parameterKeys.length > 0 ? data.parameterKeys[0] : undefined,
);
+ const hasTotpValues = Boolean(
+ data.totpIdentifier?.trim() || data.totpVerificationUrl?.trim(),
+ );
+ const credentialHasTotp = Boolean(credentialTotpIdentifier);
+ const [twoFactorRequested, setTwoFactorRequested] = useState(
+ () => hasTotpValues,
+ );
+ const showTwoFactorFields = twoFactorRequested || hasTotpValues;
+ const showCredentialTotpSummary = credentialHasTotp && !showTwoFactorFields;
return (
<div data-testid="login-block-form" className="space-y-4">
@@ -135,45 +152,97 @@ function LoginEditorBody({
}}
/>
</div>
- <div className="space-y-2">
- <div className="flex gap-2">
- <Label className="text-xs text-slate-300">2FA Identifier</Label>
- <HelpTooltip content={helpTooltips["login"]["totpIdentifier"]} />
+ {showTwoFactorFields ? (
+ <div className="space-y-3">
+ <div className="space-y-2">
+ <div className="flex gap-2">
+ <Label className="text-xs text-slate-300">2FA Identifier</Label>
+ <HelpTooltip
+ content={helpTooltips["login"]["totpIdentifier"]}
+ />
+ </div>
+ <WorkflowBlockInputTextarea
+ nodeId={blockId}
+ onChange={(value) => update({ totpIdentifier: value })}
+ value={data.totpIdentifier ?? ""}
+ placeholder={
+ !data.totpIdentifier?.trim() && credentialTotpIdentifier
+ ? `${credentialTotpIdentifier} (from credential)`
+ : placeholders["login"]["totpIdentifier"]
+ }
+ className="nopan text-xs"
+ />
+ {!data.totpIdentifier?.trim() && credentialTotpIdentifier ? (
+ <p className="text-xs text-slate-500">
+ Leave empty to use the credential's value.
+ </p>
+ ) : null}
+ </div>
+ <div className="space-y-2">
+ <div className="flex gap-2">
+ <Label className="text-xs text-slate-300">
+ 2FA Verification URL
+ </Label>
+ <HelpTooltip
+ content={helpTooltips["login"]["totpVerificationUrl"]}
+ />
+ </div>
+ <WorkflowBlockInputTextarea
+ nodeId={blockId}
+ onChange={(value) => update({ totpVerificationUrl: value })}
+ value={data.totpVerificationUrl ?? ""}
+ placeholder={placeholders["login"]["totpVerificationUrl"]}
+ className="nopan text-xs"
+ />
+ </div>
+ {!hasTotpValues ? (
+ <button
+ type="button"
+ onClick={() => setTwoFactorRequested(false)}
+ className="text-xs text-slate-400 transition-colors hover:text-slate-200"
+ >
+ Cancel
+ </button>
+ ) : null}
</div>
- <WorkflowBlockInputTextarea
- nodeId={blockId}
- onChange={(value) => update({ totpIdentifier: value })}
- value={data.totpIdentifier ?? ""}
- placeholder={
- !data.totpIdentifier?.trim() && credentialTotpIdentifier
- ? `${credentialTotpIdentifier} (from credential)`
- : placeholders["login"]["totpIdentifier"]
- }
- className="nopan text-xs"
- />
- {!data.totpIdentifier?.trim() && credentialTotpIdentifier ? (
- <p className="text-xs text-slate-500">
- Leave empty to use the credential's value.
- </p>
- ) : null}
- </div>
- <div className="space-y-2">
- <div className="flex gap-2">
- <Label className="text-xs text-slate-300">
- 2FA Verification URL
- </Label>
- <HelpTooltip
- content={helpTooltips["login"]["totpVerificationUrl"]}
- />
+ ) : showCredentialTotpSummary ? (
+ <div className="flex items-center justify-between gap-2">
+ <a
+ href="/credentials?tab=twoFactor"
+ target="_blank"
+ rel="noopener noreferrer"
+ className="min-w-0 break-words text-xs text-slate-400 transition-colors hover:text-slate-200"
+ >
+ Skyvern is waiting for 2FA codes with this identifier:{" "}
+ <span className="font-mono text-slate-300">
+ {credentialTotpIdentifier}
+ </span>
+ <ArrowTopRightIcon className="ml-0.5 inline size-3 align-text-bottom" />
+ </a>
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="h-7 shrink-0 px-2 text-xs"
+ disabled={!editable}
+ onClick={() => setTwoFactorRequested(true)}
+ >
+ Override
+ </Button>
</div>
- <WorkflowBlockInputTextarea
- nodeId={blockId}
- onChange={(value) => update({ totpVerificationUrl: value })}
- value={data.totpVerificationUrl ?? ""}
- placeholder={placeholders["login"]["totpVerificationUrl"]}
- className="nopan text-xs"
- />
- </div>
+ ) : (
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="h-7 px-2 text-xs"
+ disabled={!editable}
+ onClick={() => setTwoFactorRequested(true)}
+ >
+ <PlusIcon className="mr-1 h-3 w-3" />
+ Add two-factor authentication
+ </Button>
+ )}
</div>
<Separator />
<Accordion type="single" collapsible>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
index bfa79e8e6..7af9f414d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
@@ -291,7 +291,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
);
return (
- <div>
+ <div data-tour="node-adder">
<input
ref={fileInputRef}
type="file"
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/LoginBlockForm.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/LoginBlockForm.test.tsx
index edaf7719a..eff7ca8ae 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/LoginBlockForm.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/BlockConfigForm/LoginBlockForm.test.tsx
@@ -55,8 +55,9 @@ vi.mock("@/components/HelpTooltip", () => ({
HelpTooltip: () => <span data-testid="help-tooltip" />,
}));
+const mockCredentialTotp = vi.hoisted(() => ({ value: null as string | null }));
vi.mock("../../hooks/useSelectedCredentialTotpIdentifier", () => ({
- useSelectedCredentialTotpIdentifier: () => null,
+ useSelectedCredentialTotpIdentifier: () => mockCredentialTotp.value,
}));
vi.mock("../../nodes/LoginNode/LoginBlockCredentialSelector", () => ({
@@ -200,6 +201,7 @@ function makeLoginFixture(
beforeEach(() => {
mockNodeFixtures.clear();
+ mockCredentialTotp.value = null;
usePendingCommitsStore.setState({ commits: {} });
updateNodeDataMock.mockReset();
useDebouncedSidebarSaveMock.mockReset();
@@ -225,33 +227,104 @@ describe("LoginBlockForm (SKY-9374)", () => {
expect(useDebouncedSidebarSaveMock).not.toHaveBeenCalled();
});
- test("renders the basic fields and the advanced-settings accordion trigger", () => {
+ test("renders the basic fields and collapses 2FA behind an add button", () => {
mockNodeFixtures.set("b1", makeLoginFixture("b1"));
render(<LoginBlockForm blockId="b1" />);
- // Outside the (collapsed) Advanced Settings accordion: URL + Login
- // Goal textareas, plus the Authentication group (Credential selector +
- // 2FA Identifier + 2FA Verification URL textareas). Radix
- // AccordionContent does not render its children while closed, so
- // these are the only basic-section assertions we can make without
- // expanding it.
- expect(screen.getAllByTestId("wbi-textarea")).toHaveLength(4);
+ // Basic section: URL + Login Goal textareas (2). The Credential
+ // selector is always present, but the two 2FA textareas now sit behind
+ // the "Add two-factor authentication" affordance until the user opts in.
+ expect(screen.getAllByTestId("wbi-textarea")).toHaveLength(2);
expect(screen.getByTestId("login-credential-selector")).toBeDefined();
+ expect(
+ screen.getByRole("button", { name: "Add two-factor authentication" }),
+ ).toBeDefined();
+ expect(screen.queryByText("2FA Identifier")).toBeNull();
+ expect(screen.queryByText("2FA Verification URL")).toBeNull();
expect(screen.getByText("Advanced Settings")).toBeDefined();
});
+ test("clicking add reveals the two 2FA fields", () => {
+ mockNodeFixtures.set("b1", makeLoginFixture("b1"));
+ render(<LoginBlockForm blockId="b1" />);
+
+ fireEvent.click(
+ screen.getByRole("button", { name: "Add two-factor authentication" }),
+ );
+
+ expect(screen.getAllByTestId("wbi-textarea")).toHaveLength(4);
+ expect(screen.getByText("2FA Identifier")).toBeDefined();
+ expect(screen.getByText("2FA Verification URL")).toBeDefined();
+ });
+
+ test("auto-expands 2FA when the block already has a totp value", () => {
+ mockNodeFixtures.set(
+ "b1",
+ makeLoginFixture("b1", { totpIdentifier: "my-identifier" }),
+ );
+ render(<LoginBlockForm blockId="b1" />);
+
+ // Saved 2FA config is never hidden: the fields render without clicking
+ // add, and the add affordance is gone.
+ expect(screen.getByText("2FA Identifier")).toBeDefined();
+ expect(screen.getByText("2FA Verification URL")).toBeDefined();
+ expect(
+ screen.queryByRole("button", { name: "Add two-factor authentication" }),
+ ).toBeNull();
+ });
+
+ test("summarizes credential-provided 2FA, showing the identifier, with override", () => {
+ mockCredentialTotp.value = "credential-identifier";
+ mockNodeFixtures.set("b1", makeLoginFixture("b1"));
+ render(<LoginBlockForm blockId="b1" />);
+
+ // A credential that carries TOTP is summarized (not force-expanded) as a
+ // link to the 2FA page, surfacing its identifier — a routing label, not a
+ // secret — so the user can confirm which one is in effect.
+ const summaryLink = screen.getByRole("link", {
+ name: /waiting for 2FA codes/i,
+ });
+ expect(summaryLink.getAttribute("href")).toBe("/credentials?tab=twoFactor");
+ expect(screen.getByText("credential-identifier")).toBeDefined();
+ expect(screen.queryByText("2FA Identifier")).toBeNull();
+
+ fireEvent.click(screen.getByRole("button", { name: "Override" }));
+ expect(screen.getByText("2FA Identifier")).toBeDefined();
+ expect(screen.getByText("2FA Verification URL")).toBeDefined();
+ });
+
+ test("does not leak the 2FA expand toggle across block switches", () => {
+ // Same-type block forms are reused without remounting, so the editor is
+ // keyed by blockId. Open 2FA on b1, switch to b2, and b2 must start
+ // collapsed instead of inheriting b1's expanded section.
+ mockNodeFixtures.set("b1", makeLoginFixture("b1"));
+ mockNodeFixtures.set("b2", makeLoginFixture("b2"));
+ const { rerender } = render(<LoginBlockForm blockId="b1" />);
+
+ fireEvent.click(
+ screen.getByRole("button", { name: "Add two-factor authentication" }),
+ );
+ expect(screen.getByText("2FA Identifier")).toBeDefined();
+
+ rerender(<LoginBlockForm blockId="b2" />);
+ expect(screen.queryByText("2FA Identifier")).toBeNull();
+ expect(
+ screen.getByRole("button", { name: "Add two-factor authentication" }),
+ ).toBeDefined();
+ });
+
test("expanding Advanced Settings reveals every inline-form field", () => {
mockNodeFixtures.set("b1", makeLoginFixture("b1"));
render(<LoginBlockForm blockId="b1" />);
// Radix AccordionContent unmounts while collapsed, so reach the
- // advanced-settings widgets by clicking the trigger first. After
- // expansion every field the inline LoginNode form owns must be
- // present — pin the count so additions/removals force this test to
- // be updated explicitly instead of silently drifting parity.
+ // advanced-settings widgets by clicking the trigger first. With 2FA
+ // collapsed by default, the visible textareas are URL + Login Goal +
+ // Complete-if (3) — pin the count so additions/removals force this
+ // test to be updated explicitly instead of silently drifting parity.
fireEvent.click(screen.getByText("Advanced Settings"));
- expect(screen.getAllByTestId("wbi-textarea")).toHaveLength(5);
+ expect(screen.getAllByTestId("wbi-textarea")).toHaveLength(3);
expect(screen.getByTestId("parameters-multi-select")).toBeDefined();
expect(screen.getByTestId("model-selector")).toBeDefined();
expect(screen.getByTestId("engine-selector")).toBeDefined();
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockSidebarMigrationPopover.test.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockSidebarMigrationPopover.test.tsx
deleted file mode 100644
index bf329b411..000000000
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockSidebarMigrationPopover.test.tsx
+++ /dev/null
@@ -1,53 +0,0 @@
-// @vitest-environment jsdom
-
-import { cleanup, fireEvent, render, screen } from "@testing-library/react";
-import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
-
-import { useBlockSidebarOnboardingStore } from "@/store/BlockSidebarOnboardingStore";
-import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
-
-import { BlockSidebarMigrationPopover } from "./BlockSidebarMigrationPopover";
-
-vi.mock("../hooks/useWorkflowEditorMode", () => ({
- useWorkflowEditorMode: () => "edit",
-}));
-
-beforeEach(() => {
- localStorage.clear();
- useBlockSidebarOnboardingStore.getState().reset();
- useWorkflowPanelStore.getState().setSelectedBlockId(null);
-});
-
-afterEach(() => {
- cleanup();
-});
-
-describe("BlockSidebarMigrationPopover", () => {
- test("renders nothing when the user has already dismissed", () => {
- useBlockSidebarOnboardingStore.getState().markSeen();
- useWorkflowPanelStore.getState().setSelectedBlockId("block-1");
- const { container } = render(<BlockSidebarMigrationPopover />);
- expect(container.innerHTML).toBe("");
- });
-
- test("renders nothing when no block is selected", () => {
- const { container } = render(<BlockSidebarMigrationPopover />);
- expect(container.innerHTML).toBe("");
- });
-
- test("renders the migration copy when first-edit and a block is selected", () => {
- useWorkflowPanelStore.getState().setSelectedBlockId("block-1");
- render(<BlockSidebarMigrationPopover />);
- expect(screen.getByText(/configuration moved here/i)).toBeDefined();
- });
-
- test("clicking 'Got it' marks seen and hides the popover", () => {
- useWorkflowPanelStore.getState().setSelectedBlockId("block-1");
- render(<BlockSidebarMigrationPopover />);
- fireEvent.click(screen.getByRole("button", { name: /got it/i }));
- expect(useBlockSidebarOnboardingStore.getState().hasSeenMigration).toBe(
- true,
- );
- expect(screen.queryByText(/configuration moved here/i)).toBeNull();
- });
-});
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/BlockSidebarMigrationPopover.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/BlockSidebarMigrationPopover.tsx
deleted file mode 100644
index 3f83d6186..000000000
--- a/skyvern-frontend/src/routes/workflows/editor/panels/BlockSidebarMigrationPopover.tsx
+++ /dev/null
@@ -1,60 +0,0 @@
-import { useEffect, useState } from "react";
-
-import { Button } from "@/components/ui/button";
-import { useBlockSidebarOnboardingStore } from "@/store/BlockSidebarOnboardingStore";
-import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
-
-import { useWorkflowEditorMode } from "../hooks/useWorkflowEditorMode";
-
-function BlockSidebarMigrationPopover() {
- const mode = useWorkflowEditorMode();
- const selectedBlockId = useWorkflowPanelStore((s) => s.selectedBlockId);
- const hasSeenMigration = useBlockSidebarOnboardingStore(
- (s) => s.hasSeenMigration,
- );
- const markSeen = useBlockSidebarOnboardingStore((s) => s.markSeen);
-
- const [open, setOpen] = useState(false);
-
- useEffect(() => {
- if (mode === "edit" && selectedBlockId !== null && !hasSeenMigration) {
- setOpen(true);
- } else {
- setOpen(false);
- }
- }, [mode, selectedBlockId, hasSeenMigration]);
-
- if (!open) {
- return null;
- }
-
- return (
- <div
- data-testid="block-sidebar-migration-popover"
- role="dialog"
- aria-live="polite"
- className="absolute right-[calc(var(--block-sidebar-w)+4.5rem)] top-32 z-40 w-72 rounded-xl border border-border bg-slate-elevation3 p-4 shadow-xl"
- >
- <h3 className="text-sm font-medium text-slate-100">
- Configuration moved here
- </h3>
- <p className="mt-1 text-xs text-slate-400">
- Block settings now live in this side panel. Edits save automatically -
- no need to confirm. You can drag the panel's left edge to make it wider.
- </p>
- <div className="mt-3 flex justify-end">
- <Button
- size="sm"
- variant="tertiary"
- onClick={() => {
- markSeen();
- }}
- >
- Got it
- </Button>
- </div>
- </div>
- );
-}
-
-export { BlockSidebarMigrationPopover };
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.test.tsx b/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.test.tsx
new file mode 100644
index 000000000..67a48be9a
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.test.tsx
@@ -0,0 +1,76 @@
+// @vitest-environment jsdom
+import type { ReactNode } from "react";
+import { renderHook, waitFor, act } from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { MemoryRouter } from "react-router-dom";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const { mockPost } = vi.hoisted(() => ({ mockPost: vi.fn() }));
+
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => () => Promise.resolve("test-token"),
+}));
+
+vi.mock("@/api/AxiosClient", () => ({
+ getClient: () => Promise.resolve({ post: mockPost }),
+}));
+
+vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));
+
+vi.mock("@/util/onboarding/OnboardingTelemetry", () => ({
+ OnboardingTelemetry: { flowCompleted: vi.fn() },
+}));
+
+import { useCreateWorkflowMutation } from "./useCreateWorkflowMutation";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
+
+function wrapper({ children }: { children: ReactNode }) {
+ const queryClient = new QueryClient({
+ defaultOptions: {
+ queries: { retry: false },
+ mutations: { retry: false },
+ },
+ });
+ return (
+ <QueryClientProvider client={queryClient}>
+ <MemoryRouter>{children}</MemoryRouter>
+ </QueryClientProvider>
+ );
+}
+
+afterEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("useCreateWorkflowMutation", () => {
+ it("emits onboarding flow_completed when the creation came from the onboarding template path", async () => {
+ mockPost.mockResolvedValue({ data: { workflow_permanent_id: "wpid_x" } });
+ const { result } = renderHook(() => useCreateWorkflowMutation(), {
+ wrapper,
+ });
+
+ act(() => {
+ result.current.mutate({
+ title: "Test",
+ _via: "onboarding_template",
+ } as never);
+ });
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(OnboardingTelemetry.flowCompleted).toHaveBeenCalledWith("dashboard");
+ });
+
+ it("does not emit flow_completed for non-onboarding creations", async () => {
+ mockPost.mockResolvedValue({ data: { workflow_permanent_id: "wpid_y" } });
+ const { result } = renderHook(() => useCreateWorkflowMutation(), {
+ wrapper,
+ });
+
+ act(() => {
+ result.current.mutate({ title: "Test", _via: "sidebar" } as never);
+ });
+
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ expect(OnboardingTelemetry.flowCompleted).not.toHaveBeenCalled();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.ts b/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.ts
index 3094db026..c8a685f14 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useCreateWorkflowMutation.ts
@@ -6,6 +6,7 @@ import { useNavigate } from "react-router-dom";
import { stringify as convertToYAML } from "yaml";
import { WorkflowApiResponse } from "../types/workflowTypes";
import { toast } from "@/components/ui/use-toast";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
import axios from "axios";
type CreateWorkflowInput = WorkflowCreateYAMLRequest & { _via?: string };
@@ -53,7 +54,16 @@ function useCreateWorkflowMutation() {
queryClient.invalidateQueries({
queryKey: ["folders"],
});
+ queryClient.invalidateQueries({
+ queryKey: ["userOnboarding"],
+ });
const via = variables._via;
+ // Emit completion here rather than in the caller's mutate-level onSuccess:
+ // this navigation unmounts the onboarding modal, and React Query skips
+ // mutate-level callbacks once their observer unmounts.
+ if (via === "onboarding_template") {
+ OnboardingTelemetry.flowCompleted("dashboard");
+ }
const search = via ? `?via=${encodeURIComponent(via)}` : "";
navigate(
`/workflows/${response.data.workflow_permanent_id}/build${search}`,
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useRefreshOnboardingOnRunCompletion.test.tsx b/skyvern-frontend/src/routes/workflows/hooks/useRefreshOnboardingOnRunCompletion.test.tsx
new file mode 100644
index 000000000..eb0d2b302
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/hooks/useRefreshOnboardingOnRunCompletion.test.tsx
@@ -0,0 +1,77 @@
+// @vitest-environment jsdom
+import type { ReactNode } from "react";
+import { renderHook } from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Status } from "@/api/types";
+import { useRefreshOnboardingOnRunCompletion } from "./useRefreshOnboardingOnRunCompletion";
+
+const ONBOARDING_KEY = { queryKey: ["userOnboarding"] };
+const RUN_ID = "wr_1";
+
+function run(status: Status) {
+ return { workflow_run_id: RUN_ID, status };
+}
+
+function makeWrapper() {
+ const queryClient = new QueryClient();
+ const invalidateSpy = vi.spyOn(queryClient, "invalidateQueries");
+ const wrapper = ({ children }: { children: ReactNode }) => (
+ <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
+ );
+ return { wrapper, invalidateSpy };
+}
+
+function onboardingInvalidations(spy: ReturnType<typeof vi.spyOn>): number {
+ return spy.mock.calls.filter(
+ (call: unknown[]) =>
+ JSON.stringify(call[0]) === JSON.stringify(ONBOARDING_KEY),
+ ).length;
+}
+
+afterEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("useRefreshOnboardingOnRunCompletion", () => {
+ it("invalidates userOnboarding when an observed running run finalizes", () => {
+ const { wrapper, invalidateSpy } = makeWrapper();
+ const { rerender } = renderHook(
+ ({ wr }) => useRefreshOnboardingOnRunCompletion(wr),
+ { wrapper, initialProps: { wr: run(Status.Running) } },
+ );
+ expect(onboardingInvalidations(invalidateSpy)).toBe(0);
+
+ rerender({ wr: run(Status.Completed) });
+ expect(onboardingInvalidations(invalidateSpy)).toBe(1);
+ });
+
+ it("invalidates once when a run is already finalized on first observation", () => {
+ const { wrapper, invalidateSpy } = makeWrapper();
+ renderHook(({ wr }) => useRefreshOnboardingOnRunCompletion(wr), {
+ wrapper,
+ initialProps: { wr: run(Status.Completed) },
+ });
+ expect(onboardingInvalidations(invalidateSpy)).toBe(1);
+ });
+
+ it("does not invalidate while the run is not finalized", () => {
+ const { wrapper, invalidateSpy } = makeWrapper();
+ renderHook(({ wr }) => useRefreshOnboardingOnRunCompletion(wr), {
+ wrapper,
+ initialProps: { wr: run(Status.Running) },
+ });
+ expect(onboardingInvalidations(invalidateSpy)).toBe(0);
+ });
+
+ it("invalidates only once after completion across re-renders", () => {
+ const { wrapper, invalidateSpy } = makeWrapper();
+ const { rerender } = renderHook(
+ ({ wr }) => useRefreshOnboardingOnRunCompletion(wr),
+ { wrapper, initialProps: { wr: run(Status.Running) } },
+ );
+ rerender({ wr: run(Status.Completed) });
+ rerender({ wr: run(Status.Completed) });
+ expect(onboardingInvalidations(invalidateSpy)).toBe(1);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useRefreshOnboardingOnRunCompletion.ts b/skyvern-frontend/src/routes/workflows/hooks/useRefreshOnboardingOnRunCompletion.ts
new file mode 100644
index 000000000..af780cd2c
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/hooks/useRefreshOnboardingOnRunCompletion.ts
@@ -0,0 +1,33 @@
+import { useEffect, useRef } from "react";
+import { useQueryClient } from "@tanstack/react-query";
+import { Status } from "@/api/types";
+import { statusIsFinalized } from "@/routes/tasks/types";
+
+type RunLike = { workflow_run_id: string; status: Status };
+
+// The backend stamps first_run_at only when a run reaches a final status, so
+// refresh onboarding once per finalized run so the provider can observe the
+// milestone and emit first_run_completed. This includes runs already final on
+// first observation (a fast validation/proxy failure, or opening a completed
+// run); a repeat invalidation is a no-op since the provider only emits on the
+// null -> set transition. Repeat renders of the same run do not refetch.
+function useRefreshOnboardingOnRunCompletion(
+ workflowRun: RunLike | undefined,
+): void {
+ const queryClient = useQueryClient();
+ const refreshedRunRef = useRef<string | null>(null);
+
+ useEffect(() => {
+ if (!workflowRun || !statusIsFinalized(workflowRun)) {
+ return;
+ }
+ const runId = workflowRun.workflow_run_id;
+ if (refreshedRunRef.current === runId) {
+ return;
+ }
+ refreshedRunRef.current = runId;
+ queryClient.invalidateQueries({ queryKey: ["userOnboarding"] });
+ }, [workflowRun, queryClient]);
+}
+
+export { useRefreshOnboardingOnRunCompletion };
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowTagsBatchQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowTagsBatchQuery.ts
index 38a062b6d..e9e12b398 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowTagsBatchQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowTagsBatchQuery.ts
@@ -1,7 +1,11 @@
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery } from "@tanstack/react-query";
-import type { Tag, WorkflowTagsBatchResponse } from "../types/tagTypes";
+import {
+ normalizeWorkflowTags,
+ type Tag,
+ type WorkflowTagsBatchResponse,
+} from "../types/tagTypes";
// Backend rejects more than this many IDs per request (_BATCH_TAGS_MAX_WPIDS); the
// list page size is unbounded, so chunk rather than slice to avoid dropping tags.
@@ -32,9 +36,16 @@ function useWorkflowTagsBatchQuery(workflowPermanentIds: Array<string>) {
}),
);
// Chunks are disjoint slices, so wpid keys never collide on merge.
+ // Normalize per workflow: the SKY-10683 backend/frontend split shipped a
+ // response-shape skew once already; never let one reach render again.
const merged: Record<string, Array<Tag>> = {};
for (const response of responses) {
- Object.assign(merged, response);
+ if (response === null || typeof response !== "object") {
+ continue;
+ }
+ for (const [wpid, tags] of Object.entries(response)) {
+ merged[wpid] = normalizeWorkflowTags(tags);
+ }
}
return merged;
},
diff --git a/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts b/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts
index 0de21a4cd..a8b675871 100644
--- a/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/browserSessionTypes.ts
@@ -14,6 +14,7 @@ interface BrowserSession {
timeout: number | null;
extensions?: BrowserSessionExtension[] | null;
browser_type?: BrowserSessionType | null;
+ generate_browser_profile?: boolean;
vnc_streaming_supported: boolean;
}
diff --git a/skyvern-frontend/src/routes/workflows/types/tagTypes.test.ts b/skyvern-frontend/src/routes/workflows/types/tagTypes.test.ts
index 6656b942f..70a8b94b9 100644
--- a/skyvern-frontend/src/routes/workflows/types/tagTypes.test.ts
+++ b/skyvern-frontend/src/routes/workflows/types/tagTypes.test.ts
@@ -1,6 +1,7 @@
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
import {
+ normalizeWorkflowTags,
parseTagFilter,
parseTagFilterTerm,
parseTagInput,
@@ -143,6 +144,52 @@ describe("sortTags", () => {
});
});
+describe("normalizeWorkflowTags", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("passes a well-formed tag list through unchanged", () => {
+ const tags: Array<Tag> = [
+ { key: null, value: "prod" },
+ { key: "env", value: "dev" },
+ ];
+ expect(normalizeWorkflowTags(tags)).toEqual(tags);
+ });
+
+ it("drops malformed list entries and warns instead of throwing", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ const skewed = [
+ { key: "env", value: "prod" },
+ { key: "bad", value: { key: "nested", value: "object" } },
+ { value: 42 },
+ null,
+ "loose-string",
+ ];
+ expect(normalizeWorkflowTags(skewed)).toEqual([
+ { key: "env", value: "prod" },
+ ]);
+ expect(warn).toHaveBeenCalledTimes(1);
+ });
+
+ it("converts a legacy key-to-value record and warns", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ expect(normalizeWorkflowTags({ env: "prod", team: "growth" })).toEqual([
+ { key: "env", value: "prod" },
+ { key: "team", value: "growth" },
+ ]);
+ expect(warn).toHaveBeenCalledTimes(1);
+ });
+
+ it("degrades null, undefined, and junk to an empty list without throwing", () => {
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ expect(normalizeWorkflowTags(null)).toEqual([]);
+ expect(normalizeWorkflowTags(undefined)).toEqual([]);
+ expect(normalizeWorkflowTags("env:prod")).toEqual([]);
+ expect(normalizeWorkflowTags(7)).toEqual([]);
+ });
+});
+
describe("tagElementKey", () => {
it("namespaces standalone vs grouped so a null key can't collide with an empty key", () => {
expect(tagElementKey({ key: null, value: "foo" })).toBe("label:foo");
diff --git a/skyvern-frontend/src/routes/workflows/types/tagTypes.ts b/skyvern-frontend/src/routes/workflows/types/tagTypes.ts
index 7c197e777..20c2949c1 100644
--- a/skyvern-frontend/src/routes/workflows/types/tagTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/tagTypes.ts
@@ -238,6 +238,45 @@ export function serializeTagFilterTerm(term: TagFilterTerm): string {
return `${term.key}:${term.value}`;
}
+export function isTag(value: unknown): value is Tag {
+ if (typeof value !== "object" || value === null) {
+ return false;
+ }
+ const candidate = value as { key?: unknown; value?: unknown };
+ return (
+ (candidate.key === null || typeof candidate.key === "string") &&
+ typeof candidate.value === "string"
+ );
+}
+
+// Boundary guard for tag payloads. Accepts the current wire shape
+// (Array<Tag>) plus the pre-SKY-10683 key->value record, and degrades
+// anything else to [] - a shape skew must never throw mid-render (React #31)
+// and take the whole route down with the error boundary.
+export function normalizeWorkflowTags(input: unknown): Array<Tag> {
+ if (Array.isArray(input)) {
+ const valid = input.filter(isTag);
+ if (valid.length !== input.length) {
+ console.warn(
+ `[tags] dropped ${input.length - valid.length} malformed tag entries`,
+ );
+ }
+ return valid;
+ }
+ if (input !== null && typeof input === "object") {
+ console.warn("[tags] got a legacy record tag payload; converting");
+ return Object.entries(input)
+ .filter((entry): entry is [string, string] => {
+ return typeof entry[1] === "string";
+ })
+ .map(([key, value]) => ({ key, value }));
+ }
+ if (input !== null && input !== undefined) {
+ console.warn(`[tags] ignoring tag payload of type ${typeof input}`);
+ }
+ return [];
+}
+
// Stable React key for a tag. Standalone labels and grouped tags live in
// disjoint namespaces so a null key can't collide with an (invalid) empty key.
export function tagElementKey(tag: Tag): string {
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WebhookDeliveryStatus.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WebhookDeliveryStatus.tsx
new file mode 100644
index 000000000..2f1b6d3a6
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WebhookDeliveryStatus.tsx
@@ -0,0 +1,48 @@
+import type { WebhookDeliveryStatus as WebhookDeliveryStatusValue } from "@/api/types";
+import { cn } from "@/util/utils";
+
+type Props = {
+ webhookDeliveryStatus?: WebhookDeliveryStatusValue | null;
+ webhookFailureReason?: string | null;
+};
+
+function WebhookDeliveryStatus({
+ webhookDeliveryStatus,
+ webhookFailureReason,
+}: Props) {
+ const isFailed = webhookDeliveryStatus === "failed";
+ const isPending = webhookDeliveryStatus === "pending";
+
+ if (!isFailed && !isPending) {
+ return null;
+ }
+
+ const title = isFailed
+ ? "Webhook Failure Reason"
+ : "Webhook Delivery Pending";
+ const message = isFailed
+ ? (webhookFailureReason ?? "Skyvern recorded a webhook delivery failure.")
+ : "Skyvern has started webhook delivery or has not recorded an automatic webhook delivery attempt yet. Delivery will retry automatically.";
+
+ return (
+ <div
+ role="status"
+ className={cn("rounded border bg-slate-elevation2 p-6", {
+ "border-yellow-600/50": isFailed,
+ "border-sky-500/50": isPending,
+ })}
+ >
+ <h3 className="mb-4 text-lg font-bold">{title}</h3>
+ <div
+ className={cn("text-sm", {
+ "text-yellow-600": isFailed,
+ "text-slate-200": isPending,
+ })}
+ >
+ {message}
+ </div>
+ </div>
+ );
+}
+
+export { WebhookDeliveryStatus };
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx
index aaecc086a..d8785675b 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx
@@ -175,9 +175,6 @@ function WorkflowRunOutput() {
? getBlockDownloadedFileUrls(activeBlock.output, allFileUrls)
: allFileUrls;
const observerOutput = workflowRun?.task_v2?.output;
- const webhookFailureReasonData =
- workflowRun?.task_v2?.webhook_failure_reason ??
- workflowRun?.webhook_failure_reason;
const blockContextKey = activeBlock
? `block:${activeBlock.workflow_run_id}:${activeBlock.workflow_run_block_id}`
@@ -196,16 +193,6 @@ function WorkflowRunOutput() {
return (
<div className="space-y-5">
- {webhookFailureReasonData ? (
- <div className="rounded bg-slate-elevation2 p-6">
- <div className="space-y-4">
- <h1 className="text-lg font-bold">Webhook Failure Reason</h1>
- <div className="space-y-2 text-yellow-600">
- {webhookFailureReasonData}
- </div>
- </div>
- </div>
- ) : null}
{activeBlock ? (
<div className="rounded bg-slate-elevation2 p-6">
<div className="space-y-4">
diff --git a/skyvern-frontend/src/store/BlockSidebarOnboardingStore.test.ts b/skyvern-frontend/src/store/BlockSidebarOnboardingStore.test.ts
deleted file mode 100644
index 7c9e438ca..000000000
--- a/skyvern-frontend/src/store/BlockSidebarOnboardingStore.test.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-// @vitest-environment jsdom
-
-import { beforeEach, describe, expect, test } from "vitest";
-
-import {
- useBlockSidebarOnboardingStore,
- BLOCK_SIDEBAR_ONBOARDING_STORAGE_KEY,
-} from "./BlockSidebarOnboardingStore";
-
-beforeEach(() => {
- localStorage.clear();
- useBlockSidebarOnboardingStore.getState().reset();
-});
-
-describe("BlockSidebarOnboardingStore", () => {
- test("hasSeenMigration starts false", () => {
- expect(useBlockSidebarOnboardingStore.getState().hasSeenMigration).toBe(
- false,
- );
- });
-
- test("markSeen sets the flag and persists", () => {
- useBlockSidebarOnboardingStore.getState().markSeen();
- expect(useBlockSidebarOnboardingStore.getState().hasSeenMigration).toBe(
- true,
- );
- const raw = localStorage.getItem(BLOCK_SIDEBAR_ONBOARDING_STORAGE_KEY);
- expect(raw).not.toBeNull();
- expect(JSON.parse(raw!)).toMatchObject({
- state: { hasSeenMigration: true },
- });
- });
-
- test("reset returns flag to false", () => {
- const store = useBlockSidebarOnboardingStore.getState();
- store.markSeen();
- store.reset();
- expect(useBlockSidebarOnboardingStore.getState().hasSeenMigration).toBe(
- false,
- );
- });
-});
diff --git a/skyvern-frontend/src/store/BlockSidebarOnboardingStore.ts b/skyvern-frontend/src/store/BlockSidebarOnboardingStore.ts
deleted file mode 100644
index ee647a479..000000000
--- a/skyvern-frontend/src/store/BlockSidebarOnboardingStore.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import { create } from "zustand";
-import { persist, createJSONStorage } from "zustand/middleware";
-
-export const BLOCK_SIDEBAR_ONBOARDING_STORAGE_KEY =
- "skyvern.blockSidebarOnboarded";
-
-type BlockSidebarOnboardingStore = {
- hasSeenMigration: boolean;
- markSeen: () => void;
- reset: () => void;
-};
-
-const useBlockSidebarOnboardingStore = create<BlockSidebarOnboardingStore>()(
- persist(
- (set) => ({
- hasSeenMigration: false,
- markSeen: () => set({ hasSeenMigration: true }),
- reset: () => set({ hasSeenMigration: false }),
- }),
- {
- name: BLOCK_SIDEBAR_ONBOARDING_STORAGE_KEY,
- storage: createJSONStorage(() => localStorage),
- partialize: (state) => ({ hasSeenMigration: state.hasSeenMigration }),
- },
- ),
-);
-
-export { useBlockSidebarOnboardingStore };
diff --git a/skyvern-frontend/src/store/ProductTourStore.ts b/skyvern-frontend/src/store/ProductTourStore.ts
new file mode 100644
index 000000000..38b04f219
--- /dev/null
+++ b/skyvern-frontend/src/store/ProductTourStore.ts
@@ -0,0 +1,15 @@
+import { create } from "zustand";
+
+type ProductTourStore = {
+ requestedAt: number | null;
+ requestTour: () => void;
+ clearRequest: () => void;
+};
+
+const useProductTourStore = create<ProductTourStore>((set) => ({
+ requestedAt: null,
+ requestTour: () => set({ requestedAt: Date.now() }),
+ clearRequest: () => set({ requestedAt: null }),
+}));
+
+export { useProductTourStore };
diff --git a/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx b/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx
new file mode 100644
index 000000000..cfb6fcfea
--- /dev/null
+++ b/skyvern-frontend/src/store/onboarding/OnboardingProvider.test.tsx
@@ -0,0 +1,144 @@
+// @vitest-environment jsdom
+import {
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const { mockGet, mockPost } = vi.hoisted(() => ({
+ mockGet: vi.fn(),
+ mockPost: vi.fn(),
+}));
+
+vi.mock("@clerk/clerk-react", () => ({
+ useAuth: () => ({ isSignedIn: true }),
+}));
+
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => () => Promise.resolve("test-token"),
+}));
+
+vi.mock("@/api/AxiosClient", () => ({
+ getClient: () => Promise.resolve({ get: mockGet, post: mockPost }),
+}));
+
+vi.mock("@/util/onboarding/OnboardingTelemetry", () => ({
+ OnboardingTelemetry: {
+ error: vi.fn(),
+ registerVariant: vi.fn(),
+ firstWorkflowCreated: vi.fn(),
+ firstRunCompleted: vi.fn(),
+ },
+}));
+
+import { OnboardingProvider } from "./OnboardingProvider";
+import type { OnboardingStateResponse } from "./types";
+import { useOnboardingState } from "./useOnboardingState";
+
+const DISMISSED_AT = "2026-01-01T00:00:00.000Z";
+
+function freshResponse(): OnboardingStateResponse {
+ return {
+ onboarding_state: {
+ tour_completed_at: null,
+ modal_dismissed_at: null,
+ first_save_at: null,
+ first_run_at: null,
+ ab_variant: null,
+ user_intent: null,
+ seen_canvas: null,
+ seen_node_adder: null,
+ seen_sidebar: null,
+ seen_save_run: null,
+ },
+ launch_date_at_signup: "2025-01-01T00:00:00Z",
+ };
+}
+
+function Consumer() {
+ const { state, updateState } = useOnboardingState();
+ return (
+ <div>
+ <span data-testid="dismissed">{String(state?.modal_dismissed_at)}</span>
+ <button onClick={() => updateState({ modal_dismissed_at: DISMISSED_AT })}>
+ dismiss
+ </button>
+ </div>
+ );
+}
+
+function renderProvider() {
+ const queryClient = new QueryClient({
+ defaultOptions: {
+ queries: { retry: false },
+ mutations: { retry: false },
+ },
+ });
+ return render(
+ <QueryClientProvider client={queryClient}>
+ <OnboardingProvider>
+ <Consumer />
+ </OnboardingProvider>
+ </QueryClientProvider>,
+ );
+}
+
+afterEach(() => {
+ cleanup();
+ vi.clearAllMocks();
+});
+
+describe("OnboardingProvider write resilience", () => {
+ it("keeps the optimistic dismissal when the write fails", async () => {
+ mockGet.mockResolvedValue({ data: freshResponse() });
+ mockPost.mockRejectedValue(new Error("405 Method Not Allowed"));
+
+ renderProvider();
+ await waitFor(() =>
+ expect(screen.getByTestId("dismissed").textContent).toBe("null"),
+ );
+
+ fireEvent.click(screen.getByText("dismiss"));
+
+ // Optimistic update applies immediately.
+ await waitFor(() =>
+ expect(screen.getByTestId("dismissed").textContent).toBe(DISMISSED_AT),
+ );
+ await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(1));
+
+ // A failed write must NOT roll back the dismissal nor refetch the stale state.
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(screen.getByTestId("dismissed").textContent).toBe(DISMISSED_AT);
+ expect(mockGet).toHaveBeenCalledTimes(1);
+ });
+
+ it("refetches once after a successful write", async () => {
+ const dismissedResponse: OnboardingStateResponse = {
+ ...freshResponse(),
+ onboarding_state: {
+ ...freshResponse().onboarding_state,
+ modal_dismissed_at: DISMISSED_AT,
+ },
+ };
+ // First GET is fresh; once the write succeeds the server persists, so the
+ // refetch returns the dismissed state.
+ mockGet
+ .mockResolvedValueOnce({ data: freshResponse() })
+ .mockResolvedValue({ data: dismissedResponse });
+ mockPost.mockResolvedValue({ data: dismissedResponse });
+
+ renderProvider();
+ await waitFor(() => expect(mockGet).toHaveBeenCalledTimes(1));
+
+ fireEvent.click(screen.getByText("dismiss"));
+
+ await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(1));
+ // onSuccess invalidates -> GET refetches exactly once more.
+ await waitFor(() => expect(mockGet).toHaveBeenCalledTimes(2));
+ expect(screen.getByTestId("dismissed").textContent).toBe(DISMISSED_AT);
+ });
+});
diff --git a/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx b/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx
new file mode 100644
index 000000000..69fb16cd6
--- /dev/null
+++ b/skyvern-frontend/src/store/onboarding/OnboardingProvider.tsx
@@ -0,0 +1,119 @@
+import { useCallback, useEffect, useRef } from "react";
+import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
+import { useAuth } from "@clerk/clerk-react";
+import { getClient } from "@/api/AxiosClient";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { OnboardingContext } from "./useOnboardingState";
+import { OnboardingTelemetry } from "@/util/onboarding/OnboardingTelemetry";
+import type { OnboardingStatePatch, OnboardingStateResponse } from "./types";
+
+const QUERY_KEY = ["userOnboarding"] as const;
+
+const PRODUCT_LAUNCH_DATE = "2024-10-01T00:00:00Z";
+
+type Props = {
+ children: React.ReactNode;
+};
+
+function OnboardingProvider({ children }: Readonly<Props>) {
+ const credentialGetter = useCredentialGetter();
+ const { isSignedIn } = useAuth();
+ const queryClient = useQueryClient();
+
+ const { data, isLoading } = useQuery<OnboardingStateResponse>({
+ queryKey: QUERY_KEY,
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ const response = await client.get<OnboardingStateResponse>(
+ "/users/me/onboarding",
+ );
+ return response.data;
+ },
+ enabled: !!credentialGetter && isSignedIn === true,
+ });
+
+ const mutation = useMutation<
+ OnboardingStateResponse,
+ unknown,
+ OnboardingStatePatch
+ >({
+ mutationFn: async (patch) => {
+ const client = await getClient(credentialGetter);
+ const response = await client.post<OnboardingStateResponse>(
+ "/users/me/onboarding",
+ patch,
+ );
+ return response.data;
+ },
+ onMutate: async (patch) => {
+ await queryClient.cancelQueries({ queryKey: QUERY_KEY });
+ const previous =
+ queryClient.getQueryData<OnboardingStateResponse>(QUERY_KEY);
+ if (previous) {
+ queryClient.setQueryData<OnboardingStateResponse>(QUERY_KEY, {
+ ...previous,
+ onboarding_state: { ...previous.onboarding_state, ...patch },
+ });
+ }
+ },
+ onError: () => {
+ // Best-effort persistence: keep the optimistic state so a failed write
+ // (offline / 5xx / 405) can never resurrect a dismissed modal or tour.
+ OnboardingTelemetry.error("dashboard");
+ },
+ onSuccess: () => {
+ queryClient.invalidateQueries({ queryKey: QUERY_KEY });
+ },
+ });
+
+ const isNewUser =
+ data?.launch_date_at_signup != null &&
+ new Date(data.launch_date_at_signup) >= new Date(PRODUCT_LAUNCH_DATE);
+
+ const abVariant = data?.onboarding_state.ab_variant ?? null;
+
+ useEffect(() => {
+ if (abVariant) {
+ OnboardingTelemetry.registerVariant(abVariant);
+ }
+ }, [abVariant]);
+
+ const prevSaveAt = useRef<string | null | undefined>(undefined);
+ const prevRunAt = useRef<string | null | undefined>(undefined);
+ useEffect(() => {
+ const s = data?.onboarding_state;
+ if (!s) return;
+ // emit the activation-funnel milestones once, on the observed null -> set transition
+ if (prevSaveAt.current === null && s.first_save_at !== null) {
+ OnboardingTelemetry.firstWorkflowCreated("dashboard");
+ }
+ if (prevRunAt.current === null && s.first_run_at !== null) {
+ OnboardingTelemetry.firstRunCompleted("dashboard");
+ }
+ prevSaveAt.current = s.first_save_at;
+ prevRunAt.current = s.first_run_at;
+ }, [data?.onboarding_state]);
+
+ const updateState = useCallback(
+ (patch: OnboardingStatePatch) => {
+ mutation.mutate(patch);
+ },
+ [mutation],
+ );
+
+ return (
+ <OnboardingContext.Provider
+ value={{
+ state: data?.onboarding_state ?? null,
+ isLoading,
+ updateState,
+ isNewUser,
+ abVariant,
+ }}
+ >
+ {children}
+ </OnboardingContext.Provider>
+ );
+}
+
+export { OnboardingProvider };
diff --git a/skyvern-frontend/src/store/onboarding/types.ts b/skyvern-frontend/src/store/onboarding/types.ts
new file mode 100644
index 000000000..11912d185
--- /dev/null
+++ b/skyvern-frontend/src/store/onboarding/types.ts
@@ -0,0 +1,22 @@
+export type OnboardingState = {
+ tour_completed_at: string | null;
+ modal_dismissed_at: string | null;
+ first_save_at: string | null;
+ first_run_at: string | null;
+ ab_variant: string | null;
+ user_intent: string | null;
+ seen_canvas: boolean | null;
+ seen_node_adder: boolean | null;
+ seen_sidebar: boolean | null;
+ seen_save_run: boolean | null;
+ seen_hint_block?: boolean | null;
+ seen_hint_run?: boolean | null;
+ seen_hint_template?: boolean | null;
+};
+
+export type OnboardingStateResponse = {
+ onboarding_state: OnboardingState;
+ launch_date_at_signup: string | null;
+};
+
+export type OnboardingStatePatch = Partial<OnboardingState>;
diff --git a/skyvern-frontend/src/store/onboarding/useOnboardingState.ts b/skyvern-frontend/src/store/onboarding/useOnboardingState.ts
new file mode 100644
index 000000000..475466a4a
--- /dev/null
+++ b/skyvern-frontend/src/store/onboarding/useOnboardingState.ts
@@ -0,0 +1,29 @@
+import { createContext, useContext } from "react";
+import type { OnboardingState, OnboardingStatePatch } from "./types";
+
+type OnboardingContextValue = {
+ state: OnboardingState | null;
+ isLoading: boolean;
+ updateState: (patch: OnboardingStatePatch) => void;
+ isNewUser: boolean;
+ abVariant: string | null;
+};
+
+const OnboardingContext = createContext<OnboardingContextValue | null>(null);
+
+function useOnboardingState(): OnboardingContextValue {
+ const context = useContext(OnboardingContext);
+ if (!context) {
+ throw new Error(
+ "useOnboardingState must be used within OnboardingProvider",
+ );
+ }
+ return context;
+}
+
+function useOnboardingStateOptional(): OnboardingContextValue | null {
+ return useContext(OnboardingContext);
+}
+
+export { OnboardingContext, useOnboardingState, useOnboardingStateOptional };
+export type { OnboardingContextValue };
diff --git a/skyvern-frontend/src/util/bulkResultToast.test.ts b/skyvern-frontend/src/util/bulkResultToast.test.ts
new file mode 100644
index 000000000..4871aa8b8
--- /dev/null
+++ b/skyvern-frontend/src/util/bulkResultToast.test.ts
@@ -0,0 +1,110 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { bulkResultToast } from "./bulkResultToast";
+import { toast } from "@/components/ui/use-toast";
+
+vi.mock("@/components/ui/use-toast", () => ({
+ toast: vi.fn(),
+}));
+
+const titles = {
+ successTitle: (count: number) => `Cloned ${count}.`,
+ failureTitle: (count: number) => `Failed ${count}.`,
+ partialTitle: (succeeded: number, failed: number) =>
+ `Cloned ${succeeded}. ${failed} failed.`,
+};
+
+function rejected(message: string): PromiseSettledResult<unknown> {
+ return { status: "rejected", reason: new Error(message) };
+}
+
+const fulfilled: PromiseSettledResult<unknown> = {
+ status: "fulfilled",
+ value: undefined,
+};
+
+describe("bulkResultToast", () => {
+ beforeEach(() => {
+ vi.mocked(toast).mockClear();
+ });
+
+ it("uses success variant when everything succeeded", () => {
+ bulkResultToast({ succeeded: 3, total: 3, ...titles });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({ title: "Cloned 3.", variant: "success" }),
+ );
+ });
+
+ it("uses destructive variant when everything failed", () => {
+ bulkResultToast({ succeeded: 0, total: 2, ...titles });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({ title: "Failed 2.", variant: "destructive" }),
+ );
+ });
+
+ it("uses warning variant for partial success", () => {
+ bulkResultToast({ succeeded: 2, total: 3, ...titles });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({
+ title: "Cloned 2. 1 failed.",
+ variant: "warning",
+ }),
+ );
+ });
+
+ it("surfaces the first rejection message as the description", () => {
+ bulkResultToast({
+ succeeded: 1,
+ total: 3,
+ results: [fulfilled, rejected("403 Forbidden"), rejected("timeout")],
+ ...titles,
+ });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({ description: "403 Forbidden" }),
+ );
+ });
+
+ it("keeps the warning variant on partial success without results", () => {
+ bulkResultToast({ succeeded: 1, total: 3, ...titles });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({ variant: "warning", description: undefined }),
+ );
+ });
+
+ it("prefers the API error detail over the generic axios message", () => {
+ const axiosLike: PromiseSettledResult<unknown> = {
+ status: "rejected",
+ reason: Object.assign(new Error("Request failed with status code 403"), {
+ response: { data: { detail: "Folder is read-only" } },
+ }),
+ };
+ bulkResultToast({
+ succeeded: 0,
+ total: 1,
+ results: [axiosLike],
+ ...titles,
+ });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({ description: "Folder is read-only" }),
+ );
+ });
+
+ it("omits the description when nothing failed", () => {
+ bulkResultToast({
+ succeeded: 1,
+ total: 1,
+ results: [fulfilled],
+ ...titles,
+ });
+
+ expect(toast).toHaveBeenCalledWith(
+ expect.objectContaining({ description: undefined }),
+ );
+ });
+});
diff --git a/skyvern-frontend/src/util/bulkResultToast.ts b/skyvern-frontend/src/util/bulkResultToast.ts
new file mode 100644
index 000000000..0db8662f7
--- /dev/null
+++ b/skyvern-frontend/src/util/bulkResultToast.ts
@@ -0,0 +1,67 @@
+import { toast } from "@/components/ui/use-toast";
+
+function firstRejectionMessage(
+ results: PromiseSettledResult<unknown>[],
+): string | undefined {
+ const rejection = results.find(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ if (!rejection) {
+ return undefined;
+ }
+ const reason: unknown = rejection.reason;
+ if (reason && typeof reason === "object") {
+ // API errors carry the useful text in the FastAPI detail, not the generic
+ // axios "Request failed with status code N" message.
+ const detail = (reason as { response?: { data?: { detail?: unknown } } })
+ .response?.data?.detail;
+ if (typeof detail === "string") {
+ return detail;
+ }
+ const message = (reason as { message?: unknown }).message;
+ return typeof message === "string" ? message : undefined;
+ }
+ return typeof reason === "string" ? reason : undefined;
+}
+
+function bulkResultToast({
+ succeeded,
+ total,
+ results,
+ successTitle,
+ failureTitle,
+ partialTitle,
+}: {
+ succeeded: number;
+ total: number;
+ results?: PromiseSettledResult<unknown>[];
+ successTitle: (count: number) => string;
+ failureTitle: (count: number) => string;
+ partialTitle: (succeeded: number, failed: number) => string;
+}) {
+ const failed = total - succeeded;
+ const description = results ? firstRejectionMessage(results) : undefined;
+ if (failed === 0) {
+ toast({
+ title: successTitle(succeeded),
+ variant: "success",
+ description,
+ });
+ return;
+ }
+ if (succeeded === 0) {
+ toast({
+ title: failureTitle(failed),
+ variant: "destructive",
+ description,
+ });
+ return;
+ }
+ toast({
+ title: partialTitle(succeeded, failed),
+ variant: "warning",
+ description,
+ });
+}
+
+export { bulkResultToast };
diff --git a/skyvern-frontend/src/util/env.test.ts b/skyvern-frontend/src/util/env.test.ts
index 57895d243..004fc1ac4 100644
--- a/skyvern-frontend/src/util/env.test.ts
+++ b/skyvern-frontend/src/util/env.test.ts
@@ -1,11 +1,12 @@
import { afterEach, describe, expect, it, vi } from "vitest";
-async function loadEnv(apiKey: string | null = null) {
+async function loadEnv(apiKey: string | null = null, streamingMode = "") {
vi.resetModules();
vi.stubEnv("VITE_API_BASE_URL", "http://localhost:8000/api/v1");
vi.stubEnv("VITE_ARTIFACT_API_BASE_URL", "http://localhost:9090");
vi.stubEnv("VITE_ENVIRONMENT", "test");
vi.stubEnv("VITE_WSS_BASE_URL", "ws://localhost:8000/api/v1");
+ vi.stubEnv("VITE_BROWSER_STREAMING_MODE", streamingMode);
if (apiKey) {
vi.stubEnv("VITE_SKYVERN_API_KEY", apiKey);
} else {
@@ -56,9 +57,7 @@ describe("browserStreamingMode", () => {
});
it("uses the configured streaming mode when present", async () => {
- vi.stubEnv("VITE_BROWSER_STREAMING_MODE", "CDP");
-
- const { browserStreamingMode } = await loadEnv();
+ const { browserStreamingMode } = await loadEnv(null, "CDP");
expect(browserStreamingMode).toBe("cdp");
});
diff --git a/skyvern-frontend/src/util/featureFlags.ts b/skyvern-frontend/src/util/featureFlags.ts
index 2729b6e82..f019f2f77 100644
--- a/skyvern-frontend/src/util/featureFlags.ts
+++ b/skyvern-frontend/src/util/featureFlags.ts
@@ -1 +1,17 @@
export const ANALYTICS_DASHBOARD_FLAG = "ANALYTICS_DASHBOARD";
+
+/**
+ * Gates the workflow editor onboarding tour and A/B experiment.
+ * When disabled, users skip the tour and experiment routing entirely.
+ *
+ * Ramp plan (see rolloutConfig.ts for structured constants):
+ * 0% - ship day
+ * 10% - hold 1 day, gate check
+ * 50% - hold 3 days, gate check
+ * 100% - GA
+ *
+ * PostHog type: multivariate string flag. Control / not-enrolled reads as no
+ * variant; enabled arms are "template-first" | "copilot-first". Onboarding
+ * surfaces gate on a resolved variant (isABVariant), so 0% or rollback hides them.
+ */
+export const EDITOR_ONBOARDING_TOUR_FLAG = "EDITOR_ONBOARDING_TOUR";
diff --git a/skyvern-frontend/src/util/onboarding/OnboardingTelemetry.test.ts b/skyvern-frontend/src/util/onboarding/OnboardingTelemetry.test.ts
new file mode 100644
index 000000000..b2371f67d
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/OnboardingTelemetry.test.ts
@@ -0,0 +1,31 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import posthog from "posthog-js";
+import { OnboardingTelemetry } from "./OnboardingTelemetry";
+
+vi.mock("posthog-js", () => ({
+ default: { capture: vi.fn(), register: vi.fn() },
+}));
+
+describe("OnboardingTelemetry hint events", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("hintShown captures a layer-2 event with the hint id", () => {
+ OnboardingTelemetry.hintShown("editor", "add-another-block");
+ expect(posthog.capture).toHaveBeenCalledWith("onboarding.hint_shown", {
+ surface: "editor",
+ hint_id: "add-another-block",
+ layer: 2,
+ });
+ });
+
+ it("hintDismissed captures a layer-2 event with the hint id", () => {
+ OnboardingTelemetry.hintDismissed("runs", "run-recording");
+ expect(posthog.capture).toHaveBeenCalledWith("onboarding.hint_dismissed", {
+ surface: "runs",
+ hint_id: "run-recording",
+ layer: 2,
+ });
+ });
+});
diff --git a/skyvern-frontend/src/util/onboarding/OnboardingTelemetry.ts b/skyvern-frontend/src/util/onboarding/OnboardingTelemetry.ts
new file mode 100644
index 000000000..93f91df54
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/OnboardingTelemetry.ts
@@ -0,0 +1,196 @@
+import posthog from "posthog-js";
+
+type Surface = "dashboard" | "editor" | "runs" | "settings" | "api_docs";
+
+type TourLayer = 1 | 2;
+
+function capture(event: string, properties: Record<string, unknown>): void {
+ try {
+ posthog.capture(event, properties);
+ } catch {
+ // PostHog may be unavailable in tests or before init.
+ }
+}
+
+// -- Flow events --
+
+function flowStarted(surface: Surface): void {
+ capture("onboarding.flow_started", { surface });
+}
+
+function flowCompleted(surface: Surface): void {
+ capture("onboarding.flow_completed", { surface });
+}
+
+function dropOff(surface: Surface, step: string): void {
+ capture("onboarding.drop_off", { surface, step });
+}
+
+// -- Tour events --
+
+function tourStarted(surface: Surface): void {
+ capture("onboarding.tour_started", { surface });
+}
+
+function tourStepViewed(
+ surface: Surface,
+ stepName: string,
+ stepIndex: number,
+ layer: TourLayer,
+): void {
+ capture("onboarding.tour_step_viewed", {
+ surface,
+ step_name: stepName,
+ step_index: stepIndex,
+ layer,
+ });
+}
+
+function tourCompleted(surface: Surface): void {
+ capture("onboarding.tour_completed", { surface });
+}
+
+function tourSkipped(surface: Surface, atStep: string): void {
+ capture("onboarding.tour_skipped", { surface, at_step: atStep });
+}
+
+function stepCompleted(surface: Surface, step: string): void {
+ capture("onboarding.step_completed", { surface, step });
+}
+
+function tourDismissed(surface: Surface, lastStep: string): void {
+ capture("onboarding.tour_dismissed", { surface, last_step: lastStep });
+}
+
+// -- Modal events --
+
+function modalOpened(surface: Surface): void {
+ capture("onboarding.modal_opened", { surface });
+}
+
+function modalTemplateSelected(
+ surface: Surface,
+ templateId: string,
+ intent: string,
+): void {
+ capture("onboarding.modal_template_selected", {
+ surface,
+ template_id: templateId,
+ intent,
+ });
+}
+
+function modalCopilotClicked(
+ surface: Surface,
+ intent: string,
+ promptText: string,
+): void {
+ capture("onboarding.modal_copilot_clicked", {
+ surface,
+ intent,
+ // capture only length, never the raw prompt - it can contain customer URLs / PII
+ prompt_length: promptText.length,
+ });
+}
+
+function modalSkipped(surface: Surface): void {
+ capture("onboarding.modal_skipped", { surface });
+}
+
+// -- Activation milestones --
+
+function firstWorkflowCreated(surface: Surface): void {
+ capture("onboarding.first_workflow_created", { surface });
+}
+
+function firstRunCompleted(surface: Surface): void {
+ capture("onboarding.first_run_completed", { surface });
+}
+
+function firstApiCall(surface: Surface): void {
+ capture("onboarding.first_api_call", { surface });
+}
+
+function firstScheduleCreated(surface: Surface): void {
+ capture("onboarding.first_schedule_created", { surface });
+}
+
+// -- Empty state events --
+
+function emptyStateViewed(surface: Surface): void {
+ capture("onboarding.empty_state_viewed", { surface });
+}
+
+function emptyStateCTAClicked(surface: Surface, action: string): void {
+ capture("onboarding.empty_state_cta_clicked", { surface, action });
+}
+
+// -- Experiment --
+
+function abVariantAssigned(surface: Surface, variant: string): void {
+ capture("onboarding.ab_variant_assigned", { surface, variant });
+}
+
+function registerVariant(variant: string): void {
+ try {
+ // Super property "variant" so every onboarding event can be split by arm (matches the dashboard breakdowns).
+ posthog.register({ variant });
+ } catch {
+ // PostHog may be unavailable in tests or before init.
+ }
+}
+
+// -- Error events (rollback guardrails) --
+
+function error(surface: Surface): void {
+ capture("onboarding.error", { surface });
+}
+
+function modalRenderError(surface: Surface): void {
+ capture("onboarding.modal_render_error", { surface });
+}
+
+function tourError(surface: Surface): void {
+ capture("onboarding.tour_error", { surface });
+}
+
+// -- Contextual hint events (Layer 2) --
+
+function hintShown(surface: Surface, hintId: string): void {
+ capture("onboarding.hint_shown", { surface, hint_id: hintId, layer: 2 });
+}
+
+function hintDismissed(surface: Surface, hintId: string): void {
+ capture("onboarding.hint_dismissed", { surface, hint_id: hintId, layer: 2 });
+}
+
+export const OnboardingTelemetry = {
+ flowStarted,
+ flowCompleted,
+ dropOff,
+ tourStarted,
+ tourStepViewed,
+ tourCompleted,
+ tourSkipped,
+ stepCompleted,
+ tourDismissed,
+ modalOpened,
+ modalTemplateSelected,
+ modalCopilotClicked,
+ modalSkipped,
+ emptyStateViewed,
+ emptyStateCTAClicked,
+ firstWorkflowCreated,
+ firstRunCompleted,
+ firstApiCall,
+ firstScheduleCreated,
+ abVariantAssigned,
+ registerVariant,
+ error,
+ modalRenderError,
+ tourError,
+ hintShown,
+ hintDismissed,
+} as const;
+
+export type { Surface, TourLayer };
diff --git a/skyvern-frontend/src/util/onboarding/contextualHints.test.ts b/skyvern-frontend/src/util/onboarding/contextualHints.test.ts
new file mode 100644
index 000000000..857a598ab
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/contextualHints.test.ts
@@ -0,0 +1,56 @@
+import { describe, it, expect } from "vitest";
+import { HINT_REGISTRY } from "./contextualHints";
+
+describe("HINT_REGISTRY", () => {
+ it("contains the three first-cut hints", () => {
+ expect(HINT_REGISTRY.map((h) => h.id)).toEqual([
+ "add-another-block",
+ "run-recording",
+ "start-template",
+ ]);
+ });
+
+ it("has unique ids and unique seen keys", () => {
+ const ids = new Set(HINT_REGISTRY.map((h) => h.id));
+ const keys = new Set(HINT_REGISTRY.map((h) => h.seenKey));
+ expect(ids.size).toBe(HINT_REGISTRY.length);
+ expect(keys.size).toBe(HINT_REGISTRY.length);
+ });
+
+ it("matches the editor route only for the build editor path", () => {
+ const block = HINT_REGISTRY.find((h) => h.id === "add-another-block")!;
+ expect(block.matchRoute("/workflows/wpid_123/build")).toBe(true);
+ expect(block.matchRoute("/workflows/wpid_123/edit")).toBe(false);
+ expect(block.matchRoute("/workflows")).toBe(false);
+ expect(block.matchRoute("/workflows/wpid_123/runs")).toBe(false);
+ });
+
+ it("matches the runs and workflows list routes exactly", () => {
+ const run = HINT_REGISTRY.find((h) => h.id === "run-recording")!;
+ const tmpl = HINT_REGISTRY.find((h) => h.id === "start-template")!;
+ expect(run.matchRoute("/runs")).toBe(true);
+ expect(run.matchRoute("/runs/abc")).toBe(false);
+ expect(tmpl.matchRoute("/workflows")).toBe(true);
+ expect(tmpl.matchRoute("/workflows/wpid_1/edit")).toBe(false);
+ });
+
+ it("gates the template hint on no first save", () => {
+ const tmpl = HINT_REGISTRY.find((h) => h.id === "start-template")!;
+ const base = {
+ tour_completed_at: "2026-06-01T00:00:00Z",
+ modal_dismissed_at: null,
+ first_save_at: null,
+ first_run_at: null,
+ ab_variant: null,
+ user_intent: null,
+ seen_canvas: null,
+ seen_node_adder: null,
+ seen_sidebar: null,
+ seen_save_run: null,
+ };
+ expect(tmpl.prerequisite(base)).toBe(true);
+ expect(
+ tmpl.prerequisite({ ...base, first_save_at: "2026-06-01T00:00:00Z" }),
+ ).toBe(false);
+ });
+});
diff --git a/skyvern-frontend/src/util/onboarding/contextualHints.ts b/skyvern-frontend/src/util/onboarding/contextualHints.ts
new file mode 100644
index 000000000..b0ee15c4e
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/contextualHints.ts
@@ -0,0 +1,72 @@
+import type { OnboardingState } from "@/store/onboarding/types";
+import type { Surface } from "@/util/onboarding/OnboardingTelemetry";
+
+export type HintSeenKey =
+ | "seen_hint_block"
+ | "seen_hint_run"
+ | "seen_hint_template";
+
+export type ContextualHint = {
+ id: string;
+ surface: Surface;
+ seenKey: HintSeenKey;
+ anchor: string;
+ matchRoute: (pathname: string) => boolean;
+ prerequisite: (state: OnboardingState) => boolean;
+ popover: {
+ title: string;
+ description: string;
+ side: "top" | "bottom" | "left" | "right";
+ align: "start" | "center" | "end";
+ };
+};
+
+const EDITOR_ROUTE = /^\/workflows\/[^/]+\/build$/;
+
+export const HINT_REGISTRY: readonly ContextualHint[] = [
+ {
+ id: "add-another-block",
+ surface: "editor",
+ seenKey: "seen_hint_block",
+ anchor: "[data-tour='node-adder']",
+ matchRoute: (pathname) => EDITOR_ROUTE.test(pathname),
+ prerequisite: () => true,
+ popover: {
+ title: "Add another block",
+ description:
+ "Chain steps together - use the + to add another block and build a multi-step workflow.",
+ side: "left",
+ align: "center",
+ },
+ },
+ {
+ id: "run-recording",
+ surface: "runs",
+ seenKey: "seen_hint_run",
+ anchor: "[data-hint='run-recording']",
+ matchRoute: (pathname) => pathname === "/runs",
+ prerequisite: () => true,
+ popover: {
+ title: "Watch it run",
+ description:
+ "Click any run to watch the live browser recording and inspect each step.",
+ side: "bottom",
+ align: "start",
+ },
+ },
+ {
+ id: "start-template",
+ surface: "dashboard",
+ seenKey: "seen_hint_template",
+ anchor: "[data-hint='start-template']",
+ matchRoute: (pathname) => pathname === "/workflows",
+ prerequisite: (state) => state.first_save_at === null,
+ popover: {
+ title: "Start from a template",
+ description:
+ "Not sure where to begin? Browse ready-made templates for common automations.",
+ side: "top",
+ align: "center",
+ },
+ },
+] as const;
diff --git a/skyvern-frontend/src/util/onboarding/credentialSetupTelemetry.ts b/skyvern-frontend/src/util/onboarding/credentialSetupTelemetry.ts
new file mode 100644
index 000000000..69d091316
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/credentialSetupTelemetry.ts
@@ -0,0 +1,38 @@
+import posthog from "posthog-js";
+
+type ActivationSurface = "run_parameters" | "editor";
+
+function capture(event: string, properties: Record<string, unknown>): void {
+ try {
+ posthog.capture(event, properties);
+ } catch {
+ // PostHog may be unavailable in tests or before init.
+ }
+}
+
+function credentialSetupShown(
+ surface: ActivationSurface,
+ blockCount: number,
+): void {
+ capture("onboarding.credential_setup_shown", {
+ surface,
+ block_count: blockCount,
+ });
+}
+
+function credentialSetupCtaClicked(
+ surface: ActivationSurface,
+ blockCount: number,
+): void {
+ capture("onboarding.credential_setup_cta_clicked", {
+ surface,
+ block_count: blockCount,
+ });
+}
+
+export const CredentialSetupTelemetry = {
+ credentialSetupShown,
+ credentialSetupCtaClicked,
+} as const;
+
+export type { ActivationSurface };
diff --git a/skyvern-frontend/src/util/onboarding/experimentConfig.test.ts b/skyvern-frontend/src/util/onboarding/experimentConfig.test.ts
new file mode 100644
index 000000000..83c730921
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/experimentConfig.test.ts
@@ -0,0 +1,73 @@
+import { describe, expect, it } from "vitest";
+import {
+ EXPERIMENT,
+ ACTIVATION_FUNNEL,
+ METRICS,
+ POWER_ANALYSIS,
+ VARIANTS,
+ VARIANT_LIST,
+ DEFAULT_VARIANT,
+ isABVariant,
+} from "./experimentConfig";
+
+describe("experimentConfig", () => {
+ it("has two equally-weighted variants", () => {
+ expect(VARIANT_LIST).toHaveLength(2);
+ expect(VARIANT_LIST).toContain(VARIANTS.TEMPLATE_FIRST);
+ expect(VARIANT_LIST).toContain(VARIANTS.COPILOT_FIRST);
+ expect(EXPERIMENT.rolloutWeights["template-first"]).toBe(50);
+ expect(EXPERIMENT.rolloutWeights["copilot-first"]).toBe(50);
+ });
+
+ it("defaults to template-first", () => {
+ expect(DEFAULT_VARIANT).toBe(VARIANTS.TEMPLATE_FIRST);
+ });
+
+ it("funnel has 4 ordered steps", () => {
+ expect(ACTIVATION_FUNNEL).toHaveLength(4);
+ const orders = ACTIVATION_FUNNEL.map((s) => s.order);
+ expect(orders).toEqual([1, 2, 3, 4]);
+ });
+
+ it("funnel events reference valid onboarding telemetry events", () => {
+ for (const step of ACTIVATION_FUNNEL) {
+ expect(step.event).toMatch(/^onboarding\./);
+ }
+ });
+
+ it("has exactly one primary metric", () => {
+ const primary = METRICS.filter((m) => m.priority === "primary");
+ expect(primary).toHaveLength(1);
+ expect(primary[0]!.id).toBe("activation_rate");
+ });
+
+ it("has three secondary metrics", () => {
+ const secondary = METRICS.filter((m) => m.priority === "secondary");
+ expect(secondary).toHaveLength(3);
+ const ids = secondary.map((m) => m.id);
+ expect(ids).toContain("time_to_first_value");
+ expect(ids).toContain("onboarding_completion_rate");
+ expect(ids).toContain("seven_day_retention");
+ });
+
+ it("power analysis requires ~4400 total samples", () => {
+ expect(POWER_ANALYSIS.samplesPerVariant).toBeGreaterThanOrEqual(2000);
+ expect(POWER_ANALYSIS.totalSamplesRequired).toBe(
+ POWER_ANALYSIS.samplesPerVariant * VARIANT_LIST.length,
+ );
+ });
+
+ describe("isABVariant", () => {
+ it("accepts valid variants", () => {
+ expect(isABVariant("template-first")).toBe(true);
+ expect(isABVariant("copilot-first")).toBe(true);
+ });
+
+ it("rejects invalid values", () => {
+ expect(isABVariant("control")).toBe(false);
+ expect(isABVariant(null)).toBe(false);
+ expect(isABVariant(undefined)).toBe(false);
+ expect(isABVariant(42)).toBe(false);
+ });
+ });
+});
diff --git a/skyvern-frontend/src/util/onboarding/experimentConfig.ts b/skyvern-frontend/src/util/onboarding/experimentConfig.ts
new file mode 100644
index 000000000..d66ea2d4f
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/experimentConfig.ts
@@ -0,0 +1,123 @@
+import { EDITOR_ONBOARDING_TOUR_FLAG } from "@/util/featureFlags";
+
+// -- Variants --
+
+export type ABVariant = "template-first" | "copilot-first";
+
+export const VARIANTS = {
+ TEMPLATE_FIRST: "template-first" as const,
+ COPILOT_FIRST: "copilot-first" as const,
+};
+
+export const VARIANT_LIST: readonly ABVariant[] = [
+ VARIANTS.TEMPLATE_FIRST,
+ VARIANTS.COPILOT_FIRST,
+];
+
+export const DEFAULT_VARIANT: ABVariant = VARIANTS.TEMPLATE_FIRST;
+
+// -- Experiment metadata --
+
+export const EXPERIMENT = {
+ name: "onboarding-template-vs-copilot",
+ flagKey: EDITOR_ONBOARDING_TOUR_FLAG,
+ description:
+ "Measures whether template-first or copilot-first onboarding drives higher activation",
+ variants: VARIANT_LIST,
+ rolloutWeights: { "template-first": 50, "copilot-first": 50 } as const,
+} as const;
+
+// -- Conversion funnel --
+
+export type FunnelStep = {
+ order: number;
+ event: string;
+ label: string;
+};
+
+export const ACTIVATION_FUNNEL: readonly FunnelStep[] = [
+ { order: 1, event: "onboarding.flow_started", label: "Flow started" },
+ { order: 2, event: "onboarding.flow_completed", label: "Flow completed" },
+ {
+ order: 3,
+ event: "onboarding.first_workflow_created",
+ label: "First workflow saved",
+ },
+ {
+ order: 4,
+ event: "onboarding.first_run_completed",
+ label: "First run completed",
+ },
+] as const;
+
+// -- Metrics --
+
+export type MetricKind = "funnel" | "duration" | "retention";
+export type MetricPriority = "primary" | "secondary";
+
+export type MetricDefinition = {
+ id: string;
+ label: string;
+ priority: MetricPriority;
+ kind: MetricKind;
+ entryEvent: string;
+ successEvent: string;
+ windowDays: number;
+};
+
+export const METRICS: readonly MetricDefinition[] = [
+ {
+ id: "activation_rate",
+ label: "Activation rate (signup to first run)",
+ priority: "primary",
+ kind: "funnel",
+ entryEvent: "$pageview",
+ successEvent: "onboarding.first_run_completed",
+ windowDays: 14,
+ },
+ {
+ id: "time_to_first_value",
+ label: "Time to first value",
+ priority: "secondary",
+ kind: "duration",
+ entryEvent: "onboarding.flow_started",
+ successEvent: "onboarding.first_run_completed",
+ windowDays: 14,
+ },
+ {
+ id: "onboarding_completion_rate",
+ label: "Onboarding completion rate",
+ priority: "secondary",
+ kind: "funnel",
+ entryEvent: "onboarding.flow_started",
+ successEvent: "onboarding.flow_completed",
+ windowDays: 7,
+ },
+ {
+ id: "seven_day_retention",
+ label: "7-day retention",
+ priority: "secondary",
+ kind: "retention",
+ entryEvent: "onboarding.first_run_completed",
+ successEvent: "$pageview",
+ windowDays: 7,
+ },
+] as const;
+
+// -- Power calculation --
+
+export const POWER_ANALYSIS = {
+ baselineActivationRate: 0.25,
+ minimumDetectableEffect: 0.15,
+ significanceLevel: 0.05,
+ statisticalPower: 0.8,
+ samplesPerVariant: 2200,
+ totalSamplesRequired: 4400,
+ estimatedDurationWeeks: 6,
+} as const;
+
+// -- Helpers --
+
+export function isABVariant(value: unknown): value is ABVariant {
+ return value === VARIANTS.TEMPLATE_FIRST || value === VARIANTS.COPILOT_FIRST;
+}
diff --git a/skyvern-frontend/src/util/onboarding/product-tour.css b/skyvern-frontend/src/util/onboarding/product-tour.css
new file mode 100644
index 000000000..bb6e4f595
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/product-tour.css
@@ -0,0 +1,122 @@
+.driver-popover.skyvern-onboarding {
+ background-color: hsl(231 26% 14%);
+ border: 1px solid hsl(215 25% 27%);
+ border-radius: 8px;
+ box-shadow: 0 4px 16px -8px hsl(222 84% 5% / 0.6);
+ color: hsl(210 40% 98%);
+ padding: 16px;
+ max-width: 320px;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-arrow-side-left,
+.driver-popover.skyvern-onboarding .driver-popover-arrow-side-right,
+.driver-popover.skyvern-onboarding .driver-popover-arrow-side-top,
+.driver-popover.skyvern-onboarding .driver-popover-arrow-side-bottom {
+ border: none;
+}
+
+.driver-popover.skyvern-onboarding
+ .driver-popover-arrow-side-left
+ .driver-popover-arrow,
+.driver-popover.skyvern-onboarding
+ .driver-popover-arrow-side-right
+ .driver-popover-arrow,
+.driver-popover.skyvern-onboarding
+ .driver-popover-arrow-side-top
+ .driver-popover-arrow,
+.driver-popover.skyvern-onboarding
+ .driver-popover-arrow-side-bottom
+ .driver-popover-arrow {
+ background-color: hsl(231 26% 14%);
+ border: 1px solid hsl(215 25% 27%);
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-title {
+ margin: 0;
+ font-size: 16px;
+ font-weight: 600;
+ line-height: 1.4;
+ color: hsl(210 40% 98%);
+ font-family: inherit;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-description {
+ margin: 4px 0 0;
+ font-size: 14px;
+ font-weight: 400;
+ line-height: 20px;
+ color: hsl(215 20% 65%);
+ font-family: inherit;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-close-btn {
+ color: hsl(215 20% 65%);
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-close-btn:hover {
+ color: hsl(210 40% 98%);
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-footer {
+ margin-top: 16px;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-progress-text {
+ font-size: 12px;
+ font-weight: 500;
+ color: hsl(215 20% 65%);
+ font-variant-numeric: tabular-nums;
+ font-family: inherit;
+}
+
+.driver-popover.skyvern-onboarding
+ .driver-popover-navigation-btns
+ button
+ + button {
+ margin-left: 8px;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-prev-btn {
+ padding: 8px 16px;
+ border: 1px solid hsl(215 25% 27%);
+ border-radius: 6px;
+ background-color: transparent;
+ color: hsl(210 40% 98%);
+ font-size: 13px;
+ font-weight: 500;
+ line-height: 1.25;
+ text-shadow: none;
+ font-family: inherit;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-prev-btn:hover {
+ background-color: hsl(217 33% 18%);
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-next-btn {
+ padding: 8px 16px;
+ border: none;
+ border-radius: 6px;
+ background-color: hsl(210 40% 98%);
+ color: hsl(222 47% 11%);
+ font-size: 13px;
+ font-weight: 600;
+ line-height: 1.25;
+ text-shadow: none;
+ font-family: inherit;
+}
+
+.driver-popover.skyvern-onboarding .driver-popover-next-btn:hover {
+ background-color: hsl(210 40% 92%);
+}
+
+/* Exit dialog opens over the active tour; driver.js kills page pointer-events and overlays everything. */
+.tour-paused .driver-overlay,
+.tour-paused .driver-popover {
+ display: none !important;
+}
+
+.driver-active .skyvern-tour-exit,
+.driver-active .skyvern-tour-exit * {
+ pointer-events: auto;
+}
diff --git a/skyvern-frontend/src/util/onboarding/recoveryGuidanceTelemetry.test.ts b/skyvern-frontend/src/util/onboarding/recoveryGuidanceTelemetry.test.ts
new file mode 100644
index 000000000..3558cdc90
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/recoveryGuidanceTelemetry.test.ts
@@ -0,0 +1,68 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import posthog from "posthog-js";
+import { RecoveryGuidanceTelemetry } from "./recoveryGuidanceTelemetry";
+
+vi.mock("posthog-js", () => ({
+ default: { capture: vi.fn() },
+}));
+
+describe("RecoveryGuidanceTelemetry", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("recoveryGuidanceShown captures the surface, category, and path count", () => {
+ RecoveryGuidanceTelemetry.recoveryGuidanceShown(
+ "runs",
+ "invalid_credentials",
+ 2,
+ );
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_guidance_shown",
+ {
+ surface: "runs",
+ failure_category: "invalid_credentials",
+ path_count: 2,
+ },
+ );
+ });
+
+ it("recoveryPathChosen captures the chosen path id", () => {
+ RecoveryGuidanceTelemetry.recoveryPathChosen(
+ "runs",
+ "network_error",
+ "retry",
+ );
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_path_chosen",
+ { surface: "runs", failure_category: "network_error", path_id: "retry" },
+ );
+ });
+
+ it("recoveryOutcome captures the chosen path id and downstream outcome", () => {
+ RecoveryGuidanceTelemetry.recoveryOutcome(
+ "runs",
+ null,
+ "view_docs",
+ "opened",
+ );
+ expect(posthog.capture).toHaveBeenCalledWith(
+ "onboarding.recovery_outcome",
+ {
+ surface: "runs",
+ failure_category: null,
+ path_id: "view_docs",
+ outcome: "opened",
+ },
+ );
+ });
+
+ it("swallows posthog errors so telemetry never breaks the UI", () => {
+ vi.mocked(posthog.capture).mockImplementationOnce(() => {
+ throw new Error("posthog unavailable");
+ });
+ expect(() =>
+ RecoveryGuidanceTelemetry.recoveryGuidanceShown("runs", null, 3),
+ ).not.toThrow();
+ });
+});
diff --git a/skyvern-frontend/src/util/onboarding/recoveryGuidanceTelemetry.ts b/skyvern-frontend/src/util/onboarding/recoveryGuidanceTelemetry.ts
new file mode 100644
index 000000000..3685ecb71
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/recoveryGuidanceTelemetry.ts
@@ -0,0 +1,63 @@
+import posthog from "posthog-js";
+
+type Surface = "dashboard" | "editor" | "runs" | "settings" | "api_docs";
+
+type RecoveryOutcome =
+ | "navigated"
+ | "opened"
+ | "retry_started"
+ | "retry_failed_to_start";
+
+function capture(event: string, properties: Record<string, unknown>): void {
+ try {
+ posthog.capture(event, properties);
+ } catch {
+ // PostHog may be unavailable in tests or before init.
+ }
+}
+
+function recoveryGuidanceShown(
+ surface: Surface,
+ failureCategory: string | null,
+ pathCount: number,
+): void {
+ capture("onboarding.recovery_guidance_shown", {
+ surface,
+ failure_category: failureCategory,
+ path_count: pathCount,
+ });
+}
+
+function recoveryPathChosen(
+ surface: Surface,
+ failureCategory: string | null,
+ pathId: string,
+): void {
+ capture("onboarding.recovery_path_chosen", {
+ surface,
+ failure_category: failureCategory,
+ path_id: pathId,
+ });
+}
+
+function recoveryOutcome(
+ surface: Surface,
+ failureCategory: string | null,
+ pathId: string,
+ outcome: RecoveryOutcome,
+): void {
+ capture("onboarding.recovery_outcome", {
+ surface,
+ failure_category: failureCategory,
+ path_id: pathId,
+ outcome,
+ });
+}
+
+export const RecoveryGuidanceTelemetry = {
+ recoveryGuidanceShown,
+ recoveryPathChosen,
+ recoveryOutcome,
+} as const;
+
+export type { Surface, RecoveryOutcome };
diff --git a/skyvern-frontend/src/util/onboarding/rolloutConfig.test.ts b/skyvern-frontend/src/util/onboarding/rolloutConfig.test.ts
new file mode 100644
index 000000000..33a7a1f00
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/rolloutConfig.test.ts
@@ -0,0 +1,139 @@
+import { describe, expect, it } from "vitest";
+import {
+ RAMP_STAGES,
+ GUARDRAIL_METRICS,
+ ROLLBACK_RULES,
+ DASHBOARD_WIDGETS,
+ ROLLOUT_CONFIG,
+ getAutoRollbackRules,
+ getGuardrailForMetric,
+ getMetricDefinition,
+ isGateCheckRequired,
+} from "./rolloutConfig";
+import { EDITOR_ONBOARDING_TOUR_FLAG } from "@/util/featureFlags";
+
+describe("rolloutConfig", () => {
+ describe("RAMP_STAGES", () => {
+ it("has 4 stages: 0% -> 10% -> 50% -> 100%", () => {
+ expect(RAMP_STAGES).toHaveLength(4);
+ const percentages = RAMP_STAGES.map((s) => s.percentage);
+ expect(percentages).toEqual([0, 10, 50, 100]);
+ });
+
+ it("stages are monotonically increasing", () => {
+ for (let i = 1; i < RAMP_STAGES.length; i++) {
+ expect(RAMP_STAGES[i]!.percentage).toBeGreaterThan(
+ RAMP_STAGES[i - 1]!.percentage,
+ );
+ }
+ });
+
+ it("canary holds for 1 day, broad holds for 3 days", () => {
+ const canary = RAMP_STAGES.find((s) => s.name === "canary");
+ const broad = RAMP_STAGES.find((s) => s.name === "broad");
+ expect(canary?.holdDays).toBe(1);
+ expect(broad?.holdDays).toBe(3);
+ });
+
+ it("gate checks required at canary and broad stages", () => {
+ expect(isGateCheckRequired(1)).toBe(true);
+ expect(isGateCheckRequired(2)).toBe(true);
+ expect(isGateCheckRequired(0)).toBe(false);
+ expect(isGateCheckRequired(3)).toBe(false);
+ });
+ });
+
+ describe("GUARDRAIL_METRICS", () => {
+ it("defines error rate, modal render, tour JS, and completion drop", () => {
+ const ids = GUARDRAIL_METRICS.map((g) => g.id);
+ expect(ids).toContain("onboarding_error_rate");
+ expect(ids).toContain("modal_render_failure_rate");
+ expect(ids).toContain("tour_js_error_rate");
+ expect(ids).toContain("completion_rate_drop");
+ });
+
+ it("all error-rate guardrails threshold at 5%", () => {
+ const errorGuardrails = GUARDRAIL_METRICS.filter(
+ (g) => g.id.endsWith("_rate") && g.direction === "above",
+ );
+ for (const g of errorGuardrails) {
+ expect(g.threshold).toBe(0.05);
+ }
+ });
+
+ it("completion rate drop threshold is 20%", () => {
+ const drop = GUARDRAIL_METRICS.find(
+ (g) => g.id === "completion_rate_drop",
+ );
+ expect(drop?.threshold).toBe(0.2);
+ expect(drop?.direction).toBe("below");
+ });
+ });
+
+ describe("ROLLBACK_RULES", () => {
+ it("all rules trigger auto-rollback", () => {
+ for (const rule of ROLLBACK_RULES) {
+ expect(rule.action).toBe("auto-rollback");
+ }
+ });
+
+ it("every rule references a defined guardrail metric", () => {
+ const guardrailIds = new Set(GUARDRAIL_METRICS.map((g) => g.id));
+ for (const rule of ROLLBACK_RULES) {
+ expect(guardrailIds.has(rule.metric)).toBe(true);
+ }
+ });
+
+ it("getAutoRollbackRules returns all rules", () => {
+ expect(getAutoRollbackRules()).toEqual(ROLLBACK_RULES);
+ });
+ });
+
+ describe("DASHBOARD_WIDGETS", () => {
+ it("has 6 widgets", () => {
+ expect(DASHBOARD_WIDGETS).toHaveLength(6);
+ });
+
+ it("includes required widgets from AC", () => {
+ const ids = DASHBOARD_WIDGETS.map((w) => w.id);
+ expect(ids).toContain("activation_funnel_by_variant");
+ expect(ids).toContain("tour_completion_rate");
+ expect(ids).toContain("modal_skip_rate");
+ expect(ids).toContain("time_to_first_value_distribution");
+ });
+
+ it("activation funnel widget has variant breakdown", () => {
+ const funnel = DASHBOARD_WIDGETS.find(
+ (w) => w.id === "activation_funnel_by_variant",
+ );
+ expect(funnel?.breakdownBy).toBe("variant");
+ expect(funnel?.kind).toBe("funnel");
+ });
+ });
+
+ describe("ROLLOUT_CONFIG", () => {
+ it("references the correct feature flag", () => {
+ expect(ROLLOUT_CONFIG.flagKey).toBe(EDITOR_ONBOARDING_TOUR_FLAG);
+ });
+ });
+
+ describe("helpers", () => {
+ it("getGuardrailForMetric finds by id", () => {
+ const g = getGuardrailForMetric("onboarding_error_rate");
+ expect(g?.threshold).toBe(0.05);
+ });
+
+ it("getGuardrailForMetric returns undefined for unknown id", () => {
+ expect(getGuardrailForMetric("nonexistent")).toBeUndefined();
+ });
+
+ it("getMetricDefinition finds experiment metrics", () => {
+ const m = getMetricDefinition("activation_rate");
+ expect(m?.priority).toBe("primary");
+ });
+
+ it("isGateCheckRequired returns false for out-of-bounds", () => {
+ expect(isGateCheckRequired(99)).toBe(false);
+ });
+ });
+});
diff --git a/skyvern-frontend/src/util/onboarding/rolloutConfig.ts b/skyvern-frontend/src/util/onboarding/rolloutConfig.ts
new file mode 100644
index 000000000..75557ed67
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/rolloutConfig.ts
@@ -0,0 +1,198 @@
+import { EDITOR_ONBOARDING_TOUR_FLAG } from "@/util/featureFlags";
+import {
+ ACTIVATION_FUNNEL,
+ METRICS,
+ type MetricDefinition,
+} from "./experimentConfig";
+
+// -- Ramp stages --
+
+export type RampStage = {
+ name: string;
+ percentage: number;
+ holdDays: number;
+ gateCheck: boolean;
+};
+
+export const RAMP_STAGES: readonly RampStage[] = [
+ { name: "dark-launch", percentage: 0, holdDays: 0, gateCheck: false },
+ { name: "canary", percentage: 10, holdDays: 1, gateCheck: true },
+ { name: "broad", percentage: 50, holdDays: 3, gateCheck: true },
+ { name: "ga", percentage: 100, holdDays: 0, gateCheck: false },
+] as const;
+
+// -- Guardrail thresholds --
+
+export type GuardrailMetric = {
+ id: string;
+ label: string;
+ event: string;
+ threshold: number;
+ direction: "above" | "below";
+ windowMinutes: number;
+};
+
+export const GUARDRAIL_METRICS: readonly GuardrailMetric[] = [
+ {
+ id: "onboarding_error_rate",
+ label: "Onboarding JS error rate",
+ event: "onboarding.error",
+ threshold: 0.05,
+ direction: "above",
+ windowMinutes: 60,
+ },
+ {
+ id: "modal_render_failure_rate",
+ label: "Modal render failure rate",
+ event: "onboarding.modal_render_error",
+ threshold: 0.05,
+ direction: "above",
+ windowMinutes: 60,
+ },
+ {
+ id: "tour_js_error_rate",
+ label: "Tour JS error rate",
+ event: "onboarding.tour_error",
+ threshold: 0.05,
+ direction: "above",
+ windowMinutes: 60,
+ },
+ {
+ id: "completion_rate_drop",
+ label: "Completion rate drop from baseline",
+ event: "onboarding.flow_completed",
+ threshold: 0.2,
+ direction: "below",
+ windowMinutes: 1440,
+ },
+] as const;
+
+// -- Rollback criteria --
+
+export type RollbackRule = {
+ metric: string;
+ condition: string;
+ action: "auto-rollback" | "alert-only";
+};
+
+export const ROLLBACK_RULES: readonly RollbackRule[] = [
+ {
+ metric: "onboarding_error_rate",
+ condition: "error rate > 5% over 1h window",
+ action: "auto-rollback",
+ },
+ {
+ metric: "modal_render_failure_rate",
+ condition: "modal render failures > 5% over 1h window",
+ action: "auto-rollback",
+ },
+ {
+ metric: "tour_js_error_rate",
+ condition: "tour JS errors > 5% over 1h window",
+ action: "auto-rollback",
+ },
+ {
+ metric: "completion_rate_drop",
+ condition: "completion rate drops > 20% from baseline over 24h window",
+ action: "auto-rollback",
+ },
+] as const;
+
+// -- PostHog dashboard config --
+
+export type DashboardWidget = {
+ id: string;
+ title: string;
+ kind: "funnel" | "trend" | "distribution" | "retention";
+ events: readonly string[];
+ breakdownBy?: string;
+ dateRange: string;
+};
+
+export const DASHBOARD_WIDGETS: readonly DashboardWidget[] = [
+ {
+ id: "activation_funnel_by_variant",
+ title: "Activation funnel by variant",
+ kind: "funnel",
+ events: ACTIVATION_FUNNEL.map((s) => s.event),
+ breakdownBy: "variant",
+ dateRange: "last 14 days",
+ },
+ {
+ id: "tour_completion_rate",
+ title: "Tour completion rate",
+ kind: "funnel",
+ events: ["onboarding.tour_started", "onboarding.tour_completed"],
+ breakdownBy: "variant",
+ dateRange: "last 14 days",
+ },
+ {
+ id: "modal_skip_rate",
+ title: "Modal skip rate",
+ kind: "trend",
+ events: ["onboarding.modal_opened", "onboarding.modal_skipped"],
+ breakdownBy: "variant",
+ dateRange: "last 14 days",
+ },
+ {
+ id: "time_to_first_value_distribution",
+ title: "Time-to-first-value distribution",
+ kind: "distribution",
+ events: ["onboarding.flow_started", "onboarding.first_run_completed"],
+ breakdownBy: "variant",
+ dateRange: "last 14 days",
+ },
+ {
+ id: "error_rate_trend",
+ title: "Onboarding error rate trend",
+ kind: "trend",
+ events: [
+ "onboarding.error",
+ "onboarding.modal_render_error",
+ "onboarding.tour_error",
+ ],
+ dateRange: "last 7 days",
+ },
+ {
+ id: "seven_day_retention_by_variant",
+ title: "7-day retention by variant",
+ kind: "retention",
+ events: ["onboarding.first_run_completed", "$pageview"],
+ breakdownBy: "variant",
+ dateRange: "last 30 days",
+ },
+] as const;
+
+// -- Aggregate config --
+
+export const ROLLOUT_CONFIG = {
+ flagKey: EDITOR_ONBOARDING_TOUR_FLAG,
+ stages: RAMP_STAGES,
+ guardrails: GUARDRAIL_METRICS,
+ rollbackRules: ROLLBACK_RULES,
+ dashboard: DASHBOARD_WIDGETS,
+ metrics: METRICS,
+} as const;
+
+// -- Helpers --
+
+export function getAutoRollbackRules(): readonly RollbackRule[] {
+ return ROLLBACK_RULES.filter((r) => r.action === "auto-rollback");
+}
+
+export function getGuardrailForMetric(
+ metricId: string,
+): GuardrailMetric | undefined {
+ return GUARDRAIL_METRICS.find((g) => g.id === metricId);
+}
+
+export function getMetricDefinition(
+ metricId: string,
+): MetricDefinition | undefined {
+ return METRICS.find((m) => m.id === metricId);
+}
+
+export function isGateCheckRequired(stageIndex: number): boolean {
+ const stage = RAMP_STAGES[stageIndex];
+ return stage?.gateCheck ?? false;
+}
diff --git a/skyvern-frontend/src/util/onboarding/rolloutGating.test.ts b/skyvern-frontend/src/util/onboarding/rolloutGating.test.ts
new file mode 100644
index 000000000..ee1e47d5a
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/rolloutGating.test.ts
@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import {
+ isActivationRun,
+ isFirstFailedRunRecoveryEligible,
+} from "./rolloutGating";
+
+describe("isActivationRun", () => {
+ const loaded = { isLoading: false, state: { first_run_at: null } };
+
+ it("is true for an A/B arm when the user has no completed first run", () => {
+ expect(isActivationRun("template-first", loaded)).toBe(true);
+ expect(isActivationRun("copilot-first", loaded)).toBe(true);
+ });
+
+ it("is false when the flag is off (rollback / 0% rollout)", () => {
+ expect(isActivationRun(false, loaded)).toBe(false);
+ expect(isActivationRun(undefined, loaded)).toBe(false);
+ expect(isActivationRun("control", loaded)).toBe(false);
+ });
+
+ it("is false once a first run exists", () => {
+ expect(
+ isActivationRun("template-first", {
+ isLoading: false,
+ state: { first_run_at: "2026-01-01T00:00:00Z" },
+ }),
+ ).toBe(false);
+ });
+
+ it("is false while loading or without state", () => {
+ expect(
+ isActivationRun("template-first", {
+ isLoading: true,
+ state: { first_run_at: null },
+ }),
+ ).toBe(false);
+ expect(
+ isActivationRun("template-first", { isLoading: false, state: null }),
+ ).toBe(false);
+ expect(isActivationRun("template-first", null)).toBe(false);
+ });
+});
+
+describe("isFirstFailedRunRecoveryEligible", () => {
+ const base = {
+ flagVariant: "template-first" as string | boolean | undefined,
+ isNewUser: true,
+ isFailureRun: true,
+ hasFailureReason: true,
+ };
+
+ it("is true for a new user's failed first run under an A/B arm", () => {
+ expect(isFirstFailedRunRecoveryEligible(base)).toBe(true);
+ });
+
+ it("is false when the flag is off", () => {
+ expect(
+ isFirstFailedRunRecoveryEligible({ ...base, flagVariant: false }),
+ ).toBe(false);
+ expect(
+ isFirstFailedRunRecoveryEligible({ ...base, flagVariant: undefined }),
+ ).toBe(false);
+ });
+
+ it("is false for existing users or non-failures", () => {
+ expect(
+ isFirstFailedRunRecoveryEligible({ ...base, isNewUser: false }),
+ ).toBe(false);
+ expect(
+ isFirstFailedRunRecoveryEligible({ ...base, isFailureRun: false }),
+ ).toBe(false);
+ expect(
+ isFirstFailedRunRecoveryEligible({ ...base, hasFailureReason: false }),
+ ).toBe(false);
+ });
+});
diff --git a/skyvern-frontend/src/util/onboarding/rolloutGating.ts b/skyvern-frontend/src/util/onboarding/rolloutGating.ts
new file mode 100644
index 000000000..7fec59248
--- /dev/null
+++ b/skyvern-frontend/src/util/onboarding/rolloutGating.ts
@@ -0,0 +1,40 @@
+import { isABVariant } from "./experimentConfig";
+
+type ActivationOnboarding = {
+ isLoading: boolean;
+ state: { first_run_at: string | null } | null;
+} | null;
+
+// Onboarding surfaces must disappear when the experiment flag is off (0% rollout
+// or rollback), so each gate also checks the resolved A/B arm. Without this an
+// arm-less cloud user keeps seeing the new surfaces and loses the pre-onboarding
+// UI even after rollback.
+
+function isActivationRun(
+ flagVariant: string | boolean | undefined,
+ onboarding: ActivationOnboarding,
+): boolean {
+ return (
+ isABVariant(flagVariant) &&
+ onboarding != null &&
+ !onboarding.isLoading &&
+ onboarding.state != null &&
+ onboarding.state.first_run_at == null
+ );
+}
+
+function isFirstFailedRunRecoveryEligible(args: {
+ flagVariant: string | boolean | undefined;
+ isNewUser: boolean;
+ isFailureRun: boolean;
+ hasFailureReason: boolean;
+}): boolean {
+ return (
+ isABVariant(args.flagVariant) &&
+ args.isNewUser &&
+ args.isFailureRun &&
+ args.hasFailureReason
+ );
+}
+
+export { isActivationRun, isFirstFailedRunRecoveryEligible };
diff --git a/skyvern-frontend/src/util/runWithConcurrency.test.ts b/skyvern-frontend/src/util/runWithConcurrency.test.ts
new file mode 100644
index 000000000..0c7d227a6
--- /dev/null
+++ b/skyvern-frontend/src/util/runWithConcurrency.test.ts
@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+
+import { runWithConcurrency } from "./runWithConcurrency";
+
+describe("runWithConcurrency", () => {
+ it("runs all tasks and returns settled results", async () => {
+ const results = await runWithConcurrency(
+ [
+ () => Promise.resolve("a"),
+ () => Promise.reject(new Error("fail")),
+ () => Promise.resolve("c"),
+ ],
+ 2,
+ );
+
+ expect(results).toHaveLength(3);
+ expect(results[0]).toEqual({ status: "fulfilled", value: "a" });
+ expect(results[1]?.status).toBe("rejected");
+ expect(results[2]).toEqual({ status: "fulfilled", value: "c" });
+ });
+
+ it("respects concurrency limit", async () => {
+ let inFlight = 0;
+ let maxInFlight = 0;
+
+ await runWithConcurrency(
+ Array.from({ length: 6 }, () => async () => {
+ inFlight += 1;
+ maxInFlight = Math.max(maxInFlight, inFlight);
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ inFlight -= 1;
+ return true;
+ }),
+ 2,
+ );
+
+ expect(maxInFlight).toBeLessThanOrEqual(2);
+ });
+});
diff --git a/skyvern-frontend/src/util/runWithConcurrency.ts b/skyvern-frontend/src/util/runWithConcurrency.ts
new file mode 100644
index 000000000..801a2aab7
--- /dev/null
+++ b/skyvern-frontend/src/util/runWithConcurrency.ts
@@ -0,0 +1,27 @@
+const BULK_CONCURRENCY_LIMIT = 5;
+
+async function runWithConcurrency<T>(
+ tasks: Array<() => Promise<T>>,
+ limit: number,
+): Promise<PromiseSettledResult<T>[]> {
+ const results: Array<PromiseSettledResult<T>> = new Array(tasks.length);
+ let nextIndex = 0;
+
+ async function worker() {
+ while (nextIndex < tasks.length) {
+ const index = nextIndex++;
+ try {
+ results[index] = { status: "fulfilled", value: await tasks[index]!() };
+ } catch (reason) {
+ results[index] = { status: "rejected", reason };
+ }
+ }
+ }
+
+ await Promise.all(
+ Array.from({ length: Math.min(limit, tasks.length) }, () => worker()),
+ );
+ return results;
+}
+
+export { BULK_CONCURRENCY_LIMIT, runWithConcurrency };
diff --git a/skyvern-frontend/vitest.config.ts b/skyvern-frontend/vitest.config.ts
index 6a8a02460..70223be89 100644
--- a/skyvern-frontend/vitest.config.ts
+++ b/skyvern-frontend/vitest.config.ts
@@ -6,6 +6,7 @@ export default defineConfig({
plugins: [react()],
test: {
passWithNoTests: true,
+ exclude: ["e2e/**", "node_modules/**"],
// Tests that transitively import the editor (e.g. workflowEditorUtils via
// its `./nodes` barrel) touch zustand stores that read localStorage at
// module load — run all tests under jsdom so those imports resolve. The
diff --git a/skyvern/cli/llm_setup.py b/skyvern/cli/llm_setup.py
index f5730dd05..2db1ebe10 100644
--- a/skyvern/cli/llm_setup.py
+++ b/skyvern/cli/llm_setup.py
@@ -175,6 +175,7 @@ def setup_llm_providers(env_path: Path | str | None = None) -> None:
model_options.extend(
[
"GEMINI_3.1_PRO",
+ "GEMINI_3.5_FLASH",
"GEMINI_3.0_FLASH",
"GEMINI_3.1_FLASH_LITE",
"GEMINI_2.5_PRO",
diff --git a/skyvern/config.py b/skyvern/config.py
index 80138041f..dfce87eb9 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -492,9 +492,13 @@ class Settings(BaseSettings):
BITWARDEN_EMAIL: str | None = None
OP_SERVICE_ACCOUNT_TOKEN: str | None = None
- # Where credentials are stored: bitwarden or azure_vault
+ # Where credentials are stored: bitwarden, azure_vault, gcp, or custom
CREDENTIAL_VAULT_TYPE: str = "bitwarden"
+ # GCP Secret Manager credential vault settings
+ GCP_CREDENTIAL_VAULT_PROJECT_ID: str | None = None # project hosting the Secret Manager secrets
+ GCP_CREDENTIAL_VAULT_PREFIX: str = "skyvern-cred-" # secret-id prefix; must be unique per deployment
+
# Azure Setting
AZURE_TENANT_ID: str | None = None
AZURE_CLIENT_ID: str | None = None
@@ -651,6 +655,7 @@ class Settings(BaseSettings):
("gemini-2.5-flash", "VERTEX_GEMINI_2.5_FLASH", "GEMINI_2.5_FLASH", "Gemini 2.5 Flash"),
("gemini-3-pro-preview", "VERTEX_GEMINI_3_PRO", "GEMINI_3_PRO", "Gemini 3 Pro (Latest)"),
("gemini-3.0-flash", "VERTEX_GEMINI_3.0_FLASH", "GEMINI_3.0_FLASH", "Gemini 3 Flash"),
+ ("gemini-3.5-flash", "VERTEX_GEMINI_3.5_FLASH", "GEMINI_3.5_FLASH", "Gemini 3.5 Flash"),
]
for model_name, vertex_key, gemini_key, label in gemini_models:
mapping[model_name] = {
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index f0e1d1b2c..7bbd57827 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1322,6 +1322,15 @@ class AgentFunction:
"""
return None
+ async def validate_user_organization_membership(
+ self,
+ user_id: str,
+ organization_id: str,
+ bearer_token: str | None = None,
+ ) -> bool | None:
+ """Return whether the user belongs to the organization, or None when membership cannot be determined."""
+ return None
+
async def on_workflow_saved(
self,
organization_id: str,
diff --git a/skyvern/forge/sdk/api/gcp.py b/skyvern/forge/sdk/api/gcp.py
index d07c98f61..01ba8099a 100644
--- a/skyvern/forge/sdk/api/gcp.py
+++ b/skyvern/forge/sdk/api/gcp.py
@@ -79,7 +79,25 @@ class AsyncGcsStorageClient(Protocol):
async def close(self) -> None: ...
+class AsyncGcpSecretManagerClient(Protocol):
+ """Interface for GCP Secret Manager operations — the customer-runtime
+ credential vault. Mirrors the method surface of ``AsyncAzureVaultClient``
+ (with ``project_id`` in place of ``vault_name``) so the credential vault
+ service can be a near-copy of the Azure one.
+ """
+
+ async def get_secret(self, secret_id: str, project_id: str) -> str | None: ...
+
+ async def create_or_update_secret(self, secret_id: str, project_id: str, value: str) -> str: ...
+
+ async def delete_secret(self, secret_id: str, project_id: str) -> None: ...
+
+ async def close(self) -> None: ...
+
+
class GcpClientFactory(Protocol):
- """Interface for creating GCS storage clients."""
+ """Interface for creating GCS storage and Secret Manager clients."""
def create_storage_client(self, project_id: str | None = None) -> "AsyncGcsStorageClient": ...
+
+ def create_secret_manager_client(self) -> "AsyncGcpSecretManagerClient": ...
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 7634ee7fe..fb015bcac 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -1187,6 +1187,19 @@ if settings.ENABLE_GEMINI:
),
),
)
+ LLMConfigRegistry.register_config(
+ "GEMINI_3.5_FLASH",
+ LLMConfig(
+ "gemini/gemini-3.5-flash",
+ ["GEMINI_API_KEY"],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ max_completion_tokens=65536,
+ litellm_params=LiteLLMParams(
+ thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",
+ ),
+ ),
+ )
LLMConfigRegistry.register_config(
"GEMINI_3_PRO",
LLMConfig(
@@ -1584,6 +1597,22 @@ if settings.ENABLE_VERTEX_AI:
),
),
)
+ LLMConfigRegistry.register_config(
+ "VERTEX_GEMINI_3.5_FLASH",
+ LLMConfig(
+ "vertex_ai/gemini-3.5-flash",
+ [],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ max_completion_tokens=65536,
+ litellm_params=LiteLLMParams(
+ api_base=f"{api_base}/gemini-3.5-flash" if api_base else None,
+ vertex_location=settings.VERTEX_LOCATION,
+ thinking_level="medium" if settings.GEMINI_INCLUDE_THOUGHT else "minimal",
+ vertex_credentials=settings.VERTEX_CREDENTIALS,
+ ),
+ ),
+ )
LLMConfigRegistry.register_config(
"VERTEX_GEMINI_3.1_FLASH_LITE",
LLMConfig(
diff --git a/skyvern/forge/sdk/api/real_gcp.py b/skyvern/forge/sdk/api/real_gcp.py
index 5df0d988a..70b3d9db1 100644
--- a/skyvern/forge/sdk/api/real_gcp.py
+++ b/skyvern/forge/sdk/api/real_gcp.py
@@ -5,20 +5,22 @@
"""
import asyncio
+import threading
from datetime import timedelta
from mimetypes import add_type, guess_type
from typing import IO
import structlog
-from google.api_core.exceptions import NotFound
+from google.api_core.exceptions import Conflict, NotFound
from google.auth import default as google_auth_default
from google.auth.credentials import Credentials
from google.auth.transport.requests import Request as GoogleAuthRequest
-from google.cloud import storage
+from google.cloud import secretmanager_v1, storage
from skyvern.config import settings
from skyvern.forge.sdk.api.gcp import (
STORAGE_CLASS_STANDARD,
+ AsyncGcpSecretManagerClient,
AsyncGcsStorageClient,
GcpClientFactory,
GcsUri,
@@ -45,13 +47,19 @@ class RealAsyncGcsStorageClient(AsyncGcsStorageClient):
self.project_id = project_id or settings.GCS_PROJECT_ID
self._credentials = credentials
self._client: storage.Client | None = None
+ self._client_lock = threading.Lock()
def _get_client(self) -> storage.Client:
+ # Double-checked locking: _get_client runs inside asyncio.to_thread, so
+ # concurrent first calls would otherwise each build a client and leak all
+ # but the last.
if self._client is None:
- # When credentials are omitted the client resolves Application
- # Default Credentials (GOOGLE_APPLICATION_CREDENTIALS / Workload
- # Identity). STORAGE_EMULATOR_HOST is honored automatically.
- self._client = storage.Client(project=self.project_id, credentials=self._credentials)
+ with self._client_lock:
+ if self._client is None:
+ # When credentials are omitted the client resolves Application
+ # Default Credentials (GOOGLE_APPLICATION_CREDENTIALS / Workload
+ # Identity). STORAGE_EMULATOR_HOST is honored automatically.
+ self._client = storage.Client(project=self.project_id, credentials=self._credentials)
return self._client
def _get_blob(self, uri: str) -> storage.Blob:
@@ -225,8 +233,102 @@ class RealAsyncGcsStorageClient(AsyncGcsStorageClient):
self._client = None
+class RealAsyncGcpSecretManagerClient(AsyncGcpSecretManagerClient):
+ """Async wrapper over the synchronous Secret Manager client.
+
+ Uses the REST transport rather than the default gRPC: REST goes over plain
+ HTTPS so it works behind a TLS-intercepting proxy, and it keeps this module
+ consistent with the sync-lib-in-a-thread pattern used for storage. In a
+ GKE deployment gRPC would also work; REST is functionally equivalent for the
+ low call volume of credential CRUD.
+ """
+
+ def __init__(self, client: secretmanager_v1.SecretManagerServiceClient | None = None) -> None:
+ self._client = client
+ self._client_lock = threading.Lock()
+
+ def _get_client(self) -> secretmanager_v1.SecretManagerServiceClient:
+ # Double-checked locking: _get_client runs inside asyncio.to_thread, so
+ # concurrent first calls would otherwise each build a client and leak all
+ # but the last.
+ if self._client is None:
+ with self._client_lock:
+ if self._client is None:
+ self._client = secretmanager_v1.SecretManagerServiceClient(transport="rest")
+ return self._client
+
+ async def get_secret(self, secret_id: str, project_id: str) -> str | None:
+ def _do() -> str | None:
+ name = f"projects/{project_id}/secrets/{secret_id}/versions/latest"
+ try:
+ response = self._get_client().access_secret_version(request={"name": name})
+ return response.payload.data.decode("UTF-8")
+ except NotFound:
+ return None
+
+ return await asyncio.to_thread(_do)
+
+ async def create_or_update_secret(self, secret_id: str, project_id: str, value: str) -> str:
+ def _do() -> str:
+ client = self._get_client()
+ parent = f"projects/{project_id}"
+ try:
+ client.create_secret(
+ request={
+ "parent": parent,
+ "secret_id": secret_id,
+ "secret": {"replication": {"automatic": {}}},
+ }
+ )
+ except Conflict:
+ # Secret already exists — proceed to add a new version. The REST
+ # transport raises the generic Conflict (409); gRPC raises the
+ # AlreadyExists subclass. Conflict catches both.
+ pass
+ # The value lives in a version; "updating" a secret means adding one.
+ added = client.add_secret_version(
+ request={"parent": f"{parent}/secrets/{secret_id}", "payload": {"data": value.encode("UTF-8")}}
+ )
+ # Revoke prior versions so a rotated/leaked credential value can't
+ # be read back via an explicit version number. Only destroy versions
+ # numerically OLDER than ours: a concurrent update may have added a
+ # newer version between our add and this list, and destroying it
+ # would race both writers into destroying each other's version
+ # (destruction is permanent), leaving the secret unreadable.
+ # Comparing version numbers makes concurrent updates converge to
+ # last-writer-wins.
+ added_number = int(added.name.rsplit("/", 1)[-1])
+ destroyed_state = secretmanager_v1.SecretVersion.State.DESTROYED
+ for version in client.list_secret_versions(request={"parent": f"{parent}/secrets/{secret_id}"}):
+ if version.state == destroyed_state:
+ continue
+ if int(version.name.rsplit("/", 1)[-1]) >= added_number:
+ continue
+ client.destroy_secret_version(request={"name": version.name})
+ return secret_id
+
+ return await asyncio.to_thread(_do)
+
+ async def delete_secret(self, secret_id: str, project_id: str) -> None:
+ def _do() -> None:
+ try:
+ self._get_client().delete_secret(request={"name": f"projects/{project_id}/secrets/{secret_id}"})
+ except NotFound:
+ LOG.debug("GCP secret not found for deletion", secret_id=secret_id, project_id=project_id)
+
+ await asyncio.to_thread(_do)
+
+ async def close(self) -> None:
+ if self._client is not None:
+ await asyncio.to_thread(self._client.transport.close)
+ self._client = None
+
+
class RealGcpClientFactory(GcpClientFactory):
- """Factory for creating real GCS storage clients."""
+ """Factory for creating real GCS storage and Secret Manager clients."""
def create_storage_client(self, project_id: str | None = None) -> AsyncGcsStorageClient:
return RealAsyncGcsStorageClient(project_id=project_id)
+
+ def create_secret_manager_client(self) -> AsyncGcpSecretManagerClient:
+ return RealAsyncGcpSecretManagerClient()
diff --git a/skyvern/forge/sdk/artifact/manager.py b/skyvern/forge/sdk/artifact/manager.py
index 4bd9d4916..d74b20e47 100644
--- a/skyvern/forge/sdk/artifact/manager.py
+++ b/skyvern/forge/sdk/artifact/manager.py
@@ -428,6 +428,33 @@ class ArtifactManager:
file_size=file_size,
)
+ async def create_run_recording_artifact(
+ self,
+ *,
+ organization_id: str,
+ run_id: str,
+ uri: str,
+ workflow_run_id: str | None = None,
+ checksum: str | None = None,
+ file_size: int | None = None,
+ ) -> str:
+ """Register a per-run clip as a RECORDING Artifact scoped to ``run_id`` (``browser_session_id``
+ unset). Idempotency is the caller's responsibility.
+ """
+ artifact_id = generate_artifact_id()
+ await app.DATABASE.artifacts.create_artifact(
+ artifact_id=artifact_id,
+ artifact_type=ArtifactType.RECORDING,
+ uri=uri,
+ organization_id=organization_id,
+ run_id=run_id,
+ workflow_run_id=workflow_run_id,
+ checksum=checksum,
+ file_size=file_size,
+ )
+ LOG.debug("Registered run-scoped recording artifact", artifact_id=artifact_id, run_id=run_id, uri=uri)
+ return artifact_id
+
async def create_browser_session_replay_artifact(
self,
*,
diff --git a/skyvern/forge/sdk/artifact/storage/azure.py b/skyvern/forge/sdk/artifact/storage/azure.py
index 472889b27..696598e74 100644
--- a/skyvern/forge/sdk/artifact/storage/azure.py
+++ b/skyvern/forge/sdk/artifact/storage/azure.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.artifact.storage.base import (
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
)
+from skyvern.forge.sdk.artifact.storage.run_recording_clips import RUN_RECORDING_PATH_SEGMENT, sync_run_recording_clips
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -717,6 +718,8 @@ class AzureStorage(BaseStorage):
date: str | None = None,
) -> str:
"""Sync a file from local browser session to Azure."""
+ # Anchor per-run clip offsets to browser close, captured before the upload below.
+ recording_finalized_at = datetime.now(timezone.utc)
uri = self._build_browser_session_uri(organization_id, browser_session_id, artifact_type, remote_path, date)
tier = await self._get_storage_tier_for_org(organization_id)
tags = await self._get_tags_for_org(organization_id)
@@ -759,6 +762,26 @@ class AzureStorage(BaseStorage):
file_size=file_size,
)
+ async def _upload_clip(run_id: str, clip_path: str, filename: str) -> str:
+ clip_uri = self._build_browser_session_uri(
+ organization_id, browser_session_id, RUN_RECORDING_PATH_SEGMENT, f"{run_id}/{filename}", date
+ )
+ await self.async_client.upload_file_from_path(clip_uri, clip_path, tier=tier, tags=tags)
+ return clip_uri
+
+ try:
+ await sync_run_recording_clips(
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ source_path=local_file_path,
+ upload_clip=_upload_clip,
+ now=recording_finalized_at,
+ )
+ except Exception:
+ LOG.warning(
+ "Run recording clip generation failed", browser_session_id=browser_session_id, exc_info=True
+ )
+
return uri
async def delete_browser_session_file(
diff --git a/skyvern/forge/sdk/artifact/storage/gcs.py b/skyvern/forge/sdk/artifact/storage/gcs.py
index 288d1dc05..fe30205e2 100644
--- a/skyvern/forge/sdk/artifact/storage/gcs.py
+++ b/skyvern/forge/sdk/artifact/storage/gcs.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.artifact.storage.base import (
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
)
+from skyvern.forge.sdk.artifact.storage.run_recording_clips import RUN_RECORDING_PATH_SEGMENT, sync_run_recording_clips
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -726,6 +727,8 @@ class GcsStorage(BaseStorage):
date: str | None = None,
) -> str:
"""Sync a file from local browser session to GCS."""
+ # Anchor per-run clip offsets to browser close, captured before the upload below.
+ recording_finalized_at = datetime.now(timezone.utc)
uri = self._build_browser_session_uri(organization_id, browser_session_id, artifact_type, remote_path, date)
storage_class = await self._get_storage_class_for_org(organization_id)
tags = await self._get_tags_for_org(organization_id)
@@ -768,6 +771,28 @@ class GcsStorage(BaseStorage):
file_size=file_size,
)
+ async def _upload_clip(run_id: str, clip_path: str, filename: str) -> str:
+ clip_uri = self._build_browser_session_uri(
+ organization_id, browser_session_id, RUN_RECORDING_PATH_SEGMENT, f"{run_id}/{filename}", date
+ )
+ await self.async_client.upload_file_from_path(
+ clip_uri, clip_path, storage_class=storage_class, tags=tags
+ )
+ return clip_uri
+
+ try:
+ await sync_run_recording_clips(
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ source_path=local_file_path,
+ upload_clip=_upload_clip,
+ now=recording_finalized_at,
+ )
+ except Exception:
+ LOG.warning(
+ "Run recording clip generation failed", browser_session_id=browser_session_id, exc_info=True
+ )
+
return uri
async def delete_browser_session_file(
diff --git a/skyvern/forge/sdk/artifact/storage/run_recording_clips.py b/skyvern/forge/sdk/artifact/storage/run_recording_clips.py
new file mode 100644
index 000000000..bed41ae3e
--- /dev/null
+++ b/skyvern/forge/sdk/artifact/storage/run_recording_clips.py
@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Awaitable, Callable
+from datetime import UTC, datetime, timedelta
+
+import structlog
+
+from skyvern.forge import app
+from skyvern.forge.sdk.api.files import calculate_sha256_for_file
+from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.webeye.video_utils import cut_recording_segment, plan_run_segment, probe_media_duration_seconds
+
+LOG = structlog.get_logger()
+
+# Bound how many runs we clip per session so a heavily reused session can't fan out into an
+# unbounded number of ffmpeg jobs at close time.
+MAX_RUNS_PER_SESSION = 200
+
+# Artifact-type folder for per-run clips. Kept distinct from "videos" so the session recording
+# listing (which scans .../videos) never picks up clips, and used as the URI marker that
+# identifies a clip created by this path (vs. a run's own full recording).
+RUN_RECORDING_PATH_SEGMENT = "run_recordings"
+_CLIP_URI_MARKER = f"/{RUN_RECORDING_PATH_SEGMENT}/"
+
+# (run_id, local_clip_path, filename) -> stored uri
+UploadClipFn = Callable[[str, str, str], Awaitable[str]]
+
+
+def _file_size(path: str) -> int | None:
+ try:
+ return os.path.getsize(path)
+ except OSError:
+ return None
+
+
+async def sync_run_recording_clips(
+ *,
+ organization_id: str,
+ browser_session_id: str,
+ source_path: str,
+ upload_clip: UploadClipFn,
+ now: datetime | None = None,
+) -> None:
+ """Cut each session run's window out of the finalized recording and register it as a
+ run-scoped RECORDING artifact (``run_id`` set, ``browser_session_id`` unset).
+
+ The recording is anchored on the wall clock as ``[now - duration, now]``, ``now`` defaulting to
+ close time. Best-effort: any ffmpeg/ffprobe/DB/upload failure is logged and skipped, never raised.
+ """
+ duration = await probe_media_duration_seconds(source_path)
+ if not duration:
+ LOG.info(
+ "Skipping run recording clips: source duration unavailable",
+ browser_session_id=browser_session_id,
+ source_path=source_path,
+ )
+ return
+
+ video_start = (now or datetime.now(UTC)) - timedelta(seconds=duration)
+
+ # A session can have more than one source video (e.g. a popup page), each synced separately.
+ # Key clips on the source stem so a second source video for the same run is still clipped.
+ source_stem = os.path.splitext(os.path.basename(source_path))[0]
+
+ try:
+ runs = await app.DATABASE.workflow_runs.get_workflow_runs_for_browser_session(
+ browser_session_id=browser_session_id,
+ organization_id=organization_id,
+ page_size=MAX_RUNS_PER_SESSION,
+ )
+ except Exception:
+ LOG.warning(
+ "Skipping run recording clips: failed to list runs for browser session",
+ browser_session_id=browser_session_id,
+ exc_info=True,
+ )
+ return
+
+ if len(runs) >= MAX_RUNS_PER_SESSION:
+ LOG.warning(
+ "Run recording clip generation hit the per-session cap; some runs were not clipped",
+ browser_session_id=browser_session_id,
+ cap=MAX_RUNS_PER_SESSION,
+ )
+
+ for run in runs:
+ if run.started_at is None:
+ continue
+ segment = plan_run_segment(run.started_at, run.finished_at, video_start, duration)
+ if segment is None:
+ continue
+ start_seconds, clip_duration = segment
+ try:
+ # Scope the clip to the id the run view reads by: task_v2 runs are read by
+ # observer_cruise_id, every other run by workflow_run_id.
+ task_v2 = await app.DATABASE.observer.get_task_v2_by_workflow_run_id(run.workflow_run_id, organization_id)
+ run_id = task_v2.observer_cruise_id if task_v2 else run.workflow_run_id
+
+ existing = await app.DATABASE.artifacts.list_artifacts_for_run_by_type(
+ run_id=run_id,
+ organization_id=organization_id,
+ artifact_type=ArtifactType.RECORDING,
+ )
+ # Idempotent per (run, source): skip only a clip already made from THIS source video
+ # (a run's own full recording, or a clip from another source video, must not suppress
+ # it), so a retried sync is a no-op while popup/secondary videos still get their clip.
+ clip_basename = f"{run.workflow_run_id}-{source_stem}"
+ if any(a.uri and _CLIP_URI_MARKER in a.uri and clip_basename in a.uri for a in existing):
+ continue
+ async with cut_recording_segment(
+ source_path, start_seconds=start_seconds, duration_seconds=clip_duration
+ ) as clip_path:
+ if clip_path is None:
+ continue
+ uri = await upload_clip(run.workflow_run_id, clip_path, f"{clip_basename}.mp4")
+ await app.ARTIFACT_MANAGER.create_run_recording_artifact(
+ organization_id=organization_id,
+ run_id=run_id,
+ workflow_run_id=run.workflow_run_id,
+ uri=uri,
+ checksum=calculate_sha256_for_file(clip_path),
+ file_size=_file_size(clip_path),
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to generate run recording clip",
+ browser_session_id=browser_session_id,
+ workflow_run_id=run.workflow_run_id,
+ exc_info=True,
+ )
+ continue
diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py
index b3dce571c..cdefdbccd 100644
--- a/skyvern/forge/sdk/artifact/storage/s3.py
+++ b/skyvern/forge/sdk/artifact/storage/s3.py
@@ -30,6 +30,7 @@ from skyvern.forge.sdk.artifact.storage.base import (
_file_infos_from_artifacts,
_file_infos_from_download_artifacts,
)
+from skyvern.forge.sdk.artifact.storage.run_recording_clips import RUN_RECORDING_PATH_SEGMENT, sync_run_recording_clips
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -878,6 +879,9 @@ class S3Storage(BaseStorage):
sc = await self._get_storage_class_for_org(organization_id, self.bucket)
if artifact_type == "videos":
+ # Anchor per-run clip offsets to browser close, captured before the (potentially
+ # slow) compress+upload below so a long upload doesn't shift every clip window.
+ recording_finalized_at = datetime.now(timezone.utc)
# Compress finalized Playwright recordings before upload. The raw
# local file remains the source of truth if ffmpeg fails; the S3
# object and artifact metadata reflect the prepared upload file.
@@ -899,6 +903,26 @@ class S3Storage(BaseStorage):
checksum=checksum,
file_size=file_size,
)
+
+ async def _upload_clip(run_id: str, clip_path: str, filename: str) -> str:
+ clip_uri = self._build_browser_session_uri(
+ organization_id, browser_session_id, RUN_RECORDING_PATH_SEGMENT, f"{run_id}/{filename}", date
+ )
+ await self.async_client.upload_file_from_path(clip_uri, clip_path, storage_class=sc)
+ return clip_uri
+
+ try:
+ await sync_run_recording_clips(
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ source_path=upload_file_path,
+ upload_clip=_upload_clip,
+ now=recording_finalized_at,
+ )
+ except Exception:
+ LOG.warning(
+ "Run recording clip generation failed", browser_session_id=browser_session_id, exc_info=True
+ )
return uri
await self.async_client.upload_file_from_path(uri, local_file_path, storage_class=sc)
diff --git a/skyvern/forge/sdk/artifact/storage/test_s3_storage.py b/skyvern/forge/sdk/artifact/storage/test_s3_storage.py
index eb03963d1..08af218d4 100644
--- a/skyvern/forge/sdk/artifact/storage/test_s3_storage.py
+++ b/skyvern/forge/sdk/artifact/storage/test_s3_storage.py
@@ -1,10 +1,13 @@
import asyncio
import io
import os
-from datetime import datetime
+import shutil
+import subprocess
+from datetime import UTC, datetime, timedelta
from pathlib import Path
+from types import SimpleNamespace
from typing import Generator
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, MagicMock
import boto3
import pytest
@@ -1140,3 +1143,83 @@ class TestS3StorageZIPArchiveRetrieve:
)
assert uri.endswith(".zip")
assert not uri.endswith(".zst")
+
+
+class TestS3StoragePerRunRecordingClips:
+ """Integration test for the SKY-7220 per-run clip path: drives the real
+ ``S3Storage.sync_browser_session_file(videos)`` against moto S3 with a real
+ ffmpeg-generated recording, asserting a run-scoped clip is cut and uploaded."""
+
+ async def test_session_close_cuts_and_uploads_run_scoped_clip(
+ self,
+ s3_storage: S3Storage,
+ boto3_test_client: S3Client,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ if shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None:
+ pytest.skip("ffmpeg/ffprobe not installed")
+
+ # A real ~12s recording standing in for the finalized session video.
+ src = tmp_path / "recording.webm"
+ subprocess.check_call(
+ [
+ "ffmpeg",
+ "-y",
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc=size=320x180:rate=10:duration=12",
+ "-c:v",
+ "libvpx",
+ str(src),
+ ]
+ )
+
+ import skyvern.forge.sdk.artifact.storage.run_recording_clips as rrc
+
+ now = datetime.now(UTC)
+ run = SimpleNamespace(
+ workflow_run_id="wr_itest",
+ started_at=now - timedelta(seconds=10),
+ finished_at=now - timedelta(seconds=2),
+ )
+ create_clip = AsyncMock(return_value="a_clip")
+ fake_app = MagicMock()
+ fake_app.DATABASE.workflow_runs.get_workflow_runs_for_browser_session = AsyncMock(return_value=[run])
+ fake_app.DATABASE.artifacts.list_artifacts_for_run_by_type = AsyncMock(return_value=[])
+ fake_app.DATABASE.observer.get_task_v2_by_workflow_run_id = AsyncMock(return_value=None)
+ fake_app.ARTIFACT_MANAGER.create_run_recording_artifact = create_clip
+ monkeypatch.setattr(rrc, "app", fake_app)
+
+ uri = await s3_storage.sync_browser_session_file(
+ organization_id=TEST_ORGANIZATION_ID,
+ browser_session_id=TEST_BROWSER_SESSION_ID,
+ artifact_type="videos",
+ local_file_path=str(src),
+ remote_path="recording.webm",
+ date="2025-01-15",
+ )
+
+ # Full session recording still uploaded under videos/.
+ assert f"/browser_sessions/{TEST_BROWSER_SESSION_ID}/videos/2025-01-15/" in uri
+
+ # Exactly one run-scoped clip registered, run_id propagated, stored under run_recordings/.
+ create_clip.assert_awaited_once()
+ call = create_clip.await_args
+ assert call is not None
+ kwargs = call.kwargs
+ assert kwargs["run_id"] == "wr_itest"
+ assert kwargs["workflow_run_id"] == "wr_itest"
+ assert kwargs["file_size"] and kwargs["file_size"] > 0
+ clip_uri = kwargs["uri"]
+ assert f"/browser_sessions/{TEST_BROWSER_SESSION_ID}/run_recordings/2025-01-15/wr_itest/" in clip_uri
+
+ # The clip bytes really landed in (moto) S3 — proves the real ffmpeg cut + upload ran.
+ clip_key = S3Uri(clip_uri).key
+ head = boto3_test_client.head_object(Bucket=TEST_BUCKET, Key=clip_key)
+ assert head["ContentLength"] > 0
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
deleted file mode 100644
index 856c0a242..000000000
--- a/skyvern/forge/sdk/copilot/tools.py
+++ /dev/null
@@ -1,9528 +0,0 @@
-"""Copilot agent tools — native handlers, hooks, and registration."""
-
-from __future__ import annotations
-
-import asyncio
-import base64
-import json
-import re
-import time
-from collections import Counter, defaultdict
-from collections.abc import Mapping
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Annotated, Any, Literal, cast
-from urllib.parse import urljoin, urlparse
-
-import structlog
-import yaml
-from agents import ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData, function_tool
-from opentelemetry import trace as otel_trace
-
-try:
- from bs4 import BeautifulSoup # type: ignore[import-not-found]
-except ImportError: # pragma: no cover — bs4 is a transitive dep but discovery degrades gracefully without it.
- BeautifulSoup = None # type: ignore[assignment, misc]
-from agents.run_context import RunContextWrapper
-from jinja2.sandbox import SandboxedEnvironment
-from pydantic import AliasChoices, BaseModel, Field, ValidationError
-
-from skyvern.config import settings
-from skyvern.forge import app
-from skyvern.forge.failure_classifier import classify_from_failure_reason
-from skyvern.forge.sdk.artifact.models import ArtifactType
-from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
-from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_workflow_block_goals
-from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
-from skyvern.forge.sdk.copilot.blocker_signal import (
- CopilotToolBlockerSignal,
- RecoveryHint,
- build_loop_blocker_signal,
- clear_blocker_signal_for_reason_codes,
- stash_blocker_signal,
-)
-from skyvern.forge.sdk.copilot.build_phase import (
- BuildPhase,
- _phase_blocker_signal,
- advance_to_composing,
- advance_to_discovering,
- advance_to_testing,
-)
-from skyvern.forge.sdk.copilot.completion_verification import (
- CompletionVerificationResult,
- CriterionVerdict,
- RunEvidenceSnapshot,
- evaluate_completion_criteria,
- summarize_unsatisfied_outcomes,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRIPPED_HTML_EXPRESSION as _COMPOSITION_STRIPPED_HTML_EXPRESSION,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION as _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS as _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- scout_accessible_role_name_expression as _scout_accessible_role_name_expression,
-)
-from skyvern.forge.sdk.copilot.composition_evidence import (
- SCOUT_INTERACTION_EVIDENCE_TOOL,
- composition_page_evidence_error,
- has_bounded_page_schema,
- merge_visual_composition_evidence,
- normalize_block_observation_refs,
- page_evidence_needs_visual_fallback,
- parse_composition_html,
- parse_composition_structured,
- workflow_target_url,
-)
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
-from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
- DiagnosisRepairContract,
- build_diagnosis_repair_contract,
-)
-from skyvern.forge.sdk.copilot.enforcement import (
- POST_INTERMEDIATE_SUCCESS_NUDGE,
- PROBABLE_SITE_BLOCK_STREAK_STOP_AT,
- TOTAL_TIMEOUT_SECONDS,
- _goal_likely_needs_more_blocks,
-)
-from skyvern.forge.sdk.copilot.failure_tracking import (
- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
- ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
- PER_TOOL_BUDGET_FAILURE_CATEGORY,
- _canonical_block_config,
- compute_action_sequence_fingerprint,
- update_repeated_failure_state,
-)
-from skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handler, resolve_main_copilot_handler
-from skyvern.forge.sdk.copilot.loop_detection import (
- clear_failed_step_tracker_for_tools_in_ctx,
- detect_failed_tool_step_loop_for_ctx,
- detect_tool_loop,
- record_tool_step_result_for_ctx,
-)
-from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
-from skyvern.forge.sdk.copilot.narration import NarratorState
-from skyvern.forge.sdk.copilot.narration import handler_available as narration_handler_available
-from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
-from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
-from skyvern.forge.sdk.copilot.output_policy import (
- evaluate_output_policy,
- format_output_policy_tool_error,
- output_policy_verdict_to_trace_data,
-)
-from skyvern.forge.sdk.copilot.output_utils import (
- _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
- build_run_blocks_response,
- iter_failure_reasons,
- sanitize_tool_result_for_llm,
- truncate_output,
-)
-from skyvern.forge.sdk.copilot.request_policy import CREDENTIAL_DEFERRED_DRAFT_REASONS, RequestPolicy
-from skyvern.forge.sdk.copilot.runtime import (
- AgentContext,
- PendingBrowserInteractionObservation,
- ScoutedInteraction,
- ensure_browser_session,
-)
-from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
-from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
-from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from skyvern.forge.sdk.copilot.turn_intent import (
- NO_MUTATION_TURN_INTENT_MODES,
- READ_CONTEXT_DENIED_MODES,
- UNRESOLVED_BLOCK_REF_TARGET_ENTITY,
- TurnIntent,
- TurnIntentMode,
-)
-from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
-from skyvern.forge.sdk.schemas.credentials import Credential
-from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
-from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, get_all_blocks
-from skyvern.forge.sdk.workflow.models.parameter import (
- RESERVED_PARAMETER_KEYS,
- OutputParameter,
- Parameter,
- WorkflowParameter,
- WorkflowParameterType,
-)
-from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
-from skyvern.schemas.proxy_location import ProxyLocation
-from skyvern.schemas.workflows import BlockType
-from skyvern.utils.yaml_loader import safe_load_no_dates
-from skyvern.webeye.navigation import is_skip_inner_retry_error
-from skyvern.webeye.utils.page import SkyvernFrame
-
-LOG = structlog.get_logger()
-
-_FAILED_BLOCK_STATUSES: frozenset[str] = frozenset(
- {
- WorkflowRunStatus.failed.value,
- WorkflowRunStatus.terminated.value,
- WorkflowRunStatus.canceled.value,
- WorkflowRunStatus.timed_out.value,
- }
-)
-_DATA_PRODUCING_BLOCK_TYPES = frozenset({"EXTRACTION", "TEXT_PROMPT"})
-# Block types whose output can demonstrate an end-state outcome. Until a workflow
-# contains one of these, an unmet outcome criterion means the build is still
-# incomplete (no confirmation step yet), not a completed run that failed the goal.
-_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})
-
-# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope
-# (schemas/tasks.py:TaskOutput) rather than the raw payload. The
-# meaningful-data check must unwrap these via ``_block_data_payload`` before
-# judging output, because envelope fields (task_id, status, artifact IDs) are
-# always populated on a completed run and would otherwise mask empty
-# extractions. This is a subset of ``_DATA_PRODUCING_BLOCK_TYPES`` — keep the
-# two in sync when adding a new task-backed type. ``TEXT_PROMPT`` is
-# deliberately excluded: its block.output is the raw LLM response dict (see
-# ``TextPromptBlock.execute``), no envelope to strip.
-_TASK_ENVELOPE_BLOCK_TYPES = frozenset({"EXTRACTION"})
-assert _TASK_ENVELOPE_BLOCK_TYPES <= _DATA_PRODUCING_BLOCK_TYPES, (
- "_TASK_ENVELOPE_BLOCK_TYPES must be a subset of _DATA_PRODUCING_BLOCK_TYPES"
-)
-
-# Absolute upper bound on a single ``run_blocks`` tool invocation. Exists only
-# as a last-resort trip wire for runaway loops — progressing runs should never
-# approach this. The OpenAI Agents SDK wraps the tool in
-# ``asyncio.wait_for(..., timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS)``; the
-# inner poll loop leaves a 10 s headroom below this ceiling for orderly
-# cleanup before the SDK cancels.
-RUN_BLOCKS_SAFETY_CEILING_SECONDS = 1200 # 20 min
-
-# Per-tool-call budget for active block runs — caps a single tool invocation
-# below the session-level wall clock (``enforcement.TOTAL_TIMEOUT_SECONDS``,
-# 900 s) so a long chain cannot consume the whole budget without giving the
-# copilot a chance to issue a smaller chain. Quiet-block runs keep the longer
-# ``RUN_BLOCKS_SAFETY_CEILING_SECONDS`` above.
-PER_TOOL_CALL_BUDGET_SECONDS = 240
-_ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS = 30.0
-_ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS = 30.0
-_ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES = 8
-
-# Primary exit condition: seconds of no observed progress across the combined
-# run / block / step heartbeat. Sized to accommodate the slowest single LLM
-# round-trip (~30-60 s in practice) with headroom; going tighter risks
-# false-positives on healthy runs.
-RUN_BLOCKS_STAGNATION_WINDOW_SECONDS = 90
-
-# Reserve final-reply room; active block runs shrink their own budget near the deadline.
-COPILOT_FINAL_REPLY_RESERVE_SECONDS = 90
-
-# 5 s balances responsiveness (18 samples inside the stagnation window) against
-# DB load (240 polls worst case at the safety ceiling).
-RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0
-
-# Detached cleanup tasks held here so the garbage collector does not drop them
-# while they still have work to do, and so the "task exception was never
-# retrieved" warning cannot fire — each task adds a done-callback that logs
-# exceptions and removes itself from this set.
-_DETACHED_CLEANUP_TASKS: set[asyncio.Task] = set()
-_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
-_EVALUATE_EVIDENCE_CONFIDENCE_WITH_SCHEMA = 0.75
-_EVALUATE_EVIDENCE_CONFIDENCE_ANTIBOT_ONLY = 0.4
-
-
-@dataclass(frozen=True)
-class ActiveRunTerminalEvidenceSample:
- current_url: str | None
- page_title: str | None
- page_evidence: dict[str, Any]
- completion_verification: CompletionVerificationResult
- sample_index: int
-
-
-async def _cancel_run_task_if_not_final(
- run_task: asyncio.Task,
- workflow_run_id: str,
-) -> None:
- """Cancel ``run_task`` and reconcile the workflow run row to a terminal
- state.
-
- ``run_task.cancel()`` is synchronous — it just flips the cancel flag. We
- then wait briefly for ``execute_workflow``'s outer ``finally`` to drain
- its shielded ``_finalize_workflow_run_status`` call, which restores the
- real terminal status (``failed``/``terminated``/``timed_out``) even when
- we cancel mid-flight. After that we issue a conditional DB cancel that
- is a no-op when the row is already terminal — so a run whose finally
- block produced a proper terminal status keeps it, and a run that truly
- never finalized (e.g. cancel landed before block execution captured a
- ``pre_finally_status``) lands as ``canceled``. All awaits are
- exception-contained so teardown of the enclosing tool task doesn't
- surface a secondary error over the original cancellation.
- """
- run_task.cancel()
- try:
- # Shield run_task so OUR wait timeout does not send another cancel
- # through to it — the cancel we want is already pending.
- await asyncio.wait_for(asyncio.shield(run_task), timeout=5.0)
- except (asyncio.CancelledError, asyncio.TimeoutError):
- pass
- except Exception:
- LOG.warning(
- "Run task raised during cancellation grace window",
- workflow_run_id=workflow_run_id,
- exc_info=True,
- )
- try:
- await app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final(
- workflow_run_id=workflow_run_id,
- )
- except Exception:
- LOG.warning(
- "Conditional cancel write failed",
- workflow_run_id=workflow_run_id,
- exc_info=True,
- )
-
-
-def _log_detached_cleanup_failure(task: asyncio.Task) -> None:
- exc = task.exception() if task.done() and not task.cancelled() else None
- if exc is not None:
- LOG.warning("Detached cancel fallback failed", exc_info=exc)
-
-
-def _extract_credential_ids_from_tool_value(value: Any) -> list[str]:
- found: list[str] = []
-
- def visit(item: Any) -> None:
- if isinstance(item, str):
- found.extend(_CREDENTIAL_ID_RE.findall(item))
- elif isinstance(item, dict):
- for nested in item.values():
- visit(nested)
- elif isinstance(item, (list, tuple, set)):
- for nested in item:
- visit(nested)
- elif hasattr(item, "model_dump"):
- try:
- visit(item.model_dump(mode="json"))
- except Exception:
- return
-
- visit(value)
- return list(dict.fromkeys(found))
-
-
-def _credential_parameter_slot_field(parameter: Any) -> str | None:
- """Return the field name that legitimately carries a `cred_xxx` value for
- this parameter dict, or None if the parameter is not a credential-binding
- slot. Two shapes resolve a credential at runtime: a top-level or block-level
- `parameter_type: credential` with the ID in `credential_id`, and a
- `parameter_type: workflow` + `workflow_parameter_type: credential_id` with
- the ID in `default_value`.
- """
- if not isinstance(parameter, dict):
- return None
- parameter_type = str(parameter.get("parameter_type") or "").lower()
- if parameter_type == "credential":
- return "credential_id"
- workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
- if parameter_type == "workflow" and workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID.value:
- return "default_value"
- return None
-
-
-def _extract_credential_ids_from_workflow_parameters(parameters: Any) -> list[str]:
- if not isinstance(parameters, list):
- return []
-
- found: list[str] = []
- for parameter in parameters:
- slot_field = _credential_parameter_slot_field(parameter)
- if slot_field is None:
- continue
- found.extend(_extract_credential_ids_from_tool_value(parameter.get(slot_field)))
-
- return list(dict.fromkeys(found))
-
-
-def _workflow_definition_as_dict(workflow_definition: Any) -> dict[str, Any]:
- if workflow_definition is None:
- return {}
- if isinstance(workflow_definition, dict):
- return workflow_definition
- if hasattr(workflow_definition, "model_dump"):
- try:
- dumped = workflow_definition.model_dump(mode="json")
- except Exception:
- return {}
- return dumped if isinstance(dumped, dict) else {}
- return {}
-
-
-def _extract_credential_ids_from_workflow_definition(workflow_definition: Any) -> list[str]:
- definition = _workflow_definition_as_dict(workflow_definition)
- return _extract_credential_ids_from_workflow_parameters(definition.get("parameters"))
-
-
-def _parsed_workflow_definition(workflow_yaml: str | None) -> dict[str, Any] | None:
- if not workflow_yaml:
- return None
- try:
- parsed = safe_load_no_dates(workflow_yaml)
- except yaml.YAMLError:
- return None
- if not isinstance(parsed, dict):
- return None
- workflow_definition = parsed.get("workflow_definition")
- if not isinstance(workflow_definition, dict):
- return None
- return workflow_definition
-
-
-def _extract_credential_ids_from_workflow_yaml(workflow_yaml: str | None) -> list[str]:
- workflow_definition = _parsed_workflow_definition(workflow_yaml)
- if workflow_definition is None:
- return []
- return _extract_credential_ids_from_workflow_parameters(workflow_definition.get("parameters"))
-
-
-_MISBINDING_WORKFLOW_LOCATION = "workflow"
-
-
-def _credential_id_misbinding_findings(workflow_yaml: str | None) -> list[dict[str, str]]:
- workflow_definition = _parsed_workflow_definition(workflow_yaml)
- if workflow_definition is None:
- return []
-
- findings: list[dict[str, str]] = []
-
- def _scan_value(value: Any, location: str, field: str) -> None:
- if isinstance(value, str):
- for credential_id in _CREDENTIAL_ID_RE.findall(value):
- findings.append({"location": location, "field": field, "credential_id": credential_id})
- elif isinstance(value, list):
- for item in value:
- _scan_value(item, location, field)
- elif isinstance(value, dict):
- for nested_field, nested_value in value.items():
- _scan_value(nested_value, location, str(nested_field))
-
- def _scan_parameter(parameter: Any, location: str) -> None:
- if not isinstance(parameter, dict):
- return
- legal_slot_field = _credential_parameter_slot_field(parameter)
- for field_name, field_value in parameter.items():
- if field_name == legal_slot_field:
- continue
- _scan_value(field_value, location, str(field_name))
-
- for parameter in workflow_definition.get("parameters") or []:
- _scan_parameter(parameter, _MISBINDING_WORKFLOW_LOCATION)
-
- for block in _iter_yaml_blocks(workflow_definition.get("blocks")):
- label = str(block.get("label") or "<unlabeled>")
- for field_name, field_value in block.items():
- if field_name == "parameters":
- if isinstance(field_value, list):
- for parameter in field_value:
- _scan_parameter(parameter, label)
- continue
- if field_name == "loop_blocks":
- continue
- _scan_value(field_value, label, str(field_name))
-
- return findings
-
-
-def _credential_id_misbinding_error_message(findings: list[dict[str, str]]) -> str:
- grouped: dict[tuple[str, str], list[str]] = {}
- for finding in findings:
- key = (finding["location"], finding["credential_id"])
- grouped.setdefault(key, []).append(finding["field"])
-
- location_lines: list[str] = []
- for (location, credential_id), fields in grouped.items():
- unique_fields = list(dict.fromkeys(fields))
- joined = ", ".join(f"`{field}`" for field in unique_fields)
- scope = "workflow parameter" if location == _MISBINDING_WORKFLOW_LOCATION else f"block `{location}`"
- location_lines.append(f"- `{credential_id}` in {scope} field(s): {joined}")
- body = "\n".join(location_lines)
-
- return (
- "A credential ID is sitting in workflow fields that do not resolve it, so at runtime the agent types "
- "the literal ID into the page instead of the stored username/password:\n"
- f"{body}\n"
- "Fix BOTH halves before retrying:\n"
- "1. Bind the credential once: add a `credential` parameter (or a `workflow` parameter with "
- "`workflow_parameter_type: credential_id` and the ID in `default_value`) and reference its key from the "
- "login block's `parameter_keys`.\n"
- "2. Delete the credential ID string from every field listed above. `navigation_goal`, "
- "`complete_criterion`, `terminate_criterion` and similar fields are plain-language instructions — they "
- "must describe the outcome without naming the credential ID. Do NOT relocate the literal ID into another "
- "prose or list field; only the credential parameter slot may hold it."
- )
-
-
-def _missing_credential_reference_tool_error(missing_credential_ids: list[str]) -> str:
- formatted_ids = ", ".join(f"`{credential_id}`" for credential_id in missing_credential_ids)
- id_word = "ID" if len(missing_credential_ids) == 1 else "IDs"
- was_word = "was" if len(missing_credential_ids) == 1 else "were"
- return (
- f"The credential {id_word} {formatted_ids} {was_word} not found in this organization. "
- "Stop before creating, updating, or running the workflow. Ask the user to provide/select a valid "
- "credential ID, create the credential in the Credentials UI and return with its ID, or explicitly "
- "choose an unvalidated draft workflow that will not be run until credentials are available."
- )
-
-
-def _guardrail_tool_arguments(tool_context: Any) -> tuple[dict[str, Any], Any]:
- raw_arguments = getattr(tool_context, "tool_arguments", "")
- try:
- # Agents SDK guardrails may hand us either raw JSON or an already parsed mapping.
- parsed_arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
- except json.JSONDecodeError:
- parsed_arguments = {}
- return parsed_arguments if isinstance(parsed_arguments, dict) else {}, raw_arguments
-
-
-def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
- tool_context = data.context
- tool_arguments, raw_arguments = _guardrail_tool_arguments(tool_context)
- if not raw_arguments:
- LOG.warning(
- "workflow YAML output policy guardrail received no tool arguments",
- tool_name=getattr(tool_context, "tool_name", None),
- tool_call_id=getattr(tool_context, "tool_call_id", None),
- )
- workflow_yaml_value = tool_arguments.get("workflow_yaml")
- workflow_yaml = workflow_yaml_value if isinstance(workflow_yaml_value, str) else None
- verdict = evaluate_output_policy(
- request_policy=getattr(getattr(tool_context, "context", None), "request_policy", None),
- workflow_yaml=workflow_yaml,
- tool_arguments=tool_arguments or raw_arguments,
- )
- trace_data = output_policy_verdict_to_trace_data(
- verdict,
- surface="tool_input",
- tool_name=getattr(tool_context, "tool_name", None),
- )
- LOG.info("copilot output policy tool guardrail verdict", **trace_data)
- if not verdict.allowed:
- return ToolGuardrailFunctionOutput.reject_content(
- format_output_policy_tool_error(verdict),
- output_info=trace_data,
- )
- return ToolGuardrailFunctionOutput.allow(output_info=trace_data)
-
-
-_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL = ToolInputGuardrail(
- guardrail_function=_workflow_yaml_output_policy_guardrail,
- name="workflow_yaml_output_policy_guardrail",
-)
-
-_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA = {
- "surface": "tool_pre_side_effect",
- "tool_name": "update_and_run_blocks",
- "reason": "composition_page_evidence",
-}
-
-
-def _update_and_run_blocks_composition_evidence_precheck(
- copilot_ctx: Any,
- workflow_yaml: str | None,
- normalized_block_observation_refs: dict[str, int],
- raw_block_observation_refs: Any,
-) -> str | None:
- if copilot_ctx is None or workflow_yaml is None:
- LOG.warning(
- "update_and_run_blocks composition evidence precheck missing context or workflow yaml",
- has_context=copilot_ctx is not None,
- has_workflow_yaml=workflow_yaml is not None,
- )
- return None
-
- evidence_error = composition_page_evidence_error(
- copilot_ctx,
- workflow_yaml,
- block_observation_refs=normalized_block_observation_refs,
- raw_block_observation_refs=raw_block_observation_refs,
- )
-
- if evidence_error:
- LOG.info(
- "copilot composition page evidence pre-side-effect rejected workflow",
- workflow_permanent_id=getattr(copilot_ctx, "workflow_permanent_id", None),
- target_url=workflow_target_url(workflow_yaml),
- surface="tool_pre_side_effect",
- )
- return evidence_error
-
- return None
-
-
-async def _credential_ids_validation_error(credential_ids: list[str], ctx: AgentContext) -> str | None:
- if not credential_ids:
- return None
- try:
- existing_credentials = await app.DATABASE.credentials.get_credentials_by_ids(
- credential_ids,
- organization_id=ctx.organization_id,
- )
- except Exception:
- LOG.warning(
- "Copilot tool failed to validate credential IDs",
- organization_id=ctx.organization_id,
- credential_ids=credential_ids,
- exc_info=True,
- )
- return (
- "Credential ID validation failed, so the workflow cannot be created, updated, or run safely. "
- "Ask the user to provide/select a valid credential ID or explicitly choose an unvalidated draft "
- "workflow that will not be run until credentials are available."
- )
-
- found_ids = {credential.credential_id for credential in existing_credentials}
- missing_ids = [credential_id for credential_id in credential_ids if credential_id not in found_ids]
- if not missing_ids:
- return None
- return _missing_credential_reference_tool_error(missing_ids)
-
-
-async def _credential_reference_validation_error(value: Any, ctx: AgentContext) -> str | None:
- if isinstance(value, str):
- credential_ids = _extract_credential_ids_from_workflow_yaml(value)
- else:
- credential_ids = _extract_credential_ids_from_tool_value(value)
- return await _credential_ids_validation_error(credential_ids, ctx)
-
-
-async def _safe_read_workflow_run(
- workflow_run_id: str,
- organization_id: str,
- *,
- context: str,
-) -> WorkflowRun | None:
- """Read a workflow_runs row, logging-and-returning-None on failure.
-
- The ``context`` string distinguishes call sites in logs (e.g.
- ``"pre-cancel"`` vs ``"post-drain"``) so a failure is attributable to
- the specific phase of the timeout branch it fired from.
- """
- try:
- return await app.DATABASE.workflow_runs.get_workflow_run(
- workflow_run_id=workflow_run_id,
- organization_id=organization_id,
- )
- except Exception:
- LOG.warning(
- "Workflow run re-read failed",
- workflow_run_id=workflow_run_id,
- context=context,
- exc_info=True,
- )
- return None
-
-
-def _trusted_post_drain_status(run: WorkflowRun | None) -> str | None:
- """Return the run's status if it is one we can trust after the cancel
- helper has run; otherwise ``None``.
-
- ``canceled`` is deliberately rejected because at post-drain read time we
- can't tell a legitimate ``canceled`` (written by
- ``_finalize_workflow_run_status`` when a block/user canceled the run)
- apart from a synthetic ``canceled`` (written by the cancel helper's
- fallback). Callers that need to distinguish those cases must read the row
- BEFORE the cancel helper runs.
- """
- if run is None:
- return None
- if WorkflowRunStatus(run.status).is_final_excluding_canceled():
- return run.status
- return None
-
-
-def _active_run_terminal_evidence_detected(result: Mapping[str, object]) -> bool:
- data_value = result.get("data")
- data = data_value if isinstance(data_value, Mapping) else {}
- return data.get("active_run_terminal_evidence_detected") is True
-
-
-def _active_run_terminal_evidence_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- evidence = getattr(ctx, "workflow_verification_evidence", None)
- has_active_terminal_evidence = getattr(
- ctx, "last_failure_category_top", None
- ) == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY or bool(
- getattr(evidence, "active_run_terminal_evidence_detected", False)
- )
- if not has_active_terminal_evidence:
- return None
-
- run_id = getattr(evidence, "active_run_terminal_evidence_workflow_run_id", None) or getattr(
- ctx, "last_run_blocks_workflow_run_id", None
- )
- location = (
- getattr(evidence, "page_title", None) or getattr(evidence, "current_url", None) or "the current browser page"
- )
- run_detail = f" Workflow run: {run_id}." if isinstance(run_id, str) and run_id else ""
- agent_steering = (
- "The prior active workflow run emitted typed ACTIVE_RUN_TERMINAL_EVIDENCE while the browser task "
- f"was still running.{run_detail} The current page evidence already matched the user's terminal "
- "browser-state criteria, but the reusable workflow is not verified end-to-end. "
- f"Do not call {tool_name} again in this turn; reply with a partial-verification/blocker state that "
- "says the requested browser state was observed, the active run was interrupted before overshoot, "
- "and the workflow still needs a clean corrected run before it can be offered as tested."
- )
- user_facing = (
- f"I reached the requested browser state on {location} and stopped before continuing, "
- "but the reusable workflow still needs a clean verification run before it is ready."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- preserves_workflow_draft=True,
- renders_final_reply=True,
- internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
- blocked_tool=tool_name,
- )
-
-
-def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:
- """Clear ``pending_reconciliation_run_id`` iff the matching resolved run
- landed in a status the caller can move past: any ``is_final_excluding_canceled``
- status, or any status (including ``canceled``) when the prior exit was an
- internal per-tool-budget cancel.
- """
- pending_run_id = getattr(copilot_ctx, "pending_reconciliation_run_id", None)
- if not isinstance(pending_run_id, str) or not pending_run_id:
- return
- if not isinstance(result, dict):
- return
- data = result.get("data")
- if not isinstance(data, dict):
- return
- resolved_run_id = data.get("workflow_run_id")
- resolved_status = data.get("overall_status")
- if not isinstance(resolved_run_id, str) or resolved_run_id != pending_run_id:
- return
- if not isinstance(resolved_status, str):
- return
- is_trusted_final = WorkflowRunStatus(resolved_status).is_final_excluding_canceled()
- # ``last_failure_category_top`` reflects the prior block-running tool's outcome —
- # only ``_record_run_blocks_result`` writes it, and the reconciliation guard
- # prevents another block-running call from clobbering it before this read.
- internal_watchdog_cancel_category = getattr(copilot_ctx, "last_failure_category_top", None) in {
- PER_TOOL_BUDGET_FAILURE_CATEGORY,
- ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
- }
- if is_trusted_final or internal_watchdog_cancel_category:
- copilot_ctx.pending_reconciliation_run_id = None
- copilot_ctx.pending_reconciliation_requires_user_input = False
- clear_blocker_signal_for_reason_codes(
- copilot_ctx,
- frozenset(
- {
- "tool_error_pending_reconciliation_no_input",
- "tool_error_pending_reconciliation_requires_input",
- }
- ),
- )
- return
- if resolved_status == WorkflowRunStatus.canceled.value:
- copilot_ctx.pending_reconciliation_requires_user_input = True
- # Replace the no_input blocker with the requires-input one; unrelated
- # blockers (e.g. `loop_detected`) survive the targeted clear.
- existing_blocker = getattr(copilot_ctx, "blocker_signal", None)
- # Preserve the original blocked_tool so trace queries filtering on it correlate the no_input → requires_input transition.
- original_blocked_tool = (
- existing_blocker.blocked_tool
- if (
- isinstance(existing_blocker, CopilotToolBlockerSignal)
- and existing_blocker.internal_reason_code == "tool_error_pending_reconciliation_no_input"
- and existing_blocker.blocked_tool
- )
- else "get_run_results"
- )
- clear_blocker_signal_for_reason_codes(
- copilot_ctx,
- frozenset({"tool_error_pending_reconciliation_no_input"}),
- )
- stash_blocker_signal(
- copilot_ctx,
- _pending_reconciliation_requires_input_signal(
- pending_run_id=pending_run_id,
- blocked_tool=original_blocked_tool,
- ),
- )
-
-
-def _mark_pending_reconciliation_run(copilot_ctx: Any, workflow_run_id: str) -> None:
- copilot_ctx.pending_reconciliation_run_id = workflow_run_id
- copilot_ctx.pending_reconciliation_requires_user_input = False
-
-
-def _workflow_verification_evidence(ctx: AgentContext) -> WorkflowVerificationEvidence:
- return ctx.workflow_verification_evidence
-
-
-def _run_result_label_list(data: Mapping[str, object], key: str) -> list[str]:
- values = data.get(key)
- if not isinstance(values, list):
- return []
- return list(dict.fromkeys(str(value).strip() for value in values if str(value).strip()))
-
-
-def _run_result_blocks(data: Mapping[str, object]) -> list[Mapping[str, object]]:
- blocks = data.get("blocks")
- return [block for block in blocks if isinstance(block, dict)] if isinstance(blocks, list) else []
-
-
-def _completed_run_block_labels(data: Mapping[str, object]) -> list[str]:
- labels = [
- str(block.get("label") or "").strip()
- for block in _run_result_blocks(data)
- if _enum_or_string_name(block.get("status")) == WorkflowRunStatus.completed.value
- ]
- labels = list(dict.fromkeys(label for label in labels if label))
- return labels or _run_result_label_list(data, "executed_block_labels")
-
-
-def _failed_run_block_labels(data: Mapping[str, object]) -> list[str]:
- labels = [
- str(block.get("label") or "").strip()
- for block in _run_result_blocks(data)
- if _enum_or_string_name(block.get("status")) in _FAILED_BLOCK_STATUSES
- ]
- labels = list(dict.fromkeys(label for label in labels if label))
- if labels:
- return labels
- frontier = data.get("frontier_start_label")
- return [frontier] if isinstance(frontier, str) and frontier.strip() else []
-
-
-def _enum_or_string_name(value: Any) -> str:
- raw = getattr(value, "value", value)
- if not isinstance(raw, str):
- raw = getattr(value, "name", raw)
- return str(raw).strip().lower()
-
-
-def _per_tool_budget_problem_label_set(ctx: Any) -> set[str]:
- labels = getattr(ctx, "per_tool_budget_problem_block_labels", None)
- if not isinstance(labels, list):
- return set()
- return {label for label in labels if isinstance(label, str) and label}
-
-
-def _record_per_tool_budget_problem_blocks_from_results(copilot_ctx: Any, result: dict[str, Any]) -> None:
- if getattr(copilot_ctx, "last_failure_category_top", None) != PER_TOOL_BUDGET_FAILURE_CATEGORY:
- return
- data = result.get("data")
- if not isinstance(data, dict):
- return
- pending_run_id = getattr(copilot_ctx, "pending_reconciliation_run_id", None)
- resolved_run_id = data.get("workflow_run_id")
- if isinstance(pending_run_id, str) and pending_run_id and resolved_run_id != pending_run_id:
- return
- blocks = data.get("blocks")
- if not isinstance(blocks, list):
- return
-
- labels = _per_tool_budget_problem_label_set(copilot_ctx)
- for block in blocks:
- if not isinstance(block, dict):
- continue
- if _enum_or_string_name(block.get("block_type")) != BlockType.NAVIGATION.value:
- continue
- if _enum_or_string_name(block.get("status")) not in _FAILED_BLOCK_STATUSES:
- continue
- label = block.get("label")
- if isinstance(label, str) and label:
- labels.add(label)
- copilot_ctx.per_tool_budget_problem_block_labels = sorted(labels)
-
-
-def _navigation_labels_in_workflow(workflow: Any) -> set[str]:
- definition = getattr(workflow, "workflow_definition", None)
- blocks = getattr(definition, "blocks", None)
- if not isinstance(blocks, list):
- return set()
-
- labels: set[str] = set()
- for block in blocks:
- if _enum_or_string_name(getattr(block, "block_type", None)) != BlockType.NAVIGATION.value:
- continue
- label = getattr(block, "label", None)
- if isinstance(label, str) and label:
- labels.add(label)
- return labels
-
-
-def _clear_resolved_per_tool_budget_problem_labels(copilot_ctx: Any, workflow: Any) -> None:
- problem_labels = _per_tool_budget_problem_label_set(copilot_ctx)
- if not problem_labels:
- return
- remaining = sorted(problem_labels & _navigation_labels_in_workflow(workflow))
- copilot_ctx.per_tool_budget_problem_block_labels = remaining
-
-
-def _requested_block_label_set(arguments: dict[str, Any] | None) -> set[str] | None:
- if not isinstance(arguments, dict):
- return None
- block_labels = arguments.get("block_labels")
- if not isinstance(block_labels, list):
- return None
- return {label for label in block_labels if isinstance(label, str) and label}
-
-
-def _per_tool_budget_problem_rerun_signal(
- ctx: Any, arguments: dict[str, Any] | None, tool_name: str
-) -> CopilotToolBlockerSignal | None:
- problem_labels = _per_tool_budget_problem_label_set(ctx)
- if not problem_labels:
- return None
-
- requested_labels = _requested_block_label_set(arguments)
- blocked_labels = problem_labels if not requested_labels else problem_labels & requested_labels
- if not blocked_labels:
- return None
-
- labels = ", ".join(sorted(blocked_labels))
- agent_steering = (
- "The prior PER_TOOL_BUDGET run's get_run_results showed navigation "
- f"block(s) [{labels}] were canceled or failed while applying page state. "
- f"Do NOT rerun those block label(s) unchanged with {tool_name}. "
- "Update the workflow to split or replace the oversized navigation block first, "
- "use live-page inspection evidence to decide what state is actually missing, "
- "or run only newly-created smaller block labels that apply one missing constraint at a time."
- )
- user_facing = (
- "The previous run hit the per-step time budget before I could finish. "
- "I'll continue from the verified browser state instead of retrying blindly."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=PAGE_INSPECTION_TOOLS
- | frozenset({"get_run_results", "update_workflow", "update_and_run_blocks"}),
- internal_reason_code="tool_error_per_tool_budget_rerun",
- blocked_tool=tool_name,
- )
-
-
-def _workflow_yaml_ordered_labels(workflow_yaml: str | None) -> list[str]:
- blocks = _parse_workflow_blocks(workflow_yaml)
- if not blocks:
- return []
- labels: list[str] = []
- for block in blocks:
- if not isinstance(block, dict):
- continue
- label = _block_label_from_yaml(block)
- if label:
- labels.append(label)
- return labels
-
-
-def _post_budget_upstream_replay_signal(
- ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str
-) -> CopilotToolBlockerSignal | None:
- problem_labels = _per_tool_budget_problem_label_set(ctx)
- if not problem_labels:
- return None
-
- evidence = getattr(ctx, "composition_page_evidence", None)
- if not isinstance(evidence, dict) or evidence.get("observed_after_workflow_run") is not True:
- return None
-
- requested_labels = _requested_block_label_set(arguments)
- if not requested_labels:
- return None
-
- workflow_yaml = arguments.get("workflow_yaml") if isinstance(arguments, dict) else None
- ordered_labels = _workflow_yaml_ordered_labels(workflow_yaml if isinstance(workflow_yaml, str) else None)
- if not ordered_labels:
- ordered_labels = _current_workflow_block_labels(ctx)
- if not ordered_labels:
- return None
-
- first_problem_index = min(
- (ordered_labels.index(label) for label in problem_labels if label in ordered_labels),
- default=None,
- )
- if first_problem_index is None:
- return None
-
- upstream_requested = [label for label in ordered_labels[:first_problem_index] if label in requested_labels]
- if not upstream_requested:
- return None
-
- upstream = ", ".join(upstream_requested)
- frontier = ", ".join(sorted(problem_labels))
- agent_steering = (
- "The prior PER_TOOL_BUDGET run advanced the live browser, and you already inspected that "
- "post-run page state. Do NOT restart upstream label(s) "
- f"[{upstream}] before the budgeted frontier [{frontier}]. "
- "Answer from the observed page if it contains the requested result/no-result evidence. "
- "If more workflow testing is still required, preserve the verified upstream blocks and run only "
- "new or modified smaller labels at/after the missing frontier."
- )
- user_facing = (
- "The previous run already reached a later browser state. I'll continue from that evidence "
- "instead of restarting earlier steps."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=PAGE_INSPECTION_TOOLS
- | frozenset({"get_run_results", "update_workflow", "update_and_run_blocks"}),
- preserves_workflow_draft=True,
- renders_final_reply=False,
- internal_reason_code="tool_error_post_budget_upstream_replay",
- blocked_tool=tool_name,
- )
-
-
-def _composition_control_label(control: dict[str, Any]) -> str:
- parts: list[str] = []
- for key in ("text", "name", "id", "selector"):
- value = control.get(key)
- if isinstance(value, str) and value.strip():
- parts.append(value.strip())
- unique = list(dict.fromkeys(parts))
- return " / ".join(unique[:2])[:160] if unique else "submit/search control"
-
-
-def _disabled_submit_controls_from_evidence(evidence: dict[str, Any]) -> list[dict[str, Any]]:
- controls: list[dict[str, Any]] = []
- forms = evidence.get("forms")
- if not isinstance(forms, list):
- return controls
- for form in forms:
- if not isinstance(form, dict):
- continue
- for control in form.get("submit_controls") or []:
- if isinstance(control, dict) and control.get("disabled") is True:
- controls.append(control)
- return controls[:5]
-
-
-def _post_run_terminal_challenge_reason(evidence: dict[str, Any]) -> str | None:
- if evidence.get("observed_after_workflow_run") is not True:
- return None
-
- challenge_state = evidence.get("challenge_state")
- if isinstance(challenge_state, dict):
- gated_controls = [
- control for control in challenge_state.get("gated_submit_controls") or [] if isinstance(control, dict)
- ]
- if challenge_state.get("gates_submit_controls") is True:
- kind = str(challenge_state.get("kind") or "anti-bot challenge").replace("_", " ")
- labels = ", ".join(_composition_control_label(control) for control in gated_controls[:3])
- controls_text = f" ({labels})" if labels else ""
- return f"{kind} is still gating disabled submit/search control(s){controls_text}"
- if challenge_state.get("detected") is True and challenge_state.get("requires_human_verification") is True:
- disabled_controls = _disabled_submit_controls_from_evidence(evidence)
- if disabled_controls:
- labels = ", ".join(_composition_control_label(control) for control in disabled_controls[:3])
- return f"human-verification challenge remains while submit/search control(s) are disabled ({labels})"
-
- indicators = evidence.get("anti_bot_indicators")
- has_indicators = isinstance(indicators, list) and any(isinstance(item, str) and item.strip() for item in indicators)
- if has_indicators:
- disabled_controls = _disabled_submit_controls_from_evidence(evidence)
- if disabled_controls:
- labels = ", ".join(_composition_control_label(control) for control in disabled_controls[:3])
- return f"anti-bot evidence remains while submit/search control(s) are disabled ({labels})"
- return None
-
-
-def _post_budget_terminal_challenge_signal(
- ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str
-) -> CopilotToolBlockerSignal | None:
- evidence = getattr(ctx, "composition_page_evidence", None)
- if not isinstance(evidence, dict):
- return None
-
- # A failed run with post-run page evidence can prove the same terminal
- # challenge state even when the top-level failure category was not budget.
- prior_budget_or_failed_run = (
- getattr(ctx, "last_failure_category_top", None) == PER_TOOL_BUDGET_FAILURE_CATEGORY
- or getattr(ctx, "last_test_ok", None) is False
- or getattr(ctx, "post_run_page_observation_after_failed_test", False) is True
- or bool(_per_tool_budget_problem_label_set(ctx))
- )
- if not prior_budget_or_failed_run:
- return None
-
- reason = _post_run_terminal_challenge_reason(evidence)
- if not reason:
- return None
-
- requested_labels = _requested_block_label_set(arguments)
- labels_text = f" Requested labels: {', '.join(sorted(requested_labels))}." if requested_labels else ""
- agent_steering = (
- "The prior block-running tool hit a failed/budgeted frontier, and bounded current-page inspection "
- f"now shows: {reason}.{labels_text} Do NOT call "
- f"{tool_name} again in this turn, do NOT try another proxy/location from this evidence state, and "
- "do NOT claim registry results or no-results were verified. REPLY now with a blocker explanation "
- "that names the observed challenge/disabled control and summarizes the tested workflow state."
- )
- user_facing = (
- "The site's verification challenge is still blocking the submit/search control after live-page inspection, "
- "so I stopped without claiming results."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- preserves_workflow_draft=True,
- renders_final_reply=True,
- internal_reason_code="tool_error_post_budget_challenge_blocker",
- blocked_tool=tool_name,
- )
-
-
-_RECONCILIATION_REQUIRES_INPUT_USER_FACING = (
- "The previous run was canceled. Tell me whether to retry, keep the draft as-is, or adjust the workflow first."
-)
-_RECONCILIATION_NO_INPUT_USER_FACING = (
- "The previous run ended without a verified result. I'll check what happened before doing anything else this turn."
-)
-
-
-def _pending_reconciliation_requires_input_signal(
- *, pending_run_id: str, blocked_tool: str
-) -> CopilotToolBlockerSignal:
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=(
- f"The canceled run {pending_run_id} has already been inspected. "
- f"Do NOT run more blocks in this turn; ask the user whether to retry, "
- f"accept the unverified draft, or adjust the workflow first. This guard "
- f"prevents duplicate side effects on live sites."
- ),
- user_facing_reason=_RECONCILIATION_REQUIRES_INPUT_USER_FACING,
- recovery_hint="ask_user_clarifying",
- cleared_by_tools=frozenset(),
- internal_reason_code="tool_error_pending_reconciliation_requires_input",
- blocked_tool=blocked_tool,
- )
-
-
-def _pending_reconciliation_no_input_signal(*, pending_run_id: str, blocked_tool: str) -> CopilotToolBlockerSignal:
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=(
- f"The previous block-running tool call for run {pending_run_id} "
- f"ended without a trustworthy terminal status. "
- f'Call `get_run_results(workflow_run_id="{pending_run_id}")` '
- f"first, report the result to the user, then await user input "
- f"before running more blocks. This guard prevents duplicate "
- f"side effects on live sites."
- ),
- user_facing_reason=_RECONCILIATION_NO_INPUT_USER_FACING,
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- internal_reason_code="tool_error_pending_reconciliation_no_input",
- blocked_tool=blocked_tool,
- )
-
-
-# Streak threshold at which the copilot hard-aborts a tool call because the
-# same action sequence has repeated run-over-run with no intervening success.
-# The streak counter is incremented in ``update_repeated_failure_state`` AFTER
-# each run, so the abort fires when the 4th consecutive run against the same
-# action fingerprint enters ``_tool_loop_error`` (streak == 3 at entry, one
-# per each of the three preceding identical runs). Calibration note: the
-# repeated-frontier streak in failure_tracking.py uses STOP_AT=3 for the same
-# shape of escalation.
-REPEATED_ACTION_STREAK_ABORT_AT = 3
-MAX_CHALLENGE_GATED_PROXY_RETRIES = 1
-
-_STRUCTURED_BLOCKER_KEY_TERMS: frozenset[str] = frozenset(
- {
- "blocker",
- "blocked",
- "captcha",
- "challenge",
- "human_verification",
- "verification",
- }
-)
-_STRUCTURED_BLOCKER_MESSAGE_KEYS: frozenset[str] = frozenset(
- {
- "blocker_message",
- "blocked_message",
- "captcha_message",
- "challenge_message",
- "human_verification_message",
- }
-)
-_ANTI_BOT_BLOCKER_TERMS: tuple[str, ...] = (
- "access denied",
- "anti-bot",
- "bot block",
- "captcha",
- "challenge",
- "human verification",
- "verify you are human",
-)
-
-
-def _normalize_structured_key(value: object) -> str:
- return str(value or "").strip().lower().replace("-", "_").replace(" ", "_")
-
-
-def _looks_like_anti_bot_blocker(text: str) -> bool:
- lowered = text.lower()
- return any(term in lowered for term in _ANTI_BOT_BLOCKER_TERMS)
-
-
-def _structured_blocker_message(value: object, *, depth: int = 0) -> str | None:
- if depth > 5:
- return None
- if isinstance(value, dict):
- for key, item in value.items():
- normalized_key = _normalize_structured_key(key)
- if not isinstance(item, str) or not item.strip():
- continue
- has_blocker_key = normalized_key in _STRUCTURED_BLOCKER_MESSAGE_KEYS or any(
- term in normalized_key for term in _STRUCTURED_BLOCKER_KEY_TERMS
- )
- if has_blocker_key or (
- normalized_key in {"message", "error", "failure_reason", "reason"}
- and _looks_like_anti_bot_blocker(item)
- ):
- return item.strip()[:240]
- for item in value.values():
- nested = _structured_blocker_message(item, depth=depth + 1)
- if nested:
- return nested
- elif isinstance(value, list):
- for item in value:
- nested = _structured_blocker_message(item, depth=depth + 1)
- if nested:
- return nested
- return None
-
-
-def _run_blocks_structured_blocker_message(result: dict[str, Any]) -> str | None:
- data = result.get("data")
- if not isinstance(data, dict):
- return None
- direct = _structured_blocker_message({key: value for key, value in data.items() if key != "blocks"})
- if direct:
- return direct
- blocks = data.get("blocks")
- if not isinstance(blocks, list):
- return None
- for block in blocks:
- if not isinstance(block, dict):
- continue
- block_type = block.get("block_type")
- if block_type not in _DATA_PRODUCING_BLOCK_TYPES or block.get("status") != "completed":
- continue
- payload = _block_data_payload(block.get("extracted_data"), block_type)
- blocker = _structured_blocker_message(payload)
- if blocker:
- return blocker
- return None
-
-
-def _is_meaningful_extracted_data(extracted: Any) -> bool:
- """Return True when extracted data contains at least one non-null, non-empty value.
-
- A dict like ``{"price": None}`` is technically present but carries no signal —
- treat it the same as no output at all so enforcement can nudge the agent to
- investigate instead of declaring success.
- """
- if extracted is None:
- return False
- if isinstance(extracted, (str, bytes)):
- return bool(extracted)
- if isinstance(extracted, dict):
- return any(_is_meaningful_extracted_data(v) for v in extracted.values())
- if isinstance(extracted, (list, tuple, set)):
- return any(_is_meaningful_extracted_data(v) for v in extracted)
- # Numbers, booleans, and other scalars count as meaningful output.
- return True
-
-
-# Payload fields inside a ``TaskOutput.from_task()`` envelope
-# (schemas/tasks.py:TaskOutput). Only these carry "did the block produce
-# something useful?" signal; the rest (task_id, status, artifact IDs, etc.)
-# are always populated on a completed run and would short-circuit
-# _is_meaningful_extracted_data to True even when nothing useful was produced.
-_TASK_OUTPUT_PAYLOAD_FIELDS: tuple[str, ...] = (
- "extracted_information",
- "downloaded_files",
- "downloaded_file_urls",
-)
-
-
-def _block_data_payload(extracted_data: Any, block_type: str | None) -> Any:
- """Return the payload view of a block's output for the meaningful-data check.
-
- For task-envelope block types (``_TASK_ENVELOPE_BLOCK_TYPES``), slice the
- envelope down to ``_TASK_OUTPUT_PAYLOAD_FIELDS`` so envelope metadata
- can't mask an empty result. Other data-producing types pass through
- unchanged — e.g. TEXT_PROMPT's ``block.output`` is the raw LLM response
- dict (TextPromptBlock.execute records ``output_parameter_value=response``
- directly), so scoping the unwrap avoids slicing a user-defined
- json_schema that happens to include an ``extracted_information`` field.
- """
- if block_type in _TASK_ENVELOPE_BLOCK_TYPES and isinstance(extracted_data, dict):
- return {field: extracted_data.get(field) for field in _TASK_OUTPUT_PAYLOAD_FIELDS}
- return extracted_data
-
-
-async def _attach_action_traces(
- blocks: list,
- results: list[dict[str, Any]],
- organization_id: str,
-) -> None:
- """For non-success blocks with a task_id, fetch and attach a compact action trace."""
- failed_task_ids = [
- b.task_id for b, r in zip(blocks, results) if b.task_id and r.get("status") in _FAILED_BLOCK_STATUSES
- ]
- if not failed_task_ids:
- return
-
- rows = await app.DATABASE.tasks.get_recent_actions_for_tasks(
- task_ids=failed_task_ids,
- organization_id=organization_id,
- )
-
- actions_by_task: dict[str, list] = defaultdict(list)
- for row in rows:
- if row.task_id is not None:
- actions_by_task[row.task_id].append(row)
-
- for block, block_result in zip(blocks, results):
- if block_result.get("status") not in _FAILED_BLOCK_STATUSES or not block.task_id:
- continue
- task_actions = actions_by_task.get(block.task_id, [])
- block_result["action_trace"] = [
- {
- "action": a.action_type,
- "status": a.status,
- "reasoning": a.reasoning[:150] if a.reasoning else None,
- "element": a.element_id,
- }
- for a in task_actions
- ]
-
-
-async def _fetch_last_screenshot_b64(task_id: str, organization_id: str) -> str | None:
- try:
- artifacts = await app.DATABASE.artifacts.get_artifacts_for_task_v2(
- task_v2_id=task_id,
- organization_id=organization_id,
- artifact_types=[ArtifactType.SCREENSHOT_LLM],
- )
- if not artifacts:
- return None
- # The last artifact is the one captured closest to the failure.
- artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifacts[-1])
- if not artifact_bytes:
- return None
- return base64.b64encode(artifact_bytes).decode("utf-8")
- except Exception:
- LOG.debug("Failed to fetch screenshot for failed block", task_id=task_id, exc_info=True)
- return None
-
-
-async def _attach_failed_block_screenshots(
- blocks: list,
- results: list[dict[str, Any]],
- organization_id: str,
-) -> None:
- """For failed blocks with a task_id, fetch the last SCREENSHOT_LLM artifact."""
- task_id_to_block: dict[str, dict] = {
- block.task_id: block_result
- for block, block_result in zip(blocks, results)
- if block.task_id and block_result.get("status") in _FAILED_BLOCK_STATUSES
- }
- if not task_id_to_block:
- return
-
- task_ids = list(task_id_to_block.keys())
- screenshots = await asyncio.gather(
- *(_fetch_last_screenshot_b64(task_id, organization_id) for task_id in task_ids),
- )
- for task_id, b64 in zip(task_ids, screenshots):
- if b64 is not None:
- task_id_to_block[task_id]["screenshot_b64"] = b64
-
-
-BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run_blocks"})
-WORKFLOW_MUTATION_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})
-ANSWER_ONLY_CONTEXT_TOOLS = frozenset({"get_run_results"})
-CREDENTIAL_METADATA_TOOLS = frozenset({"list_credentials"})
-PAGE_INSPECTION_TOOLS = frozenset({"inspect_page_for_composition", "evaluate", "get_browser_screenshot"})
-PAGE_SCHEMA_CONTEXT_TOOLS = frozenset({"inspect_page_for_composition"})
-_CURRENT_PAGE_INSPECTION_TARGETS = frozenset({"", "current", "current_page", "__current_page__"})
-
-
-def _copilot_seconds_remaining(ctx: AgentContext) -> float | None:
- started_at = getattr(ctx, "copilot_run_start_monotonic", None)
- if not isinstance(started_at, int | float):
- return None
- return TOTAL_TIMEOUT_SECONDS - (time.monotonic() - float(started_at))
-
-
-def _active_block_run_budget_seconds(ctx: AgentContext) -> int:
- remaining = _copilot_seconds_remaining(ctx)
- if remaining is None:
- return PER_TOOL_CALL_BUDGET_SECONDS
- remaining_after_reply_reserve = remaining - COPILOT_FINAL_REPLY_RESERVE_SECONDS
- return max(1, min(PER_TOOL_CALL_BUDGET_SECONDS, int(remaining_after_reply_reserve)))
-
-
-def _late_block_running_call_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- remaining = _copilot_seconds_remaining(ctx)
- if remaining is None or remaining > COPILOT_FINAL_REPLY_RESERVE_SECONDS:
- return None
-
- last_failed_workflow_yaml = getattr(ctx, "last_failed_workflow_yaml", None)
- last_good_workflow_yaml = getattr(ctx, "last_good_workflow_yaml", None)
- if (
- isinstance(last_failed_workflow_yaml, str)
- and last_failed_workflow_yaml
- and isinstance(last_good_workflow_yaml, str)
- and last_good_workflow_yaml
- ):
- agent_steering = (
- f"Wall-clock budget too low to retry: about {int(max(0.0, remaining))}s remain of the "
- f"{TOTAL_TIMEOUT_SECONDS}s session budget. A verified workflow exists from before the failure. "
- "Do NOT call update_and_run_blocks or run_blocks_and_collect_debug again. REPLY now: summarize "
- "what worked, name the block that failed, and tell the user they can keep the verified prefix or discard."
- )
- elif isinstance(last_failed_workflow_yaml, str) and last_failed_workflow_yaml:
- agent_steering = (
- f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn "
- "after the previous workflow run failed. Do NOT retry block-running tools. Use only existing "
- "run evidence and quick browser inspection tools such as get_run_results, evaluate, or "
- "get_browser_screenshot if one more read is needed. If the current page contains the requested "
- "answer, answer from that observed page evidence. If evidence is incomplete, report exactly "
- "which browser state was verified and which requested data remains unverified. Never repeat "
- "this tool-error text as the user-facing answer."
- )
- else:
- agent_steering = (
- f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn. "
- "Do NOT start another block-running tool call; reply to the user with the workflow draft and "
- "progress gathered so far, and make clear which parts have not been verified end-to-end."
- )
-
- user_facing = "I'm running out of time on this turn. I'll wrap up with what I have so far."
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="stop",
- cleared_by_tools=frozenset(),
- # The "wrap up with what we have" semantic means the draft saved
- # earlier in the turn should still surface — only the chat reply
- # is overridden by the renderer.
- preserves_workflow_draft=True,
- renders_final_reply=False,
- internal_reason_code="tool_error_late_block_running",
- blocked_tool=tool_name,
- )
-
-
-def _mark_page_inspected(ctx: AgentContext) -> None:
- ctx.post_budget_page_inspection_required = False
- ctx.post_budget_page_inspection_url = None
- ctx.post_budget_page_inspection_run_id = None
-
-
-def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
- if not left or not right:
- return False
- try:
- left_parsed = urlparse(left)
- right_parsed = urlparse(right)
- except Exception as exc:
- LOG.debug("copilot_same_page_url_parse_failed", left=left, right=right, error=str(exc))
- return False
- left_url = left_parsed._replace(fragment="").geturl().rstrip("/")
- right_url = right_parsed._replace(fragment="").geturl().rstrip("/")
- return left_url == right_url
-
-
-def _clear_pending_browser_interaction_observation(ctx: AgentContext) -> None:
- ctx.pending_browser_interaction_observation = None
-
-
-def _mark_pending_browser_interaction_observation(ctx: AgentContext, *, tool_name: str, url: str) -> None:
- if not url.strip():
- _clear_pending_browser_interaction_observation(ctx)
- return
- ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(
- tool_name=tool_name,
- url=url.strip(),
- )
-
-
-def _consume_pending_browser_interaction_observation(
- ctx: AgentContext,
- *,
- current_url: str,
- evidence: dict[str, Any],
-) -> bool:
- pending = ctx.pending_browser_interaction_observation
- if pending is None:
- return False
- _clear_pending_browser_interaction_observation(ctx)
- if not has_bounded_page_schema(evidence):
- return False
- if not _same_page_ignoring_fragment(pending.url, current_url):
- LOG.warning(
- "copilot_pending_browser_interaction_observation_page_mismatch",
- tool_name=pending.tool_name,
- pending_url=pending.url,
- current_url=current_url,
- )
- return False
- return True
-
-
-_MAX_SCOUTED_INTERACTIONS = 20
-
-
-async def _live_working_page_url(ctx: AgentContext) -> str | None:
- if not ctx.browser_session_id:
- return None
- try:
- browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
- session_id=ctx.browser_session_id,
- organization_id=ctx.organization_id,
- )
- if not browser_state:
- return None
- page = await browser_state.get_or_create_page()
- return page.url if page else None
- except Exception:
- return None
-
-
-async def _capture_scout_source_url(ctx: AgentContext) -> None:
- # Pre-action: a navigating click/Enter would leave only the destination URL, not the page the selector acted on.
- ctx.pending_scout_source_url = await _live_working_page_url(ctx)
-
-
-def _consume_scout_source_url(ctx: AgentContext) -> str | None:
- source_url = ctx.pending_scout_source_url
- # Cleared unconditionally so a non-recording action can't bleed its source page into a later interaction.
- ctx.pending_scout_source_url = None
- return source_url
-
-
-_ROLE_NAME_SELECTOR_RE = re.compile(r'^role=([a-zA-Z]+)(?:\[name="((?:[^"\\]|\\.)*)"\])?(.*)$')
-
-
-def _role_name_from_selector(selector: str) -> tuple[str, str] | None:
- """Parse the ``role=<role>[name="<name>"]`` form (ref_to_selector) — TIER 1, no browser read.
-
- Returns (role, accessible_name) when the selector is a plain role/name locator;
- None for bare CSS/xpath or when an engine chain (`>> nth=`) trails the role/name.
- """
- selector = selector.strip()
- match = _ROLE_NAME_SELECTOR_RE.match(selector)
- if not match:
- return None
- role, raw_name, suffix = match.group(1), match.group(2), match.group(3)
- if suffix.strip():
- return None
- name = raw_name.replace('\\"', '"') if raw_name is not None else ""
- return role, name
-
-
-async def _capture_accessible_role_name(ctx: AgentContext, selector: str) -> tuple[str, str] | None:
- """TIER 2: read the element's role/accessible name for a bare CSS/xpath selector.
-
- A failed read degrades gracefully to None so the selector-only auto-credit
- path (SKY-10712) stays intact.
- """
- selector = selector.strip()
- if not selector:
- return None
- server = getattr(ctx, "discovery_mcp_server", None)
- if server is None:
- return None
- try:
- result = await asyncio.wait_for(
- server.call_internal_tool(
- "skyvern_evaluate",
- {"expression": _scout_accessible_role_name_expression(selector)},
- ),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except Exception:
- return None
- if not isinstance(result, dict) or not result.get("ok"):
- return None
- value = (result.get("data") or {}).get("result")
- if not isinstance(value, dict):
- return None
- role = str(value.get("role") or "").strip()
- name = str(value.get("accessible_name") or "").strip()
- if not role and not name:
- return None
- return role, name
-
-
-async def _resolve_scout_role_name(
- ctx: AgentContext, selector: str, *, allow_browser_read: bool = True
-) -> tuple[str, str]:
- """Resolve (role, accessible_name) for a scouted selector. TIER 1 parse first;
- TIER 2 browser read only for bare CSS/xpath. Always degrades to ("", "").
-
- ``allow_browser_read=False`` skips TIER 2 when the action navigated: a post-action
- read against the landing page would capture the wrong element's name, so the bare
- selector is kept verbatim (the synthesizer prefers it anyway)."""
- selector = selector.strip()
- if not selector:
- return "", ""
- parsed = _role_name_from_selector(selector)
- if parsed is not None:
- return parsed
- if not allow_browser_read:
- return "", ""
- captured = await _capture_accessible_role_name(ctx, selector)
- if captured is not None:
- return captured
- return "", ""
-
-
-def _record_scouted_interaction(
- ctx: AgentContext,
- *,
- tool_name: str,
- selector: str = "",
- source_url: str | None = None,
- value: str = "",
- key: str = "",
- typed_length: int = 0,
- role: str = "",
- accessible_name: str = "",
-) -> None:
- selector = selector.strip()
- # press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
- if tool_name != "press_key" and not selector:
- return
- artifact: ScoutedInteraction = {"tool_name": tool_name}
- if selector:
- artifact["selector"] = selector
- if source_url and source_url.strip():
- artifact["source_url"] = source_url.strip()
- if value:
- artifact["value"] = value
- if key:
- artifact["key"] = key
- if typed_length:
- artifact["typed_length"] = typed_length
- if role:
- artifact["role"] = role
- if accessible_name:
- artifact["accessible_name"] = accessible_name
- interactions = [
- item
- for item in ctx.scouted_interactions
- if not (
- item.get("tool_name") == artifact["tool_name"]
- and item.get("selector") == artifact.get("selector")
- and item.get("source_url") == artifact.get("source_url")
- )
- ]
- interactions.append(artifact)
- ctx.scouted_interactions = interactions[-_MAX_SCOUTED_INTERACTIONS:]
-
- trajectory = list(ctx.scout_trajectory)
- trajectory_artifact = cast(ScoutedInteraction, artifact.copy())
- trajectory_artifact["trajectory_index"] = len(trajectory)
- trajectory.append(trajectory_artifact)
- ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]
-
- LOG.info(
- "copilot_scout_interaction_captured",
- tool_name=tool_name,
- selector=selector or None,
- source_url=artifact.get("source_url"),
- role=role or None,
- total_scouted_interactions=len(ctx.scouted_interactions),
- total_scout_trajectory=len(ctx.scout_trajectory),
- )
-
-
-def _register_scout_interaction_observation(
- ctx: AgentContext, *, tool_name: str, selector: str, source_url: str | None, url: str
-) -> int | None:
- # A successful scout interaction reaches the post-action page; record it as an
- # interaction-reached observation so a click-reached block can be authored
- # against it without a separate inspect_page_for_composition.
- selector = selector.strip()
- if not selector or not url:
- return None
- evidence: dict[str, Any] = {
- "inspected_url": url,
- "current_url": url,
- "source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,
- "interaction_tool": tool_name,
- "interaction_selector": selector,
- }
- if source_url and source_url.strip():
- evidence["interaction_source_url"] = source_url.strip()
- return _append_flow_evidence(ctx, evidence, reached_via="interaction")
-
-
-def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:
- run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
- if not isinstance(run_id, str) or not run_id:
- return
- ctx.post_run_page_observation_tool = source_tool
- ctx.post_run_page_observation_url = url
- ctx.post_run_page_observation_workflow_run_id = run_id
- ctx.post_run_page_observation_after_failed_test = getattr(ctx, "last_test_ok", None) is False
- evidence = _workflow_verification_evidence(ctx)
- evidence.live_page_state_verified = True
- evidence.verified_from_current_browser_state = True
- evidence.workflow_run_id = run_id
- if url:
- evidence.current_url = url
- evidence.current_url_observed_after_workflow_run = True
- evidence.current_url_may_encode_runtime_state = bool(urlparse(url).query)
-
-
-def _allows_post_run_current_page_inspection_budget_bypass(ctx: AgentContext, *, use_current_page: bool) -> bool:
- if not use_current_page:
- return False
- run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
- if not isinstance(run_id, str) or not run_id:
- return False
- if getattr(ctx, "last_test_ok", None) is None:
- return False
- return getattr(ctx, "post_run_current_page_inspection_workflow_run_id", None) != run_id
-
-
-def _post_budget_page_inspection_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- if getattr(ctx, "post_budget_page_inspection_required", False) is not True:
- return None
-
- url = getattr(ctx, "post_budget_page_inspection_url", None)
- run_id = getattr(ctx, "post_budget_page_inspection_run_id", None)
- url_text = f" at {url}" if isinstance(url, str) and url else ""
- run_text = f" for run {run_id}" if isinstance(run_id, str) and run_id else ""
- agent_steering = (
- f"The prior PER_TOOL_BUDGET run{run_text} advanced the live browser{url_text}. "
- "Before another block-running tool, inspect the current browser page with "
- 'inspect_page_for_composition(target_url="current_page"). Generic screenshot/evaluate reads can '
- "help answer the user, but they do not satisfy the bounded page-evidence contract for workflow "
- "mutations. If the observed page evidence already contains the requested result or a no-results "
- "state, answer from that evidence instead of rerunning the search. If evidence shows a missing "
- "page-state change, then run only the smaller missing block after that inspection."
- )
- user_facing = "I need to inspect the current page state before running more steps."
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=PAGE_SCHEMA_CONTEXT_TOOLS,
- preserves_workflow_draft=True,
- renders_final_reply=False,
- internal_reason_code="tool_error_post_budget_page_inspection_required",
- blocked_tool=tool_name,
- )
-
-
-def _proxy_value_signature(value: object) -> str:
- try:
- return json.dumps(value, sort_keys=True, default=str)
- except (TypeError, ValueError):
- return str(value)
-
-
-def _anti_bot_retry_changes_proxy(ctx: AgentContext, arguments: dict[str, Any] | None) -> bool:
- if not isinstance(arguments, dict):
- return False
- workflow_yaml = arguments.get("workflow_yaml")
- if not isinstance(workflow_yaml, str) or not workflow_yaml.strip():
- return False
-
- prior_yaml = getattr(ctx, "last_failed_workflow_yaml", None) or getattr(ctx, "workflow_yaml", None)
- if not isinstance(prior_yaml, str) or not prior_yaml.strip():
- return False
-
- new_proxy_present, new_proxy = _raw_yaml_proxy_location(workflow_yaml)
- if not new_proxy_present:
- return False
- old_proxy_present, old_proxy = _raw_yaml_proxy_location(prior_yaml)
- return (not old_proxy_present) or _proxy_value_signature(new_proxy) != _proxy_value_signature(old_proxy)
-
-
-def _challenge_gated_proxy_retry_allowed(ctx: AgentContext, arguments: dict[str, Any] | None) -> bool:
- if not _anti_bot_retry_changes_proxy(ctx, arguments):
- return False
- retry_count = getattr(ctx, "challenge_gated_proxy_retry_count", 0)
- if not isinstance(retry_count, int):
- retry_count = 0
- if retry_count >= MAX_CHALLENGE_GATED_PROXY_RETRIES:
- return False
- ctx.challenge_gated_proxy_retry_count = retry_count + 1
- return True
-
-
-def _last_run_has_terminal_anti_bot_blocker(ctx: AgentContext) -> bool:
- anti_bot_reason = getattr(ctx, "last_test_anti_bot", None)
- if not isinstance(anti_bot_reason, str) or not anti_bot_reason.strip():
- return False
- failure_reason = getattr(ctx, "last_test_failure_reason", None)
- if not isinstance(failure_reason, str) or not failure_reason.strip():
- return False
- lowered = failure_reason.lower()
- if "blocker" in lowered and _looks_like_anti_bot_blocker(lowered):
- return True
-
- evidence = getattr(ctx, "composition_page_evidence", None)
- challenge_state = evidence.get("challenge_state") if isinstance(evidence, dict) else None
- challenge_gates_submit = isinstance(challenge_state, dict) and challenge_state.get("gates_submit_controls") is True
- if not (challenge_gates_submit or "challenge-gated disabled submit/search control" in anti_bot_reason):
- return False
-
- return "disabled" in lowered and any(
- term in lowered for term in ("submit", "search", "button", "control", "element")
- )
-
-
-def _challenge_gated_anti_bot_rerun_signal(
- ctx: AgentContext,
- arguments: dict[str, Any] | None,
- tool_name: str,
-) -> CopilotToolBlockerSignal | None:
- if not _last_run_has_terminal_anti_bot_blocker(ctx):
- return None
- if tool_name == "update_and_run_blocks" and _challenge_gated_proxy_retry_allowed(ctx, arguments):
- return None
-
- failure_reason = getattr(ctx, "last_test_failure_reason", "")
- agent_steering = (
- "The prior run confirmed an anti-bot challenge or blocker on the submit/search path, "
- f"and the latest failure_reason was: {str(failure_reason)[:240]}. Do NOT call "
- f"{tool_name} again with the same workflow/browser path. REPLY now with a blocker "
- "explanation that names the observed challenge/blocker or disabled submit/search control "
- "and describes what was tried. "
- "Ask whether to try a materially different proxy/location, entrypoint, or alternate source."
- )
- user_facing = (
- "The site's verification challenge is still keeping the submit/search control disabled, so I stopped "
- "instead of retrying the same workflow path."
- )
- return CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="report_blocker_to_user",
- cleared_by_tools=frozenset(),
- preserves_workflow_draft=True,
- renders_final_reply=True,
- internal_reason_code="tool_error_challenge_gated_submit_disabled",
- blocked_tool=tool_name,
- )
-
-
-def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:
- detected = detect_failed_tool_step_loop_for_ctx(ctx, tool_name, arguments or {})
- if detected is not None:
- return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
-
- # Consecutive same-name guard: false-positives on the intended iterative
- # build (one new block per update_and_run_blocks). Block-running tools
- # rely on the progress-aware checks below instead.
- tracker = getattr(ctx, "consecutive_tool_tracker", None)
- if isinstance(tracker, list) and tool_name not in BLOCK_RUNNING_TOOLS:
- detected = detect_tool_loop(tracker, tool_name)
- if detected is not None:
- return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
-
- if tool_name == "update_workflow" or tool_name in BLOCK_RUNNING_TOOLS:
- active_terminal_signal = _active_run_terminal_evidence_signal(ctx, tool_name)
- if active_terminal_signal is not None:
- return _emit_tool_blocker_signal(ctx, active_terminal_signal)
-
- # Hard-abort when the agent has re-fired the same action sequence against
- # the page N times without intervening success. This is the signal that
- # the form is blocked (captcha / anti-bot / error banner the agent isn't
- # detecting) and further attempts will just burn the tool timeout. Scoped
- # to block-running tools so planning/metadata tools (update_workflow,
- # list_credentials, get_run_results) stay unaffected.
- if tool_name in BLOCK_RUNNING_TOOLS:
- # Reconciliation guard: the previous block-running tool call exited
- # without a trustworthy terminal status for its workflow run (the
- # watchdog's stagnation / ceiling / task_exit_unfinalized paths, or
- # the SKY-9167 post-drain branch where the row read as ``canceled``,
- # non-final, or unreadable). Block further block-running calls until
- # ``get_run_results`` clears the flag — prevents the LLM from
- # auto-retrying a mutation block whose side effects may already
- # have landed.
- pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
- if isinstance(pending_run_id, str) and pending_run_id:
- if getattr(ctx, "pending_reconciliation_requires_user_input", False) is True:
- return _emit_tool_blocker_signal(
- ctx,
- _pending_reconciliation_requires_input_signal(
- pending_run_id=pending_run_id, blocked_tool=tool_name
- ),
- )
- return _emit_tool_blocker_signal(
- ctx,
- _pending_reconciliation_no_input_signal(pending_run_id=pending_run_id, blocked_tool=tool_name),
- )
-
- inspection_signal = _post_budget_page_inspection_signal(ctx, tool_name)
- if inspection_signal is not None:
- return _emit_tool_blocker_signal(ctx, inspection_signal)
-
- # Terminal anti-bot evidence should produce the final user-facing reply
- # before the generic budget rerun path can ask for another attempt.
- post_budget_challenge_signal = _post_budget_terminal_challenge_signal(ctx, arguments, tool_name)
- if post_budget_challenge_signal is not None:
- return _emit_tool_blocker_signal(ctx, post_budget_challenge_signal)
-
- budget_signal = _per_tool_budget_problem_rerun_signal(ctx, arguments, tool_name)
- if budget_signal is not None:
- return _emit_tool_blocker_signal(ctx, budget_signal)
-
- upstream_replay_signal = _post_budget_upstream_replay_signal(ctx, arguments, tool_name)
- if upstream_replay_signal is not None:
- return _emit_tool_blocker_signal(ctx, upstream_replay_signal)
-
- challenge_signal = _challenge_gated_anti_bot_rerun_signal(ctx, arguments, tool_name)
- if challenge_signal is not None:
- return _emit_tool_blocker_signal(ctx, challenge_signal)
-
- streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)
- streak = streak_raw if isinstance(streak_raw, int) else 0
- if streak >= REPEATED_ACTION_STREAK_ABORT_AT:
- agent_steering = (
- f"Repeated-action abort: the last {streak} runs fired the same "
- "action sequence against the page without making progress. "
- "The site is likely blocked by a captcha, popup, anti-bot "
- "challenge, or hidden validation error that the agent is not "
- "detecting. Do NOT retry this tool — conclude the workflow is "
- "not automatable as-is and report back to the user."
- )
- user_facing = (
- "I tried the same actions a few times without making progress. The site looks blocked. I'll stop here."
- )
- return _emit_tool_blocker_signal(
- ctx,
- CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="stop",
- cleared_by_tools=frozenset(),
- internal_reason_code="tool_error_repeated_action_abort",
- blocked_tool=tool_name,
- ),
- )
-
- # Within-turn fail-fast for permanent navigation errors (DNS / cert /
- # SSL / invalid URL). The enforcement-loop stop nudge only runs
- # BETWEEN agent turns, so without this check the LLM is free to make
- # speculative within-turn retries (e.g. drop the `www` subdomain and
- # try again) before the nudge fires. update_and_run_blocks internally
- # calls _update_workflow which clears the flag, so this check must
- # run before that — hence at the tool entrypoint, not inside the run
- # body.
- prior_nav_error = getattr(ctx, "last_test_non_retriable_nav_error", None)
- if isinstance(prior_nav_error, str) and prior_nav_error:
- agent_steering = (
- f"Prior run in this turn hit a permanent navigation error "
- f"({prior_nav_error[:200]}). Do NOT retry — the URL is unreachable "
- "regardless of subdomain or path variations. Reply to the user "
- "explaining the failure and asking them to verify the URL."
- )
- user_facing = "The URL I tried isn't reachable. Tell me the correct address and I'll try again."
- return _emit_tool_blocker_signal(
- ctx,
- CopilotToolBlockerSignal(
- blocker_kind="tool_error",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="ask_user_clarifying",
- cleared_by_tools=frozenset(),
- internal_reason_code="tool_error_non_retriable_nav",
- blocked_tool=tool_name,
- ),
- )
-
- late_signal = _late_block_running_call_signal(ctx, tool_name)
- if late_signal is not None:
- return _emit_tool_blocker_signal(ctx, late_signal)
- return None
-
-
-_build_loop_blocker_signal = build_loop_blocker_signal
-
-
-def _request_policy_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- policy = getattr(ctx, "request_policy", None)
- if not isinstance(policy, RequestPolicy):
- return None
-
- agent_steering: str | None = None
- user_facing: str | None = None
- reason_code: str | None = None
- recovery_hint: RecoveryHint = "report_blocker_to_user"
- cleared_by: frozenset[str] = frozenset()
-
- if tool_name == "update_workflow" and not policy.allow_update_workflow:
- reason_code = "request_policy_blocks_update_workflow"
- agent_steering = (
- "Request policy blocks workflow updates for the latest user message. "
- "Ask the user for safe stored credential metadata instead."
- )
- user_facing = "I need stored credential metadata before I can update this workflow."
- recovery_hint = "ask_user_clarifying"
- elif tool_name in BLOCK_RUNNING_TOOLS and not policy.allow_run_blocks:
- if policy.testing_intent == "skip_test":
- reason_code = "request_policy_blocks_run_blocks_skip_test"
- agent_steering = (
- "Request policy says the latest user message asked for an untested draft. Use update_workflow only."
- )
- user_facing = "I'll save this as an untested draft because the request asked me not to run it."
- recovery_hint = "retry_with_different_tool"
- cleared_by = frozenset({"update_workflow"})
- elif policy.clarification_reason == "workflow_credential_inputs_unbound":
- reason_code = "request_policy_blocks_run_blocks_credential_unbound"
- agent_steering = (
- "Skipped test run: the existing workflow references credential parameters "
- "whose keys point to workflow inputs that are not configured. REPLY to the user "
- "with: 'I applied your requested change. I couldn't test the modified workflow "
- "because I couldn't find the required credentials — please add them via the "
- "Credentials UI, then I can try again.' Keep the unvalidated draft surfaced."
- )
- user_facing = (
- "I applied your requested change. I couldn't test the modified workflow because "
- "the required credentials aren't set up. Add them in the Credentials UI and ask "
- "me to test it."
- )
- recovery_hint = "report_blocker_to_user"
- else:
- reason_code = "request_policy_blocks_run_blocks_generic"
- agent_steering = (
- "Request policy blocks block-running tools for the latest user message. "
- "Ask the user for the required safe credential or clarification before testing."
- )
- user_facing = "I need a credential or clarification from you before I can test this workflow."
- recovery_hint = "ask_user_clarifying"
-
- if reason_code is None or agent_steering is None or user_facing is None:
- return None
-
- LOG.info(
- "copilot authority gate evaluated tool",
- authority_gate_layer="request_policy",
- blocked_tool=tool_name,
- request_policy_allow_update_workflow=policy.allow_update_workflow,
- request_policy_allow_run_blocks=policy.allow_run_blocks,
- request_policy_testing_intent=policy.testing_intent,
- request_policy_clarification_reason=policy.clarification_reason,
- safe_reason_code=reason_code,
- )
- return CopilotToolBlockerSignal(
- blocker_kind="authority_denied",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint=recovery_hint,
- cleared_by_tools=cleared_by,
- internal_reason_code=reason_code,
- blocked_tool=tool_name,
- )
-
-
-def _request_policy_allows_credential_deferred_draft(ctx: AgentContext) -> bool:
- policy = getattr(ctx, "request_policy", None)
- return (
- isinstance(policy, RequestPolicy)
- and policy.allow_update_workflow
- and not policy.allow_run_blocks
- and policy.allow_missing_credentials_in_draft
- and policy.clarification_reason in CREDENTIAL_DEFERRED_DRAFT_REASONS
- )
-
-
-def _request_policy_allows_update_and_skip_run(ctx: AgentContext, tool_name: str) -> bool:
- return tool_name == "update_and_run_blocks" and _request_policy_allows_credential_deferred_draft(ctx)
-
-
-def _turn_intent_has_edit_target(intent: TurnIntent) -> bool:
- # Keep this aligned with TurnIntent target kinds that make an edit specific enough to mutate safely.
- if any(
- intent.target_entities.get(entity_type)
- for entity_type in (
- "block",
- "run",
- "proposed_workflow",
- "latest_assistant_proposal",
- "proposal",
- "workflow_change",
- )
- ):
- return True
- return any(target != "current_workflow" for target in intent.target_entities.get("workflow", []))
-
-
-def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
- intent = getattr(ctx, "turn_intent", None)
- if not isinstance(intent, TurnIntent):
- return None
-
- authority = intent.authority
- unresolved_refs = intent.target_entities.get(UNRESOLVED_BLOCK_REF_TARGET_ENTITY, [])
- if intent.mode == TurnIntentMode.EDIT and tool_name in WORKFLOW_MUTATION_TOOLS and unresolved_refs:
- reason_code = "turn_intent_unresolved_edit_target"
- labels = sorted(_workflow_yaml_blocks_by_label(getattr(ctx, "workflow_yaml", None)))
- label_hint = ", ".join(labels[:8]) if labels else "no labeled blocks"
- LOG.info(
- "copilot authority gate evaluated tool",
- authority_gate_layer="turn_intent",
- turn_intent_mode=intent.mode.value,
- turn_intent_target_entity_types=sorted(intent.target_entities),
- turn_intent_unresolved_refs=unresolved_refs,
- blocked_tool=tool_name,
- safe_reason_code=reason_code,
- )
- unresolved_str = ", ".join(unresolved_refs)
- return _build_turn_intent_signal(
- tool_name=tool_name,
- classifier_mode=intent.mode.value,
- reason_code=reason_code,
- agent_steering_text=(
- f"The latest user message references workflow/block identifier(s) that are not present in the "
- f"current workflow: {unresolved_str}. Current workflow labels include: {label_hint}. Ask the user "
- f"which current block should change before mutating or running blocks."
- ),
- user_facing_reason=(
- f"I couldn't find the block(s) you mentioned ({unresolved_str}). "
- f"Tell me which existing block to change."
- ),
- recovery_hint="ask_user_clarifying",
- )
-
- if (
- intent.mode == TurnIntentMode.EDIT
- and tool_name in WORKFLOW_MUTATION_TOOLS
- and not _turn_intent_has_edit_target(intent)
- ):
- reason_code = "turn_intent_missing_edit_target"
- LOG.info(
- "copilot authority gate evaluated tool",
- authority_gate_layer="turn_intent",
- turn_intent_mode=intent.mode.value,
- turn_intent_target_entity_types=sorted(intent.target_entities),
- blocked_tool=tool_name,
- safe_reason_code=reason_code,
- )
- return _build_turn_intent_signal(
- tool_name=tool_name,
- classifier_mode=intent.mode.value,
- reason_code=reason_code,
- agent_steering_text=(
- "Could not identify a specific workflow edit target. Ask the user which workflow/block should "
- "change before mutating."
- ),
- user_facing_reason="Tell me which block or workflow you'd like me to change.",
- recovery_hint="ask_user_clarifying",
- )
-
- blocks_update = tool_name in WORKFLOW_MUTATION_TOOLS and not authority.may_update_workflow
- blocks_run = tool_name in BLOCK_RUNNING_TOOLS and not authority.may_run_blocks
- blocks_page_inspection = tool_name in PAGE_SCHEMA_CONTEXT_TOOLS and (
- intent.mode in NO_MUTATION_TURN_INTENT_MODES or not authority.may_update_workflow
- )
- blocks_credential_metadata = tool_name in CREDENTIAL_METADATA_TOOLS and not (
- authority.may_update_workflow or authority.may_run_blocks
- )
- # Two paths grant read access to ANSWER_ONLY_CONTEXT_TOOLS, both excluded for DOCS_ANSWER/REFUSE/CLARIFY:
- # (1) authority.may_read_run_context — classifier-derived (DIAGNOSE turns)
- # (2) pending_reconciliation_run_id — within-turn override anchored to the run-blocks watchdog
- may_read_run_context = authority.may_read_run_context and intent.mode not in READ_CONTEXT_DENIED_MODES
- blocks_context_read = tool_name in ANSWER_ONLY_CONTEXT_TOOLS and not may_read_run_context
-
- within_turn_read_override = False
- if blocks_context_read and intent.mode not in READ_CONTEXT_DENIED_MODES:
- pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
- if isinstance(pending_run_id, str) and pending_run_id:
- blocks_context_read = False
- within_turn_read_override = True
- else:
- same_turn_run_id = getattr(ctx, "last_successful_run_blocks_workflow_run_id", None) or getattr(
- ctx,
- "last_run_blocks_workflow_run_id",
- None,
- )
- if isinstance(same_turn_run_id, str) and same_turn_run_id:
- blocks_context_read = False
- within_turn_read_override = True
-
- if blocks_run and not blocks_update and _request_policy_allows_update_and_skip_run(ctx, tool_name):
- return None
- if (
- not blocks_update
- and not blocks_run
- and not blocks_context_read
- and not blocks_page_inspection
- and not blocks_credential_metadata
- ):
- if within_turn_read_override:
- LOG.info(
- "copilot authority gate allowed tool via within-turn read override",
- authority_gate_layer="turn_intent",
- turn_intent_mode=intent.mode.value,
- tool_name=tool_name,
- turn_intent_within_turn_read_override=True,
- )
- return None
-
- if intent.mode in NO_MUTATION_TURN_INTENT_MODES and blocks_run:
- reason_code = "turn_intent_no_mutation_run_blocked"
- elif intent.mode in NO_MUTATION_TURN_INTENT_MODES and blocks_update:
- reason_code = "turn_intent_no_mutation_update_blocked"
- elif blocks_update and blocks_run:
- reason_code = "turn_intent_no_mutation_run_blocked"
- elif blocks_update:
- reason_code = "turn_intent_update_blocked"
- elif blocks_page_inspection:
- reason_code = "turn_intent_page_inspection_blocked"
- elif blocks_context_read:
- reason_code = "turn_intent_context_read_blocked"
- elif blocks_credential_metadata:
- reason_code = "turn_intent_credential_metadata_blocked"
- else:
- reason_code = "turn_intent_run_blocked"
- LOG.info(
- "copilot authority gate evaluated tool",
- authority_gate_layer="turn_intent",
- turn_intent_mode=intent.mode.value,
- turn_intent_target_entity_types=sorted(intent.target_entities),
- turn_intent_may_update_workflow=authority.may_update_workflow,
- turn_intent_may_run_blocks=authority.may_run_blocks,
- turn_intent_may_read_run_context=authority.may_read_run_context,
- blocked_tool=tool_name,
- safe_reason_code=reason_code,
- )
- action = "ask the user" if authority.requires_user_input else "answer the user"
- detail = f" Ask: {intent.missing_context_question}" if intent.missing_context_question else ""
-
- if blocks_run and not blocks_update and authority.may_update_workflow:
- return _build_turn_intent_signal(
- tool_name=tool_name,
- classifier_mode=intent.mode.value,
- reason_code=reason_code,
- agent_steering_text=(
- "Browser blocks may not run for the latest user message. Use update_workflow only and keep the "
- f"draft unvalidated.{detail}"
- ),
- user_facing_reason="I'll save the change as a draft without running it.",
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=frozenset({"update_workflow"}),
- )
- if blocks_context_read and not blocks_update and not blocks_run:
- return _build_turn_intent_signal(
- tool_name=tool_name,
- classifier_mode=intent.mode.value,
- reason_code=reason_code,
- agent_steering_text=(
- "Run context may not be read for the latest user message. Answer using the context already "
- f"provided.{detail}"
- ),
- user_facing_reason="I'll answer with the information I already have.",
- recovery_hint="report_blocker_to_user",
- )
- return _build_turn_intent_signal(
- tool_name=tool_name,
- classifier_mode=intent.mode.value,
- reason_code=reason_code,
- agent_steering_text=(
- "This tool is not allowed for the latest user message. Do not update workflow YAML or run browser "
- "blocks, and do not fetch additional run context with tools; "
- f"{action} using the available context instead.{detail}"
- ),
- user_facing_reason="I'll respond with the information I already have.",
- recovery_hint="ask_user_clarifying" if authority.requires_user_input else "report_blocker_to_user",
- )
-
-
-def _build_turn_intent_signal(
- *,
- tool_name: str,
- classifier_mode: str,
- reason_code: str,
- agent_steering_text: str,
- user_facing_reason: str,
- recovery_hint: RecoveryHint,
- cleared_by_tools: frozenset[str] = frozenset(),
-) -> CopilotToolBlockerSignal:
- return CopilotToolBlockerSignal(
- blocker_kind="authority_denied",
- agent_steering_text=agent_steering_text,
- user_facing_reason=user_facing_reason,
- recovery_hint=recovery_hint,
- cleared_by_tools=cleared_by_tools,
- internal_reason_code=reason_code,
- blocked_tool=tool_name,
- classifier_mode=classifier_mode,
- )
-
-
-def _emit_tool_blocker_signal(ctx: AgentContext, signal: CopilotToolBlockerSignal) -> str:
- return stash_blocker_signal(ctx, signal)
-
-
-def _authority_tool_error(
- ctx: AgentContext,
- tool_name: str,
- *,
- ignore_request_policy_error: bool = False,
-) -> str | None:
- # Request-policy precedes turn-intent unless explicitly ignored.
- turn_intent_signal = _turn_intent_tool_error(ctx, tool_name)
- request_policy_signal = _request_policy_tool_error(ctx, tool_name)
- if turn_intent_signal is not None and request_policy_signal is not None:
- LOG.info(
- "copilot authority gate blocked tool",
- authority_gate_layer="both",
- blocked_tool=tool_name,
- )
- chosen = (
- request_policy_signal
- if (request_policy_signal is not None and not ignore_request_policy_error)
- else turn_intent_signal
- )
- if chosen is not None:
- return _emit_tool_blocker_signal(ctx, chosen)
- phase_signal = _phase_blocker_signal(ctx, tool_name)
- if phase_signal is not None:
- LOG.info(
- "copilot authority gate blocked tool",
- authority_gate_layer="build_phase",
- blocked_tool=tool_name,
- build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),
- )
- return _emit_tool_blocker_signal(ctx, phase_signal)
- return None
-
-
-_PARAMETER_TYPE_PLACEHOLDERS: dict[WorkflowParameterType, Any] = {
- WorkflowParameterType.STRING: "",
- WorkflowParameterType.INTEGER: 0,
- WorkflowParameterType.FLOAT: 0.0,
- WorkflowParameterType.BOOLEAN: False,
- WorkflowParameterType.JSON: {},
- WorkflowParameterType.FILE_URL: "",
-}
-
-
-def _placeholder_for_parameter_type(param_type: WorkflowParameterType) -> Any:
- return _PARAMETER_TYPE_PLACEHOLDERS.get(param_type)
-
-
-def _parameter_binding_invariant_error(
- workflow: Workflow,
- persisted_workflow_params: list[WorkflowParameter],
- persisted_output_params: list[OutputParameter],
-) -> tuple[str, dict[str, list[str]], dict[str, list[str]]] | None:
- """Return a ``(summary, missing_persisted, missing_from_definition)`` tuple
- when ``workflow.workflow_definition`` disagrees with persisted
- definition-parameter rows for runtime-relevant classes. Returns ``None``
- when aligned.
-
- Compares ``WorkflowParameter`` rows by ``(key, workflow_parameter_type)``
- and ``OutputParameter`` rows by ``key``. Secret/credential and context
- parameters are intentionally out of scope — runtime reads those from the
- definition JSON.
- """
- definition = getattr(workflow, "workflow_definition", None)
- parameters = getattr(definition, "parameters", None) if definition else None
- parameters = list(parameters) if parameters else []
-
- def_workflow_ids: set[tuple[str, str]] = set()
- def_output_keys: set[str] = set()
- for parameter in parameters:
- if isinstance(parameter, WorkflowParameter):
- def_workflow_ids.add((parameter.key, parameter.workflow_parameter_type.value))
- elif isinstance(parameter, OutputParameter):
- def_output_keys.add(parameter.key)
-
- persisted_workflow_ids: set[tuple[str, str]] = {
- (wp.key, wp.workflow_parameter_type.value) for wp in persisted_workflow_params
- }
- persisted_output_keys: set[str] = {op.key for op in persisted_output_params}
-
- missing_persisted_workflow = sorted(
- f"{key} ({ptype})" for (key, ptype) in def_workflow_ids - persisted_workflow_ids
- )
- extra_persisted_workflow = sorted(f"{key} ({ptype})" for (key, ptype) in persisted_workflow_ids - def_workflow_ids)
- missing_persisted_output = sorted(def_output_keys - persisted_output_keys)
- extra_persisted_output = sorted(persisted_output_keys - def_output_keys)
-
- if (
- not missing_persisted_workflow
- and not extra_persisted_workflow
- and not missing_persisted_output
- and not extra_persisted_output
- ):
- return None
-
- summary = (
- "Pre-run invariant: workflow_definition and persisted parameter rows disagree. "
- f"workflow missing persisted: {missing_persisted_workflow or '[]'}; "
- f"workflow missing from definition: {extra_persisted_workflow or '[]'}; "
- f"output missing persisted: {missing_persisted_output or '[]'}; "
- f"output missing from definition: {extra_persisted_output or '[]'}"
- )
- return (
- summary,
- {"workflow": missing_persisted_workflow, "output": missing_persisted_output},
- {"workflow": extra_persisted_workflow, "output": extra_persisted_output},
- )
-
-
-class BlockObservationRef(BaseModel):
- label: str
- observation_step: Annotated[int, Field(ge=0)]
-
-
-ArtifactEvidenceStatus = Literal["satisfied", "missing", "diagnostic_only", "observed_not_verified"]
-
-
-class CodeArtifactClaimedOutcome(BaseModel):
- id: str
- scope: str
- text: str
- status: ArtifactEvidenceStatus
- depends_on: list[str] = Field(default_factory=list)
- covered_criteria: list[str] = Field(default_factory=list)
- criteria_ids: list[str] = Field(default_factory=list)
- evidence_refs: list[str] = Field(default_factory=list)
- observation_refs: list[str] = Field(default_factory=list)
- required_tokens: list[str] = Field(default_factory=list)
- entities: list[str] = Field(default_factory=list)
-
-
-class CodeArtifactPageDependency(BaseModel):
- id: str
- scope: str
- status: ArtifactEvidenceStatus
- url_hint: str | None = None
- page_state_hint: str | None = None
- required_affordances: list[str] = Field(default_factory=list)
- required_outcomes: list[str] = Field(default_factory=list)
- evidence_refs: list[str] = Field(default_factory=list, description="Dependency-scoped evidence_ref ids.")
- observation_refs: list[str] = Field(default_factory=list, description="Dependency-scoped observation_ref ids.")
-
-
-class CodeArtifactCompletionCriterion(BaseModel):
- id: str
- text: str
- level: Literal["terminal", "outcome", "prefix", "method"]
- outcome: str | None = None
- terminal: bool | None = None
-
-
-class CodeArtifactScopedRef(BaseModel):
- claim_id: str | None = None
- dependency_id: str | None = None
- criterion_id: str | None = None
- evidence_ref: str | None = None
- observation_ref: str | None = None
- status: ArtifactEvidenceStatus = Field(validation_alias=AliasChoices("status", "evidence_status"))
- source_tool: str | None = None
- observation_step: Annotated[int, Field(ge=0)] | None = None
- run_sample_id: str | None = None
- current_url: str | None = None
- source_label: str | None = None
- checkpoint_next_mode: Literal["advance", "stop"] | None = None
-
-
-class CodeArtifactTerminalVerifierExpectation(BaseModel):
- id: str
- text: str
- criteria_ids: list[str] = Field(default_factory=list)
- claimed_outcome_ids: list[str] = Field(default_factory=list)
-
-
-class CodeArtifactExplorationObservation(BaseModel):
- id: str
- text: str
- status: Literal["observed_not_verified"] = Field(
- default="observed_not_verified",
- validation_alias=AliasChoices("status", "evidence_status"),
- )
- observation_ref: str | None = None
- source_tool: str | None = None
- observation_step: Annotated[int, Field(ge=0)] | None = None
- current_url: str | None = None
- source_label: str | None = None
- checkpoint_next_mode: Literal["stop"] | None = None
-
-
-class CodeArtifactMetadata(BaseModel):
- artifact_id: str | None = None
- block_label: str | None = None
- block_id: str | None = None
- declared_goal: str
- claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(default_factory=list)
- page_dependencies: list[CodeArtifactPageDependency] = Field(default_factory=list)
- completion_criteria: list[CodeArtifactCompletionCriterion] = Field(default_factory=list)
- evidence_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)
- observation_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)
- terminal_verifier_expectations: list[CodeArtifactTerminalVerifierExpectation] = Field(default_factory=list)
- exploration_observations: list[CodeArtifactExplorationObservation] = Field(default_factory=list)
-
-
-_CODE_ARTIFACT_REQUIRED_LIST_FIELDS = (
- "claimed_outcomes",
- "page_dependencies",
- "completion_criteria",
- "terminal_verifier_expectations",
-)
-
-
-def _code_artifact_metadata_as_tool_argument(
- metadata: list[CodeArtifactMetadata] | None,
-) -> list[dict[str, Any]]:
- if not metadata:
- return []
- return [item.model_dump(mode="json", exclude_none=True) for item in metadata]
-
-
-def _format_code_artifact_violations(violations: list[str]) -> str:
- # Surface every contract violation at once so the agent fixes them in a single
- # update instead of round-tripping one error per `update_and_run_blocks` call.
- if len(violations) == 1:
- return violations[0]
- numbered = "\n".join(f"{index}. {message}" for index, message in enumerate(violations, start=1))
- return f"Artifact metadata has {len(violations)} contract violations; fix all of them in one update:\n{numbered}"
-
-
-def _normalize_code_artifact_metadata(
- raw_metadata: Any,
- workflow_yaml: str,
-) -> tuple[dict[str, dict[str, Any]], str | None]:
- if raw_metadata in (None, [], {}):
- return {}, None
- items = _code_artifact_metadata_items(raw_metadata)
- code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
- normalized: dict[str, dict[str, Any]] = {}
- violations: list[str] = []
- seen_labels: set[str] = set()
- for raw_item in items:
- try:
- metadata = (
- raw_item
- if isinstance(raw_item, CodeArtifactMetadata)
- else CodeArtifactMetadata.model_validate(raw_item)
- )
- except ValidationError as exc:
- violations.append(f"Artifact metadata is malformed: {exc}")
- continue
- dumped = metadata.model_dump(mode="json", exclude_none=True)
- label = str(dumped.get("block_label") or "").strip()
- if not label:
- violations.append("Artifact metadata requires a non-empty `block_label` for each artifact.")
- continue
- if label not in code_blocks:
- violations.append(
- f"Artifact metadata for `{label}` does not reference an existing code block label. "
- "Attach metadata only to authored code blocks."
- )
- continue
- if label in seen_labels:
- violations.append(f"Artifact metadata contains duplicate entries for `{label}`.")
- continue
- seen_labels.add(label)
- dumped["block_label"] = label
- item_violations: list[str] = []
- artifact_id = str(dumped.get("artifact_id") or "").strip()
- if artifact_id and not artifact_id.startswith("code_artifact:"):
- item_violations.append(
- f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."
- )
- dumped["artifact_id"] = artifact_id or _artifact_id_for_block_label(label)
- block_id = str(
- dumped.get("block_id") or code_blocks[label].get("block_id") or code_blocks[label].get("id") or ""
- ).strip()
- if block_id:
- dumped["block_id"] = block_id
- declared_goal = str(dumped.get("declared_goal") or "").strip()
- if not declared_goal:
- item_violations.append(f"Artifact metadata for `{label}` requires a non-empty `declared_goal`.")
- for field_name in _CODE_ARTIFACT_REQUIRED_LIST_FIELDS:
- value = dumped.get(field_name)
- if not isinstance(value, list) or not value:
- item_violations.append(f"Artifact metadata for `{label}` requires non-empty `{field_name}`.")
- if not dumped.get("evidence_refs") and not dumped.get("observation_refs"):
- item_violations.append(f"Artifact metadata for `{label}` requires `evidence_refs` or `observation_refs`.")
- item_violations.extend(_code_artifact_metadata_shape_errors(label, dumped))
- if item_violations:
- violations.extend(item_violations)
- continue
- normalized[label] = dumped
- if violations:
- return {}, _format_code_artifact_violations(violations)
- return normalized, None
-
-
-def _code_artifact_metadata_items(raw_metadata: Any) -> list[Any]:
- if isinstance(raw_metadata, Mapping):
- items: list[Any] = []
- for block_label, value in raw_metadata.items():
- if isinstance(value, Mapping) and "block_label" not in value:
- items.append({"block_label": block_label, **value})
- else:
- items.append(value)
- return items
- if isinstance(raw_metadata, list):
- return raw_metadata
- return [raw_metadata]
-
-
-def _workflow_yaml_code_blocks_by_label(workflow_yaml: str | None) -> dict[str, Mapping[str, Any]]:
- blocks: dict[str, Mapping[str, Any]] = {}
- for label, block in _workflow_yaml_blocks_by_label(workflow_yaml).items():
- if _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:
- blocks[label] = block
- return blocks
-
-
-def _artifact_id_for_block_label(label: str) -> str:
- fragment = re.sub(r"[^A-Za-z0-9_.-]+", "_", label).strip("_") or "artifact"
- return f"code_artifact:{fragment}"
-
-
-def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:
- """Return every shape violation for one artifact; the caller aggregates them."""
- errors: list[str] = []
- for field_name, ref_key in (("evidence_refs", "evidence_ref"), ("observation_refs", "observation_ref")):
- for index, ref in enumerate(_artifact_rows(artifact.get(field_name))):
- if not str(ref.get(ref_key) or "").strip():
- errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `{ref_key}`.")
- if not any(str(ref.get(key) or "").strip() for key in ("claim_id", "dependency_id", "criterion_id")):
- errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires a scoped id.")
- status = str(ref.get("status") or "").strip()
- if ref.get("checkpoint_next_mode") == "advance" and status != "diagnostic_only":
- errors.append(
- f"Artifact metadata for `{label}` `{field_name}` entry {index} has "
- "`checkpoint_next_mode=advance`; it must stay `diagnostic_only`."
- )
- if ref.get("checkpoint_next_mode") == "stop" and status not in {"observed_not_verified", "diagnostic_only"}:
- errors.append(
- f"Artifact metadata for `{label}` `{field_name}` entry {index} has "
- "`checkpoint_next_mode=stop`; it must remain `observed_not_verified` or `diagnostic_only`."
- )
- if status != "missing" and not str(ref.get("source_tool") or "").strip():
- errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `source_tool`.")
-
- for index, claim in enumerate(_artifact_rows(artifact.get("claimed_outcomes"))):
- claim_id = str(claim.get("id") or "").strip()
- if not _artifact_string_list(claim.get("depends_on")):
- errors.append(f"Artifact metadata claim `{claim_id or index}` for `{label}` requires `depends_on`.")
- claim_criteria = _artifact_string_list(claim.get("covered_criteria")) or _artifact_string_list(
- claim.get("criteria_ids")
- )
- if not claim_criteria:
- errors.append(f"Artifact metadata claim `{claim_id}` for `{label}` requires covered criterion ids.")
- claim_evidence_refs = _artifact_string_list(claim.get("evidence_refs"))
- claim_observation_refs = _artifact_string_list(claim.get("observation_refs"))
- if claim.get("status") == "satisfied" and not claim_evidence_refs:
- errors.append(
- f"Artifact metadata claim `{claim_id}` for `{label}` is `satisfied` but has no "
- "claim-scoped `evidence_refs`."
- )
- if claim.get("status") != "missing" and not claim_evidence_refs and not claim_observation_refs:
- errors.append(
- f"Artifact metadata claim `{claim_id}` for `{label}` requires claim-scoped "
- "`evidence_refs` or `observation_refs` unless status is `missing`."
- )
-
- for dependency in _artifact_rows(artifact.get("page_dependencies")):
- dependency_id = str(dependency.get("id") or "").strip()
- dependency_evidence_refs = _artifact_string_list(dependency.get("evidence_refs"))
- dependency_observation_refs = _artifact_string_list(dependency.get("observation_refs"))
- if dependency.get("status") == "satisfied" and not dependency_evidence_refs:
- errors.append(
- f"Artifact metadata dependency `{dependency_id}` for `{label}` is `satisfied` but has no "
- "dependency-scoped `evidence_refs`."
- )
- if dependency.get("status") != "missing" and not dependency_evidence_refs and not dependency_observation_refs:
- errors.append(
- f"Artifact metadata dependency `{dependency_id}` for `{label}` requires scoped "
- "`evidence_refs` or `observation_refs` unless status is `missing`."
- )
-
- for index, expectation in enumerate(_artifact_rows(artifact.get("terminal_verifier_expectations"))):
- expectation_id = str(expectation.get("id") or "").strip()
- if not _artifact_string_list(expectation.get("criteria_ids")) and not _artifact_string_list(
- expectation.get("claimed_outcome_ids")
- ):
- errors.append(
- f"Artifact metadata terminal verifier expectation `{expectation_id or index}` for `{label}` "
- "requires `criteria_ids` or `claimed_outcome_ids`."
- )
-
- for index, observation in enumerate(_artifact_rows(artifact.get("exploration_observations"))):
- if observation.get("status") != "observed_not_verified":
- errors.append(
- f"Artifact metadata for `{label}` exploration observation {index} must be marked "
- "`observed_not_verified` until authored execution and terminal verification pass."
- )
- if observation.get("checkpoint_next_mode") == "advance":
- errors.append(
- f"Artifact metadata for `{label}` exploration observation {index} cannot carry "
- "`checkpoint_next_mode=advance`; record that as `diagnostic_only` evidence instead."
- )
- return errors
-
-
-def _artifact_rows(value: Any) -> list[Mapping[str, Any]]:
- return [row for row in value if isinstance(row, Mapping)] if isinstance(value, list) else []
-
-
-def _artifact_string_list(value: Any) -> list[str]:
- if not isinstance(value, list):
- return []
- return [str(item).strip() for item in value if str(item).strip()]
-
-
-async def _update_workflow(
- params: dict[str, Any],
- ctx: AgentContext,
- *,
- allow_missing_credentials: bool | None = None,
-) -> dict[str, Any]:
- authority_error = _authority_tool_error(ctx, "update_workflow")
- if authority_error is not None:
- return {"ok": False, "error": authority_error}
-
- workflow_yaml = params["workflow_yaml"]
- # Tool wrappers run authority/loop guards before calling here. The composition
- # gate below consumes these refs, so they must be visible before validation.
- ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
- ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
- ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
- code_artifact_metadata, code_artifact_metadata_error = _normalize_code_artifact_metadata(
- params.get("code_artifact_metadata"),
- workflow_yaml,
- )
- if code_artifact_metadata_error is not None:
- return {"ok": False, "error": code_artifact_metadata_error}
- if code_artifact_metadata:
- ctx.code_artifact_metadata = code_artifact_metadata
- ctx.workflow_verification_evidence.code_artifact_metadata = code_artifact_metadata
- params["code_artifact_metadata"] = code_artifact_metadata
- if allow_missing_credentials is None:
- allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
- if not allow_missing_credentials:
- credential_error = await _credential_reference_validation_error(workflow_yaml, ctx)
- if credential_error is not None:
- return {"ok": False, "error": credential_error}
-
- misbinding_findings = _credential_id_misbinding_findings(workflow_yaml)
- if misbinding_findings:
- LOG.info(
- "copilot credential id misbinding rejected",
- organization_id=ctx.organization_id,
- workflow_id=ctx.workflow_id,
- findings=misbinding_findings,
- )
- return {"ok": False, "error": _credential_id_misbinding_error_message(misbinding_findings)}
-
- output_policy_verdict = evaluate_output_policy(
- request_policy=getattr(ctx, "request_policy", None),
- workflow_yaml=workflow_yaml,
- tool_arguments=params,
- )
- if not output_policy_verdict.allowed:
- LOG.info(
- "copilot output policy tool body verdict",
- **output_policy_verdict_to_trace_data(
- output_policy_verdict,
- surface="tool_body",
- tool_name="update_workflow",
- ),
- )
- return {"ok": False, "error": format_output_policy_tool_error(output_policy_verdict)}
-
- # Prefer the most-recent in-turn emission so cross-path flows (inline
- # REPLACE_WORKFLOW followed by update_workflow) compare against what the
- # LLM actually saw, not the turn-start persisted state.
- last_yaml = getattr(ctx, "last_workflow_yaml", None)
- prior_yaml = last_yaml if isinstance(last_yaml, str) and last_yaml else ctx.workflow_yaml
- stale_metadata = _detect_stale_block_metadata(workflow_yaml, prior_yaml)
- if stale_metadata:
- return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}
-
- wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
- if wait_block_error:
- return {"ok": False, "error": wait_block_error}
-
- challenge_http_error = _challenge_http_request_reject_message(ctx, workflow_yaml, ctx.workflow_yaml)
- if challenge_http_error:
- return {"ok": False, "error": challenge_http_error}
-
- # Post-emission reject of copilot-v2 writes that introduce a banned
- # block type. The schema pre_hook only fires when the LLM consults the
- # schema; this safety net fires regardless of emission path. Label-based
- # diff preserves legacy workflows — only NEW banned labels trip the reject.
- banned_items = _detect_new_banned_blocks(
- workflow_yaml,
- ctx.workflow_yaml,
- banned_types=_copilot_banned_block_types(ctx),
- )
- if banned_items:
- _record_banned_block_reject_span("_update_workflow", banned_items)
- return {"ok": False, "error": _banned_block_reject_message(banned_items, ctx)}
-
- composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)
- if composition_evidence_error:
- LOG.info(
- "copilot composition page evidence rejected workflow",
- workflow_permanent_id=ctx.workflow_permanent_id,
- target_url=workflow_target_url(workflow_yaml),
- )
- return {"ok": False, "error": composition_evidence_error}
-
- try:
- workflow = _process_workflow_yaml(
- workflow_id=ctx.workflow_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- workflow_yaml=workflow_yaml,
- )
- _record_workflow_proxy_location_span(workflow_yaml, workflow)
-
- # Param / top-level setting changes go through canonical because
- # prepare_workflow and the runtime parameter-row read consume canonical
- # values; terminal handlers roll back on non-auto-accept.
- prior_workflow = await _get_prior_workflow(ctx)
- requires_canonical_persist = _workflow_requires_canonical_persist(prior_workflow, workflow)
- if requires_canonical_persist:
- created_by_stamp = await resolve_copilot_created_by_stamp(ctx.workflow_id, ctx.organization_id)
- await app.WORKFLOW_SERVICE.update_workflow_definition(
- workflow_id=ctx.workflow_id,
- organization_id=ctx.organization_id,
- title=workflow.title,
- description=workflow.description,
- workflow_definition=workflow.workflow_definition,
- proxy_location=workflow.proxy_location,
- webhook_callback_url=workflow.webhook_callback_url,
- totp_verification_url=workflow.totp_verification_url,
- totp_identifier=workflow.totp_identifier,
- persist_browser_session=workflow.persist_browser_session,
- browser_profile_id=workflow.browser_profile_id,
- model=workflow.model,
- max_screenshot_scrolling_times=workflow.max_screenshot_scrolls,
- extra_http_headers=workflow.extra_http_headers,
- cdp_connect_headers=workflow.cdp_connect_headers,
- run_with=workflow.run_with,
- ai_fallback=workflow.ai_fallback,
- cache_key=workflow.cache_key,
- adaptive_caching=workflow.adaptive_caching,
- code_version=workflow.code_version,
- run_sequentially=workflow.run_sequentially,
- sequential_key=workflow.sequential_key,
- created_by=created_by_stamp,
- edited_by="copilot",
- )
- ctx.canonical_was_persisted_due_to_param_change = True
- ctx.staged_workflow_yaml = workflow_yaml
- ctx.staged_workflow = workflow
- ctx.has_staged_proposal = True
- ctx.workflow_yaml = workflow_yaml
- # Best-effort — narrative emit failures must never abort an
- # otherwise-successful update_workflow tool call. ``isinstance``
- # narrows the parameter's declared ``AgentContext`` to the
- # envelope-aware ``CopilotContext`` for mypy.
- if isinstance(ctx, CopilotContext) and ctx.stream is not None:
- try:
- await maybe_emit_design_end(ctx.stream, ctx)
- await emit_workflow_draft(ctx.stream, ctx, workflow)
- except Exception as emit_err:
- LOG.warning("copilot_narrative_workflow_draft_emit_failed", error=str(emit_err))
- return {
- "ok": True,
- "data": {
- "message": "Workflow updated successfully.",
- "block_count": len(workflow.workflow_definition.blocks) if workflow.workflow_definition else 0,
- },
- "_workflow": workflow,
- }
- except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:
- return {
- "ok": False,
- "error": f"Workflow validation failed: {e}",
- }
-
-
-async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
- page = params.get("page", 1)
- page_size = min(params.get("page_size", 10), 50)
- credentials = await app.DATABASE.credentials.get_credentials(
- organization_id=ctx.organization_id,
- page=page,
- page_size=page_size,
- )
- serialized = []
- for cred in credentials:
- entry: dict[str, Any] = {
- "credential_id": cred.credential_id,
- "name": cred.name,
- "credential_type": str(cred.credential_type),
- }
- if cred.username:
- entry["username"] = cred.username
- entry["totp_type"] = str(cred.totp_type) if cred.totp_type else None
- elif cred.card_last4:
- entry["card_last_four"] = cred.card_last4
- entry["card_brand"] = cred.card_brand
- elif cred.secret_label:
- entry["secret_label"] = cred.secret_label
- serialized.append(entry)
- _record_discovered_credentials_on_policy(ctx, credentials)
- return {
- "ok": True,
- "data": {
- "credentials": serialized,
- "page": page,
- "page_size": page_size,
- "count": len(serialized),
- "has_more": len(serialized) == page_size,
- },
- }
-
-
-def _record_discovered_credentials_on_policy(ctx: AgentContext, credentials: list[Credential]) -> None:
- request_policy = ctx.request_policy
- if not isinstance(request_policy, RequestPolicy):
- return
- known_ids = {credential.credential_id for credential in request_policy.discovered_credentials}
- for credential in credentials:
- if credential.credential_id not in known_ids:
- request_policy.discovered_credentials.append(credential)
-
-
-# Block types that establish browser state (loaded page / authenticated
-# session / navigation target). These are valid upstream anchors to walk back
-# to when a downstream edit invalidates part of the chain.
-#
-# We intentionally do NOT maintain a companion "rerunnable from current
-# browser state" set. We have no signal that the persistent browser session
-# is actually anchored at the frontier boundary — after a successful
-# [A, B, C] the browser is at post-C state, not pre-C — so rerunning only
-# an edited block is unsafe even for read-only types. Every edit walks back
-# to an upstream state-establisher, or falls back to the full requested list.
-_BLOCK_TYPES_STATE_ESTABLISHER = frozenset({"navigation", "login", "goto_url"})
-
-_JINJA_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*"
-_OUTPUT_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})_output\s*(?=[\.|}}])")
-_BLOCK_FORM_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*\.")
-_JINJA_ROOT_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*(?=[\.|}}])")
-
-_JINJA_RUNTIME_GLOBAL_ROOTS = frozenset(SandboxedEnvironment().globals)
-_JINJA_LITERAL_ROOTS = frozenset({"none", "true", "false"})
-_JINJA_SPECIAL_CONTEXT_ROOTS = frozenset({"loop", "self", "varargs", "kwargs"})
-_SKYVERN_TEMPLATE_CONTEXT_ROOTS = frozenset(RESERVED_PARAMETER_KEYS) | frozenset(
- {
- "parameters",
- "browser_session_id",
- "organization_id",
- # Conditional / branch evaluation roots — see BranchEvaluationContext.build_template_data.
- "params",
- "outputs",
- "environment",
- "env",
- "llm",
- }
-)
-_TEMPLATE_BUILTIN_ROOTS = (
- _JINJA_RUNTIME_GLOBAL_ROOTS | _JINJA_LITERAL_ROOTS | _JINJA_SPECIAL_CONTEXT_ROOTS | _SKYVERN_TEMPLATE_CONTEXT_ROOTS
-)
-
-# Keep this to grammatical glue only. Workflow/action words are intentionally
-# not filtered; the two-token stale threshold is the conservative guardrail.
-_BLOCK_METADATA_STOPWORDS = frozenset({"and", "for", "the", "with"})
-
-
-def _block_type_name(block: object) -> str:
- """Lowercase string name of a block's type, for both YAML and runtime blocks."""
- bt = getattr(block, "block_type", None)
- if bt is None:
- return ""
- name = getattr(bt, "value", None) or getattr(bt, "name", None) or str(bt)
- return str(name).lower()
-
-
-def _blocks_by_label(workflow_definition: object | None) -> dict[str, object]:
- blocks = getattr(workflow_definition, "blocks", None) if workflow_definition else None
- by_label: dict[str, object] = {}
- if not blocks:
- return by_label
- for block in blocks:
- label = getattr(block, "label", None)
- if isinstance(label, str):
- by_label[label] = block
- return by_label
-
-
-def _workflow_definition_block_labels(workflow_definition: object | None) -> list[str]:
- blocks = getattr(workflow_definition, "blocks", None) if workflow_definition else None
- labels: list[str] = []
- if not blocks:
- return labels
- for block in blocks:
- label = getattr(block, "label", None)
- if isinstance(label, str) and label:
- labels.append(label)
- return labels
-
-
-def _current_workflow_block_labels(ctx: object) -> list[str]:
- workflow = getattr(ctx, "last_workflow", None)
- labels = _workflow_definition_block_labels(getattr(workflow, "workflow_definition", None))
- if labels:
- return labels
- workflow_yaml = getattr(ctx, "last_workflow_yaml", None)
- if not isinstance(workflow_yaml, str):
- return []
- blocks = _parse_workflow_blocks(workflow_yaml)
- if not blocks:
- return []
- yaml_labels: list[str] = []
- for block in blocks:
- if isinstance(block, dict):
- label = _block_label_from_yaml(block)
- if label:
- yaml_labels.append(label)
- return yaml_labels
-
-
-def _current_workflow_has_evidence_block(ctx: object) -> bool:
- workflow = getattr(ctx, "last_workflow", None)
- blocks = getattr(getattr(workflow, "workflow_definition", None), "blocks", None)
- if blocks:
- return any(_block_type_name(block) in _OUTCOME_EVIDENCE_BLOCK_TYPES for block in blocks)
- workflow_yaml = getattr(ctx, "last_workflow_yaml", None)
- if not isinstance(workflow_yaml, str):
- return False
- return any(
- isinstance(block, dict) and _enum_or_string_name(block.get("block_type")) in _OUTCOME_EVIDENCE_BLOCK_TYPES
- for block in (_parse_workflow_blocks(workflow_yaml) or [])
- )
-
-
-def _unverified_current_workflow_labels(ctx: object) -> list[str]:
- labels = _current_workflow_block_labels(ctx)
- verified = set(getattr(ctx, "verified_prefix_labels", []) or [])
- return [label for label in labels if label not in verified]
-
-
-# Minimum length to apply the trailing-``s`` plural strip; below this we
-# leave the token alone so words like ``is``/``us``/``has`` aren't mangled.
-_MIN_STEMMABLE_TOKEN_LEN = 5
-
-
-def _metadata_token(token: str) -> str:
- token = token.lower()
- if len(token) >= _MIN_STEMMABLE_TOKEN_LEN and token.endswith("s"):
- token = token[:-1]
- return token
-
-
-def _metadata_tokens(value: Any) -> set[str]:
- if not isinstance(value, str):
- return set()
- tokens: set[str] = set()
- for token in re.findall(r"[A-Za-z0-9]+", value):
- normalized = _metadata_token(token)
- if len(normalized) <= 2 or normalized in _BLOCK_METADATA_STOPWORDS:
- continue
- tokens.add(normalized)
- return tokens
-
-
-def _iter_yaml_blocks(blocks: Any) -> list[dict[str, Any]]:
- found: list[dict[str, Any]] = []
- if not isinstance(blocks, list):
- return found
- for block in blocks:
- if not isinstance(block, dict):
- continue
- found.append(block)
- loop_blocks = block.get("loop_blocks")
- if isinstance(loop_blocks, list):
- found.extend(_iter_yaml_blocks(loop_blocks))
- return found
-
-
-def _workflow_yaml_blocks_by_label(workflow_yaml: str | None) -> dict[str, dict[str, Any]]:
- if not workflow_yaml:
- return {}
- try:
- parsed = safe_load_no_dates(workflow_yaml)
- except yaml.YAMLError:
- return {}
- if not isinstance(parsed, dict):
- return {}
- workflow_definition = parsed.get("workflow_definition")
- if not isinstance(workflow_definition, dict):
- return {}
- by_label: dict[str, dict[str, Any]] = {}
- for block in _iter_yaml_blocks(workflow_definition.get("blocks")):
- label = block.get("label")
- if isinstance(label, str):
- by_label[label] = block
- return by_label
-
-
-def _semantic_tokens_from_yaml(value: Any, *, exclude_keys: frozenset[str]) -> set[str]:
- tokens: set[str] = set()
- if isinstance(value, str):
- return _metadata_tokens(value)
- if isinstance(value, list):
- for item in value:
- tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))
- return tokens
- if isinstance(value, dict):
- for key, item in value.items():
- if key in exclude_keys:
- continue
- tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))
- return tokens
-
-
-def _stale_metadata_reason(
- *,
- field_name: str,
- field_value: Any,
- prior_block: dict[str, Any],
- submitted_block: dict[str, Any],
- current_exclude_keys: frozenset[str],
-) -> str | None:
- tokens = _metadata_tokens(field_value)
- if len(tokens) < 2:
- return None
-
- prior_tokens = _semantic_tokens_from_yaml(prior_block, exclude_keys=current_exclude_keys)
- current_tokens = _semantic_tokens_from_yaml(submitted_block, exclude_keys=current_exclude_keys)
- removed_tokens = prior_tokens - current_tokens
- if len(tokens & removed_tokens) < 2:
- return None
-
- return f"{field_name} {field_value!r} appears stale"
-
-
-def _prior_blocks_by_unique_title(prior_by_label: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
- titled = [(b["title"], b) for b in prior_by_label.values() if isinstance(b.get("title"), str) and b["title"]]
- counts = Counter(title for title, _ in titled)
- return {title: block for title, block in titled if counts[title] == 1}
-
-
-_STALE_BASE_EXCLUDE = frozenset({"label", "next_block_label"})
-_STALE_TITLE_EXCLUDE = frozenset({"label", "title", "next_block_label"})
-
-
-def _stale_for_renamed_label(
- label: str,
- submitted_block: dict[str, Any],
- prior_by_unique_title: dict[str, dict[str, Any]],
-) -> dict[str, Any] | None:
- submitted_title = submitted_block.get("title")
- # Title-less blocks intentionally skip the renamed-label path: with no
- # title there is no cross-reference to a prior block, so we'd be
- # guessing whether the rename was warranted.
- if not (isinstance(submitted_title, str) and submitted_title):
- return None
- prior_block = prior_by_unique_title.get(submitted_title)
- if prior_block is None:
- return None
- title_reason = _stale_metadata_reason(
- field_name="title",
- field_value=submitted_title,
- prior_block=prior_block,
- submitted_block=submitted_block,
- current_exclude_keys=_STALE_TITLE_EXCLUDE,
- )
- if not title_reason:
- return None
- return {"label": label, "reasons": [title_reason]}
-
-
-def _stale_for_matched_label(
- label: str,
- submitted_block: dict[str, Any],
- prior_block: dict[str, Any],
-) -> dict[str, Any] | None:
- reasons: list[str] = []
- submitted_title = submitted_block.get("title")
- prior_title = prior_block.get("title")
- title_reason = None
- if isinstance(submitted_title, str) and submitted_title and submitted_title == prior_title:
- title_reason = _stale_metadata_reason(
- field_name="title",
- field_value=submitted_title,
- prior_block=prior_block,
- submitted_block=submitted_block,
- current_exclude_keys=_STALE_TITLE_EXCLUDE,
- )
- if title_reason:
- reasons.append(title_reason)
-
- # When the title was already flagged stale, exclude its tokens from the
- # label-stale comparison so the same words don't double-count; otherwise
- # the title's tokens are stable content that should weigh in.
- label_exclude_keys = _STALE_TITLE_EXCLUDE if title_reason else _STALE_BASE_EXCLUDE
- label_reason = _stale_metadata_reason(
- field_name="label",
- field_value=label,
- prior_block=prior_block,
- submitted_block=submitted_block,
- current_exclude_keys=label_exclude_keys,
- )
- if label_reason:
- reasons.insert(0, label_reason)
-
- if not reasons:
- return None
- return {"label": label, "reasons": reasons}
-
-
-def _detect_stale_block_metadata(submitted_yaml: str | None, prior_yaml: str | None) -> list[dict[str, Any]]:
- """Find corrected blocks whose old label/title no longer matches their revised goal text."""
- prior_by_label = _workflow_yaml_blocks_by_label(prior_yaml)
- submitted_by_label = _workflow_yaml_blocks_by_label(submitted_yaml)
- if not prior_by_label or not submitted_by_label:
- return []
-
- prior_by_unique_title = _prior_blocks_by_unique_title(prior_by_label)
-
- stale_items: list[dict[str, Any]] = []
- for label, submitted_block in submitted_by_label.items():
- prior_block = prior_by_label.get(label)
- if prior_block is None:
- item = _stale_for_renamed_label(label, submitted_block, prior_by_unique_title)
- else:
- item = _stale_for_matched_label(label, submitted_block, prior_block)
- if item is not None:
- stale_items.append(item)
- return stale_items
-
-
-_STALE_BLOCK_METADATA_MESSAGE_LIMIT = 5
-
-
-def _stale_block_metadata_message(items: list[dict[str, Any]]) -> str:
- details = []
- for item in items[:_STALE_BLOCK_METADATA_MESSAGE_LIMIT]:
- label = item.get("label", "?")
- reasons = item.get("reasons") or []
- detail = "; ".join(str(reason) for reason in reasons)
- details.append(f"{label}: {detail}")
- if len(items) > _STALE_BLOCK_METADATA_MESSAGE_LIMIT:
- details.append(f"(and {len(items) - _STALE_BLOCK_METADATA_MESSAGE_LIMIT} more)")
- joined = "; ".join(details)
- return (
- "Workflow validation failed: corrected block metadata still appears stale. "
- "When changing a user's requested subject, URL, or action, rename affected block labels and titles "
- "to match the revised goal, and update next_block_label, block_labels, and Jinja references accordingly. "
- f"Stale metadata: {joined}"
- )
-
-
-def _find_invalidated_labels(
- old_definition: object | None,
- new_definition: object | None,
- requested_labels: list[str],
-) -> set[str]:
- """Return the set of requested labels whose behavior is invalidated.
-
- A label is invalidated when its own config changed or when any upstream
- label in the requested chain was invalidated (downstream trust propagates
- forward).
- """
- old_by_label = _blocks_by_label(old_definition)
- new_by_label = _blocks_by_label(new_definition)
- invalidated: set[str] = set()
- upstream_invalidated = False
- for label in requested_labels:
- if upstream_invalidated:
- invalidated.add(label)
- continue
- old_block = old_by_label.get(label)
- new_block = new_by_label.get(label)
- if old_block is None or new_block is None:
- invalidated.add(label)
- upstream_invalidated = True
- continue
- if _canonical_block_config(old_block) != _canonical_block_config(new_block):
- invalidated.add(label)
- upstream_invalidated = True
- return invalidated
-
-
-def _earliest_invalidated(requested_labels: list[str], invalidated: set[str]) -> str | None:
- for label in requested_labels:
- if label in invalidated:
- return label
- return None
-
-
-def _clear_runtime_anchor_evidence(copilot_ctx: Any) -> None:
- # Clears the runtime-anchor *trust* flags only. evidence.current_url /
- # page_title / workflow_run_id are left intact: an edit does not move the
- # browser, so they remain accurate observational context — the cleared flags
- # are what mark that state as no longer verified.
- evidence = copilot_ctx.workflow_verification_evidence
- copilot_ctx.verified_prefix_current_url = None
- evidence.live_page_state_verified = False
- evidence.verified_from_current_browser_state = False
- evidence.current_url_observed_after_workflow_run = False
- evidence.current_url_may_encode_runtime_state = False
-
-
-def _reset_all_verified_trust(copilot_ctx: Any) -> None:
- evidence = copilot_ctx.workflow_verification_evidence
- copilot_ctx.verified_prefix_labels = []
- copilot_ctx.verified_block_outputs = {}
- copilot_ctx.last_full_workflow_test_ok = False
- evidence.block_verified = []
- evidence.full_workflow_verified = False
- _clear_runtime_anchor_evidence(copilot_ctx)
-
-
-def _workflow_parameters_changed(prior_definition: object | None, new_definition: object | None) -> bool:
- new_by_key = {getattr(p, "key", None): p for p in (getattr(new_definition, "parameters", None) or [])}
- prior_by_key = {getattr(p, "key", None): p for p in (getattr(prior_definition, "parameters", None) or [])}
- # Any added or removed parameter is a change: a block may reference a key by
- # template without a config edit, so an added/removed key can alter behavior
- # the block-diff alone won't catch. Pure reordering is ignored (keyed access).
- if set(prior_by_key) != set(new_by_key):
- return True
- for key, prior_param in prior_by_key.items():
- try:
- if _stable_parameter_fingerprint(prior_param) != _stable_parameter_fingerprint(new_by_key[key]):
- return True
- except Exception:
- LOG.debug("Parameter fingerprint comparison failed on edit", exc_info=True)
- return True
- return False
-
-
-def _invalidate_verified_state_on_edit(
- copilot_ctx: Any,
- prior_definition: object | None,
- new_definition: object | None,
-) -> None:
- evidence = copilot_ctx.workflow_verification_evidence
- if new_definition is None:
- # An unknown new definition can't be reconciled against; fail closed.
- if copilot_ctx.verified_prefix_labels or evidence.block_verified:
- _reset_all_verified_trust(copilot_ctx)
- return
- prior_labels = _workflow_definition_block_labels(prior_definition)
- trusted = set(copilot_ctx.verified_prefix_labels or []) | set(evidence.block_verified or [])
-
- invalidated: set[str] = set()
- if trusted:
- # No reconcilable prior, or a parameter change that could alter any
- # block's behavior — fail closed rather than reuse unproven trust.
- if prior_definition is None or _workflow_parameters_changed(prior_definition, new_definition):
- _reset_all_verified_trust(copilot_ctx)
- return
- # Diff the full prior chain, not just trusted labels, so a change to an
- # unverified upstream block still propagates to downstream trusted ones.
- full_order = list(prior_labels)
- for label in list(copilot_ctx.verified_prefix_labels or []) + list(evidence.block_verified or []):
- if label not in full_order:
- full_order.append(label)
- try:
- invalidated = _find_invalidated_labels(prior_definition, new_definition, full_order) & trusted
- except Exception:
- LOG.debug("Verified-state invalidation diff failed on edit", exc_info=True)
- _reset_all_verified_trust(copilot_ctx)
- return
-
- if invalidated:
- copilot_ctx.verified_prefix_labels = [
- label for label in copilot_ctx.verified_prefix_labels if label not in invalidated
- ]
- for label in invalidated:
- copilot_ctx.verified_block_outputs.pop(label, None)
- evidence.block_verified = [label for label in evidence.block_verified if label not in invalidated]
- # The recorded prefix-end URL came from a run that included a now-invalid
- # block, so it is no longer a safe runtime anchor for a re-run.
- _clear_runtime_anchor_evidence(copilot_ctx)
-
- # The kept prefix must still be a contiguous leading run of the new workflow;
- # a reorder or upstream insertion breaks that (set-membership trust is
- # order-blind), so fail closed when it no longer is.
- current_labels = _workflow_definition_block_labels(new_definition)
- remaining = list(copilot_ctx.verified_prefix_labels or [])
- if remaining and remaining != current_labels[: len(remaining)]:
- _reset_all_verified_trust(copilot_ctx)
- return
-
- # The end-to-end claim survives only an identical block list whose every block
- # is still verified, so append/removal/reorder/config edits all drop it.
- verified = set(remaining) | set(evidence.block_verified or [])
- if not (current_labels == prior_labels and all(label in verified for label in current_labels)):
- evidence.full_workflow_verified = False
- copilot_ctx.last_full_workflow_test_ok = False
-
-
-def _nearest_upstream_state_establisher(
- requested_labels: list[str], target_label: str, new_definition: object | None
-) -> str | None:
- by_label = _blocks_by_label(new_definition)
- try:
- idx = requested_labels.index(target_label)
- except ValueError:
- return None
- for candidate in reversed(requested_labels[:idx]):
- block = by_label.get(candidate)
- if block is None:
- continue
- if _block_type_name(block) in _BLOCK_TYPES_STATE_ESTABLISHER:
- return candidate
- return None
-
-
-def _block_can_start_browser_run(block: object) -> bool:
- if _block_type_name(block) == BlockType.GOTO_URL.value:
- return True
- return _valid_runtime_anchor_url(getattr(block, "url", None)) is not None
-
-
-def _nearest_upstream_runnable_anchor(
- workflow_labels: list[str], target_label: str, new_definition: object | None
-) -> str | None:
- by_label = _blocks_by_label(new_definition)
- try:
- idx = workflow_labels.index(target_label)
- except ValueError:
- return None
- for candidate in reversed(workflow_labels[:idx]):
- block = by_label.get(candidate)
- if block is not None and _block_can_start_browser_run(block):
- return candidate
- return workflow_labels[0] if workflow_labels[:idx] else None
-
-
-def _serialized_frontier_block_configs(frontier_labels: list[str], new_definition: object | None) -> list[str]:
- by_label = _blocks_by_label(new_definition)
- serialized_configs: list[str] = []
- for label in frontier_labels:
- block = by_label.get(label)
- if block is None:
- continue
- try:
- serialized_configs.append(json.dumps(_canonical_block_config(block), default=str, separators=(",", ":")))
- except (TypeError, ValueError):
- serialized_configs.append(repr(block))
- return serialized_configs
-
-
-def _workflow_parameter_keys(definition: object | None) -> set[str]:
- parameters = getattr(definition, "parameters", None) if definition else None
- keys: set[str] = set()
- if not parameters:
- return keys
- for parameter in parameters:
- key = getattr(parameter, "key", None)
- if isinstance(key, str):
- keys.add(key)
- return keys
-
-
-_CREDENTIAL_REAL_VALUE_SUFFIXES = ("_real_username", "_real_password")
-
-
-def _classify_frontier_jinja_refs(
- frontier_labels: list[str],
- new_definition: object | None,
- serialized_configs: list[str] | None = None,
-) -> tuple[set[str], set[str], set[str]]:
- """Single pass over frontier blocks; returns ``(suffix_form_refs, block_form_refs, unknown_roots)``."""
- if serialized_configs is None:
- serialized_configs = _serialized_frontier_block_configs(frontier_labels, new_definition)
- known_labels = set(_blocks_by_label(new_definition))
- parameter_keys = _workflow_parameter_keys(new_definition)
- known_roots = known_labels | parameter_keys | _TEMPLATE_BUILTIN_ROOTS
-
- suffix_form_refs: set[str] = set()
- block_form_refs: set[str] = set()
- unknown_roots: set[str] = set()
-
- for serialized in serialized_configs:
- for match in _OUTPUT_REF_RE.findall(serialized):
- if match in known_labels:
- suffix_form_refs.add(match)
- for match in _BLOCK_FORM_REF_RE.findall(serialized):
- if match in known_labels:
- block_form_refs.add(match)
- for root in _JINJA_ROOT_RE.findall(serialized):
- if root in known_roots:
- continue
- if root.endswith("_output") and root[: -len("_output")] in known_labels:
- continue
- if any(
- root.endswith(suffix) and root[: -len(suffix)] in parameter_keys
- for suffix in _CREDENTIAL_REAL_VALUE_SUFFIXES
- ):
- continue
- unknown_roots.add(root)
-
- return suffix_form_refs, block_form_refs, unknown_roots
-
-
-def _referenced_output_labels(
- frontier_labels: list[str],
- new_definition: object | None,
- serialized_configs: list[str] | None = None,
-) -> set[str]:
- suffix_refs, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
- return suffix_refs | block_form_refs
-
-
-def _block_form_output_labels(
- frontier_labels: list[str],
- new_definition: object | None,
- serialized_configs: list[str] | None = None,
-) -> set[str]:
- _, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
- return block_form_refs
-
-
-def _unknown_jinja_roots(
- frontier_labels: list[str],
- new_definition: object | None,
- serialized_configs: list[str] | None = None,
-) -> set[str]:
- _, _, unknown_roots = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
- return unknown_roots
-
-
-def _summarize_action_trace(action_trace: list[dict[str, Any]] | None) -> list[str]:
- """Compact, stringified summary of action entries for the compact packet."""
- if not action_trace:
- return []
- summary: list[str] = []
- for entry in action_trace[-6:]:
- if not isinstance(entry, dict):
- continue
- action = entry.get("action") or "?"
- status = entry.get("status") or ""
- element = entry.get("element")
- bits = [str(action)]
- if element:
- bits.append(str(element))
- if status:
- bits.append(str(status))
- summary.append(" ".join(bits).strip())
- return summary
-
-
-async def _get_prior_workflow_definition(ctx: AgentContext) -> object | None:
- """Hybrid: prefer ctx.last_workflow, fall back to DB fetch on cold start."""
- last_workflow = getattr(ctx, "last_workflow", None)
- if last_workflow is not None:
- definition = getattr(last_workflow, "workflow_definition", None)
- if definition is not None:
- return definition
- last_yaml = getattr(ctx, "last_workflow_yaml", None)
- if last_yaml:
- try:
- workflow = _process_workflow_yaml(
- workflow_id=ctx.workflow_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- workflow_yaml=last_yaml,
- )
- return workflow.workflow_definition
- except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException):
- pass
- try:
- fetched = await app.DATABASE.workflows.get_workflow_by_permanent_id(
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- )
- if fetched is not None:
- return fetched.workflow_definition
- except Exception:
- LOG.debug("Failed to fetch prior workflow definition for frontier diff", exc_info=True)
- return None
-
-
-async def _get_prior_workflow(ctx: AgentContext) -> Workflow | None:
- """Return the prior Workflow; in-memory > re-parsed yaml > DB."""
- last_workflow = ctx.last_workflow
- if last_workflow is not None:
- return last_workflow
- last_yaml = ctx.last_workflow_yaml
- if last_yaml:
- try:
- return _process_workflow_yaml(
- workflow_id=ctx.workflow_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- workflow_yaml=last_yaml,
- )
- except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException):
- pass
- try:
- return await app.DATABASE.workflows.get_workflow_by_permanent_id(
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- )
- except Exception:
- LOG.warning(
- "Failed to fetch prior workflow for staging comparison; staging may skip a needed canonical write",
- exc_info=True,
- )
- return None
-
-
-# Must stay in lockstep with the writers calling update_workflow_definition;
-# missing fields silently drop accepted settings on auto-accept.
-_CANONICAL_WORKFLOW_SETTING_FIELDS: tuple[str, ...] = (
- "title",
- "description",
- "proxy_location",
- "webhook_callback_url",
- "totp_verification_url",
- "totp_identifier",
- "persist_browser_session",
- "browser_profile_id",
- "model",
- "max_screenshot_scrolls",
- "extra_http_headers",
- "cdp_connect_headers",
- "run_with",
- "ai_fallback",
- "cache_key",
- "adaptive_caching",
- "code_version",
- "run_sequentially",
- "sequential_key",
-)
-
-# convert_workflow_definition regenerates ids/timestamps per call; ignore
-# them when comparing parameters to react only to user intent.
-_PARAMETER_FINGERPRINT_VOLATILE_KEYS = frozenset(
- {
- "workflow_parameter_id",
- "output_parameter_id",
- "aws_secret_parameter_id",
- "azure_secret_parameter_id",
- "azure_vault_credential_parameter_id",
- "bitwarden_credit_card_data_parameter_id",
- "bitwarden_login_credential_parameter_id",
- "bitwarden_sensitive_information_parameter_id",
- "credential_parameter_id",
- "onepassword_credential_parameter_id",
- "created_at",
- "modified_at",
- }
-)
-
-
-def _stable_parameter_fingerprint(parameter: Parameter) -> dict[str, Any]:
- dump = parameter.model_dump(mode="json")
- return {k: v for k, v in dump.items() if k not in _PARAMETER_FINGERPRINT_VOLATILE_KEYS}
-
-
-def _workflow_requires_canonical_persist(prior: Workflow | None, new: Workflow) -> bool:
- if prior is None:
- return False
- for field_name in _CANONICAL_WORKFLOW_SETTING_FIELDS:
- if getattr(prior, field_name, None) != getattr(new, field_name, None):
- return True
- prior_params = prior.workflow_definition.parameters
- new_params = new.workflow_definition.parameters
- if len(prior_params) != len(new_params):
- return True
- prior_fingerprints = [_stable_parameter_fingerprint(p) for p in prior_params]
- new_fingerprints = [_stable_parameter_fingerprint(p) for p in new_params]
- return prior_fingerprints != new_fingerprints
-
-
-def _plan_frontier(
- ctx: AgentContext,
- requested_labels: list[str],
- old_definition: object | None,
- new_definition: object | None,
-) -> tuple[list[str], dict[str, Any], str | None]:
- """Plan the frontier execution.
-
- Returns ``(labels_to_execute, block_outputs_to_seed, frontier_start_label)``.
-
- Falls back to the full requested list on any ambiguity. When there is no
- workflow change (plain run path) the frontier is the first requested label,
- and we seed verified outputs referenced by the suffix plus prior
- browser-state outputs needed to start a downstream frontier.
- """
- if not requested_labels:
- return requested_labels, {}, None
- if new_definition is None:
- return requested_labels, {}, requested_labels[0]
-
- verified_outputs: dict[str, Any] = dict(ctx.verified_block_outputs or {})
- verified_prefix: list[str] = list(ctx.verified_prefix_labels or [])
- verified_prefix_set = set(verified_prefix)
-
- # No old definition (cold start or parse failure) OR no diff signal → plain path.
- if old_definition is None:
- frontier = requested_labels[0]
- return _seed_for_frontier(requested_labels, frontier, verified_outputs, new_definition)
-
- try:
- invalidated = _find_invalidated_labels(old_definition, new_definition, requested_labels)
- except Exception:
- LOG.debug("Frontier diff failed, falling back to full run", exc_info=True)
- return requested_labels, {}, requested_labels[0]
-
- earliest = _earliest_invalidated(requested_labels, invalidated)
- if earliest is None:
- # No invalidation at all — unchanged request. Continue from the
- # first unverified requested label so a model may keep passing the
- # complete chain while the tool advances the browser in small
- # verified frontiers.
- next_frontier = _first_unverified_requested_label(requested_labels, verified_prefix_set)
- if next_frontier is not None:
- return _seed_for_frontier(requested_labels, next_frontier, verified_outputs, new_definition)
-
- # If the model accidentally asks to rerun an already-verified prefix,
- # keep the browser moving forward instead of spending another tool call
- # on work the current session has already covered.
- workflow_labels = _workflow_definition_block_labels(new_definition)
- next_workflow_frontier = _first_unverified_requested_label(workflow_labels, verified_prefix_set)
- if next_workflow_frontier is not None:
- frontier_idx = workflow_labels.index(next_workflow_frontier)
- return _seed_for_frontier(
- workflow_labels[: frontier_idx + 1],
- next_workflow_frontier,
- verified_outputs,
- new_definition,
- )
-
- return _seed_for_frontier(requested_labels, requested_labels[0], verified_outputs, new_definition)
-
- # Ensure the prefix before the earliest invalidated label is all in the
- # verified prefix from a successful prior run. Otherwise we have no
- # trusted anchor — fall back to the full requested list.
- prefix_in_requested = [label for label in requested_labels if label != earliest]
- prefix_in_requested = prefix_in_requested[: requested_labels.index(earliest)]
- if not all(label in verified_prefix_set for label in prefix_in_requested):
- return requested_labels, {}, requested_labels[0]
-
- old_by_label = _blocks_by_label(old_definition)
- is_append_only = earliest not in old_by_label
- if is_append_only:
- # Case A — append-after-success. The earliest invalidated label is a
- # new block that didn't exist in the prior definition, so the verified
- # prefix represents the browser state just before it. Start there.
- workflow_labels = _workflow_definition_block_labels(new_definition)
- if earliest in workflow_labels:
- workflow_prefix = workflow_labels[: workflow_labels.index(earliest)]
- if not all(label in verified_prefix_set for label in workflow_prefix):
- anchor = _nearest_upstream_runnable_anchor(workflow_labels, earliest, new_definition)
- if anchor is not None:
- return _seed_for_frontier(
- workflow_labels[workflow_labels.index(anchor) : workflow_labels.index(earliest) + 1],
- anchor,
- verified_outputs,
- new_definition,
- )
- return _seed_for_frontier(requested_labels, earliest, verified_outputs, new_definition)
-
- # Edit-in-place. We lack a browser-anchor signal, so we cannot safely
- # rerun just the edited block (the browser is at post-prefix state, not
- # pre-edit state). Walk back to the nearest upstream state-establishing
- # block within the requested chain. Falls back to the full requested list
- # if no safe upstream anchor can be identified.
- anchor = _nearest_upstream_state_establisher(requested_labels, earliest, new_definition)
- if anchor is None:
- return requested_labels, {}, requested_labels[0]
- return _seed_for_frontier(requested_labels, anchor, verified_outputs, new_definition)
-
-
-def _first_unverified_requested_label(requested_labels: list[str], verified_prefix_set: set[str]) -> str | None:
- for label in requested_labels:
- if label not in verified_prefix_set:
- return label
- return None
-
-
-def _seed_for_frontier(
- requested_labels: list[str],
- frontier: str,
- verified_outputs: dict[str, Any],
- new_definition: object | None,
-) -> tuple[list[str], dict[str, Any], str]:
- try:
- idx = requested_labels.index(frontier)
- except ValueError:
- return requested_labels, {}, requested_labels[0]
- labels_to_execute = requested_labels[idx:]
- workflow_labels = _workflow_definition_block_labels(new_definition)
- if frontier in workflow_labels:
- prefix_labels = workflow_labels[: workflow_labels.index(frontier)]
- else:
- prefix_labels = requested_labels[:idx]
- if not prefix_labels:
- return labels_to_execute, {}, frontier
- serialized_configs = _serialized_frontier_block_configs(labels_to_execute, new_definition)
- suffix_refs, block_form_refs, unknown_roots = _classify_frontier_jinja_refs(
- labels_to_execute, new_definition, serialized_configs
- )
- if any(label in block_form_refs for label in prefix_labels):
- # Seeded block_outputs only register <label>_output; block-form refs
- # need a normal upstream execution to populate the <label> namespace.
- return requested_labels, {}, requested_labels[0]
- needed = suffix_refs | block_form_refs
- seed: dict[str, Any] = {}
- for label in prefix_labels:
- if label not in needed:
- continue
- if label not in verified_outputs:
- return requested_labels, {}, requested_labels[0]
- seed[label] = verified_outputs[label]
- if unknown_roots:
- return requested_labels, {}, requested_labels[0]
- by_label = _blocks_by_label(new_definition)
- for label in prefix_labels:
- block = by_label.get(label)
- if block is None or _block_type_name(block) not in _BLOCK_TYPES_STATE_ESTABLISHER:
- continue
- if label in verified_outputs:
- seed.setdefault(label, verified_outputs[label])
- return labels_to_execute, seed, frontier
-
-
-_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS = 2
-_PAGE_CHANGING_FRONTIER_BLOCK_TYPES: frozenset[str] = frozenset(
- {
- BlockType.ACTION.value,
- BlockType.FILE_DOWNLOAD.value,
- BlockType.FILE_UPLOAD.value,
- BlockType.LOGIN.value,
- BlockType.NAVIGATION.value,
- }
-)
-
-
-def _frontier_block_type_names(labels: list[str], workflow_definition: object | None) -> list[str]:
- by_label = _blocks_by_label(workflow_definition)
- type_names: list[str] = []
- for label in labels:
- block = by_label.get(label)
- if block is None:
- continue
- type_name = _block_type_name(block)
- if type_name:
- type_names.append(type_name)
- return type_names
-
-
-def _frontier_has_several_page_changing_stages(labels: list[str], workflow_definition: object | None) -> bool:
- type_names = _frontier_block_type_names(labels, workflow_definition)
- if len(type_names) <= _MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:
- return False
- page_changing_count = sum(1 for type_name in type_names if type_name in _PAGE_CHANGING_FRONTIER_BLOCK_TYPES)
- return page_changing_count >= 2 or (page_changing_count >= 1 and len(type_names) >= 4)
-
-
-def _frontier_includes_required_runtime_anchor(block_labels: list[str], labels_to_execute: list[str]) -> bool:
- if not block_labels or len(labels_to_execute) <= len(block_labels):
- return False
- return labels_to_execute[-len(block_labels) :] == block_labels
-
-
-def _frontier_run_size_error(
- copilot_ctx: object,
- block_labels: list[str],
- labels_to_execute: list[str],
- workflow_definition: object | None,
-) -> str | None:
- if len(labels_to_execute) <= _MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:
- return None
- if _frontier_includes_required_runtime_anchor(block_labels, labels_to_execute):
- return None
- if getattr(copilot_ctx, "build_phase", None) not in (BuildPhase.COMPOSING, BuildPhase.TESTING):
- return None
- if not _frontier_has_several_page_changing_stages(labels_to_execute, workflow_definition):
- return None
-
- suggested = labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS]
- remaining = labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]
- return (
- "Workflow validation failed: this browser test frontier is too long for a multi-stage "
- "page-changing workflow. Keep the same complete workflow YAML, but shrink only the "
- f"block_labels argument to the next 1-2 unverified labels: {suggested!r}. "
- "If a prior run already advanced the browser, inspect that reached page "
- '(inspect_page_for_composition(target_url="current_page")) to ground the next labels in '
- "what is actually there rather than shrinking the frontier blind. "
- f"Do not remove later blocks from the YAML; test them after this frontier succeeds. "
- f"Deferred labels: {remaining!r}. Requested labels: {block_labels!r}."
- )
-
-
-# Watchdog exit reasons. ``success`` means the run reached a trustworthy
-# terminal status inside the poll loop OR after the post-drain reconcile.
-# The three non-success reasons share the reconcile path but produce distinct
-# error messages: ``stagnation`` is the primary trip (no progress signals
-# for ``RUN_BLOCKS_STAGNATION_WINDOW_SECONDS`` seconds), ``ceiling`` is the
-# last-resort budget-exhausted branch, and ``task_exit_unfinalized`` is the
-# rare race where ``execute_workflow`` naturally exits before writing a
-# terminal row.
-WatchdogExitReason = Literal[
- "success",
- "stagnation",
- "ceiling",
- "per_tool_budget",
- "task_exit_unfinalized",
- "active_run_terminal_evidence",
-]
-
-
-def _watchdog_exit_allows_terminal_promotion(exit_reason: WatchdogExitReason | None) -> bool:
- return exit_reason != "active_run_terminal_evidence"
-
-
-# Block types that legitimately execute long silent periods: one DB write on
-# entry, work done without intermediate writes (sleep / LLM call / await human
-# input / browser download wait), one write on finish. The watchdog can't
-# distinguish these from "stuck", so any invocation that includes one disables
-# stagnation for the whole run and relies on the safety ceiling alone.
-_QUIET_BLOCK_TYPES: frozenset[str] = frozenset(
- {
- BlockType.WAIT.value,
- BlockType.TEXT_PROMPT.value,
- BlockType.HUMAN_INTERACTION.value,
- BlockType.FILE_DOWNLOAD.value,
- }
-)
-
-
-def _any_quiet_block_requested(
- copilot_ctx: CopilotContext,
- labels: list[str] | None,
-) -> bool:
- """Return True if any of ``labels`` refers to a block whose type is in
- ``_QUIET_BLOCK_TYPES``. Reuses ``_blocks_by_label`` on the already-loaded
- workflow definition — no DB call.
- """
- if not labels:
- return False
- last_workflow = getattr(copilot_ctx, "last_workflow", None)
- if last_workflow is None:
- return False
- by_label = _blocks_by_label(getattr(last_workflow, "workflow_definition", None))
- for label in labels:
- block = by_label.get(label)
- if block is None:
- continue
- block_type = getattr(block, "block_type", None)
- if block_type is None:
- continue
- block_type_str = block_type.value if hasattr(block_type, "value") else str(block_type)
- if block_type_str in _QUIET_BLOCK_TYPES:
- return True
- return False
-
-
-async def _read_progress_sources(
- ctx: CopilotContext,
- workflow_run_id: str,
-) -> tuple[WorkflowRun | None, datetime | None, datetime | None]:
- """Read one ``workflow_runs`` row + the two progress aggregates needed
- by the watchdog marker. Three cheap indexed queries; no row hydration
- on the aggregate side. The two repo calls run concurrently — they open
- separate async sessions and hit different tables.
- """
-
- async def _read_timestamps() -> tuple[datetime | None, datetime | None]:
- try:
- return await app.DATABASE.tasks.get_workflow_run_progress_timestamps(
- workflow_run_id=workflow_run_id,
- organization_id=ctx.organization_id,
- )
- except Exception:
- LOG.warning(
- "Workflow run progress timestamps read failed",
- workflow_run_id=workflow_run_id,
- exc_info=True,
- )
- return None, None
-
- run, (step_ts, block_ts) = await asyncio.gather(
- _safe_read_workflow_run(workflow_run_id, ctx.organization_id, context="watchdog-poll"),
- _read_timestamps(),
- )
- return run, step_ts, block_ts
-
-
-def _progress_marker(
- run: WorkflowRun | None,
- step_ts: datetime | None,
- block_ts: datetime | None,
-) -> tuple[Any, ...]:
- """Hashable scalar snapshot. Changes iff any observable progress has
- occurred at the run, step, or block level since the last poll. Every
- ``update_step`` fires during action execution (including incremental
- token/cost accumulators at ``forge/agent.py:1449``), so
- ``max(step.modified_at)`` is the per-LLM-call heartbeat. Non-task blocks
- (CODE, TEXT_PROMPT) don't create step rows — ``max(workflow_run_block.modified_at)``
- covers that case. ``run.modified_at`` and ``run.status`` catch the
- run-level transitions that happen outside those two tables.
- """
- return (
- run.status if run else None,
- run.modified_at if run is not None else None,
- step_ts,
- block_ts,
- )
-
-
-async def _watchdog_error_message(
- exit_reason: WatchdogExitReason,
- ctx: AgentContext,
- workflow_run_id: str,
- run: WorkflowRun | None,
- budget_seconds: int,
-) -> str:
- """LLM-facing error string for a non-success watchdog exit. No variant uses
- "timed out" or other retry-inviting phrasing — those are SKY-9163 traps.
- """
- if exit_reason == "stagnation":
- body = (
- f"The run has not made progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s. "
- f"No step, block, or workflow-run row updates were observed in that window. "
- f"The page is most likely blocked by a captcha, popup, anti-bot challenge, "
- f"hidden validation error, or an infinite-retry loop on an action the agent "
- f"cannot detect is failing."
- )
- elif exit_reason == "per_tool_budget":
- message = (
- f"The run exceeded the {budget_seconds}s per-tool-call budget while still "
- f"making progress. This budget exists so a single in-flight call cannot "
- f"consume the whole copilot session.\n"
- f"Run ID: {workflow_run_id}.\n"
- f"Next step: call get_run_results with this workflow_run_id to inspect what "
- f"the cancelled run actually completed (the in-flight block was cancelled "
- f"mid-execution and may have left partial side effects). If the result "
- f"includes a current_url, inspect that current page before any further "
- f'block-running call with inspect_page_for_composition(target_url="current_page"). '
- f"Generic screenshot/evaluate reads can help answer the user, but they do not "
- f"satisfy the bounded page-evidence contract for workflow mutations. Use the "
- f"bounded evidence to decide whether the answer is already visible, whether "
- f"a challenge-gated submit/search control is still disabled, or which page-state "
- f"change is still missing. If challenge_state.gates_submit_controls=true and "
- f"the requested answer is not visible, stop and report the observed anti-bot "
- f"blocker instead of retrying the same solve/wait/submit chain. Only then call "
- f"update_and_run_blocks with a smaller chain — the first 1-2 unverified blocks. "
- f"Verified-prefix state is preserved, so the next call only re-runs from the new frontier. "
- f"Do NOT retry the same chain unchanged — a longer "
- f"run won't fit either."
- )
- current_url, _ = await _fallback_page_info(ctx)
- if current_url:
- message += f" Browser was on: {current_url}"
- return message
- elif exit_reason == "active_run_terminal_evidence":
- message = (
- "The active run was interrupted because bounded current-page evidence matched the requested "
- "browser terminal state while the workflow run was still in progress.\n"
- f"Run ID: {workflow_run_id}.\n"
- "This is NOT full workflow verification: the requested browser state was observed, but the durable "
- "workflow chain still needs diagnosis/repair and a clean verification run. Next step: call "
- "get_run_results with this workflow_run_id to inspect the active run boundary, preserve the observed "
- "current-page evidence, and update only the block(s) that overshot or kept running after the state "
- "was reached. Do NOT report end-to-end success unless a corrected workflow run verifies cleanly."
- )
- current_url, _ = await _fallback_page_info(ctx)
- if current_url:
- message += f" Browser was on: {current_url}"
- return message
- elif exit_reason == "ceiling":
- body = (
- f"The run exceeded the {budget_seconds}s absolute ceiling "
- f"while still showing progress. The workflow is too long to fit in a single "
- f"tool invocation — split it into smaller block groups."
- )
- else: # task_exit_unfinalized
- last_observed = f"last observed status: {run.status}" if run is not None else "workflow run row was unreadable"
- body = (
- f"The run ended but did not record a trustworthy terminal status in the "
- f"cancellation grace window ({last_observed})."
- )
-
- message = (
- f"{body} Run ID: {workflow_run_id}. Outcome is uncertain. "
- f"Do NOT re-invoke block-running tools in this session without first calling "
- f"`get_run_results` with this workflow_run_id and reporting the result to the user."
- )
- current_url, _ = await _fallback_page_info(ctx)
- if current_url:
- message += f" Browser was on: {current_url}"
- return message
-
-
-def _watchdog_user_failure_reason(
- exit_reason: WatchdogExitReason,
- workflow_run_id: str,
- budget_seconds: int,
- run: WorkflowRun | None,
-) -> str:
- if exit_reason == "stagnation":
- body = f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."
- elif exit_reason == "per_tool_budget":
- body = f"The run exceeded the {budget_seconds}s per-tool-call budget while still making progress."
- elif exit_reason == "active_run_terminal_evidence":
- body = (
- "The active run reached the requested browser state before the workflow finished, "
- "so it was interrupted for diagnosis/repair. Full workflow verification is still required."
- )
- elif exit_reason == "ceiling":
- body = f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."
- else:
- status = f" Last observed status: {run.status}." if run is not None else ""
- body = "The run ended before recording a trustworthy terminal status." + status
- return f"{body} Run ID: {workflow_run_id}. Outcome is uncertain."
-
-
-def _watchdog_user_facing_summary(
- exit_reason: WatchdogExitReason,
- budget_seconds: int,
- run: WorkflowRun | None,
-) -> str:
- if exit_reason == "stagnation":
- return f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."
- if exit_reason == "per_tool_budget":
- return f"The run exceeded the {budget_seconds}s per-tool-call budget while still making progress."
- if exit_reason == "ceiling":
- return f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."
- if run is not None:
- return f"The run ended before recording a trustworthy terminal status. Last observed status: {run.status}."
- return "The run ended before recording a trustworthy terminal status."
-
-
-def _workflow_with_runtime_block_goal_context(workflow: Workflow, ctx: CopilotContext) -> Workflow:
- block_goal_main_goal = ctx.block_goal_main_goal or ctx.user_message or ""
- if not block_goal_main_goal:
- LOG.warning("run_blocks invoked without block-goal context; using persisted workflow goals unchanged")
- return workflow
- return wrap_workflow_block_goals(workflow, block_goal_main_goal)
-
-
-def _valid_runtime_anchor_url(value: object) -> str | None:
- if not isinstance(value, str):
- return None
- url = value.strip()
- if not url or url in {"about:blank", ":"}:
- return None
- try:
- parsed = urlparse(url)
- except ValueError:
- return None
- if parsed.scheme not in {"http", "https"} or not parsed.netloc:
- return None
- return url
-
-
-def _blank_runtime_page_url(value: object) -> bool:
- if not isinstance(value, str):
- return True
- # Chrome/CDP can expose ":" while the page is still in early blank-page initialization.
- return value.strip() in {"about:blank", "", ":"}
-
-
-def _missing_runtime_frontier_block_url(value: object) -> bool:
- if value is None:
- return True
- if isinstance(value, str):
- return not value.strip()
- return False
-
-
-def _same_runtime_page(left: str | None, right: str | None) -> bool:
- if not left or not right:
- return False
- try:
- left_parsed = urlparse(left)
- right_parsed = urlparse(right)
- except ValueError:
- return False
- return (
- left_parsed.scheme.lower(),
- left_parsed.netloc.lower(),
- left_parsed.path,
- left_parsed.query,
- ) == (
- right_parsed.scheme.lower(),
- right_parsed.netloc.lower(),
- right_parsed.path,
- right_parsed.query,
- )
-
-
-def _frontier_anchor_url_from_value(value: object, *, depth: int = 0) -> str | None:
- if depth > 3:
- return None
- direct = _valid_runtime_anchor_url(value)
- if direct is not None:
- return direct
- if isinstance(value, dict):
- for key in ("current_url", "url", "page_url"):
- candidate = _valid_runtime_anchor_url(value.get(key))
- if candidate is not None:
- return candidate
- for nested in value.values():
- candidate = _frontier_anchor_url_from_value(nested, depth=depth + 1)
- if candidate is not None:
- return candidate
- if isinstance(value, list):
- for nested in value:
- candidate = _frontier_anchor_url_from_value(nested, depth=depth + 1)
- if candidate is not None:
- return candidate
- return None
-
-
-def _frontier_runtime_anchor_url(ctx: CopilotContext, block_outputs_to_seed: dict[str, Any]) -> str | None:
- for value in (
- ctx.verified_prefix_current_url,
- getattr(ctx, "composition_page_evidence", None),
- ):
- candidate = _frontier_anchor_url_from_value(value)
- if candidate is not None:
- return candidate
- for value in reversed(list((block_outputs_to_seed or {}).values())):
- candidate = _frontier_anchor_url_from_value(value)
- if candidate is not None:
- return candidate
- return None
-
-
-def _iter_workflow_model_blocks(blocks: list[BlockTypeVar] | None) -> list[BlockTypeVar]:
- if not isinstance(blocks, list):
- return []
- return get_all_blocks(blocks)
-
-
-def _workflow_model_block_by_label(workflow_definition: object | None, label: str | None) -> BlockTypeVar | None:
- if workflow_definition is None or not label:
- return None
- blocks = workflow_definition.blocks if hasattr(workflow_definition, "blocks") else None
- for block in _iter_workflow_model_blocks(blocks):
- if block.label == label:
- return block
- return None
-
-
-def _has_verified_prefix_before_frontier(
- ctx: CopilotContext, workflow_definition: object | None, frontier_label: str | None
-) -> bool:
- if not frontier_label:
- return False
- workflow_labels = _workflow_definition_block_labels(workflow_definition)
- if frontier_label not in workflow_labels:
- return False
- prefix_labels = workflow_labels[: workflow_labels.index(frontier_label)]
- if not prefix_labels:
- return False
- verified = set(ctx.verified_prefix_labels or [])
- return all(label in verified for label in prefix_labels)
-
-
-def _workflow_with_runtime_frontier_anchor(
- workflow: Workflow,
- ctx: CopilotContext,
- *,
- labels_to_execute: list[str],
- frontier_start_label: str | None,
- block_outputs_to_seed: dict[str, Any],
-) -> tuple[Workflow, str | None]:
- if not labels_to_execute:
- return workflow, None
- workflow_definition = workflow.workflow_definition
- if not _has_verified_prefix_before_frontier(ctx, workflow_definition, frontier_start_label):
- return workflow, None
-
- first_label = labels_to_execute[0]
- first_block = _workflow_model_block_by_label(workflow_definition, first_label)
- if first_block is None or not hasattr(first_block, "url"):
- return workflow, None
-
- anchor_url = _frontier_runtime_anchor_url(ctx, block_outputs_to_seed)
- if anchor_url is None:
- return workflow, None
-
- existing_url = _valid_runtime_anchor_url(first_block.url if hasattr(first_block, "url") else None)
- if existing_url is not None and not _same_runtime_page(existing_url, anchor_url):
- return workflow, None
-
- if existing_url is not None:
- if _block_type_name(first_block) != BlockType.NAVIGATION.value:
- return workflow, None
- anchored = workflow.model_copy(deep=True)
- anchored_block = _workflow_model_block_by_label(anchored.workflow_definition, first_label)
- if anchored_block is None or not hasattr(anchored_block, "url"):
- return workflow, None
- anchored_block.url = None
- LOG.info(
- "Cleared runtime frontier URL to preserve browser state",
- frontier_start_label=frontier_start_label,
- first_block_label=first_label,
- existing_url=existing_url,
- continuation_url=anchor_url,
- )
- return anchored, anchor_url
-
- LOG.info(
- "Preserved runtime frontier browser state without URL reload",
- frontier_start_label=frontier_start_label,
- first_block_label=first_label,
- continuation_url=anchor_url,
- )
- return workflow, anchor_url
-
-
-async def _workflow_with_runtime_frontier_starter_url_seed(
- workflow: Workflow,
- ctx: CopilotContext,
- *,
- labels_to_execute: list[str],
- runtime_frontier_anchor_url: str | None,
-) -> Workflow:
- if not labels_to_execute or runtime_frontier_anchor_url is None or not ctx.browser_session_id:
- return workflow
-
- first_label = labels_to_execute[0]
- first_block = _workflow_model_block_by_label(workflow.workflow_definition, first_label)
- if (
- first_block is None
- or not hasattr(first_block, "url")
- or _block_type_name(first_block) != BlockType.NAVIGATION.value
- or not _missing_runtime_frontier_block_url(first_block.url)
- ):
- return workflow
-
- current_page_url: str | None = None
- try:
- browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
- session_id=ctx.browser_session_id,
- organization_id=ctx.organization_id,
- )
- if browser_state is not None:
- page = await browser_state.get_working_page()
- # Playwright Page.url is exposed as a dynamic property at this boundary.
- current_page_url = page.url if page is not None else None
- except Exception:
- LOG.debug(
- "Failed to inspect runtime frontier browser page before starter URL seed",
- browser_session_id=ctx.browser_session_id,
- frontier_start_label=first_label,
- exc_info=True,
- )
-
- if not _blank_runtime_page_url(current_page_url):
- LOG.info(
- "Preserved attached runtime frontier browser page",
- browser_session_id=ctx.browser_session_id,
- frontier_start_label=first_label,
- current_url=current_page_url,
- continuation_url=runtime_frontier_anchor_url,
- )
- return workflow
-
- seeded = workflow.model_copy(deep=True)
- seeded_block = _workflow_model_block_by_label(seeded.workflow_definition, first_label)
- # Defensive: the copied workflow definition should preserve labels and URL fields.
- if seeded_block is None or not hasattr(seeded_block, "url"):
- return workflow
- seeded_block.url = runtime_frontier_anchor_url
- LOG.info(
- "Seeded runtime frontier starter URL because attached browser page was blank",
- browser_session_id=ctx.browser_session_id,
- frontier_start_label=first_label,
- current_url=current_page_url,
- continuation_url=runtime_frontier_anchor_url,
- )
- return seeded
-
-
-async def _run_blocks_and_collect_debug(
- params: dict[str, Any],
- ctx: CopilotContext,
- *,
- labels_to_execute: list[str] | None = None,
- block_outputs_to_seed: dict[str, Any] | None = None,
- frontier_start_label: str | None = None,
-) -> dict[str, Any]:
- block_labels = params["block_labels"]
- if not block_labels:
- return {"ok": False, "error": "block_labels must not be empty"}
-
- labels_to_execute = list(labels_to_execute) if labels_to_execute else list(block_labels)
- block_outputs_to_seed = block_outputs_to_seed or {}
- if frontier_start_label is None:
- frontier_start_label = labels_to_execute[0] if labels_to_execute else None
-
- ctx.last_requested_block_labels = list(block_labels)
- ctx.last_executed_block_labels = list(labels_to_execute)
- ctx.last_frontier_start_label = frontier_start_label
-
- # Verified state is NOT invalidated pre-run. On a failed / partial run we
- # want the prior verified prefix preserved so the next edit can still use
- # the optimization. YAML-diff-based invalidation for edited/downstream
- # labels happens in update_and_run_blocks_tool at edit time, which is the
- # right moment to drop stale outputs. Full success at the end of this
- # function updates verified state in place (overwriting re-run labels).
-
- # Common-case staging leaves the canonical row stale; prefer the staged copy.
- workflow = ctx.staged_workflow
- if workflow is None:
- workflow = await app.DATABASE.workflows.get_workflow_by_permanent_id(
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- )
- if not workflow:
- return {"ok": False, "error": f"Workflow not found: {ctx.workflow_permanent_id}"}
-
- credential_ids = list(
- dict.fromkeys(
- _extract_credential_ids_from_tool_value(params.get("parameters") or {})
- + _extract_credential_ids_from_workflow_definition(workflow.workflow_definition)
- )
- )
- credential_error = await _credential_ids_validation_error(credential_ids, ctx)
- if credential_error is not None:
- return {"ok": False, "error": credential_error}
-
- for label in block_labels:
- if not workflow.get_output_parameter(label):
- return {"ok": False, "error": f"Block label not found in saved workflow: {label!r}"}
-
- from skyvern.forge.sdk.schemas.organizations import Organization
- from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody
- from skyvern.services import workflow_service
-
- org = await app.DATABASE.organizations.get_organization(organization_id=ctx.organization_id)
- if not org:
- return {"ok": False, "error": "Organization not found"}
-
- organization = Organization.model_validate(org)
- runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)
- runtime_workflow, runtime_frontier_anchor_url = _workflow_with_runtime_frontier_anchor(
- runtime_workflow,
- ctx,
- labels_to_execute=labels_to_execute,
- frontier_start_label=frontier_start_label,
- block_outputs_to_seed=block_outputs_to_seed,
- )
- runtime_frontier_starter_url_seeded = False
-
- user_params: dict[str, Any] = params.get("parameters") or {}
- all_workflow_params, all_output_params = await asyncio.gather(
- app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow.workflow_id),
- app.DATABASE.workflow_params.get_workflow_output_parameters(workflow_id=workflow.workflow_id),
- )
-
- # Short-circuit before a wasted workflow execution when the definition
- # JSON has drifted from the persisted parameter rows that runtime reads.
- invariant_error = _parameter_binding_invariant_error(workflow, all_workflow_params, all_output_params)
- if invariant_error is not None:
- summary, missing_persisted, missing_from_definition = invariant_error
- return {
- "ok": False,
- "error": summary,
- "data": {
- "workflow_run_id": None,
- "overall_status": "failed",
- "failure_reason": summary,
- "requested_block_labels": list(block_labels),
- "executed_block_labels": [],
- "frontier_start_label": None,
- "blocks": [],
- "failure_categories": [
- {
- "category": "PARAMETER_BINDING_ERROR",
- "confidence_float": 0.99,
- "reasoning": "Pre-run invariant: workflow_definition and persisted parameter rows disagree",
- }
- ],
- "missing_persisted": missing_persisted,
- "missing_from_definition": missing_from_definition,
- },
- }
-
- data: dict[str, Any] = {}
- for wp in all_workflow_params:
- if wp.key in user_params:
- data[wp.key] = user_params[wp.key]
- elif wp.default_value is not None:
- data[wp.key] = wp.default_value
- else:
- placeholder = _placeholder_for_parameter_type(wp.workflow_parameter_type)
- if placeholder is not None:
- data[wp.key] = placeholder
- LOG.info(
- "Auto-filled missing workflow parameter for copilot test run",
- parameter_key=wp.key,
- parameter_type=str(wp.workflow_parameter_type),
- )
-
- # Without a session, the workflow service launches the browser in-process,
- # which only works in worker pods (cloakbrowser isn't in the API image).
- session_err = await ensure_browser_session(ctx)
- if session_err is not None:
- return session_err
-
- seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(
- runtime_workflow,
- ctx,
- labels_to_execute=labels_to_execute,
- runtime_frontier_anchor_url=runtime_frontier_anchor_url,
- )
- runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
- runtime_workflow = seeded_runtime_workflow
-
- workflow_request = WorkflowRequestBody(
- data=data if data else None,
- browser_session_id=ctx.browser_session_id,
- # Copilot test runs don't need scrolling post-action screenshots;
- # the ForgeAgent's split screenshots (used for LLM context) are unaffected.
- max_screenshot_scrolls=0,
- )
-
- workflow_run = await workflow_service.prepare_workflow(
- workflow_id=ctx.workflow_permanent_id,
- organization=organization,
- workflow_request=workflow_request,
- template=False,
- version=None,
- max_steps=None,
- request_id=None,
- copilot_session_id=ctx.workflow_copilot_chat_id,
- )
-
- from skyvern.utils.files import initialize_skyvern_state_file
-
- await initialize_skyvern_state_file(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=ctx.organization_id,
- )
-
- run_task = asyncio.create_task(
- app.WORKFLOW_SERVICE.execute_workflow(
- workflow_run_id=workflow_run.workflow_run_id,
- api_key="copilot-agent",
- organization=organization,
- browser_session_id=ctx.browser_session_id,
- block_labels=labels_to_execute,
- block_outputs=block_outputs_to_seed or None,
- workflow_override=runtime_workflow,
- )
- )
-
- # The OpenAI Agents SDK wraps this tool in
- # ``asyncio.wait_for(..., timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS)``, so
- # the inner budget leaves 10 s of headroom for the cancel-drain and
- # post-drain reconcile to finish before the SDK's own cancel fires.
- #
- # Do NOT short-circuit on client disconnect: the agent loop runs to
- # completion after the SSE stream is gone so its reply persists
- # (SKY-8986); aborting mid-block would strand the run without debug
- # output for the final chat message.
- initial_run, initial_step_ts, initial_block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
- progress_marker = _progress_marker(initial_run, initial_step_ts, initial_block_ts)
- last_progress_monotonic = time.monotonic()
- started_monotonic = last_progress_monotonic
- final_status: str | None = None
- run: Any = initial_run
- exit_reason: WatchdogExitReason | None = None
- run_cancelled_by_watchdog = False
- active_run_terminal_evidence: ActiveRunTerminalEvidenceSample | None = None
- # Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have
- # DB-silent periods; disable stagnation for any invocation that includes
- # one. Safety ceiling still applies.
- stagnation_enabled = not _any_quiet_block_requested(ctx, labels_to_execute)
- # Active block runs use the tighter per-tool budget so a single in-flight
- # call cannot consume the whole copilot session. Quiet-block runs keep the
- # long safety ceiling because HumanInteractionBlock can legitimately pause
- # indefinitely.
- budget_exit_reason: WatchdogExitReason
- if stagnation_enabled:
- budget_seconds = _active_block_run_budget_seconds(ctx)
- budget_exit_reason = "per_tool_budget"
- else:
- budget_seconds = max(1, RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10)
- budget_exit_reason = "ceiling"
-
- # Mid-tool narrator bridge: feed block-status changes and step-level
- # heartbeats into NarratorState so the narration ticker keeps emitting
- # while a long workflow run is in flight.
- narrator_state: NarratorState | None = getattr(ctx, "narrator_state", None)
- narrator_enabled = narrator_state is not None and narration_handler_available()
- seen_block_states: dict[str, str] = {}
- prior_block_ts: datetime | None = initial_block_ts
- last_block_fetch_monotonic = 0.0
- active_terminal_monitor_enabled = await _active_run_terminal_monitor_enabled(ctx)
- next_active_terminal_monitor_monotonic = (
- started_monotonic + _ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS
- if active_terminal_monitor_enabled
- else float("inf")
- )
- active_terminal_monitor_samples = 0
-
- try:
- while True:
- await asyncio.sleep(RUN_BLOCKS_POLL_INTERVAL_SECONDS)
-
- run, step_ts, block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
-
- if narrator_enabled:
- assert narrator_state is not None # narrator_enabled implies non-None
- tick_result = await narrator_poll_tick(
- narrator_state,
- current_block_ts=block_ts,
- prior_block_ts=prior_block_ts,
- last_block_fetch_monotonic=last_block_fetch_monotonic,
- seen_block_states=seen_block_states,
- fetch_block_statuses=lambda: app.DATABASE.observer.get_workflow_run_blocks(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=ctx.organization_id,
- ),
- stream=ctx.stream,
- block_state_map=ctx.block_state_map,
- block_started_at_map=ctx.block_started_at_map,
- block_ended_at_map=ctx.block_ended_at_map,
- )
- prior_block_ts = tick_result.prior_block_ts
- last_block_fetch_monotonic = tick_result.last_block_fetch_monotonic
-
- if run and WorkflowRunStatus(run.status).is_final():
- final_status = run.status
- exit_reason = "success"
- break
-
- if run_task.done():
- # Row not terminal yet — shared reconcile path below flips
- # most of these back to success after post-drain reread.
- exit_reason = "task_exit_unfinalized"
- break
-
- now = time.monotonic()
- new_marker = _progress_marker(run, step_ts, block_ts)
- # A run in ``paused`` status (e.g. HumanInteractionBlock) is a
- # user-driven wait, not stagnation — never trip.
- is_paused = run is not None and run.status == WorkflowRunStatus.paused.value
- stagnation_active = stagnation_enabled and not is_paused
-
- if (
- active_terminal_monitor_enabled
- and not is_paused
- and active_terminal_monitor_samples < _ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES
- and now >= next_active_terminal_monitor_monotonic
- ):
- active_terminal_monitor_samples += 1
- next_active_terminal_monitor_monotonic += _ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS
- sample = await _active_run_terminal_evidence_sample(
- ctx,
- workflow_run_id=workflow_run.workflow_run_id,
- labels_to_execute=labels_to_execute,
- sample_index=active_terminal_monitor_samples,
- )
- if sample is not None:
- post_sample_run = await _safe_read_workflow_run(
- workflow_run.workflow_run_id,
- ctx.organization_id,
- context="active-terminal-post-sample",
- )
- if post_sample_run is not None:
- run = post_sample_run
- active_run_terminal_evidence = sample
- exit_reason = "active_run_terminal_evidence"
- break
-
- if new_marker != progress_marker:
- progress_marker = new_marker
- last_progress_monotonic = now
- elif stagnation_active and now - last_progress_monotonic >= RUN_BLOCKS_STAGNATION_WINDOW_SECONDS:
- exit_reason = "stagnation"
- break
-
- if now - started_monotonic >= budget_seconds:
- exit_reason = budget_exit_reason
- break
-
- if exit_reason is not None and exit_reason != "success":
- # Pre-cancel read first: a legitimate self-finalize (user/block
- # cancel, or any terminal the run wrote itself) can land between
- # the last poll and here, and trusting it avoids the
- # synthetic-``canceled`` ambiguity that the post-drain reread
- # has to exclude. Then cancel + reread +
- # ``_trusted_post_drain_status`` applies SKY-9167's success-race
- # recovery uniformly to all three non-success exit reasons.
- pre_cancel_run = await _safe_read_workflow_run(
- workflow_run.workflow_run_id, ctx.organization_id, context="pre-cancel"
- )
- if (
- _watchdog_exit_allows_terminal_promotion(exit_reason)
- and pre_cancel_run is not None
- and WorkflowRunStatus(pre_cancel_run.status).is_final()
- ):
- final_status = pre_cancel_run.status
- run = pre_cancel_run
- exit_reason = "success"
- else:
- if pre_cancel_run is not None:
- run = pre_cancel_run
- if run is None or not WorkflowRunStatus(run.status).is_final():
- await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)
- run_cancelled_by_watchdog = True
- run = await _safe_read_workflow_run(
- workflow_run.workflow_run_id, ctx.organization_id, context="post-drain"
- )
- if _watchdog_exit_allows_terminal_promotion(exit_reason):
- trusted = _trusted_post_drain_status(run)
- if trusted is not None:
- final_status = trusted
- exit_reason = "success"
-
- if exit_reason != "success":
- assert exit_reason is not None # narrows for mypy; outer check excludes "success" but not None
- _mark_pending_reconciliation_run(ctx, workflow_run.workflow_run_id)
- error_msg = await _watchdog_error_message(
- exit_reason, ctx, workflow_run.workflow_run_id, run, budget_seconds
- )
- user_failure_reason = _watchdog_user_failure_reason(
- exit_reason, workflow_run.workflow_run_id, budget_seconds, run
- )
- user_facing_summary = _watchdog_user_facing_summary(exit_reason, budget_seconds, run)
- current_url, page_title = await _fallback_page_info(ctx)
- if exit_reason == "active_run_terminal_evidence" and active_run_terminal_evidence is not None:
- result: dict[str, Any] = _active_run_terminal_evidence_result(
- workflow_run_id=workflow_run.workflow_run_id,
- run_status=run.status if run is not None else None,
- sample=active_run_terminal_evidence,
- requested_block_labels=list(block_labels),
- executed_block_labels=list(labels_to_execute),
- current_url=current_url,
- page_title=page_title,
- )
- result["error"] = error_msg
- result["data"]["failure_reason"] = user_failure_reason
- else:
- result = {
- "ok": False,
- "error": error_msg,
- "data": {
- "workflow_run_id": workflow_run.workflow_run_id,
- "overall_status": run.status if run is not None else None,
- "failure_reason": user_failure_reason,
- "current_url": current_url,
- "page_title": page_title,
- },
- }
- result["data"]["control_signal"] = {
- "kind": f"watchdog_{exit_reason}",
- "user_facing_summary": user_facing_summary,
- }
- result["data"]["user_facing_summary"] = user_facing_summary
- if exit_reason == "per_tool_budget":
- # Stable failure_categories entry so consecutive budget trips
- # hash to the same streak signature; without it the run_id in
- # ``error_msg`` would make every trip unique.
- result["data"]["failure_categories"] = [
- {
- "category": PER_TOOL_BUDGET_FAILURE_CATEGORY,
- "confidence_float": 1.0,
- "reasoning": (
- f"Per-tool-call budget of {budget_seconds}s exceeded; "
- "the run was making progress but cannot fit in a single tool call."
- ),
- }
- ]
- if run_cancelled_by_watchdog:
- result[_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY] = True
- return result
- except asyncio.CancelledError:
- # The SDK's @function_tool(timeout=...) cancelled us mid-poll. Shield
- # the cleanup so the parent cancellation can't interrupt it mid-await.
- # If the shield itself is cancelled, fall back to a detached task
- # that outlives tool teardown and still reconciles workflow state.
- try:
- await asyncio.shield(_cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id))
- except asyncio.CancelledError:
- fallback = asyncio.ensure_future(_cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id))
- _DETACHED_CLEANUP_TASKS.add(fallback)
- fallback.add_done_callback(_DETACHED_CLEANUP_TASKS.discard)
- fallback.add_done_callback(_log_detached_cleanup_failure)
- raise
- finally:
- # Belt and braces. If any exit path above missed a cancel — e.g. an
- # unexpected exception bubbling out of the poll loop — make sure the
- # run_task is at least signaled to cancel so we don't leak it.
- if not run_task.done():
- run_task.cancel()
-
- if run and run.browser_session_id:
- ctx.browser_session_id = run.browser_session_id
-
- blocks = await app.DATABASE.observer.get_workflow_run_blocks(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=ctx.organization_id,
- )
-
- results = []
- block_outputs_by_label: dict[str, Any] = {}
- for block in blocks:
- block_result: dict[str, Any] = {
- "label": block.label,
- "block_type": block.block_type.name if hasattr(block.block_type, "name") else str(block.block_type),
- "status": block.status,
- }
- if block.failure_reason:
- block_result["failure_reason"] = block.failure_reason
- if hasattr(block, "output") and block.output:
- block_result["extracted_data"] = block.output
- if block.label is not None:
- block_outputs_by_label[block.label] = block.output
- results.append(block_result)
-
- await _attach_action_traces(blocks, results, ctx.organization_id)
-
- # final_status is guaranteed set here: every non-success exit returns
- # above, and the success path always populates final_status.
- assert final_status is not None
- run_ok = WorkflowRunStatus(final_status) == WorkflowRunStatus.completed
-
- action_trace_summary: list[str] = []
- first_failed = next(
- (r for r in results if r.get("status") in _FAILED_BLOCK_STATUSES and r.get("action_trace")),
- None,
- )
- if first_failed is not None:
- action_trace_summary = _summarize_action_trace(first_failed.get("action_trace"))
-
- # Compute the action-sequence fingerprint BEFORE we strip action_trace.
- # Stash it on a pending ctx field so update_repeated_failure_state can
- # compare the NEW fingerprint against ctx.last_action_sequence_fingerprint
- # (the PRIOR value) and increment the streak. Never enters the LLM-visible
- # packet. Drives the repeated-action streak that hard-aborts a stuck
- # fill→click→re-fill loop in _tool_loop_error.
- ctx.pending_action_sequence_fingerprint = compute_action_sequence_fingerprint(results)
-
- # Per-block action_trace is for derivation only — keep it out of the
- # compact packet. get_run_results remains the heavier inspection path.
- for entry in results:
- entry.pop("action_trace", None)
-
- current_url, page_title = await _fallback_page_info(ctx)
-
- screenshot_b64: str | None = None
- if not run_ok and ctx.browser_session_id:
- try:
- browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
- session_id=ctx.browser_session_id,
- organization_id=ctx.organization_id,
- )
- if browser_state:
- page = await browser_state.get_or_create_page()
- if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
- try:
- await SkyvernFrame.hide_cursor_overlay(page)
- except Exception:
- pass
- try:
- screenshot_bytes = await page.screenshot(type="png")
- finally:
- if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
- try:
- await SkyvernFrame.show_cursor_overlay(page)
- except Exception:
- pass
- screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
- except Exception:
- LOG.debug("Failed to capture post-run screenshot", exc_info=True)
-
- result_data: dict[str, Any] = {
- "workflow_run_id": workflow_run.workflow_run_id,
- "browser_session_id": ctx.browser_session_id,
- "overall_status": final_status,
- "requested_block_labels": list(block_labels),
- "executed_block_labels": list(labels_to_execute),
- "frontier_start_label": frontier_start_label,
- "blocks": results,
- "current_url": current_url,
- "page_title": page_title,
- "action_trace_summary": action_trace_summary,
- }
- if runtime_frontier_anchor_url is not None:
- result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_url
- if runtime_frontier_starter_url_seeded:
- result_data["runtime_frontier_starter_url_seeded"] = True
- if screenshot_b64 is not None:
- result_data["screenshot_base64"] = screenshot_b64
- if not run_ok and run and getattr(run, "failure_reason", None):
- result_data["failure_reason"] = run.failure_reason
-
- # Update verified prefix state ONLY on a fully-successful run. A failed
- # suffix run leaves the browser in post-failure state, so we must not
- # trust blocks that individually succeeded inside it.
- if run_ok and all(r.get("status") == "completed" for r in results):
- for label, output in block_outputs_by_label.items():
- ctx.verified_block_outputs[label] = output
- existing_prefix = list(getattr(ctx, "verified_prefix_labels", []) or [])
- existing_set = set(existing_prefix)
- for label in labels_to_execute:
- if label not in existing_set:
- existing_prefix.append(label)
- existing_set.add(label)
- ctx.verified_prefix_labels = existing_prefix
- verified_current_url = _valid_runtime_anchor_url(current_url)
- if verified_current_url is not None:
- ctx.verified_prefix_current_url = verified_current_url
-
- return build_run_blocks_response(run_ok, result_data)
-
-
-async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
- workflow_run_id = params.get("workflow_run_id")
- pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
- if isinstance(pending_run_id, str) and pending_run_id:
- if workflow_run_id and workflow_run_id != pending_run_id:
- return {
- "ok": False,
- "error": (
- f"Run inspection is pending for {pending_run_id}; "
- "call get_run_results with that workflow_run_id first."
- ),
- }
- workflow_run_id = pending_run_id
- if not workflow_run_id:
- same_turn_run_id = getattr(ctx, "last_successful_run_blocks_workflow_run_id", None)
- if not isinstance(same_turn_run_id, str) or not same_turn_run_id:
- same_turn_run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
- if isinstance(same_turn_run_id, str) and same_turn_run_id:
- workflow_run_id = same_turn_run_id
-
- if not workflow_run_id:
- # Include every final state so the agent can inspect failures via the
- # fallback. Non-final states (created/queued/running/paused) remain
- # excluded — reading block records from an in-flight run is unsafe.
- runs = await app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id(
- workflow_permanent_id=ctx.workflow_permanent_id,
- organization_id=ctx.organization_id,
- page=1,
- page_size=1,
- status=[
- WorkflowRunStatus.completed,
- WorkflowRunStatus.failed,
- WorkflowRunStatus.terminated,
- WorkflowRunStatus.canceled,
- WorkflowRunStatus.timed_out,
- ],
- )
- if not runs:
- return {"ok": False, "error": "No runs found for this workflow."}
- workflow_run_id = runs[0].workflow_run_id
-
- run = await app.DATABASE.workflow_runs.get_workflow_run(
- workflow_run_id=workflow_run_id,
- organization_id=ctx.organization_id,
- )
- if not run:
- return {"ok": False, "error": f"Workflow run not found: {workflow_run_id}"}
- if getattr(run, "workflow_permanent_id", None) != ctx.workflow_permanent_id:
- return {"ok": False, "error": f"Workflow run not found for this workflow: {workflow_run_id}"}
-
- blocks = await app.DATABASE.observer.get_workflow_run_blocks(
- workflow_run_id=workflow_run_id,
- organization_id=ctx.organization_id,
- )
-
- results = []
- for block in blocks:
- block_result: dict[str, Any] = {
- "label": block.label,
- "block_type": block.block_type.name if hasattr(block.block_type, "name") else str(block.block_type),
- "status": block.status,
- }
- if block.failure_reason:
- block_result["failure_reason"] = block.failure_reason
- output = truncate_output(getattr(block, "output", None))
- if output:
- block_result["output"] = output
- results.append(block_result)
-
- await _attach_action_traces(blocks, results, ctx.organization_id)
- await _attach_failed_block_screenshots(blocks, results, ctx.organization_id)
-
- result_data: dict[str, Any] = {
- "workflow_run_id": workflow_run_id,
- "overall_status": run.status,
- "blocks": results,
- }
- current_url, page_title = await _fallback_page_info(ctx)
- if current_url:
- result_data["current_url"] = current_url
- result_data["page_title"] = page_title
- if getattr(run, "failure_reason", None):
- result_data["failure_reason"] = run.failure_reason
-
- return {
- "ok": True,
- "data": result_data,
- }
-
-
-async def _fallback_page_info(ctx: AgentContext) -> tuple[str, str]:
- if not ctx.browser_session_id:
- return "", ""
- try:
- browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
- session_id=ctx.browser_session_id,
- organization_id=ctx.organization_id,
- )
- if not browser_state:
- return "", ""
- page = await browser_state.get_or_create_page()
- if page:
- return page.url, await page.title()
- except Exception:
- pass
- return "", ""
-
-
-async def _resolve_url_title(raw: dict[str, Any], ctx: AgentContext) -> tuple[str, str]:
- """Extract URL and title from raw MCP result, falling back to live page info."""
- browser_ctx = raw.get("browser_context", {})
- url = browser_ctx.get("url", "")
- title = browser_ctx.get("title", "")
- if not url:
- url, fallback_title = await _fallback_page_info(ctx)
- if fallback_title:
- title = fallback_title
- return url, title
-
-
-def _bounded_observation_text(value: object, limit: int = 240) -> str:
- return " ".join(str(value or "").split())[:limit]
-
-
-@dataclass(frozen=True)
-class _ObservedFieldEvidence:
- name: str
- id: str
- label: str
- type: str
- placeholder: str
- required: bool
- selector: str
-
- @classmethod
- def from_raw(cls, raw_field: dict[str, Any]) -> _ObservedFieldEvidence:
- label = raw_field.get("label")
- if not label and isinstance(raw_field.get("labels"), list):
- label = " ".join(str(item) for item in raw_field["labels"][:2])
- return cls(
- name=_bounded_observation_text(raw_field.get("name"), 120),
- id=_bounded_observation_text(raw_field.get("id"), 120),
- label=_bounded_observation_text(label, 240),
- type=_bounded_observation_text(raw_field.get("type"), 40),
- placeholder=_bounded_observation_text(raw_field.get("placeholder"), 240),
- required=bool(raw_field.get("required")),
- selector=_bounded_observation_text(raw_field.get("selector"), 160),
- )
-
- def as_evidence(self) -> dict[str, Any]:
- return {
- "name": self.name,
- "id": self.id,
- "label": self.label,
- "type": self.type,
- "placeholder": self.placeholder,
- "required": self.required,
- "selector": self.selector,
- }
-
-
-def _normalize_observed_fields(raw_fields: object) -> list[dict[str, Any]]:
- if not isinstance(raw_fields, list):
- return []
- fields: list[dict[str, Any]] = []
- for raw_field in raw_fields[:20]:
- if not isinstance(raw_field, dict):
- continue
- fields.append(_ObservedFieldEvidence.from_raw(raw_field).as_evidence())
- return fields
-
-
-def _normalize_observed_forms(observed_data: dict[str, Any]) -> list[dict[str, Any]]:
- raw_forms = observed_data.get("forms")
- forms: list[dict[str, Any]] = []
- if isinstance(raw_forms, list):
- for raw_form in raw_forms[:5]:
- if not isinstance(raw_form, dict):
- continue
- fields = _normalize_observed_fields(raw_form.get("fields") or raw_form.get("inputs"))
- submit_controls = _normalize_observed_fields(
- raw_form.get("submit_controls") or raw_form.get("buttons") or raw_form.get("submits")
- )
- forms.append(
- {
- "id": _bounded_observation_text(raw_form.get("id"), 120),
- "name": _bounded_observation_text(raw_form.get("name"), 120),
- "action": _bounded_observation_text(raw_form.get("action"), 240),
- "method": _bounded_observation_text(raw_form.get("method"), 20),
- "fields": fields,
- "submit_controls": submit_controls[:10],
- }
- )
- if forms:
- return forms
- fields = _normalize_observed_fields(observed_data.get("inputs") or observed_data.get("fields"))
- if not fields:
- return []
- return [{"id": "", "name": "", "action": "", "method": "", "fields": fields, "submit_controls": []}]
-
-
-def _evaluate_observed_payload(observed_data: object) -> dict[str, Any]:
- if not isinstance(observed_data, dict):
- return {}
- raw_result = observed_data.get("result")
- if isinstance(raw_result, dict):
- payload: dict[str, Any] = dict(raw_result)
- elif isinstance(raw_result, list):
- payload = {"rows": raw_result}
- else:
- payload = {}
- for key, value in observed_data.items():
- if key == "result":
- continue
- if key not in payload or payload.get(key) in (None, "", [], {}):
- payload[key] = value
- return payload or observed_data
-
-
-def _observed_row_text(row: object) -> str:
- if isinstance(row, str):
- return _bounded_observation_text(row, 500)
- if not isinstance(row, dict):
- return ""
- parts: list[str] = []
- for key in ("text", "label", "name", "title"):
- value = row.get(key)
- if isinstance(value, str):
- parts.append(value)
- cells = row.get("cells")
- if isinstance(cells, list):
- for cell in cells[:12]:
- if isinstance(cell, dict):
- for key in ("text", "value"):
- value = cell.get(key)
- if isinstance(value, str):
- parts.append(value)
- elif isinstance(cell, str):
- parts.append(cell)
- return _bounded_observation_text(" ".join(parts), 500)
-
-
-def _normalize_observed_result_containers(observed_data: dict[str, Any]) -> list[dict[str, Any]]:
- raw_results = observed_data.get("result_containers") or observed_data.get("tables")
- containers: list[dict[str, Any]] = []
- if isinstance(raw_results, list):
- for item in raw_results[:8]:
- if isinstance(item, dict):
- containers.append(
- {
- "tag": _bounded_observation_text(item.get("tag") or item.get("type"), 40),
- "id": _bounded_observation_text(item.get("id"), 120),
- "selector": _bounded_observation_text(item.get("selector"), 160),
- }
- )
- if containers:
- return containers
- raw_rows = observed_data.get("rows")
- if isinstance(raw_rows, list) and raw_rows:
- sample_rows = [text for row in raw_rows[:5] if (text := _observed_row_text(row))]
- return [
- {
- "tag": "table",
- "id": _bounded_observation_text(observed_data.get("id"), 120),
- "selector": _bounded_observation_text(observed_data.get("selector"), 160),
- "row_count": len(raw_rows),
- "sample_rows": sample_rows,
- }
- ]
- table = observed_data.get("table")
- if table:
- return [{"tag": "table", "id": "", "selector": ""}]
- return []
-
-
-def _normalize_evaluate_challenge_state(
- observed_data: dict[str, Any],
- anti_bot_indicators: list[str],
-) -> dict[str, Any]:
- detected = bool(anti_bot_indicators)
- gated_controls: list[dict[str, Any]] = []
- for key, value in observed_data.items():
- normalized_key = str(key).strip().lower().replace("_", " ").replace("-", " ")
- if "disabled" not in normalized_key or value is not True:
- continue
- if any(token in normalized_key for token in ("submit", "search", "button", "btn")):
- gated_controls.append(
- {
- "text": _bounded_observation_text(str(key), 120),
- "id": "",
- "name": _bounded_observation_text(str(key), 120),
- "selector": "",
- "disabled": True,
- }
- )
- indicator_text = " ".join(anti_bot_indicators).lower()
- if "access denied" in indicator_text:
- kind = "access_denied"
- elif "turnstile" in indicator_text or "captcha" in indicator_text or "are you a robot" in indicator_text:
- kind = "captcha"
- elif detected:
- kind = "human_verification"
- else:
- kind = "none"
- return {
- "detected": detected,
- "kind": kind,
- "source": "mcp_evaluate" if detected else "",
- "indicators": anti_bot_indicators[:8],
- "requires_human_verification": detected,
- "visual_location": "",
- "gates_submit_controls": bool(detected and gated_controls),
- "gated_submit_controls": gated_controls[:5] if detected else [],
- }
-
-
-def _evaluate_anti_bot_indicators(observed_data: dict[str, Any], text: str) -> list[str]:
- key_text = " ".join(str(key) for key in observed_data.keys()).lower()
- combined = f"{text} {key_text}"
- return [pattern for pattern in _DISCOVERY_ANTI_BOT_PATTERNS if pattern in combined]
-
-
-def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:
- observed_data = _evaluate_observed_payload(observed_data)
- if not observed_data:
- return {}
- text_parts = [
- observed_data.get("body"),
- observed_data.get("bodyText"),
- observed_data.get("text"),
- observed_data.get("html"),
- ]
- text = " ".join(part for part in text_parts if isinstance(part, str)).lower()[:4096]
- anti_bot = _evaluate_anti_bot_indicators(observed_data, text)
- forms = _normalize_observed_forms(observed_data)
- result_containers = _normalize_observed_result_containers(observed_data)
- if not forms and not result_containers and not anti_bot:
- return {}
- return {
- "forms": forms,
- "result_containers": result_containers,
- "anti_bot_indicators": anti_bot,
- "challenge_state": _normalize_evaluate_challenge_state(observed_data, anti_bot),
- "evidence_sources": ["mcp_evaluate"],
- "evidence_confidence": (
- _EVALUATE_EVIDENCE_CONFIDENCE_WITH_SCHEMA
- if forms or result_containers
- else _EVALUATE_EVIDENCE_CONFIDENCE_ANTIBOT_ONLY
- ),
- }
-
-
-def _record_composition_page_observation(
- ctx: AgentContext,
- *,
- source_tool: str,
- url: str,
- title: str = "",
- observed_data: object | None = None,
- append_to_flow: bool = False,
- reached_via: str = "current_page",
-) -> int | None:
- if not url:
- return None
- _mark_post_run_page_observed(ctx, source_tool=source_tool, url=url)
- if title:
- _workflow_verification_evidence(ctx).page_title = title[:160]
- existing = ctx.composition_page_evidence
-
- evidence: dict[str, Any] = {
- "inspected_url": url,
- "current_url": url,
- "page_title": title[:240],
- "forms": [],
- "navigation_targets": [],
- "result_containers": [],
- "anti_bot_indicators": [],
- "evidence_confidence": 0.0,
- "source_tool": source_tool,
- "observed_after_workflow_run": False,
- }
- if isinstance(observed_data, dict):
- observed_title = observed_data.get("title")
- if isinstance(observed_title, str) and observed_title and not evidence["page_title"]:
- evidence["page_title"] = observed_title[:240]
- if source_tool == "get_browser_screenshot":
- evidence.update(
- {
- "evidence_sources": ["screenshot"],
- "screenshot_used": True,
- }
- )
- elif source_tool == "evaluate":
- evidence.update(_normalize_evaluate_page_schema(observed_data))
- run_id = ctx.last_run_blocks_workflow_run_id
- if isinstance(run_id, str) and run_id:
- evidence["workflow_run_id"] = run_id
- evidence["observed_after_workflow_run"] = True
-
- observation_step: int | None = None
- if append_to_flow and has_bounded_page_schema(evidence):
- actual_reached_via = reached_via
- if reached_via == "auto":
- if isinstance(run_id, str) and run_id:
- actual_reached_via = "post_run"
- elif _consume_pending_browser_interaction_observation(ctx, current_url=url, evidence=evidence):
- actual_reached_via = "interaction"
- else:
- actual_reached_via = "current_page"
- observation_step = _append_flow_evidence(ctx, evidence, reached_via=actual_reached_via)
-
- if not _should_keep_existing_composition_page_evidence(existing, evidence):
- ctx.composition_page_evidence = evidence
- return observation_step
-
-
-def _composition_evidence_page_url(evidence: dict[str, Any] | None) -> str | None:
- if not isinstance(evidence, dict):
- return None
- for key in ("current_url", "inspected_url"):
- value = evidence.get(key)
- if isinstance(value, str) and value.strip() and value != "current_page":
- return value.strip()
- return None
-
-
-def _should_keep_existing_composition_page_evidence(
- existing: dict[str, Any] | None,
- incoming: dict[str, Any],
-) -> bool:
- if not isinstance(existing, dict) or existing.get("source_tool") != "inspect_page_for_composition":
- return False
- if incoming.get("source_tool") != "evaluate":
- return True
- if not has_bounded_page_schema(incoming):
- return True
- if not has_bounded_page_schema(existing):
- return False
- existing_url = _composition_evidence_page_url(existing)
- incoming_url = _composition_evidence_page_url(incoming)
- if existing_url and incoming_url and not _same_page_ignoring_fragment(existing_url, incoming_url):
- return False
- return True
-
-
-def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:
- policy = getattr(copilot_ctx, "request_policy", None)
- # A method-mandated criterion asserts HOW the goal was reached; the outcome
- # judge sees only end-state evidence.
- return [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]
-
-
-def _verified_context_block_labels_for_snapshot(
- copilot_ctx: Any, current_labels: list[str], executed_labels: list[str]
-) -> list[str]:
- current_label_set = set(current_labels)
- executed_label_set = set(executed_labels)
- candidates: set[str] = set()
- for raw_values in (
- getattr(copilot_ctx, "verified_prefix_labels", None),
- getattr(getattr(copilot_ctx, "workflow_verification_evidence", None), "block_verified", None),
- ):
- if not isinstance(raw_values, list):
- continue
- candidates.update(str(label) for label in raw_values if isinstance(label, str) and label in current_label_set)
- return [label for label in current_labels if label in candidates and label not in executed_label_set]
-
-
-def _build_page_observation_evidence_snapshot(
- copilot_ctx: Any,
- *,
- url: str,
- title: str = "",
- observed_data: object | None = None,
-) -> RunEvidenceSnapshot:
- run_id = getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)
- block_outputs: dict[str, Any] = {}
- if isinstance(observed_data, dict) and observed_data:
- block_outputs["current_page_observation"] = observed_data
- elif observed_data is not None:
- block_outputs["current_page_observation"] = str(observed_data)
- return RunEvidenceSnapshot(
- workflow_run_id=run_id if isinstance(run_id, str) else None,
- block_outputs=block_outputs,
- current_url=_valid_runtime_anchor_url(url),
- page_title=title if isinstance(title, str) and title.strip() else None,
- )
-
-
-async def _maybe_run_completion_verification_from_page_observation(
- copilot_ctx: Any,
- *,
- url: str,
- title: str = "",
- observed_data: object | None = None,
-) -> CompletionVerificationResult | None:
- """Verify completion only for post-run page observations after failed tests."""
-
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return None
- existing = getattr(copilot_ctx, "completion_verification_result", None)
- if isinstance(existing, CompletionVerificationResult) and existing.is_fully_satisfied():
- return existing
- if getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False) is not True:
- return None
- criteria = _completion_verification_criteria(copilot_ctx)
- if not criteria:
- return None
- handler = await _completion_verification_handler(copilot_ctx)
- if handler is None:
- return None
- remaining = _copilot_seconds_remaining(copilot_ctx)
- if (
- remaining is not None
- and remaining
- <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
- ):
- verification = CompletionVerificationResult(status="unavailable")
- else:
- snapshot = _build_page_observation_evidence_snapshot(
- copilot_ctx,
- url=url,
- title=title,
- observed_data=observed_data,
- )
- if not snapshot.has_evidence():
- verification = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=[criterion.id for criterion in criteria],
- verdicts=[
- CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")
- for criterion in criteria
- ],
- )
- else:
- verification = await evaluate_completion_criteria(criteria, snapshot, handler)
-
- if (
- isinstance(existing, CompletionVerificationResult)
- and not verification.is_fully_satisfied()
- and not (existing.status == "unavailable" and verification.status == "evaluated")
- ):
- return existing
-
- copilot_ctx.completion_verification_result = verification
- record_completion_verification(copilot_ctx, verification)
- if verification.status == "evaluated":
- _emit_completion_verification_trace(copilot_ctx, verification)
- return verification
-
-
-# Block types the copilot must never emit. They delegate the entire goal to
-# a separate agent, which bypasses copilot-level block decomposition and
-# obfuscates issues the copilot should surface/handle directly.
-_COPILOT_BANNED_BLOCK_TYPES: frozenset[str] = frozenset({"task", "task_v2"})
-# Code-only browser mode uses this temporary hard ban to force iteration on
-# code-first Copilot behavior. The GA policy is not settled: it may allow some
-# of these block types again, or make the allow/ban list configurable.
-_COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES: frozenset[str] = _COPILOT_BANNED_BLOCK_TYPES | frozenset(
- {
- "action",
- "browser_task",
- "extraction",
- "file_download",
- "file_upload",
- "goto_url",
- "login",
- "navigation",
- "print_page",
- "validation",
- }
-)
-
-# Shared suffix across every LLM-facing rejection message for banned
-# block emission — the pre-hook (schema-lookup reject) and the post-
-# emission detector both steer the LLM toward the same alternatives.
-_COPILOT_BANNED_BLOCK_ALTERNATIVES = (
- "Use `navigation` for page actions (filling forms, clicking, multi-step flows), "
- "`extraction` for data extraction, `validation` for completion checks, "
- "`login` for authentication, or `goto_url` for pure URL navigation."
-)
-_COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES = (
- "Browser/page workflow block types are unavailable in code-only browser mode. Use focused `code` "
- "blocks for durable page or browser-session work."
-)
-_CODE_ONLY_TARGET_EVIDENCE_KEYS = frozenset(
- {
- "buttons",
- "fields",
- "forms",
- "inputs",
- "links",
- "options",
- "result",
- "results",
- "rows",
- "selects",
- "tables",
- "textareas",
- "url",
- }
-)
-_CODE_ONLY_SELECTOR_ACTION_TOOLS = frozenset({"click", "type_text", "select_option", "press_key"})
-
-
-def _copilot_block_authoring_policy(ctx: AgentContext | None) -> BlockAuthoringPolicy:
- if ctx is None:
- return BlockAuthoringPolicy.STANDARD
- return normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None))
-
-
-def _copilot_banned_block_types(ctx: AgentContext | None) -> frozenset[str]:
- if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return _COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES
- return _COPILOT_BANNED_BLOCK_TYPES
-
-
-def _copilot_banned_block_alternatives(ctx: AgentContext | None) -> str:
- if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES
- return _COPILOT_BANNED_BLOCK_ALTERNATIVES
-
-
-def _banned_block_reject_message(items: list[tuple[str, str]], ctx: AgentContext | None = None) -> str:
- """Uniform error text for the post-emission reject, sharing the
- alternatives suffix with the schema pre-hook."""
- labels = ", ".join(sorted({label for label, _ in items}))
- types = sorted({block_type for _, block_type in items})
- types_part = " / ".join(repr(t) for t in types)
- return (
- f"Block type {types_part} is not available in the workflow copilot. "
- f"Offending labels: [{labels}]. "
- f"{_copilot_banned_block_alternatives(ctx)}"
- )
-
-
-def _record_banned_block_reject_span(source_tool: str, items: list[tuple[str, str]]) -> None:
- """Emit the dedicated ``update_workflow_banned_block_reject`` span used
- by post-rollout logfire trend queries."""
- with copilot_span(
- "update_workflow_banned_block_reject",
- data={
- "labels": [label for label, _ in items],
- "block_types": sorted({block_type for _, block_type in items}),
- "source_tool": source_tool,
- },
- ):
- pass
-
-
-def _proxy_location_trace_value(proxy_location: Any) -> Any:
- if proxy_location is None:
- return None
- if hasattr(proxy_location, "value"):
- return proxy_location.value
- if hasattr(proxy_location, "model_dump"):
- return proxy_location.model_dump(mode="json")
- return proxy_location
-
-
-def _raw_yaml_proxy_location(workflow_yaml: str) -> tuple[bool, Any]:
- try:
- parsed_yaml = safe_load_no_dates(workflow_yaml)
- except yaml.YAMLError:
- return False, None
-
- if not isinstance(parsed_yaml, dict) or "proxy_location" not in parsed_yaml:
- return False, None
- return True, _proxy_location_trace_value(parsed_yaml.get("proxy_location"))
-
-
-def _record_workflow_proxy_location_span(workflow_yaml: str, workflow: Workflow) -> None:
- input_present, input_proxy_location = _raw_yaml_proxy_location(workflow_yaml)
- effective_proxy_location = _proxy_location_trace_value(workflow.proxy_location)
- with copilot_span(
- "workflow_proxy_location_normalized",
- data={
- "input_proxy_location_present": input_present,
- "input_proxy_location": input_proxy_location,
- "effective_proxy_location": effective_proxy_location,
- },
- ):
- pass
-
-
-def _collect_banned_block_items(
- blocks: list[Any],
- banned_types: frozenset[str] | None = None,
-) -> list[tuple[str, str]]:
- """Recursively walk ``blocks`` (mirroring
- :func:`skyvern.forge.sdk.copilot.block_goal_wrapping._wrap_blocks_in_place`)
- and return ``(label, normalized_block_type)`` for every block whose type is
- in :data:`_COPILOT_BANNED_BLOCK_TYPES`. Blocks missing ``label`` are
- skipped — the downstream Pydantic validator surfaces those errors on its
- own."""
- active_banned_types = banned_types or _COPILOT_BANNED_BLOCK_TYPES
- items: list[tuple[str, str]] = []
- for block in blocks:
- if not isinstance(block, dict):
- continue
- raw_type = block.get("block_type")
- if isinstance(raw_type, str):
- normalized = raw_type.strip().lower()
- raw_normalized = normalize_copilot_block_type_alias(normalized)
- if normalized in active_banned_types or raw_normalized in active_banned_types:
- label = block.get("label")
- if isinstance(label, str):
- items.append((label, raw_normalized))
- loop_blocks = block.get("loop_blocks")
- if isinstance(loop_blocks, list):
- items.extend(_collect_banned_block_items(loop_blocks, active_banned_types))
- return items
-
-
-def _parse_workflow_blocks(yaml_str: str | None) -> list[Any] | None:
- """Parse ``yaml_str`` and return ``workflow_definition.blocks`` as a list,
- or ``None`` if the YAML is missing, unparseable, or not in the expected
- shape. Graceful on every failure so callers can treat ``None`` as 'nothing
- to compare against.'"""
- if not yaml_str:
- return None
- try:
- parsed = safe_load_no_dates(yaml_str)
- except yaml.YAMLError:
- return None
- if not isinstance(parsed, dict):
- return None
- definition = parsed.get("workflow_definition")
- if not isinstance(definition, dict):
- return None
- blocks = definition.get("blocks")
- return blocks if isinstance(blocks, list) else None
-
-
-def _block_label_from_yaml(block: dict[str, Any]) -> str | None:
- label = block.get("label")
- return label if isinstance(label, str) and label else None
-
-
-def _detect_new_banned_blocks(
- submitted_yaml: str,
- prior_workflow_yaml: str | None,
- *,
- banned_types: frozenset[str] | None = None,
-) -> list[tuple[str, str]]:
- """Return ``[(label, block_type), ...]`` for every banned-type block in
- ``submitted_yaml`` whose label is NOT present as a banned-type block in
- ``prior_workflow_yaml``. Pure: no I/O, no logging.
-
- Recurses into ``for_loop.loop_blocks`` mirroring
- :func:`skyvern.forge.sdk.copilot.block_goal_wrapping._wrap_blocks_in_place`.
- Legacy workflows that carry ``task`` / ``task_v2`` blocks under unchanged
- labels produce an empty list and therefore do not reject.
-
- Malformed YAML, missing ``workflow_definition``, or a non-list ``blocks``
- all produce an empty list — the downstream Pydantic validation in
- ``_process_workflow_yaml`` surfaces the specific parse / shape error on
- its own path.
- """
- submitted_blocks = _parse_workflow_blocks(submitted_yaml)
- if submitted_blocks is None:
- return []
- active_banned_types = banned_types or _COPILOT_BANNED_BLOCK_TYPES
- submitted_items = _collect_banned_block_items(submitted_blocks, active_banned_types)
- if not submitted_items:
- return []
- prior_blocks = _parse_workflow_blocks(prior_workflow_yaml)
- prior_labels = {label for label, _ in _collect_banned_block_items(prior_blocks or [], active_banned_types)}
- return [(label, block_type) for label, block_type in submitted_items if label not in prior_labels]
-
-
-_CHALLENGE_WAIT_PATTERN = re.compile(
- r"\b(anti[-_\s]?bot|bot[-_\s]?block|captcha|challenge|human[-_\s]?verification|ip[-_\s]?block|waf)\b",
- re.IGNORECASE,
-)
-
-
-def _has_confirmed_waf_or_site_block(ctx: Any) -> bool:
- if getattr(ctx, "last_test_anti_bot", None):
- return True
- return _get_int_attr(ctx, "probable_site_block_streak_count") >= PROBABLE_SITE_BLOCK_STREAK_STOP_AT
-
-
-def _get_int_attr(ctx: Any, name: str, default: int = 0) -> int:
- value = getattr(ctx, name, default)
- return value if isinstance(value, int) else default
-
-
-def _block_challenge_wait_text(block: dict[str, Any]) -> str:
- values = []
- for key in ("label", "title", "description", "navigation_goal", "complete_criterion"):
- value = block.get(key)
- if isinstance(value, str):
- values.append(value)
- return " ".join(values)
-
-
-def _detect_timing_only_challenge_wait_blocks(submitted_yaml: str | None) -> list[str]:
- submitted_blocks = _parse_workflow_blocks(submitted_yaml)
- if submitted_blocks is None:
- return []
- labels: list[str] = []
- for block in _iter_yaml_blocks(submitted_blocks):
- raw_type = block.get("block_type")
- if not isinstance(raw_type, str) or raw_type.strip().lower() != "wait":
- continue
- label = block.get("label")
- if not isinstance(label, str):
- continue
- if _CHALLENGE_WAIT_PATTERN.search(_block_challenge_wait_text(block)):
- labels.append(label)
- return labels
-
-
-def _composition_evidence_has_challenge(ctx: AgentContext) -> bool:
- evidence = getattr(ctx, "composition_page_evidence", None)
- if not isinstance(evidence, dict):
- return False
- if evidence.get("anti_bot_indicators") or evidence.get("challenge_controls"):
- return True
- challenge_state = evidence.get("challenge_state")
- return isinstance(challenge_state, dict) and challenge_state.get("detected") is True
-
-
-def _detect_new_http_request_blocks(submitted_yaml: str | None, prior_workflow_yaml: str | None) -> list[str]:
- submitted_blocks = _parse_workflow_blocks(submitted_yaml)
- if submitted_blocks is None:
- return []
- prior_blocks = _parse_workflow_blocks(prior_workflow_yaml)
- prior_labels: set[str] = set()
- for block in _iter_yaml_blocks(prior_blocks or []):
- if str(block.get("block_type") or "").strip().lower() != "http_request":
- continue
- label = block.get("label")
- if isinstance(label, str):
- prior_labels.add(label)
- labels: list[str] = []
- for block in _iter_yaml_blocks(submitted_blocks):
- if str(block.get("block_type") or "").strip().lower() != "http_request":
- continue
- label = block.get("label")
- if isinstance(label, str) and label not in prior_labels:
- labels.append(label)
- return labels
-
-
-def _challenge_http_request_reject_message(
- ctx: AgentContext, submitted_yaml: str | None, prior_workflow_yaml: str | None
-) -> str | None:
- if not _composition_evidence_has_challenge(ctx):
- return None
- labels = _detect_new_http_request_blocks(submitted_yaml, prior_workflow_yaml)
- if not labels:
- return None
- labels_text = ", ".join(sorted(set(labels)))
- return (
- "Workflow validation failed: raw http_request blocks are not allowed for a page with observed "
- "anti-bot or human-verification challenge evidence. "
- f"Offending labels: [{labels_text}]. "
- "Use browser workflow blocks grounded in the observed page, include challenge handling only when visible, "
- "or stop and report the observed challenge blocker if it cannot be completed."
- )
-
-
-def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | None) -> str | None:
- if not _has_confirmed_waf_or_site_block(ctx):
- return None
- labels = _detect_timing_only_challenge_wait_blocks(submitted_yaml)
- if not labels:
- return None
- labels_text = ", ".join(sorted(set(labels)))
- return (
- "Workflow validation failed: timing-only challenge wait blocks are not allowed after confirmed "
- "anti-bot/WAF or repeated site-block evidence. "
- f"Offending labels: [{labels_text}]. "
- "Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "
- "real action, try a materially different proxy/source if allowed, or stop and explain the blocker."
- )
-
-
-async def _get_block_schema_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- """Short-circuit requests for banned block types with an explicit error.
- Without this pre-hook the underlying MCP tool silently redirects ``task``
- and ``task_v2`` queries to ``navigation``'s schema, which makes the LLM
- think the banned types are available."""
- block_type = params.get("block_type")
- if not isinstance(block_type, str):
- return None
- normalized = normalize_copilot_block_type_alias(block_type)
- if normalized != block_type.strip().lower():
- params["block_type"] = normalized
- if normalized not in _copilot_banned_block_types(ctx):
- return None
- return {
- "ok": False,
- "error": f"Block type {block_type!r} is not available in the workflow copilot. {_copilot_banned_block_alternatives(ctx)}",
- }
-
-
-async def _validate_block_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- return {
- "ok": False,
- "error": (
- "CODE-ONLY BLOCK VALIDATION DISABLED: do not use validate_block, dummy code blocks, or probe code "
- "blocks in code-only browser mode. Use MCP browser tools to explore the page, then call "
- "update_and_run_blocks with real focused code blocks that implement the workflow behavior."
- ),
- }
-
-
-async def _get_block_schema_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- """Scrub banned block types from list-mode responses. Belt-and-suspenders
- against future drift in ``BLOCK_SUMMARIES`` (which currently omits them)."""
- data = result.get("data")
- if isinstance(data, dict):
- block_types = data.get("block_types")
- if isinstance(block_types, dict):
- for banned in _copilot_banned_block_types(ctx):
- block_types.pop(banned, None)
- block_type = data.get("block_type")
- if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and block_type == "code":
- ctx.code_only_code_schema_seen = True
- data["code_only_note"] = _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES
- data["code_only_guidance"] = [
- "Use one focused code block per durable browser goal, such as open, search, submit, expand, or extract.",
- "Do not persist navigation/action/login/extraction/validation blocks for browser page work.",
- "Use concrete selectors and text anchors found during exploration. If only intent targeting is available, inspect the page again before mutating.",
- "Call update_and_run_blocks with a connected runnable set of real code blocks instead of validating dummy or probe blocks.",
- "Keep block outputs JSON-safe and include visible evidence text when extracting records, products, totals, confirmations, or identifiers.",
- ]
- return result
-
-
-def _code_only_pre_run_results_error(ctx: CopilotContext) -> dict[str, Any] | None:
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
- if ctx.workflow_persisted or ctx.update_workflow_called:
- return None
- for value in (
- ctx.pending_reconciliation_run_id,
- ctx.last_run_blocks_workflow_run_id,
- ctx.last_successful_run_blocks_workflow_run_id,
- ):
- if isinstance(value, str) and value:
- return None
- return {
- "ok": False,
- "error": (
- "CODE-ONLY EXPLORATION PHASE: get_run_results is unavailable before a real workflow run exists. "
- "Use MCP browser tools such as navigate_browser, evaluate, click, type_text, get_browser_screenshot, "
- "console_messages, scroll, select_option, or press_key to understand the page, then call "
- "update_and_run_blocks with real focused code blocks."
- ),
- }
-
-
-async def _evaluate_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- expr = params.get("expression", "").lower()
- if ".click()" in expr or ".click(" in expr:
- return {
- "ok": False,
- "error": "Do not use evaluate to click elements. Use the 'click' tool with a CSS selector instead.",
- }
- return None
-
-
-def _code_only_deterministic_targeting_error(tool_name: str) -> str:
- return (
- f"In code-only browser mode, {tool_name} requires a CSS/XPath selector for page mutations "
- "after the reusable workflow has been verified. Use evaluate, screenshots, or page inspection "
- "to derive a selector, then retry with selector only."
- )
-
-
-def _code_only_selector_action_requires_deterministic_target(ctx: AgentContext) -> bool:
- if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return False
- if not getattr(ctx, "workflow_persisted", False):
- return False
- return bool(getattr(ctx, "last_full_workflow_test_ok", False))
-
-
-def _strip_intent_for_code_only_selector_action(
- params: dict[str, Any],
- ctx: AgentContext,
- *,
- tool_name: str,
-) -> dict[str, Any] | None:
- if not _code_only_selector_action_requires_deterministic_target(ctx):
- return None
- if tool_name not in _CODE_ONLY_SELECTOR_ACTION_TOOLS:
- return None
- selector = params.get("selector")
- if isinstance(selector, str) and selector.strip():
- if "intent" in params:
- params["intent"] = None
- return None
- if params.get("intent"):
- return {"ok": False, "error": _code_only_deterministic_targeting_error(tool_name)}
- return None
-
-
-def _code_only_has_target_page_evidence(data: object) -> bool:
- if not isinstance(data, dict):
- return False
- for key, value in data.items():
- normalized = str(key).strip().lower()
- if normalized in _CODE_ONLY_TARGET_EVIDENCE_KEYS and bool(value):
- return True
- if isinstance(value, dict) and _code_only_has_target_page_evidence(value):
- return True
- if isinstance(value, list) and any(_code_only_has_target_page_evidence(item) for item in value):
- return True
- return False
-
-
-_JQUERY_SELECTOR_RE = re.compile(r":(?:contains|eq|first|last|gt|lt|nth|visible|hidden|checked)\s*\(", re.IGNORECASE)
-
-
-async def _click_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- await _capture_scout_source_url(ctx)
- deterministic_result = _strip_intent_for_code_only_selector_action(params, ctx, tool_name="click")
- if deterministic_result is not None:
- return deterministic_result
- selector = params.get("selector", "")
- if not selector:
- return None
- if _JQUERY_SELECTOR_RE.search(selector):
- return {
- "ok": False,
- "error": (
- f"Invalid selector: {selector!r}. "
- "jQuery pseudo-selectors like :contains(), :eq(), :first, :visible are NOT valid CSS. "
- "Use standard CSS selectors instead. Examples: "
- "nth-of-type() instead of :eq(), "
- "[data-attr] or tag.class for filtering, "
- "or use the 'evaluate' tool with JS: "
- "document.querySelectorAll('button').forEach("
- "b => {{ if (b.textContent.includes('Download')) b.click() }})"
- ),
- }
- return None
-
-
-async def _type_text_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- await _capture_scout_source_url(ctx)
- return _strip_intent_for_code_only_selector_action(params, ctx, tool_name="type_text")
-
-
-async def _select_option_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- await _capture_scout_source_url(ctx)
- return _strip_intent_for_code_only_selector_action(params, ctx, tool_name="select_option")
-
-
-async def _press_key_pre_hook(
- params: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any] | None:
- await _capture_scout_source_url(ctx)
- return _strip_intent_for_code_only_selector_action(params, ctx, tool_name="press_key")
-
-
-async def _navigate_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- _clear_pending_browser_interaction_observation(ctx)
- if result.get("ok"):
- data = result.pop("data", {})
- result["url"] = data.get("url", "")
- result["next_step"] = (
- "Page loaded. You MUST now use evaluate, "
- "get_browser_screenshot, or click to inspect page content "
- "before responding."
- )
- if (
- _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
- and isinstance(ctx, CopilotContext)
- and ctx.build_phase in {BuildPhase.INITIAL, BuildPhase.DISCOVERING}
- ):
- advance_to_composing(ctx, reason="code_only_browser_navigation_succeeded")
- return result
-
-
-async def _screenshot_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- if result.get("ok") and result.get("data"):
- _mark_page_inspected(ctx)
- data = result["data"]
- url, title = await _resolve_url_title(raw, ctx)
- _record_composition_page_observation(ctx, source_tool="get_browser_screenshot", url=url, title=title)
- result["data"] = {
- "screenshot_base64": data.get("data", ""),
- "url": url,
- "title": title,
- }
- return result
-
-
-async def _click_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- _clear_pending_browser_interaction_observation(ctx)
- source_url = _consume_scout_source_url(ctx)
- if result.get("ok") and result.get("data"):
- data = result["data"]
- url, title = await _resolve_url_title(raw, ctx)
- _mark_pending_browser_interaction_observation(ctx, tool_name="click", url=url)
- result["data"] = {
- "selector": data.get("selector", ""),
- "url": url,
- "title": title,
- }
- navigated = bool(source_url) and bool(url) and source_url != url
- role, accessible_name = await _resolve_scout_role_name(
- ctx, data.get("selector", ""), allow_browser_read=not navigated
- )
- _record_scouted_interaction(
- ctx,
- tool_name="click",
- selector=data.get("selector", ""),
- source_url=source_url,
- role=role,
- accessible_name=accessible_name,
- )
- observation_step = _register_scout_interaction_observation(
- ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url, url=url
- )
- if observation_step is not None:
- result["observation_step"] = observation_step
- result["data"]["observation_step"] = observation_step
- return result
-
-
-_TYPE_READBACK_SETTLE_SECONDS = 0.3
-
-
-async def _verify_scout_type_landed(
- ctx: AgentContext,
- *,
- selector: str,
- typed_length: Any,
-) -> dict[str, Any] | None:
- """Confirm a non-empty type actually entered the field, else return a failure.
-
- A marketing/cookie overlay can consume the focus or keystrokes — the field
- stays empty while `skyvern_type` still reports success (the first interaction
- on an overlaid page often just dismisses the overlay). Read the field back; a
- field still empty after a non-empty type means the input did not land. Only
- fires when there is a selector to read and a positive typed length, so it never
- second-guesses intent-only types or masked/formatted values, which keep a
- non-empty value.
- """
- if not isinstance(selector, str) or not selector.strip():
- return None
- if not isinstance(typed_length, int) or typed_length <= 0:
- return None
- server = getattr(ctx, "discovery_mcp_server", None)
- if server is None:
- return None
-
- async def _read_back() -> Any:
- try:
- readback = await asyncio.wait_for(
- server.call_internal_tool("skyvern_get_value", {"selector": selector}),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except Exception:
- LOG.debug("scout type-landed read-back failed; leaving the type result unverified", exc_info=True)
- return None
- if not isinstance(readback, dict) or not readback.get("ok"):
- return None
- return (readback.get("data") or {}).get("value")
-
- value = await _read_back()
- if isinstance(value, str) and value.strip() == "":
- # A controlled/React input can mirror its value asynchronously, so a first read may be
- # transiently empty; settle briefly and re-read once before declaring the type lost.
- await asyncio.sleep(_TYPE_READBACK_SETTLE_SECONDS)
- value = await _read_back()
- if isinstance(value, str) and value.strip() == "":
- return {
- "ok": False,
- "error": (
- "type_text reported success but the field is still empty — an overlay "
- "(cookie/marketing popup) likely consumed the keystrokes or focus. "
- "Re-inspect the current page and retry typing into the target field; "
- "the overlay is usually dismissed by that first interaction."
- ),
- }
- return None
-
-
-async def _type_text_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- _clear_pending_browser_interaction_observation(ctx)
- source_url = _consume_scout_source_url(ctx)
- if result.get("ok") and result.get("data"):
- data = result["data"]
- selector = data.get("selector", "")
- typed_length = data.get("text_length", 0)
- url, _ = await _resolve_url_title(raw, ctx)
- result["data"] = {
- "selector": selector,
- "typed_length": typed_length,
- "url": url,
- }
- landing_failure = await _verify_scout_type_landed(ctx, selector=selector, typed_length=typed_length)
- if landing_failure is not None:
- return landing_failure
- _mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)
- role, accessible_name = await _resolve_scout_role_name(ctx, selector)
- _record_scouted_interaction(
- ctx,
- tool_name="type_text",
- selector=selector,
- source_url=source_url,
- typed_length=typed_length,
- role=role,
- accessible_name=accessible_name,
- )
- observation_step = _register_scout_interaction_observation(
- ctx, tool_name="type_text", selector=selector, source_url=source_url, url=url
- )
- if observation_step is not None:
- result["observation_step"] = observation_step
- result["data"]["observation_step"] = observation_step
- return result
-
-
-async def _evaluate_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- if result.get("ok") and result.get("data"):
- _mark_page_inspected(ctx)
- result["data"].pop("sdk_equivalent", None)
- if "url" not in result["data"]:
- url, _ = await _resolve_url_title(raw, ctx)
- if url:
- result["data"]["url"] = url
- url = str(result["data"].get("url") or "")
- title = str(result["data"].get("title") or "")
- if not title:
- _, title = await _resolve_url_title(raw, ctx)
- observation_step = _record_composition_page_observation(
- ctx,
- source_tool="evaluate",
- url=url,
- title=title,
- observed_data=result["data"],
- append_to_flow=True,
- reached_via="auto",
- )
- if observation_step is not None:
- result["observation_step"] = observation_step
- result["data"]["observation_step"] = observation_step
- if _copilot_block_authoring_policy(
- ctx
- ) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(result["data"]):
- ctx.code_only_target_page_evidence_seen = True
- await _maybe_run_completion_verification_from_page_observation(
- ctx,
- url=url,
- title=title,
- observed_data=result["data"],
- )
- return result
-
-
-async def _scroll_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- if result.get("ok") and result.get("data"):
- data = result["data"]
- url, _ = await _resolve_url_title(raw, ctx)
- result["data"] = {
- "direction": data.get("direction", ""),
- "amount": data.get("pixels") or data.get("amount"),
- "url": url,
- }
- return result
-
-
-async def _select_option_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- _clear_pending_browser_interaction_observation(ctx)
- source_url = _consume_scout_source_url(ctx)
- if result.get("ok") and result.get("data"):
- data = result["data"]
- url, _ = await _resolve_url_title(raw, ctx)
- _mark_pending_browser_interaction_observation(ctx, tool_name="select_option", url=url)
- result["data"] = {
- "selector": data.get("selector", ""),
- "value": data.get("value", ""),
- "url": url,
- }
- role, accessible_name = await _resolve_scout_role_name(ctx, data.get("selector", ""))
- _record_scouted_interaction(
- ctx,
- tool_name="select_option",
- selector=data.get("selector", ""),
- source_url=source_url,
- value=data.get("value", ""),
- role=role,
- accessible_name=accessible_name,
- )
- observation_step = _register_scout_interaction_observation(
- ctx, tool_name="select_option", selector=data.get("selector", ""), source_url=source_url, url=url
- )
- if observation_step is not None:
- result["observation_step"] = observation_step
- result["data"]["observation_step"] = observation_step
- return result
-
-
-async def _press_key_post_hook(
- result: dict[str, Any],
- raw: dict[str, Any],
- ctx: AgentContext,
-) -> dict[str, Any]:
- _clear_pending_browser_interaction_observation(ctx)
- source_url = _consume_scout_source_url(ctx)
- if result.get("ok") and result.get("data"):
- data = result["data"]
- url, _ = await _resolve_url_title(raw, ctx)
- _mark_pending_browser_interaction_observation(ctx, tool_name="press_key", url=url)
- result["data"] = {
- "key": data.get("key", ""),
- "selector": data.get("selector", ""),
- "url": url,
- }
- _record_scouted_interaction(
- ctx,
- tool_name="press_key",
- selector=data.get("selector", ""),
- source_url=source_url,
- key=data.get("key", ""),
- )
- return result
-
-
-def get_skyvern_mcp_alias_map() -> dict[str, str]:
- return {
- "get_block_schema": "skyvern_block_schema",
- "validate_block": "skyvern_block_validate",
- "navigate_browser": "skyvern_navigate",
- "get_browser_screenshot": "skyvern_screenshot",
- "evaluate": "skyvern_evaluate",
- "click": "skyvern_click",
- "type_text": "skyvern_type",
- "scroll": "skyvern_scroll",
- "console_messages": "skyvern_console_messages",
- "select_option": "skyvern_select_option",
- "press_key": "skyvern_press_key",
- }
-
-
-def _build_skyvern_mcp_overlays() -> dict[str, SchemaOverlay]:
- return {
- "get_block_schema": SchemaOverlay(
- pre_hook=_get_block_schema_pre_hook,
- post_hook=_get_block_schema_post_hook,
- ),
- "validate_block": SchemaOverlay(pre_hook=_validate_block_pre_hook),
- "navigate_browser": SchemaOverlay(
- description=(
- "Navigate the debug browser to a URL. "
- "Use this to reset browser state or navigate to a starting page before running blocks."
- ),
- hide_params=frozenset({"session_id", "cdp_url"}),
- requires_browser=True,
- post_hook=_navigate_post_hook,
- ),
- "get_browser_screenshot": SchemaOverlay(
- description=(
- "Take a screenshot of the current debug browser session. "
- "Returns a base64-encoded PNG image. "
- "Use this to see what the browser looks like after running blocks."
- ),
- hide_params=frozenset({"session_id", "cdp_url", "selector"}),
- forced_args={"inline": True},
- requires_browser=True,
- post_hook=_screenshot_post_hook,
- ),
- "evaluate": SchemaOverlay(
- description=(
- "Execute JavaScript in the browser and return the result. "
- "Use this to inspect DOM state, read values, or run arbitrary JS."
- ),
- hide_params=frozenset({"session_id", "cdp_url"}),
- requires_browser=True,
- timeout=30,
- pre_hook=_evaluate_pre_hook,
- post_hook=_evaluate_post_hook,
- ),
- "click": SchemaOverlay(
- description=(
- "Click an element in the browser. Prefer a CSS selector ALONE for a target "
- "you can identify from page evidence — a selector-only click is instant and "
- "deterministic. Use `intent` only when you cannot derive a selector: an "
- "`intent`-only click routes through a slower full-page AI scan, and if you "
- "pass both, the selector wins and the `intent` is ignored. When a shared class "
- "matches many elements (e.g. one button per result row), scope the selector to "
- "the specific item (its container, a unique attribute, or :nth-of-type) instead "
- "of relying on `intent` to disambiguate. "
- "IMPORTANT: jQuery pseudo-selectors like :contains(), :eq(), :first, "
- ":visible are NOT valid CSS. Use standard selectors: "
- "'button.download', 'a[href*=\"pdf\"]', '#submit-btn', "
- "'table tr:nth-of-type(2) td a'."
- ),
- hide_params=frozenset({"session_id", "cdp_url", "button", "click_count"}),
- forced_args={"selector_mode": "direct"},
- requires_browser=True,
- timeout=15,
- pre_hook=_click_pre_hook,
- post_hook=_click_post_hook,
- ),
- "type_text": SchemaOverlay(
- description=(
- "Type text into an input element. Prefer a CSS selector ALONE to target the "
- "field — a selector-only type is instant and deterministic. Use `intent` only "
- "when you cannot derive a selector: an `intent`-only type routes through a "
- "slower full-page AI scan, and if you pass both, the selector wins and the "
- "`intent` is ignored. "
- "Optionally clear the field first. Use this for form filling. "
- "NEVER type inline passwords, API keys, tokens, cookies, TOTP/OTP "
- "codes, private keys, or other raw credentials/secrets received in "
- "chat — stop and follow the CREDENTIAL HANDLING refusal rule in the "
- "system prompt instead."
- ),
- hide_params=frozenset({"session_id", "cdp_url", "delay"}),
- forced_args={"selector_mode": "direct"},
- required_overrides=["text"],
- arg_transforms={"clear_first": "clear"},
- requires_browser=True,
- timeout=15,
- pre_hook=_type_text_pre_hook,
- post_hook=_type_text_post_hook,
- ),
- "scroll": SchemaOverlay(
- description=(
- "Scroll the page in a direction (up/down/left/right) by pixel amount, "
- "or scroll a specific element into view using intent or selector. "
- "Use this to reveal content below the fold."
- ),
- hide_params=frozenset({"session_id", "cdp_url"}),
- requires_browser=True,
- post_hook=_scroll_post_hook,
- ),
- "console_messages": SchemaOverlay(
- description=(
- "Read console log messages from the browser. "
- "Use level='error' to find JavaScript errors. "
- "This is a read-only diagnostic tool."
- ),
- hide_params=frozenset({"session_id", "cdp_url"}),
- requires_browser=True,
- ),
- "select_option": SchemaOverlay(
- description=(
- "Select an option from a <select> dropdown. Provide the value to select and a "
- "selector to target the element precisely; use `intent` (alone) only when you "
- "cannot derive a selector — passing both lets the selector win and ignores the "
- "`intent`. For free-text inputs, use type_text instead."
- ),
- hide_params=frozenset({"session_id", "cdp_url", "timeout"}),
- forced_args={"selector_mode": "direct"},
- required_overrides=["value"],
- requires_browser=True,
- timeout=15,
- pre_hook=_select_option_pre_hook,
- post_hook=_select_option_post_hook,
- ),
- "press_key": SchemaOverlay(
- description=(
- "Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.). "
- "Optionally focus an element first via selector or intent. "
- "Use for form submission, tab navigation, or closing dialogs."
- ),
- hide_params=frozenset({"session_id", "cdp_url"}),
- required_overrides=["key"],
- requires_browser=True,
- pre_hook=_press_key_pre_hook,
- post_hook=_press_key_post_hook,
- ),
- }
-
-
-def _record_workflow_update_result(
- copilot_ctx: Any, result: dict[str, Any], prior_definition: object | None = None
-) -> None:
- if not (result.get("ok") and "_workflow" in result):
- return
-
- wf = result["_workflow"]
- copilot_ctx.last_workflow = wf
- _clear_resolved_per_tool_budget_problem_labels(copilot_ctx, wf)
- copilot_ctx.last_workflow_yaml = copilot_ctx.workflow_yaml or None
- copilot_ctx.effective_workflow_proxy_location = getattr(wf, "proxy_location", None) or ProxyLocation.RESIDENTIAL
- data = result.get("data")
- if isinstance(data, dict):
- block_count = data.get("block_count")
- if isinstance(block_count, int):
- copilot_ctx.last_update_block_count = block_count
- copilot_ctx.last_test_ok = None
- copilot_ctx.last_test_failure_reason = None
- # A fresh workflow edit invalidates the prior test's failure state —
- # otherwise an exhausted POST_UPDATE_NUDGE on the new draft would raise
- # CopilotNonRetriableNavError with the old run's error, telling the user
- # to "verify the URL" for a URL they just corrected in the new draft.
- copilot_ctx.last_test_non_retriable_nav_error = None
- copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
-
- # Block-running failures keyed off (labels, parameters) go stale once the
- # workflow itself changes — without this clear, a user who fixes the bug
- # via update_workflow gets a LOOP DETECTED on the next legitimate run.
- clear_failed_step_tracker_for_tools_in_ctx(copilot_ctx, BLOCK_RUNNING_TOOLS)
-
- _invalidate_verified_state_on_edit(copilot_ctx, prior_definition, getattr(wf, "workflow_definition", None))
-
-
-def _last_update_is_single_goto_bootstrap(copilot_ctx: CopilotContext) -> bool:
- last_workflow = copilot_ctx.last_workflow
- definition = _workflow_definition_as_dict(last_workflow.workflow_definition if last_workflow is not None else None)
- blocks = definition.get("blocks")
- if not isinstance(blocks, list) or len(blocks) != 1:
- return False
- block = blocks[0]
- if not isinstance(block, dict):
- return False
- return str(block.get("block_type") or "").strip().lower() == "goto_url"
-
-
-def _pre_run_workflow_coverage_error(copilot_ctx: Any) -> str | None:
- block_count = getattr(copilot_ctx, "last_update_block_count", None)
- if not isinstance(block_count, int):
- return None
- if block_count == 1 and _last_update_is_single_goto_bootstrap(copilot_ctx):
- return None
-
- user_message = getattr(copilot_ctx, "user_message", "")
- request_policy = getattr(copilot_ctx, "request_policy", None)
- completion_contract = getattr(request_policy, "completion_contract", None)
- if isinstance(completion_contract, str):
- completion_contract = completion_contract.strip() or None
- else:
- completion_contract = None
-
- if not _goal_likely_needs_more_blocks(user_message, block_count, completion_contract):
- return None
-
- nudge_count = getattr(copilot_ctx, "coverage_nudge_count", 0)
- if nudge_count >= 1:
- return None
- copilot_ctx.coverage_nudge_count = nudge_count + 1
- return (
- f"{POST_INTERMEDIATE_SUCCESS_NUDGE} The workflow was saved with {block_count} block"
- f"{'' if block_count == 1 else 's'}, but it has not been run because the request-policy "
- "completion contract still leaves distinct requested actions uncovered."
- )
-
-
-def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[dict] | None]:
- """Single-pass analysis of run result blocks.
-
- Returns ``(anti_bot_match, has_empty_data_blocks, failure_categories)``
- by iterating the block list once. Classification delegates to
- :func:`~skyvern.forge.failure_classifier.classify_from_failure_reason`.
- When ``data["failure_categories"]`` is already populated (pre-run
- short-circuit with no blocks), honor it instead of re-classifying.
- """
- data = result.get("data")
- if not isinstance(data, dict):
- return None, False, None
-
- anti_bot_match: str | None = None
-
- precomputed_categories = data.get("failure_categories")
- if isinstance(precomputed_categories, list) and precomputed_categories:
- for cat in precomputed_categories:
- if isinstance(cat, dict) and cat.get("category") == "ANTI_BOT_DETECTION":
- anti_bot_match = cat.get("reasoning", "anti-bot pattern detected")
- break
- return anti_bot_match, False, precomputed_categories
-
- # Collect texts for scanning and data-block stats in one pass
- texts_to_scan: list[str] = []
- error = result.get("error")
- if isinstance(error, str):
- texts_to_scan.append(error)
- html = data.get("visible_elements_html")
- if isinstance(html, str):
- texts_to_scan.append(html)
- failure_reason = data.get("failure_reason")
- if isinstance(failure_reason, str):
- texts_to_scan.append(failure_reason)
- page_title = data.get("page_title")
- if isinstance(page_title, str):
- texts_to_scan.append(page_title)
- action_trace_summary = data.get("action_trace_summary")
- if isinstance(action_trace_summary, list):
- texts_to_scan.extend(str(item) for item in action_trace_summary if isinstance(item, str))
-
- has_data_blocks = False
- any_data_output = False
-
- blocks = data.get("blocks")
- if isinstance(blocks, list):
- for block in blocks:
- if not isinstance(block, dict):
- continue
- reason = block.get("failure_reason")
- if isinstance(reason, str):
- texts_to_scan.append(reason)
- block_type = block.get("block_type")
- if block_type in _DATA_PRODUCING_BLOCK_TYPES and block.get("status") == "completed":
- has_data_blocks = True
- payload = _block_data_payload(block.get("extracted_data"), block_type)
- structured_blocker = _structured_blocker_message(payload)
- if structured_blocker:
- texts_to_scan.append(structured_blocker)
- if _is_meaningful_extracted_data(payload):
- any_data_output = True
-
- combined = "\n".join(texts_to_scan)
- categories = classify_from_failure_reason(combined)
- if categories:
- for cat in categories:
- if cat.get("category") == "ANTI_BOT_DETECTION":
- anti_bot_match = cat.get("reasoning", "anti-bot pattern detected")
- break
-
- empty_data_blocks = has_data_blocks and not any_data_output
- return anti_bot_match, empty_data_blocks, categories
-
-
-def _composition_anti_bot_reason(copilot_ctx: object) -> str | None:
- evidence = getattr(copilot_ctx, "composition_page_evidence", None)
- if not isinstance(evidence, dict):
- return None
- indicators = evidence.get("anti_bot_indicators")
- challenge_controls = evidence.get("challenge_controls")
- challenge_state = evidence.get("challenge_state")
- normalized_indicators = (
- [str(item) for item in indicators if isinstance(item, str)] if isinstance(indicators, list) else []
- )
- control_count = len(challenge_controls) if isinstance(challenge_controls, list) else 0
- challenge_detected = isinstance(challenge_state, dict) and challenge_state.get("detected") is True
- if not normalized_indicators and control_count == 0 and not challenge_detected:
- return None
- detail_parts = normalized_indicators[:4]
- if isinstance(challenge_state, dict):
- state_indicators = challenge_state.get("indicators")
- if isinstance(state_indicators, list):
- detail_parts.extend(str(item) for item in state_indicators if isinstance(item, str))
- challenge_kind = challenge_state.get("kind")
- if isinstance(challenge_kind, str) and challenge_kind and challenge_kind != "none":
- detail_parts.append(challenge_kind)
- gated_controls = challenge_state.get("gated_submit_controls")
- if challenge_state.get("gates_submit_controls") is True:
- gated_control_items = gated_controls if isinstance(gated_controls, list) else []
- control_labels = [
- str(item.get("text") or item.get("value") or item.get("id") or item.get("name") or item.get("selector"))
- for item in gated_control_items
- if isinstance(item, dict)
- and (
- item.get("text") or item.get("value") or item.get("id") or item.get("name") or item.get("selector")
- )
- ]
- labels = ", ".join(list(dict.fromkeys(control_labels))[:3]) or "submit/search control"
- detail_parts.append(f"challenge-gated disabled submit/search control: {labels}")
- if control_count:
- detail_parts.append(f"{control_count} challenge control(s)")
- details = ", ".join(list(dict.fromkeys(detail_parts))[:6])
- return f"Observed anti-bot challenge evidence before the run: {details}"
-
-
-# Generic failure-reason template emitted by the shared agent when the
-# browser-side scraper catches ScrapingFailed / NoElementFound. Matching on
-# the template (not the shared classifier) lets the copilot notice a repeated
-# site-block/unreadable-page pattern even though the classifier routes it to
-# DATA_EXTRACTION_FAILURE, not ANTI_BOT_DETECTION.
-# Coupling note: these substrings come from the run-level failure_reason
-# produced when the shared scraper raises ScrapingFailed. If the template
-# wording changes, update this tuple and the test that locks it in
-# (tests/unit/test_copilot_probable_site_block.py).
-_PROBABLE_SITE_BLOCK_FAILURE_REASON_SUBSTRINGS = (
- "failed to load the website",
- "page may have navigated unexpectedly",
-)
-
-
-def _detect_probable_site_block_wall(result: dict[str, Any]) -> bool:
- """True when a block failed with the site-load template and the failure is
- not a non-retriable nav error (DNS / SSL / invalid URL are owned by
- :func:`_detect_non_retriable_nav_error`)."""
- if bool(result.get("ok", False)):
- return False
- if _detect_non_retriable_nav_error(result):
- return False
- data = result.get("data")
- if not isinstance(data, dict):
- return False
- blocks = data.get("blocks")
- if not isinstance(blocks, list):
- return False
-
- for block in blocks:
- if not isinstance(block, dict):
- continue
- reason = block.get("failure_reason")
- if isinstance(reason, str):
- lowered = reason.lower()
- if any(sub in lowered for sub in _PROBABLE_SITE_BLOCK_FAILURE_REASON_SUBSTRINGS):
- return True
- return False
-
-
-def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:
- """Return the first failure_reason that matches SKIP_INNER_NAV_RETRY_ERRORS
- (DNS / cert / SSL / invalid URL), preferring run-level over block-level.
- Same set is_skip_inner_retry_error uses at the browser layer, so the copilot
- classifies on exactly the patterns that already short-circuit retries in
- navigate_with_retry (skyvern/webeye/navigation.py)."""
- return next((reason for reason in iter_failure_reasons(result) if is_skip_inner_retry_error(reason)), None)
-
-
-def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, result: Mapping[str, object]) -> None:
- evidence = _workflow_verification_evidence(copilot_ctx)
- data_value = result.get("data")
- data: Mapping[str, object] = data_value if isinstance(data_value, dict) else {}
- run_ok = bool(result.get("ok", False))
- full_workflow_verified = run_ok and copilot_ctx.last_full_workflow_test_ok is True
- evidence.full_workflow_verified = full_workflow_verified
- evidence.test_attempted_but_incomplete = not full_workflow_verified
-
- run_id = data.get("workflow_run_id")
- if isinstance(run_id, str) and run_id.strip():
- evidence.workflow_run_id = run_id.strip()
- if _active_run_terminal_evidence_detected(result):
- evidence.active_run_terminal_evidence_detected = True
- evidence.live_page_state_verified = True
- evidence.verified_from_current_browser_state = True
- evidence.full_workflow_verified = False
- evidence.test_attempted_but_incomplete = True
- if isinstance(run_id, str) and run_id.strip():
- evidence.active_run_terminal_evidence_workflow_run_id = run_id.strip()
- sample_index = data.get("active_run_terminal_evidence_sample_index")
- if isinstance(sample_index, int):
- evidence.active_run_terminal_evidence_sample_index = sample_index
- current_url = _valid_runtime_anchor_url(data.get("current_url"))
- if current_url is not None:
- evidence.current_url = current_url
- evidence.live_page_state_verified = True
- if data.get("observed_after_workflow_run") is True:
- evidence.current_url_observed_after_workflow_run = True
- evidence.current_url_may_encode_runtime_state = bool(urlparse(current_url).query)
- page_title = data.get("page_title")
- if isinstance(page_title, str) and page_title.strip():
- evidence.page_title = " ".join(page_title.split())[:160]
-
- if run_ok:
- evidence.merge_verified_blocks(_completed_run_block_labels(data))
- unverified = list(copilot_ctx.last_unverified_block_labels or [])
- evidence.unverified_block_labels = list(dict.fromkeys(str(label) for label in unverified if str(label)))
- evidence.failed_block_labels = []
- # A completed-but-suspicious run (outcome unverified, null data, blocker)
- # keeps its failure reason so the evidence stays consistent with
- # test_attempted_but_incomplete instead of reading as a clean success.
- suspicious_reason = copilot_ctx.last_test_failure_reason if copilot_ctx.last_test_suspicious_success else None
- evidence.failure_reason = (
- " ".join(suspicious_reason.split())[:240]
- if isinstance(suspicious_reason, str) and suspicious_reason.strip()
- else None
- )
- if evidence.unverified_block_labels:
- evidence.verified_from_current_browser_state = True
- return
-
- failed_labels = _failed_run_block_labels(data)
- evidence.failed_block_labels = failed_labels
- if copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY:
- evidence.merge_per_tool_budget_blocks(failed_labels)
- failure_reason = copilot_ctx.last_test_failure_reason or result.get("error")
- if isinstance(failure_reason, str) and failure_reason.strip():
- evidence.failure_reason = " ".join(failure_reason.split())[:240]
-
-
-_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0
-
-
-async def _completion_verification_handler(copilot_ctx: Any) -> Any | None:
- return await resolve_main_copilot_handler(
- getattr(copilot_ctx, "workflow_permanent_id", None),
- getattr(copilot_ctx, "organization_id", None),
- )
-
-
-async def _active_run_terminal_monitor_enabled(copilot_ctx: Any) -> bool:
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return False
- if not getattr(copilot_ctx, "browser_session_id", None):
- return False
- if not getattr(copilot_ctx, "discovery_mcp_server", None):
- return False
- if not _completion_verification_criteria(copilot_ctx):
- return False
- return await _completion_verification_handler(copilot_ctx) is not None
-
-
-def _active_run_terminal_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:
- if page_evidence_needs_visual_fallback(evidence):
- return True
- return evidence.get("screenshot_used") is not True
-
-
-async def _active_run_terminal_evidence_sample(
- copilot_ctx: Any,
- *,
- workflow_run_id: str,
- labels_to_execute: list[str],
- sample_index: int,
-) -> ActiveRunTerminalEvidenceSample | None:
- criteria = _completion_verification_criteria(copilot_ctx)
- if not criteria:
- return None
- handler = await _completion_verification_handler(copilot_ctx)
- if handler is None:
- return None
-
- current_url_raw, page_title_raw = await _fallback_page_info(copilot_ctx)
- current_url = _valid_runtime_anchor_url(current_url_raw)
- if current_url is None:
- return None
-
- evidence, html_error = await _capture_composition_evidence(
- copilot_ctx,
- inspected_url=current_url,
- current_url=current_url,
- active_run_terminal_sample=True,
- )
- if html_error is not None or evidence is None:
- LOG.info(
- "copilot active-run terminal evidence sample skipped",
- workflow_run_id=workflow_run_id,
- sample_index=sample_index,
- html_error=html_error,
- )
- return None
-
- page_title = evidence.get("page_title")
- if not isinstance(page_title, str) or not page_title.strip():
- page_title = page_title_raw if isinstance(page_title_raw, str) and page_title_raw.strip() else None
- evidence = {
- **evidence,
- "workflow_run_id": workflow_run_id,
- "observed_during_active_workflow_run": True,
- }
- snapshot = RunEvidenceSnapshot(
- workflow_run_id=workflow_run_id,
- current_url=current_url,
- page_title=page_title,
- executed_block_labels=list(labels_to_execute),
- page_evidence=evidence,
- )
- if not snapshot.has_evidence():
- return None
-
- result = await evaluate_completion_criteria(criteria, snapshot, handler)
- LOG.info(
- "copilot active-run terminal evidence sample",
- workflow_run_id=workflow_run_id,
- sample_index=sample_index,
- completion_verification_status=result.status,
- completion_verification_fully_satisfied=result.is_fully_satisfied(),
- )
- if result.status != "evaluated" or not result.is_fully_satisfied():
- return None
- copilot_ctx.composition_page_evidence = evidence
- return ActiveRunTerminalEvidenceSample(
- current_url=current_url,
- page_title=page_title,
- page_evidence=evidence,
- completion_verification=result,
- sample_index=sample_index,
- )
-
-
-def _active_run_terminal_evidence_result(
- *,
- workflow_run_id: str,
- run_status: str | None,
- sample: Any,
- requested_block_labels: list[str],
- executed_block_labels: list[str],
- current_url: str | None = None,
- page_title: str | None = None,
-) -> dict[str, Any]:
- observed_url = current_url or getattr(sample, "current_url", None)
- observed_title = page_title or getattr(sample, "page_title", None)
- completion = getattr(sample, "completion_verification", None)
- completion_trace = completion.to_trace_data() if isinstance(completion, CompletionVerificationResult) else {}
- reason = (
- "The active run reached the requested browser state while the workflow was still running, "
- "so Copilot interrupted it before further browser actions could overshoot that state. "
- "The current page evidence is not a durable full-workflow verification; inspect the run boundary, "
- "repair the workflow if needed, and verify the corrected workflow run."
- )
- return {
- "ok": False,
- "error": reason,
- "data": {
- "workflow_run_id": workflow_run_id,
- "overall_status": run_status,
- "failure_reason": reason,
- "requested_block_labels": list(requested_block_labels),
- "executed_block_labels": list(executed_block_labels),
- "current_url": observed_url,
- "page_title": observed_title,
- "active_run_terminal_evidence_detected": True,
- "active_run_terminal_evidence_sample_index": getattr(sample, "sample_index", None),
- "full_workflow_verified": False,
- "current_page_evidence": getattr(sample, "page_evidence", None),
- "active_run_terminal_completion_verification": completion_trace,
- "failure_categories": [
- {
- "category": ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
- "confidence_float": 1.0,
- "reasoning": (
- "Bounded current-page evidence matched the request-policy completion criteria "
- "while the workflow run was still active; the run was interrupted for diagnosis/repair."
- ),
- }
- ],
- },
- }
-
-
-def _is_outcome_evidence_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
- """A clean ok=True run worth judging on its whole-workflow outcome.
-
- Recognition is governed by the outcome evidence the user can observe, not by
- whether every block was verified as an end-to-end prefix (SKY-10576). The judge
- requires positive evidence for every criterion, and ``empty_data_blocks`` rejects
- a run whose outcome block produced nothing, so an incomplete run never passes --
- this only lets an already-reached goal be recognized without a redundant
- full-prefix re-run.
- """
- if not bool(result.get("ok", False)):
- return False
- if _run_blocks_structured_blocker_message(result):
- return False
- _anti_bot, empty_data_blocks, _categories = _analyze_run_blocks(result)
- return not empty_data_blocks
-
-
-def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
- """A canceled/partial run (ok=False) still worth judging because it left runtime
- evidence behind. The judge confirms a criterion only on positive evidence, so a
- broken run never spuriously passes; this only lets a reached goal be recognized
- even though the run did not finish cleanly — recognition must not key on run status.
- """
- if bool(result.get("ok", False)):
- return False
- if _active_run_terminal_evidence_detected(result):
- return False
- if _run_blocks_structured_blocker_message(result):
- return False
- data = result.get("data")
- if not isinstance(data, dict):
- return False
- return _valid_runtime_anchor_url(data.get("current_url")) is not None
-
-
-def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
- data = result.get("data")
- data = data if isinstance(data, dict) else {}
- current_label_order = _current_workflow_block_labels(copilot_ctx)
- current_labels = set(current_label_order)
- # Evidence must be what THIS run produced. ``verified_block_outputs`` accumulates
- # across incremental runs, so sourcing from it would let an output from a prior
- # run satisfy a criterion the current run never re-produced.
- blocks = data.get("blocks")
- block_outputs: dict[str, Any] = {}
- if isinstance(blocks, list):
- for block in blocks:
- if not isinstance(block, dict):
- continue
- label = block.get("label")
- output = block.get("extracted_data")
- if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):
- block_outputs[label] = output
- executed = data.get("executed_block_labels")
- executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []
- page_title = data.get("page_title")
- run_id = data.get("workflow_run_id")
- return RunEvidenceSnapshot(
- workflow_run_id=run_id if isinstance(run_id, str) else None,
- block_outputs=block_outputs,
- current_url=_valid_runtime_anchor_url(data.get("current_url")),
- page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
- executed_block_labels=executed_block_labels,
- verified_context_block_labels=_verified_context_block_labels_for_snapshot(
- copilot_ctx,
- current_label_order,
- executed_block_labels,
- ),
- )
-
-
-async def _maybe_run_completion_verification(
- copilot_ctx: Any, result: dict[str, Any], handler_start: float
-) -> CompletionVerificationResult | None:
- if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
- return None
- if getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):
- return None
- criteria = _completion_verification_criteria(copilot_ctx)
- if not criteria:
- return None
- if not (
- _is_outcome_evidence_candidate(copilot_ctx, result)
- or _is_unfinished_run_verification_candidate(copilot_ctx, result)
- ):
- return None
- # A missing judge handler is an infra/config state, not a transient failure:
- # fall back to the prior gate rather than fail closed on every run.
- handler = await _completion_verification_handler(copilot_ctx)
- if handler is None:
- return None
- # Too little budget to verify a candidate run: fail closed (unavailable) rather
- # than let the run-status proxy claim an unverified outcome as success.
- remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)
- if remaining <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:
- return CompletionVerificationResult(status="unavailable")
- snapshot = _build_run_evidence_snapshot(copilot_ctx, result)
- if not snapshot.has_evidence():
- return CompletionVerificationResult(
- status="evaluated",
- criterion_ids=[criterion.id for criterion in criteria],
- verdicts=[
- CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")
- for criterion in criteria
- ],
- )
- return await evaluate_completion_criteria(criteria, snapshot, handler)
-
-
-def _outcome_unverified_reason(
- copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
-) -> str | None:
- if completion_verification is None:
- return None
- if completion_verification.status == "evaluated":
- if completion_verification.is_fully_satisfied():
- return None
- policy = getattr(copilot_ctx, "request_policy", None)
- criteria = list(policy.completion_criteria) if policy is not None else []
- unmet = summarize_unsatisfied_outcomes(completion_verification, criteria)
- detail = f": {unmet}" if unmet else ""
- return (
- f"The run completed but did not demonstrate the goal outcome(s){detail}. "
- "Add an end-state confirmation (an extraction or validation block) that observes the outcome, then re-run."
- )
- # An 'unavailable' result reaches here only when verification was required;
- # fail closed and ask for a re-run rather than claim an unverified success.
- return (
- "The run completed but the goal outcome could not be verified (verification was unavailable). "
- "Re-run to verify the outcome before reporting success."
- )
-
-
-def _outcome_failure_warrants_repair(
- copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
-) -> bool:
- """Whether an unmet outcome should route to suspicious-success/repair rather
- than continue-building.
-
- Contradicting evidence is always a real failure. Absence of evidence is a
- failure only once the workflow has an outcome-producing block; while the agent
- is still adding blocks toward the goal, an unmet criterion is expected, and the
- run should keep building. Terminal success stays withheld in both cases via the
- verification result, so this only governs the repair directive, not the gate.
- """
- if completion_verification is None:
- return False
- if any(verdict.reason_code == "evidence_contradicts" for verdict in completion_verification.verdicts):
- return True
- return _current_workflow_has_evidence_block(copilot_ctx)
-
-
-def _tool_visible_result_after_completion_verification(
- copilot_ctx: Any,
- result: dict[str, Any],
- completion_verification: CompletionVerificationResult | None,
-) -> dict[str, Any]:
- outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
- if outcome_unverified_reason is None:
- return result
- if not _outcome_failure_warrants_repair(copilot_ctx, completion_verification):
- return result
-
- data = result.get("data")
- copied_data = dict(data) if isinstance(data, dict) else {}
- copied_data["failure_reason"] = outcome_unverified_reason
- copied_data["completion_verification"] = (
- completion_verification.to_trace_data() if completion_verification is not None else None
- )
- categories = copied_data.get("failure_categories")
- copied_categories = list(categories) if isinstance(categories, list) else []
- copied_categories.insert(
- 0,
- {
- "category": "OUTCOME_UNVERIFIED",
- "confidence_float": 1.0,
- "reasoning": outcome_unverified_reason,
- },
- )
- copied_data["failure_categories"] = copied_categories
- return {
- **result,
- "ok": False,
- "error": outcome_unverified_reason,
- "data": copied_data,
- }
-
-
-def _emit_completion_verification_trace(
- copilot_ctx: Any, completion_verification: CompletionVerificationResult
-) -> None:
- block_count = getattr(copilot_ctx, "last_update_block_count", None)
- policy = getattr(copilot_ctx, "request_policy", None)
- contract = policy.completion_contract if policy is not None else None
- heuristic_would_block = isinstance(block_count, int) and _goal_likely_needs_more_blocks(
- getattr(copilot_ctx, "user_message", ""), block_count, contract
- )
- trace_data = {
- **completion_verification.to_trace_data(),
- "heuristic_would_block": heuristic_would_block,
- "evidence_block_present": _current_workflow_has_evidence_block(copilot_ctx),
- "warrants_repair": _outcome_failure_warrants_repair(copilot_ctx, completion_verification),
- }
- LOG.info(
- "copilot completion verification",
- **{f"completion_verification_{key}": value for key, value in trace_data.items()},
- )
- with copilot_span("completion_verification", data=trace_data):
- pass
-
-
-def _record_run_blocks_result(
- copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None
-) -> None:
- run_ok = bool(result.get("ok", False))
- data = result.get("data")
- run_id = data.get("workflow_run_id") if isinstance(data, dict) else None
- copilot_ctx.completion_verification_result = completion_verification
- record_completion_verification(copilot_ctx, completion_verification)
- if completion_verification is not None and completion_verification.status == "evaluated":
- _emit_completion_verification_trace(copilot_ctx, completion_verification)
- copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None
- copilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else None
- # Watchdog cancels normally count as ok=False; only a coincident total
- # timeout softens to ``None`` to keep the unvalidated WIP rescue open.
- cancelled_by_watchdog = result.get(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY) is True
- timeout_latched = bool(copilot_ctx.copilot_total_timeout_exceeded)
- copilot_ctx.last_test_ok = None if (cancelled_by_watchdog and timeout_latched) else run_ok
- copilot_ctx.last_full_workflow_test_ok = False
- copilot_ctx.last_unverified_block_labels = _unverified_current_workflow_labels(copilot_ctx)
- copilot_ctx.last_test_failure_reason = None
- copilot_ctx.last_test_suspicious_success = False
- copilot_ctx.suspicious_success_nudge_count = 0
- copilot_ctx.last_test_anti_bot = None
- prior_budget_flag = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY
- copilot_ctx.last_failure_category_top = None
- copilot_ctx.last_test_non_retriable_nav_error = None
- copilot_ctx.post_run_page_observation_tool = None
- copilot_ctx.post_run_page_observation_url = None
- copilot_ctx.post_run_page_observation_workflow_run_id = None
- copilot_ctx.post_run_page_observation_after_failed_test = False
- copilot_ctx.post_run_current_page_inspection_workflow_run_id = None
-
- structured_blocker = _run_blocks_structured_blocker_message(result)
- anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result)
- if not anti_bot_match:
- anti_bot_match = _composition_anti_bot_reason(copilot_ctx)
- if anti_bot_match:
- copilot_ctx.last_test_anti_bot = anti_bot_match
- if failure_categories:
- top = failure_categories[0]
- if isinstance(top, dict):
- top_category = top.get("category")
- if isinstance(top_category, str):
- copilot_ctx.last_failure_category_top = top_category
-
- if copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY and isinstance(data, dict):
- current_url = _valid_runtime_anchor_url(data.get("current_url"))
- if current_url is not None:
- copilot_ctx.post_budget_page_inspection_required = True
- copilot_ctx.post_budget_page_inspection_url = current_url
- copilot_ctx.post_budget_page_inspection_run_id = run_id if isinstance(run_id, str) else None
-
- # A fresh budget trip on a different chain should get the dedicated split
- # nudge again rather than falling through to the generic failed-test path,
- # so reset the cap when the latest run is not itself a budget trip.
- if prior_budget_flag and copilot_ctx.last_failure_category_top != PER_TOOL_BUDGET_FAILURE_CATEGORY:
- copilot_ctx.per_tool_budget_nudge_count = 0
-
- # Expose full failure classification in tool output for agent reasoning
- if failure_categories:
- data = result.get("data")
- if isinstance(data, dict):
- data["failure_categories"] = failure_categories
-
- if _active_run_terminal_evidence_detected(result):
- _update_verification_evidence_from_run_result(copilot_ctx, result)
- signal = _active_run_terminal_evidence_signal(copilot_ctx, "update_and_run_blocks")
- if signal is not None:
- stash_blocker_signal(copilot_ctx, signal)
-
- if run_ok:
- _mark_page_inspected(copilot_ctx)
- if structured_blocker:
- failure_reason = f"Run completed, but extracted data reported a blocker: {structured_blocker}"
- copilot_ctx.last_test_ok = False
- copilot_ctx.last_test_suspicious_success = True
- copilot_ctx.last_test_failure_reason = failure_reason
- copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
- if not copilot_ctx.last_test_anti_bot and _looks_like_anti_bot_blocker(structured_blocker):
- copilot_ctx.last_test_anti_bot = f"Extracted data reported anti-bot blocker: {structured_blocker[:160]}"
- data = result.get("data")
- if isinstance(data, dict):
- data.setdefault("failure_reason", failure_reason)
- if copilot_ctx.last_test_anti_bot and not failure_categories:
- anti_bot_category = {
- "category": "ANTI_BOT_DETECTION",
- "confidence_float": 0.7,
- "reasoning": "Structured extracted data reported an anti-bot blocker.",
- }
- data["failure_categories"] = [anti_bot_category]
- update_repeated_failure_state(copilot_ctx, result)
- _update_verification_evidence_from_run_result(copilot_ctx, result)
- return
- if empty_data_blocks:
- copilot_ctx.last_test_ok = None
- copilot_ctx.last_test_suspicious_success = True
- copilot_ctx.null_data_streak_count = getattr(copilot_ctx, "null_data_streak_count", 0) + 1
- copilot_ctx.last_test_failure_reason = (
- "All blocks completed but data-producing blocks "
- "(extraction/text_prompt) produced no meaningful output "
- "(missing, empty, or all-null fields). "
- "The workflow may not be working correctly."
- )
- # Clean-ish success (no scrape-fail pattern): reset the streak.
- copilot_ctx.probable_site_block_streak_count = 0
- update_repeated_failure_state(copilot_ctx, result)
- _update_verification_evidence_from_run_result(copilot_ctx, result)
- return
- copilot_ctx.failed_test_nudge_count = 0
- copilot_ctx.null_data_streak_count = 0
- copilot_ctx.probable_site_block_streak_count = 0
- copilot_ctx.last_failed_workflow_yaml = None
- # Real success: clear the signature latch so a subsequent bad URL in
- # the same session can re-fire the stop nudge.
- copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
- unverified = _unverified_current_workflow_labels(copilot_ctx)
- copilot_ctx.last_unverified_block_labels = unverified
- outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
- if outcome_unverified_reason is not None and _outcome_failure_warrants_repair(
- copilot_ctx, completion_verification
- ):
- # The workflow already has a confirmation block, yet the produced
- # evidence does not demonstrate the outcome (or contradicts it). Treat
- # it as a suspicious success so the existing repair/partial machinery
- # fires. A mid-build run with no confirmation block yet falls through to
- # keep-building below; terminal success stays withheld either way via
- # the verification result.
- copilot_ctx.last_test_suspicious_success = True
- copilot_ctx.last_test_failure_reason = outcome_unverified_reason
- if isinstance(data, dict):
- data.setdefault("failure_reason", outcome_unverified_reason)
- elif not unverified:
- copilot_ctx.last_full_workflow_test_ok = True
- copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
- copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
- else:
- copilot_ctx.last_test_failure_reason = (
- "The last run verified only the current browser frontier; unverified workflow blocks remain: "
- + ", ".join(unverified[:8])
- )
- update_repeated_failure_state(copilot_ctx, result)
- _update_verification_evidence_from_run_result(copilot_ctx, result)
- return
-
- copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
- copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)
- if _detect_probable_site_block_wall(result):
- copilot_ctx.probable_site_block_streak_count += 1
- else:
- copilot_ctx.probable_site_block_streak_count = 0
-
- data = result.get("data")
- if isinstance(data, dict):
- blocks = data.get("blocks")
- if isinstance(blocks, list):
- for block in blocks:
- if isinstance(block, dict) and block.get("failure_reason"):
- copilot_ctx.last_test_failure_reason = str(block["failure_reason"])
- break
- if copilot_ctx.last_test_failure_reason is None:
- copilot_ctx.last_test_failure_reason = next(iter_failure_reasons(result), None)
- if result.get("error") and copilot_ctx.last_test_failure_reason is None:
- copilot_ctx.last_test_failure_reason = str(result["error"])
- update_repeated_failure_state(copilot_ctx, result)
- _update_verification_evidence_from_run_result(copilot_ctx, result)
-
-
-def _record_diagnosis_repair_contract(
- copilot_ctx: Any,
- *,
- source_tool: str,
- result: dict[str, Any],
- workflow_updated: bool = False,
-) -> DiagnosisRepairContract:
- contract = build_diagnosis_repair_contract(
- source_tool=source_tool,
- result=result,
- ctx=copilot_ctx,
- workflow_updated=workflow_updated,
- )
- copilot_ctx.latest_diagnosis_repair_contract = contract
- trace_data = contract.to_trace_data()
- LOG.info(
- "copilot diagnosis repair contract shadow",
- **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},
- )
- with copilot_span("diagnosis_repair_contract", data=trace_data):
- pass
- return contract
-
-
-def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:
- result = {"ok": False, "error": error}
- _record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)
- return json.dumps(result)
-
-
-@function_tool(
- name_override="update_workflow",
- tool_input_guardrails=[_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL],
-)
-async def update_workflow_tool(
- ctx: RunContextWrapper,
- workflow_yaml: str,
- block_observation_refs: list[BlockObservationRef] | None = None,
- code_artifact_metadata: list[CodeArtifactMetadata] | None = None,
-) -> str:
- """Validate and update the workflow YAML definition.
- Provide the complete workflow YAML as a string.
- Returns the validated workflow or validation errors.
-
- Top-level workflow parameter keys appear in the run-input UI. When you
- add runtime inputs in `workflow_definition.parameters`, name keys for the
- reusable domain value the user supplies, not the page widget or action used
- to enter it.
-
- Use browser inspection and run evidence to fill knowledge gaps while
- building or editing the workflow. Do not invent URL params, form fields,
- result affordances, or page structure from memory; ground workflow blocks
- in observed MCP evidence or information the user supplied.
- When you compose no-url blocks from a page reached by prior clicks, include
- `block_observation_refs` entries with each block label and the
- `observation_step` returned by inspect_page_for_composition for the page
- that block acts on.
- For authored code blocks, include `code_artifact_metadata` rows describing
- declared goals, claimed outcomes, page dependencies, criteria, evidence
- refs, observation refs, and terminal verifier expectations.
- """
- copilot_ctx = ctx.context
- serialized_code_artifact_metadata = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
- normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
- arguments = {
- "workflow_yaml": workflow_yaml,
- "block_observation_refs": normalized_block_observation_refs,
- "code_artifact_metadata": serialized_code_artifact_metadata,
- }
- loop_error = _tool_loop_error(copilot_ctx, "update_workflow", arguments)
- if loop_error:
- return json.dumps({"ok": False, "error": loop_error})
- credential_deferred_draft = _request_policy_allows_credential_deferred_draft(copilot_ctx)
- # A credential-deferred draft is redirected to update_and_run_blocks' skip-run
- # save path, which saves the draft and skips the browser run with the required
- # credential setup message.
- if credential_deferred_draft:
- agent_steering = (
- "Use update_and_run_blocks for this credential-deferred draft. It will save the workflow draft "
- "and skip the browser run with the required credential setup message."
- )
- user_facing = (
- "I can save this as a draft without running it because the credentials aren't set up yet. "
- "Add them in the Credentials UI and ask me to test the workflow."
- )
- signal = CopilotToolBlockerSignal(
- blocker_kind="authority_denied",
- agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
- recovery_hint="retry_with_different_tool",
- cleared_by_tools=frozenset({"update_and_run_blocks"}),
- internal_reason_code="request_policy_credential_deferred_redirect",
- blocked_tool="update_workflow",
- )
- payload = _emit_tool_blocker_signal(copilot_ctx, signal)
- result = {"ok": False, "error": payload}
- record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
- return json.dumps(result)
-
- with copilot_span(
- "composition_evidence_precheck",
- data={**_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA, "tool_name": "update_workflow"},
- ):
- composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
- copilot_ctx,
- workflow_yaml,
- normalized_block_observation_refs,
- block_observation_refs,
- )
- if composition_evidence_error:
- result = {"ok": False, "error": composition_evidence_error}
- record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_workflow",
- result=result,
- )
- sanitized = sanitize_tool_result_for_llm("update_workflow", result)
- return json.dumps(sanitized)
-
- prior_definition = await _get_prior_workflow_definition(copilot_ctx)
- with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
- result = await _update_workflow(
- {
- **arguments,
- "raw_block_observation_refs": block_observation_refs,
- "raw_code_artifact_metadata": code_artifact_metadata,
- },
- copilot_ctx,
- allow_missing_credentials=getattr(copilot_ctx, "allow_untested_workflow_draft", False) is True,
- )
- _record_workflow_update_result(copilot_ctx, result, prior_definition)
- record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
- sanitized = sanitize_tool_result_for_llm("update_workflow", result)
- return json.dumps(sanitized)
-
-
-@function_tool(name_override="list_credentials")
-async def list_credentials_tool(
- ctx: RunContextWrapper,
- page: int = 1,
- page_size: int = 10,
-) -> str:
- """List stored credentials (metadata only — never passwords or secrets).
- Use this to find credential IDs for login blocks.
-
- Paginated. `page_size` caps at 50. The response includes `has_more`;
- before concluding no credential exists, keep incrementing `page` until
- `has_more` is `false` — otherwise you risk telling the user to create
- a credential they have already stored on a later page.
- """
- copilot_ctx = ctx.context
- arguments = {"page": page, "page_size": page_size}
- loop_error = _tool_loop_error(copilot_ctx, "list_credentials", arguments)
- if loop_error:
- return json.dumps({"ok": False, "error": loop_error})
-
- authority_error = _authority_tool_error(copilot_ctx, "list_credentials")
- if authority_error:
- result = {"ok": False, "error": authority_error}
- record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)
- return json.dumps(result)
-
- result = await _list_credentials(arguments, copilot_ctx)
- record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)
- sanitized = sanitize_tool_result_for_llm("list_credentials", result)
- return json.dumps(sanitized)
-
-
-def _run_blocks_span_data(
- block_labels: list[str],
- labels_to_execute: list[str],
- frontier_start_label: str | None,
- seeded_outputs: dict[str, Any],
- ctx: object,
-) -> dict[str, Any]:
- return {
- "requested_block_labels": block_labels,
- "executed_block_labels": labels_to_execute,
- "frontier_start_label": frontier_start_label,
- "seeded_output_count": len(seeded_outputs or {}),
- "repeated_failure_streak_count": int(getattr(ctx, "repeated_failure_streak_count", 0) or 0),
- "block_count": len(block_labels),
- }
-
-
-def _frontier_run_size_result(
- error: str,
- block_labels: list[str],
- labels_to_execute: list[str],
-) -> dict[str, Any]:
- suggested_labels = list(labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS])
- user_facing_summary = (
- "Workflow draft saved; I still need to test the next smaller browser frontier before continuing."
- )
- return {
- "ok": False,
- "error": error,
- "data": {
- "workflow_run_id": None,
- "overall_status": "skipped",
- "workflow_run_skipped": True,
- "requested_block_labels": list(block_labels),
- "executed_block_labels": [],
- "planned_block_labels": list(labels_to_execute),
- "suggested_block_labels": suggested_labels,
- "deferred_block_labels": list(labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]),
- "control_signal": {
- "kind": "intermediate_success",
- "user_facing_summary": user_facing_summary,
- "next_tool": "run_blocks_and_collect_debug",
- "next_block_labels": suggested_labels,
- "preserve_workflow_yaml": True,
- },
- "user_facing_summary": user_facing_summary,
- },
- }
-
-
-@function_tool(
- name_override="run_blocks_and_collect_debug",
- timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS,
- strict_mode=False,
-)
-async def run_blocks_tool(
- ctx: RunContextWrapper,
- block_labels: list[str],
- parameters: dict[str, Any] | None = None,
-) -> Any:
- """Run one or more blocks of the current workflow, wait for completion,
- and return compact debug output (status, failure reason, visible elements).
- The workflow must be saved before running blocks.
- Block labels must match labels in the saved workflow.
-
- For diagnostic complaints, follow the system prompt's ASK-vs-EDIT routing.
- If the complaint has no prior edit goal, inspect current workflow context
- and existing run evidence before deciding whether a fresh run is needed.
- If prior context establishes a resolvable edit, use `update_and_run_blocks`
- instead of rerunning unchanged blocks.
-
- Pass runtime values for workflow parameters via the `parameters` dict —
- keys must match the workflow parameter `key` field. When the user has
- supplied concrete non-secret values in their message (names, emails, IDs),
- pass them on the first call rather than letting the workflow fall back to
- placeholders. For sensitive values (password, secret, token, api_key,
- credential, totp, otp, one_time_code, private_key, auth) — call
- `list_credentials` and use a credential parameter whose default_value is
- the stored `credential_id`. If no stored credential matches, do NOT pass
- the inline value via `parameters`; stop and follow the CREDENTIAL
- HANDLING refusal rule in the system prompt.
-
- Use browser inspection and run evidence to fill knowledge gaps before
- changing the workflow. If visible state is uncertain, inspect the live
- page and then compose the next normal workflow action from observed
- evidence instead of retrying guessed URL params or page structure.
- """
- copilot_ctx = ctx.context
- copilot_ctx.completion_verification_result = None
- handler_start = time.monotonic()
- arguments = {"block_labels": block_labels, "parameters": parameters or {}}
- authority_error = _authority_tool_error(copilot_ctx, "run_blocks_and_collect_debug")
- if authority_error:
- return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", authority_error)
-
- loop_error = _tool_loop_error(copilot_ctx, "run_blocks_and_collect_debug", arguments)
- if loop_error:
- return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", loop_error)
-
- prior_definition = await _get_prior_workflow_definition(copilot_ctx)
- labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(
- copilot_ctx, block_labels, prior_definition, prior_definition
- )
- frontier_error = _frontier_run_size_error(copilot_ctx, block_labels, labels_to_execute, prior_definition)
- if frontier_error:
- result = _frontier_run_size_result(frontier_error, block_labels, labels_to_execute)
- record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="run_blocks_and_collect_debug",
- result=result,
- )
- return json.dumps(result)
-
- with copilot_span(
- "run_blocks",
- data=_run_blocks_span_data(
- block_labels,
- labels_to_execute,
- frontier_start_label,
- block_outputs_to_seed,
- copilot_ctx,
- ),
- ):
- result = await _run_blocks_and_collect_debug(
- arguments,
- copilot_ctx,
- labels_to_execute=labels_to_execute,
- block_outputs_to_seed=block_outputs_to_seed,
- frontier_start_label=frontier_start_label,
- )
- completion_verification = await _maybe_run_completion_verification(copilot_ctx, result, handler_start)
- _record_run_blocks_result(copilot_ctx, result, completion_verification=completion_verification)
- tool_visible_result = _tool_visible_result_after_completion_verification(
- copilot_ctx,
- result,
- completion_verification,
- )
- record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, tool_visible_result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="run_blocks_and_collect_debug",
- result=tool_visible_result,
- )
- enqueue_screenshot_from_result(copilot_ctx, result)
-
- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)
- return json.dumps(sanitized)
-
-
-@function_tool(name_override="get_run_results")
-async def get_run_results_tool(
- ctx: RunContextWrapper,
- workflow_run_id: str | None = None,
-) -> str:
- """Fetch results from a previous workflow run.
- Returns block statuses, failure reasons, and output data.
- If workflow_run_id is omitted, fetches the most recently created finished
- run (completed, failed, canceled, terminated, or timed_out — excludes
- in-flight runs). For unambiguous results in concurrent-run scenarios,
- pass an explicit workflow_run_id from a prior tool response.
- """
- copilot_ctx = ctx.context
- params: dict[str, Any] = {}
- if workflow_run_id:
- params["workflow_run_id"] = workflow_run_id
- loop_error = _tool_loop_error(copilot_ctx, "get_run_results", params)
- if loop_error:
- return json.dumps({"ok": False, "error": loop_error})
- authority_error = _authority_tool_error(copilot_ctx, "get_run_results")
- if authority_error:
- return json.dumps({"ok": False, "error": authority_error})
- if isinstance(copilot_ctx, CopilotContext):
- code_only_pre_run_error = _code_only_pre_run_results_error(copilot_ctx)
- if code_only_pre_run_error is not None:
- record_tool_step_result_for_ctx(copilot_ctx, "get_run_results", params, code_only_pre_run_error)
- return json.dumps(code_only_pre_run_error)
-
- result = await _get_run_results(params, copilot_ctx)
- _record_per_tool_budget_problem_blocks_from_results(copilot_ctx, result)
- _maybe_clear_reconciliation_flag(copilot_ctx, result)
- record_tool_step_result_for_ctx(copilot_ctx, "get_run_results", params, result)
-
- sanitized = sanitize_tool_result_for_llm("get_run_results", result)
- return json.dumps(sanitized)
-
-
-@function_tool(
- name_override="update_and_run_blocks",
- timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS,
- strict_mode=False,
- tool_input_guardrails=[_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL],
-)
-async def update_and_run_blocks_tool(
- ctx: RunContextWrapper,
- workflow_yaml: str,
- block_labels: list[str],
- block_observation_refs: list[BlockObservationRef] | None = None,
- code_artifact_metadata: list[CodeArtifactMetadata] | None = None,
- parameters: dict[str, Any] | None = None,
-) -> Any:
- """Update the workflow YAML and immediately run the specified blocks in one step.
- Use this instead of calling update_workflow and run_blocks_and_collect_debug separately.
- The workflow must validate successfully before blocks are run.
- `block_labels` may be a tested frontier subset of the full workflow YAML;
- save the complete reusable workflow, then run only the next 1-2 unverified
- blocks when a long form/search/result chain can be verified incrementally.
-
- Top-level workflow parameter keys appear in the run-input UI. When you
- add runtime inputs in `workflow_definition.parameters`, name keys for the
- reusable domain value the user supplies, not the page widget or action used
- to enter it.
-
- For diagnostic complaints, follow the system prompt's ASK-vs-EDIT routing.
- A complaint with no prior edit goal needs context inspection or
- clarification first. A diagnostic follow-up after an explicit edit goal may
- update/run once the correction is clear.
-
- Pass runtime values for workflow parameters via the `parameters` dict —
- keys must match the workflow parameter `key` field. When the user has
- supplied concrete non-secret values in their message (names, emails, IDs),
- pass them on the first call rather than letting the workflow fall back to
- placeholders. For sensitive values (password, secret, token, api_key,
- credential, totp, otp, one_time_code, private_key, auth) — call
- `list_credentials` and use a credential parameter whose default_value is
- the stored `credential_id`. If no stored credential matches, do NOT pass
- the inline value via `parameters`; stop and follow the CREDENTIAL
- HANDLING refusal rule in the system prompt.
-
- Use browser inspection and run evidence to fill knowledge gaps while
- building, editing, or debugging the workflow. Do not invent URL params,
- form fields, result affordances, or page structure from memory; ground
- workflow blocks in observed MCP evidence or information the user supplied.
- Only refine URL params when they are grounded in observed DOM/link/form
- state or observed URL deltas.
- Browser inspection is build-time context; add durable workflow blocks only
- for the reusable actions/checks the workflow actually needs.
- When you compose no-url blocks from a page reached by prior clicks, include
- `block_observation_refs` entries with each block label and the
- `observation_step` returned by inspect_page_for_composition or evaluate for
- the page that block acts on.
- For authored code blocks, include `code_artifact_metadata` rows describing
- declared goals, claimed outcomes, page dependencies, criteria, evidence
- refs, observation refs, and terminal verifier expectations.
- When inspected evidence shows an anti-bot challenge gating a disabled
- submit/search control, account for challenge resolution before submit;
- do not compose a click against a control observed as disabled.
- """
- copilot_ctx = ctx.context
- copilot_ctx.completion_verification_result = None
- handler_start = time.monotonic()
- serialized_code_artifact_metadata = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
- normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
- arguments = {
- "workflow_yaml": workflow_yaml,
- "block_labels": block_labels,
- "block_observation_refs": normalized_block_observation_refs,
- "code_artifact_metadata": serialized_code_artifact_metadata,
- "parameters": parameters or {},
- }
- skip_run_after_update = _request_policy_allows_update_and_skip_run(copilot_ctx, "update_and_run_blocks")
- authority_error = _authority_tool_error(
- copilot_ctx,
- "update_and_run_blocks",
- ignore_request_policy_error=skip_run_after_update,
- )
- if authority_error:
- return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", authority_error)
-
- loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
- if loop_error:
- return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)
-
- with copilot_span("composition_evidence_precheck", data=_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA):
- composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
- copilot_ctx,
- workflow_yaml,
- normalized_block_observation_refs,
- block_observation_refs,
- )
- if composition_evidence_error:
- result = {"ok": False, "error": composition_evidence_error}
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=result,
- )
- sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)
- return json.dumps(sanitized)
-
- _clear_pending_browser_interaction_observation(copilot_ctx)
-
- # Snapshot the prior workflow definition BEFORE _update_workflow saves
- # the new one — we need the pre-update state to diff against.
- prior_definition = await _get_prior_workflow_definition(copilot_ctx)
-
- # Step 1: Update the workflow
- with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
- update_result = await _update_workflow(
- {
- "workflow_yaml": workflow_yaml,
- "block_observation_refs": normalized_block_observation_refs,
- "raw_block_observation_refs": block_observation_refs,
- "code_artifact_metadata": serialized_code_artifact_metadata,
- "raw_code_artifact_metadata": code_artifact_metadata,
- },
- copilot_ctx,
- allow_missing_credentials=skip_run_after_update,
- )
- _record_workflow_update_result(copilot_ctx, update_result, prior_definition)
-
- if not update_result.get("ok"):
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, update_result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=update_result,
- )
- sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)
- return json.dumps(sanitized)
-
- coverage_error = _pre_run_workflow_coverage_error(copilot_ctx)
- if coverage_error:
- user_facing_summary = (
- "Workflow draft saved; I still need to add the remaining requested actions before testing it."
- )
- result = {
- "ok": False,
- "error": coverage_error,
- "data": {
- "block_count": copilot_ctx.last_update_block_count,
- "workflow_updated": True,
- "workflow_run_skipped": True,
- "control_signal": {
- "kind": "intermediate_success",
- "user_facing_summary": user_facing_summary,
- },
- "user_facing_summary": user_facing_summary,
- },
- }
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=result,
- workflow_updated=True,
- )
- return json.dumps(result)
-
- if skip_run_after_update:
- skip_message = "Skipped test run: required credentials are not configured."
- skip_result = {
- "ok": True,
- "message": skip_message,
- "data": {
- "block_count": copilot_ctx.last_update_block_count,
- "workflow_updated": True,
- "skipped_run": True,
- "skip_reason": "workflow_credential_inputs_unbound",
- },
- }
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, skip_result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=skip_result,
- workflow_updated=True,
- )
- LOG.info(
- "update_and_run_blocks skipped run on unbound credential workflow inputs",
- workflow_permanent_id=copilot_ctx.workflow_permanent_id,
- )
- return json.dumps(skip_result)
-
- # Step 2: Compute frontier and run the blocks.
- new_definition = None
- if copilot_ctx.last_workflow is not None:
- new_definition = getattr(copilot_ctx.last_workflow, "workflow_definition", None)
-
- labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(
- copilot_ctx, block_labels, prior_definition, new_definition
- )
- frontier_error = _frontier_run_size_error(copilot_ctx, block_labels, labels_to_execute, new_definition)
- if frontier_error:
- result = _frontier_run_size_result(frontier_error, block_labels, labels_to_execute)
- data = result.get("data")
- if isinstance(data, dict):
- data["workflow_updated"] = True
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=result,
- workflow_updated=True,
- )
- return json.dumps(result)
-
- with copilot_span(
- "run_blocks",
- data=_run_blocks_span_data(
- block_labels,
- labels_to_execute,
- frontier_start_label,
- block_outputs_to_seed,
- copilot_ctx,
- ),
- ):
- run_result = await _run_blocks_and_collect_debug(
- {"block_labels": block_labels, "parameters": parameters or {}},
- copilot_ctx,
- labels_to_execute=labels_to_execute,
- block_outputs_to_seed=block_outputs_to_seed,
- frontier_start_label=frontier_start_label,
- )
- completion_verification = await _maybe_run_completion_verification(copilot_ctx, run_result, handler_start)
- _record_run_blocks_result(copilot_ctx, run_result, completion_verification=completion_verification)
- tool_visible_result = _tool_visible_result_after_completion_verification(
- copilot_ctx,
- run_result,
- completion_verification,
- )
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, tool_visible_result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=tool_visible_result,
- workflow_updated=True,
- )
- enqueue_screenshot_from_result(copilot_ctx, run_result)
- if run_result.get("ok"):
- advance_to_testing(copilot_ctx)
-
- sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)
- return json.dumps(sanitized)
-
-
-# Build-time entrypoint discovery: navigates and reads pages, returns a
-# candidate URL into the agent's context. Never mutates workflow YAML.
-# Available only during INITIAL / DISCOVERING phases.
-
-_DISCOVERY_PER_CHAT_BUDGET = 3
-_DISCOVERY_PER_TURN_BUDGET = 1
-_COMPOSITION_INSPECTION_PER_CHAT_BUDGET = 6
-_COMPOSITION_INSPECTION_PER_TURN_BUDGET = 4
-_DISCOVERY_WALL_CLOCK_SECONDS = 60.0
-_DISCOVERY_STEP_CAP = 8
-_DISCOVERY_EVIDENCE_TRAIL_MAX = 8
-_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX = 10
-_DISCOVERY_HTML_BYTES_MAX = 200_000
-_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE = 0.6
-_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS = 10.0
-_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME = "workflow-copilot-page-evidence-vision"
-
-_DISCOVERY_URL_SCHEME_RE = re.compile(r"^https?://", re.IGNORECASE)
-# host + optional path/query/fragment — handles `example.com/login`,
-# `the-internet.herokuapp.com/tables?x=y`, `site.com?q=x`, `site.com#frag`.
-_DISCOVERY_DOMAIN_WITH_PATH_RE = re.compile(
- r"^[a-z0-9-]+(\.[a-z]{2,})+([/?#][^\s]*)?$",
- re.IGNORECASE,
-)
-_DISCOVERY_BARE_WORD_RE = re.compile(r"^[a-z0-9-]{2,32}$", re.IGNORECASE)
-_DISCOVERY_TOKEN_RE = re.compile(r"[a-z0-9]+", re.IGNORECASE)
-_DISCOVERY_CANDIDATE_EVIDENCE_STOPWORDS = frozenset({"a", "an", "and", "for", "in", "of", "on", "or", "the", "to"})
-_DISCOVERY_LOGIN_TITLE_RE = re.compile(r"\b(sign\s*in|log\s*in|login)\b", re.IGNORECASE)
-_DISCOVERY_PASSWORD_INPUT_RE = re.compile(
- r"<input[^>]*type\s*=\s*[\"']password[\"']",
- re.IGNORECASE,
-)
-_DISCOVERY_ANTI_BOT_PATTERNS = (
- "just a moment",
- "captcha",
- "challenge",
- "turnstile",
- "cf-turnstile",
- "human-verification",
- "human verification",
- "verify you are human",
- "access denied",
- "are you a robot",
-)
-
-
-def _resolve_discovery_entry_url(site_or_url: str) -> tuple[str | None, str]:
- """Resolve the user-supplied site name/URL into a navigable URL.
-
- Returns ``(resolved_url, kind)`` where ``kind`` is one of:
- ``url`` / ``domain`` / ``word`` / ``unresolved``.
- """
- token = (site_or_url or "").strip()
- if not token:
- return None, "unresolved"
- if _DISCOVERY_URL_SCHEME_RE.match(token):
- return token, "url"
- if _DISCOVERY_DOMAIN_WITH_PATH_RE.match(token):
- return f"https://{token}", "domain"
- if _DISCOVERY_BARE_WORD_RE.match(token):
- return f"https://www.{token.lower()}.com", "word"
- return None, "unresolved"
-
-
-def _concrete_homepage_entrypoint(entry_url: str | None, kind: str) -> str | None:
- if kind not in {"domain", "url"} or not entry_url:
- return None
- try:
- parsed = urlparse(entry_url)
- except Exception:
- return None
- if parsed.scheme not in {"http", "https"} or not parsed.netloc:
- return None
- if parsed.path not in {"", "/"} or parsed.params or parsed.query or parsed.fragment:
- return None
- return f"{parsed.scheme}://{parsed.netloc}/"
-
-
-def _discovery_anchor_score(
- anchor_text: str,
- anchor_title: str,
- href_path: str,
- intent_tokens: set[str],
-) -> int:
- """Count intent tokens that appear as substrings of the combined anchor text.
-
- Substring (not exact-token) matching handles ``sort`` ↔ ``sortable`` and
- ``table`` ↔ ``tables`` without a full stemmer.
- """
- if not intent_tokens:
- return 0
- combined = f"{anchor_text} {anchor_title} {href_path}".lower()
- return sum(1 for token in intent_tokens if token in combined)
-
-
-def _discovery_title_score(page_title: str, intent_tokens: set[str]) -> int:
- if not intent_tokens or not page_title:
- return 0
- lowered = page_title.lower()
- return sum(1 for token in intent_tokens if token in lowered)
-
-
-def _discovery_candidate_evidence_tokens(intent_tokens: set[str]) -> set[str]:
- return {
- token
- for token in intent_tokens
- if len(token) > 2 and token.lower() not in _DISCOVERY_CANDIDATE_EVIDENCE_STOPWORDS
- }
-
-
-def _discovery_detect_login_wall(html: str, page_title: str) -> bool:
- if _DISCOVERY_LOGIN_TITLE_RE.search(page_title or ""):
- return True
- return bool(_DISCOVERY_PASSWORD_INPUT_RE.search(html or ""))
-
-
-def _discovery_detect_anti_bot(html: str, page_title: str) -> bool:
- lowered_title = (page_title or "").lower()
- lowered_html = (html or "")[:_DISCOVERY_HTML_BYTES_MAX].lower()
- return any(pat in lowered_title or pat in lowered_html for pat in _DISCOVERY_ANTI_BOT_PATTERNS)
-
-
-def _discovery_build_result(
- *,
- candidate_url: str | None,
- candidate_form_fields: list[dict[str, Any]],
- evidence_trail: list[dict[str, Any]],
- confidence: float,
- failure_reason: str | None,
- ok: bool = True,
- error: str | None = None,
-) -> dict[str, Any]:
- """Shape a `discover_workflow_entrypoint` result envelope.
-
- Convention: ``ok=True`` for any *completed* walk — including controlled
- outcomes that report a ``failure_reason`` and ``candidate_url=None``.
- ``ok=False`` is reserved for actual tool errors (MCP unavailable, browser
- boot failure, internal exception). Matches the existing copilot
- ``_request_policy_tool_error`` convention so the eval harness counts a
- controlled failure as a successful tool call.
- """
- return {
- "ok": ok,
- "data": {
- "candidate_url": candidate_url,
- "candidate_form_fields": candidate_form_fields[:_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX],
- "evidence_trail": evidence_trail[:_DISCOVERY_EVIDENCE_TRAIL_MAX],
- "confidence": float(confidence),
- "failure_reason": failure_reason,
- },
- "error": error,
- }
-
-
-def _record_discovery_resolution_on_ctx(ctx: Any, result: Mapping[str, Any]) -> None:
- data_payload = result.get("data")
- data: Mapping[str, Any] = data_payload if isinstance(data_payload, Mapping) else {}
- candidate_url = data.get("candidate_url")
- failure_reason = data.get("failure_reason")
- if isinstance(candidate_url, str) and candidate_url:
- prior_candidate_url = getattr(ctx, "resolved_discovery_entrypoint_url", None)
- ctx.resolved_discovery_entrypoint_url = candidate_url
- ctx.resolved_discovery_failure_reason = (
- failure_reason if isinstance(failure_reason, str) and failure_reason else None
- )
- if prior_candidate_url != candidate_url:
- ctx.resolved_discovery_entrypoint_inspection_baseline = int(
- getattr(ctx, "page_inspection_calls_this_turn", 0) or 0
- )
- ctx.discovery_entrypoint_url_question_nudge_count = 0
- # Prior successful candidates remain authoritative over later no-candidate failures.
- elif not getattr(ctx, "resolved_discovery_entrypoint_url", None):
- # No prior candidate and no new one: clear URL state while recording the failure.
- ctx.resolved_discovery_entrypoint_url = None
- ctx.resolved_discovery_failure_reason = (
- failure_reason if isinstance(failure_reason, str) and failure_reason else None
- )
- ctx.resolved_discovery_entrypoint_inspection_baseline = 0
- try:
- current_span = otel_trace.get_current_span()
- if ctx.resolved_discovery_entrypoint_url is not None:
- current_span.set_attribute("copilot.discovery_candidate_url", ctx.resolved_discovery_entrypoint_url)
- if ctx.resolved_discovery_failure_reason is not None:
- current_span.set_attribute("copilot.discovery_failure_reason", ctx.resolved_discovery_failure_reason)
- except Exception:
- LOG.debug("Unable to set discovery resolution span attributes", exc_info=True)
-
-
-def _discovery_parse_html(html: str) -> tuple[str, list[dict[str, str]], list[dict[str, str]]]:
- """Parse HTML for page title, link anchors, and form-field metadata.
-
- Uses BeautifulSoup if available (a transitive Skyvern dep). Falls back to
- empty results if not — discovery degrades gracefully rather than crashing.
- """
- if BeautifulSoup is None:
- return "", [], []
-
- try:
- soup = BeautifulSoup(html or "", "html.parser")
- except Exception:
- return "", [], []
-
- title_text = ""
- title_tag = soup.find("title")
- if title_tag and title_tag.string:
- title_text = title_tag.string.strip()
- h1_tag = soup.find("h1")
- if h1_tag:
- h1_text = h1_tag.get_text(strip=True)
- if h1_text:
- title_text = f"{title_text} {h1_text}".strip()
-
- anchors: list[dict[str, str]] = []
- for a in soup.find_all("a", href=True):
- href = a.get("href", "").strip()
- if not href or href.startswith("#") or href.lower().startswith("javascript:"):
- continue
- text = a.get_text(" ", strip=True)
- anchors.append(
- {
- "href": href,
- "text": text[:240],
- "title": (a.get("title") or "")[:240],
- }
- )
-
- form_fields: list[dict[str, str]] = []
- form = soup.find("form")
- if form is not None:
- for inp in form.find_all(["input", "select", "textarea"]):
- field_type = inp.get("type", inp.name) or "text"
- if field_type.lower() in {"hidden", "submit", "button"}:
- continue
- field_name = inp.get("name") or inp.get("id") or ""
- label_text = ""
- label_id = inp.get("id")
- if label_id:
- label_tag = soup.find("label", attrs={"for": label_id})
- if label_tag is not None:
- label_text = label_tag.get_text(" ", strip=True)
- form_fields.append(
- {
- "name": field_name[:120],
- "label": label_text[:240],
- "type": str(field_type)[:40],
- "value_hint": (inp.get("placeholder") or "")[:240],
- }
- )
-
- return title_text, anchors, form_fields
-
-
-def _discovery_resolve_href(base_url: str, href: str) -> str | None:
- try:
- absolute = urljoin(base_url, href)
- except Exception:
- return None
- parsed_abs = urlparse(absolute)
- parsed_base = urlparse(base_url)
- if parsed_abs.scheme not in {"http", "https"}:
- return None
- # Same-origin only — discovery does not follow cross-origin links to keep
- # the entrypoint search bounded to the user's named site.
- if parsed_abs.netloc and parsed_base.netloc and parsed_abs.netloc != parsed_base.netloc:
- return None
- return absolute
-
-
-def _discovery_origin_url(url: str) -> str | None:
- try:
- parsed = urlparse(url)
- except Exception:
- return None
- if parsed.scheme not in {"http", "https"} or not parsed.netloc:
- return None
- return f"{parsed.scheme}://{parsed.netloc}/"
-
-
-def _discovery_should_retry_from_origin(entry_url: str, current_url: str) -> bool:
- try:
- parsed_entry = urlparse(entry_url)
- parsed_current = urlparse(current_url)
- except Exception:
- return False
- if parsed_current.scheme not in {"http", "https"} or not parsed_current.netloc:
- return False
- if parsed_entry.netloc and parsed_entry.netloc != parsed_current.netloc:
- return False
- return bool(parsed_current.path not in {"", "/"} or parsed_current.query)
-
-
-def _discovery_anchor_selector(anchor: dict[str, str]) -> str | None:
- href = (anchor.get("href") or "").strip()
- if not href or any(char in href for char in {'"', "\\", "\n", "\r"}):
- return None
- return f'a[href="{href}"]'
-
-
-# Per-call timeout for each MCP primitive inside the discovery walker. The
-# walker also checks the cumulative 60s wall clock between steps, but without
-# a per-call cap a single hung navigate or get_html could block past the
-# cumulative cap (cumulative is only checked between awaits).
-_DISCOVERY_PER_CALL_TIMEOUT_SECONDS = 20.0
-_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE = 0.2
-# Scorer-miss outcomes (page loaded, no keyword match); wall outcomes are excluded as real blocks.
-_DISCOVERY_SCORER_MISS_REASONS = frozenset({"no_candidate", "step_limit", "wall_clock_limit"})
-
-
-def _discovery_last_loaded_url(evidence_trail: list[dict[str, Any]]) -> str | None:
- """URL of the last page the walk loaded, skipping failed nav/click steps and detected walls."""
- for entry in reversed(evidence_trail):
- reason = str(entry.get("transition_reason", ""))
- if reason.startswith("navigate_failed") or reason.startswith("anchor_click_failed"):
- continue
- if entry.get("wall"):
- continue
- url = entry.get("url")
- if isinstance(url, str) and url:
- return url
- return None
-
-
-async def _discovery_navigate(
- ctx: CopilotContext,
- url: str,
- *,
- wait_until: str | None = None,
- timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
-) -> dict[str, Any]:
- server = getattr(ctx, "discovery_mcp_server", None)
- if server is None:
- return {"ok": False, "error": "discovery MCP server not attached to context"}
- nav_args: dict[str, Any] = {"url": url}
- cap = timeout_seconds
- if wait_until:
- # `load` waits for every resource (analytics/marketing beacons on heavy
- # commerce pages keep it pending past the cap, so the navigate aborts before
- # any HTML is captured). `domcontentloaded` returns once the server-rendered
- # DOM is parsed — the forms/links are already present — and the recapture
- # loop settles anything still hydrating.
- nav_args["wait_until"] = wait_until
- nav_args["timeout"] = int(timeout_seconds * 1000)
- cap = timeout_seconds + 5
- try:
- return await asyncio.wait_for(
- server.call_internal_tool("skyvern_navigate", nav_args),
- timeout=cap,
- )
- except asyncio.TimeoutError:
- return {"ok": False, "error": f"skyvern_navigate timed out after {timeout_seconds:g}s"}
-
-
-async def _discovery_click_anchor(ctx: CopilotContext, anchor: dict[str, str]) -> dict[str, Any]:
- server = getattr(ctx, "discovery_mcp_server", None)
- if server is None:
- return {"ok": False, "error": "discovery MCP server not attached to context"}
- selector = _discovery_anchor_selector(anchor)
- if selector is None:
- return {"ok": False, "error": "anchor href could not be converted to a bounded CSS selector"}
- try:
- return await asyncio.wait_for(
- # call_internal_tool bypasses the schema overlays, so selector_mode="direct" must be
- # passed explicitly here (it is not picked up from the overlay's forced_args).
- server.call_internal_tool("skyvern_click", {"selector": selector, "selector_mode": "direct"}),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- return {"ok": False, "error": f"skyvern_click timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
-
-
-async def _discovery_get_html(ctx: CopilotContext) -> dict[str, Any]:
- """Read the full page body. ``skyvern_get_html`` requires a selector arg;
- pass ``body`` so the walker receives the full document body. Without this
- the raw MCP call fails validation since the inspection tool has a
- required positional ``selector``.
- """
- server = getattr(ctx, "discovery_mcp_server", None)
- if server is None:
- return {"ok": False, "error": "discovery MCP server not attached to context"}
- try:
- return await asyncio.wait_for(
- server.call_internal_tool("skyvern_get_html", {"selector": "body"}),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- return {"ok": False, "error": f"skyvern_get_html timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
-
-
-async def _composition_get_screenshot(ctx: CopilotContext) -> dict[str, Any]:
- server = getattr(ctx, "discovery_mcp_server", None)
- if server is None:
- return {"ok": False, "error": "discovery MCP server not attached to context"}
- try:
- return await asyncio.wait_for(
- server.call_internal_tool("skyvern_screenshot", {"inline": True}),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- return {"ok": False, "error": f"skyvern_screenshot timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
-
-
-def _discovery_extract_html_payload(result: dict[str, Any]) -> str:
- data = result.get("data")
- if isinstance(data, dict):
- for key in ("html", "outer_html", "text", "content"):
- value = data.get(key)
- if isinstance(value, str):
- return value
- return ""
-
-
-def _discovery_extract_current_url(result: dict[str, Any], fallback: str) -> str:
- data = result.get("data")
- if isinstance(data, dict):
- url = data.get("url") or data.get("current_url")
- if isinstance(url, str) and url:
- return url
- return fallback
-
-
-def _composition_extract_screenshot_b64(result: dict[str, Any]) -> str:
- data = result.get("data")
- if isinstance(data, dict):
- for key in ("screenshot_base64", "data", "image_base64"):
- value = data.get(key)
- if isinstance(value, str) and value.strip():
- return value.strip()
- return ""
-
-
-def _composition_visual_prompt(evidence: dict[str, Any]) -> str:
- context = {
- "page_title": evidence.get("page_title") or "",
- "current_url": evidence.get("current_url") or "",
- "anti_bot_indicators": evidence.get("anti_bot_indicators") or [],
- "challenge_state": evidence.get("challenge_state") or {},
- "form_count": len(evidence.get("forms") or []),
- "result_container_count": len(evidence.get("result_containers") or []),
- "page_obstruction_count": len(evidence.get("page_obstructions") or []),
- "visual_obstruction_candidate_count": len(evidence.get("visual_obstruction_candidates") or []),
- "schema_empty_page": evidence.get("schema_empty_page") is True,
- }
- return (
- "Summarize this screenshot for Workflow Copilot build-time page evidence. "
- "Return JSON only with keys: summary, challenge_detected, challenge_kind, "
- "challenge_location, submit_blocked, blocked_submit_controls, empty_page_visible, "
- "loading_state_visible, page_obstruction_detected, obstruction_kind, "
- "obstruction_location, underlying_page_blocked, visible_dismiss_controls, omissions. "
- "In summary, include the visible page state that would help verify an end-state outcome, "
- "such as cart items, "
- "record rows, visible identifiers, quantities, statuses, prices, confirmations, search results, "
- "or selected values when legible. Also note visible anti-bot or human-verification state, "
- "whether it appears to gate a submit/search control, whether any visible artificial barrier "
- "mechanically blocks the underlying page, and where it appears relative to the page controls. "
- "Do not include raw DOM, code, selectors, personal data, or workflow instructions. "
- "If no challenge is visible, set challenge_detected to false and submit_blocked to false. "
- "If no page obstruction is visible, set page_obstruction_detected to false. "
- "If DOM context shows a schema-empty page, set empty_page_visible to true only when the "
- "screenshot shows a settled page with no visible forms, controls, result data, challenge, "
- "or loading/progress state; set loading_state_visible to true when the page appears to be "
- "waiting, loading, redirecting, or still rendering.\n\n"
- f"DOM evidence context:\n{json.dumps(context, sort_keys=True)}"
- )
-
-
-async def _composition_visual_handler(ctx: CopilotContext) -> Any | None:
- return await resolve_fast_copilot_handler(
- getattr(ctx, "workflow_permanent_id", None),
- getattr(ctx, "organization_id", None),
- )
-
-
-def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:
- if not isinstance(value, dict):
- return None
- summary = value.get("summary")
- challenge_detected = value.get("challenge_detected")
- challenge_kind = value.get("challenge_kind")
- challenge_location = value.get("challenge_location")
- submit_blocked = value.get("submit_blocked")
- blocked_submit_controls = value.get("blocked_submit_controls")
- empty_page_visible = value.get("empty_page_visible")
- loading_state_visible = value.get("loading_state_visible")
- page_obstruction_detected = value.get("page_obstruction_detected")
- obstruction_kind = value.get("obstruction_kind")
- obstruction_location = value.get("obstruction_location")
- underlying_page_blocked = value.get("underlying_page_blocked")
- visible_dismiss_controls = value.get("visible_dismiss_controls")
- omissions = value.get("omissions")
- return {
- "summary": summary if isinstance(summary, str) else "",
- "challenge_detected": challenge_detected if isinstance(challenge_detected, bool) else None,
- "challenge_kind": challenge_kind if isinstance(challenge_kind, str) else "",
- "challenge_location": challenge_location if isinstance(challenge_location, str) else "",
- "submit_blocked": submit_blocked if isinstance(submit_blocked, bool) else None,
- "empty_page_visible": empty_page_visible if isinstance(empty_page_visible, bool) else None,
- "loading_state_visible": loading_state_visible if isinstance(loading_state_visible, bool) else None,
- "page_obstruction_detected": page_obstruction_detected if isinstance(page_obstruction_detected, bool) else None,
- "obstruction_kind": obstruction_kind if isinstance(obstruction_kind, str) else "",
- "obstruction_location": obstruction_location if isinstance(obstruction_location, str) else "",
- "underlying_page_blocked": underlying_page_blocked if isinstance(underlying_page_blocked, bool) else None,
- "blocked_submit_controls": [item for item in blocked_submit_controls if isinstance(item, str)]
- if isinstance(blocked_submit_controls, list)
- else [],
- "visible_dismiss_controls": [item for item in visible_dismiss_controls if isinstance(item, str)]
- if isinstance(visible_dismiss_controls, list)
- else [],
- "omissions": [item for item in omissions if isinstance(item, str)] if isinstance(omissions, list) else [],
- }
-
-
-async def _composition_summarize_screenshot(
- ctx: CopilotContext,
- *,
- evidence: dict[str, Any],
- screenshot_b64: str,
-) -> tuple[dict[str, Any] | None, str | None]:
- handler = await _composition_visual_handler(ctx)
- if handler is None:
- return None, "workflow copilot LLM handler is not configured"
- try:
- screenshot_bytes = base64.b64decode(screenshot_b64, validate=True)
- except Exception:
- return None, "screenshot payload was not valid base64"
- try:
- response = await asyncio.wait_for(
- handler(
- prompt=_composition_visual_prompt(evidence),
- prompt_name=_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME,
- screenshots=[screenshot_bytes],
- organization_id=getattr(ctx, "organization_id", None),
- force_dict=True,
- ),
- timeout=_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- return None, f"visual summary timed out after {_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS:g}s"
- except Exception as exc:
- LOG.warning("Composition screenshot visual summary failed", error=str(exc), exc_info=True)
- return None, str(exc)
- normalized = _normalize_visual_summary(response)
- if normalized is None:
- return None, "visual summary response was not a JSON object"
- return normalized, None
-
-
-async def _augment_composition_evidence_with_visual_fallback(
- ctx: CopilotContext,
- evidence: dict[str, Any],
-) -> dict[str, Any]:
- screenshot_result = await _composition_get_screenshot(ctx)
- if not screenshot_result.get("ok"):
- return _composition_add_evidence_omission(
- evidence,
- f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",
- )
- screenshot_b64 = _composition_extract_screenshot_b64(screenshot_result)
- visual_summary, visual_error = await _composition_summarize_screenshot(
- ctx,
- evidence=evidence,
- screenshot_b64=screenshot_b64,
- )
- return merge_visual_composition_evidence(evidence, visual_summary=visual_summary, visual_error=visual_error)
-
-
-def _composition_add_evidence_omission(evidence: dict[str, Any], message: str) -> dict[str, Any]:
- merged = dict(evidence)
- omissions = [item for item in merged.get("visual_evidence_omissions") or [] if isinstance(item, str)]
- if message:
- omissions.append(message[:160])
- merged["visual_evidence_omissions"] = list(dict.fromkeys(omissions))[:5]
- return merged
-
-
-def _composition_add_inspection_warning(evidence: dict[str, Any], message: str) -> dict[str, Any]:
- merged = dict(evidence)
- warnings = [item for item in merged.get("inspection_warnings") or [] if isinstance(item, str)]
- if message:
- warnings.append(message[:240])
- merged["inspection_warnings"] = list(dict.fromkeys(warnings))[:5]
- return merged
-
-
-async def _composition_evidence_after_navigation_failure(
- ctx: CopilotContext,
- *,
- inspected_url: str,
- navigation_error: str,
-) -> dict[str, Any] | None:
- current_url, _ = await _fallback_page_info(ctx)
- current_url = current_url or inspected_url
- structured = await _composition_get_structured_evidence(ctx, inspected_url=inspected_url, current_url=current_url)
- if structured is not None and has_bounded_page_schema(structured):
- evidence = _composition_add_inspection_warning(
- structured,
- f"navigation_error_before_html_capture: {navigation_error}",
- )
- if page_evidence_needs_visual_fallback(evidence):
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence
- # Same size-cap survival as the success path: a heavy page that rendered before the nav
- # error still parses via the stripped-body evaluate instead of yielding hollow evidence.
- html, html_error, html_truncated, _ = await _composition_get_html(ctx)
- if html_error is None:
- evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
- evidence = _composition_add_inspection_warning(
- evidence,
- f"navigation_error_before_html_capture: {navigation_error}",
- )
- if html_truncated:
- evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")
- evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(ctx, evidence)
- if page_evidence_needs_visual_fallback(evidence):
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence
-
- evidence = parse_composition_html("", inspected_url=inspected_url, current_url=current_url)
- evidence = _composition_add_inspection_warning(
- evidence,
- f"navigation_error_before_evidence_capture: {navigation_error}",
- )
- evidence = _composition_add_inspection_warning(
- evidence,
- f"html_capture_failed_after_navigation_error: {html_error}",
- )
- evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
- return evidence if evidence.get("screenshot_used") else None
-
-
-# Large enough for long single-turn reconnaissance: block_observation_refs are
-# chosen after scouting, so this cap must stay above the citation window a turn
-# can realistically compose against. Entries carry bounded parse summaries, not
-# raw HTML or screenshots.
-_MAX_FLOW_EVIDENCE = 64
-
-
-def _inspection_reached_via(*, use_current_page: bool, post_run: bool, earned_interaction: bool) -> str:
- """How the just-inspected state was reached, for the flow-evidence trajectory.
-
- A target_url inspection navigates there itself ("navigate"); a post-run
- current-page inspection observes the page the run left behind ("post_run"); a
- normal current-page inspection counts as an interaction only when a successful
- browser action immediately earned that credit.
- """
- if not use_current_page:
- return "navigate"
- if post_run:
- return "post_run"
- return "interaction" if earned_interaction else "current_page"
-
-
-def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached_via: str) -> int | None:
- """Append a typed entry to the bounded flow-evidence trajectory (SKY-10562).
-
- One entry per scouted page: the page-evidence packet plus how it was reached
- and whether bounded schema was captured. Feeds the per-acted-page composition
- gate and the cross-turn observed-page summary; never written into the YAML.
- """
- trajectory = getattr(copilot_ctx, "flow_evidence", None)
- if not isinstance(trajectory, list):
- return None
- prior_steps = [entry.get("step") for entry in trajectory if isinstance(entry, dict)]
- step = (
- max((value for value in prior_steps if isinstance(value, int) and not isinstance(value, bool)), default=-1) + 1
- )
- trajectory.append(
- {
- "evidence": evidence,
- "reached_via": reached_via,
- "had_bounded_schema": has_bounded_page_schema(evidence),
- "step": step,
- }
- )
- if len(trajectory) > _MAX_FLOW_EVIDENCE:
- overflow_entry_count = len(trajectory) - _MAX_FLOW_EVIDENCE
- LOG.warning(
- "copilot_flow_evidence_evicted",
- overflow_entry_count=overflow_entry_count,
- max_flow_evidence=_MAX_FLOW_EVIDENCE,
- retained_window_size=_MAX_FLOW_EVIDENCE,
- latest_step=step,
- )
- del trajectory[:-_MAX_FLOW_EVIDENCE]
- return step
-
-
-def _latest_interaction_reached_flow_evidence(copilot_ctx: Any) -> tuple[int, str, dict[str, Any]] | None:
- trajectory = getattr(copilot_ctx, "flow_evidence", None)
- if not isinstance(trajectory, list):
- return None
- for entry in reversed(trajectory):
- if not isinstance(entry, dict):
- continue
- reached_via = str(entry.get("reached_via") or "")
- if reached_via not in {"interaction", "post_run"}:
- continue
- evidence = entry.get("evidence")
- step = entry.get("step")
- if isinstance(step, bool) or not isinstance(step, int) or not isinstance(evidence, dict):
- continue
- if not has_bounded_page_schema(evidence):
- continue
- observed_url = _composition_evidence_page_url(evidence)
- if observed_url:
- return step, observed_url, evidence
- return None
-
-
-def _non_current_inspection_regression_error(copilot_ctx: Any, *, entry_url: str) -> dict[str, Any] | None:
- latest = _latest_interaction_reached_flow_evidence(copilot_ctx)
- if latest is None:
- return None
- observation_step, observed_url, _ = latest
- if _same_page_ignoring_fragment(observed_url, entry_url):
- return None
- return {
- "ok": False,
- "data": {
- "current_url": observed_url,
- "observation_step": observation_step,
- },
- "error": (
- "inspect_page_for_composition would navigate away from the latest interaction-reached page "
- f'({observed_url}). Use inspect_page_for_composition(target_url="current_page") to inspect '
- "the live page, or compose from the existing page evidence and pass observation_step "
- f"{observation_step} in block_observation_refs for blocks that act on that reached page."
- ),
- }
-
-
-def _page_inspection_budget_error(copilot_ctx: Any, *, scope: Literal["turn", "chat"]) -> str:
- scope_label = "turn" if scope == "turn" else "chat"
- return (
- f"inspect_page_for_composition reached the page-inspection budget for this {scope_label}. "
- "This is not evidence that scouting is complete. Use evaluate, get_browser_screenshot, or a browser "
- "action on the current page to determine whether the goal is already satisfied, whether progress is still "
- "possible, or whether a real blocker exists. Do not author downstream result, extraction, or confirmation "
- "blocks unless the existing evidence already shows the page state those blocks will act on."
- )
-
-
-_COMPOSITION_HOLLOW_RECAPTURE_RETRIES = 2
-_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS = 2.5
-# The composition inspect navigates with `domcontentloaded`, so a heavier cap than
-# the discovery walker's is safe — the navigate returns at DOM parse, well before
-# this ceiling, and only a genuinely stuck load reaches it.
-_COMPOSITION_NAVIGATE_TIMEOUT_SECONDS = 30.0
-
-
-async def _composition_get_stripped_html(copilot_ctx: Any) -> tuple[str | None, bool]:
- """Return (stripped_body_html, truncated). truncated is True when the expression sliced
- the body at the cap, so the tail (below-fold forms/controls) is missing from the evidence."""
- server = getattr(copilot_ctx, "discovery_mcp_server", None)
- if server is None:
- return None, False
- try:
- result = await asyncio.wait_for(
- server.call_internal_tool("skyvern_evaluate", {"expression": _COMPOSITION_STRIPPED_HTML_EXPRESSION}),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except Exception:
- return None, False
- if not isinstance(result, dict) or not result.get("ok"):
- return None, False
- value = (result.get("data") or {}).get("result")
- if not isinstance(value, str):
- return None, False
- return value, len(value) >= _COMPOSITION_STRIPPED_HTML_MAX_CHARS
-
-
-def _normalize_visual_obstruction_candidates(value: Any) -> list[dict[str, Any]]:
- if not isinstance(value, list):
- return []
- candidates: list[dict[str, Any]] = []
- for item in value:
- if len(candidates) >= 5:
- break
- if not isinstance(item, dict):
- continue
- position = item.get("position")
- coverage = item.get("coverage")
- if position not in {"fixed", "sticky"} or coverage != "viewport":
- continue
- candidates.append(
- {
- "source": "computed_style",
- "position": position,
- "coverage": "viewport",
- "has_visible_controls": item.get("has_visible_controls") is True,
- }
- )
- return candidates
-
-
-def _merge_visual_obstruction_candidates(
- evidence: dict[str, Any],
- candidates: list[dict[str, Any]],
-) -> dict[str, Any]:
- if not candidates:
- return evidence
- merged = dict(evidence)
- existing = [item for item in merged.get("visual_obstruction_candidates") or [] if isinstance(item, dict)]
- for candidate in candidates:
- if len(existing) >= 5:
- break
- if candidate not in existing:
- existing.append(candidate)
- merged["visual_obstruction_candidates"] = existing[:5]
- return merged
-
-
-async def _composition_get_computed_visual_obstruction_candidates(copilot_ctx: Any) -> list[dict[str, Any]]:
- server = getattr(copilot_ctx, "discovery_mcp_server", None)
- if server is None:
- return []
- try:
- result = await asyncio.wait_for(
- server.call_internal_tool(
- "skyvern_evaluate",
- {"expression": _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION},
- ),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except Exception:
- return []
- if not isinstance(result, dict) or not result.get("ok"):
- return []
- value = (result.get("data") or {}).get("result")
- return _normalize_visual_obstruction_candidates(value)
-
-
-async def _augment_composition_evidence_with_computed_obstruction_candidates(
- copilot_ctx: Any,
- evidence: dict[str, Any],
-) -> dict[str, Any]:
- if page_evidence_needs_visual_fallback(evidence) or not has_bounded_page_schema(evidence):
- return evidence
- candidates = await _composition_get_computed_visual_obstruction_candidates(copilot_ctx)
- return _merge_visual_obstruction_candidates(evidence, candidates)
-
-
-async def _composition_get_html(copilot_ctx: Any, *, skip_raw: bool = False) -> tuple[str, str | None, bool, bool]:
- """Return body HTML for composition parsing, surviving the MCP response size cap.
-
- `skyvern_get_html("body")` is the fast path, but the shared size cap DROPS the
- payload (no `html` field, just truncation metadata) when the serialized body
- exceeds the limit — heavy commerce pages routinely do, so the inspector would
- parse an empty string and report hollow evidence. On an empty/capped read, fall
- back to an `evaluate` that returns the body with script/style/svg/etc. stripped
- and length-bounded; that fits under the cap while preserving the form/link
- structure. Returns (html, error, truncated, used_stripped): error is set only on
- a hard read failure; truncated is True when the stripped fallback was sliced at
- the cap; used_stripped is True when the bounded read was the source (raw skipped
- or cap-dropped). `skip_raw` goes straight to the stripped read so a caller that
- has already seen the raw serialization get cap-dropped for this page need not
- re-issue it.
- """
- html_result: dict[str, Any] = {}
- if not skip_raw:
- html_result = await _discovery_get_html(copilot_ctx)
- if html_result.get("ok"):
- html = _discovery_extract_html_payload(html_result)
- if html.strip():
- return html, None, False, False
- stripped, truncated = await _composition_get_stripped_html(copilot_ctx)
- if stripped and stripped.strip():
- return stripped, None, truncated, True
- error = html_result.get("error")
- return "", str(error) if error else None, False, True
-
-
-async def _composition_get_structured_evidence(
- copilot_ctx: Any,
- *,
- inspected_url: str,
- current_url: str,
-) -> dict[str, Any] | None:
- """Capture composition evidence via the page-side extractor; None when it can't yield a usable payload."""
- server = getattr(copilot_ctx, "discovery_mcp_server", None)
- if server is None:
- return None
- with copilot_span("composition_structured_extract"):
- try:
- result = await asyncio.wait_for(
- server.call_internal_tool(
- "skyvern_evaluate", {"expression": _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION}
- ),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except Exception:
- return None
- if not isinstance(result, dict) or not result.get("ok"):
- return None
- raw = (result.get("data") or {}).get("result")
- if isinstance(raw, str):
- if len(raw) > _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS:
- return None
- try:
- payload = json.loads(raw)
- except (ValueError, TypeError):
- return None
- elif isinstance(raw, dict):
- payload = raw
- else:
- return None
- return parse_composition_structured(payload, inspected_url=inspected_url, current_url=current_url)
-
-
-async def _capture_composition_evidence(
- copilot_ctx: Any,
- *,
- inspected_url: str,
- current_url: str,
- active_run_terminal_sample: bool = False,
-) -> tuple[dict[str, Any] | None, str | None]:
- """Parse composition evidence (cheap extractor first, get_html fallback); html_error is set only on a failed HTML read."""
- evidence: dict[str, Any] | None = None
- html_truncated = False
- used_structured = False
- skip_raw = False
- for attempt in range(_COMPOSITION_HOLLOW_RECAPTURE_RETRIES + 1):
- structured = await _composition_get_structured_evidence(
- copilot_ctx, inspected_url=inspected_url, current_url=current_url
- )
- if structured is not None:
- evidence = structured
- used_structured = True
- if has_bounded_page_schema(evidence):
- break
- if attempt < _COMPOSITION_HOLLOW_RECAPTURE_RETRIES:
- await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
- continue
- html, html_error, html_truncated, used_stripped = await _composition_get_html(copilot_ctx, skip_raw=skip_raw)
- if html_error is not None:
- if evidence is not None:
- break
- return None, html_error
- # On a heavy page the raw get_html serialization is dropped over the MCP size cap and
- # falls back to the stripped read; once that happens, settle-and-recapture via the
- # stripped path only so a slow page is still retried without re-serializing the full DOM.
- if used_stripped:
- skip_raw = True
- evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
- used_structured = False
- if has_bounded_page_schema(evidence):
- break
- if attempt < _COMPOSITION_HOLLOW_RECAPTURE_RETRIES:
- await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
- if evidence is not None and html_truncated and not used_structured:
- evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")
- # Structured evidence already carries computed obstruction candidates; only the get_html path augments.
- if evidence is not None and not used_structured:
- evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(copilot_ctx, evidence)
- if evidence is not None and (
- page_evidence_needs_visual_fallback(evidence)
- or (active_run_terminal_sample and _active_run_terminal_evidence_needs_visual_fallback(evidence))
- or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))
- ):
- evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)
- return evidence, None
-
-
-def _normalized_inspect_url(url: str | None) -> str | None:
- """Normalized full URL for strict same-page comparison, or None when not comparable.
-
- Preserves scheme, the path's trailing slash, query, and fragment so distinct rendered
- states (http vs https, /p vs /p/, ?q=a vs ?q=b, hash-routed SPA states) never collide;
- only netloc case and an empty root path are normalized.
- """
- if not isinstance(url, str) or not url.strip():
- return None
- try:
- parsed = urlparse(url.strip())
- except Exception:
- return None
- if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
- return None
- path = parsed.path or "/"
- query = f"?{parsed.query}" if parsed.query else ""
- fragment = f"#{parsed.fragment}" if parsed.fragment else ""
- return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}{path}{query}{fragment}"
-
-
-def _same_inspect_target(live_url: str | None, target_url: str | None) -> bool:
- """True when the live page is the exact page a URL-target inspect would navigate to.
-
- Strict full-URL equality, so a different scheme, trailing slash, query, or fragment
- still navigates. Used to skip the re-navigation when the agent is already standing on
- the requested page.
- """
- live_key = _normalized_inspect_url(live_url)
- target_key = _normalized_inspect_url(target_url)
- return live_key is not None and live_key == target_key
-
-
-async def _inspect_page_for_composition_impl(
- copilot_ctx: Any,
- target_url: str,
-) -> dict[str, Any]:
- """Inspect a known target page and store form/search evidence on ctx.
-
- This is composition context, not workflow YAML. It is intentionally separate
- from `discover_workflow_entrypoint`: discovery answers "which page?";
- inspection answers "what fields and controls are actually on this page?".
- """
- arguments = {"target_url": target_url}
- authority_error = _authority_tool_error(copilot_ctx, "inspect_page_for_composition")
- if authority_error:
- result = {"ok": False, "error": authority_error}
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
-
- use_current_page = (target_url or "").strip().lower() in _CURRENT_PAGE_INSPECTION_TARGETS
- if not use_current_page:
- _clear_pending_browser_interaction_observation(copilot_ctx)
- bypass_budget_for_post_run_current_page = _allows_post_run_current_page_inspection_budget_bypass(
- copilot_ctx,
- use_current_page=use_current_page,
- )
-
- entry_url: str
- kind: str
- if use_current_page:
- current_url, _ = await _fallback_page_info(copilot_ctx)
- entry_url = current_url or "current_page"
- kind = "current_page"
- else:
- resolved_entry_url, kind = _resolve_discovery_entry_url(target_url)
- if resolved_entry_url is None:
- result = {
- "ok": False,
- "data": None,
- "error": "inspect_page_for_composition requires a URL, domain with an explicit path, or target_url='current_page'.",
- }
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
- entry_url = resolved_entry_url
- regression_error = _non_current_inspection_regression_error(copilot_ctx, entry_url=entry_url)
- if regression_error is not None:
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, regression_error)
- return regression_error
-
- # Skip re-navigation when the inspect target is the page the browser is already on. A
- # passive client-side redirect can move the browser without a tool, so for a URL target
- # confirm against the live URL; for current_page the live URL is the target by definition.
- if use_current_page:
- inspect_target_url = current_url
- on_target_page = True
- else:
- live_url, _ = await _fallback_page_info(copilot_ctx)
- on_target_page = _same_inspect_target(live_url, entry_url)
- inspect_target_url = live_url if on_target_page else entry_url
-
- if (
- not bypass_budget_for_post_run_current_page
- and copilot_ctx.page_inspection_calls_this_turn >= _COMPOSITION_INSPECTION_PER_TURN_BUDGET
- ):
- result = {
- "ok": False,
- "data": None,
- "error": _page_inspection_budget_error(copilot_ctx, scope="turn"),
- }
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
-
- cumulative = copilot_ctx.prior_page_inspection_calls_made + copilot_ctx.page_inspection_calls_this_turn
- if not bypass_budget_for_post_run_current_page and cumulative >= _COMPOSITION_INSPECTION_PER_CHAT_BUDGET:
- result = {
- "ok": False,
- "data": None,
- "error": _page_inspection_budget_error(copilot_ctx, scope="chat"),
- }
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
-
- evidence = None
- html_error: str | None = None
- with copilot_span(
- "inspect_page_for_composition",
- data={"target_url_kind": kind},
- ):
- if on_target_page:
- # current_page, or a URL target the agent is already on — capture without navigating.
- current_url = inspect_target_url or entry_url
- evidence, html_error = await _capture_composition_evidence(
- copilot_ctx, inspected_url=entry_url, current_url=current_url
- )
- else:
- nav_result = await _discovery_navigate(
- copilot_ctx,
- entry_url,
- wait_until="domcontentloaded",
- timeout_seconds=_COMPOSITION_NAVIGATE_TIMEOUT_SECONDS,
- )
- if not nav_result.get("ok"):
- nav_error = str(nav_result.get("error") or "unknown")
- failure_evidence = await _composition_evidence_after_navigation_failure(
- copilot_ctx,
- inspected_url=entry_url,
- navigation_error=nav_error,
- )
- if failure_evidence is None:
- result = {
- "ok": False,
- "data": None,
- "error": f"inspect_page_for_composition could not navigate: {nav_error}",
- }
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
- evidence = failure_evidence
- current_url = str(evidence.get("current_url") or entry_url)
- else:
- current_url = _discovery_extract_current_url(nav_result, entry_url)
- evidence, html_error = await _capture_composition_evidence(
- copilot_ctx, inspected_url=entry_url, current_url=current_url
- )
-
- if html_error is not None:
- result = {
- "ok": False,
- "data": None,
- "error": f"inspect_page_for_composition could not read page HTML: {html_error}",
- }
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
-
- if evidence is None:
- result = {
- "ok": False,
- "data": None,
- "error": "inspect_page_for_composition could not read page HTML.",
- }
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
-
- run_id = getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)
- if isinstance(run_id, str) and run_id:
- evidence = {
- **evidence,
- "workflow_run_id": run_id,
- "observed_after_workflow_run": True,
- }
- _mark_post_run_page_observed(copilot_ctx, source_tool="inspect_page_for_composition", url=current_url)
- page_title = evidence.get("page_title")
- if isinstance(page_title, str) and page_title:
- _workflow_verification_evidence(copilot_ctx).page_title = page_title[:160]
-
- if not bypass_budget_for_post_run_current_page:
- copilot_ctx.page_inspection_calls_this_turn += 1
- if bypass_budget_for_post_run_current_page:
- copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_id
- copilot_ctx.composition_page_evidence = evidence
- if (
- isinstance(run_id, str)
- and run_id
- and getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False)
- ):
- page_title = evidence.get("page_title")
- await _maybe_run_completion_verification_from_page_observation(
- copilot_ctx,
- url=str(evidence.get("current_url") or current_url or ""),
- title=page_title if isinstance(page_title, str) else "",
- observed_data=evidence,
- )
- earned_interaction = False
- if use_current_page and not run_id:
- earned_interaction = _consume_pending_browser_interaction_observation(
- copilot_ctx,
- current_url=str(evidence.get("current_url") or current_url or ""),
- evidence=evidence,
- )
- reached_via = _inspection_reached_via(
- use_current_page=use_current_page,
- post_run=bool(run_id),
- earned_interaction=earned_interaction,
- )
- observation_step = _append_flow_evidence(copilot_ctx, evidence, reached_via=reached_via)
- if observation_step is None:
- LOG.warning("copilot_flow_evidence_append_failed_no_trajectory")
- _mark_page_inspected(copilot_ctx)
- # Surface the reached page at the top level so the model registers that the
- # inspection already navigated there and does not re-issue navigate_browser.
- current_url = evidence.get("current_url") or evidence.get("inspected_url") or ""
- result = {
- "ok": True,
- "current_url": current_url,
- "reached_via": reached_via,
- "data": evidence,
- }
- if observation_step is not None:
- result["observation_step"] = observation_step
- record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
- return result
-
-
-async def _discovery_walk(
- ctx: CopilotContext,
- *,
- entry_url: str,
- intent_hint: str,
-) -> dict[str, Any]:
- """Deterministic anchor-scoring walker. No inner LLM call.
-
- Reads each visited page, looks for a strong title/H1 match with the
- intent_hint, surfaces a form if one is present, and otherwise follows
- the highest-scored same-origin anchor whose text matches the
- intent_hint. Bounded by step / wall-clock caps.
- """
- intent_tokens = set(_DISCOVERY_TOKEN_RE.findall(intent_hint.lower())) if intent_hint else set()
- evidence_trail: list[dict[str, Any]] = []
- current_url = entry_url
- current_page_loaded = False
- retried_deep_link_from_origin = False
- started = ctx.discovery_started_monotonic or time.monotonic()
-
- for step in range(_DISCOVERY_STEP_CAP):
- ctx.discovery_step_count = step + 1
- elapsed = time.monotonic() - started
- if elapsed > _DISCOVERY_WALL_CLOCK_SECONDS:
- return _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="wall_clock_limit",
- )
-
- if current_page_loaded:
- current_page_loaded = False
- else:
- nav_result = await _discovery_navigate(ctx, current_url)
- if not nav_result.get("ok"):
- evidence_trail.append(
- {
- "url": current_url,
- "page_title": "",
- "transition_reason": f"navigate_failed: {nav_result.get('error', 'unknown')}"[:240],
- }
- )
- # A pre-composition browser/session failure is not evidence that
- # the user omitted a page URL. Return the resolved entry URL so the
- # agent can test a minimal goto_url block and gather real run/debug
- # evidence before deciding whether to ask a follow-up.
- return _discovery_build_result(
- candidate_url=current_url,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
- failure_reason=None,
- )
-
- current_url = _discovery_extract_current_url(nav_result, current_url)
- # Survive the MCP size cap: a heavy DOM exceeds it and the html field is dropped, so
- # fall back to a stripped-body evaluate that keeps the links/forms the resolver needs
- # to identify a usable entrypoint. (Discovery only resolves the entrypoint, so a sliced
- # tail does not matter here.)
- html, _, _, _ = await _composition_get_html(ctx)
- page_title, anchors, form_fields = _discovery_parse_html(html)
-
- evidence_trail.append(
- {
- "url": current_url,
- "page_title": page_title[:240],
- "transition_reason": "initial" if step == 0 else "anchor_match",
- }
- )
-
- title_score = _discovery_title_score(page_title, intent_tokens)
-
- best_score = 0
- best_href: str | None = None
- best_anchor: dict[str, str] | None = None
- for anchor in anchors:
- score = _discovery_anchor_score(
- anchor.get("text", ""),
- anchor.get("title", ""),
- anchor.get("href", ""),
- intent_tokens,
- )
- if score > best_score:
- resolved = _discovery_resolve_href(current_url, anchor.get("href", ""))
- if resolved is None:
- continue
- best_score = score
- best_href = resolved
- best_anchor = anchor
-
- evidence_tokens = _discovery_candidate_evidence_tokens(intent_tokens)
- candidate_title_score = _discovery_title_score(page_title, evidence_tokens)
- candidate_anchor_score = 0
- for anchor in anchors:
- candidate_anchor_score = max(
- candidate_anchor_score,
- _discovery_anchor_score(
- anchor.get("text", ""),
- anchor.get("title", ""),
- anchor.get("href", ""),
- evidence_tokens,
- ),
- )
-
- anti_bot_detected = _discovery_detect_anti_bot(html, page_title)
- login_wall_detected = _discovery_detect_login_wall(html, page_title)
- anti_bot_has_no_candidate_evidence = (
- not form_fields and candidate_title_score == 0 and candidate_anchor_score == 0
- )
- if anti_bot_detected or login_wall_detected:
- # Tag walls so the scorer-miss fallback refuses them even when they degrade to no_candidate.
- evidence_trail[-1]["wall"] = True
- if anti_bot_detected and anti_bot_has_no_candidate_evidence:
- origin_url = _discovery_origin_url(current_url)
- if (
- not retried_deep_link_from_origin
- and origin_url
- and origin_url != current_url
- and _discovery_should_retry_from_origin(entry_url, current_url)
- ):
- evidence_trail[-1]["transition_reason"] = "direct_deep_link_anti_bot"
- current_url = origin_url
- retried_deep_link_from_origin = True
- continue
- return _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="anti_bot_wall",
- )
- if login_wall_detected:
- return _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="login_wall",
- )
-
- if intent_tokens and title_score >= 2 and (form_fields or best_score <= title_score):
- confidence = min(1.0, title_score / max(1, len(intent_tokens)))
- return _discovery_build_result(
- candidate_url=current_url,
- candidate_form_fields=form_fields,
- evidence_trail=evidence_trail,
- confidence=confidence,
- failure_reason=None,
- )
-
- if form_fields and (title_score >= 1 or step > 0):
- confidence = 0.6 if title_score >= 1 else 0.4
- return _discovery_build_result(
- candidate_url=current_url,
- candidate_form_fields=form_fields,
- evidence_trail=evidence_trail,
- confidence=confidence,
- failure_reason=None,
- )
-
- if not intent_tokens:
- return _discovery_build_result(
- candidate_url=current_url,
- candidate_form_fields=form_fields,
- evidence_trail=evidence_trail,
- confidence=0.3,
- failure_reason=None,
- )
-
- if best_score == 0 or best_href is None:
- return _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="no_candidate",
- )
-
- if retried_deep_link_from_origin and best_anchor is not None:
- click_result = await _discovery_click_anchor(ctx, best_anchor)
- if click_result.get("ok"):
- current_url = _discovery_extract_current_url(click_result, best_href)
- # The next loop should inspect the clicked page instead of
- # navigating back to the original entry URL.
- current_page_loaded = True
- continue
- evidence_trail.append(
- {
- "url": best_href,
- "page_title": "",
- "transition_reason": f"anchor_click_failed: {click_result.get('error', 'unknown')}"[:240],
- }
- )
-
- current_url = best_href
-
- return _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=0.0,
- failure_reason="step_limit",
- )
-
-
-async def _discover_workflow_entrypoint_impl(
- copilot_ctx: Any,
- site_or_url: str,
- intent_hint: str,
-) -> dict[str, Any]:
- """Discovery tool body — separated from the @function_tool wrapper so
- tests can drive it with a stand-in ctx without the SDK's invocation
- machinery.
- """
- arguments = {"site_or_url": site_or_url, "intent_hint": intent_hint}
-
- def finish(result: dict[str, Any], *, site_or_url_kind: str | None = None) -> dict[str, Any]:
- _record_discovery_resolution_on_ctx(copilot_ctx, result)
- record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
- data_payload = result.get("data")
- data = data_payload if isinstance(data_payload, Mapping) else {}
- candidate_url = data.get("candidate_url")
- failure_reason = data.get("failure_reason")
- if not isinstance(failure_reason, str) or not failure_reason:
- error = result.get("error")
- failure_reason = error if isinstance(error, str) and error else None
- LOG.info(
- "discover_workflow_entrypoint completed",
- ok=result.get("ok"),
- candidate_url=candidate_url if isinstance(candidate_url, str) and candidate_url else None,
- failure_reason=failure_reason,
- site_or_url_kind=site_or_url_kind,
- )
- return result
-
- authority_error = _authority_tool_error(copilot_ctx, "discover_workflow_entrypoint")
- if authority_error:
- result = {"ok": False, "error": authority_error}
- return finish(result)
-
- if copilot_ctx.discovery_calls_this_turn >= _DISCOVERY_PER_TURN_BUDGET:
- result = _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=list(copilot_ctx.discovery_evidence_trail),
- confidence=0.0,
- failure_reason="discovery_already_completed_this_turn",
- )
- return finish(result)
-
- cumulative = copilot_ctx.prior_discovery_calls_made + copilot_ctx.discovery_calls_this_turn
- if cumulative >= _DISCOVERY_PER_CHAT_BUDGET:
- result = _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=[],
- confidence=0.0,
- failure_reason="discovery_budget_exhausted_for_chat",
- )
- return finish(result)
-
- entry_url, kind = _resolve_discovery_entry_url(site_or_url)
- if entry_url is None:
- result = _discovery_build_result(
- candidate_url=None,
- candidate_form_fields=[],
- evidence_trail=[],
- confidence=0.0,
- failure_reason="could_not_resolve_site_name",
- )
- return finish(result, site_or_url_kind=kind)
-
- if copilot_ctx.build_phase == BuildPhase.INITIAL:
- try:
- advance_to_discovering(copilot_ctx)
- except ValueError as exc:
- # Race or unexpected prior advance — proceed without re-transitioning,
- # but surface the impossible state so it shows up in production logs.
- LOG.warning(
- "discover_workflow_entrypoint phase transition to discovering rejected",
- error=str(exc),
- build_phase=copilot_ctx.build_phase.value,
- )
- copilot_ctx.discovery_calls_this_turn += 1
-
- concrete_homepage_url = _concrete_homepage_entrypoint(entry_url, kind)
- if concrete_homepage_url is not None:
- evidence_trail = [
- {
- "url": concrete_homepage_url,
- "page_title": "",
- "transition_reason": "concrete_domain_homepage",
- }
- ]
- result = _discovery_build_result(
- candidate_url=concrete_homepage_url,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- # Lower than a scraped-page match because the fast path skips page inspection.
- confidence=_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE,
- failure_reason=None,
- )
- copilot_ctx.discovery_evidence_trail = list(evidence_trail)
- try:
- advance_to_composing(copilot_ctx, reason="discovery_concrete_domain_homepage")
- except ValueError as exc:
- LOG.warning(
- "discover_workflow_entrypoint phase transition to composing rejected",
- error=str(exc),
- build_phase=copilot_ctx.build_phase.value,
- )
- return finish(result, site_or_url_kind=kind)
-
- with copilot_span(
- "discover_workflow_entrypoint",
- data={
- "site_or_url_kind": kind,
- "intent_hint_len": len(intent_hint or ""),
- "phase_entered": copilot_ctx.build_phase.value,
- },
- ):
- try:
- result = await _discovery_walk(
- copilot_ctx,
- entry_url=entry_url,
- intent_hint=intent_hint or "",
- )
- except Exception as exc:
- LOG.exception("discover_workflow_entrypoint walker raised")
- result = {
- "ok": False,
- "data": {
- "candidate_url": None,
- "candidate_form_fields": [],
- "evidence_trail": [],
- "confidence": 0.0,
- "failure_reason": None,
- },
- "error": f"discover_workflow_entrypoint failed: {exc}",
- }
-
- data_payload = result.get("data") or {}
- data: dict[str, Any] = data_payload if isinstance(data_payload, dict) else {}
- evidence_trail = data.get("evidence_trail") or []
- copilot_ctx.discovery_evidence_trail = list(evidence_trail)
- if (
- result.get("ok")
- and not data.get("candidate_url")
- and data.get("failure_reason") in _DISCOVERY_SCORER_MISS_REASONS
- ):
- navigated_url = _discovery_last_loaded_url(evidence_trail)
- if navigated_url:
- # Don't ask for a URL we already reached; hand the loaded page to composition at low confidence.
- result = _discovery_build_result(
- candidate_url=navigated_url,
- candidate_form_fields=[],
- evidence_trail=evidence_trail,
- confidence=_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
- failure_reason=None,
- )
- rebuilt_data = result.get("data")
- data = rebuilt_data if isinstance(rebuilt_data, dict) else {}
- copilot_ctx.discovery_evidence_trail = list(data.get("evidence_trail", []))
- if result.get("ok") and data.get("candidate_url"):
- try:
- advance_to_composing(copilot_ctx, reason="discovery_returned_candidate")
- except ValueError as exc:
- LOG.warning(
- "discover_workflow_entrypoint phase transition to composing rejected",
- error=str(exc),
- build_phase=copilot_ctx.build_phase.value,
- )
-
- return finish(result, site_or_url_kind=kind)
-
-
-@function_tool(name_override="discover_workflow_entrypoint", strict_mode=False)
-async def discover_workflow_entrypoint_tool(
- ctx: RunContextWrapper,
- site_or_url: str,
- intent_hint: str,
-) -> str:
- """Find the page a new workflow should start at when the user named a site but not the page.
-
- Use this BEFORE writing blocks when the user named a website (with a URL,
- a bare domain, or a single brand word) but no specific page. Accepts:
- a URL with or without scheme (``example.com/login`` is fine), a bare
- domain (``example.com``), or a single brand word (resolved as
- ``https://www.<word>.com``). English phrases ("the X website") return
- ``failure_reason=could_not_resolve_site_name`` — ASK_QUESTION for a URL.
-
- Returns ``candidate_url`` plus a short ``evidence_trail`` and any
- ``candidate_form_fields``. Use ``candidate_url`` as the ``url`` value
- on a ``goto_url`` block. Do NOT paste the evidence into workflow YAML.
-
- Budget: one successful call per turn, three per chat, eight page hops,
- sixty seconds. On any ``failure_reason``, ASK_QUESTION for a URL — do not
- retry. Discovery navigates and reads pages; it will NOT type, click form
- buttons, run JavaScript, or submit forms.
- """
- result = await _discover_workflow_entrypoint_impl(ctx.context, site_or_url, intent_hint)
- return json.dumps(result)
-
-
-@function_tool(name_override="inspect_page_for_composition", strict_mode=False)
-async def inspect_page_for_composition_tool(
- ctx: RunContextWrapper,
- target_url: str,
-) -> str:
- """Inspect a known page before composing form/search workflow blocks.
-
- Use this after the entrypoint URL is known and before authoring blocks that
- fill fields, submit searches, filter results, or expand result rows. It
- can also inspect the current browser page after a run by passing
- target_url="current_page"; use that after partial/budgeted runs so you do
- not replay a search that already advanced the page.
-
- Returns observed page evidence: current URL, title, navigation targets, form
- fields with labels and selectors, submit/search controls, result containers,
- compact visible text excerpts, anti-bot indicators, and bounded visual
- challenge evidence when DOM evidence shows challenge state. The returned
- `observation_step` is the side-channel id to pass in `block_observation_refs`
- when a newly authored block acts on this observed page. Do NOT paste the
- evidence into workflow YAML; use it to ground concise block prompts. If a
- block run changes pages, inspect the reached page before authoring downstream
- form/search/result blocks. If the
- evidence shows required fields or controls that the user did not supply
- enough information for, ASK_QUESTION with that observed missing input. If
- evidence is sufficient, compose and run workflow blocks from the observed fields.
- If challenge_state.gates_submit_controls is true, treat challenge resolution
- as a prerequisite for submit/search; do not click a submit control while the
- latest inspected evidence says it is disabled. If a later test still leaves
- that submit/search control disabled after a challenge-resolution attempt,
- report the observed anti-bot blocker rather than retrying the same flow.
- """
- result = await _inspect_page_for_composition_impl(ctx.context, target_url)
- return json.dumps(result)
-
-
-NATIVE_TOOLS = [
- update_workflow_tool,
- list_credentials_tool,
- run_blocks_tool,
- get_run_results_tool,
- update_and_run_blocks_tool,
- discover_workflow_entrypoint_tool,
- inspect_page_for_composition_tool,
-]
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
new file mode 100644
index 000000000..2ce073605
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -0,0 +1,835 @@
+"""Copilot agent tools — native handlers, hooks, and registration."""
+
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+
+import structlog
+from agents import function_tool
+from agents.run_context import RunContextWrapper
+
+from skyvern.forge import app as app
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.build_phase import (
+ advance_to_testing,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ composition_page_evidence_error as composition_page_evidence_error,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema as has_bounded_page_schema
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ normalize_block_observation_refs,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import workflow_target_url as workflow_target_url
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.failure_tracking import (
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY as ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+)
+from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.output_utils import (
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY as _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
+)
+from skyvern.forge.sdk.copilot.output_utils import (
+ sanitize_tool_result_for_llm,
+)
+from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml as _process_workflow_yaml
+
+from ._shared import _COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS
+from ._shared import _FAILED_BLOCK_STATUSES as _FAILED_BLOCK_STATUSES
+from ._shared import BLOCK_RUNNING_TOOLS as BLOCK_RUNNING_TOOLS
+from ._shared import COPILOT_FINAL_REPLY_RESERVE_SECONDS as COPILOT_FINAL_REPLY_RESERVE_SECONDS
+from ._shared import PER_TOOL_CALL_BUDGET_SECONDS as PER_TOOL_CALL_BUDGET_SECONDS
+from ._shared import (
+ RUN_BLOCKS_SAFETY_CEILING_SECONDS,
+)
+from ._shared import _composition_get_html as _composition_get_html
+from ._shared import _current_workflow_has_evidence_block as _current_workflow_has_evidence_block
+from ._shared import (
+ _emit_tool_blocker_signal,
+)
+from ._shared import _fallback_page_info as _fallback_page_info
+from ._shared import _is_meaningful_extracted_data as _is_meaningful_extracted_data
+from ._shared import _proxy_location_trace_value as _proxy_location_trace_value
+from ._shared import _raw_yaml_proxy_location as _raw_yaml_proxy_location
+from ._shared import _same_page_ignoring_fragment as _same_page_ignoring_fragment
+from ._shared import _unverified_current_workflow_labels as _unverified_current_workflow_labels
+from .banned_blocks import _COPILOT_BANNED_BLOCK_TYPES as _COPILOT_BANNED_BLOCK_TYPES
+from .banned_blocks import _banned_block_reject_message as _banned_block_reject_message
+from .banned_blocks import _challenge_http_request_reject_message as _challenge_http_request_reject_message
+from .banned_blocks import _detect_new_banned_blocks as _detect_new_banned_blocks
+from .banned_blocks import _detect_timing_only_challenge_wait_blocks as _detect_timing_only_challenge_wait_blocks
+from .banned_blocks import _record_banned_block_reject_span as _record_banned_block_reject_span
+from .banned_blocks import _timing_only_challenge_wait_reject_message as _timing_only_challenge_wait_reject_message
+from .blockers import REPEATED_ACTION_STREAK_ABORT_AT as REPEATED_ACTION_STREAK_ABORT_AT
+from .blockers import _active_block_run_budget_seconds as _active_block_run_budget_seconds
+from .blockers import _analyze_run_blocks as _analyze_run_blocks
+from .blockers import _build_loop_blocker_signal as _build_loop_blocker_signal
+from .blockers import _last_run_has_terminal_anti_bot_blocker as _last_run_has_terminal_anti_bot_blocker
+from .blockers import _per_tool_budget_problem_rerun_signal as _per_tool_budget_problem_rerun_signal
+from .blockers import (
+ _record_per_tool_budget_problem_blocks_from_results,
+ _tool_loop_error,
+)
+from .blockers import _trusted_post_drain_status as _trusted_post_drain_status
+from .completion import _build_run_evidence_snapshot as _build_run_evidence_snapshot
+from .completion import _completion_verification_handler as _completion_verification_handler
+from .completion import _is_outcome_evidence_candidate as _is_outcome_evidence_candidate
+from .completion import _is_unfinished_run_verification_candidate as _is_unfinished_run_verification_candidate
+from .completion import (
+ _maybe_run_completion_verification,
+)
+from .completion import (
+ _maybe_run_completion_verification_from_page_observation as _maybe_run_completion_verification_from_page_observation,
+)
+from .completion import _outcome_failure_warrants_repair as _outcome_failure_warrants_repair
+from .completion import _outcome_unverified_reason as _outcome_unverified_reason
+from .completion import (
+ _tool_visible_result_after_completion_verification,
+)
+from .composition_capture import _COMPOSITION_INSPECTION_PER_CHAT_BUDGET as _COMPOSITION_INSPECTION_PER_CHAT_BUDGET
+from .composition_capture import _COMPOSITION_INSPECTION_PER_TURN_BUDGET as _COMPOSITION_INSPECTION_PER_TURN_BUDGET
+from .composition_capture import (
+ _active_run_terminal_evidence_needs_visual_fallback as _active_run_terminal_evidence_needs_visual_fallback,
+)
+from .composition_capture import _active_run_terminal_evidence_result as _active_run_terminal_evidence_result
+from .composition_capture import _active_run_terminal_evidence_sample as _active_run_terminal_evidence_sample
+from .composition_capture import _capture_composition_evidence as _capture_composition_evidence
+from .composition_capture import (
+ _composition_evidence_after_navigation_failure as _composition_evidence_after_navigation_failure,
+)
+from .composition_capture import _composition_visual_handler as _composition_visual_handler
+from .composition_capture import _composition_visual_prompt as _composition_visual_prompt
+from .composition_capture import (
+ _inspect_page_for_composition_impl,
+)
+from .composition_capture import _normalized_inspect_url as _normalized_inspect_url
+from .composition_capture import _same_inspect_target as _same_inspect_target
+from .credentials import _credential_id_misbinding_error_message as _credential_id_misbinding_error_message
+from .credentials import _credential_id_misbinding_findings as _credential_id_misbinding_findings
+from .credentials import _credential_reference_validation_error as _credential_reference_validation_error
+from .credentials import _extract_credential_ids_from_tool_value as _extract_credential_ids_from_tool_value
+from .credentials import _extract_credential_ids_from_workflow_yaml as _extract_credential_ids_from_workflow_yaml
+from .credentials import (
+ _list_credentials,
+)
+from .discovery import _DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE as _DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE
+from .discovery import _DISCOVERY_PER_CHAT_BUDGET as _DISCOVERY_PER_CHAT_BUDGET
+from .discovery import _DISCOVERY_PER_TURN_BUDGET as _DISCOVERY_PER_TURN_BUDGET
+from .discovery import _DISCOVERY_STEP_CAP as _DISCOVERY_STEP_CAP
+from .discovery import _discover_workflow_entrypoint_impl as _discover_workflow_entrypoint_impl
+from .discovery import _discovery_anchor_score as _discovery_anchor_score
+from .discovery import _discovery_click_anchor as _discovery_click_anchor
+from .discovery import _discovery_detect_anti_bot as _discovery_detect_anti_bot
+from .discovery import _discovery_detect_login_wall as _discovery_detect_login_wall
+from .discovery import _discovery_resolve_href as _discovery_resolve_href
+from .discovery import _discovery_walk as _discovery_walk
+from .discovery import _resolve_discovery_entry_url as _resolve_discovery_entry_url
+from .frontier import _CANONICAL_WORKFLOW_SETTING_FIELDS as _CANONICAL_WORKFLOW_SETTING_FIELDS
+from .frontier import _JINJA_LITERAL_ROOTS as _JINJA_LITERAL_ROOTS
+from .frontier import _JINJA_RUNTIME_GLOBAL_ROOTS as _JINJA_RUNTIME_GLOBAL_ROOTS
+from .frontier import _JINJA_SPECIAL_CONTEXT_ROOTS as _JINJA_SPECIAL_CONTEXT_ROOTS
+from .frontier import _SKYVERN_TEMPLATE_CONTEXT_ROOTS as _SKYVERN_TEMPLATE_CONTEXT_ROOTS
+from .frontier import _TEMPLATE_BUILTIN_ROOTS as _TEMPLATE_BUILTIN_ROOTS
+from .frontier import _detect_stale_block_metadata as _detect_stale_block_metadata
+from .frontier import _find_invalidated_labels as _find_invalidated_labels
+from .frontier import _frontier_run_size_error as _frontier_run_size_error
+from .frontier import _get_prior_workflow as _get_prior_workflow
+from .frontier import _get_prior_workflow_definition as _get_prior_workflow_definition
+from .frontier import _invalidate_verified_state_on_edit as _invalidate_verified_state_on_edit
+from .frontier import _plan_frontier as _plan_frontier
+from .frontier import _referenced_output_labels as _referenced_output_labels
+from .frontier import _stale_block_metadata_message as _stale_block_metadata_message
+from .frontier import _unknown_jinja_roots as _unknown_jinja_roots
+from .frontier import _workflow_requires_canonical_persist as _workflow_requires_canonical_persist
+from .frontier import _workflow_with_runtime_frontier_anchor as _workflow_with_runtime_frontier_anchor
+from .frontier import (
+ _workflow_with_runtime_frontier_starter_url_seed as _workflow_with_runtime_frontier_starter_url_seed,
+)
+from .guardrails import (
+ _COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA,
+ _WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL,
+ _authority_tool_error,
+)
+from .guardrails import _parameter_binding_invariant_error as _parameter_binding_invariant_error
+from .guardrails import (
+ _request_policy_allows_credential_deferred_draft,
+ _request_policy_allows_update_and_skip_run,
+)
+from .guardrails import _turn_intent_tool_error as _turn_intent_tool_error
+from .guardrails import (
+ _update_and_run_blocks_composition_evidence_precheck,
+)
+from .mcp_hooks import _build_skyvern_mcp_overlays as _build_skyvern_mcp_overlays
+from .mcp_hooks import _click_post_hook as _click_post_hook
+from .mcp_hooks import (
+ _code_only_pre_run_results_error,
+)
+from .mcp_hooks import _evaluate_post_hook as _evaluate_post_hook
+from .mcp_hooks import _get_block_schema_post_hook as _get_block_schema_post_hook
+from .mcp_hooks import _get_block_schema_pre_hook as _get_block_schema_pre_hook
+from .mcp_hooks import _navigate_post_hook as _navigate_post_hook
+from .mcp_hooks import _press_key_post_hook as _press_key_post_hook
+from .mcp_hooks import _screenshot_post_hook as _screenshot_post_hook
+from .mcp_hooks import _select_option_post_hook as _select_option_post_hook
+from .mcp_hooks import _type_text_post_hook as _type_text_post_hook
+from .mcp_hooks import _verify_scout_type_landed as _verify_scout_type_landed
+from .mcp_hooks import get_skyvern_mcp_alias_map as get_skyvern_mcp_alias_map
+from .page_observation import _record_composition_page_observation as _record_composition_page_observation
+from .page_observation import _resolve_url_title as _resolve_url_title
+from .run_execution import RUN_BLOCKS_STAGNATION_WINDOW_SECONDS as RUN_BLOCKS_STAGNATION_WINDOW_SECONDS
+from .run_execution import WatchdogExitReason as WatchdogExitReason
+from .run_execution import _any_quiet_block_requested as _any_quiet_block_requested
+from .run_execution import _attach_action_traces as _attach_action_traces
+from .run_execution import _cancel_run_task_if_not_final as _cancel_run_task_if_not_final
+from .run_execution import _composition_anti_bot_reason as _composition_anti_bot_reason
+from .run_execution import _detect_non_retriable_nav_error as _detect_non_retriable_nav_error
+from .run_execution import _detect_probable_site_block_wall as _detect_probable_site_block_wall
+from .run_execution import (
+ _diagnosis_repair_tool_error,
+ _frontier_run_size_result,
+ _get_run_results,
+)
+from .run_execution import _mark_pending_reconciliation_run as _mark_pending_reconciliation_run
+from .run_execution import (
+ _maybe_clear_reconciliation_flag,
+)
+from .run_execution import _progress_marker as _progress_marker
+from .run_execution import _read_progress_sources as _read_progress_sources
+from .run_execution import (
+ _record_diagnosis_repair_contract,
+ _record_run_blocks_result,
+ _run_blocks_and_collect_debug,
+ _run_blocks_span_data,
+)
+from .run_execution import _watchdog_error_message as _watchdog_error_message
+from .run_execution import _watchdog_exit_allows_terminal_promotion as _watchdog_exit_allows_terminal_promotion
+from .run_execution import _watchdog_user_failure_reason as _watchdog_user_failure_reason
+from .scouting import _MAX_SCOUTED_INTERACTIONS as _MAX_SCOUTED_INTERACTIONS
+from .scouting import _capture_accessible_role_name as _capture_accessible_role_name
+from .scouting import _capture_scout_source_url as _capture_scout_source_url
+from .scouting import _clear_pending_browser_interaction_observation as _clear_pending_browser_interaction_observation
+from .scouting import (
+ _consume_pending_browser_interaction_observation as _consume_pending_browser_interaction_observation,
+)
+from .scouting import _consume_scout_source_url as _consume_scout_source_url
+from .scouting import _mark_page_inspected as _mark_page_inspected
+from .scouting import _mark_pending_browser_interaction_observation as _mark_pending_browser_interaction_observation
+from .scouting import _mark_post_run_page_observed as _mark_post_run_page_observed
+from .scouting import _record_scouted_interaction as _record_scouted_interaction
+from .scouting import _register_scout_interaction_observation as _register_scout_interaction_observation
+from .scouting import _resolve_scout_role_name as _resolve_scout_role_name
+from .scouting import _role_name_from_selector as _role_name_from_selector
+from .workflow_update import BlockObservationRef as BlockObservationRef
+from .workflow_update import CodeArtifactMetadata as CodeArtifactMetadata
+from .workflow_update import _code_artifact_metadata_as_tool_argument as _code_artifact_metadata_as_tool_argument
+from .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata
+from .workflow_update import _pre_run_workflow_coverage_error as _pre_run_workflow_coverage_error
+from .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_span
+from .workflow_update import _record_workflow_update_result as _record_workflow_update_result
+from .workflow_update import _update_workflow as _update_workflow
+
+LOG = structlog.get_logger()
+
+
+@function_tool(
+ name_override="update_workflow",
+ tool_input_guardrails=[_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL],
+)
+async def update_workflow_tool(
+ ctx: RunContextWrapper,
+ workflow_yaml: str,
+ block_observation_refs: list[BlockObservationRef] | None = None,
+ code_artifact_metadata: list[CodeArtifactMetadata] | None = None,
+) -> str:
+ """Validate and update the workflow YAML definition.
+ Provide the complete workflow YAML as a string.
+ Returns the validated workflow or validation errors.
+
+ Top-level workflow parameter keys appear in the run-input UI. When you
+ add runtime inputs in `workflow_definition.parameters`, name keys for the
+ reusable domain value the user supplies, not the page widget or action used
+ to enter it.
+
+ Use browser inspection and run evidence to fill knowledge gaps while
+ building or editing the workflow. Do not invent URL params, form fields,
+ result affordances, or page structure from memory; ground workflow blocks
+ in observed MCP evidence or information the user supplied.
+ When you compose no-url blocks from a page reached by prior clicks, include
+ `block_observation_refs` entries with each block label and the
+ `observation_step` returned by inspect_page_for_composition for the page
+ that block acts on.
+ For authored code blocks, include `code_artifact_metadata` rows describing
+ declared goals, claimed outcomes, page dependencies, criteria, evidence
+ refs, observation refs, and terminal verifier expectations.
+ """
+ copilot_ctx = ctx.context
+ serialized_code_artifact_metadata = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
+ normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
+ arguments = {
+ "workflow_yaml": workflow_yaml,
+ "block_observation_refs": normalized_block_observation_refs,
+ "code_artifact_metadata": serialized_code_artifact_metadata,
+ }
+ loop_error = _tool_loop_error(copilot_ctx, "update_workflow", arguments)
+ if loop_error:
+ return json.dumps({"ok": False, "error": loop_error})
+ credential_deferred_draft = _request_policy_allows_credential_deferred_draft(copilot_ctx)
+ # A credential-deferred draft is redirected to update_and_run_blocks' skip-run
+ # save path, which saves the draft and skips the browser run with the required
+ # credential setup message.
+ if credential_deferred_draft:
+ agent_steering = (
+ "Use update_and_run_blocks for this credential-deferred draft. It will save the workflow draft "
+ "and skip the browser run with the required credential setup message."
+ )
+ user_facing = (
+ "I can save this as a draft without running it because the credentials aren't set up yet. "
+ "Add them in the Credentials UI and ask me to test the workflow."
+ )
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_and_run_blocks"}),
+ internal_reason_code="request_policy_credential_deferred_redirect",
+ blocked_tool="update_workflow",
+ )
+ payload = _emit_tool_blocker_signal(copilot_ctx, signal)
+ result = {"ok": False, "error": payload}
+ record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
+ return json.dumps(result)
+
+ with copilot_span(
+ "composition_evidence_precheck",
+ data={**_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA, "tool_name": "update_workflow"},
+ ):
+ composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
+ copilot_ctx,
+ workflow_yaml,
+ normalized_block_observation_refs,
+ block_observation_refs,
+ )
+ if composition_evidence_error:
+ result = {"ok": False, "error": composition_evidence_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_workflow",
+ result=result,
+ )
+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)
+ return json.dumps(sanitized)
+
+ prior_definition = await _get_prior_workflow_definition(copilot_ctx)
+ with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
+ result = await _update_workflow(
+ {
+ **arguments,
+ "raw_block_observation_refs": block_observation_refs,
+ "raw_code_artifact_metadata": code_artifact_metadata,
+ },
+ copilot_ctx,
+ allow_missing_credentials=getattr(copilot_ctx, "allow_untested_workflow_draft", False) is True,
+ )
+ _record_workflow_update_result(copilot_ctx, result, prior_definition)
+ record_tool_step_result_for_ctx(copilot_ctx, "update_workflow", arguments, result)
+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)
+ return json.dumps(sanitized)
+
+
+@function_tool(name_override="list_credentials")
+async def list_credentials_tool(
+ ctx: RunContextWrapper,
+ page: int = 1,
+ page_size: int = 10,
+) -> str:
+ """List stored credentials (metadata only — never passwords or secrets).
+ Use this to find credential IDs for login blocks.
+
+ Paginated. `page_size` caps at 50. The response includes `has_more`;
+ before concluding no credential exists, keep incrementing `page` until
+ `has_more` is `false` — otherwise you risk telling the user to create
+ a credential they have already stored on a later page.
+ """
+ copilot_ctx = ctx.context
+ arguments = {"page": page, "page_size": page_size}
+ loop_error = _tool_loop_error(copilot_ctx, "list_credentials", arguments)
+ if loop_error:
+ return json.dumps({"ok": False, "error": loop_error})
+
+ authority_error = _authority_tool_error(copilot_ctx, "list_credentials")
+ if authority_error:
+ result = {"ok": False, "error": authority_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)
+ return json.dumps(result)
+
+ result = await _list_credentials(arguments, copilot_ctx)
+ record_tool_step_result_for_ctx(copilot_ctx, "list_credentials", arguments, result)
+ sanitized = sanitize_tool_result_for_llm("list_credentials", result)
+ return json.dumps(sanitized)
+
+
+@function_tool(
+ name_override="run_blocks_and_collect_debug",
+ timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS,
+ strict_mode=False,
+)
+async def run_blocks_tool(
+ ctx: RunContextWrapper,
+ block_labels: list[str],
+ parameters: dict[str, Any] | None = None,
+) -> Any:
+ """Run one or more blocks of the current workflow, wait for completion,
+ and return compact debug output (status, failure reason, visible elements).
+ The workflow must be saved before running blocks.
+ Block labels must match labels in the saved workflow.
+
+ For diagnostic complaints, follow the system prompt's ASK-vs-EDIT routing.
+ If the complaint has no prior edit goal, inspect current workflow context
+ and existing run evidence before deciding whether a fresh run is needed.
+ If prior context establishes a resolvable edit, use `update_and_run_blocks`
+ instead of rerunning unchanged blocks.
+
+ Pass runtime values for workflow parameters via the `parameters` dict —
+ keys must match the workflow parameter `key` field. When the user has
+ supplied concrete non-secret values in their message (names, emails, IDs),
+ pass them on the first call rather than letting the workflow fall back to
+ placeholders. For sensitive values (password, secret, token, api_key,
+ credential, totp, otp, one_time_code, private_key, auth) — call
+ `list_credentials` and use a credential parameter whose default_value is
+ the stored `credential_id`. If no stored credential matches, do NOT pass
+ the inline value via `parameters`; stop and follow the CREDENTIAL
+ HANDLING refusal rule in the system prompt.
+
+ Use browser inspection and run evidence to fill knowledge gaps before
+ changing the workflow. If visible state is uncertain, inspect the live
+ page and then compose the next normal workflow action from observed
+ evidence instead of retrying guessed URL params or page structure.
+ """
+ copilot_ctx = ctx.context
+ copilot_ctx.completion_verification_result = None
+ handler_start = time.monotonic()
+ arguments = {"block_labels": block_labels, "parameters": parameters or {}}
+ authority_error = _authority_tool_error(copilot_ctx, "run_blocks_and_collect_debug")
+ if authority_error:
+ return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", authority_error)
+
+ loop_error = _tool_loop_error(copilot_ctx, "run_blocks_and_collect_debug", arguments)
+ if loop_error:
+ return _diagnosis_repair_tool_error(copilot_ctx, "run_blocks_and_collect_debug", loop_error)
+
+ prior_definition = await _get_prior_workflow_definition(copilot_ctx)
+ labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(
+ copilot_ctx, block_labels, prior_definition, prior_definition
+ )
+ frontier_error = _frontier_run_size_error(copilot_ctx, block_labels, labels_to_execute, prior_definition)
+ if frontier_error:
+ result = _frontier_run_size_result(frontier_error, block_labels, labels_to_execute)
+ record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="run_blocks_and_collect_debug",
+ result=result,
+ )
+ return json.dumps(result)
+
+ with copilot_span(
+ "run_blocks",
+ data=_run_blocks_span_data(
+ block_labels,
+ labels_to_execute,
+ frontier_start_label,
+ block_outputs_to_seed,
+ copilot_ctx,
+ ),
+ ):
+ result = await _run_blocks_and_collect_debug(
+ arguments,
+ copilot_ctx,
+ labels_to_execute=labels_to_execute,
+ block_outputs_to_seed=block_outputs_to_seed,
+ frontier_start_label=frontier_start_label,
+ )
+ completion_verification = await _maybe_run_completion_verification(copilot_ctx, result, handler_start)
+ _record_run_blocks_result(copilot_ctx, result, completion_verification=completion_verification)
+ tool_visible_result = _tool_visible_result_after_completion_verification(
+ copilot_ctx,
+ result,
+ completion_verification,
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "run_blocks_and_collect_debug", arguments, tool_visible_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="run_blocks_and_collect_debug",
+ result=tool_visible_result,
+ )
+ enqueue_screenshot_from_result(copilot_ctx, result)
+
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)
+ return json.dumps(sanitized)
+
+
+@function_tool(name_override="get_run_results")
+async def get_run_results_tool(
+ ctx: RunContextWrapper,
+ workflow_run_id: str | None = None,
+) -> str:
+ """Fetch results from a previous workflow run.
+ Returns block statuses, failure reasons, and output data.
+ If workflow_run_id is omitted, fetches the most recently created finished
+ run (completed, failed, canceled, terminated, or timed_out — excludes
+ in-flight runs). For unambiguous results in concurrent-run scenarios,
+ pass an explicit workflow_run_id from a prior tool response.
+ """
+ copilot_ctx = ctx.context
+ params: dict[str, Any] = {}
+ if workflow_run_id:
+ params["workflow_run_id"] = workflow_run_id
+ loop_error = _tool_loop_error(copilot_ctx, "get_run_results", params)
+ if loop_error:
+ return json.dumps({"ok": False, "error": loop_error})
+ authority_error = _authority_tool_error(copilot_ctx, "get_run_results")
+ if authority_error:
+ return json.dumps({"ok": False, "error": authority_error})
+ if isinstance(copilot_ctx, CopilotContext):
+ code_only_pre_run_error = _code_only_pre_run_results_error(copilot_ctx)
+ if code_only_pre_run_error is not None:
+ record_tool_step_result_for_ctx(copilot_ctx, "get_run_results", params, code_only_pre_run_error)
+ return json.dumps(code_only_pre_run_error)
+
+ result = await _get_run_results(params, copilot_ctx)
+ _record_per_tool_budget_problem_blocks_from_results(copilot_ctx, result)
+ _maybe_clear_reconciliation_flag(copilot_ctx, result)
+ record_tool_step_result_for_ctx(copilot_ctx, "get_run_results", params, result)
+
+ sanitized = sanitize_tool_result_for_llm("get_run_results", result)
+ return json.dumps(sanitized)
+
+
+@function_tool(
+ name_override="update_and_run_blocks",
+ timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS,
+ strict_mode=False,
+ tool_input_guardrails=[_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL],
+)
+async def update_and_run_blocks_tool(
+ ctx: RunContextWrapper,
+ workflow_yaml: str,
+ block_labels: list[str],
+ block_observation_refs: list[BlockObservationRef] | None = None,
+ code_artifact_metadata: list[CodeArtifactMetadata] | None = None,
+ parameters: dict[str, Any] | None = None,
+) -> Any:
+ """Update the workflow YAML and immediately run the specified blocks in one step.
+ Use this instead of calling update_workflow and run_blocks_and_collect_debug separately.
+ The workflow must validate successfully before blocks are run.
+ `block_labels` may be a tested frontier subset of the full workflow YAML;
+ save the complete reusable workflow, then run only the next 1-2 unverified
+ blocks when a long form/search/result chain can be verified incrementally.
+
+ Top-level workflow parameter keys appear in the run-input UI. When you
+ add runtime inputs in `workflow_definition.parameters`, name keys for the
+ reusable domain value the user supplies, not the page widget or action used
+ to enter it.
+
+ For diagnostic complaints, follow the system prompt's ASK-vs-EDIT routing.
+ A complaint with no prior edit goal needs context inspection or
+ clarification first. A diagnostic follow-up after an explicit edit goal may
+ update/run once the correction is clear.
+
+ Pass runtime values for workflow parameters via the `parameters` dict —
+ keys must match the workflow parameter `key` field. When the user has
+ supplied concrete non-secret values in their message (names, emails, IDs),
+ pass them on the first call rather than letting the workflow fall back to
+ placeholders. For sensitive values (password, secret, token, api_key,
+ credential, totp, otp, one_time_code, private_key, auth) — call
+ `list_credentials` and use a credential parameter whose default_value is
+ the stored `credential_id`. If no stored credential matches, do NOT pass
+ the inline value via `parameters`; stop and follow the CREDENTIAL
+ HANDLING refusal rule in the system prompt.
+
+ Use browser inspection and run evidence to fill knowledge gaps while
+ building, editing, or debugging the workflow. Do not invent URL params,
+ form fields, result affordances, or page structure from memory; ground
+ workflow blocks in observed MCP evidence or information the user supplied.
+ Only refine URL params when they are grounded in observed DOM/link/form
+ state or observed URL deltas.
+ Browser inspection is build-time context; add durable workflow blocks only
+ for the reusable actions/checks the workflow actually needs.
+ When you compose no-url blocks from a page reached by prior clicks, include
+ `block_observation_refs` entries with each block label and the
+ `observation_step` returned by inspect_page_for_composition or evaluate for
+ the page that block acts on.
+ For authored code blocks, include `code_artifact_metadata` rows describing
+ declared goals, claimed outcomes, page dependencies, criteria, evidence
+ refs, observation refs, and terminal verifier expectations.
+ When inspected evidence shows an anti-bot challenge gating a disabled
+ submit/search control, account for challenge resolution before submit;
+ do not compose a click against a control observed as disabled.
+ """
+ copilot_ctx = ctx.context
+ copilot_ctx.completion_verification_result = None
+ handler_start = time.monotonic()
+ serialized_code_artifact_metadata = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
+ normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
+ arguments = {
+ "workflow_yaml": workflow_yaml,
+ "block_labels": block_labels,
+ "block_observation_refs": normalized_block_observation_refs,
+ "code_artifact_metadata": serialized_code_artifact_metadata,
+ "parameters": parameters or {},
+ }
+ skip_run_after_update = _request_policy_allows_update_and_skip_run(copilot_ctx, "update_and_run_blocks")
+ authority_error = _authority_tool_error(
+ copilot_ctx,
+ "update_and_run_blocks",
+ ignore_request_policy_error=skip_run_after_update,
+ )
+ if authority_error:
+ return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", authority_error)
+
+ loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
+ if loop_error:
+ return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)
+
+ with copilot_span("composition_evidence_precheck", data=_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA):
+ composition_evidence_error = _update_and_run_blocks_composition_evidence_precheck(
+ copilot_ctx,
+ workflow_yaml,
+ normalized_block_observation_refs,
+ block_observation_refs,
+ )
+ if composition_evidence_error:
+ result = {"ok": False, "error": composition_evidence_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=result,
+ )
+ sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)
+ return json.dumps(sanitized)
+
+ _clear_pending_browser_interaction_observation(copilot_ctx)
+
+ # Snapshot the prior workflow definition BEFORE _update_workflow saves
+ # the new one — we need the pre-update state to diff against.
+ prior_definition = await _get_prior_workflow_definition(copilot_ctx)
+
+ # Step 1: Update the workflow
+ with copilot_span("update_workflow", data={"yaml_length": len(workflow_yaml)}):
+ update_result = await _update_workflow(
+ {
+ "workflow_yaml": workflow_yaml,
+ "block_observation_refs": normalized_block_observation_refs,
+ "raw_block_observation_refs": block_observation_refs,
+ "code_artifact_metadata": serialized_code_artifact_metadata,
+ "raw_code_artifact_metadata": code_artifact_metadata,
+ },
+ copilot_ctx,
+ allow_missing_credentials=skip_run_after_update,
+ )
+ _record_workflow_update_result(copilot_ctx, update_result, prior_definition)
+
+ if not update_result.get("ok"):
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, update_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=update_result,
+ )
+ sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)
+ return json.dumps(sanitized)
+
+ coverage_error = _pre_run_workflow_coverage_error(copilot_ctx)
+ if coverage_error:
+ user_facing_summary = (
+ "Workflow draft saved; I still need to add the remaining requested actions before testing it."
+ )
+ result = {
+ "ok": False,
+ "error": coverage_error,
+ "data": {
+ "block_count": copilot_ctx.last_update_block_count,
+ "workflow_updated": True,
+ "workflow_run_skipped": True,
+ "control_signal": {
+ "kind": "intermediate_success",
+ "user_facing_summary": user_facing_summary,
+ },
+ "user_facing_summary": user_facing_summary,
+ },
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=result,
+ workflow_updated=True,
+ )
+ return json.dumps(result)
+
+ if skip_run_after_update:
+ skip_message = "Skipped test run: required credentials are not configured."
+ skip_result = {
+ "ok": True,
+ "message": skip_message,
+ "data": {
+ "block_count": copilot_ctx.last_update_block_count,
+ "workflow_updated": True,
+ "skipped_run": True,
+ "skip_reason": "workflow_credential_inputs_unbound",
+ },
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, skip_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=skip_result,
+ workflow_updated=True,
+ )
+ LOG.info(
+ "update_and_run_blocks skipped run on unbound credential workflow inputs",
+ workflow_permanent_id=copilot_ctx.workflow_permanent_id,
+ )
+ return json.dumps(skip_result)
+
+ # Step 2: Compute frontier and run the blocks.
+ new_definition = None
+ if copilot_ctx.last_workflow is not None:
+ new_definition = getattr(copilot_ctx.last_workflow, "workflow_definition", None)
+
+ labels_to_execute, block_outputs_to_seed, frontier_start_label = _plan_frontier(
+ copilot_ctx, block_labels, prior_definition, new_definition
+ )
+ frontier_error = _frontier_run_size_error(copilot_ctx, block_labels, labels_to_execute, new_definition)
+ if frontier_error:
+ result = _frontier_run_size_result(frontier_error, block_labels, labels_to_execute)
+ data = result.get("data")
+ if isinstance(data, dict):
+ data["workflow_updated"] = True
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=result,
+ workflow_updated=True,
+ )
+ return json.dumps(result)
+
+ with copilot_span(
+ "run_blocks",
+ data=_run_blocks_span_data(
+ block_labels,
+ labels_to_execute,
+ frontier_start_label,
+ block_outputs_to_seed,
+ copilot_ctx,
+ ),
+ ):
+ run_result = await _run_blocks_and_collect_debug(
+ {"block_labels": block_labels, "parameters": parameters or {}},
+ copilot_ctx,
+ labels_to_execute=labels_to_execute,
+ block_outputs_to_seed=block_outputs_to_seed,
+ frontier_start_label=frontier_start_label,
+ )
+ completion_verification = await _maybe_run_completion_verification(copilot_ctx, run_result, handler_start)
+ _record_run_blocks_result(copilot_ctx, run_result, completion_verification=completion_verification)
+ tool_visible_result = _tool_visible_result_after_completion_verification(
+ copilot_ctx,
+ run_result,
+ completion_verification,
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, tool_visible_result)
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=tool_visible_result,
+ workflow_updated=True,
+ )
+ enqueue_screenshot_from_result(copilot_ctx, run_result)
+ if run_result.get("ok"):
+ advance_to_testing(copilot_ctx)
+
+ sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", tool_visible_result)
+ return json.dumps(sanitized)
+
+
+@function_tool(name_override="discover_workflow_entrypoint", strict_mode=False)
+async def discover_workflow_entrypoint_tool(
+ ctx: RunContextWrapper,
+ site_or_url: str,
+ intent_hint: str,
+) -> str:
+ """Find the page a new workflow should start at when the user named a site but not the page.
+
+ Use this BEFORE writing blocks when the user named a website (with a URL,
+ a bare domain, or a single brand word) but no specific page. Accepts:
+ a URL with or without scheme (``example.com/login`` is fine), a bare
+ domain (``example.com``), or a single brand word (resolved as
+ ``https://www.<word>.com``). English phrases ("the X website") return
+ ``failure_reason=could_not_resolve_site_name`` — ASK_QUESTION for a URL.
+
+ Returns ``candidate_url`` plus a short ``evidence_trail`` and any
+ ``candidate_form_fields``. Use ``candidate_url`` as the ``url`` value
+ on a ``goto_url`` block. Do NOT paste the evidence into workflow YAML.
+
+ Budget: one successful call per turn, three per chat, eight page hops,
+ sixty seconds. On any ``failure_reason``, ASK_QUESTION for a URL — do not
+ retry. Discovery navigates and reads pages; it will NOT type, click form
+ buttons, run JavaScript, or submit forms.
+ """
+ result = await _discover_workflow_entrypoint_impl(ctx.context, site_or_url, intent_hint)
+ return json.dumps(result)
+
+
+@function_tool(name_override="inspect_page_for_composition", strict_mode=False)
+async def inspect_page_for_composition_tool(
+ ctx: RunContextWrapper,
+ target_url: str,
+) -> str:
+ """Inspect a known page before composing form/search workflow blocks.
+
+ Use this after the entrypoint URL is known and before authoring blocks that
+ fill fields, submit searches, filter results, or expand result rows. It
+ can also inspect the current browser page after a run by passing
+ target_url="current_page"; use that after partial/budgeted runs so you do
+ not replay a search that already advanced the page.
+
+ Returns observed page evidence: current URL, title, navigation targets, form
+ fields with labels and selectors, submit/search controls, result containers,
+ compact visible text excerpts, anti-bot indicators, and bounded visual
+ challenge evidence when DOM evidence shows challenge state. The returned
+ `observation_step` is the side-channel id to pass in `block_observation_refs`
+ when a newly authored block acts on this observed page. Do NOT paste the
+ evidence into workflow YAML; use it to ground concise block prompts. If a
+ block run changes pages, inspect the reached page before authoring downstream
+ form/search/result blocks. If the
+ evidence shows required fields or controls that the user did not supply
+ enough information for, ASK_QUESTION with that observed missing input. If
+ evidence is sufficient, compose and run workflow blocks from the observed fields.
+ If challenge_state.gates_submit_controls is true, treat challenge resolution
+ as a prerequisite for submit/search; do not click a submit control while the
+ latest inspected evidence says it is disabled. If a later test still leaves
+ that submit/search control disabled after a challenge-resolution attempt,
+ report the observed anti-bot blocker rather than retrying the same flow.
+ """
+ result = await _inspect_page_for_composition_impl(ctx.context, target_url)
+ return json.dumps(result)
+
+
+NATIVE_TOOLS = [
+ update_workflow_tool,
+ list_credentials_tool,
+ run_blocks_tool,
+ get_run_results_tool,
+ update_and_run_blocks_tool,
+ discover_workflow_entrypoint_tool,
+ inspect_page_for_composition_tool,
+]
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
new file mode 100644
index 000000000..0c9b43a19
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -0,0 +1,602 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urlparse
+
+import structlog
+import yaml
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_STRIPPED_HTML_EXPRESSION as _COMPOSITION_STRIPPED_HTML_EXPRESSION,
+)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.schemas.workflows import BlockType
+from skyvern.utils.yaml_loader import safe_load_no_dates
+
+LOG = structlog.get_logger()
+
+
+_FAILED_BLOCK_STATUSES: frozenset[str] = frozenset(
+ {
+ WorkflowRunStatus.failed.value,
+ WorkflowRunStatus.terminated.value,
+ WorkflowRunStatus.canceled.value,
+ WorkflowRunStatus.timed_out.value,
+ }
+)
+
+
+_DATA_PRODUCING_BLOCK_TYPES = frozenset({"EXTRACTION", "TEXT_PROMPT"})
+
+
+# Block types whose output can demonstrate an end-state outcome. Until a workflow
+# contains one of these, an unmet outcome criterion means the build is still
+# incomplete (no confirmation step yet), not a completed run that failed the goal.
+_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})
+
+
+# Block types whose ``block.output`` is a ``TaskOutput.from_task()`` envelope
+# (schemas/tasks.py:TaskOutput) rather than the raw payload. The
+# meaningful-data check must unwrap these via ``_block_data_payload`` before
+# judging output, because envelope fields (task_id, status, artifact IDs) are
+# always populated on a completed run and would otherwise mask empty
+# extractions. This is a subset of ``_DATA_PRODUCING_BLOCK_TYPES`` — keep the
+# two in sync when adding a new task-backed type. ``TEXT_PROMPT`` is
+# deliberately excluded: its block.output is the raw LLM response dict (see
+# ``TextPromptBlock.execute``), no envelope to strip.
+_TASK_ENVELOPE_BLOCK_TYPES = frozenset({"EXTRACTION"})
+assert _TASK_ENVELOPE_BLOCK_TYPES <= _DATA_PRODUCING_BLOCK_TYPES, (
+ "_TASK_ENVELOPE_BLOCK_TYPES must be a subset of _DATA_PRODUCING_BLOCK_TYPES"
+)
+
+
+# Absolute upper bound on a single ``run_blocks`` tool invocation. Exists only
+# as a last-resort trip wire for runaway loops — progressing runs should never
+# approach this. The OpenAI Agents SDK wraps the tool in
+# ``asyncio.wait_for(..., timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS)``; the
+# inner poll loop leaves a 10 s headroom below this ceiling for orderly
+# cleanup before the SDK cancels.
+RUN_BLOCKS_SAFETY_CEILING_SECONDS = 1200 # 20 min
+
+
+# Per-tool-call budget for active block runs — caps a single tool invocation
+# below the session-level wall clock (``enforcement.TOTAL_TIMEOUT_SECONDS``,
+# 900 s) so a long chain cannot consume the whole budget without giving the
+# copilot a chance to issue a smaller chain. Quiet-block runs keep the longer
+# ``RUN_BLOCKS_SAFETY_CEILING_SECONDS`` above.
+PER_TOOL_CALL_BUDGET_SECONDS = 240
+
+
+# Reserve final-reply room; active block runs shrink their own budget near the deadline.
+COPILOT_FINAL_REPLY_RESERVE_SECONDS = 90
+
+
+def _workflow_definition_as_dict(workflow_definition: Any) -> dict[str, Any]:
+ if workflow_definition is None:
+ return {}
+ if isinstance(workflow_definition, dict):
+ return workflow_definition
+ if hasattr(workflow_definition, "model_dump"):
+ try:
+ dumped = workflow_definition.model_dump(mode="json")
+ except Exception:
+ return {}
+ return dumped if isinstance(dumped, dict) else {}
+ return {}
+
+
+def _workflow_verification_evidence(ctx: AgentContext) -> WorkflowVerificationEvidence:
+ return ctx.workflow_verification_evidence
+
+
+def _run_result_label_list(data: Mapping[str, object], key: str) -> list[str]:
+ values = data.get(key)
+ if not isinstance(values, list):
+ return []
+ return list(dict.fromkeys(str(value).strip() for value in values if str(value).strip()))
+
+
+def _run_result_blocks(data: Mapping[str, object]) -> list[Mapping[str, object]]:
+ blocks = data.get("blocks")
+ return [block for block in blocks if isinstance(block, dict)] if isinstance(blocks, list) else []
+
+
+def _completed_run_block_labels(data: Mapping[str, object]) -> list[str]:
+ labels = [
+ str(block.get("label") or "").strip()
+ for block in _run_result_blocks(data)
+ if _enum_or_string_name(block.get("status")) == WorkflowRunStatus.completed.value
+ ]
+ labels = list(dict.fromkeys(label for label in labels if label))
+ return labels or _run_result_label_list(data, "executed_block_labels")
+
+
+def _failed_run_block_labels(data: Mapping[str, object]) -> list[str]:
+ labels = [
+ str(block.get("label") or "").strip()
+ for block in _run_result_blocks(data)
+ if _enum_or_string_name(block.get("status")) in _FAILED_BLOCK_STATUSES
+ ]
+ labels = list(dict.fromkeys(label for label in labels if label))
+ if labels:
+ return labels
+ frontier = data.get("frontier_start_label")
+ return [frontier] if isinstance(frontier, str) and frontier.strip() else []
+
+
+def _enum_or_string_name(value: Any) -> str:
+ raw = getattr(value, "value", value)
+ if not isinstance(raw, str):
+ raw = getattr(value, "name", raw)
+ return str(raw).strip().lower()
+
+
+def _is_meaningful_extracted_data(extracted: Any) -> bool:
+ """Return True when extracted data contains at least one non-null, non-empty value.
+
+ A dict like ``{"price": None}`` is technically present but carries no signal —
+ treat it the same as no output at all so enforcement can nudge the agent to
+ investigate instead of declaring success.
+ """
+ if extracted is None:
+ return False
+ if isinstance(extracted, (str, bytes)):
+ return bool(extracted)
+ if isinstance(extracted, dict):
+ return any(_is_meaningful_extracted_data(v) for v in extracted.values())
+ if isinstance(extracted, (list, tuple, set)):
+ return any(_is_meaningful_extracted_data(v) for v in extracted)
+ # Numbers, booleans, and other scalars count as meaningful output.
+ return True
+
+
+# Payload fields inside a ``TaskOutput.from_task()`` envelope
+# (schemas/tasks.py:TaskOutput). Only these carry "did the block produce
+# something useful?" signal; the rest (task_id, status, artifact IDs, etc.)
+# are always populated on a completed run and would short-circuit
+# _is_meaningful_extracted_data to True even when nothing useful was produced.
+_TASK_OUTPUT_PAYLOAD_FIELDS: tuple[str, ...] = (
+ "extracted_information",
+ "downloaded_files",
+ "downloaded_file_urls",
+)
+
+
+def _block_data_payload(extracted_data: Any, block_type: str | None) -> Any:
+ """Return the payload view of a block's output for the meaningful-data check.
+
+ For task-envelope block types (``_TASK_ENVELOPE_BLOCK_TYPES``), slice the
+ envelope down to ``_TASK_OUTPUT_PAYLOAD_FIELDS`` so envelope metadata
+ can't mask an empty result. Other data-producing types pass through
+ unchanged — e.g. TEXT_PROMPT's ``block.output`` is the raw LLM response
+ dict (TextPromptBlock.execute records ``output_parameter_value=response``
+ directly), so scoping the unwrap avoids slicing a user-defined
+ json_schema that happens to include an ``extracted_information`` field.
+ """
+ if block_type in _TASK_ENVELOPE_BLOCK_TYPES and isinstance(extracted_data, dict):
+ return {field: extracted_data.get(field) for field in _TASK_OUTPUT_PAYLOAD_FIELDS}
+ return extracted_data
+
+
+BLOCK_RUNNING_TOOLS = frozenset({"run_blocks_and_collect_debug", "update_and_run_blocks"})
+
+
+WORKFLOW_MUTATION_TOOLS = frozenset({"update_workflow", "update_and_run_blocks"})
+
+
+ANSWER_ONLY_CONTEXT_TOOLS = frozenset({"get_run_results"})
+
+
+CREDENTIAL_METADATA_TOOLS = frozenset({"list_credentials"})
+
+
+PAGE_INSPECTION_TOOLS = frozenset({"inspect_page_for_composition", "evaluate", "get_browser_screenshot"})
+
+
+PAGE_SCHEMA_CONTEXT_TOOLS = frozenset({"inspect_page_for_composition"})
+
+
+_CURRENT_PAGE_INSPECTION_TARGETS = frozenset({"", "current", "current_page", "__current_page__"})
+
+
+def _copilot_seconds_remaining(ctx: AgentContext) -> float | None:
+ started_at = getattr(ctx, "copilot_run_start_monotonic", None)
+ if not isinstance(started_at, int | float):
+ return None
+ return TOTAL_TIMEOUT_SECONDS - (time.monotonic() - float(started_at))
+
+
+def _same_page_ignoring_fragment(left: str | None, right: str | None) -> bool:
+ if not left or not right:
+ return False
+ try:
+ left_parsed = urlparse(left)
+ right_parsed = urlparse(right)
+ except Exception as exc:
+ LOG.debug("copilot_same_page_url_parse_failed", left=left, right=right, error=str(exc))
+ return False
+ left_url = left_parsed._replace(fragment="").geturl().rstrip("/")
+ right_url = right_parsed._replace(fragment="").geturl().rstrip("/")
+ return left_url == right_url
+
+
+def _emit_tool_blocker_signal(ctx: AgentContext, signal: CopilotToolBlockerSignal) -> str:
+ return stash_blocker_signal(ctx, signal)
+
+
+def _block_type_name(block: object) -> str:
+ """Lowercase string name of a block's type, for both YAML and runtime blocks."""
+ bt = getattr(block, "block_type", None)
+ if bt is None:
+ return ""
+ name = getattr(bt, "value", None) or getattr(bt, "name", None) or str(bt)
+ return str(name).lower()
+
+
+def _workflow_definition_block_labels(workflow_definition: object | None) -> list[str]:
+ blocks = getattr(workflow_definition, "blocks", None) if workflow_definition else None
+ labels: list[str] = []
+ if not blocks:
+ return labels
+ for block in blocks:
+ label = getattr(block, "label", None)
+ if isinstance(label, str) and label:
+ labels.append(label)
+ return labels
+
+
+def _current_workflow_block_labels(ctx: object) -> list[str]:
+ workflow = getattr(ctx, "last_workflow", None)
+ labels = _workflow_definition_block_labels(getattr(workflow, "workflow_definition", None))
+ if labels:
+ return labels
+ workflow_yaml = getattr(ctx, "last_workflow_yaml", None)
+ if not isinstance(workflow_yaml, str):
+ return []
+ blocks = _parse_workflow_blocks(workflow_yaml)
+ if not blocks:
+ return []
+ yaml_labels: list[str] = []
+ for block in blocks:
+ if isinstance(block, dict):
+ label = _block_label_from_yaml(block)
+ if label:
+ yaml_labels.append(label)
+ return yaml_labels
+
+
+def _current_workflow_has_evidence_block(ctx: object) -> bool:
+ workflow = getattr(ctx, "last_workflow", None)
+ blocks = getattr(getattr(workflow, "workflow_definition", None), "blocks", None)
+ if blocks:
+ return any(_block_type_name(block) in _OUTCOME_EVIDENCE_BLOCK_TYPES for block in blocks)
+ workflow_yaml = getattr(ctx, "last_workflow_yaml", None)
+ if not isinstance(workflow_yaml, str):
+ return False
+ return any(
+ isinstance(block, dict) and _enum_or_string_name(block.get("block_type")) in _OUTCOME_EVIDENCE_BLOCK_TYPES
+ for block in (_parse_workflow_blocks(workflow_yaml) or [])
+ )
+
+
+def _unverified_current_workflow_labels(ctx: object) -> list[str]:
+ labels = _current_workflow_block_labels(ctx)
+ verified = set(getattr(ctx, "verified_prefix_labels", []) or [])
+ return [label for label in labels if label not in verified]
+
+
+def _iter_yaml_blocks(blocks: Any) -> list[dict[str, Any]]:
+ found: list[dict[str, Any]] = []
+ if not isinstance(blocks, list):
+ return found
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ found.append(block)
+ loop_blocks = block.get("loop_blocks")
+ if isinstance(loop_blocks, list):
+ found.extend(_iter_yaml_blocks(loop_blocks))
+ return found
+
+
+def _workflow_yaml_blocks_by_label(workflow_yaml: str | None) -> dict[str, dict[str, Any]]:
+ if not workflow_yaml:
+ return {}
+ try:
+ parsed = safe_load_no_dates(workflow_yaml)
+ except yaml.YAMLError:
+ return {}
+ if not isinstance(parsed, dict):
+ return {}
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, dict):
+ return {}
+ by_label: dict[str, dict[str, Any]] = {}
+ for block in _iter_yaml_blocks(workflow_definition.get("blocks")):
+ label = block.get("label")
+ if isinstance(label, str):
+ by_label[label] = block
+ return by_label
+
+
+def _valid_runtime_anchor_url(value: object) -> str | None:
+ if not isinstance(value, str):
+ return None
+ url = value.strip()
+ if not url or url in {"about:blank", ":"}:
+ return None
+ try:
+ parsed = urlparse(url)
+ except ValueError:
+ return None
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ return None
+ return url
+
+
+async def _fallback_page_info(ctx: AgentContext) -> tuple[str, str]:
+ if not ctx.browser_session_id:
+ return "", ""
+ try:
+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
+ session_id=ctx.browser_session_id,
+ organization_id=ctx.organization_id,
+ )
+ if not browser_state:
+ return "", ""
+ page = await browser_state.get_or_create_page()
+ if page:
+ return page.url, await page.title()
+ except Exception:
+ pass
+ return "", ""
+
+
+def _composition_evidence_page_url(evidence: dict[str, Any] | None) -> str | None:
+ if not isinstance(evidence, dict):
+ return None
+ for key in ("current_url", "inspected_url"):
+ value = evidence.get(key)
+ if isinstance(value, str) and value.strip() and value != "current_page":
+ return value.strip()
+ return None
+
+
+def _proxy_location_trace_value(proxy_location: Any) -> Any:
+ if proxy_location is None:
+ return None
+ if hasattr(proxy_location, "value"):
+ return proxy_location.value
+ if hasattr(proxy_location, "model_dump"):
+ return proxy_location.model_dump(mode="json")
+ return proxy_location
+
+
+def _raw_yaml_proxy_location(workflow_yaml: str) -> tuple[bool, Any]:
+ try:
+ parsed_yaml = safe_load_no_dates(workflow_yaml)
+ except yaml.YAMLError:
+ return False, None
+
+ if not isinstance(parsed_yaml, dict) or "proxy_location" not in parsed_yaml:
+ return False, None
+ return True, _proxy_location_trace_value(parsed_yaml.get("proxy_location"))
+
+
+def _parse_workflow_blocks(yaml_str: str | None) -> list[Any] | None:
+ """Parse ``yaml_str`` and return ``workflow_definition.blocks`` as a list,
+ or ``None`` if the YAML is missing, unparseable, or not in the expected
+ shape. Graceful on every failure so callers can treat ``None`` as 'nothing
+ to compare against.'"""
+ if not yaml_str:
+ return None
+ try:
+ parsed = safe_load_no_dates(yaml_str)
+ except yaml.YAMLError:
+ return None
+ if not isinstance(parsed, dict):
+ return None
+ definition = parsed.get("workflow_definition")
+ if not isinstance(definition, dict):
+ return None
+ blocks = definition.get("blocks")
+ return blocks if isinstance(blocks, list) else None
+
+
+def _block_label_from_yaml(block: dict[str, Any]) -> str | None:
+ label = block.get("label")
+ return label if isinstance(label, str) and label else None
+
+
+_DISCOVERY_ANTI_BOT_PATTERNS = (
+ "just a moment",
+ "captcha",
+ "challenge",
+ "turnstile",
+ "cf-turnstile",
+ "human-verification",
+ "human verification",
+ "verify you are human",
+ "access denied",
+ "are you a robot",
+)
+
+
+# Per-call timeout for each MCP primitive inside the discovery walker. The
+# walker also checks the cumulative 60s wall clock between steps, but without
+# a per-call cap a single hung navigate or get_html could block past the
+# cumulative cap (cumulative is only checked between awaits).
+_DISCOVERY_PER_CALL_TIMEOUT_SECONDS = 20.0
+
+
+async def _discovery_navigate(
+ ctx: CopilotContext,
+ url: str,
+ *,
+ wait_until: str | None = None,
+ timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+) -> dict[str, Any]:
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return {"ok": False, "error": "discovery MCP server not attached to context"}
+ nav_args: dict[str, Any] = {"url": url}
+ cap = timeout_seconds
+ if wait_until:
+ # `load` waits for every resource (analytics/marketing beacons on heavy
+ # commerce pages keep it pending past the cap, so the navigate aborts before
+ # any HTML is captured). `domcontentloaded` returns once the server-rendered
+ # DOM is parsed — the forms/links are already present — and the recapture
+ # loop settles anything still hydrating.
+ nav_args["wait_until"] = wait_until
+ nav_args["timeout"] = int(timeout_seconds * 1000)
+ cap = timeout_seconds + 5
+ try:
+ return await asyncio.wait_for(
+ server.call_internal_tool("skyvern_navigate", nav_args),
+ timeout=cap,
+ )
+ except asyncio.TimeoutError:
+ return {"ok": False, "error": f"skyvern_navigate timed out after {timeout_seconds:g}s"}
+
+
+async def _discovery_get_html(ctx: CopilotContext) -> dict[str, Any]:
+ """Read the full page body. ``skyvern_get_html`` requires a selector arg;
+ pass ``body`` so the walker receives the full document body. Without this
+ the raw MCP call fails validation since the inspection tool has a
+ required positional ``selector``.
+ """
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return {"ok": False, "error": "discovery MCP server not attached to context"}
+ try:
+ return await asyncio.wait_for(
+ server.call_internal_tool("skyvern_get_html", {"selector": "body"}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ return {"ok": False, "error": f"skyvern_get_html timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
+
+
+def _discovery_extract_html_payload(result: dict[str, Any]) -> str:
+ data = result.get("data")
+ if isinstance(data, dict):
+ for key in ("html", "outer_html", "text", "content"):
+ value = data.get(key)
+ if isinstance(value, str):
+ return value
+ return ""
+
+
+def _discovery_extract_current_url(result: dict[str, Any], fallback: str) -> str:
+ data = result.get("data")
+ if isinstance(data, dict):
+ url = data.get("url") or data.get("current_url")
+ if isinstance(url, str) and url:
+ return url
+ return fallback
+
+
+# Large enough for long single-turn reconnaissance: block_observation_refs are
+# chosen after scouting, so this cap must stay above the citation window a turn
+# can realistically compose against. Entries carry bounded parse summaries, not
+# raw HTML or screenshots.
+_MAX_FLOW_EVIDENCE = 64
+
+
+def _append_flow_evidence(copilot_ctx: Any, evidence: dict[str, Any], *, reached_via: str) -> int | None:
+ """Append a typed entry to the bounded flow-evidence trajectory (SKY-10562).
+
+ One entry per scouted page: the page-evidence packet plus how it was reached
+ and whether bounded schema was captured. Feeds the per-acted-page composition
+ gate and the cross-turn observed-page summary; never written into the YAML.
+ """
+ trajectory = getattr(copilot_ctx, "flow_evidence", None)
+ if not isinstance(trajectory, list):
+ return None
+ prior_steps = [entry.get("step") for entry in trajectory if isinstance(entry, dict)]
+ step = (
+ max((value for value in prior_steps if isinstance(value, int) and not isinstance(value, bool)), default=-1) + 1
+ )
+ trajectory.append(
+ {
+ "evidence": evidence,
+ "reached_via": reached_via,
+ "had_bounded_schema": has_bounded_page_schema(evidence),
+ "step": step,
+ }
+ )
+ if len(trajectory) > _MAX_FLOW_EVIDENCE:
+ overflow_entry_count = len(trajectory) - _MAX_FLOW_EVIDENCE
+ LOG.warning(
+ "copilot_flow_evidence_evicted",
+ overflow_entry_count=overflow_entry_count,
+ max_flow_evidence=_MAX_FLOW_EVIDENCE,
+ retained_window_size=_MAX_FLOW_EVIDENCE,
+ latest_step=step,
+ )
+ del trajectory[:-_MAX_FLOW_EVIDENCE]
+ return step
+
+
+async def _composition_get_stripped_html(copilot_ctx: Any) -> tuple[str | None, bool]:
+ """Return (stripped_body_html, truncated). truncated is True when the expression sliced
+ the body at the cap, so the tail (below-fold forms/controls) is missing from the evidence."""
+ server = getattr(copilot_ctx, "discovery_mcp_server", None)
+ if server is None:
+ return None, False
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool("skyvern_evaluate", {"expression": _COMPOSITION_STRIPPED_HTML_EXPRESSION}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return None, False
+ if not isinstance(result, dict) or not result.get("ok"):
+ return None, False
+ value = (result.get("data") or {}).get("result")
+ if not isinstance(value, str):
+ return None, False
+ return value, len(value) >= _COMPOSITION_STRIPPED_HTML_MAX_CHARS
+
+
+async def _composition_get_html(copilot_ctx: Any, *, skip_raw: bool = False) -> tuple[str, str | None, bool, bool]:
+ """Return body HTML for composition parsing, surviving the MCP response size cap.
+
+ `skyvern_get_html("body")` is the fast path, but the shared size cap DROPS the
+ payload (no `html` field, just truncation metadata) when the serialized body
+ exceeds the limit — heavy commerce pages routinely do, so the inspector would
+ parse an empty string and report hollow evidence. On an empty/capped read, fall
+ back to an `evaluate` that returns the body with script/style/svg/etc. stripped
+ and length-bounded; that fits under the cap while preserving the form/link
+ structure. Returns (html, error, truncated, used_stripped): error is set only on
+ a hard read failure; truncated is True when the stripped fallback was sliced at
+ the cap; used_stripped is True when the bounded read was the source (raw skipped
+ or cap-dropped). `skip_raw` goes straight to the stripped read so a caller that
+ has already seen the raw serialization get cap-dropped for this page need not
+ re-issue it.
+ """
+ html_result: dict[str, Any] = {}
+ if not skip_raw:
+ html_result = await _discovery_get_html(copilot_ctx)
+ if html_result.get("ok"):
+ html = _discovery_extract_html_payload(html_result)
+ if html.strip():
+ return html, None, False, False
+ stripped, truncated = await _composition_get_stripped_html(copilot_ctx)
+ if stripped and stripped.strip():
+ return stripped, None, truncated, True
+ error = html_result.get("error")
+ return "", str(error) if error else None, False, True
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
new file mode 100644
index 000000000..764db194f
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, normalize_block_authoring_policy
+from skyvern.forge.sdk.copilot.enforcement import PROBABLE_SITE_BLOCK_STREAK_STOP_AT
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+
+from ._shared import _iter_yaml_blocks, _parse_workflow_blocks
+
+# Block types the copilot must never emit. They delegate the entire goal to
+# a separate agent, which bypasses copilot-level block decomposition and
+# obfuscates issues the copilot should surface/handle directly.
+_COPILOT_BANNED_BLOCK_TYPES: frozenset[str] = frozenset({"task", "task_v2"})
+# Code-only browser mode uses this temporary hard ban to force iteration on
+# code-first Copilot behavior. The GA policy is not settled: it may allow some
+# of these block types again, or make the allow/ban list configurable.
+_COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES: frozenset[str] = _COPILOT_BANNED_BLOCK_TYPES | frozenset(
+ {
+ "action",
+ "browser_task",
+ "extraction",
+ "file_download",
+ "file_upload",
+ "goto_url",
+ "login",
+ "navigation",
+ "print_page",
+ "validation",
+ }
+)
+
+# Shared suffix across every LLM-facing rejection message for banned
+# block emission — the pre-hook (schema-lookup reject) and the post-
+# emission detector both steer the LLM toward the same alternatives.
+_COPILOT_BANNED_BLOCK_ALTERNATIVES = (
+ "Use `navigation` for page actions (filling forms, clicking, multi-step flows), "
+ "`extraction` for data extraction, `validation` for completion checks, "
+ "`login` for authentication, or `goto_url` for pure URL navigation."
+)
+_COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES = (
+ "Browser/page workflow block types are unavailable in code-only browser mode. Use focused `code` "
+ "blocks for durable page or browser-session work."
+)
+_CODE_ONLY_TARGET_EVIDENCE_KEYS = frozenset(
+ {
+ "buttons",
+ "fields",
+ "forms",
+ "inputs",
+ "links",
+ "options",
+ "result",
+ "results",
+ "rows",
+ "selects",
+ "tables",
+ "textareas",
+ "url",
+ }
+)
+_CODE_ONLY_SELECTOR_ACTION_TOOLS = frozenset({"click", "type_text", "select_option", "press_key"})
+
+
+def _copilot_block_authoring_policy(ctx: AgentContext | None) -> BlockAuthoringPolicy:
+ if ctx is None:
+ return BlockAuthoringPolicy.STANDARD
+ return normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None))
+
+
+def _copilot_banned_block_types(ctx: AgentContext | None) -> frozenset[str]:
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return _COPILOT_CODE_ONLY_BROWSER_BANNED_BLOCK_TYPES
+ return _COPILOT_BANNED_BLOCK_TYPES
+
+
+def _copilot_banned_block_alternatives(ctx: AgentContext | None) -> str:
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES
+ return _COPILOT_BANNED_BLOCK_ALTERNATIVES
+
+
+def _banned_block_reject_message(items: list[tuple[str, str]], ctx: AgentContext | None = None) -> str:
+ """Uniform error text for the post-emission reject, sharing the
+ alternatives suffix with the schema pre-hook."""
+ labels = ", ".join(sorted({label for label, _ in items}))
+ types = sorted({block_type for _, block_type in items})
+ types_part = " / ".join(repr(t) for t in types)
+ return (
+ f"Block type {types_part} is not available in the workflow copilot. "
+ f"Offending labels: [{labels}]. "
+ f"{_copilot_banned_block_alternatives(ctx)}"
+ )
+
+
+def _record_banned_block_reject_span(source_tool: str, items: list[tuple[str, str]]) -> None:
+ """Emit the dedicated ``update_workflow_banned_block_reject`` span used
+ by post-rollout logfire trend queries."""
+ with copilot_span(
+ "update_workflow_banned_block_reject",
+ data={
+ "labels": [label for label, _ in items],
+ "block_types": sorted({block_type for _, block_type in items}),
+ "source_tool": source_tool,
+ },
+ ):
+ pass
+
+
+def _collect_banned_block_items(
+ blocks: list[Any],
+ banned_types: frozenset[str] | None = None,
+) -> list[tuple[str, str]]:
+ """Recursively walk ``blocks`` (mirroring
+ :func:`skyvern.forge.sdk.copilot.block_goal_wrapping._wrap_blocks_in_place`)
+ and return ``(label, normalized_block_type)`` for every block whose type is
+ in :data:`_COPILOT_BANNED_BLOCK_TYPES`. Blocks missing ``label`` are
+ skipped — the downstream Pydantic validator surfaces those errors on its
+ own."""
+ active_banned_types = banned_types or _COPILOT_BANNED_BLOCK_TYPES
+ items: list[tuple[str, str]] = []
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ raw_type = block.get("block_type")
+ if isinstance(raw_type, str):
+ normalized = raw_type.strip().lower()
+ raw_normalized = normalize_copilot_block_type_alias(normalized)
+ if normalized in active_banned_types or raw_normalized in active_banned_types:
+ label = block.get("label")
+ if isinstance(label, str):
+ items.append((label, raw_normalized))
+ loop_blocks = block.get("loop_blocks")
+ if isinstance(loop_blocks, list):
+ items.extend(_collect_banned_block_items(loop_blocks, active_banned_types))
+ return items
+
+
+def _detect_new_banned_blocks(
+ submitted_yaml: str,
+ prior_workflow_yaml: str | None,
+ *,
+ banned_types: frozenset[str] | None = None,
+) -> list[tuple[str, str]]:
+ """Return ``[(label, block_type), ...]`` for every banned-type block in
+ ``submitted_yaml`` whose label is NOT present as a banned-type block in
+ ``prior_workflow_yaml``. Pure: no I/O, no logging.
+
+ Recurses into ``for_loop.loop_blocks`` mirroring
+ :func:`skyvern.forge.sdk.copilot.block_goal_wrapping._wrap_blocks_in_place`.
+ Legacy workflows that carry ``task`` / ``task_v2`` blocks under unchanged
+ labels produce an empty list and therefore do not reject.
+
+ Malformed YAML, missing ``workflow_definition``, or a non-list ``blocks``
+ all produce an empty list — the downstream Pydantic validation in
+ ``_process_workflow_yaml`` surfaces the specific parse / shape error on
+ its own path.
+ """
+ submitted_blocks = _parse_workflow_blocks(submitted_yaml)
+ if submitted_blocks is None:
+ return []
+ active_banned_types = banned_types or _COPILOT_BANNED_BLOCK_TYPES
+ submitted_items = _collect_banned_block_items(submitted_blocks, active_banned_types)
+ if not submitted_items:
+ return []
+ prior_blocks = _parse_workflow_blocks(prior_workflow_yaml)
+ prior_labels = {label for label, _ in _collect_banned_block_items(prior_blocks or [], active_banned_types)}
+ return [(label, block_type) for label, block_type in submitted_items if label not in prior_labels]
+
+
+_CHALLENGE_WAIT_PATTERN = re.compile(
+ r"\b(anti[-_\s]?bot|bot[-_\s]?block|captcha|challenge|human[-_\s]?verification|ip[-_\s]?block|waf)\b",
+ re.IGNORECASE,
+)
+
+
+def _has_confirmed_waf_or_site_block(ctx: Any) -> bool:
+ if getattr(ctx, "last_test_anti_bot", None):
+ return True
+ return _get_int_attr(ctx, "probable_site_block_streak_count") >= PROBABLE_SITE_BLOCK_STREAK_STOP_AT
+
+
+def _get_int_attr(ctx: Any, name: str, default: int = 0) -> int:
+ value = getattr(ctx, name, default)
+ return value if isinstance(value, int) else default
+
+
+def _block_challenge_wait_text(block: dict[str, Any]) -> str:
+ values = []
+ for key in ("label", "title", "description", "navigation_goal", "complete_criterion"):
+ value = block.get(key)
+ if isinstance(value, str):
+ values.append(value)
+ return " ".join(values)
+
+
+def _detect_timing_only_challenge_wait_blocks(submitted_yaml: str | None) -> list[str]:
+ submitted_blocks = _parse_workflow_blocks(submitted_yaml)
+ if submitted_blocks is None:
+ return []
+ labels: list[str] = []
+ for block in _iter_yaml_blocks(submitted_blocks):
+ raw_type = block.get("block_type")
+ if not isinstance(raw_type, str) or raw_type.strip().lower() != "wait":
+ continue
+ label = block.get("label")
+ if not isinstance(label, str):
+ continue
+ if _CHALLENGE_WAIT_PATTERN.search(_block_challenge_wait_text(block)):
+ labels.append(label)
+ return labels
+
+
+def _composition_evidence_has_challenge(ctx: AgentContext) -> bool:
+ evidence = getattr(ctx, "composition_page_evidence", None)
+ if not isinstance(evidence, dict):
+ return False
+ if evidence.get("anti_bot_indicators") or evidence.get("challenge_controls"):
+ return True
+ challenge_state = evidence.get("challenge_state")
+ return isinstance(challenge_state, dict) and challenge_state.get("detected") is True
+
+
+def _detect_new_http_request_blocks(submitted_yaml: str | None, prior_workflow_yaml: str | None) -> list[str]:
+ submitted_blocks = _parse_workflow_blocks(submitted_yaml)
+ if submitted_blocks is None:
+ return []
+ prior_blocks = _parse_workflow_blocks(prior_workflow_yaml)
+ prior_labels: set[str] = set()
+ for block in _iter_yaml_blocks(prior_blocks or []):
+ if str(block.get("block_type") or "").strip().lower() != "http_request":
+ continue
+ label = block.get("label")
+ if isinstance(label, str):
+ prior_labels.add(label)
+ labels: list[str] = []
+ for block in _iter_yaml_blocks(submitted_blocks):
+ if str(block.get("block_type") or "").strip().lower() != "http_request":
+ continue
+ label = block.get("label")
+ if isinstance(label, str) and label not in prior_labels:
+ labels.append(label)
+ return labels
+
+
+def _challenge_http_request_reject_message(
+ ctx: AgentContext, submitted_yaml: str | None, prior_workflow_yaml: str | None
+) -> str | None:
+ if not _composition_evidence_has_challenge(ctx):
+ return None
+ labels = _detect_new_http_request_blocks(submitted_yaml, prior_workflow_yaml)
+ if not labels:
+ return None
+ labels_text = ", ".join(sorted(set(labels)))
+ return (
+ "Workflow validation failed: raw http_request blocks are not allowed for a page with observed "
+ "anti-bot or human-verification challenge evidence. "
+ f"Offending labels: [{labels_text}]. "
+ "Use browser workflow blocks grounded in the observed page, include challenge handling only when visible, "
+ "or stop and report the observed challenge blocker if it cannot be completed."
+ )
+
+
+def _timing_only_challenge_wait_reject_message(ctx: Any, submitted_yaml: str | None) -> str | None:
+ if not _has_confirmed_waf_or_site_block(ctx):
+ return None
+ labels = _detect_timing_only_challenge_wait_blocks(submitted_yaml)
+ if not labels:
+ return None
+ labels_text = ", ".join(sorted(set(labels)))
+ return (
+ "Workflow validation failed: timing-only challenge wait blocks are not allowed after confirmed "
+ "anti-bot/WAF or repeated site-block evidence. "
+ f"Offending labels: [{labels_text}]. "
+ "Do not add wait/delay-only blocks for this blocker; use a conditional challenge check that takes a "
+ "real action, try a materially different proxy/source if allowed, or stop and explain the blocker."
+ )
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
new file mode 100644
index 000000000..eee55372d
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -0,0 +1,972 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from typing import Any
+
+import structlog
+
+from skyvern.forge import app
+from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, build_loop_blocker_signal
+from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
+from skyvern.forge.sdk.copilot.failure_tracking import (
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+ PER_TOOL_BUDGET_FAILURE_CATEGORY,
+)
+from skyvern.forge.sdk.copilot.loop_detection import detect_failed_tool_step_loop_for_ctx, detect_tool_loop
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+from skyvern.schemas.workflows import BlockType
+
+from ._shared import (
+ _DATA_PRODUCING_BLOCK_TYPES,
+ _FAILED_BLOCK_STATUSES,
+ BLOCK_RUNNING_TOOLS,
+ COPILOT_FINAL_REPLY_RESERVE_SECONDS,
+ PAGE_INSPECTION_TOOLS,
+ PAGE_SCHEMA_CONTEXT_TOOLS,
+ PER_TOOL_CALL_BUDGET_SECONDS,
+ _block_data_payload,
+ _block_label_from_yaml,
+ _copilot_seconds_remaining,
+ _current_workflow_block_labels,
+ _emit_tool_blocker_signal,
+ _enum_or_string_name,
+ _is_meaningful_extracted_data,
+ _parse_workflow_blocks,
+ _raw_yaml_proxy_location,
+)
+
+LOG = structlog.get_logger()
+
+
+async def _safe_read_workflow_run(
+ workflow_run_id: str,
+ organization_id: str,
+ *,
+ context: str,
+) -> WorkflowRun | None:
+ """Read a workflow_runs row, logging-and-returning-None on failure.
+
+ The ``context`` string distinguishes call sites in logs (e.g.
+ ``"pre-cancel"`` vs ``"post-drain"``) so a failure is attributable to
+ the specific phase of the timeout branch it fired from.
+ """
+ try:
+ return await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Workflow run re-read failed",
+ workflow_run_id=workflow_run_id,
+ context=context,
+ exc_info=True,
+ )
+ return None
+
+
+def _trusted_post_drain_status(run: WorkflowRun | None) -> str | None:
+ """Return the run's status if it is one we can trust after the cancel
+ helper has run; otherwise ``None``.
+
+ ``canceled`` is deliberately rejected because at post-drain read time we
+ can't tell a legitimate ``canceled`` (written by
+ ``_finalize_workflow_run_status`` when a block/user canceled the run)
+ apart from a synthetic ``canceled`` (written by the cancel helper's
+ fallback). Callers that need to distinguish those cases must read the row
+ BEFORE the cancel helper runs.
+ """
+ if run is None:
+ return None
+ if WorkflowRunStatus(run.status).is_final_excluding_canceled():
+ return run.status
+ return None
+
+
+def _active_run_terminal_evidence_detected(result: Mapping[str, object]) -> bool:
+ data_value = result.get("data")
+ data = data_value if isinstance(data_value, Mapping) else {}
+ return data.get("active_run_terminal_evidence_detected") is True
+
+
+def _active_run_terminal_evidence_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
+ evidence = getattr(ctx, "workflow_verification_evidence", None)
+ has_active_terminal_evidence = getattr(
+ ctx, "last_failure_category_top", None
+ ) == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY or bool(
+ getattr(evidence, "active_run_terminal_evidence_detected", False)
+ )
+ if not has_active_terminal_evidence:
+ return None
+
+ run_id = getattr(evidence, "active_run_terminal_evidence_workflow_run_id", None) or getattr(
+ ctx, "last_run_blocks_workflow_run_id", None
+ )
+ location = (
+ getattr(evidence, "page_title", None) or getattr(evidence, "current_url", None) or "the current browser page"
+ )
+ run_detail = f" Workflow run: {run_id}." if isinstance(run_id, str) and run_id else ""
+ agent_steering = (
+ "The prior active workflow run emitted typed ACTIVE_RUN_TERMINAL_EVIDENCE while the browser task "
+ f"was still running.{run_detail} The current page evidence already matched the user's terminal "
+ "browser-state criteria, but the reusable workflow is not verified end-to-end. "
+ f"Do not call {tool_name} again in this turn; reply with a partial-verification/blocker state that "
+ "says the requested browser state was observed, the active run was interrupted before overshoot, "
+ "and the workflow still needs a clean corrected run before it can be offered as tested."
+ )
+ user_facing = (
+ f"I reached the requested browser state on {location} and stopped before continuing, "
+ "but the reusable workflow still needs a clean verification run before it is ready."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+ blocked_tool=tool_name,
+ )
+
+
+def _per_tool_budget_problem_label_set(ctx: Any) -> set[str]:
+ labels = getattr(ctx, "per_tool_budget_problem_block_labels", None)
+ if not isinstance(labels, list):
+ return set()
+ return {label for label in labels if isinstance(label, str) and label}
+
+
+def _record_per_tool_budget_problem_blocks_from_results(copilot_ctx: Any, result: dict[str, Any]) -> None:
+ if getattr(copilot_ctx, "last_failure_category_top", None) != PER_TOOL_BUDGET_FAILURE_CATEGORY:
+ return
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return
+ pending_run_id = getattr(copilot_ctx, "pending_reconciliation_run_id", None)
+ resolved_run_id = data.get("workflow_run_id")
+ if isinstance(pending_run_id, str) and pending_run_id and resolved_run_id != pending_run_id:
+ return
+ blocks = data.get("blocks")
+ if not isinstance(blocks, list):
+ return
+
+ labels = _per_tool_budget_problem_label_set(copilot_ctx)
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ if _enum_or_string_name(block.get("block_type")) != BlockType.NAVIGATION.value:
+ continue
+ if _enum_or_string_name(block.get("status")) not in _FAILED_BLOCK_STATUSES:
+ continue
+ label = block.get("label")
+ if isinstance(label, str) and label:
+ labels.add(label)
+ copilot_ctx.per_tool_budget_problem_block_labels = sorted(labels)
+
+
+def _navigation_labels_in_workflow(workflow: Any) -> set[str]:
+ definition = getattr(workflow, "workflow_definition", None)
+ blocks = getattr(definition, "blocks", None)
+ if not isinstance(blocks, list):
+ return set()
+
+ labels: set[str] = set()
+ for block in blocks:
+ if _enum_or_string_name(getattr(block, "block_type", None)) != BlockType.NAVIGATION.value:
+ continue
+ label = getattr(block, "label", None)
+ if isinstance(label, str) and label:
+ labels.add(label)
+ return labels
+
+
+def _clear_resolved_per_tool_budget_problem_labels(copilot_ctx: Any, workflow: Any) -> None:
+ problem_labels = _per_tool_budget_problem_label_set(copilot_ctx)
+ if not problem_labels:
+ return
+ remaining = sorted(problem_labels & _navigation_labels_in_workflow(workflow))
+ copilot_ctx.per_tool_budget_problem_block_labels = remaining
+
+
+def _requested_block_label_set(arguments: dict[str, Any] | None) -> set[str] | None:
+ if not isinstance(arguments, dict):
+ return None
+ block_labels = arguments.get("block_labels")
+ if not isinstance(block_labels, list):
+ return None
+ return {label for label in block_labels if isinstance(label, str) and label}
+
+
+def _per_tool_budget_problem_rerun_signal(
+ ctx: Any, arguments: dict[str, Any] | None, tool_name: str
+) -> CopilotToolBlockerSignal | None:
+ problem_labels = _per_tool_budget_problem_label_set(ctx)
+ if not problem_labels:
+ return None
+
+ requested_labels = _requested_block_label_set(arguments)
+ blocked_labels = problem_labels if not requested_labels else problem_labels & requested_labels
+ if not blocked_labels:
+ return None
+
+ labels = ", ".join(sorted(blocked_labels))
+ agent_steering = (
+ "The prior PER_TOOL_BUDGET run's get_run_results showed navigation "
+ f"block(s) [{labels}] were canceled or failed while applying page state. "
+ f"Do NOT rerun those block label(s) unchanged with {tool_name}. "
+ "Update the workflow to split or replace the oversized navigation block first, "
+ "use live-page inspection evidence to decide what state is actually missing, "
+ "or run only newly-created smaller block labels that apply one missing constraint at a time."
+ )
+ user_facing = (
+ "The previous run hit the per-step time budget before I could finish. "
+ "I'll continue from the verified browser state instead of retrying blindly."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=PAGE_INSPECTION_TOOLS
+ | frozenset({"get_run_results", "update_workflow", "update_and_run_blocks"}),
+ internal_reason_code="tool_error_per_tool_budget_rerun",
+ blocked_tool=tool_name,
+ )
+
+
+def _workflow_yaml_ordered_labels(workflow_yaml: str | None) -> list[str]:
+ blocks = _parse_workflow_blocks(workflow_yaml)
+ if not blocks:
+ return []
+ labels: list[str] = []
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ label = _block_label_from_yaml(block)
+ if label:
+ labels.append(label)
+ return labels
+
+
+def _post_budget_upstream_replay_signal(
+ ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str
+) -> CopilotToolBlockerSignal | None:
+ problem_labels = _per_tool_budget_problem_label_set(ctx)
+ if not problem_labels:
+ return None
+
+ evidence = getattr(ctx, "composition_page_evidence", None)
+ if not isinstance(evidence, dict) or evidence.get("observed_after_workflow_run") is not True:
+ return None
+
+ requested_labels = _requested_block_label_set(arguments)
+ if not requested_labels:
+ return None
+
+ workflow_yaml = arguments.get("workflow_yaml") if isinstance(arguments, dict) else None
+ ordered_labels = _workflow_yaml_ordered_labels(workflow_yaml if isinstance(workflow_yaml, str) else None)
+ if not ordered_labels:
+ ordered_labels = _current_workflow_block_labels(ctx)
+ if not ordered_labels:
+ return None
+
+ first_problem_index = min(
+ (ordered_labels.index(label) for label in problem_labels if label in ordered_labels),
+ default=None,
+ )
+ if first_problem_index is None:
+ return None
+
+ upstream_requested = [label for label in ordered_labels[:first_problem_index] if label in requested_labels]
+ if not upstream_requested:
+ return None
+
+ upstream = ", ".join(upstream_requested)
+ frontier = ", ".join(sorted(problem_labels))
+ agent_steering = (
+ "The prior PER_TOOL_BUDGET run advanced the live browser, and you already inspected that "
+ "post-run page state. Do NOT restart upstream label(s) "
+ f"[{upstream}] before the budgeted frontier [{frontier}]. "
+ "Answer from the observed page if it contains the requested result/no-result evidence. "
+ "If more workflow testing is still required, preserve the verified upstream blocks and run only "
+ "new or modified smaller labels at/after the missing frontier."
+ )
+ user_facing = (
+ "The previous run already reached a later browser state. I'll continue from that evidence "
+ "instead of restarting earlier steps."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=PAGE_INSPECTION_TOOLS
+ | frozenset({"get_run_results", "update_workflow", "update_and_run_blocks"}),
+ preserves_workflow_draft=True,
+ renders_final_reply=False,
+ internal_reason_code="tool_error_post_budget_upstream_replay",
+ blocked_tool=tool_name,
+ )
+
+
+def _composition_control_label(control: dict[str, Any]) -> str:
+ parts: list[str] = []
+ for key in ("text", "name", "id", "selector"):
+ value = control.get(key)
+ if isinstance(value, str) and value.strip():
+ parts.append(value.strip())
+ unique = list(dict.fromkeys(parts))
+ return " / ".join(unique[:2])[:160] if unique else "submit/search control"
+
+
+def _disabled_submit_controls_from_evidence(evidence: dict[str, Any]) -> list[dict[str, Any]]:
+ controls: list[dict[str, Any]] = []
+ forms = evidence.get("forms")
+ if not isinstance(forms, list):
+ return controls
+ for form in forms:
+ if not isinstance(form, dict):
+ continue
+ for control in form.get("submit_controls") or []:
+ if isinstance(control, dict) and control.get("disabled") is True:
+ controls.append(control)
+ return controls[:5]
+
+
+def _post_run_terminal_challenge_reason(evidence: dict[str, Any]) -> str | None:
+ if evidence.get("observed_after_workflow_run") is not True:
+ return None
+
+ challenge_state = evidence.get("challenge_state")
+ if isinstance(challenge_state, dict):
+ gated_controls = [
+ control for control in challenge_state.get("gated_submit_controls") or [] if isinstance(control, dict)
+ ]
+ if challenge_state.get("gates_submit_controls") is True:
+ kind = str(challenge_state.get("kind") or "anti-bot challenge").replace("_", " ")
+ labels = ", ".join(_composition_control_label(control) for control in gated_controls[:3])
+ controls_text = f" ({labels})" if labels else ""
+ return f"{kind} is still gating disabled submit/search control(s){controls_text}"
+ if challenge_state.get("detected") is True and challenge_state.get("requires_human_verification") is True:
+ disabled_controls = _disabled_submit_controls_from_evidence(evidence)
+ if disabled_controls:
+ labels = ", ".join(_composition_control_label(control) for control in disabled_controls[:3])
+ return f"human-verification challenge remains while submit/search control(s) are disabled ({labels})"
+
+ indicators = evidence.get("anti_bot_indicators")
+ has_indicators = isinstance(indicators, list) and any(isinstance(item, str) and item.strip() for item in indicators)
+ if has_indicators:
+ disabled_controls = _disabled_submit_controls_from_evidence(evidence)
+ if disabled_controls:
+ labels = ", ".join(_composition_control_label(control) for control in disabled_controls[:3])
+ return f"anti-bot evidence remains while submit/search control(s) are disabled ({labels})"
+ return None
+
+
+def _post_budget_terminal_challenge_signal(
+ ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str
+) -> CopilotToolBlockerSignal | None:
+ evidence = getattr(ctx, "composition_page_evidence", None)
+ if not isinstance(evidence, dict):
+ return None
+
+ # A failed run with post-run page evidence can prove the same terminal
+ # challenge state even when the top-level failure category was not budget.
+ prior_budget_or_failed_run = (
+ getattr(ctx, "last_failure_category_top", None) == PER_TOOL_BUDGET_FAILURE_CATEGORY
+ or getattr(ctx, "last_test_ok", None) is False
+ or getattr(ctx, "post_run_page_observation_after_failed_test", False) is True
+ or bool(_per_tool_budget_problem_label_set(ctx))
+ )
+ if not prior_budget_or_failed_run:
+ return None
+
+ reason = _post_run_terminal_challenge_reason(evidence)
+ if not reason:
+ return None
+
+ requested_labels = _requested_block_label_set(arguments)
+ labels_text = f" Requested labels: {', '.join(sorted(requested_labels))}." if requested_labels else ""
+ agent_steering = (
+ "The prior block-running tool hit a failed/budgeted frontier, and bounded current-page inspection "
+ f"now shows: {reason}.{labels_text} Do NOT call "
+ f"{tool_name} again in this turn, do NOT try another proxy/location from this evidence state, and "
+ "do NOT claim registry results or no-results were verified. REPLY now with a blocker explanation "
+ "that names the observed challenge/disabled control and summarizes the tested workflow state."
+ )
+ user_facing = (
+ "The site's verification challenge is still blocking the submit/search control after live-page inspection, "
+ "so I stopped without claiming results."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code="tool_error_post_budget_challenge_blocker",
+ blocked_tool=tool_name,
+ )
+
+
+_RECONCILIATION_REQUIRES_INPUT_USER_FACING = (
+ "The previous run was canceled. Tell me whether to retry, keep the draft as-is, or adjust the workflow first."
+)
+_RECONCILIATION_NO_INPUT_USER_FACING = (
+ "The previous run ended without a verified result. I'll check what happened before doing anything else this turn."
+)
+
+
+def _pending_reconciliation_requires_input_signal(
+ *, pending_run_id: str, blocked_tool: str
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ f"The canceled run {pending_run_id} has already been inspected. "
+ f"Do NOT run more blocks in this turn; ask the user whether to retry, "
+ f"accept the unverified draft, or adjust the workflow first. This guard "
+ f"prevents duplicate side effects on live sites."
+ ),
+ user_facing_reason=_RECONCILIATION_REQUIRES_INPUT_USER_FACING,
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_pending_reconciliation_requires_input",
+ blocked_tool=blocked_tool,
+ )
+
+
+def _pending_reconciliation_no_input_signal(*, pending_run_id: str, blocked_tool: str) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=(
+ f"The previous block-running tool call for run {pending_run_id} "
+ f"ended without a trustworthy terminal status. "
+ f'Call `get_run_results(workflow_run_id="{pending_run_id}")` '
+ f"first, report the result to the user, then await user input "
+ f"before running more blocks. This guard prevents duplicate "
+ f"side effects on live sites."
+ ),
+ user_facing_reason=_RECONCILIATION_NO_INPUT_USER_FACING,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_pending_reconciliation_no_input",
+ blocked_tool=blocked_tool,
+ )
+
+
+# Streak threshold at which the copilot hard-aborts a tool call because the
+# same action sequence has repeated run-over-run with no intervening success.
+# The streak counter is incremented in ``update_repeated_failure_state`` AFTER
+# each run, so the abort fires when the 4th consecutive run against the same
+# action fingerprint enters ``_tool_loop_error`` (streak == 3 at entry, one
+# per each of the three preceding identical runs). Calibration note: the
+# repeated-frontier streak in failure_tracking.py uses STOP_AT=3 for the same
+# shape of escalation.
+REPEATED_ACTION_STREAK_ABORT_AT = 3
+MAX_CHALLENGE_GATED_PROXY_RETRIES = 1
+
+_STRUCTURED_BLOCKER_KEY_TERMS: frozenset[str] = frozenset(
+ {
+ "blocker",
+ "blocked",
+ "captcha",
+ "challenge",
+ "human_verification",
+ "verification",
+ }
+)
+_STRUCTURED_BLOCKER_MESSAGE_KEYS: frozenset[str] = frozenset(
+ {
+ "blocker_message",
+ "blocked_message",
+ "captcha_message",
+ "challenge_message",
+ "human_verification_message",
+ }
+)
+_ANTI_BOT_BLOCKER_TERMS: tuple[str, ...] = (
+ "access denied",
+ "anti-bot",
+ "bot block",
+ "captcha",
+ "challenge",
+ "human verification",
+ "verify you are human",
+)
+
+
+def _normalize_structured_key(value: object) -> str:
+ return str(value or "").strip().lower().replace("-", "_").replace(" ", "_")
+
+
+def _looks_like_anti_bot_blocker(text: str) -> bool:
+ lowered = text.lower()
+ return any(term in lowered for term in _ANTI_BOT_BLOCKER_TERMS)
+
+
+def _structured_blocker_message(value: object, *, depth: int = 0) -> str | None:
+ if depth > 5:
+ return None
+ if isinstance(value, dict):
+ for key, item in value.items():
+ normalized_key = _normalize_structured_key(key)
+ if not isinstance(item, str) or not item.strip():
+ continue
+ has_blocker_key = normalized_key in _STRUCTURED_BLOCKER_MESSAGE_KEYS or any(
+ term in normalized_key for term in _STRUCTURED_BLOCKER_KEY_TERMS
+ )
+ if has_blocker_key or (
+ normalized_key in {"message", "error", "failure_reason", "reason"}
+ and _looks_like_anti_bot_blocker(item)
+ ):
+ return item.strip()[:240]
+ for item in value.values():
+ nested = _structured_blocker_message(item, depth=depth + 1)
+ if nested:
+ return nested
+ elif isinstance(value, list):
+ for item in value:
+ nested = _structured_blocker_message(item, depth=depth + 1)
+ if nested:
+ return nested
+ return None
+
+
+def _run_blocks_structured_blocker_message(result: dict[str, Any]) -> str | None:
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return None
+ direct = _structured_blocker_message({key: value for key, value in data.items() if key != "blocks"})
+ if direct:
+ return direct
+ blocks = data.get("blocks")
+ if not isinstance(blocks, list):
+ return None
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ block_type = block.get("block_type")
+ if block_type not in _DATA_PRODUCING_BLOCK_TYPES or block.get("status") != "completed":
+ continue
+ payload = _block_data_payload(block.get("extracted_data"), block_type)
+ blocker = _structured_blocker_message(payload)
+ if blocker:
+ return blocker
+ return None
+
+
+def _active_block_run_budget_seconds(ctx: AgentContext) -> int:
+ remaining = _copilot_seconds_remaining(ctx)
+ if remaining is None:
+ return PER_TOOL_CALL_BUDGET_SECONDS
+ remaining_after_reply_reserve = remaining - COPILOT_FINAL_REPLY_RESERVE_SECONDS
+ return max(1, min(PER_TOOL_CALL_BUDGET_SECONDS, int(remaining_after_reply_reserve)))
+
+
+def _late_block_running_call_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
+ remaining = _copilot_seconds_remaining(ctx)
+ if remaining is None or remaining > COPILOT_FINAL_REPLY_RESERVE_SECONDS:
+ return None
+
+ last_failed_workflow_yaml = getattr(ctx, "last_failed_workflow_yaml", None)
+ last_good_workflow_yaml = getattr(ctx, "last_good_workflow_yaml", None)
+ if (
+ isinstance(last_failed_workflow_yaml, str)
+ and last_failed_workflow_yaml
+ and isinstance(last_good_workflow_yaml, str)
+ and last_good_workflow_yaml
+ ):
+ agent_steering = (
+ f"Wall-clock budget too low to retry: about {int(max(0.0, remaining))}s remain of the "
+ f"{TOTAL_TIMEOUT_SECONDS}s session budget. A verified workflow exists from before the failure. "
+ "Do NOT call update_and_run_blocks or run_blocks_and_collect_debug again. REPLY now: summarize "
+ "what worked, name the block that failed, and tell the user they can keep the verified prefix or discard."
+ )
+ elif isinstance(last_failed_workflow_yaml, str) and last_failed_workflow_yaml:
+ agent_steering = (
+ f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn "
+ "after the previous workflow run failed. Do NOT retry block-running tools. Use only existing "
+ "run evidence and quick browser inspection tools such as get_run_results, evaluate, or "
+ "get_browser_screenshot if one more read is needed. If the current page contains the requested "
+ "answer, answer from that observed page evidence. If evidence is incomplete, report exactly "
+ "which browser state was verified and which requested data remains unverified. Never repeat "
+ "this tool-error text as the user-facing answer."
+ )
+ else:
+ agent_steering = (
+ f"Less than {COPILOT_FINAL_REPLY_RESERVE_SECONDS} seconds remain in this Copilot turn. "
+ "Do NOT start another block-running tool call; reply to the user with the workflow draft and "
+ "progress gathered so far, and make clear which parts have not been verified end-to-end."
+ )
+
+ user_facing = "I'm running out of time on this turn. I'll wrap up with what I have so far."
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="stop",
+ cleared_by_tools=frozenset(),
+ # The "wrap up with what we have" semantic means the draft saved
+ # earlier in the turn should still surface — only the chat reply
+ # is overridden by the renderer.
+ preserves_workflow_draft=True,
+ renders_final_reply=False,
+ internal_reason_code="tool_error_late_block_running",
+ blocked_tool=tool_name,
+ )
+
+
+def _allows_post_run_current_page_inspection_budget_bypass(ctx: AgentContext, *, use_current_page: bool) -> bool:
+ if not use_current_page:
+ return False
+ run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
+ if not isinstance(run_id, str) or not run_id:
+ return False
+ if getattr(ctx, "last_test_ok", None) is None:
+ return False
+ return getattr(ctx, "post_run_current_page_inspection_workflow_run_id", None) != run_id
+
+
+def _post_budget_page_inspection_signal(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
+ if getattr(ctx, "post_budget_page_inspection_required", False) is not True:
+ return None
+
+ url = getattr(ctx, "post_budget_page_inspection_url", None)
+ run_id = getattr(ctx, "post_budget_page_inspection_run_id", None)
+ url_text = f" at {url}" if isinstance(url, str) and url else ""
+ run_text = f" for run {run_id}" if isinstance(run_id, str) and run_id else ""
+ agent_steering = (
+ f"The prior PER_TOOL_BUDGET run{run_text} advanced the live browser{url_text}. "
+ "Before another block-running tool, inspect the current browser page with "
+ 'inspect_page_for_composition(target_url="current_page"). Generic screenshot/evaluate reads can '
+ "help answer the user, but they do not satisfy the bounded page-evidence contract for workflow "
+ "mutations. If the observed page evidence already contains the requested result or a no-results "
+ "state, answer from that evidence instead of rerunning the search. If evidence shows a missing "
+ "page-state change, then run only the smaller missing block after that inspection."
+ )
+ user_facing = "I need to inspect the current page state before running more steps."
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=PAGE_SCHEMA_CONTEXT_TOOLS,
+ preserves_workflow_draft=True,
+ renders_final_reply=False,
+ internal_reason_code="tool_error_post_budget_page_inspection_required",
+ blocked_tool=tool_name,
+ )
+
+
+def _proxy_value_signature(value: object) -> str:
+ try:
+ return json.dumps(value, sort_keys=True, default=str)
+ except (TypeError, ValueError):
+ return str(value)
+
+
+def _anti_bot_retry_changes_proxy(ctx: AgentContext, arguments: dict[str, Any] | None) -> bool:
+ if not isinstance(arguments, dict):
+ return False
+ workflow_yaml = arguments.get("workflow_yaml")
+ if not isinstance(workflow_yaml, str) or not workflow_yaml.strip():
+ return False
+
+ prior_yaml = getattr(ctx, "last_failed_workflow_yaml", None) or getattr(ctx, "workflow_yaml", None)
+ if not isinstance(prior_yaml, str) or not prior_yaml.strip():
+ return False
+
+ new_proxy_present, new_proxy = _raw_yaml_proxy_location(workflow_yaml)
+ if not new_proxy_present:
+ return False
+ old_proxy_present, old_proxy = _raw_yaml_proxy_location(prior_yaml)
+ return (not old_proxy_present) or _proxy_value_signature(new_proxy) != _proxy_value_signature(old_proxy)
+
+
+def _challenge_gated_proxy_retry_allowed(ctx: AgentContext, arguments: dict[str, Any] | None) -> bool:
+ if not _anti_bot_retry_changes_proxy(ctx, arguments):
+ return False
+ retry_count = getattr(ctx, "challenge_gated_proxy_retry_count", 0)
+ if not isinstance(retry_count, int):
+ retry_count = 0
+ if retry_count >= MAX_CHALLENGE_GATED_PROXY_RETRIES:
+ return False
+ ctx.challenge_gated_proxy_retry_count = retry_count + 1
+ return True
+
+
+def _last_run_has_terminal_anti_bot_blocker(ctx: AgentContext) -> bool:
+ anti_bot_reason = getattr(ctx, "last_test_anti_bot", None)
+ if not isinstance(anti_bot_reason, str) or not anti_bot_reason.strip():
+ return False
+ failure_reason = getattr(ctx, "last_test_failure_reason", None)
+ if not isinstance(failure_reason, str) or not failure_reason.strip():
+ return False
+ lowered = failure_reason.lower()
+ if "blocker" in lowered and _looks_like_anti_bot_blocker(lowered):
+ return True
+
+ evidence = getattr(ctx, "composition_page_evidence", None)
+ challenge_state = evidence.get("challenge_state") if isinstance(evidence, dict) else None
+ challenge_gates_submit = isinstance(challenge_state, dict) and challenge_state.get("gates_submit_controls") is True
+ if not (challenge_gates_submit or "challenge-gated disabled submit/search control" in anti_bot_reason):
+ return False
+
+ return "disabled" in lowered and any(
+ term in lowered for term in ("submit", "search", "button", "control", "element")
+ )
+
+
+def _challenge_gated_anti_bot_rerun_signal(
+ ctx: AgentContext,
+ arguments: dict[str, Any] | None,
+ tool_name: str,
+) -> CopilotToolBlockerSignal | None:
+ if not _last_run_has_terminal_anti_bot_blocker(ctx):
+ return None
+ if tool_name == "update_and_run_blocks" and _challenge_gated_proxy_retry_allowed(ctx, arguments):
+ return None
+
+ failure_reason = getattr(ctx, "last_test_failure_reason", "")
+ agent_steering = (
+ "The prior run confirmed an anti-bot challenge or blocker on the submit/search path, "
+ f"and the latest failure_reason was: {str(failure_reason)[:240]}. Do NOT call "
+ f"{tool_name} again with the same workflow/browser path. REPLY now with a blocker "
+ "explanation that names the observed challenge/blocker or disabled submit/search control "
+ "and describes what was tried. "
+ "Ask whether to try a materially different proxy/location, entrypoint, or alternate source."
+ )
+ user_facing = (
+ "The site's verification challenge is still keeping the submit/search control disabled, so I stopped "
+ "instead of retrying the same workflow path."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code="tool_error_challenge_gated_submit_disabled",
+ blocked_tool=tool_name,
+ )
+
+
+def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any] | None = None) -> str | None:
+ detected = detect_failed_tool_step_loop_for_ctx(ctx, tool_name, arguments or {})
+ if detected is not None:
+ return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
+
+ # Consecutive same-name guard: false-positives on the intended iterative
+ # build (one new block per update_and_run_blocks). Block-running tools
+ # rely on the progress-aware checks below instead.
+ tracker = getattr(ctx, "consecutive_tool_tracker", None)
+ if isinstance(tracker, list) and tool_name not in BLOCK_RUNNING_TOOLS:
+ detected = detect_tool_loop(tracker, tool_name)
+ if detected is not None:
+ return _emit_tool_blocker_signal(ctx, _build_loop_blocker_signal(detected, tool_name=tool_name))
+
+ if tool_name == "update_workflow" or tool_name in BLOCK_RUNNING_TOOLS:
+ active_terminal_signal = _active_run_terminal_evidence_signal(ctx, tool_name)
+ if active_terminal_signal is not None:
+ return _emit_tool_blocker_signal(ctx, active_terminal_signal)
+
+ # Hard-abort when the agent has re-fired the same action sequence against
+ # the page N times without intervening success. This is the signal that
+ # the form is blocked (captcha / anti-bot / error banner the agent isn't
+ # detecting) and further attempts will just burn the tool timeout. Scoped
+ # to block-running tools so planning/metadata tools (update_workflow,
+ # list_credentials, get_run_results) stay unaffected.
+ if tool_name in BLOCK_RUNNING_TOOLS:
+ # Reconciliation guard: the previous block-running tool call exited
+ # without a trustworthy terminal status for its workflow run (the
+ # watchdog's stagnation / ceiling / task_exit_unfinalized paths, or
+ # the SKY-9167 post-drain branch where the row read as ``canceled``,
+ # non-final, or unreadable). Block further block-running calls until
+ # ``get_run_results`` clears the flag — prevents the LLM from
+ # auto-retrying a mutation block whose side effects may already
+ # have landed.
+ pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
+ if isinstance(pending_run_id, str) and pending_run_id:
+ if getattr(ctx, "pending_reconciliation_requires_user_input", False) is True:
+ return _emit_tool_blocker_signal(
+ ctx,
+ _pending_reconciliation_requires_input_signal(
+ pending_run_id=pending_run_id, blocked_tool=tool_name
+ ),
+ )
+ return _emit_tool_blocker_signal(
+ ctx,
+ _pending_reconciliation_no_input_signal(pending_run_id=pending_run_id, blocked_tool=tool_name),
+ )
+
+ inspection_signal = _post_budget_page_inspection_signal(ctx, tool_name)
+ if inspection_signal is not None:
+ return _emit_tool_blocker_signal(ctx, inspection_signal)
+
+ # Terminal anti-bot evidence should produce the final user-facing reply
+ # before the generic budget rerun path can ask for another attempt.
+ post_budget_challenge_signal = _post_budget_terminal_challenge_signal(ctx, arguments, tool_name)
+ if post_budget_challenge_signal is not None:
+ return _emit_tool_blocker_signal(ctx, post_budget_challenge_signal)
+
+ budget_signal = _per_tool_budget_problem_rerun_signal(ctx, arguments, tool_name)
+ if budget_signal is not None:
+ return _emit_tool_blocker_signal(ctx, budget_signal)
+
+ upstream_replay_signal = _post_budget_upstream_replay_signal(ctx, arguments, tool_name)
+ if upstream_replay_signal is not None:
+ return _emit_tool_blocker_signal(ctx, upstream_replay_signal)
+
+ challenge_signal = _challenge_gated_anti_bot_rerun_signal(ctx, arguments, tool_name)
+ if challenge_signal is not None:
+ return _emit_tool_blocker_signal(ctx, challenge_signal)
+
+ streak_raw = getattr(ctx, "repeated_action_fingerprint_streak_count", 0)
+ streak = streak_raw if isinstance(streak_raw, int) else 0
+ if streak >= REPEATED_ACTION_STREAK_ABORT_AT:
+ agent_steering = (
+ f"Repeated-action abort: the last {streak} runs fired the same "
+ "action sequence against the page without making progress. "
+ "The site is likely blocked by a captcha, popup, anti-bot "
+ "challenge, or hidden validation error that the agent is not "
+ "detecting. Do NOT retry this tool — conclude the workflow is "
+ "not automatable as-is and report back to the user."
+ )
+ user_facing = (
+ "I tried the same actions a few times without making progress. The site looks blocked. I'll stop here."
+ )
+ return _emit_tool_blocker_signal(
+ ctx,
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="stop",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_repeated_action_abort",
+ blocked_tool=tool_name,
+ ),
+ )
+
+ # Within-turn fail-fast for permanent navigation errors (DNS / cert /
+ # SSL / invalid URL). The enforcement-loop stop nudge only runs
+ # BETWEEN agent turns, so without this check the LLM is free to make
+ # speculative within-turn retries (e.g. drop the `www` subdomain and
+ # try again) before the nudge fires. update_and_run_blocks internally
+ # calls _update_workflow which clears the flag, so this check must
+ # run before that — hence at the tool entrypoint, not inside the run
+ # body.
+ prior_nav_error = getattr(ctx, "last_test_non_retriable_nav_error", None)
+ if isinstance(prior_nav_error, str) and prior_nav_error:
+ agent_steering = (
+ f"Prior run in this turn hit a permanent navigation error "
+ f"({prior_nav_error[:200]}). Do NOT retry — the URL is unreachable "
+ "regardless of subdomain or path variations. Reply to the user "
+ "explaining the failure and asking them to verify the URL."
+ )
+ user_facing = "The URL I tried isn't reachable. Tell me the correct address and I'll try again."
+ return _emit_tool_blocker_signal(
+ ctx,
+ CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint="ask_user_clarifying",
+ cleared_by_tools=frozenset(),
+ internal_reason_code="tool_error_non_retriable_nav",
+ blocked_tool=tool_name,
+ ),
+ )
+
+ late_signal = _late_block_running_call_signal(ctx, tool_name)
+ if late_signal is not None:
+ return _emit_tool_blocker_signal(ctx, late_signal)
+ return None
+
+
+_build_loop_blocker_signal = build_loop_blocker_signal
+
+
+def _analyze_run_blocks(result: dict[str, Any]) -> tuple[str | None, bool, list[dict] | None]:
+ """Single-pass analysis of run result blocks.
+
+ Returns ``(anti_bot_match, has_empty_data_blocks, failure_categories)``
+ by iterating the block list once. Classification delegates to
+ :func:`~skyvern.forge.failure_classifier.classify_from_failure_reason`.
+ When ``data["failure_categories"]`` is already populated (pre-run
+ short-circuit with no blocks), honor it instead of re-classifying.
+ """
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return None, False, None
+
+ anti_bot_match: str | None = None
+
+ precomputed_categories = data.get("failure_categories")
+ if isinstance(precomputed_categories, list) and precomputed_categories:
+ for cat in precomputed_categories:
+ if isinstance(cat, dict) and cat.get("category") == "ANTI_BOT_DETECTION":
+ anti_bot_match = cat.get("reasoning", "anti-bot pattern detected")
+ break
+ return anti_bot_match, False, precomputed_categories
+
+ # Collect texts for scanning and data-block stats in one pass
+ texts_to_scan: list[str] = []
+ error = result.get("error")
+ if isinstance(error, str):
+ texts_to_scan.append(error)
+ html = data.get("visible_elements_html")
+ if isinstance(html, str):
+ texts_to_scan.append(html)
+ failure_reason = data.get("failure_reason")
+ if isinstance(failure_reason, str):
+ texts_to_scan.append(failure_reason)
+ page_title = data.get("page_title")
+ if isinstance(page_title, str):
+ texts_to_scan.append(page_title)
+ action_trace_summary = data.get("action_trace_summary")
+ if isinstance(action_trace_summary, list):
+ texts_to_scan.extend(str(item) for item in action_trace_summary if isinstance(item, str))
+
+ has_data_blocks = False
+ any_data_output = False
+
+ blocks = data.get("blocks")
+ if isinstance(blocks, list):
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ reason = block.get("failure_reason")
+ if isinstance(reason, str):
+ texts_to_scan.append(reason)
+ block_type = block.get("block_type")
+ if block_type in _DATA_PRODUCING_BLOCK_TYPES and block.get("status") == "completed":
+ has_data_blocks = True
+ payload = _block_data_payload(block.get("extracted_data"), block_type)
+ structured_blocker = _structured_blocker_message(payload)
+ if structured_blocker:
+ texts_to_scan.append(structured_blocker)
+ if _is_meaningful_extracted_data(payload):
+ any_data_output = True
+
+ combined = "\n".join(texts_to_scan)
+ categories = classify_from_failure_reason(combined)
+ if categories:
+ for cat in categories:
+ if cat.get("category") == "ANTI_BOT_DETECTION":
+ anti_bot_match = cat.get("reasoning", "anti-bot pattern detected")
+ break
+
+ empty_data_blocks = has_data_blocks and not any_data_output
+ return anti_bot_match, empty_data_blocks, categories
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
new file mode 100644
index 000000000..23922187e
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -0,0 +1,361 @@
+import time
+from typing import Any
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ RunEvidenceSnapshot,
+ evaluate_completion_criteria,
+ summarize_unsatisfied_outcomes,
+)
+from skyvern.forge.sdk.copilot.enforcement import _goal_likely_needs_more_blocks
+from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
+from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+
+from ._shared import (
+ RUN_BLOCKS_SAFETY_CEILING_SECONDS,
+ _copilot_seconds_remaining,
+ _current_workflow_block_labels,
+ _current_workflow_has_evidence_block,
+ _is_meaningful_extracted_data,
+ _valid_runtime_anchor_url,
+)
+from .blockers import (
+ _active_run_terminal_evidence_detected,
+ _analyze_run_blocks,
+ _run_blocks_structured_blocker_message,
+)
+
+LOG = structlog.get_logger()
+
+
+def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:
+ policy = getattr(copilot_ctx, "request_policy", None)
+ # A method-mandated criterion asserts HOW the goal was reached; the outcome
+ # judge sees only end-state evidence.
+ return [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]
+
+
+def _verified_context_block_labels_for_snapshot(
+ copilot_ctx: Any, current_labels: list[str], executed_labels: list[str]
+) -> list[str]:
+ current_label_set = set(current_labels)
+ executed_label_set = set(executed_labels)
+ candidates: set[str] = set()
+ for raw_values in (
+ getattr(copilot_ctx, "verified_prefix_labels", None),
+ getattr(getattr(copilot_ctx, "workflow_verification_evidence", None), "block_verified", None),
+ ):
+ if not isinstance(raw_values, list):
+ continue
+ candidates.update(str(label) for label in raw_values if isinstance(label, str) and label in current_label_set)
+ return [label for label in current_labels if label in candidates and label not in executed_label_set]
+
+
+def _build_page_observation_evidence_snapshot(
+ copilot_ctx: Any,
+ *,
+ url: str,
+ title: str = "",
+ observed_data: object | None = None,
+) -> RunEvidenceSnapshot:
+ run_id = getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)
+ block_outputs: dict[str, Any] = {}
+ if isinstance(observed_data, dict) and observed_data:
+ block_outputs["current_page_observation"] = observed_data
+ elif observed_data is not None:
+ block_outputs["current_page_observation"] = str(observed_data)
+ return RunEvidenceSnapshot(
+ workflow_run_id=run_id if isinstance(run_id, str) else None,
+ block_outputs=block_outputs,
+ current_url=_valid_runtime_anchor_url(url),
+ page_title=title if isinstance(title, str) and title.strip() else None,
+ )
+
+
+async def _maybe_run_completion_verification_from_page_observation(
+ copilot_ctx: Any,
+ *,
+ url: str,
+ title: str = "",
+ observed_data: object | None = None,
+) -> CompletionVerificationResult | None:
+ """Verify completion only for post-run page observations after failed tests."""
+
+ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
+ return None
+ existing = getattr(copilot_ctx, "completion_verification_result", None)
+ if isinstance(existing, CompletionVerificationResult) and existing.is_fully_satisfied():
+ return existing
+ if getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False) is not True:
+ return None
+ criteria = _completion_verification_criteria(copilot_ctx)
+ if not criteria:
+ return None
+ handler = await _completion_verification_handler(copilot_ctx)
+ if handler is None:
+ return None
+ remaining = _copilot_seconds_remaining(copilot_ctx)
+ if (
+ remaining is not None
+ and remaining
+ <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS
+ ):
+ verification = CompletionVerificationResult(status="unavailable")
+ else:
+ snapshot = _build_page_observation_evidence_snapshot(
+ copilot_ctx,
+ url=url,
+ title=title,
+ observed_data=observed_data,
+ )
+ if not snapshot.has_evidence():
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[criterion.id for criterion in criteria],
+ verdicts=[
+ CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")
+ for criterion in criteria
+ ],
+ )
+ else:
+ verification = await evaluate_completion_criteria(criteria, snapshot, handler)
+
+ if (
+ isinstance(existing, CompletionVerificationResult)
+ and not verification.is_fully_satisfied()
+ and not (existing.status == "unavailable" and verification.status == "evaluated")
+ ):
+ return existing
+
+ copilot_ctx.completion_verification_result = verification
+ record_completion_verification(copilot_ctx, verification)
+ if verification.status == "evaluated":
+ _emit_completion_verification_trace(copilot_ctx, verification)
+ return verification
+
+
+_COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS = 5.0
+
+
+async def _completion_verification_handler(copilot_ctx: Any) -> Any | None:
+ return await resolve_main_copilot_handler(
+ getattr(copilot_ctx, "workflow_permanent_id", None),
+ getattr(copilot_ctx, "organization_id", None),
+ )
+
+
+def _is_outcome_evidence_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
+ """A clean ok=True run worth judging on its whole-workflow outcome.
+
+ Recognition is governed by the outcome evidence the user can observe, not by
+ whether every block was verified as an end-to-end prefix (SKY-10576). The judge
+ requires positive evidence for every criterion, and ``empty_data_blocks`` rejects
+ a run whose outcome block produced nothing, so an incomplete run never passes --
+ this only lets an already-reached goal be recognized without a redundant
+ full-prefix re-run.
+ """
+ if not bool(result.get("ok", False)):
+ return False
+ if _run_blocks_structured_blocker_message(result):
+ return False
+ _anti_bot, empty_data_blocks, _categories = _analyze_run_blocks(result)
+ return not empty_data_blocks
+
+
+def _is_unfinished_run_verification_candidate(copilot_ctx: Any, result: dict[str, Any]) -> bool:
+ """A canceled/partial run (ok=False) still worth judging because it left runtime
+ evidence behind. The judge confirms a criterion only on positive evidence, so a
+ broken run never spuriously passes; this only lets a reached goal be recognized
+ even though the run did not finish cleanly — recognition must not key on run status.
+ """
+ if bool(result.get("ok", False)):
+ return False
+ if _active_run_terminal_evidence_detected(result):
+ return False
+ if _run_blocks_structured_blocker_message(result):
+ return False
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return False
+ return _valid_runtime_anchor_url(data.get("current_url")) is not None
+
+
+def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
+ data = result.get("data")
+ data = data if isinstance(data, dict) else {}
+ current_label_order = _current_workflow_block_labels(copilot_ctx)
+ current_labels = set(current_label_order)
+ # Evidence must be what THIS run produced. ``verified_block_outputs`` accumulates
+ # across incremental runs, so sourcing from it would let an output from a prior
+ # run satisfy a criterion the current run never re-produced.
+ blocks = data.get("blocks")
+ block_outputs: dict[str, Any] = {}
+ if isinstance(blocks, list):
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ label = block.get("label")
+ output = block.get("extracted_data")
+ if isinstance(label, str) and label in current_labels and _is_meaningful_extracted_data(output):
+ block_outputs[label] = output
+ executed = data.get("executed_block_labels")
+ executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []
+ page_title = data.get("page_title")
+ run_id = data.get("workflow_run_id")
+ return RunEvidenceSnapshot(
+ workflow_run_id=run_id if isinstance(run_id, str) else None,
+ block_outputs=block_outputs,
+ current_url=_valid_runtime_anchor_url(data.get("current_url")),
+ page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
+ executed_block_labels=executed_block_labels,
+ verified_context_block_labels=_verified_context_block_labels_for_snapshot(
+ copilot_ctx,
+ current_label_order,
+ executed_block_labels,
+ ),
+ )
+
+
+async def _maybe_run_completion_verification(
+ copilot_ctx: Any, result: dict[str, Any], handler_start: float
+) -> CompletionVerificationResult | None:
+ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
+ return None
+ if getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):
+ return None
+ criteria = _completion_verification_criteria(copilot_ctx)
+ if not criteria:
+ return None
+ if not (
+ _is_outcome_evidence_candidate(copilot_ctx, result)
+ or _is_unfinished_run_verification_candidate(copilot_ctx, result)
+ ):
+ return None
+ # A missing judge handler is an infra/config state, not a transient failure:
+ # fall back to the prior gate rather than fail closed on every run.
+ handler = await _completion_verification_handler(copilot_ctx)
+ if handler is None:
+ return None
+ # Too little budget to verify a candidate run: fail closed (unavailable) rather
+ # than let the run-status proxy claim an unverified outcome as success.
+ remaining = RUN_BLOCKS_SAFETY_CEILING_SECONDS - (time.monotonic() - handler_start)
+ if remaining <= settings.COPILOT_FEASIBILITY_GATE_TIMEOUT_SECONDS + _COMPLETION_VERIFICATION_BUDGET_MARGIN_SECONDS:
+ return CompletionVerificationResult(status="unavailable")
+ snapshot = _build_run_evidence_snapshot(copilot_ctx, result)
+ if not snapshot.has_evidence():
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[criterion.id for criterion in criteria],
+ verdicts=[
+ CriterionVerdict(criterion_id=criterion.id, satisfied=False, reason_code="no_evidence")
+ for criterion in criteria
+ ],
+ )
+ return await evaluate_completion_criteria(criteria, snapshot, handler)
+
+
+def _outcome_unverified_reason(
+ copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
+) -> str | None:
+ if completion_verification is None:
+ return None
+ if completion_verification.status == "evaluated":
+ if completion_verification.is_fully_satisfied():
+ return None
+ policy = getattr(copilot_ctx, "request_policy", None)
+ criteria = list(policy.completion_criteria) if policy is not None else []
+ unmet = summarize_unsatisfied_outcomes(completion_verification, criteria)
+ detail = f": {unmet}" if unmet else ""
+ return (
+ f"The run completed but did not demonstrate the goal outcome(s){detail}. "
+ "Add an end-state confirmation (an extraction or validation block) that observes the outcome, then re-run."
+ )
+ # An 'unavailable' result reaches here only when verification was required;
+ # fail closed and ask for a re-run rather than claim an unverified success.
+ return (
+ "The run completed but the goal outcome could not be verified (verification was unavailable). "
+ "Re-run to verify the outcome before reporting success."
+ )
+
+
+def _outcome_failure_warrants_repair(
+ copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
+) -> bool:
+ """Whether an unmet outcome should route to suspicious-success/repair rather
+ than continue-building.
+
+ Contradicting evidence is always a real failure. Absence of evidence is a
+ failure only once the workflow has an outcome-producing block; while the agent
+ is still adding blocks toward the goal, an unmet criterion is expected, and the
+ run should keep building. Terminal success stays withheld in both cases via the
+ verification result, so this only governs the repair directive, not the gate.
+ """
+ if completion_verification is None:
+ return False
+ if any(verdict.reason_code == "evidence_contradicts" for verdict in completion_verification.verdicts):
+ return True
+ return _current_workflow_has_evidence_block(copilot_ctx)
+
+
+def _tool_visible_result_after_completion_verification(
+ copilot_ctx: Any,
+ result: dict[str, Any],
+ completion_verification: CompletionVerificationResult | None,
+) -> dict[str, Any]:
+ outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ if outcome_unverified_reason is None:
+ return result
+ if not _outcome_failure_warrants_repair(copilot_ctx, completion_verification):
+ return result
+
+ data = result.get("data")
+ copied_data = dict(data) if isinstance(data, dict) else {}
+ copied_data["failure_reason"] = outcome_unverified_reason
+ copied_data["completion_verification"] = (
+ completion_verification.to_trace_data() if completion_verification is not None else None
+ )
+ categories = copied_data.get("failure_categories")
+ copied_categories = list(categories) if isinstance(categories, list) else []
+ copied_categories.insert(
+ 0,
+ {
+ "category": "OUTCOME_UNVERIFIED",
+ "confidence_float": 1.0,
+ "reasoning": outcome_unverified_reason,
+ },
+ )
+ copied_data["failure_categories"] = copied_categories
+ return {
+ **result,
+ "ok": False,
+ "error": outcome_unverified_reason,
+ "data": copied_data,
+ }
+
+
+def _emit_completion_verification_trace(
+ copilot_ctx: Any, completion_verification: CompletionVerificationResult
+) -> None:
+ block_count = getattr(copilot_ctx, "last_update_block_count", None)
+ policy = getattr(copilot_ctx, "request_policy", None)
+ contract = policy.completion_contract if policy is not None else None
+ heuristic_would_block = isinstance(block_count, int) and _goal_likely_needs_more_blocks(
+ getattr(copilot_ctx, "user_message", ""), block_count, contract
+ )
+ trace_data = {
+ **completion_verification.to_trace_data(),
+ "heuristic_would_block": heuristic_would_block,
+ "evidence_block_present": _current_workflow_has_evidence_block(copilot_ctx),
+ "warrants_repair": _outcome_failure_warrants_repair(copilot_ctx, completion_verification),
+ }
+ LOG.info(
+ "copilot completion verification",
+ **{f"completion_verification_{key}": value for key, value in trace_data.items()},
+ )
+ with copilot_span("completion_verification", data=trace_data):
+ pass
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
new file mode 100644
index 000000000..31d01fe1f
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -0,0 +1,918 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+from dataclasses import dataclass
+from typing import Any, Literal
+from urllib.parse import urlparse
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ RunEvidenceSnapshot,
+ evaluate_completion_criteria,
+)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION as _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION,
+)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS as _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS,
+)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ has_bounded_page_schema,
+ merge_visual_composition_evidence,
+ page_evidence_needs_visual_fallback,
+ parse_composition_html,
+ parse_composition_structured,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handler
+from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+
+from ._shared import (
+ _CURRENT_PAGE_INSPECTION_TARGETS,
+ _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ _append_flow_evidence,
+ _composition_evidence_page_url,
+ _composition_get_html,
+ _discovery_extract_current_url,
+ _discovery_navigate,
+ _fallback_page_info,
+ _same_page_ignoring_fragment,
+ _valid_runtime_anchor_url,
+ _workflow_verification_evidence,
+)
+from .blockers import _allows_post_run_current_page_inspection_budget_bypass
+from .completion import (
+ _completion_verification_criteria,
+ _completion_verification_handler,
+ _maybe_run_completion_verification_from_page_observation,
+)
+from .discovery import _resolve_discovery_entry_url
+from .guardrails import _authority_tool_error
+from .scouting import (
+ _clear_pending_browser_interaction_observation,
+ _consume_pending_browser_interaction_observation,
+ _mark_page_inspected,
+ _mark_post_run_page_observed,
+)
+
+LOG = structlog.get_logger()
+
+
+@dataclass(frozen=True)
+class ActiveRunTerminalEvidenceSample:
+ current_url: str | None
+ page_title: str | None
+ page_evidence: dict[str, Any]
+ completion_verification: CompletionVerificationResult
+ sample_index: int
+
+
+async def _active_run_terminal_monitor_enabled(copilot_ctx: Any) -> bool:
+ if not settings.COPILOT_OUTCOME_VERIFICATION_ENABLED:
+ return False
+ if not getattr(copilot_ctx, "browser_session_id", None):
+ return False
+ if not getattr(copilot_ctx, "discovery_mcp_server", None):
+ return False
+ if not _completion_verification_criteria(copilot_ctx):
+ return False
+ return await _completion_verification_handler(copilot_ctx) is not None
+
+
+def _active_run_terminal_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:
+ if page_evidence_needs_visual_fallback(evidence):
+ return True
+ return evidence.get("screenshot_used") is not True
+
+
+async def _active_run_terminal_evidence_sample(
+ copilot_ctx: Any,
+ *,
+ workflow_run_id: str,
+ labels_to_execute: list[str],
+ sample_index: int,
+) -> ActiveRunTerminalEvidenceSample | None:
+ criteria = _completion_verification_criteria(copilot_ctx)
+ if not criteria:
+ return None
+ handler = await _completion_verification_handler(copilot_ctx)
+ if handler is None:
+ return None
+
+ current_url_raw, page_title_raw = await _fallback_page_info(copilot_ctx)
+ current_url = _valid_runtime_anchor_url(current_url_raw)
+ if current_url is None:
+ return None
+
+ evidence, html_error = await _capture_composition_evidence(
+ copilot_ctx,
+ inspected_url=current_url,
+ current_url=current_url,
+ active_run_terminal_sample=True,
+ )
+ if html_error is not None or evidence is None:
+ LOG.info(
+ "copilot active-run terminal evidence sample skipped",
+ workflow_run_id=workflow_run_id,
+ sample_index=sample_index,
+ html_error=html_error,
+ )
+ return None
+
+ page_title = evidence.get("page_title")
+ if not isinstance(page_title, str) or not page_title.strip():
+ page_title = page_title_raw if isinstance(page_title_raw, str) and page_title_raw.strip() else None
+ evidence = {
+ **evidence,
+ "workflow_run_id": workflow_run_id,
+ "observed_during_active_workflow_run": True,
+ }
+ snapshot = RunEvidenceSnapshot(
+ workflow_run_id=workflow_run_id,
+ current_url=current_url,
+ page_title=page_title,
+ executed_block_labels=list(labels_to_execute),
+ page_evidence=evidence,
+ )
+ if not snapshot.has_evidence():
+ return None
+
+ result = await evaluate_completion_criteria(criteria, snapshot, handler)
+ LOG.info(
+ "copilot active-run terminal evidence sample",
+ workflow_run_id=workflow_run_id,
+ sample_index=sample_index,
+ completion_verification_status=result.status,
+ completion_verification_fully_satisfied=result.is_fully_satisfied(),
+ )
+ if result.status != "evaluated" or not result.is_fully_satisfied():
+ return None
+ copilot_ctx.composition_page_evidence = evidence
+ return ActiveRunTerminalEvidenceSample(
+ current_url=current_url,
+ page_title=page_title,
+ page_evidence=evidence,
+ completion_verification=result,
+ sample_index=sample_index,
+ )
+
+
+def _active_run_terminal_evidence_result(
+ *,
+ workflow_run_id: str,
+ run_status: str | None,
+ sample: Any,
+ requested_block_labels: list[str],
+ executed_block_labels: list[str],
+ current_url: str | None = None,
+ page_title: str | None = None,
+) -> dict[str, Any]:
+ observed_url = current_url or getattr(sample, "current_url", None)
+ observed_title = page_title or getattr(sample, "page_title", None)
+ completion = getattr(sample, "completion_verification", None)
+ completion_trace = completion.to_trace_data() if isinstance(completion, CompletionVerificationResult) else {}
+ reason = (
+ "The active run reached the requested browser state while the workflow was still running, "
+ "so Copilot interrupted it before further browser actions could overshoot that state. "
+ "The current page evidence is not a durable full-workflow verification; inspect the run boundary, "
+ "repair the workflow if needed, and verify the corrected workflow run."
+ )
+ return {
+ "ok": False,
+ "error": reason,
+ "data": {
+ "workflow_run_id": workflow_run_id,
+ "overall_status": run_status,
+ "failure_reason": reason,
+ "requested_block_labels": list(requested_block_labels),
+ "executed_block_labels": list(executed_block_labels),
+ "current_url": observed_url,
+ "page_title": observed_title,
+ "active_run_terminal_evidence_detected": True,
+ "active_run_terminal_evidence_sample_index": getattr(sample, "sample_index", None),
+ "full_workflow_verified": False,
+ "current_page_evidence": getattr(sample, "page_evidence", None),
+ "active_run_terminal_completion_verification": completion_trace,
+ "failure_categories": [
+ {
+ "category": ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ "confidence_float": 1.0,
+ "reasoning": (
+ "Bounded current-page evidence matched the request-policy completion criteria "
+ "while the workflow run was still active; the run was interrupted for diagnosis/repair."
+ ),
+ }
+ ],
+ },
+ }
+
+
+_COMPOSITION_INSPECTION_PER_CHAT_BUDGET = 6
+_COMPOSITION_INSPECTION_PER_TURN_BUDGET = 4
+_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS = 10.0
+_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME = "workflow-copilot-page-evidence-vision"
+
+
+async def _composition_get_screenshot(ctx: CopilotContext) -> dict[str, Any]:
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return {"ok": False, "error": "discovery MCP server not attached to context"}
+ try:
+ return await asyncio.wait_for(
+ server.call_internal_tool("skyvern_screenshot", {"inline": True}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ return {"ok": False, "error": f"skyvern_screenshot timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
+
+
+def _composition_extract_screenshot_b64(result: dict[str, Any]) -> str:
+ data = result.get("data")
+ if isinstance(data, dict):
+ for key in ("screenshot_base64", "data", "image_base64"):
+ value = data.get(key)
+ if isinstance(value, str) and value.strip():
+ return value.strip()
+ return ""
+
+
+def _composition_visual_prompt(evidence: dict[str, Any]) -> str:
+ context = {
+ "page_title": evidence.get("page_title") or "",
+ "current_url": evidence.get("current_url") or "",
+ "anti_bot_indicators": evidence.get("anti_bot_indicators") or [],
+ "challenge_state": evidence.get("challenge_state") or {},
+ "form_count": len(evidence.get("forms") or []),
+ "result_container_count": len(evidence.get("result_containers") or []),
+ "page_obstruction_count": len(evidence.get("page_obstructions") or []),
+ "visual_obstruction_candidate_count": len(evidence.get("visual_obstruction_candidates") or []),
+ "schema_empty_page": evidence.get("schema_empty_page") is True,
+ }
+ return (
+ "Summarize this screenshot for Workflow Copilot build-time page evidence. "
+ "Return JSON only with keys: summary, challenge_detected, challenge_kind, "
+ "challenge_location, submit_blocked, blocked_submit_controls, empty_page_visible, "
+ "loading_state_visible, page_obstruction_detected, obstruction_kind, "
+ "obstruction_location, underlying_page_blocked, visible_dismiss_controls, omissions. "
+ "In summary, include the visible page state that would help verify an end-state outcome, "
+ "such as cart items, "
+ "record rows, visible identifiers, quantities, statuses, prices, confirmations, search results, "
+ "or selected values when legible. Also note visible anti-bot or human-verification state, "
+ "whether it appears to gate a submit/search control, whether any visible artificial barrier "
+ "mechanically blocks the underlying page, and where it appears relative to the page controls. "
+ "Do not include raw DOM, code, selectors, personal data, or workflow instructions. "
+ "If no challenge is visible, set challenge_detected to false and submit_blocked to false. "
+ "If no page obstruction is visible, set page_obstruction_detected to false. "
+ "If DOM context shows a schema-empty page, set empty_page_visible to true only when the "
+ "screenshot shows a settled page with no visible forms, controls, result data, challenge, "
+ "or loading/progress state; set loading_state_visible to true when the page appears to be "
+ "waiting, loading, redirecting, or still rendering.\n\n"
+ f"DOM evidence context:\n{json.dumps(context, sort_keys=True)}"
+ )
+
+
+async def _composition_visual_handler(ctx: CopilotContext) -> Any | None:
+ return await resolve_fast_copilot_handler(
+ getattr(ctx, "workflow_permanent_id", None),
+ getattr(ctx, "organization_id", None),
+ )
+
+
+def _normalize_visual_summary(value: Any) -> dict[str, Any] | None:
+ if not isinstance(value, dict):
+ return None
+ summary = value.get("summary")
+ challenge_detected = value.get("challenge_detected")
+ challenge_kind = value.get("challenge_kind")
+ challenge_location = value.get("challenge_location")
+ submit_blocked = value.get("submit_blocked")
+ blocked_submit_controls = value.get("blocked_submit_controls")
+ empty_page_visible = value.get("empty_page_visible")
+ loading_state_visible = value.get("loading_state_visible")
+ page_obstruction_detected = value.get("page_obstruction_detected")
+ obstruction_kind = value.get("obstruction_kind")
+ obstruction_location = value.get("obstruction_location")
+ underlying_page_blocked = value.get("underlying_page_blocked")
+ visible_dismiss_controls = value.get("visible_dismiss_controls")
+ omissions = value.get("omissions")
+ return {
+ "summary": summary if isinstance(summary, str) else "",
+ "challenge_detected": challenge_detected if isinstance(challenge_detected, bool) else None,
+ "challenge_kind": challenge_kind if isinstance(challenge_kind, str) else "",
+ "challenge_location": challenge_location if isinstance(challenge_location, str) else "",
+ "submit_blocked": submit_blocked if isinstance(submit_blocked, bool) else None,
+ "empty_page_visible": empty_page_visible if isinstance(empty_page_visible, bool) else None,
+ "loading_state_visible": loading_state_visible if isinstance(loading_state_visible, bool) else None,
+ "page_obstruction_detected": page_obstruction_detected if isinstance(page_obstruction_detected, bool) else None,
+ "obstruction_kind": obstruction_kind if isinstance(obstruction_kind, str) else "",
+ "obstruction_location": obstruction_location if isinstance(obstruction_location, str) else "",
+ "underlying_page_blocked": underlying_page_blocked if isinstance(underlying_page_blocked, bool) else None,
+ "blocked_submit_controls": [item for item in blocked_submit_controls if isinstance(item, str)]
+ if isinstance(blocked_submit_controls, list)
+ else [],
+ "visible_dismiss_controls": [item for item in visible_dismiss_controls if isinstance(item, str)]
+ if isinstance(visible_dismiss_controls, list)
+ else [],
+ "omissions": [item for item in omissions if isinstance(item, str)] if isinstance(omissions, list) else [],
+ }
+
+
+async def _composition_summarize_screenshot(
+ ctx: CopilotContext,
+ *,
+ evidence: dict[str, Any],
+ screenshot_b64: str,
+) -> tuple[dict[str, Any] | None, str | None]:
+ handler = await _composition_visual_handler(ctx)
+ if handler is None:
+ return None, "workflow copilot LLM handler is not configured"
+ try:
+ screenshot_bytes = base64.b64decode(screenshot_b64, validate=True)
+ except Exception:
+ return None, "screenshot payload was not valid base64"
+ try:
+ response = await asyncio.wait_for(
+ handler(
+ prompt=_composition_visual_prompt(evidence),
+ prompt_name=_COMPOSITION_VISUAL_SUMMARY_PROMPT_NAME,
+ screenshots=[screenshot_bytes],
+ organization_id=getattr(ctx, "organization_id", None),
+ force_dict=True,
+ ),
+ timeout=_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ return None, f"visual summary timed out after {_COMPOSITION_VISUAL_SUMMARY_TIMEOUT_SECONDS:g}s"
+ except Exception as exc:
+ LOG.warning("Composition screenshot visual summary failed", error=str(exc), exc_info=True)
+ return None, str(exc)
+ normalized = _normalize_visual_summary(response)
+ if normalized is None:
+ return None, "visual summary response was not a JSON object"
+ return normalized, None
+
+
+async def _augment_composition_evidence_with_visual_fallback(
+ ctx: CopilotContext,
+ evidence: dict[str, Any],
+) -> dict[str, Any]:
+ screenshot_result = await _composition_get_screenshot(ctx)
+ if not screenshot_result.get("ok"):
+ return _composition_add_evidence_omission(
+ evidence,
+ f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",
+ )
+ screenshot_b64 = _composition_extract_screenshot_b64(screenshot_result)
+ visual_summary, visual_error = await _composition_summarize_screenshot(
+ ctx,
+ evidence=evidence,
+ screenshot_b64=screenshot_b64,
+ )
+ return merge_visual_composition_evidence(evidence, visual_summary=visual_summary, visual_error=visual_error)
+
+
+def _composition_add_evidence_omission(evidence: dict[str, Any], message: str) -> dict[str, Any]:
+ merged = dict(evidence)
+ omissions = [item for item in merged.get("visual_evidence_omissions") or [] if isinstance(item, str)]
+ if message:
+ omissions.append(message[:160])
+ merged["visual_evidence_omissions"] = list(dict.fromkeys(omissions))[:5]
+ return merged
+
+
+def _composition_add_inspection_warning(evidence: dict[str, Any], message: str) -> dict[str, Any]:
+ merged = dict(evidence)
+ warnings = [item for item in merged.get("inspection_warnings") or [] if isinstance(item, str)]
+ if message:
+ warnings.append(message[:240])
+ merged["inspection_warnings"] = list(dict.fromkeys(warnings))[:5]
+ return merged
+
+
+async def _composition_evidence_after_navigation_failure(
+ ctx: CopilotContext,
+ *,
+ inspected_url: str,
+ navigation_error: str,
+) -> dict[str, Any] | None:
+ current_url, _ = await _fallback_page_info(ctx)
+ current_url = current_url or inspected_url
+ structured = await _composition_get_structured_evidence(ctx, inspected_url=inspected_url, current_url=current_url)
+ if structured is not None and has_bounded_page_schema(structured):
+ evidence = _composition_add_inspection_warning(
+ structured,
+ f"navigation_error_before_html_capture: {navigation_error}",
+ )
+ if page_evidence_needs_visual_fallback(evidence):
+ evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return evidence
+ # Same size-cap survival as the success path: a heavy page that rendered before the nav
+ # error still parses via the stripped-body evaluate instead of yielding hollow evidence.
+ html, html_error, html_truncated, _ = await _composition_get_html(ctx)
+ if html_error is None:
+ evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
+ evidence = _composition_add_inspection_warning(
+ evidence,
+ f"navigation_error_before_html_capture: {navigation_error}",
+ )
+ if html_truncated:
+ evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")
+ evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(ctx, evidence)
+ if page_evidence_needs_visual_fallback(evidence):
+ evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return evidence
+
+ evidence = parse_composition_html("", inspected_url=inspected_url, current_url=current_url)
+ evidence = _composition_add_inspection_warning(
+ evidence,
+ f"navigation_error_before_evidence_capture: {navigation_error}",
+ )
+ evidence = _composition_add_inspection_warning(
+ evidence,
+ f"html_capture_failed_after_navigation_error: {html_error}",
+ )
+ evidence = await _augment_composition_evidence_with_visual_fallback(ctx, evidence)
+ return evidence if evidence.get("screenshot_used") else None
+
+
+def _inspection_reached_via(*, use_current_page: bool, post_run: bool, earned_interaction: bool) -> str:
+ """How the just-inspected state was reached, for the flow-evidence trajectory.
+
+ A target_url inspection navigates there itself ("navigate"); a post-run
+ current-page inspection observes the page the run left behind ("post_run"); a
+ normal current-page inspection counts as an interaction only when a successful
+ browser action immediately earned that credit.
+ """
+ if not use_current_page:
+ return "navigate"
+ if post_run:
+ return "post_run"
+ return "interaction" if earned_interaction else "current_page"
+
+
+def _latest_interaction_reached_flow_evidence(copilot_ctx: Any) -> tuple[int, str, dict[str, Any]] | None:
+ trajectory = getattr(copilot_ctx, "flow_evidence", None)
+ if not isinstance(trajectory, list):
+ return None
+ for entry in reversed(trajectory):
+ if not isinstance(entry, dict):
+ continue
+ reached_via = str(entry.get("reached_via") or "")
+ if reached_via not in {"interaction", "post_run"}:
+ continue
+ evidence = entry.get("evidence")
+ step = entry.get("step")
+ if isinstance(step, bool) or not isinstance(step, int) or not isinstance(evidence, dict):
+ continue
+ if not has_bounded_page_schema(evidence):
+ continue
+ observed_url = _composition_evidence_page_url(evidence)
+ if observed_url:
+ return step, observed_url, evidence
+ return None
+
+
+def _non_current_inspection_regression_error(copilot_ctx: Any, *, entry_url: str) -> dict[str, Any] | None:
+ latest = _latest_interaction_reached_flow_evidence(copilot_ctx)
+ if latest is None:
+ return None
+ observation_step, observed_url, _ = latest
+ if _same_page_ignoring_fragment(observed_url, entry_url):
+ return None
+ return {
+ "ok": False,
+ "data": {
+ "current_url": observed_url,
+ "observation_step": observation_step,
+ },
+ "error": (
+ "inspect_page_for_composition would navigate away from the latest interaction-reached page "
+ f'({observed_url}). Use inspect_page_for_composition(target_url="current_page") to inspect '
+ "the live page, or compose from the existing page evidence and pass observation_step "
+ f"{observation_step} in block_observation_refs for blocks that act on that reached page."
+ ),
+ }
+
+
+def _page_inspection_budget_error(copilot_ctx: Any, *, scope: Literal["turn", "chat"]) -> str:
+ scope_label = "turn" if scope == "turn" else "chat"
+ return (
+ f"inspect_page_for_composition reached the page-inspection budget for this {scope_label}. "
+ "This is not evidence that scouting is complete. Use evaluate, get_browser_screenshot, or a browser "
+ "action on the current page to determine whether the goal is already satisfied, whether progress is still "
+ "possible, or whether a real blocker exists. Do not author downstream result, extraction, or confirmation "
+ "blocks unless the existing evidence already shows the page state those blocks will act on."
+ )
+
+
+_COMPOSITION_HOLLOW_RECAPTURE_RETRIES = 2
+_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS = 2.5
+# The composition inspect navigates with `domcontentloaded`, so a heavier cap than
+# the discovery walker's is safe — the navigate returns at DOM parse, well before
+# this ceiling, and only a genuinely stuck load reaches it.
+_COMPOSITION_NAVIGATE_TIMEOUT_SECONDS = 30.0
+
+
+def _normalize_visual_obstruction_candidates(value: Any) -> list[dict[str, Any]]:
+ if not isinstance(value, list):
+ return []
+ candidates: list[dict[str, Any]] = []
+ for item in value:
+ if len(candidates) >= 5:
+ break
+ if not isinstance(item, dict):
+ continue
+ position = item.get("position")
+ coverage = item.get("coverage")
+ if position not in {"fixed", "sticky"} or coverage != "viewport":
+ continue
+ candidates.append(
+ {
+ "source": "computed_style",
+ "position": position,
+ "coverage": "viewport",
+ "has_visible_controls": item.get("has_visible_controls") is True,
+ }
+ )
+ return candidates
+
+
+def _merge_visual_obstruction_candidates(
+ evidence: dict[str, Any],
+ candidates: list[dict[str, Any]],
+) -> dict[str, Any]:
+ if not candidates:
+ return evidence
+ merged = dict(evidence)
+ existing = [item for item in merged.get("visual_obstruction_candidates") or [] if isinstance(item, dict)]
+ for candidate in candidates:
+ if len(existing) >= 5:
+ break
+ if candidate not in existing:
+ existing.append(candidate)
+ merged["visual_obstruction_candidates"] = existing[:5]
+ return merged
+
+
+async def _composition_get_computed_visual_obstruction_candidates(copilot_ctx: Any) -> list[dict[str, Any]]:
+ server = getattr(copilot_ctx, "discovery_mcp_server", None)
+ if server is None:
+ return []
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool(
+ "skyvern_evaluate",
+ {"expression": _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION},
+ ),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return []
+ if not isinstance(result, dict) or not result.get("ok"):
+ return []
+ value = (result.get("data") or {}).get("result")
+ return _normalize_visual_obstruction_candidates(value)
+
+
+async def _augment_composition_evidence_with_computed_obstruction_candidates(
+ copilot_ctx: Any,
+ evidence: dict[str, Any],
+) -> dict[str, Any]:
+ if page_evidence_needs_visual_fallback(evidence) or not has_bounded_page_schema(evidence):
+ return evidence
+ candidates = await _composition_get_computed_visual_obstruction_candidates(copilot_ctx)
+ return _merge_visual_obstruction_candidates(evidence, candidates)
+
+
+async def _composition_get_structured_evidence(
+ copilot_ctx: Any,
+ *,
+ inspected_url: str,
+ current_url: str,
+) -> dict[str, Any] | None:
+ """Capture composition evidence via the page-side extractor; None when it can't yield a usable payload."""
+ server = getattr(copilot_ctx, "discovery_mcp_server", None)
+ if server is None:
+ return None
+ with copilot_span("composition_structured_extract"):
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool(
+ "skyvern_evaluate", {"expression": _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION}
+ ),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return None
+ if not isinstance(result, dict) or not result.get("ok"):
+ return None
+ raw = (result.get("data") or {}).get("result")
+ if isinstance(raw, str):
+ if len(raw) > _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS:
+ return None
+ try:
+ payload = json.loads(raw)
+ except (ValueError, TypeError):
+ return None
+ elif isinstance(raw, dict):
+ payload = raw
+ else:
+ return None
+ return parse_composition_structured(payload, inspected_url=inspected_url, current_url=current_url)
+
+
+async def _capture_composition_evidence(
+ copilot_ctx: Any,
+ *,
+ inspected_url: str,
+ current_url: str,
+ active_run_terminal_sample: bool = False,
+) -> tuple[dict[str, Any] | None, str | None]:
+ """Parse composition evidence (cheap extractor first, get_html fallback); html_error is set only on a failed HTML read."""
+ evidence: dict[str, Any] | None = None
+ html_truncated = False
+ used_structured = False
+ skip_raw = False
+ for attempt in range(_COMPOSITION_HOLLOW_RECAPTURE_RETRIES + 1):
+ structured = await _composition_get_structured_evidence(
+ copilot_ctx, inspected_url=inspected_url, current_url=current_url
+ )
+ if structured is not None:
+ evidence = structured
+ used_structured = True
+ if has_bounded_page_schema(evidence):
+ break
+ if attempt < _COMPOSITION_HOLLOW_RECAPTURE_RETRIES:
+ await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
+ continue
+ html, html_error, html_truncated, used_stripped = await _composition_get_html(copilot_ctx, skip_raw=skip_raw)
+ if html_error is not None:
+ if evidence is not None:
+ break
+ return None, html_error
+ # On a heavy page the raw get_html serialization is dropped over the MCP size cap and
+ # falls back to the stripped read; once that happens, settle-and-recapture via the
+ # stripped path only so a slow page is still retried without re-serializing the full DOM.
+ if used_stripped:
+ skip_raw = True
+ evidence = parse_composition_html(html, inspected_url=inspected_url, current_url=current_url)
+ used_structured = False
+ if has_bounded_page_schema(evidence):
+ break
+ if attempt < _COMPOSITION_HOLLOW_RECAPTURE_RETRIES:
+ await asyncio.sleep(_COMPOSITION_HOLLOW_RECAPTURE_DELAY_SECONDS)
+ if evidence is not None and html_truncated and not used_structured:
+ evidence = _composition_add_inspection_warning(evidence, "html_sliced_at_cap")
+ # Structured evidence already carries computed obstruction candidates; only the get_html path augments.
+ if evidence is not None and not used_structured:
+ evidence = await _augment_composition_evidence_with_computed_obstruction_candidates(copilot_ctx, evidence)
+ if evidence is not None and (
+ page_evidence_needs_visual_fallback(evidence)
+ or (active_run_terminal_sample and _active_run_terminal_evidence_needs_visual_fallback(evidence))
+ or (evidence.get("schema_empty_page") is True and not has_bounded_page_schema(evidence))
+ ):
+ evidence = await _augment_composition_evidence_with_visual_fallback(copilot_ctx, evidence)
+ return evidence, None
+
+
+def _normalized_inspect_url(url: str | None) -> str | None:
+ """Normalized full URL for strict same-page comparison, or None when not comparable.
+
+ Preserves scheme, the path's trailing slash, query, and fragment so distinct rendered
+ states (http vs https, /p vs /p/, ?q=a vs ?q=b, hash-routed SPA states) never collide;
+ only netloc case and an empty root path are normalized.
+ """
+ if not isinstance(url, str) or not url.strip():
+ return None
+ try:
+ parsed = urlparse(url.strip())
+ except Exception:
+ return None
+ if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
+ return None
+ path = parsed.path or "/"
+ query = f"?{parsed.query}" if parsed.query else ""
+ fragment = f"#{parsed.fragment}" if parsed.fragment else ""
+ return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}{path}{query}{fragment}"
+
+
+def _same_inspect_target(live_url: str | None, target_url: str | None) -> bool:
+ """True when the live page is the exact page a URL-target inspect would navigate to.
+
+ Strict full-URL equality, so a different scheme, trailing slash, query, or fragment
+ still navigates. Used to skip the re-navigation when the agent is already standing on
+ the requested page.
+ """
+ live_key = _normalized_inspect_url(live_url)
+ target_key = _normalized_inspect_url(target_url)
+ return live_key is not None and live_key == target_key
+
+
+async def _inspect_page_for_composition_impl(
+ copilot_ctx: Any,
+ target_url: str,
+) -> dict[str, Any]:
+ """Inspect a known target page and store form/search evidence on ctx.
+
+ This is composition context, not workflow YAML. It is intentionally separate
+ from `discover_workflow_entrypoint`: discovery answers "which page?";
+ inspection answers "what fields and controls are actually on this page?".
+ """
+ arguments = {"target_url": target_url}
+ authority_error = _authority_tool_error(copilot_ctx, "inspect_page_for_composition")
+ if authority_error:
+ result = {"ok": False, "error": authority_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+
+ use_current_page = (target_url or "").strip().lower() in _CURRENT_PAGE_INSPECTION_TARGETS
+ if not use_current_page:
+ _clear_pending_browser_interaction_observation(copilot_ctx)
+ bypass_budget_for_post_run_current_page = _allows_post_run_current_page_inspection_budget_bypass(
+ copilot_ctx,
+ use_current_page=use_current_page,
+ )
+
+ entry_url: str
+ kind: str
+ if use_current_page:
+ current_url, _ = await _fallback_page_info(copilot_ctx)
+ entry_url = current_url or "current_page"
+ kind = "current_page"
+ else:
+ resolved_entry_url, kind = _resolve_discovery_entry_url(target_url)
+ if resolved_entry_url is None:
+ result = {
+ "ok": False,
+ "data": None,
+ "error": "inspect_page_for_composition requires a URL, domain with an explicit path, or target_url='current_page'.",
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+ entry_url = resolved_entry_url
+ regression_error = _non_current_inspection_regression_error(copilot_ctx, entry_url=entry_url)
+ if regression_error is not None:
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, regression_error)
+ return regression_error
+
+ # Skip re-navigation when the inspect target is the page the browser is already on. A
+ # passive client-side redirect can move the browser without a tool, so for a URL target
+ # confirm against the live URL; for current_page the live URL is the target by definition.
+ if use_current_page:
+ inspect_target_url = current_url
+ on_target_page = True
+ else:
+ live_url, _ = await _fallback_page_info(copilot_ctx)
+ on_target_page = _same_inspect_target(live_url, entry_url)
+ inspect_target_url = live_url if on_target_page else entry_url
+
+ if (
+ not bypass_budget_for_post_run_current_page
+ and copilot_ctx.page_inspection_calls_this_turn >= _COMPOSITION_INSPECTION_PER_TURN_BUDGET
+ ):
+ result = {
+ "ok": False,
+ "data": None,
+ "error": _page_inspection_budget_error(copilot_ctx, scope="turn"),
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+
+ cumulative = copilot_ctx.prior_page_inspection_calls_made + copilot_ctx.page_inspection_calls_this_turn
+ if not bypass_budget_for_post_run_current_page and cumulative >= _COMPOSITION_INSPECTION_PER_CHAT_BUDGET:
+ result = {
+ "ok": False,
+ "data": None,
+ "error": _page_inspection_budget_error(copilot_ctx, scope="chat"),
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+
+ evidence = None
+ html_error: str | None = None
+ with copilot_span(
+ "inspect_page_for_composition",
+ data={"target_url_kind": kind},
+ ):
+ if on_target_page:
+ # current_page, or a URL target the agent is already on — capture without navigating.
+ current_url = inspect_target_url or entry_url
+ evidence, html_error = await _capture_composition_evidence(
+ copilot_ctx, inspected_url=entry_url, current_url=current_url
+ )
+ else:
+ nav_result = await _discovery_navigate(
+ copilot_ctx,
+ entry_url,
+ wait_until="domcontentloaded",
+ timeout_seconds=_COMPOSITION_NAVIGATE_TIMEOUT_SECONDS,
+ )
+ if not nav_result.get("ok"):
+ nav_error = str(nav_result.get("error") or "unknown")
+ failure_evidence = await _composition_evidence_after_navigation_failure(
+ copilot_ctx,
+ inspected_url=entry_url,
+ navigation_error=nav_error,
+ )
+ if failure_evidence is None:
+ result = {
+ "ok": False,
+ "data": None,
+ "error": f"inspect_page_for_composition could not navigate: {nav_error}",
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+ evidence = failure_evidence
+ current_url = str(evidence.get("current_url") or entry_url)
+ else:
+ current_url = _discovery_extract_current_url(nav_result, entry_url)
+ evidence, html_error = await _capture_composition_evidence(
+ copilot_ctx, inspected_url=entry_url, current_url=current_url
+ )
+
+ if html_error is not None:
+ result = {
+ "ok": False,
+ "data": None,
+ "error": f"inspect_page_for_composition could not read page HTML: {html_error}",
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+
+ if evidence is None:
+ result = {
+ "ok": False,
+ "data": None,
+ "error": "inspect_page_for_composition could not read page HTML.",
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
+
+ run_id = getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)
+ if isinstance(run_id, str) and run_id:
+ evidence = {
+ **evidence,
+ "workflow_run_id": run_id,
+ "observed_after_workflow_run": True,
+ }
+ _mark_post_run_page_observed(copilot_ctx, source_tool="inspect_page_for_composition", url=current_url)
+ page_title = evidence.get("page_title")
+ if isinstance(page_title, str) and page_title:
+ _workflow_verification_evidence(copilot_ctx).page_title = page_title[:160]
+
+ if not bypass_budget_for_post_run_current_page:
+ copilot_ctx.page_inspection_calls_this_turn += 1
+ if bypass_budget_for_post_run_current_page:
+ copilot_ctx.post_run_current_page_inspection_workflow_run_id = run_id
+ copilot_ctx.composition_page_evidence = evidence
+ if (
+ isinstance(run_id, str)
+ and run_id
+ and getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False)
+ ):
+ page_title = evidence.get("page_title")
+ await _maybe_run_completion_verification_from_page_observation(
+ copilot_ctx,
+ url=str(evidence.get("current_url") or current_url or ""),
+ title=page_title if isinstance(page_title, str) else "",
+ observed_data=evidence,
+ )
+ earned_interaction = False
+ if use_current_page and not run_id:
+ earned_interaction = _consume_pending_browser_interaction_observation(
+ copilot_ctx,
+ current_url=str(evidence.get("current_url") or current_url or ""),
+ evidence=evidence,
+ )
+ reached_via = _inspection_reached_via(
+ use_current_page=use_current_page,
+ post_run=bool(run_id),
+ earned_interaction=earned_interaction,
+ )
+ observation_step = _append_flow_evidence(copilot_ctx, evidence, reached_via=reached_via)
+ if observation_step is None:
+ LOG.warning("copilot_flow_evidence_append_failed_no_trajectory")
+ _mark_page_inspected(copilot_ctx)
+ # Surface the reached page at the top level so the model registers that the
+ # inspection already navigated there and does not re-issue navigate_browser.
+ current_url = evidence.get("current_url") or evidence.get("inspected_url") or ""
+ result = {
+ "ok": True,
+ "current_url": current_url,
+ "reached_via": reached_via,
+ "data": evidence,
+ }
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ record_tool_step_result_for_ctx(copilot_ctx, "inspect_page_for_composition", arguments, result)
+ return result
diff --git a/skyvern/forge/sdk/copilot/tools/credentials.py b/skyvern/forge/sdk/copilot/tools/credentials.py
new file mode 100644
index 000000000..3fb6e81ff
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/credentials.py
@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+import structlog
+import yaml
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.schemas.credentials import Credential
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType
+from skyvern.utils.yaml_loader import safe_load_no_dates
+
+from ._shared import _iter_yaml_blocks, _workflow_definition_as_dict
+
+LOG = structlog.get_logger()
+
+
+_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
+
+
+def _extract_credential_ids_from_tool_value(value: Any) -> list[str]:
+ found: list[str] = []
+
+ def visit(item: Any) -> None:
+ if isinstance(item, str):
+ found.extend(_CREDENTIAL_ID_RE.findall(item))
+ elif isinstance(item, dict):
+ for nested in item.values():
+ visit(nested)
+ elif isinstance(item, (list, tuple, set)):
+ for nested in item:
+ visit(nested)
+ elif hasattr(item, "model_dump"):
+ try:
+ visit(item.model_dump(mode="json"))
+ except Exception:
+ return
+
+ visit(value)
+ return list(dict.fromkeys(found))
+
+
+def _credential_parameter_slot_field(parameter: Any) -> str | None:
+ """Return the field name that legitimately carries a `cred_xxx` value for
+ this parameter dict, or None if the parameter is not a credential-binding
+ slot. Two shapes resolve a credential at runtime: a top-level or block-level
+ `parameter_type: credential` with the ID in `credential_id`, and a
+ `parameter_type: workflow` + `workflow_parameter_type: credential_id` with
+ the ID in `default_value`.
+ """
+ if not isinstance(parameter, dict):
+ return None
+ parameter_type = str(parameter.get("parameter_type") or "").lower()
+ if parameter_type == "credential":
+ return "credential_id"
+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
+ if parameter_type == "workflow" and workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID.value:
+ return "default_value"
+ return None
+
+
+def _extract_credential_ids_from_workflow_parameters(parameters: Any) -> list[str]:
+ if not isinstance(parameters, list):
+ return []
+
+ found: list[str] = []
+ for parameter in parameters:
+ slot_field = _credential_parameter_slot_field(parameter)
+ if slot_field is None:
+ continue
+ found.extend(_extract_credential_ids_from_tool_value(parameter.get(slot_field)))
+
+ return list(dict.fromkeys(found))
+
+
+def _extract_credential_ids_from_workflow_definition(workflow_definition: Any) -> list[str]:
+ definition = _workflow_definition_as_dict(workflow_definition)
+ return _extract_credential_ids_from_workflow_parameters(definition.get("parameters"))
+
+
+def _parsed_workflow_definition(workflow_yaml: str | None) -> dict[str, Any] | None:
+ if not workflow_yaml:
+ return None
+ try:
+ parsed = safe_load_no_dates(workflow_yaml)
+ except yaml.YAMLError:
+ return None
+ if not isinstance(parsed, dict):
+ return None
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, dict):
+ return None
+ return workflow_definition
+
+
+def _extract_credential_ids_from_workflow_yaml(workflow_yaml: str | None) -> list[str]:
+ workflow_definition = _parsed_workflow_definition(workflow_yaml)
+ if workflow_definition is None:
+ return []
+ return _extract_credential_ids_from_workflow_parameters(workflow_definition.get("parameters"))
+
+
+_MISBINDING_WORKFLOW_LOCATION = "workflow"
+
+
+def _credential_id_misbinding_findings(workflow_yaml: str | None) -> list[dict[str, str]]:
+ workflow_definition = _parsed_workflow_definition(workflow_yaml)
+ if workflow_definition is None:
+ return []
+
+ findings: list[dict[str, str]] = []
+
+ def _scan_value(value: Any, location: str, field: str) -> None:
+ if isinstance(value, str):
+ for credential_id in _CREDENTIAL_ID_RE.findall(value):
+ findings.append({"location": location, "field": field, "credential_id": credential_id})
+ elif isinstance(value, list):
+ for item in value:
+ _scan_value(item, location, field)
+ elif isinstance(value, dict):
+ for nested_field, nested_value in value.items():
+ _scan_value(nested_value, location, str(nested_field))
+
+ def _scan_parameter(parameter: Any, location: str) -> None:
+ if not isinstance(parameter, dict):
+ return
+ legal_slot_field = _credential_parameter_slot_field(parameter)
+ for field_name, field_value in parameter.items():
+ if field_name == legal_slot_field:
+ continue
+ _scan_value(field_value, location, str(field_name))
+
+ for parameter in workflow_definition.get("parameters") or []:
+ _scan_parameter(parameter, _MISBINDING_WORKFLOW_LOCATION)
+
+ for block in _iter_yaml_blocks(workflow_definition.get("blocks")):
+ label = str(block.get("label") or "<unlabeled>")
+ for field_name, field_value in block.items():
+ if field_name == "parameters":
+ if isinstance(field_value, list):
+ for parameter in field_value:
+ _scan_parameter(parameter, label)
+ continue
+ if field_name == "loop_blocks":
+ continue
+ _scan_value(field_value, label, str(field_name))
+
+ return findings
+
+
+def _credential_id_misbinding_error_message(findings: list[dict[str, str]]) -> str:
+ grouped: dict[tuple[str, str], list[str]] = {}
+ for finding in findings:
+ key = (finding["location"], finding["credential_id"])
+ grouped.setdefault(key, []).append(finding["field"])
+
+ location_lines: list[str] = []
+ for (location, credential_id), fields in grouped.items():
+ unique_fields = list(dict.fromkeys(fields))
+ joined = ", ".join(f"`{field}`" for field in unique_fields)
+ scope = "workflow parameter" if location == _MISBINDING_WORKFLOW_LOCATION else f"block `{location}`"
+ location_lines.append(f"- `{credential_id}` in {scope} field(s): {joined}")
+ body = "\n".join(location_lines)
+
+ return (
+ "A credential ID is sitting in workflow fields that do not resolve it, so at runtime the agent types "
+ "the literal ID into the page instead of the stored username/password:\n"
+ f"{body}\n"
+ "Fix BOTH halves before retrying:\n"
+ "1. Bind the credential once: add a `credential` parameter (or a `workflow` parameter with "
+ "`workflow_parameter_type: credential_id` and the ID in `default_value`) and reference its key from the "
+ "login block's `parameter_keys`.\n"
+ "2. Delete the credential ID string from every field listed above. `navigation_goal`, "
+ "`complete_criterion`, `terminate_criterion` and similar fields are plain-language instructions — they "
+ "must describe the outcome without naming the credential ID. Do NOT relocate the literal ID into another "
+ "prose or list field; only the credential parameter slot may hold it."
+ )
+
+
+def _missing_credential_reference_tool_error(missing_credential_ids: list[str]) -> str:
+ formatted_ids = ", ".join(f"`{credential_id}`" for credential_id in missing_credential_ids)
+ id_word = "ID" if len(missing_credential_ids) == 1 else "IDs"
+ was_word = "was" if len(missing_credential_ids) == 1 else "were"
+ return (
+ f"The credential {id_word} {formatted_ids} {was_word} not found in this organization. "
+ "Stop before creating, updating, or running the workflow. Ask the user to provide/select a valid "
+ "credential ID, create the credential in the Credentials UI and return with its ID, or explicitly "
+ "choose an unvalidated draft workflow that will not be run until credentials are available."
+ )
+
+
+async def _credential_ids_validation_error(credential_ids: list[str], ctx: AgentContext) -> str | None:
+ if not credential_ids:
+ return None
+ try:
+ existing_credentials = await app.DATABASE.credentials.get_credentials_by_ids(
+ credential_ids,
+ organization_id=ctx.organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Copilot tool failed to validate credential IDs",
+ organization_id=ctx.organization_id,
+ credential_ids=credential_ids,
+ exc_info=True,
+ )
+ return (
+ "Credential ID validation failed, so the workflow cannot be created, updated, or run safely. "
+ "Ask the user to provide/select a valid credential ID or explicitly choose an unvalidated draft "
+ "workflow that will not be run until credentials are available."
+ )
+
+ found_ids = {credential.credential_id for credential in existing_credentials}
+ missing_ids = [credential_id for credential_id in credential_ids if credential_id not in found_ids]
+ if not missing_ids:
+ return None
+ return _missing_credential_reference_tool_error(missing_ids)
+
+
+async def _credential_reference_validation_error(value: Any, ctx: AgentContext) -> str | None:
+ if isinstance(value, str):
+ credential_ids = _extract_credential_ids_from_workflow_yaml(value)
+ else:
+ credential_ids = _extract_credential_ids_from_tool_value(value)
+ return await _credential_ids_validation_error(credential_ids, ctx)
+
+
+async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
+ page = params.get("page", 1)
+ page_size = min(params.get("page_size", 10), 50)
+ credentials = await app.DATABASE.credentials.get_credentials(
+ organization_id=ctx.organization_id,
+ page=page,
+ page_size=page_size,
+ )
+ serialized = []
+ for cred in credentials:
+ entry: dict[str, Any] = {
+ "credential_id": cred.credential_id,
+ "name": cred.name,
+ "credential_type": str(cred.credential_type),
+ }
+ if cred.username:
+ entry["username"] = cred.username
+ entry["totp_type"] = str(cred.totp_type) if cred.totp_type else None
+ elif cred.card_last4:
+ entry["card_last_four"] = cred.card_last4
+ entry["card_brand"] = cred.card_brand
+ elif cred.secret_label:
+ entry["secret_label"] = cred.secret_label
+ serialized.append(entry)
+ _record_discovered_credentials_on_policy(ctx, credentials)
+ return {
+ "ok": True,
+ "data": {
+ "credentials": serialized,
+ "page": page,
+ "page_size": page_size,
+ "count": len(serialized),
+ "has_more": len(serialized) == page_size,
+ },
+ }
+
+
+def _record_discovered_credentials_on_policy(ctx: AgentContext, credentials: list[Credential]) -> None:
+ request_policy = ctx.request_policy
+ if not isinstance(request_policy, RequestPolicy):
+ return
+ known_ids = {credential.credential_id for credential in request_policy.discovered_credentials}
+ for credential in credentials:
+ if credential.credential_id not in known_ids:
+ request_policy.discovered_credentials.append(credential)
diff --git a/skyvern/forge/sdk/copilot/tools/discovery.py b/skyvern/forge/sdk/copilot/tools/discovery.py
new file mode 100644
index 000000000..84823410e
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/discovery.py
@@ -0,0 +1,732 @@
+from __future__ import annotations
+
+import asyncio
+import re
+import time
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import structlog
+from opentelemetry import trace as otel_trace
+
+try:
+ from bs4 import BeautifulSoup # type: ignore[import-not-found]
+except ImportError: # pragma: no cover — bs4 is a transitive dep but discovery degrades gracefully without it.
+ BeautifulSoup = None # type: ignore[assignment, misc]
+
+from skyvern.forge.sdk.copilot.build_phase import (
+ BuildPhase,
+ advance_to_composing,
+ advance_to_discovering,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+
+from ._shared import (
+ _DISCOVERY_ANTI_BOT_PATTERNS,
+ _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ _composition_get_html,
+ _discovery_extract_current_url,
+ _discovery_navigate,
+)
+from .guardrails import _authority_tool_error
+
+LOG = structlog.get_logger()
+
+# Build-time entrypoint discovery: navigates and reads pages, returns a
+# candidate URL into the agent's context. Never mutates workflow YAML.
+# Available only during INITIAL / DISCOVERING phases.
+
+_DISCOVERY_PER_CHAT_BUDGET = 3
+_DISCOVERY_PER_TURN_BUDGET = 1
+_DISCOVERY_WALL_CLOCK_SECONDS = 60.0
+_DISCOVERY_STEP_CAP = 8
+_DISCOVERY_EVIDENCE_TRAIL_MAX = 8
+_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX = 10
+_DISCOVERY_HTML_BYTES_MAX = 200_000
+_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE = 0.6
+
+_DISCOVERY_URL_SCHEME_RE = re.compile(r"^https?://", re.IGNORECASE)
+# host + optional path/query/fragment — handles `example.com/login`,
+# `the-internet.herokuapp.com/tables?x=y`, `site.com?q=x`, `site.com#frag`.
+_DISCOVERY_DOMAIN_WITH_PATH_RE = re.compile(
+ r"^[a-z0-9-]+(\.[a-z]{2,})+([/?#][^\s]*)?$",
+ re.IGNORECASE,
+)
+_DISCOVERY_BARE_WORD_RE = re.compile(r"^[a-z0-9-]{2,32}$", re.IGNORECASE)
+_DISCOVERY_TOKEN_RE = re.compile(r"[a-z0-9]+", re.IGNORECASE)
+_DISCOVERY_CANDIDATE_EVIDENCE_STOPWORDS = frozenset({"a", "an", "and", "for", "in", "of", "on", "or", "the", "to"})
+_DISCOVERY_LOGIN_TITLE_RE = re.compile(r"\b(sign\s*in|log\s*in|login)\b", re.IGNORECASE)
+_DISCOVERY_PASSWORD_INPUT_RE = re.compile(
+ r"<input[^>]*type\s*=\s*[\"']password[\"']",
+ re.IGNORECASE,
+)
+
+
+def _resolve_discovery_entry_url(site_or_url: str) -> tuple[str | None, str]:
+ """Resolve the user-supplied site name/URL into a navigable URL.
+
+ Returns ``(resolved_url, kind)`` where ``kind`` is one of:
+ ``url`` / ``domain`` / ``word`` / ``unresolved``.
+ """
+ token = (site_or_url or "").strip()
+ if not token:
+ return None, "unresolved"
+ if _DISCOVERY_URL_SCHEME_RE.match(token):
+ return token, "url"
+ if _DISCOVERY_DOMAIN_WITH_PATH_RE.match(token):
+ return f"https://{token}", "domain"
+ if _DISCOVERY_BARE_WORD_RE.match(token):
+ return f"https://www.{token.lower()}.com", "word"
+ return None, "unresolved"
+
+
+def _concrete_homepage_entrypoint(entry_url: str | None, kind: str) -> str | None:
+ if kind not in {"domain", "url"} or not entry_url:
+ return None
+ try:
+ parsed = urlparse(entry_url)
+ except Exception:
+ return None
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ return None
+ if parsed.path not in {"", "/"} or parsed.params or parsed.query or parsed.fragment:
+ return None
+ return f"{parsed.scheme}://{parsed.netloc}/"
+
+
+def _discovery_anchor_score(
+ anchor_text: str,
+ anchor_title: str,
+ href_path: str,
+ intent_tokens: set[str],
+) -> int:
+ """Count intent tokens that appear as substrings of the combined anchor text.
+
+ Substring (not exact-token) matching handles ``sort`` ↔ ``sortable`` and
+ ``table`` ↔ ``tables`` without a full stemmer.
+ """
+ if not intent_tokens:
+ return 0
+ combined = f"{anchor_text} {anchor_title} {href_path}".lower()
+ return sum(1 for token in intent_tokens if token in combined)
+
+
+def _discovery_title_score(page_title: str, intent_tokens: set[str]) -> int:
+ if not intent_tokens or not page_title:
+ return 0
+ lowered = page_title.lower()
+ return sum(1 for token in intent_tokens if token in lowered)
+
+
+def _discovery_candidate_evidence_tokens(intent_tokens: set[str]) -> set[str]:
+ return {
+ token
+ for token in intent_tokens
+ if len(token) > 2 and token.lower() not in _DISCOVERY_CANDIDATE_EVIDENCE_STOPWORDS
+ }
+
+
+def _discovery_detect_login_wall(html: str, page_title: str) -> bool:
+ if _DISCOVERY_LOGIN_TITLE_RE.search(page_title or ""):
+ return True
+ return bool(_DISCOVERY_PASSWORD_INPUT_RE.search(html or ""))
+
+
+def _discovery_detect_anti_bot(html: str, page_title: str) -> bool:
+ lowered_title = (page_title or "").lower()
+ lowered_html = (html or "")[:_DISCOVERY_HTML_BYTES_MAX].lower()
+ return any(pat in lowered_title or pat in lowered_html for pat in _DISCOVERY_ANTI_BOT_PATTERNS)
+
+
+def _discovery_build_result(
+ *,
+ candidate_url: str | None,
+ candidate_form_fields: list[dict[str, Any]],
+ evidence_trail: list[dict[str, Any]],
+ confidence: float,
+ failure_reason: str | None,
+ ok: bool = True,
+ error: str | None = None,
+) -> dict[str, Any]:
+ """Shape a `discover_workflow_entrypoint` result envelope.
+
+ Convention: ``ok=True`` for any *completed* walk — including controlled
+ outcomes that report a ``failure_reason`` and ``candidate_url=None``.
+ ``ok=False`` is reserved for actual tool errors (MCP unavailable, browser
+ boot failure, internal exception). Matches the existing copilot
+ ``_request_policy_tool_error`` convention so the eval harness counts a
+ controlled failure as a successful tool call.
+ """
+ return {
+ "ok": ok,
+ "data": {
+ "candidate_url": candidate_url,
+ "candidate_form_fields": candidate_form_fields[:_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX],
+ "evidence_trail": evidence_trail[:_DISCOVERY_EVIDENCE_TRAIL_MAX],
+ "confidence": float(confidence),
+ "failure_reason": failure_reason,
+ },
+ "error": error,
+ }
+
+
+def _record_discovery_resolution_on_ctx(ctx: Any, result: Mapping[str, Any]) -> None:
+ data_payload = result.get("data")
+ data: Mapping[str, Any] = data_payload if isinstance(data_payload, Mapping) else {}
+ candidate_url = data.get("candidate_url")
+ failure_reason = data.get("failure_reason")
+ if isinstance(candidate_url, str) and candidate_url:
+ prior_candidate_url = getattr(ctx, "resolved_discovery_entrypoint_url", None)
+ ctx.resolved_discovery_entrypoint_url = candidate_url
+ ctx.resolved_discovery_failure_reason = (
+ failure_reason if isinstance(failure_reason, str) and failure_reason else None
+ )
+ if prior_candidate_url != candidate_url:
+ ctx.resolved_discovery_entrypoint_inspection_baseline = int(
+ getattr(ctx, "page_inspection_calls_this_turn", 0) or 0
+ )
+ ctx.discovery_entrypoint_url_question_nudge_count = 0
+ # Prior successful candidates remain authoritative over later no-candidate failures.
+ elif not getattr(ctx, "resolved_discovery_entrypoint_url", None):
+ # No prior candidate and no new one: clear URL state while recording the failure.
+ ctx.resolved_discovery_entrypoint_url = None
+ ctx.resolved_discovery_failure_reason = (
+ failure_reason if isinstance(failure_reason, str) and failure_reason else None
+ )
+ ctx.resolved_discovery_entrypoint_inspection_baseline = 0
+ try:
+ current_span = otel_trace.get_current_span()
+ if ctx.resolved_discovery_entrypoint_url is not None:
+ current_span.set_attribute("copilot.discovery_candidate_url", ctx.resolved_discovery_entrypoint_url)
+ if ctx.resolved_discovery_failure_reason is not None:
+ current_span.set_attribute("copilot.discovery_failure_reason", ctx.resolved_discovery_failure_reason)
+ except Exception:
+ LOG.debug("Unable to set discovery resolution span attributes", exc_info=True)
+
+
+def _discovery_parse_html(html: str) -> tuple[str, list[dict[str, str]], list[dict[str, str]]]:
+ """Parse HTML for page title, link anchors, and form-field metadata.
+
+ Uses BeautifulSoup if available (a transitive Skyvern dep). Falls back to
+ empty results if not — discovery degrades gracefully rather than crashing.
+ """
+ if BeautifulSoup is None:
+ return "", [], []
+
+ try:
+ soup = BeautifulSoup(html or "", "html.parser")
+ except Exception:
+ return "", [], []
+
+ title_text = ""
+ title_tag = soup.find("title")
+ if title_tag and title_tag.string:
+ title_text = title_tag.string.strip()
+ h1_tag = soup.find("h1")
+ if h1_tag:
+ h1_text = h1_tag.get_text(strip=True)
+ if h1_text:
+ title_text = f"{title_text} {h1_text}".strip()
+
+ anchors: list[dict[str, str]] = []
+ for a in soup.find_all("a", href=True):
+ href = a.get("href", "").strip()
+ if not href or href.startswith("#") or href.lower().startswith("javascript:"):
+ continue
+ text = a.get_text(" ", strip=True)
+ anchors.append(
+ {
+ "href": href,
+ "text": text[:240],
+ "title": (a.get("title") or "")[:240],
+ }
+ )
+
+ form_fields: list[dict[str, str]] = []
+ form = soup.find("form")
+ if form is not None:
+ for inp in form.find_all(["input", "select", "textarea"]):
+ field_type = inp.get("type", inp.name) or "text"
+ if field_type.lower() in {"hidden", "submit", "button"}:
+ continue
+ field_name = inp.get("name") or inp.get("id") or ""
+ label_text = ""
+ label_id = inp.get("id")
+ if label_id:
+ label_tag = soup.find("label", attrs={"for": label_id})
+ if label_tag is not None:
+ label_text = label_tag.get_text(" ", strip=True)
+ form_fields.append(
+ {
+ "name": field_name[:120],
+ "label": label_text[:240],
+ "type": str(field_type)[:40],
+ "value_hint": (inp.get("placeholder") or "")[:240],
+ }
+ )
+
+ return title_text, anchors, form_fields
+
+
+def _discovery_resolve_href(base_url: str, href: str) -> str | None:
+ try:
+ absolute = urljoin(base_url, href)
+ except Exception:
+ return None
+ parsed_abs = urlparse(absolute)
+ parsed_base = urlparse(base_url)
+ if parsed_abs.scheme not in {"http", "https"}:
+ return None
+ # Same-origin only — discovery does not follow cross-origin links to keep
+ # the entrypoint search bounded to the user's named site.
+ if parsed_abs.netloc and parsed_base.netloc and parsed_abs.netloc != parsed_base.netloc:
+ return None
+ return absolute
+
+
+def _discovery_origin_url(url: str) -> str | None:
+ try:
+ parsed = urlparse(url)
+ except Exception:
+ return None
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ return None
+ return f"{parsed.scheme}://{parsed.netloc}/"
+
+
+def _discovery_should_retry_from_origin(entry_url: str, current_url: str) -> bool:
+ try:
+ parsed_entry = urlparse(entry_url)
+ parsed_current = urlparse(current_url)
+ except Exception:
+ return False
+ if parsed_current.scheme not in {"http", "https"} or not parsed_current.netloc:
+ return False
+ if parsed_entry.netloc and parsed_entry.netloc != parsed_current.netloc:
+ return False
+ return bool(parsed_current.path not in {"", "/"} or parsed_current.query)
+
+
+def _discovery_anchor_selector(anchor: dict[str, str]) -> str | None:
+ href = (anchor.get("href") or "").strip()
+ if not href or any(char in href for char in {'"', "\\", "\n", "\r"}):
+ return None
+ return f'a[href="{href}"]'
+
+
+_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE = 0.2
+# Scorer-miss outcomes (page loaded, no keyword match); wall outcomes are excluded as real blocks.
+_DISCOVERY_SCORER_MISS_REASONS = frozenset({"no_candidate", "step_limit", "wall_clock_limit"})
+
+
+def _discovery_last_loaded_url(evidence_trail: list[dict[str, Any]]) -> str | None:
+ """URL of the last page the walk loaded, skipping failed nav/click steps and detected walls."""
+ for entry in reversed(evidence_trail):
+ reason = str(entry.get("transition_reason", ""))
+ if reason.startswith("navigate_failed") or reason.startswith("anchor_click_failed"):
+ continue
+ if entry.get("wall"):
+ continue
+ url = entry.get("url")
+ if isinstance(url, str) and url:
+ return url
+ return None
+
+
+async def _discovery_click_anchor(ctx: CopilotContext, anchor: dict[str, str]) -> dict[str, Any]:
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return {"ok": False, "error": "discovery MCP server not attached to context"}
+ selector = _discovery_anchor_selector(anchor)
+ if selector is None:
+ return {"ok": False, "error": "anchor href could not be converted to a bounded CSS selector"}
+ try:
+ return await asyncio.wait_for(
+ # call_internal_tool bypasses the schema overlays, so selector_mode="direct" must be
+ # passed explicitly here (it is not picked up from the overlay's forced_args).
+ server.call_internal_tool("skyvern_click", {"selector": selector, "selector_mode": "direct"}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ return {"ok": False, "error": f"skyvern_click timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
+
+
+async def _discovery_walk(
+ ctx: CopilotContext,
+ *,
+ entry_url: str,
+ intent_hint: str,
+) -> dict[str, Any]:
+ """Deterministic anchor-scoring walker. No inner LLM call.
+
+ Reads each visited page, looks for a strong title/H1 match with the
+ intent_hint, surfaces a form if one is present, and otherwise follows
+ the highest-scored same-origin anchor whose text matches the
+ intent_hint. Bounded by step / wall-clock caps.
+ """
+ intent_tokens = set(_DISCOVERY_TOKEN_RE.findall(intent_hint.lower())) if intent_hint else set()
+ evidence_trail: list[dict[str, Any]] = []
+ current_url = entry_url
+ current_page_loaded = False
+ retried_deep_link_from_origin = False
+ started = ctx.discovery_started_monotonic or time.monotonic()
+
+ for step in range(_DISCOVERY_STEP_CAP):
+ ctx.discovery_step_count = step + 1
+ elapsed = time.monotonic() - started
+ if elapsed > _DISCOVERY_WALL_CLOCK_SECONDS:
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="wall_clock_limit",
+ )
+
+ if current_page_loaded:
+ current_page_loaded = False
+ else:
+ nav_result = await _discovery_navigate(ctx, current_url)
+ if not nav_result.get("ok"):
+ evidence_trail.append(
+ {
+ "url": current_url,
+ "page_title": "",
+ "transition_reason": f"navigate_failed: {nav_result.get('error', 'unknown')}"[:240],
+ }
+ )
+ # A pre-composition browser/session failure is not evidence that
+ # the user omitted a page URL. Return the resolved entry URL so the
+ # agent can test a minimal goto_url block and gather real run/debug
+ # evidence before deciding whether to ask a follow-up.
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
+ failure_reason=None,
+ )
+
+ current_url = _discovery_extract_current_url(nav_result, current_url)
+ # Survive the MCP size cap: a heavy DOM exceeds it and the html field is dropped, so
+ # fall back to a stripped-body evaluate that keeps the links/forms the resolver needs
+ # to identify a usable entrypoint. (Discovery only resolves the entrypoint, so a sliced
+ # tail does not matter here.)
+ html, _, _, _ = await _composition_get_html(ctx)
+ page_title, anchors, form_fields = _discovery_parse_html(html)
+
+ evidence_trail.append(
+ {
+ "url": current_url,
+ "page_title": page_title[:240],
+ "transition_reason": "initial" if step == 0 else "anchor_match",
+ }
+ )
+
+ title_score = _discovery_title_score(page_title, intent_tokens)
+
+ best_score = 0
+ best_href: str | None = None
+ best_anchor: dict[str, str] | None = None
+ for anchor in anchors:
+ score = _discovery_anchor_score(
+ anchor.get("text", ""),
+ anchor.get("title", ""),
+ anchor.get("href", ""),
+ intent_tokens,
+ )
+ if score > best_score:
+ resolved = _discovery_resolve_href(current_url, anchor.get("href", ""))
+ if resolved is None:
+ continue
+ best_score = score
+ best_href = resolved
+ best_anchor = anchor
+
+ evidence_tokens = _discovery_candidate_evidence_tokens(intent_tokens)
+ candidate_title_score = _discovery_title_score(page_title, evidence_tokens)
+ candidate_anchor_score = 0
+ for anchor in anchors:
+ candidate_anchor_score = max(
+ candidate_anchor_score,
+ _discovery_anchor_score(
+ anchor.get("text", ""),
+ anchor.get("title", ""),
+ anchor.get("href", ""),
+ evidence_tokens,
+ ),
+ )
+
+ anti_bot_detected = _discovery_detect_anti_bot(html, page_title)
+ login_wall_detected = _discovery_detect_login_wall(html, page_title)
+ anti_bot_has_no_candidate_evidence = (
+ not form_fields and candidate_title_score == 0 and candidate_anchor_score == 0
+ )
+ if anti_bot_detected or login_wall_detected:
+ # Tag walls so the scorer-miss fallback refuses them even when they degrade to no_candidate.
+ evidence_trail[-1]["wall"] = True
+ if anti_bot_detected and anti_bot_has_no_candidate_evidence:
+ origin_url = _discovery_origin_url(current_url)
+ if (
+ not retried_deep_link_from_origin
+ and origin_url
+ and origin_url != current_url
+ and _discovery_should_retry_from_origin(entry_url, current_url)
+ ):
+ evidence_trail[-1]["transition_reason"] = "direct_deep_link_anti_bot"
+ current_url = origin_url
+ retried_deep_link_from_origin = True
+ continue
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="anti_bot_wall",
+ )
+ if login_wall_detected:
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="login_wall",
+ )
+
+ if intent_tokens and title_score >= 2 and (form_fields or best_score <= title_score):
+ confidence = min(1.0, title_score / max(1, len(intent_tokens)))
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=form_fields,
+ evidence_trail=evidence_trail,
+ confidence=confidence,
+ failure_reason=None,
+ )
+
+ if form_fields and (title_score >= 1 or step > 0):
+ confidence = 0.6 if title_score >= 1 else 0.4
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=form_fields,
+ evidence_trail=evidence_trail,
+ confidence=confidence,
+ failure_reason=None,
+ )
+
+ if not intent_tokens:
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=form_fields,
+ evidence_trail=evidence_trail,
+ confidence=0.3,
+ failure_reason=None,
+ )
+
+ if best_score == 0 or best_href is None:
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="no_candidate",
+ )
+
+ if retried_deep_link_from_origin and best_anchor is not None:
+ click_result = await _discovery_click_anchor(ctx, best_anchor)
+ if click_result.get("ok"):
+ current_url = _discovery_extract_current_url(click_result, best_href)
+ # The next loop should inspect the clicked page instead of
+ # navigating back to the original entry URL.
+ current_page_loaded = True
+ continue
+ evidence_trail.append(
+ {
+ "url": best_href,
+ "page_title": "",
+ "transition_reason": f"anchor_click_failed: {click_result.get('error', 'unknown')}"[:240],
+ }
+ )
+
+ current_url = best_href
+
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="step_limit",
+ )
+
+
+async def _discover_workflow_entrypoint_impl(
+ copilot_ctx: Any,
+ site_or_url: str,
+ intent_hint: str,
+) -> dict[str, Any]:
+ """Discovery tool body — separated from the @function_tool wrapper so
+ tests can drive it with a stand-in ctx without the SDK's invocation
+ machinery.
+ """
+ arguments = {"site_or_url": site_or_url, "intent_hint": intent_hint}
+
+ def finish(result: dict[str, Any], *, site_or_url_kind: str | None = None) -> dict[str, Any]:
+ _record_discovery_resolution_on_ctx(copilot_ctx, result)
+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
+ data_payload = result.get("data")
+ data = data_payload if isinstance(data_payload, Mapping) else {}
+ candidate_url = data.get("candidate_url")
+ failure_reason = data.get("failure_reason")
+ if not isinstance(failure_reason, str) or not failure_reason:
+ error = result.get("error")
+ failure_reason = error if isinstance(error, str) and error else None
+ LOG.info(
+ "discover_workflow_entrypoint completed",
+ ok=result.get("ok"),
+ candidate_url=candidate_url if isinstance(candidate_url, str) and candidate_url else None,
+ failure_reason=failure_reason,
+ site_or_url_kind=site_or_url_kind,
+ )
+ return result
+
+ authority_error = _authority_tool_error(copilot_ctx, "discover_workflow_entrypoint")
+ if authority_error:
+ result = {"ok": False, "error": authority_error}
+ return finish(result)
+
+ if copilot_ctx.discovery_calls_this_turn >= _DISCOVERY_PER_TURN_BUDGET:
+ result = _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=list(copilot_ctx.discovery_evidence_trail),
+ confidence=0.0,
+ failure_reason="discovery_already_completed_this_turn",
+ )
+ return finish(result)
+
+ cumulative = copilot_ctx.prior_discovery_calls_made + copilot_ctx.discovery_calls_this_turn
+ if cumulative >= _DISCOVERY_PER_CHAT_BUDGET:
+ result = _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=[],
+ confidence=0.0,
+ failure_reason="discovery_budget_exhausted_for_chat",
+ )
+ return finish(result)
+
+ entry_url, kind = _resolve_discovery_entry_url(site_or_url)
+ if entry_url is None:
+ result = _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=[],
+ confidence=0.0,
+ failure_reason="could_not_resolve_site_name",
+ )
+ return finish(result, site_or_url_kind=kind)
+
+ if copilot_ctx.build_phase == BuildPhase.INITIAL:
+ try:
+ advance_to_discovering(copilot_ctx)
+ except ValueError as exc:
+ # Race or unexpected prior advance — proceed without re-transitioning,
+ # but surface the impossible state so it shows up in production logs.
+ LOG.warning(
+ "discover_workflow_entrypoint phase transition to discovering rejected",
+ error=str(exc),
+ build_phase=copilot_ctx.build_phase.value,
+ )
+ copilot_ctx.discovery_calls_this_turn += 1
+
+ concrete_homepage_url = _concrete_homepage_entrypoint(entry_url, kind)
+ if concrete_homepage_url is not None:
+ evidence_trail = [
+ {
+ "url": concrete_homepage_url,
+ "page_title": "",
+ "transition_reason": "concrete_domain_homepage",
+ }
+ ]
+ result = _discovery_build_result(
+ candidate_url=concrete_homepage_url,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ # Lower than a scraped-page match because the fast path skips page inspection.
+ confidence=_DISCOVERY_CONCRETE_HOMEPAGE_CONFIDENCE,
+ failure_reason=None,
+ )
+ copilot_ctx.discovery_evidence_trail = list(evidence_trail)
+ try:
+ advance_to_composing(copilot_ctx, reason="discovery_concrete_domain_homepage")
+ except ValueError as exc:
+ LOG.warning(
+ "discover_workflow_entrypoint phase transition to composing rejected",
+ error=str(exc),
+ build_phase=copilot_ctx.build_phase.value,
+ )
+ return finish(result, site_or_url_kind=kind)
+
+ with copilot_span(
+ "discover_workflow_entrypoint",
+ data={
+ "site_or_url_kind": kind,
+ "intent_hint_len": len(intent_hint or ""),
+ "phase_entered": copilot_ctx.build_phase.value,
+ },
+ ):
+ try:
+ result = await _discovery_walk(
+ copilot_ctx,
+ entry_url=entry_url,
+ intent_hint=intent_hint or "",
+ )
+ except Exception as exc:
+ LOG.exception("discover_workflow_entrypoint walker raised")
+ result = {
+ "ok": False,
+ "data": {
+ "candidate_url": None,
+ "candidate_form_fields": [],
+ "evidence_trail": [],
+ "confidence": 0.0,
+ "failure_reason": None,
+ },
+ "error": f"discover_workflow_entrypoint failed: {exc}",
+ }
+
+ data_payload = result.get("data") or {}
+ data: dict[str, Any] = data_payload if isinstance(data_payload, dict) else {}
+ evidence_trail = data.get("evidence_trail") or []
+ copilot_ctx.discovery_evidence_trail = list(evidence_trail)
+ if (
+ result.get("ok")
+ and not data.get("candidate_url")
+ and data.get("failure_reason") in _DISCOVERY_SCORER_MISS_REASONS
+ ):
+ navigated_url = _discovery_last_loaded_url(evidence_trail)
+ if navigated_url:
+ # Don't ask for a URL we already reached; hand the loaded page to composition at low confidence.
+ result = _discovery_build_result(
+ candidate_url=navigated_url,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=_DISCOVERY_NAVIGATION_FALLBACK_CONFIDENCE,
+ failure_reason=None,
+ )
+ rebuilt_data = result.get("data")
+ data = rebuilt_data if isinstance(rebuilt_data, dict) else {}
+ copilot_ctx.discovery_evidence_trail = list(data.get("evidence_trail", []))
+ if result.get("ok") and data.get("candidate_url"):
+ try:
+ advance_to_composing(copilot_ctx, reason="discovery_returned_candidate")
+ except ValueError as exc:
+ LOG.warning(
+ "discover_workflow_entrypoint phase transition to composing rejected",
+ error=str(exc),
+ build_phase=copilot_ctx.build_phase.value,
+ )
+
+ return finish(result, site_or_url_kind=kind)
diff --git a/skyvern/forge/sdk/copilot/tools/frontier.py b/skyvern/forge/sdk/copilot/tools/frontier.py
new file mode 100644
index 000000000..d4a5d4057
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/frontier.py
@@ -0,0 +1,1120 @@
+"""Copilot agent tools — native handlers, hooks, and registration."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+from typing import Any
+from urllib.parse import urlparse
+
+import structlog
+import yaml
+
+try:
+ from bs4 import BeautifulSoup # type: ignore[import-not-found]
+except ImportError: # pragma: no cover — bs4 is a transitive dep but discovery degrades gracefully without it.
+ BeautifulSoup = None # type: ignore[assignment, misc]
+from jinja2.sandbox import SandboxedEnvironment
+from pydantic import ValidationError
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_workflow_block_goals
+from skyvern.forge.sdk.copilot.build_phase import (
+ BuildPhase,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.failure_tracking import (
+ _canonical_block_config,
+)
+from skyvern.forge.sdk.copilot.runtime import (
+ AgentContext,
+)
+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
+from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, get_all_blocks
+from skyvern.forge.sdk.workflow.models.parameter import (
+ RESERVED_PARAMETER_KEYS,
+ Parameter,
+)
+from skyvern.forge.sdk.workflow.models.workflow import Workflow
+from skyvern.schemas.workflows import BlockType
+
+from ._shared import (
+ _block_type_name,
+ _valid_runtime_anchor_url,
+ _workflow_definition_block_labels,
+ _workflow_yaml_blocks_by_label,
+)
+
+LOG = structlog.get_logger()
+
+
+_BLOCK_TYPES_STATE_ESTABLISHER = frozenset({"navigation", "login", "goto_url"})
+
+_JINJA_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*"
+_OUTPUT_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})_output\s*(?=[\.|}}])")
+_BLOCK_FORM_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*\.")
+_JINJA_ROOT_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*(?=[\.|}}])")
+
+_JINJA_RUNTIME_GLOBAL_ROOTS = frozenset(SandboxedEnvironment().globals)
+_JINJA_LITERAL_ROOTS = frozenset({"none", "true", "false"})
+_JINJA_SPECIAL_CONTEXT_ROOTS = frozenset({"loop", "self", "varargs", "kwargs"})
+_SKYVERN_TEMPLATE_CONTEXT_ROOTS = frozenset(RESERVED_PARAMETER_KEYS) | frozenset(
+ {
+ "parameters",
+ "browser_session_id",
+ "organization_id",
+ # Conditional / branch evaluation roots — see BranchEvaluationContext.build_template_data.
+ "params",
+ "outputs",
+ "environment",
+ "env",
+ "llm",
+ }
+)
+_TEMPLATE_BUILTIN_ROOTS = (
+ _JINJA_RUNTIME_GLOBAL_ROOTS | _JINJA_LITERAL_ROOTS | _JINJA_SPECIAL_CONTEXT_ROOTS | _SKYVERN_TEMPLATE_CONTEXT_ROOTS
+)
+
+# Keep this to grammatical glue only. Workflow/action words are intentionally
+# not filtered; the two-token stale threshold is the conservative guardrail.
+_BLOCK_METADATA_STOPWORDS = frozenset({"and", "for", "the", "with"})
+
+
+def _blocks_by_label(workflow_definition: object | None) -> dict[str, object]:
+ blocks = getattr(workflow_definition, "blocks", None) if workflow_definition else None
+ by_label: dict[str, object] = {}
+ if not blocks:
+ return by_label
+ for block in blocks:
+ label = getattr(block, "label", None)
+ if isinstance(label, str):
+ by_label[label] = block
+ return by_label
+
+
+# Minimum length to apply the trailing-``s`` plural strip; below this we
+# leave the token alone so words like ``is``/``us``/``has`` aren't mangled.
+_MIN_STEMMABLE_TOKEN_LEN = 5
+
+
+def _metadata_token(token: str) -> str:
+ token = token.lower()
+ if len(token) >= _MIN_STEMMABLE_TOKEN_LEN and token.endswith("s"):
+ token = token[:-1]
+ return token
+
+
+def _metadata_tokens(value: Any) -> set[str]:
+ if not isinstance(value, str):
+ return set()
+ tokens: set[str] = set()
+ for token in re.findall(r"[A-Za-z0-9]+", value):
+ normalized = _metadata_token(token)
+ if len(normalized) <= 2 or normalized in _BLOCK_METADATA_STOPWORDS:
+ continue
+ tokens.add(normalized)
+ return tokens
+
+
+def _semantic_tokens_from_yaml(value: Any, *, exclude_keys: frozenset[str]) -> set[str]:
+ tokens: set[str] = set()
+ if isinstance(value, str):
+ return _metadata_tokens(value)
+ if isinstance(value, list):
+ for item in value:
+ tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))
+ return tokens
+ if isinstance(value, dict):
+ for key, item in value.items():
+ if key in exclude_keys:
+ continue
+ tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))
+ return tokens
+
+
+def _stale_metadata_reason(
+ *,
+ field_name: str,
+ field_value: Any,
+ prior_block: dict[str, Any],
+ submitted_block: dict[str, Any],
+ current_exclude_keys: frozenset[str],
+) -> str | None:
+ tokens = _metadata_tokens(field_value)
+ if len(tokens) < 2:
+ return None
+
+ prior_tokens = _semantic_tokens_from_yaml(prior_block, exclude_keys=current_exclude_keys)
+ current_tokens = _semantic_tokens_from_yaml(submitted_block, exclude_keys=current_exclude_keys)
+ removed_tokens = prior_tokens - current_tokens
+ if len(tokens & removed_tokens) < 2:
+ return None
+
+ return f"{field_name} {field_value!r} appears stale"
+
+
+def _prior_blocks_by_unique_title(prior_by_label: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
+ titled = [(b["title"], b) for b in prior_by_label.values() if isinstance(b.get("title"), str) and b["title"]]
+ counts = Counter(title for title, _ in titled)
+ return {title: block for title, block in titled if counts[title] == 1}
+
+
+_STALE_BASE_EXCLUDE = frozenset({"label", "next_block_label"})
+_STALE_TITLE_EXCLUDE = frozenset({"label", "title", "next_block_label"})
+
+
+def _stale_for_renamed_label(
+ label: str,
+ submitted_block: dict[str, Any],
+ prior_by_unique_title: dict[str, dict[str, Any]],
+) -> dict[str, Any] | None:
+ submitted_title = submitted_block.get("title")
+ # Title-less blocks intentionally skip the renamed-label path: with no
+ # title there is no cross-reference to a prior block, so we'd be
+ # guessing whether the rename was warranted.
+ if not (isinstance(submitted_title, str) and submitted_title):
+ return None
+ prior_block = prior_by_unique_title.get(submitted_title)
+ if prior_block is None:
+ return None
+ title_reason = _stale_metadata_reason(
+ field_name="title",
+ field_value=submitted_title,
+ prior_block=prior_block,
+ submitted_block=submitted_block,
+ current_exclude_keys=_STALE_TITLE_EXCLUDE,
+ )
+ if not title_reason:
+ return None
+ return {"label": label, "reasons": [title_reason]}
+
+
+def _stale_for_matched_label(
+ label: str,
+ submitted_block: dict[str, Any],
+ prior_block: dict[str, Any],
+) -> dict[str, Any] | None:
+ reasons: list[str] = []
+ submitted_title = submitted_block.get("title")
+ prior_title = prior_block.get("title")
+ title_reason = None
+ if isinstance(submitted_title, str) and submitted_title and submitted_title == prior_title:
+ title_reason = _stale_metadata_reason(
+ field_name="title",
+ field_value=submitted_title,
+ prior_block=prior_block,
+ submitted_block=submitted_block,
+ current_exclude_keys=_STALE_TITLE_EXCLUDE,
+ )
+ if title_reason:
+ reasons.append(title_reason)
+
+ # When the title was already flagged stale, exclude its tokens from the
+ # label-stale comparison so the same words don't double-count; otherwise
+ # the title's tokens are stable content that should weigh in.
+ label_exclude_keys = _STALE_TITLE_EXCLUDE if title_reason else _STALE_BASE_EXCLUDE
+ label_reason = _stale_metadata_reason(
+ field_name="label",
+ field_value=label,
+ prior_block=prior_block,
+ submitted_block=submitted_block,
+ current_exclude_keys=label_exclude_keys,
+ )
+ if label_reason:
+ reasons.insert(0, label_reason)
+
+ if not reasons:
+ return None
+ return {"label": label, "reasons": reasons}
+
+
+def _detect_stale_block_metadata(submitted_yaml: str | None, prior_yaml: str | None) -> list[dict[str, Any]]:
+ """Find corrected blocks whose old label/title no longer matches their revised goal text."""
+ prior_by_label = _workflow_yaml_blocks_by_label(prior_yaml)
+ submitted_by_label = _workflow_yaml_blocks_by_label(submitted_yaml)
+ if not prior_by_label or not submitted_by_label:
+ return []
+
+ prior_by_unique_title = _prior_blocks_by_unique_title(prior_by_label)
+
+ stale_items: list[dict[str, Any]] = []
+ for label, submitted_block in submitted_by_label.items():
+ prior_block = prior_by_label.get(label)
+ if prior_block is None:
+ item = _stale_for_renamed_label(label, submitted_block, prior_by_unique_title)
+ else:
+ item = _stale_for_matched_label(label, submitted_block, prior_block)
+ if item is not None:
+ stale_items.append(item)
+ return stale_items
+
+
+_STALE_BLOCK_METADATA_MESSAGE_LIMIT = 5
+
+
+def _stale_block_metadata_message(items: list[dict[str, Any]]) -> str:
+ details = []
+ for item in items[:_STALE_BLOCK_METADATA_MESSAGE_LIMIT]:
+ label = item.get("label", "?")
+ reasons = item.get("reasons") or []
+ detail = "; ".join(str(reason) for reason in reasons)
+ details.append(f"{label}: {detail}")
+ if len(items) > _STALE_BLOCK_METADATA_MESSAGE_LIMIT:
+ details.append(f"(and {len(items) - _STALE_BLOCK_METADATA_MESSAGE_LIMIT} more)")
+ joined = "; ".join(details)
+ return (
+ "Workflow validation failed: corrected block metadata still appears stale. "
+ "When changing a user's requested subject, URL, or action, rename affected block labels and titles "
+ "to match the revised goal, and update next_block_label, block_labels, and Jinja references accordingly. "
+ f"Stale metadata: {joined}"
+ )
+
+
+def _find_invalidated_labels(
+ old_definition: object | None,
+ new_definition: object | None,
+ requested_labels: list[str],
+) -> set[str]:
+ """Return the set of requested labels whose behavior is invalidated.
+
+ A label is invalidated when its own config changed or when any upstream
+ label in the requested chain was invalidated (downstream trust propagates
+ forward).
+ """
+ old_by_label = _blocks_by_label(old_definition)
+ new_by_label = _blocks_by_label(new_definition)
+ invalidated: set[str] = set()
+ upstream_invalidated = False
+ for label in requested_labels:
+ if upstream_invalidated:
+ invalidated.add(label)
+ continue
+ old_block = old_by_label.get(label)
+ new_block = new_by_label.get(label)
+ if old_block is None or new_block is None:
+ invalidated.add(label)
+ upstream_invalidated = True
+ continue
+ if _canonical_block_config(old_block) != _canonical_block_config(new_block):
+ invalidated.add(label)
+ upstream_invalidated = True
+ return invalidated
+
+
+def _earliest_invalidated(requested_labels: list[str], invalidated: set[str]) -> str | None:
+ for label in requested_labels:
+ if label in invalidated:
+ return label
+ return None
+
+
+def _clear_runtime_anchor_evidence(copilot_ctx: Any) -> None:
+ # Clears the runtime-anchor *trust* flags only. evidence.current_url /
+ # page_title / workflow_run_id are left intact: an edit does not move the
+ # browser, so they remain accurate observational context — the cleared flags
+ # are what mark that state as no longer verified.
+ evidence = copilot_ctx.workflow_verification_evidence
+ copilot_ctx.verified_prefix_current_url = None
+ evidence.live_page_state_verified = False
+ evidence.verified_from_current_browser_state = False
+ evidence.current_url_observed_after_workflow_run = False
+ evidence.current_url_may_encode_runtime_state = False
+
+
+def _reset_all_verified_trust(copilot_ctx: Any) -> None:
+ evidence = copilot_ctx.workflow_verification_evidence
+ copilot_ctx.verified_prefix_labels = []
+ copilot_ctx.verified_block_outputs = {}
+ copilot_ctx.last_full_workflow_test_ok = False
+ evidence.block_verified = []
+ evidence.full_workflow_verified = False
+ _clear_runtime_anchor_evidence(copilot_ctx)
+
+
+def _workflow_parameters_changed(prior_definition: object | None, new_definition: object | None) -> bool:
+ new_by_key = {getattr(p, "key", None): p for p in (getattr(new_definition, "parameters", None) or [])}
+ prior_by_key = {getattr(p, "key", None): p for p in (getattr(prior_definition, "parameters", None) or [])}
+ # Any added or removed parameter is a change: a block may reference a key by
+ # template without a config edit, so an added/removed key can alter behavior
+ # the block-diff alone won't catch. Pure reordering is ignored (keyed access).
+ if set(prior_by_key) != set(new_by_key):
+ return True
+ for key, prior_param in prior_by_key.items():
+ try:
+ if _stable_parameter_fingerprint(prior_param) != _stable_parameter_fingerprint(new_by_key[key]):
+ return True
+ except Exception:
+ LOG.debug("Parameter fingerprint comparison failed on edit", exc_info=True)
+ return True
+ return False
+
+
+def _invalidate_verified_state_on_edit(
+ copilot_ctx: Any,
+ prior_definition: object | None,
+ new_definition: object | None,
+) -> None:
+ evidence = copilot_ctx.workflow_verification_evidence
+ if new_definition is None:
+ # An unknown new definition can't be reconciled against; fail closed.
+ if copilot_ctx.verified_prefix_labels or evidence.block_verified:
+ _reset_all_verified_trust(copilot_ctx)
+ return
+ prior_labels = _workflow_definition_block_labels(prior_definition)
+ trusted = set(copilot_ctx.verified_prefix_labels or []) | set(evidence.block_verified or [])
+
+ invalidated: set[str] = set()
+ if trusted:
+ # No reconcilable prior, or a parameter change that could alter any
+ # block's behavior — fail closed rather than reuse unproven trust.
+ if prior_definition is None or _workflow_parameters_changed(prior_definition, new_definition):
+ _reset_all_verified_trust(copilot_ctx)
+ return
+ # Diff the full prior chain, not just trusted labels, so a change to an
+ # unverified upstream block still propagates to downstream trusted ones.
+ full_order = list(prior_labels)
+ for label in list(copilot_ctx.verified_prefix_labels or []) + list(evidence.block_verified or []):
+ if label not in full_order:
+ full_order.append(label)
+ try:
+ invalidated = _find_invalidated_labels(prior_definition, new_definition, full_order) & trusted
+ except Exception:
+ LOG.debug("Verified-state invalidation diff failed on edit", exc_info=True)
+ _reset_all_verified_trust(copilot_ctx)
+ return
+
+ if invalidated:
+ copilot_ctx.verified_prefix_labels = [
+ label for label in copilot_ctx.verified_prefix_labels if label not in invalidated
+ ]
+ for label in invalidated:
+ copilot_ctx.verified_block_outputs.pop(label, None)
+ evidence.block_verified = [label for label in evidence.block_verified if label not in invalidated]
+ # The recorded prefix-end URL came from a run that included a now-invalid
+ # block, so it is no longer a safe runtime anchor for a re-run.
+ _clear_runtime_anchor_evidence(copilot_ctx)
+
+ # The kept prefix must still be a contiguous leading run of the new workflow;
+ # a reorder or upstream insertion breaks that (set-membership trust is
+ # order-blind), so fail closed when it no longer is.
+ current_labels = _workflow_definition_block_labels(new_definition)
+ remaining = list(copilot_ctx.verified_prefix_labels or [])
+ if remaining and remaining != current_labels[: len(remaining)]:
+ _reset_all_verified_trust(copilot_ctx)
+ return
+
+ # The end-to-end claim survives only an identical block list whose every block
+ # is still verified, so append/removal/reorder/config edits all drop it.
+ verified = set(remaining) | set(evidence.block_verified or [])
+ if not (current_labels == prior_labels and all(label in verified for label in current_labels)):
+ evidence.full_workflow_verified = False
+ copilot_ctx.last_full_workflow_test_ok = False
+
+
+def _nearest_upstream_state_establisher(
+ requested_labels: list[str], target_label: str, new_definition: object | None
+) -> str | None:
+ by_label = _blocks_by_label(new_definition)
+ try:
+ idx = requested_labels.index(target_label)
+ except ValueError:
+ return None
+ for candidate in reversed(requested_labels[:idx]):
+ block = by_label.get(candidate)
+ if block is None:
+ continue
+ if _block_type_name(block) in _BLOCK_TYPES_STATE_ESTABLISHER:
+ return candidate
+ return None
+
+
+def _block_can_start_browser_run(block: object) -> bool:
+ if _block_type_name(block) == BlockType.GOTO_URL.value:
+ return True
+ return _valid_runtime_anchor_url(getattr(block, "url", None)) is not None
+
+
+def _nearest_upstream_runnable_anchor(
+ workflow_labels: list[str], target_label: str, new_definition: object | None
+) -> str | None:
+ by_label = _blocks_by_label(new_definition)
+ try:
+ idx = workflow_labels.index(target_label)
+ except ValueError:
+ return None
+ for candidate in reversed(workflow_labels[:idx]):
+ block = by_label.get(candidate)
+ if block is not None and _block_can_start_browser_run(block):
+ return candidate
+ return workflow_labels[0] if workflow_labels[:idx] else None
+
+
+def _serialized_frontier_block_configs(frontier_labels: list[str], new_definition: object | None) -> list[str]:
+ by_label = _blocks_by_label(new_definition)
+ serialized_configs: list[str] = []
+ for label in frontier_labels:
+ block = by_label.get(label)
+ if block is None:
+ continue
+ try:
+ serialized_configs.append(json.dumps(_canonical_block_config(block), default=str, separators=(",", ":")))
+ except (TypeError, ValueError):
+ serialized_configs.append(repr(block))
+ return serialized_configs
+
+
+def _workflow_parameter_keys(definition: object | None) -> set[str]:
+ parameters = getattr(definition, "parameters", None) if definition else None
+ keys: set[str] = set()
+ if not parameters:
+ return keys
+ for parameter in parameters:
+ key = getattr(parameter, "key", None)
+ if isinstance(key, str):
+ keys.add(key)
+ return keys
+
+
+_CREDENTIAL_REAL_VALUE_SUFFIXES = ("_real_username", "_real_password")
+
+
+def _classify_frontier_jinja_refs(
+ frontier_labels: list[str],
+ new_definition: object | None,
+ serialized_configs: list[str] | None = None,
+) -> tuple[set[str], set[str], set[str]]:
+ """Single pass over frontier blocks; returns ``(suffix_form_refs, block_form_refs, unknown_roots)``."""
+ if serialized_configs is None:
+ serialized_configs = _serialized_frontier_block_configs(frontier_labels, new_definition)
+ known_labels = set(_blocks_by_label(new_definition))
+ parameter_keys = _workflow_parameter_keys(new_definition)
+ known_roots = known_labels | parameter_keys | _TEMPLATE_BUILTIN_ROOTS
+
+ suffix_form_refs: set[str] = set()
+ block_form_refs: set[str] = set()
+ unknown_roots: set[str] = set()
+
+ for serialized in serialized_configs:
+ for match in _OUTPUT_REF_RE.findall(serialized):
+ if match in known_labels:
+ suffix_form_refs.add(match)
+ for match in _BLOCK_FORM_REF_RE.findall(serialized):
+ if match in known_labels:
+ block_form_refs.add(match)
+ for root in _JINJA_ROOT_RE.findall(serialized):
+ if root in known_roots:
+ continue
+ if root.endswith("_output") and root[: -len("_output")] in known_labels:
+ continue
+ if any(
+ root.endswith(suffix) and root[: -len(suffix)] in parameter_keys
+ for suffix in _CREDENTIAL_REAL_VALUE_SUFFIXES
+ ):
+ continue
+ unknown_roots.add(root)
+
+ return suffix_form_refs, block_form_refs, unknown_roots
+
+
+def _referenced_output_labels(
+ frontier_labels: list[str],
+ new_definition: object | None,
+ serialized_configs: list[str] | None = None,
+) -> set[str]:
+ suffix_refs, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
+ return suffix_refs | block_form_refs
+
+
+def _block_form_output_labels(
+ frontier_labels: list[str],
+ new_definition: object | None,
+ serialized_configs: list[str] | None = None,
+) -> set[str]:
+ _, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
+ return block_form_refs
+
+
+def _unknown_jinja_roots(
+ frontier_labels: list[str],
+ new_definition: object | None,
+ serialized_configs: list[str] | None = None,
+) -> set[str]:
+ _, _, unknown_roots = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
+ return unknown_roots
+
+
+async def _get_prior_workflow_definition(ctx: AgentContext) -> object | None:
+ """Hybrid: prefer ctx.last_workflow, fall back to DB fetch on cold start."""
+ last_workflow = getattr(ctx, "last_workflow", None)
+ if last_workflow is not None:
+ definition = getattr(last_workflow, "workflow_definition", None)
+ if definition is not None:
+ return definition
+ last_yaml = getattr(ctx, "last_workflow_yaml", None)
+ if last_yaml:
+ try:
+ workflow = _process_workflow_yaml(
+ workflow_id=ctx.workflow_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ workflow_yaml=last_yaml,
+ )
+ return workflow.workflow_definition
+ except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException):
+ pass
+ try:
+ fetched = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ )
+ if fetched is not None:
+ return fetched.workflow_definition
+ except Exception:
+ LOG.debug("Failed to fetch prior workflow definition for frontier diff", exc_info=True)
+ return None
+
+
+async def _get_prior_workflow(ctx: AgentContext) -> Workflow | None:
+ """Return the prior Workflow; in-memory > re-parsed yaml > DB."""
+ last_workflow = ctx.last_workflow
+ if last_workflow is not None:
+ return last_workflow
+ last_yaml = ctx.last_workflow_yaml
+ if last_yaml:
+ try:
+ return _process_workflow_yaml(
+ workflow_id=ctx.workflow_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ workflow_yaml=last_yaml,
+ )
+ except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException):
+ pass
+ try:
+ return await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to fetch prior workflow for staging comparison; staging may skip a needed canonical write",
+ exc_info=True,
+ )
+ return None
+
+
+# Must stay in lockstep with the writers calling update_workflow_definition;
+# missing fields silently drop accepted settings on auto-accept.
+_CANONICAL_WORKFLOW_SETTING_FIELDS: tuple[str, ...] = (
+ "title",
+ "description",
+ "proxy_location",
+ "webhook_callback_url",
+ "totp_verification_url",
+ "totp_identifier",
+ "persist_browser_session",
+ "browser_profile_id",
+ "model",
+ "max_screenshot_scrolls",
+ "extra_http_headers",
+ "cdp_connect_headers",
+ "run_with",
+ "ai_fallback",
+ "cache_key",
+ "adaptive_caching",
+ "code_version",
+ "run_sequentially",
+ "sequential_key",
+)
+
+# convert_workflow_definition regenerates ids/timestamps per call; ignore
+# them when comparing parameters to react only to user intent.
+_PARAMETER_FINGERPRINT_VOLATILE_KEYS = frozenset(
+ {
+ "workflow_parameter_id",
+ "output_parameter_id",
+ "aws_secret_parameter_id",
+ "azure_secret_parameter_id",
+ "azure_vault_credential_parameter_id",
+ "bitwarden_credit_card_data_parameter_id",
+ "bitwarden_login_credential_parameter_id",
+ "bitwarden_sensitive_information_parameter_id",
+ "credential_parameter_id",
+ "onepassword_credential_parameter_id",
+ "created_at",
+ "modified_at",
+ }
+)
+
+
+def _stable_parameter_fingerprint(parameter: Parameter) -> dict[str, Any]:
+ dump = parameter.model_dump(mode="json")
+ return {k: v for k, v in dump.items() if k not in _PARAMETER_FINGERPRINT_VOLATILE_KEYS}
+
+
+def _workflow_requires_canonical_persist(prior: Workflow | None, new: Workflow) -> bool:
+ if prior is None:
+ return False
+ for field_name in _CANONICAL_WORKFLOW_SETTING_FIELDS:
+ if getattr(prior, field_name, None) != getattr(new, field_name, None):
+ return True
+ prior_params = prior.workflow_definition.parameters
+ new_params = new.workflow_definition.parameters
+ if len(prior_params) != len(new_params):
+ return True
+ prior_fingerprints = [_stable_parameter_fingerprint(p) for p in prior_params]
+ new_fingerprints = [_stable_parameter_fingerprint(p) for p in new_params]
+ return prior_fingerprints != new_fingerprints
+
+
+def _plan_frontier(
+ ctx: AgentContext,
+ requested_labels: list[str],
+ old_definition: object | None,
+ new_definition: object | None,
+) -> tuple[list[str], dict[str, Any], str | None]:
+ """Plan the frontier execution.
+
+ Returns ``(labels_to_execute, block_outputs_to_seed, frontier_start_label)``.
+
+ Falls back to the full requested list on any ambiguity. When there is no
+ workflow change (plain run path) the frontier is the first requested label,
+ and we seed verified outputs referenced by the suffix plus prior
+ browser-state outputs needed to start a downstream frontier.
+ """
+ if not requested_labels:
+ return requested_labels, {}, None
+ if new_definition is None:
+ return requested_labels, {}, requested_labels[0]
+
+ verified_outputs: dict[str, Any] = dict(ctx.verified_block_outputs or {})
+ verified_prefix: list[str] = list(ctx.verified_prefix_labels or [])
+ verified_prefix_set = set(verified_prefix)
+
+ # No old definition (cold start or parse failure) OR no diff signal → plain path.
+ if old_definition is None:
+ frontier = requested_labels[0]
+ return _seed_for_frontier(requested_labels, frontier, verified_outputs, new_definition)
+
+ try:
+ invalidated = _find_invalidated_labels(old_definition, new_definition, requested_labels)
+ except Exception:
+ LOG.debug("Frontier diff failed, falling back to full run", exc_info=True)
+ return requested_labels, {}, requested_labels[0]
+
+ earliest = _earliest_invalidated(requested_labels, invalidated)
+ if earliest is None:
+ # No invalidation at all — unchanged request. Continue from the
+ # first unverified requested label so a model may keep passing the
+ # complete chain while the tool advances the browser in small
+ # verified frontiers.
+ next_frontier = _first_unverified_requested_label(requested_labels, verified_prefix_set)
+ if next_frontier is not None:
+ return _seed_for_frontier(requested_labels, next_frontier, verified_outputs, new_definition)
+
+ # If the model accidentally asks to rerun an already-verified prefix,
+ # keep the browser moving forward instead of spending another tool call
+ # on work the current session has already covered.
+ workflow_labels = _workflow_definition_block_labels(new_definition)
+ next_workflow_frontier = _first_unverified_requested_label(workflow_labels, verified_prefix_set)
+ if next_workflow_frontier is not None:
+ frontier_idx = workflow_labels.index(next_workflow_frontier)
+ return _seed_for_frontier(
+ workflow_labels[: frontier_idx + 1],
+ next_workflow_frontier,
+ verified_outputs,
+ new_definition,
+ )
+
+ return _seed_for_frontier(requested_labels, requested_labels[0], verified_outputs, new_definition)
+
+ # Ensure the prefix before the earliest invalidated label is all in the
+ # verified prefix from a successful prior run. Otherwise we have no
+ # trusted anchor — fall back to the full requested list.
+ prefix_in_requested = [label for label in requested_labels if label != earliest]
+ prefix_in_requested = prefix_in_requested[: requested_labels.index(earliest)]
+ if not all(label in verified_prefix_set for label in prefix_in_requested):
+ return requested_labels, {}, requested_labels[0]
+
+ old_by_label = _blocks_by_label(old_definition)
+ is_append_only = earliest not in old_by_label
+ if is_append_only:
+ # Case A — append-after-success. The earliest invalidated label is a
+ # new block that didn't exist in the prior definition, so the verified
+ # prefix represents the browser state just before it. Start there.
+ workflow_labels = _workflow_definition_block_labels(new_definition)
+ if earliest in workflow_labels:
+ workflow_prefix = workflow_labels[: workflow_labels.index(earliest)]
+ if not all(label in verified_prefix_set for label in workflow_prefix):
+ anchor = _nearest_upstream_runnable_anchor(workflow_labels, earliest, new_definition)
+ if anchor is not None:
+ return _seed_for_frontier(
+ workflow_labels[workflow_labels.index(anchor) : workflow_labels.index(earliest) + 1],
+ anchor,
+ verified_outputs,
+ new_definition,
+ )
+ return _seed_for_frontier(requested_labels, earliest, verified_outputs, new_definition)
+
+ # Edit-in-place. We lack a browser-anchor signal, so we cannot safely
+ # rerun just the edited block (the browser is at post-prefix state, not
+ # pre-edit state). Walk back to the nearest upstream state-establishing
+ # block within the requested chain. Falls back to the full requested list
+ # if no safe upstream anchor can be identified.
+ anchor = _nearest_upstream_state_establisher(requested_labels, earliest, new_definition)
+ if anchor is None:
+ return requested_labels, {}, requested_labels[0]
+ return _seed_for_frontier(requested_labels, anchor, verified_outputs, new_definition)
+
+
+def _first_unverified_requested_label(requested_labels: list[str], verified_prefix_set: set[str]) -> str | None:
+ for label in requested_labels:
+ if label not in verified_prefix_set:
+ return label
+ return None
+
+
+def _seed_for_frontier(
+ requested_labels: list[str],
+ frontier: str,
+ verified_outputs: dict[str, Any],
+ new_definition: object | None,
+) -> tuple[list[str], dict[str, Any], str]:
+ try:
+ idx = requested_labels.index(frontier)
+ except ValueError:
+ return requested_labels, {}, requested_labels[0]
+ labels_to_execute = requested_labels[idx:]
+ workflow_labels = _workflow_definition_block_labels(new_definition)
+ if frontier in workflow_labels:
+ prefix_labels = workflow_labels[: workflow_labels.index(frontier)]
+ else:
+ prefix_labels = requested_labels[:idx]
+ if not prefix_labels:
+ return labels_to_execute, {}, frontier
+ serialized_configs = _serialized_frontier_block_configs(labels_to_execute, new_definition)
+ suffix_refs, block_form_refs, unknown_roots = _classify_frontier_jinja_refs(
+ labels_to_execute, new_definition, serialized_configs
+ )
+ if any(label in block_form_refs for label in prefix_labels):
+ # Seeded block_outputs only register <label>_output; block-form refs
+ # need a normal upstream execution to populate the <label> namespace.
+ return requested_labels, {}, requested_labels[0]
+ needed = suffix_refs | block_form_refs
+ seed: dict[str, Any] = {}
+ for label in prefix_labels:
+ if label not in needed:
+ continue
+ if label not in verified_outputs:
+ return requested_labels, {}, requested_labels[0]
+ seed[label] = verified_outputs[label]
+ if unknown_roots:
+ return requested_labels, {}, requested_labels[0]
+ by_label = _blocks_by_label(new_definition)
+ for label in prefix_labels:
+ block = by_label.get(label)
+ if block is None or _block_type_name(block) not in _BLOCK_TYPES_STATE_ESTABLISHER:
+ continue
+ if label in verified_outputs:
+ seed.setdefault(label, verified_outputs[label])
+ return labels_to_execute, seed, frontier
+
+
+_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS = 2
+_PAGE_CHANGING_FRONTIER_BLOCK_TYPES: frozenset[str] = frozenset(
+ {
+ BlockType.ACTION.value,
+ BlockType.FILE_DOWNLOAD.value,
+ BlockType.FILE_UPLOAD.value,
+ BlockType.LOGIN.value,
+ BlockType.NAVIGATION.value,
+ }
+)
+
+
+def _frontier_block_type_names(labels: list[str], workflow_definition: object | None) -> list[str]:
+ by_label = _blocks_by_label(workflow_definition)
+ type_names: list[str] = []
+ for label in labels:
+ block = by_label.get(label)
+ if block is None:
+ continue
+ type_name = _block_type_name(block)
+ if type_name:
+ type_names.append(type_name)
+ return type_names
+
+
+def _frontier_has_several_page_changing_stages(labels: list[str], workflow_definition: object | None) -> bool:
+ type_names = _frontier_block_type_names(labels, workflow_definition)
+ if len(type_names) <= _MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:
+ return False
+ page_changing_count = sum(1 for type_name in type_names if type_name in _PAGE_CHANGING_FRONTIER_BLOCK_TYPES)
+ return page_changing_count >= 2 or (page_changing_count >= 1 and len(type_names) >= 4)
+
+
+def _frontier_includes_required_runtime_anchor(block_labels: list[str], labels_to_execute: list[str]) -> bool:
+ if not block_labels or len(labels_to_execute) <= len(block_labels):
+ return False
+ return labels_to_execute[-len(block_labels) :] == block_labels
+
+
+def _frontier_run_size_error(
+ copilot_ctx: object,
+ block_labels: list[str],
+ labels_to_execute: list[str],
+ workflow_definition: object | None,
+) -> str | None:
+ if len(labels_to_execute) <= _MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:
+ return None
+ if _frontier_includes_required_runtime_anchor(block_labels, labels_to_execute):
+ return None
+ if getattr(copilot_ctx, "build_phase", None) not in (BuildPhase.COMPOSING, BuildPhase.TESTING):
+ return None
+ if not _frontier_has_several_page_changing_stages(labels_to_execute, workflow_definition):
+ return None
+
+ suggested = labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS]
+ remaining = labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]
+ return (
+ "Workflow validation failed: this browser test frontier is too long for a multi-stage "
+ "page-changing workflow. Keep the same complete workflow YAML, but shrink only the "
+ f"block_labels argument to the next 1-2 unverified labels: {suggested!r}. "
+ "If a prior run already advanced the browser, inspect that reached page "
+ '(inspect_page_for_composition(target_url="current_page")) to ground the next labels in '
+ "what is actually there rather than shrinking the frontier blind. "
+ f"Do not remove later blocks from the YAML; test them after this frontier succeeds. "
+ f"Deferred labels: {remaining!r}. Requested labels: {block_labels!r}."
+ )
+
+
+def _workflow_with_runtime_block_goal_context(workflow: Workflow, ctx: CopilotContext) -> Workflow:
+ block_goal_main_goal = ctx.block_goal_main_goal or ctx.user_message or ""
+ if not block_goal_main_goal:
+ LOG.warning("run_blocks invoked without block-goal context; using persisted workflow goals unchanged")
+ return workflow
+ return wrap_workflow_block_goals(workflow, block_goal_main_goal)
+
+
+def _blank_runtime_page_url(value: object) -> bool:
+ if not isinstance(value, str):
+ return True
+ # Chrome/CDP can expose ":" while the page is still in early blank-page initialization.
+ return value.strip() in {"about:blank", "", ":"}
+
+
+def _missing_runtime_frontier_block_url(value: object) -> bool:
+ if value is None:
+ return True
+ if isinstance(value, str):
+ return not value.strip()
+ return False
+
+
+def _same_runtime_page(left: str | None, right: str | None) -> bool:
+ if not left or not right:
+ return False
+ try:
+ left_parsed = urlparse(left)
+ right_parsed = urlparse(right)
+ except ValueError:
+ return False
+ return (
+ left_parsed.scheme.lower(),
+ left_parsed.netloc.lower(),
+ left_parsed.path,
+ left_parsed.query,
+ ) == (
+ right_parsed.scheme.lower(),
+ right_parsed.netloc.lower(),
+ right_parsed.path,
+ right_parsed.query,
+ )
+
+
+def _frontier_anchor_url_from_value(value: object, *, depth: int = 0) -> str | None:
+ if depth > 3:
+ return None
+ direct = _valid_runtime_anchor_url(value)
+ if direct is not None:
+ return direct
+ if isinstance(value, dict):
+ for key in ("current_url", "url", "page_url"):
+ candidate = _valid_runtime_anchor_url(value.get(key))
+ if candidate is not None:
+ return candidate
+ for nested in value.values():
+ candidate = _frontier_anchor_url_from_value(nested, depth=depth + 1)
+ if candidate is not None:
+ return candidate
+ if isinstance(value, list):
+ for nested in value:
+ candidate = _frontier_anchor_url_from_value(nested, depth=depth + 1)
+ if candidate is not None:
+ return candidate
+ return None
+
+
+def _frontier_runtime_anchor_url(ctx: CopilotContext, block_outputs_to_seed: dict[str, Any]) -> str | None:
+ for value in (
+ ctx.verified_prefix_current_url,
+ getattr(ctx, "composition_page_evidence", None),
+ ):
+ candidate = _frontier_anchor_url_from_value(value)
+ if candidate is not None:
+ return candidate
+ for value in reversed(list((block_outputs_to_seed or {}).values())):
+ candidate = _frontier_anchor_url_from_value(value)
+ if candidate is not None:
+ return candidate
+ return None
+
+
+def _iter_workflow_model_blocks(blocks: list[BlockTypeVar] | None) -> list[BlockTypeVar]:
+ if not isinstance(blocks, list):
+ return []
+ return get_all_blocks(blocks)
+
+
+def _workflow_model_block_by_label(workflow_definition: object | None, label: str | None) -> BlockTypeVar | None:
+ if workflow_definition is None or not label:
+ return None
+ blocks = workflow_definition.blocks if hasattr(workflow_definition, "blocks") else None
+ for block in _iter_workflow_model_blocks(blocks):
+ if block.label == label:
+ return block
+ return None
+
+
+def _has_verified_prefix_before_frontier(
+ ctx: CopilotContext, workflow_definition: object | None, frontier_label: str | None
+) -> bool:
+ if not frontier_label:
+ return False
+ workflow_labels = _workflow_definition_block_labels(workflow_definition)
+ if frontier_label not in workflow_labels:
+ return False
+ prefix_labels = workflow_labels[: workflow_labels.index(frontier_label)]
+ if not prefix_labels:
+ return False
+ verified = set(ctx.verified_prefix_labels or [])
+ return all(label in verified for label in prefix_labels)
+
+
+def _workflow_with_runtime_frontier_anchor(
+ workflow: Workflow,
+ ctx: CopilotContext,
+ *,
+ labels_to_execute: list[str],
+ frontier_start_label: str | None,
+ block_outputs_to_seed: dict[str, Any],
+) -> tuple[Workflow, str | None]:
+ if not labels_to_execute:
+ return workflow, None
+ workflow_definition = workflow.workflow_definition
+ if not _has_verified_prefix_before_frontier(ctx, workflow_definition, frontier_start_label):
+ return workflow, None
+
+ first_label = labels_to_execute[0]
+ first_block = _workflow_model_block_by_label(workflow_definition, first_label)
+ if first_block is None or not hasattr(first_block, "url"):
+ return workflow, None
+
+ anchor_url = _frontier_runtime_anchor_url(ctx, block_outputs_to_seed)
+ if anchor_url is None:
+ return workflow, None
+
+ existing_url = _valid_runtime_anchor_url(first_block.url if hasattr(first_block, "url") else None)
+ if existing_url is not None and not _same_runtime_page(existing_url, anchor_url):
+ return workflow, None
+
+ if existing_url is not None:
+ if _block_type_name(first_block) != BlockType.NAVIGATION.value:
+ return workflow, None
+ anchored = workflow.model_copy(deep=True)
+ anchored_block = _workflow_model_block_by_label(anchored.workflow_definition, first_label)
+ if anchored_block is None or not hasattr(anchored_block, "url"):
+ return workflow, None
+ anchored_block.url = None
+ LOG.info(
+ "Cleared runtime frontier URL to preserve browser state",
+ frontier_start_label=frontier_start_label,
+ first_block_label=first_label,
+ existing_url=existing_url,
+ continuation_url=anchor_url,
+ )
+ return anchored, anchor_url
+
+ LOG.info(
+ "Preserved runtime frontier browser state without URL reload",
+ frontier_start_label=frontier_start_label,
+ first_block_label=first_label,
+ continuation_url=anchor_url,
+ )
+ return workflow, anchor_url
+
+
+async def _workflow_with_runtime_frontier_starter_url_seed(
+ workflow: Workflow,
+ ctx: CopilotContext,
+ *,
+ labels_to_execute: list[str],
+ runtime_frontier_anchor_url: str | None,
+) -> Workflow:
+ if not labels_to_execute or runtime_frontier_anchor_url is None or not ctx.browser_session_id:
+ return workflow
+
+ first_label = labels_to_execute[0]
+ first_block = _workflow_model_block_by_label(workflow.workflow_definition, first_label)
+ if (
+ first_block is None
+ or not hasattr(first_block, "url")
+ or _block_type_name(first_block) != BlockType.NAVIGATION.value
+ or not _missing_runtime_frontier_block_url(first_block.url)
+ ):
+ return workflow
+
+ current_page_url: str | None = None
+ try:
+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
+ session_id=ctx.browser_session_id,
+ organization_id=ctx.organization_id,
+ )
+ if browser_state is not None:
+ page = await browser_state.get_working_page()
+ # Playwright Page.url is exposed as a dynamic property at this boundary.
+ current_page_url = page.url if page is not None else None
+ except Exception:
+ LOG.debug(
+ "Failed to inspect runtime frontier browser page before starter URL seed",
+ browser_session_id=ctx.browser_session_id,
+ frontier_start_label=first_label,
+ exc_info=True,
+ )
+
+ if not _blank_runtime_page_url(current_page_url):
+ LOG.info(
+ "Preserved attached runtime frontier browser page",
+ browser_session_id=ctx.browser_session_id,
+ frontier_start_label=first_label,
+ current_url=current_page_url,
+ continuation_url=runtime_frontier_anchor_url,
+ )
+ return workflow
+
+ seeded = workflow.model_copy(deep=True)
+ seeded_block = _workflow_model_block_by_label(seeded.workflow_definition, first_label)
+ # Defensive: the copied workflow definition should preserve labels and URL fields.
+ if seeded_block is None or not hasattr(seeded_block, "url"):
+ return workflow
+ seeded_block.url = runtime_frontier_anchor_url
+ LOG.info(
+ "Seeded runtime frontier starter URL because attached browser page was blank",
+ browser_session_id=ctx.browser_session_id,
+ frontier_start_label=first_label,
+ current_url=current_page_url,
+ continuation_url=runtime_frontier_anchor_url,
+ )
+ return seeded
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
new file mode 100644
index 000000000..ab26156b2
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -0,0 +1,551 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import structlog
+from agents import ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData
+
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, RecoveryHint
+from skyvern.forge.sdk.copilot.build_phase import _phase_blocker_signal
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ composition_page_evidence_error,
+ workflow_target_url,
+)
+from skyvern.forge.sdk.copilot.output_policy import (
+ evaluate_output_policy,
+ format_output_policy_tool_error,
+ output_policy_verdict_to_trace_data,
+)
+from skyvern.forge.sdk.copilot.request_policy import CREDENTIAL_DEFERRED_DRAFT_REASONS, RequestPolicy
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.turn_intent import (
+ NO_MUTATION_TURN_INTENT_MODES,
+ READ_CONTEXT_DENIED_MODES,
+ UNRESOLVED_BLOCK_REF_TARGET_ENTITY,
+ TurnIntent,
+ TurnIntentMode,
+)
+from skyvern.forge.sdk.workflow.models.parameter import (
+ OutputParameter,
+ WorkflowParameter,
+ WorkflowParameterType,
+)
+from skyvern.forge.sdk.workflow.models.workflow import Workflow
+
+from ._shared import (
+ ANSWER_ONLY_CONTEXT_TOOLS,
+ BLOCK_RUNNING_TOOLS,
+ CREDENTIAL_METADATA_TOOLS,
+ PAGE_SCHEMA_CONTEXT_TOOLS,
+ WORKFLOW_MUTATION_TOOLS,
+ _emit_tool_blocker_signal,
+ _workflow_yaml_blocks_by_label,
+)
+
+LOG = structlog.get_logger()
+
+
+def _guardrail_tool_arguments(tool_context: Any) -> tuple[dict[str, Any], Any]:
+ raw_arguments = getattr(tool_context, "tool_arguments", "")
+ try:
+ # Agents SDK guardrails may hand us either raw JSON or an already parsed mapping.
+ parsed_arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
+ except json.JSONDecodeError:
+ parsed_arguments = {}
+ return parsed_arguments if isinstance(parsed_arguments, dict) else {}, raw_arguments
+
+
+def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> ToolGuardrailFunctionOutput:
+ tool_context = data.context
+ tool_arguments, raw_arguments = _guardrail_tool_arguments(tool_context)
+ if not raw_arguments:
+ LOG.warning(
+ "workflow YAML output policy guardrail received no tool arguments",
+ tool_name=getattr(tool_context, "tool_name", None),
+ tool_call_id=getattr(tool_context, "tool_call_id", None),
+ )
+ workflow_yaml_value = tool_arguments.get("workflow_yaml")
+ workflow_yaml = workflow_yaml_value if isinstance(workflow_yaml_value, str) else None
+ verdict = evaluate_output_policy(
+ request_policy=getattr(getattr(tool_context, "context", None), "request_policy", None),
+ workflow_yaml=workflow_yaml,
+ tool_arguments=tool_arguments or raw_arguments,
+ )
+ trace_data = output_policy_verdict_to_trace_data(
+ verdict,
+ surface="tool_input",
+ tool_name=getattr(tool_context, "tool_name", None),
+ )
+ LOG.info("copilot output policy tool guardrail verdict", **trace_data)
+ if not verdict.allowed:
+ return ToolGuardrailFunctionOutput.reject_content(
+ format_output_policy_tool_error(verdict),
+ output_info=trace_data,
+ )
+ return ToolGuardrailFunctionOutput.allow(output_info=trace_data)
+
+
+_WORKFLOW_YAML_OUTPUT_POLICY_GUARDRAIL = ToolInputGuardrail(
+ guardrail_function=_workflow_yaml_output_policy_guardrail,
+ name="workflow_yaml_output_policy_guardrail",
+)
+
+_COMPOSITION_EVIDENCE_PRECHECK_TRACE_DATA = {
+ "surface": "tool_pre_side_effect",
+ "tool_name": "update_and_run_blocks",
+ "reason": "composition_page_evidence",
+}
+
+
+def _update_and_run_blocks_composition_evidence_precheck(
+ copilot_ctx: Any,
+ workflow_yaml: str | None,
+ normalized_block_observation_refs: dict[str, int],
+ raw_block_observation_refs: Any,
+) -> str | None:
+ if copilot_ctx is None or workflow_yaml is None:
+ LOG.warning(
+ "update_and_run_blocks composition evidence precheck missing context or workflow yaml",
+ has_context=copilot_ctx is not None,
+ has_workflow_yaml=workflow_yaml is not None,
+ )
+ return None
+
+ evidence_error = composition_page_evidence_error(
+ copilot_ctx,
+ workflow_yaml,
+ block_observation_refs=normalized_block_observation_refs,
+ raw_block_observation_refs=raw_block_observation_refs,
+ )
+
+ if evidence_error:
+ LOG.info(
+ "copilot composition page evidence pre-side-effect rejected workflow",
+ workflow_permanent_id=getattr(copilot_ctx, "workflow_permanent_id", None),
+ target_url=workflow_target_url(workflow_yaml),
+ surface="tool_pre_side_effect",
+ )
+ return evidence_error
+
+ return None
+
+
+def _request_policy_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
+ policy = getattr(ctx, "request_policy", None)
+ if not isinstance(policy, RequestPolicy):
+ return None
+
+ agent_steering: str | None = None
+ user_facing: str | None = None
+ reason_code: str | None = None
+ recovery_hint: RecoveryHint = "report_blocker_to_user"
+ cleared_by: frozenset[str] = frozenset()
+
+ if tool_name == "update_workflow" and not policy.allow_update_workflow:
+ reason_code = "request_policy_blocks_update_workflow"
+ agent_steering = (
+ "Request policy blocks workflow updates for the latest user message. "
+ "Ask the user for safe stored credential metadata instead."
+ )
+ user_facing = "I need stored credential metadata before I can update this workflow."
+ recovery_hint = "ask_user_clarifying"
+ elif tool_name in BLOCK_RUNNING_TOOLS and not policy.allow_run_blocks:
+ if policy.testing_intent == "skip_test":
+ reason_code = "request_policy_blocks_run_blocks_skip_test"
+ agent_steering = (
+ "Request policy says the latest user message asked for an untested draft. Use update_workflow only."
+ )
+ user_facing = "I'll save this as an untested draft because the request asked me not to run it."
+ recovery_hint = "retry_with_different_tool"
+ cleared_by = frozenset({"update_workflow"})
+ elif policy.clarification_reason == "workflow_credential_inputs_unbound":
+ reason_code = "request_policy_blocks_run_blocks_credential_unbound"
+ agent_steering = (
+ "Skipped test run: the existing workflow references credential parameters "
+ "whose keys point to workflow inputs that are not configured. REPLY to the user "
+ "with: 'I applied your requested change. I couldn't test the modified workflow "
+ "because I couldn't find the required credentials — please add them via the "
+ "Credentials UI, then I can try again.' Keep the unvalidated draft surfaced."
+ )
+ user_facing = (
+ "I applied your requested change. I couldn't test the modified workflow because "
+ "the required credentials aren't set up. Add them in the Credentials UI and ask "
+ "me to test it."
+ )
+ recovery_hint = "report_blocker_to_user"
+ else:
+ reason_code = "request_policy_blocks_run_blocks_generic"
+ agent_steering = (
+ "Request policy blocks block-running tools for the latest user message. "
+ "Ask the user for the required safe credential or clarification before testing."
+ )
+ user_facing = "I need a credential or clarification from you before I can test this workflow."
+ recovery_hint = "ask_user_clarifying"
+
+ if reason_code is None or agent_steering is None or user_facing is None:
+ return None
+
+ LOG.info(
+ "copilot authority gate evaluated tool",
+ authority_gate_layer="request_policy",
+ blocked_tool=tool_name,
+ request_policy_allow_update_workflow=policy.allow_update_workflow,
+ request_policy_allow_run_blocks=policy.allow_run_blocks,
+ request_policy_testing_intent=policy.testing_intent,
+ request_policy_clarification_reason=policy.clarification_reason,
+ safe_reason_code=reason_code,
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text=agent_steering,
+ user_facing_reason=user_facing,
+ recovery_hint=recovery_hint,
+ cleared_by_tools=cleared_by,
+ internal_reason_code=reason_code,
+ blocked_tool=tool_name,
+ )
+
+
+def _request_policy_allows_credential_deferred_draft(ctx: AgentContext) -> bool:
+ policy = getattr(ctx, "request_policy", None)
+ return (
+ isinstance(policy, RequestPolicy)
+ and policy.allow_update_workflow
+ and not policy.allow_run_blocks
+ and policy.allow_missing_credentials_in_draft
+ and policy.clarification_reason in CREDENTIAL_DEFERRED_DRAFT_REASONS
+ )
+
+
+def _request_policy_allows_update_and_skip_run(ctx: AgentContext, tool_name: str) -> bool:
+ return tool_name == "update_and_run_blocks" and _request_policy_allows_credential_deferred_draft(ctx)
+
+
+def _turn_intent_has_edit_target(intent: TurnIntent) -> bool:
+ # Keep this aligned with TurnIntent target kinds that make an edit specific enough to mutate safely.
+ if any(
+ intent.target_entities.get(entity_type)
+ for entity_type in (
+ "block",
+ "run",
+ "proposed_workflow",
+ "latest_assistant_proposal",
+ "proposal",
+ "workflow_change",
+ )
+ ):
+ return True
+ return any(target != "current_workflow" for target in intent.target_entities.get("workflow", []))
+
+
+def _turn_intent_tool_error(ctx: AgentContext, tool_name: str) -> CopilotToolBlockerSignal | None:
+ intent = getattr(ctx, "turn_intent", None)
+ if not isinstance(intent, TurnIntent):
+ return None
+
+ authority = intent.authority
+ unresolved_refs = intent.target_entities.get(UNRESOLVED_BLOCK_REF_TARGET_ENTITY, [])
+ if intent.mode == TurnIntentMode.EDIT and tool_name in WORKFLOW_MUTATION_TOOLS and unresolved_refs:
+ reason_code = "turn_intent_unresolved_edit_target"
+ labels = sorted(_workflow_yaml_blocks_by_label(getattr(ctx, "workflow_yaml", None)))
+ label_hint = ", ".join(labels[:8]) if labels else "no labeled blocks"
+ LOG.info(
+ "copilot authority gate evaluated tool",
+ authority_gate_layer="turn_intent",
+ turn_intent_mode=intent.mode.value,
+ turn_intent_target_entity_types=sorted(intent.target_entities),
+ turn_intent_unresolved_refs=unresolved_refs,
+ blocked_tool=tool_name,
+ safe_reason_code=reason_code,
+ )
+ unresolved_str = ", ".join(unresolved_refs)
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ f"The latest user message references workflow/block identifier(s) that are not present in the "
+ f"current workflow: {unresolved_str}. Current workflow labels include: {label_hint}. Ask the user "
+ f"which current block should change before mutating or running blocks."
+ ),
+ user_facing_reason=(
+ f"I couldn't find the block(s) you mentioned ({unresolved_str}). "
+ f"Tell me which existing block to change."
+ ),
+ recovery_hint="ask_user_clarifying",
+ )
+
+ if (
+ intent.mode == TurnIntentMode.EDIT
+ and tool_name in WORKFLOW_MUTATION_TOOLS
+ and not _turn_intent_has_edit_target(intent)
+ ):
+ reason_code = "turn_intent_missing_edit_target"
+ LOG.info(
+ "copilot authority gate evaluated tool",
+ authority_gate_layer="turn_intent",
+ turn_intent_mode=intent.mode.value,
+ turn_intent_target_entity_types=sorted(intent.target_entities),
+ blocked_tool=tool_name,
+ safe_reason_code=reason_code,
+ )
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "Could not identify a specific workflow edit target. Ask the user which workflow/block should "
+ "change before mutating."
+ ),
+ user_facing_reason="Tell me which block or workflow you'd like me to change.",
+ recovery_hint="ask_user_clarifying",
+ )
+
+ blocks_update = tool_name in WORKFLOW_MUTATION_TOOLS and not authority.may_update_workflow
+ blocks_run = tool_name in BLOCK_RUNNING_TOOLS and not authority.may_run_blocks
+ blocks_page_inspection = tool_name in PAGE_SCHEMA_CONTEXT_TOOLS and (
+ intent.mode in NO_MUTATION_TURN_INTENT_MODES or not authority.may_update_workflow
+ )
+ blocks_credential_metadata = tool_name in CREDENTIAL_METADATA_TOOLS and not (
+ authority.may_update_workflow or authority.may_run_blocks
+ )
+ # Two paths grant read access to ANSWER_ONLY_CONTEXT_TOOLS, both excluded for DOCS_ANSWER/REFUSE/CLARIFY:
+ # (1) authority.may_read_run_context — classifier-derived (DIAGNOSE turns)
+ # (2) pending_reconciliation_run_id — within-turn override anchored to the run-blocks watchdog
+ may_read_run_context = authority.may_read_run_context and intent.mode not in READ_CONTEXT_DENIED_MODES
+ blocks_context_read = tool_name in ANSWER_ONLY_CONTEXT_TOOLS and not may_read_run_context
+
+ within_turn_read_override = False
+ if blocks_context_read and intent.mode not in READ_CONTEXT_DENIED_MODES:
+ pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
+ if isinstance(pending_run_id, str) and pending_run_id:
+ blocks_context_read = False
+ within_turn_read_override = True
+ else:
+ same_turn_run_id = getattr(ctx, "last_successful_run_blocks_workflow_run_id", None) or getattr(
+ ctx,
+ "last_run_blocks_workflow_run_id",
+ None,
+ )
+ if isinstance(same_turn_run_id, str) and same_turn_run_id:
+ blocks_context_read = False
+ within_turn_read_override = True
+
+ if blocks_run and not blocks_update and _request_policy_allows_update_and_skip_run(ctx, tool_name):
+ return None
+ if (
+ not blocks_update
+ and not blocks_run
+ and not blocks_context_read
+ and not blocks_page_inspection
+ and not blocks_credential_metadata
+ ):
+ if within_turn_read_override:
+ LOG.info(
+ "copilot authority gate allowed tool via within-turn read override",
+ authority_gate_layer="turn_intent",
+ turn_intent_mode=intent.mode.value,
+ tool_name=tool_name,
+ turn_intent_within_turn_read_override=True,
+ )
+ return None
+
+ if intent.mode in NO_MUTATION_TURN_INTENT_MODES and blocks_run:
+ reason_code = "turn_intent_no_mutation_run_blocked"
+ elif intent.mode in NO_MUTATION_TURN_INTENT_MODES and blocks_update:
+ reason_code = "turn_intent_no_mutation_update_blocked"
+ elif blocks_update and blocks_run:
+ reason_code = "turn_intent_no_mutation_run_blocked"
+ elif blocks_update:
+ reason_code = "turn_intent_update_blocked"
+ elif blocks_page_inspection:
+ reason_code = "turn_intent_page_inspection_blocked"
+ elif blocks_context_read:
+ reason_code = "turn_intent_context_read_blocked"
+ elif blocks_credential_metadata:
+ reason_code = "turn_intent_credential_metadata_blocked"
+ else:
+ reason_code = "turn_intent_run_blocked"
+ LOG.info(
+ "copilot authority gate evaluated tool",
+ authority_gate_layer="turn_intent",
+ turn_intent_mode=intent.mode.value,
+ turn_intent_target_entity_types=sorted(intent.target_entities),
+ turn_intent_may_update_workflow=authority.may_update_workflow,
+ turn_intent_may_run_blocks=authority.may_run_blocks,
+ turn_intent_may_read_run_context=authority.may_read_run_context,
+ blocked_tool=tool_name,
+ safe_reason_code=reason_code,
+ )
+ action = "ask the user" if authority.requires_user_input else "answer the user"
+ detail = f" Ask: {intent.missing_context_question}" if intent.missing_context_question else ""
+
+ if blocks_run and not blocks_update and authority.may_update_workflow:
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "Browser blocks may not run for the latest user message. Use update_workflow only and keep the "
+ f"draft unvalidated.{detail}"
+ ),
+ user_facing_reason="I'll save the change as a draft without running it.",
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=frozenset({"update_workflow"}),
+ )
+ if blocks_context_read and not blocks_update and not blocks_run:
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "Run context may not be read for the latest user message. Answer using the context already "
+ f"provided.{detail}"
+ ),
+ user_facing_reason="I'll answer with the information I already have.",
+ recovery_hint="report_blocker_to_user",
+ )
+ return _build_turn_intent_signal(
+ tool_name=tool_name,
+ classifier_mode=intent.mode.value,
+ reason_code=reason_code,
+ agent_steering_text=(
+ "This tool is not allowed for the latest user message. Do not update workflow YAML or run browser "
+ "blocks, and do not fetch additional run context with tools; "
+ f"{action} using the available context instead.{detail}"
+ ),
+ user_facing_reason="I'll respond with the information I already have.",
+ recovery_hint="ask_user_clarifying" if authority.requires_user_input else "report_blocker_to_user",
+ )
+
+
+def _build_turn_intent_signal(
+ *,
+ tool_name: str,
+ classifier_mode: str,
+ reason_code: str,
+ agent_steering_text: str,
+ user_facing_reason: str,
+ recovery_hint: RecoveryHint,
+ cleared_by_tools: frozenset[str] = frozenset(),
+) -> CopilotToolBlockerSignal:
+ return CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text=agent_steering_text,
+ user_facing_reason=user_facing_reason,
+ recovery_hint=recovery_hint,
+ cleared_by_tools=cleared_by_tools,
+ internal_reason_code=reason_code,
+ blocked_tool=tool_name,
+ classifier_mode=classifier_mode,
+ )
+
+
+def _authority_tool_error(
+ ctx: AgentContext,
+ tool_name: str,
+ *,
+ ignore_request_policy_error: bool = False,
+) -> str | None:
+ # Request-policy precedes turn-intent unless explicitly ignored.
+ turn_intent_signal = _turn_intent_tool_error(ctx, tool_name)
+ request_policy_signal = _request_policy_tool_error(ctx, tool_name)
+ if turn_intent_signal is not None and request_policy_signal is not None:
+ LOG.info(
+ "copilot authority gate blocked tool",
+ authority_gate_layer="both",
+ blocked_tool=tool_name,
+ )
+ chosen = (
+ request_policy_signal
+ if (request_policy_signal is not None and not ignore_request_policy_error)
+ else turn_intent_signal
+ )
+ if chosen is not None:
+ return _emit_tool_blocker_signal(ctx, chosen)
+ phase_signal = _phase_blocker_signal(ctx, tool_name)
+ if phase_signal is not None:
+ LOG.info(
+ "copilot authority gate blocked tool",
+ authority_gate_layer="build_phase",
+ blocked_tool=tool_name,
+ build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),
+ )
+ return _emit_tool_blocker_signal(ctx, phase_signal)
+ return None
+
+
+_PARAMETER_TYPE_PLACEHOLDERS: dict[WorkflowParameterType, Any] = {
+ WorkflowParameterType.STRING: "",
+ WorkflowParameterType.INTEGER: 0,
+ WorkflowParameterType.FLOAT: 0.0,
+ WorkflowParameterType.BOOLEAN: False,
+ WorkflowParameterType.JSON: {},
+ WorkflowParameterType.FILE_URL: "",
+}
+
+
+def _placeholder_for_parameter_type(param_type: WorkflowParameterType) -> Any:
+ return _PARAMETER_TYPE_PLACEHOLDERS.get(param_type)
+
+
+def _parameter_binding_invariant_error(
+ workflow: Workflow,
+ persisted_workflow_params: list[WorkflowParameter],
+ persisted_output_params: list[OutputParameter],
+) -> tuple[str, dict[str, list[str]], dict[str, list[str]]] | None:
+ """Return a ``(summary, missing_persisted, missing_from_definition)`` tuple
+ when ``workflow.workflow_definition`` disagrees with persisted
+ definition-parameter rows for runtime-relevant classes. Returns ``None``
+ when aligned.
+
+ Compares ``WorkflowParameter`` rows by ``(key, workflow_parameter_type)``
+ and ``OutputParameter`` rows by ``key``. Secret/credential and context
+ parameters are intentionally out of scope — runtime reads those from the
+ definition JSON.
+ """
+ definition = getattr(workflow, "workflow_definition", None)
+ parameters = getattr(definition, "parameters", None) if definition else None
+ parameters = list(parameters) if parameters else []
+
+ def_workflow_ids: set[tuple[str, str]] = set()
+ def_output_keys: set[str] = set()
+ for parameter in parameters:
+ if isinstance(parameter, WorkflowParameter):
+ def_workflow_ids.add((parameter.key, parameter.workflow_parameter_type.value))
+ elif isinstance(parameter, OutputParameter):
+ def_output_keys.add(parameter.key)
+
+ persisted_workflow_ids: set[tuple[str, str]] = {
+ (wp.key, wp.workflow_parameter_type.value) for wp in persisted_workflow_params
+ }
+ persisted_output_keys: set[str] = {op.key for op in persisted_output_params}
+
+ missing_persisted_workflow = sorted(
+ f"{key} ({ptype})" for (key, ptype) in def_workflow_ids - persisted_workflow_ids
+ )
+ extra_persisted_workflow = sorted(f"{key} ({ptype})" for (key, ptype) in persisted_workflow_ids - def_workflow_ids)
+ missing_persisted_output = sorted(def_output_keys - persisted_output_keys)
+ extra_persisted_output = sorted(persisted_output_keys - def_output_keys)
+
+ if (
+ not missing_persisted_workflow
+ and not extra_persisted_workflow
+ and not missing_persisted_output
+ and not extra_persisted_output
+ ):
+ return None
+
+ summary = (
+ "Pre-run invariant: workflow_definition and persisted parameter rows disagree. "
+ f"workflow missing persisted: {missing_persisted_workflow or '[]'}; "
+ f"workflow missing from definition: {extra_persisted_workflow or '[]'}; "
+ f"output missing persisted: {missing_persisted_output or '[]'}; "
+ f"output missing from definition: {extra_persisted_output or '[]'}"
+ )
+ return (
+ summary,
+ {"workflow": missing_persisted_workflow, "output": missing_persisted_output},
+ {"workflow": extra_persisted_workflow, "output": extra_persisted_output},
+ )
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
new file mode 100644
index 000000000..38f428f1b
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -0,0 +1,692 @@
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any
+
+import structlog
+
+from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
+from skyvern.forge.sdk.copilot.build_phase import (
+ BuildPhase,
+ advance_to_composing,
+)
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+
+from ._shared import _DISCOVERY_PER_CALL_TIMEOUT_SECONDS
+from .banned_blocks import (
+ _CODE_ONLY_SELECTOR_ACTION_TOOLS,
+ _CODE_ONLY_TARGET_EVIDENCE_KEYS,
+ _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES,
+ _copilot_banned_block_alternatives,
+ _copilot_banned_block_types,
+ _copilot_block_authoring_policy,
+)
+from .completion import _maybe_run_completion_verification_from_page_observation
+from .page_observation import (
+ _record_composition_page_observation,
+ _resolve_url_title,
+)
+from .scouting import (
+ _capture_scout_source_url,
+ _clear_pending_browser_interaction_observation,
+ _consume_scout_source_url,
+ _mark_page_inspected,
+ _mark_pending_browser_interaction_observation,
+ _record_scouted_interaction,
+ _register_scout_interaction_observation,
+ _resolve_scout_role_name,
+)
+
+LOG = structlog.get_logger()
+
+
+async def _get_block_schema_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ """Short-circuit requests for banned block types with an explicit error.
+ Without this pre-hook the underlying MCP tool silently redirects ``task``
+ and ``task_v2`` queries to ``navigation``'s schema, which makes the LLM
+ think the banned types are available."""
+ block_type = params.get("block_type")
+ if not isinstance(block_type, str):
+ return None
+ normalized = normalize_copilot_block_type_alias(block_type)
+ if normalized != block_type.strip().lower():
+ params["block_type"] = normalized
+ if normalized not in _copilot_banned_block_types(ctx):
+ return None
+ return {
+ "ok": False,
+ "error": f"Block type {block_type!r} is not available in the workflow copilot. {_copilot_banned_block_alternatives(ctx)}",
+ }
+
+
+async def _validate_block_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return None
+ return {
+ "ok": False,
+ "error": (
+ "CODE-ONLY BLOCK VALIDATION DISABLED: do not use validate_block, dummy code blocks, or probe code "
+ "blocks in code-only browser mode. Use MCP browser tools to explore the page, then call "
+ "update_and_run_blocks with real focused code blocks that implement the workflow behavior."
+ ),
+ }
+
+
+async def _get_block_schema_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ """Scrub banned block types from list-mode responses. Belt-and-suspenders
+ against future drift in ``BLOCK_SUMMARIES`` (which currently omits them)."""
+ data = result.get("data")
+ if isinstance(data, dict):
+ block_types = data.get("block_types")
+ if isinstance(block_types, dict):
+ for banned in _copilot_banned_block_types(ctx):
+ block_types.pop(banned, None)
+ block_type = data.get("block_type")
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and block_type == "code":
+ ctx.code_only_code_schema_seen = True
+ data["code_only_note"] = _COPILOT_CODE_ONLY_BROWSER_ALTERNATIVES
+ data["code_only_guidance"] = [
+ "Use one focused code block per durable browser goal, such as open, search, submit, expand, or extract.",
+ "Do not persist navigation/action/login/extraction/validation blocks for browser page work.",
+ "Use concrete selectors and text anchors found during exploration. If only intent targeting is available, inspect the page again before mutating.",
+ "Call update_and_run_blocks with a connected runnable set of real code blocks instead of validating dummy or probe blocks.",
+ "Keep block outputs JSON-safe and include visible evidence text when extracting records, products, totals, confirmations, or identifiers.",
+ ]
+ return result
+
+
+def _code_only_pre_run_results_error(ctx: CopilotContext) -> dict[str, Any] | None:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return None
+ if ctx.workflow_persisted or ctx.update_workflow_called:
+ return None
+ for value in (
+ ctx.pending_reconciliation_run_id,
+ ctx.last_run_blocks_workflow_run_id,
+ ctx.last_successful_run_blocks_workflow_run_id,
+ ):
+ if isinstance(value, str) and value:
+ return None
+ return {
+ "ok": False,
+ "error": (
+ "CODE-ONLY EXPLORATION PHASE: get_run_results is unavailable before a real workflow run exists. "
+ "Use MCP browser tools such as navigate_browser, evaluate, click, type_text, get_browser_screenshot, "
+ "console_messages, scroll, select_option, or press_key to understand the page, then call "
+ "update_and_run_blocks with real focused code blocks."
+ ),
+ }
+
+
+async def _evaluate_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ expr = params.get("expression", "").lower()
+ if ".click()" in expr or ".click(" in expr:
+ return {
+ "ok": False,
+ "error": "Do not use evaluate to click elements. Use the 'click' tool with a CSS selector instead.",
+ }
+ return None
+
+
+def _code_only_deterministic_targeting_error(tool_name: str) -> str:
+ return (
+ f"In code-only browser mode, {tool_name} requires a CSS/XPath selector for page mutations "
+ "after the reusable workflow has been verified. Use evaluate, screenshots, or page inspection "
+ "to derive a selector, then retry with selector only."
+ )
+
+
+def _code_only_selector_action_requires_deterministic_target(ctx: AgentContext) -> bool:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return False
+ if not getattr(ctx, "workflow_persisted", False):
+ return False
+ return bool(getattr(ctx, "last_full_workflow_test_ok", False))
+
+
+def _strip_intent_for_code_only_selector_action(
+ params: dict[str, Any],
+ ctx: AgentContext,
+ *,
+ tool_name: str,
+) -> dict[str, Any] | None:
+ if not _code_only_selector_action_requires_deterministic_target(ctx):
+ return None
+ if tool_name not in _CODE_ONLY_SELECTOR_ACTION_TOOLS:
+ return None
+ selector = params.get("selector")
+ if isinstance(selector, str) and selector.strip():
+ if "intent" in params:
+ params["intent"] = None
+ return None
+ if params.get("intent"):
+ return {"ok": False, "error": _code_only_deterministic_targeting_error(tool_name)}
+ return None
+
+
+def _code_only_has_target_page_evidence(data: object) -> bool:
+ if not isinstance(data, dict):
+ return False
+ for key, value in data.items():
+ normalized = str(key).strip().lower()
+ if normalized in _CODE_ONLY_TARGET_EVIDENCE_KEYS and bool(value):
+ return True
+ if isinstance(value, dict) and _code_only_has_target_page_evidence(value):
+ return True
+ if isinstance(value, list) and any(_code_only_has_target_page_evidence(item) for item in value):
+ return True
+ return False
+
+
+_JQUERY_SELECTOR_RE = re.compile(r":(?:contains|eq|first|last|gt|lt|nth|visible|hidden|checked)\s*\(", re.IGNORECASE)
+
+
+async def _click_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ await _capture_scout_source_url(ctx)
+ deterministic_result = _strip_intent_for_code_only_selector_action(params, ctx, tool_name="click")
+ if deterministic_result is not None:
+ return deterministic_result
+ selector = params.get("selector", "")
+ if not selector:
+ return None
+ if _JQUERY_SELECTOR_RE.search(selector):
+ return {
+ "ok": False,
+ "error": (
+ f"Invalid selector: {selector!r}. "
+ "jQuery pseudo-selectors like :contains(), :eq(), :first, :visible are NOT valid CSS. "
+ "Use standard CSS selectors instead. Examples: "
+ "nth-of-type() instead of :eq(), "
+ "[data-attr] or tag.class for filtering, "
+ "or use the 'evaluate' tool with JS: "
+ "document.querySelectorAll('button').forEach("
+ "b => {{ if (b.textContent.includes('Download')) b.click() }})"
+ ),
+ }
+ return None
+
+
+async def _type_text_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ await _capture_scout_source_url(ctx)
+ return _strip_intent_for_code_only_selector_action(params, ctx, tool_name="type_text")
+
+
+async def _select_option_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ await _capture_scout_source_url(ctx)
+ return _strip_intent_for_code_only_selector_action(params, ctx, tool_name="select_option")
+
+
+async def _press_key_pre_hook(
+ params: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any] | None:
+ await _capture_scout_source_url(ctx)
+ return _strip_intent_for_code_only_selector_action(params, ctx, tool_name="press_key")
+
+
+async def _navigate_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
+ if result.get("ok"):
+ data = result.pop("data", {})
+ result["url"] = data.get("url", "")
+ result["next_step"] = (
+ "Page loaded. You MUST now use evaluate, "
+ "get_browser_screenshot, or click to inspect page content "
+ "before responding."
+ )
+ if (
+ _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ and isinstance(ctx, CopilotContext)
+ and ctx.build_phase in {BuildPhase.INITIAL, BuildPhase.DISCOVERING}
+ ):
+ advance_to_composing(ctx, reason="code_only_browser_navigation_succeeded")
+ return result
+
+
+async def _screenshot_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ if result.get("ok") and result.get("data"):
+ _mark_page_inspected(ctx)
+ data = result["data"]
+ url, title = await _resolve_url_title(raw, ctx)
+ _record_composition_page_observation(ctx, source_tool="get_browser_screenshot", url=url, title=title)
+ result["data"] = {
+ "screenshot_base64": data.get("data", ""),
+ "url": url,
+ "title": title,
+ }
+ return result
+
+
+async def _click_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
+ source_url = _consume_scout_source_url(ctx)
+ if result.get("ok") and result.get("data"):
+ data = result["data"]
+ url, title = await _resolve_url_title(raw, ctx)
+ _mark_pending_browser_interaction_observation(ctx, tool_name="click", url=url)
+ result["data"] = {
+ "selector": data.get("selector", ""),
+ "url": url,
+ "title": title,
+ }
+ navigated = bool(source_url) and bool(url) and source_url != url
+ role, accessible_name = await _resolve_scout_role_name(
+ ctx, data.get("selector", ""), allow_browser_read=not navigated
+ )
+ _record_scouted_interaction(
+ ctx,
+ tool_name="click",
+ selector=data.get("selector", ""),
+ source_url=source_url,
+ role=role,
+ accessible_name=accessible_name,
+ )
+ observation_step = _register_scout_interaction_observation(
+ ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url, url=url
+ )
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ result["data"]["observation_step"] = observation_step
+ return result
+
+
+_TYPE_READBACK_SETTLE_SECONDS = 0.3
+
+
+async def _verify_scout_type_landed(
+ ctx: AgentContext,
+ *,
+ selector: str,
+ typed_length: Any,
+) -> dict[str, Any] | None:
+ """Confirm a non-empty type actually entered the field, else return a failure.
+
+ A marketing/cookie overlay can consume the focus or keystrokes — the field
+ stays empty while `skyvern_type` still reports success (the first interaction
+ on an overlaid page often just dismisses the overlay). Read the field back; a
+ field still empty after a non-empty type means the input did not land. Only
+ fires when there is a selector to read and a positive typed length, so it never
+ second-guesses intent-only types or masked/formatted values, which keep a
+ non-empty value.
+ """
+ if not isinstance(selector, str) or not selector.strip():
+ return None
+ if not isinstance(typed_length, int) or typed_length <= 0:
+ return None
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return None
+
+ async def _read_back() -> Any:
+ try:
+ readback = await asyncio.wait_for(
+ server.call_internal_tool("skyvern_get_value", {"selector": selector}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ LOG.debug("scout type-landed read-back failed; leaving the type result unverified", exc_info=True)
+ return None
+ if not isinstance(readback, dict) or not readback.get("ok"):
+ return None
+ return (readback.get("data") or {}).get("value")
+
+ value = await _read_back()
+ if isinstance(value, str) and value.strip() == "":
+ # A controlled/React input can mirror its value asynchronously, so a first read may be
+ # transiently empty; settle briefly and re-read once before declaring the type lost.
+ await asyncio.sleep(_TYPE_READBACK_SETTLE_SECONDS)
+ value = await _read_back()
+ if isinstance(value, str) and value.strip() == "":
+ return {
+ "ok": False,
+ "error": (
+ "type_text reported success but the field is still empty — an overlay "
+ "(cookie/marketing popup) likely consumed the keystrokes or focus. "
+ "Re-inspect the current page and retry typing into the target field; "
+ "the overlay is usually dismissed by that first interaction."
+ ),
+ }
+ return None
+
+
+async def _type_text_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
+ source_url = _consume_scout_source_url(ctx)
+ if result.get("ok") and result.get("data"):
+ data = result["data"]
+ selector = data.get("selector", "")
+ typed_length = data.get("text_length", 0)
+ url, _ = await _resolve_url_title(raw, ctx)
+ result["data"] = {
+ "selector": selector,
+ "typed_length": typed_length,
+ "url": url,
+ }
+ landing_failure = await _verify_scout_type_landed(ctx, selector=selector, typed_length=typed_length)
+ if landing_failure is not None:
+ return landing_failure
+ _mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)
+ role, accessible_name = await _resolve_scout_role_name(ctx, selector)
+ _record_scouted_interaction(
+ ctx,
+ tool_name="type_text",
+ selector=selector,
+ source_url=source_url,
+ typed_length=typed_length,
+ role=role,
+ accessible_name=accessible_name,
+ )
+ observation_step = _register_scout_interaction_observation(
+ ctx, tool_name="type_text", selector=selector, source_url=source_url, url=url
+ )
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ result["data"]["observation_step"] = observation_step
+ return result
+
+
+async def _evaluate_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ if result.get("ok") and result.get("data"):
+ _mark_page_inspected(ctx)
+ result["data"].pop("sdk_equivalent", None)
+ if "url" not in result["data"]:
+ url, _ = await _resolve_url_title(raw, ctx)
+ if url:
+ result["data"]["url"] = url
+ url = str(result["data"].get("url") or "")
+ title = str(result["data"].get("title") or "")
+ if not title:
+ _, title = await _resolve_url_title(raw, ctx)
+ observation_step = _record_composition_page_observation(
+ ctx,
+ source_tool="evaluate",
+ url=url,
+ title=title,
+ observed_data=result["data"],
+ append_to_flow=True,
+ reached_via="auto",
+ )
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ result["data"]["observation_step"] = observation_step
+ if _copilot_block_authoring_policy(
+ ctx
+ ) == BlockAuthoringPolicy.CODE_ONLY_BROWSER and _code_only_has_target_page_evidence(result["data"]):
+ ctx.code_only_target_page_evidence_seen = True
+ await _maybe_run_completion_verification_from_page_observation(
+ ctx,
+ url=url,
+ title=title,
+ observed_data=result["data"],
+ )
+ return result
+
+
+async def _scroll_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ if result.get("ok") and result.get("data"):
+ data = result["data"]
+ url, _ = await _resolve_url_title(raw, ctx)
+ result["data"] = {
+ "direction": data.get("direction", ""),
+ "amount": data.get("pixels") or data.get("amount"),
+ "url": url,
+ }
+ return result
+
+
+async def _select_option_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
+ source_url = _consume_scout_source_url(ctx)
+ if result.get("ok") and result.get("data"):
+ data = result["data"]
+ url, _ = await _resolve_url_title(raw, ctx)
+ _mark_pending_browser_interaction_observation(ctx, tool_name="select_option", url=url)
+ result["data"] = {
+ "selector": data.get("selector", ""),
+ "value": data.get("value", ""),
+ "url": url,
+ }
+ role, accessible_name = await _resolve_scout_role_name(ctx, data.get("selector", ""))
+ _record_scouted_interaction(
+ ctx,
+ tool_name="select_option",
+ selector=data.get("selector", ""),
+ source_url=source_url,
+ value=data.get("value", ""),
+ role=role,
+ accessible_name=accessible_name,
+ )
+ observation_step = _register_scout_interaction_observation(
+ ctx, tool_name="select_option", selector=data.get("selector", ""), source_url=source_url, url=url
+ )
+ if observation_step is not None:
+ result["observation_step"] = observation_step
+ result["data"]["observation_step"] = observation_step
+ return result
+
+
+async def _press_key_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ _clear_pending_browser_interaction_observation(ctx)
+ source_url = _consume_scout_source_url(ctx)
+ if result.get("ok") and result.get("data"):
+ data = result["data"]
+ url, _ = await _resolve_url_title(raw, ctx)
+ _mark_pending_browser_interaction_observation(ctx, tool_name="press_key", url=url)
+ result["data"] = {
+ "key": data.get("key", ""),
+ "selector": data.get("selector", ""),
+ "url": url,
+ }
+ _record_scouted_interaction(
+ ctx,
+ tool_name="press_key",
+ selector=data.get("selector", ""),
+ source_url=source_url,
+ key=data.get("key", ""),
+ )
+ return result
+
+
+def get_skyvern_mcp_alias_map() -> dict[str, str]:
+ return {
+ "get_block_schema": "skyvern_block_schema",
+ "validate_block": "skyvern_block_validate",
+ "navigate_browser": "skyvern_navigate",
+ "get_browser_screenshot": "skyvern_screenshot",
+ "evaluate": "skyvern_evaluate",
+ "click": "skyvern_click",
+ "type_text": "skyvern_type",
+ "scroll": "skyvern_scroll",
+ "console_messages": "skyvern_console_messages",
+ "select_option": "skyvern_select_option",
+ "press_key": "skyvern_press_key",
+ }
+
+
+def _build_skyvern_mcp_overlays() -> dict[str, SchemaOverlay]:
+ return {
+ "get_block_schema": SchemaOverlay(
+ pre_hook=_get_block_schema_pre_hook,
+ post_hook=_get_block_schema_post_hook,
+ ),
+ "validate_block": SchemaOverlay(pre_hook=_validate_block_pre_hook),
+ "navigate_browser": SchemaOverlay(
+ description=(
+ "Navigate the debug browser to a URL. "
+ "Use this to reset browser state or navigate to a starting page before running blocks."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url"}),
+ requires_browser=True,
+ post_hook=_navigate_post_hook,
+ ),
+ "get_browser_screenshot": SchemaOverlay(
+ description=(
+ "Take a screenshot of the current debug browser session. "
+ "Returns a base64-encoded PNG image. "
+ "Use this to see what the browser looks like after running blocks."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url", "selector"}),
+ forced_args={"inline": True},
+ requires_browser=True,
+ post_hook=_screenshot_post_hook,
+ ),
+ "evaluate": SchemaOverlay(
+ description=(
+ "Execute JavaScript in the browser and return the result. "
+ "Use this to inspect DOM state, read values, or run arbitrary JS."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url"}),
+ requires_browser=True,
+ timeout=30,
+ pre_hook=_evaluate_pre_hook,
+ post_hook=_evaluate_post_hook,
+ ),
+ "click": SchemaOverlay(
+ description=(
+ "Click an element in the browser. Prefer a CSS selector ALONE for a target "
+ "you can identify from page evidence — a selector-only click is instant and "
+ "deterministic. Use `intent` only when you cannot derive a selector: an "
+ "`intent`-only click routes through a slower full-page AI scan, and if you "
+ "pass both, the selector wins and the `intent` is ignored. When a shared class "
+ "matches many elements (e.g. one button per result row), scope the selector to "
+ "the specific item (its container, a unique attribute, or :nth-of-type) instead "
+ "of relying on `intent` to disambiguate. "
+ "IMPORTANT: jQuery pseudo-selectors like :contains(), :eq(), :first, "
+ ":visible are NOT valid CSS. Use standard selectors: "
+ "'button.download', 'a[href*=\"pdf\"]', '#submit-btn', "
+ "'table tr:nth-of-type(2) td a'."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url", "button", "click_count"}),
+ forced_args={"selector_mode": "direct"},
+ requires_browser=True,
+ timeout=15,
+ pre_hook=_click_pre_hook,
+ post_hook=_click_post_hook,
+ ),
+ "type_text": SchemaOverlay(
+ description=(
+ "Type text into an input element. Prefer a CSS selector ALONE to target the "
+ "field — a selector-only type is instant and deterministic. Use `intent` only "
+ "when you cannot derive a selector: an `intent`-only type routes through a "
+ "slower full-page AI scan, and if you pass both, the selector wins and the "
+ "`intent` is ignored. "
+ "Optionally clear the field first. Use this for form filling. "
+ "NEVER type inline passwords, API keys, tokens, cookies, TOTP/OTP "
+ "codes, private keys, or other raw credentials/secrets received in "
+ "chat — stop and follow the CREDENTIAL HANDLING refusal rule in the "
+ "system prompt instead."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url", "delay"}),
+ forced_args={"selector_mode": "direct"},
+ required_overrides=["text"],
+ arg_transforms={"clear_first": "clear"},
+ requires_browser=True,
+ timeout=15,
+ pre_hook=_type_text_pre_hook,
+ post_hook=_type_text_post_hook,
+ ),
+ "scroll": SchemaOverlay(
+ description=(
+ "Scroll the page in a direction (up/down/left/right) by pixel amount, "
+ "or scroll a specific element into view using intent or selector. "
+ "Use this to reveal content below the fold."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url"}),
+ requires_browser=True,
+ post_hook=_scroll_post_hook,
+ ),
+ "console_messages": SchemaOverlay(
+ description=(
+ "Read console log messages from the browser. "
+ "Use level='error' to find JavaScript errors. "
+ "This is a read-only diagnostic tool."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url"}),
+ requires_browser=True,
+ ),
+ "select_option": SchemaOverlay(
+ description=(
+ "Select an option from a <select> dropdown. Provide the value to select and a "
+ "selector to target the element precisely; use `intent` (alone) only when you "
+ "cannot derive a selector — passing both lets the selector win and ignores the "
+ "`intent`. For free-text inputs, use type_text instead."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url", "timeout"}),
+ forced_args={"selector_mode": "direct"},
+ required_overrides=["value"],
+ requires_browser=True,
+ timeout=15,
+ pre_hook=_select_option_pre_hook,
+ post_hook=_select_option_post_hook,
+ ),
+ "press_key": SchemaOverlay(
+ description=(
+ "Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.). "
+ "Optionally focus an element first via selector or intent. "
+ "Use for form submission, tab navigation, or closing dialogs."
+ ),
+ hide_params=frozenset({"session_id", "cdp_url"}),
+ required_overrides=["key"],
+ requires_browser=True,
+ pre_hook=_press_key_pre_hook,
+ post_hook=_press_key_post_hook,
+ ),
+ }
diff --git a/skyvern/forge/sdk/copilot/tools/page_observation.py b/skyvern/forge/sdk/copilot/tools/page_observation.py
new file mode 100644
index 000000000..1f94ad95c
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/page_observation.py
@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+
+from ._shared import (
+ _DISCOVERY_ANTI_BOT_PATTERNS,
+ _append_flow_evidence,
+ _composition_evidence_page_url,
+ _fallback_page_info,
+ _same_page_ignoring_fragment,
+ _workflow_verification_evidence,
+)
+from .scouting import _consume_pending_browser_interaction_observation, _mark_post_run_page_observed
+
+_EVALUATE_EVIDENCE_CONFIDENCE_WITH_SCHEMA = 0.75
+_EVALUATE_EVIDENCE_CONFIDENCE_ANTIBOT_ONLY = 0.4
+
+
+async def _resolve_url_title(raw: dict[str, Any], ctx: AgentContext) -> tuple[str, str]:
+ """Extract URL and title from raw MCP result, falling back to live page info."""
+ browser_ctx = raw.get("browser_context", {})
+ url = browser_ctx.get("url", "")
+ title = browser_ctx.get("title", "")
+ if not url:
+ url, fallback_title = await _fallback_page_info(ctx)
+ if fallback_title:
+ title = fallback_title
+ return url, title
+
+
+def _bounded_observation_text(value: object, limit: int = 240) -> str:
+ return " ".join(str(value or "").split())[:limit]
+
+
+@dataclass(frozen=True)
+class _ObservedFieldEvidence:
+ name: str
+ id: str
+ label: str
+ type: str
+ placeholder: str
+ required: bool
+ selector: str
+
+ @classmethod
+ def from_raw(cls, raw_field: dict[str, Any]) -> _ObservedFieldEvidence:
+ label = raw_field.get("label")
+ if not label and isinstance(raw_field.get("labels"), list):
+ label = " ".join(str(item) for item in raw_field["labels"][:2])
+ return cls(
+ name=_bounded_observation_text(raw_field.get("name"), 120),
+ id=_bounded_observation_text(raw_field.get("id"), 120),
+ label=_bounded_observation_text(label, 240),
+ type=_bounded_observation_text(raw_field.get("type"), 40),
+ placeholder=_bounded_observation_text(raw_field.get("placeholder"), 240),
+ required=bool(raw_field.get("required")),
+ selector=_bounded_observation_text(raw_field.get("selector"), 160),
+ )
+
+ def as_evidence(self) -> dict[str, Any]:
+ return {
+ "name": self.name,
+ "id": self.id,
+ "label": self.label,
+ "type": self.type,
+ "placeholder": self.placeholder,
+ "required": self.required,
+ "selector": self.selector,
+ }
+
+
+def _normalize_observed_fields(raw_fields: object) -> list[dict[str, Any]]:
+ if not isinstance(raw_fields, list):
+ return []
+ fields: list[dict[str, Any]] = []
+ for raw_field in raw_fields[:20]:
+ if not isinstance(raw_field, dict):
+ continue
+ fields.append(_ObservedFieldEvidence.from_raw(raw_field).as_evidence())
+ return fields
+
+
+def _normalize_observed_forms(observed_data: dict[str, Any]) -> list[dict[str, Any]]:
+ raw_forms = observed_data.get("forms")
+ forms: list[dict[str, Any]] = []
+ if isinstance(raw_forms, list):
+ for raw_form in raw_forms[:5]:
+ if not isinstance(raw_form, dict):
+ continue
+ fields = _normalize_observed_fields(raw_form.get("fields") or raw_form.get("inputs"))
+ submit_controls = _normalize_observed_fields(
+ raw_form.get("submit_controls") or raw_form.get("buttons") or raw_form.get("submits")
+ )
+ forms.append(
+ {
+ "id": _bounded_observation_text(raw_form.get("id"), 120),
+ "name": _bounded_observation_text(raw_form.get("name"), 120),
+ "action": _bounded_observation_text(raw_form.get("action"), 240),
+ "method": _bounded_observation_text(raw_form.get("method"), 20),
+ "fields": fields,
+ "submit_controls": submit_controls[:10],
+ }
+ )
+ if forms:
+ return forms
+ fields = _normalize_observed_fields(observed_data.get("inputs") or observed_data.get("fields"))
+ if not fields:
+ return []
+ return [{"id": "", "name": "", "action": "", "method": "", "fields": fields, "submit_controls": []}]
+
+
+def _evaluate_observed_payload(observed_data: object) -> dict[str, Any]:
+ if not isinstance(observed_data, dict):
+ return {}
+ raw_result = observed_data.get("result")
+ if isinstance(raw_result, dict):
+ payload: dict[str, Any] = dict(raw_result)
+ elif isinstance(raw_result, list):
+ payload = {"rows": raw_result}
+ else:
+ payload = {}
+ for key, value in observed_data.items():
+ if key == "result":
+ continue
+ if key not in payload or payload.get(key) in (None, "", [], {}):
+ payload[key] = value
+ return payload or observed_data
+
+
+def _observed_row_text(row: object) -> str:
+ if isinstance(row, str):
+ return _bounded_observation_text(row, 500)
+ if not isinstance(row, dict):
+ return ""
+ parts: list[str] = []
+ for key in ("text", "label", "name", "title"):
+ value = row.get(key)
+ if isinstance(value, str):
+ parts.append(value)
+ cells = row.get("cells")
+ if isinstance(cells, list):
+ for cell in cells[:12]:
+ if isinstance(cell, dict):
+ for key in ("text", "value"):
+ value = cell.get(key)
+ if isinstance(value, str):
+ parts.append(value)
+ elif isinstance(cell, str):
+ parts.append(cell)
+ return _bounded_observation_text(" ".join(parts), 500)
+
+
+def _normalize_observed_result_containers(observed_data: dict[str, Any]) -> list[dict[str, Any]]:
+ raw_results = observed_data.get("result_containers") or observed_data.get("tables")
+ containers: list[dict[str, Any]] = []
+ if isinstance(raw_results, list):
+ for item in raw_results[:8]:
+ if isinstance(item, dict):
+ containers.append(
+ {
+ "tag": _bounded_observation_text(item.get("tag") or item.get("type"), 40),
+ "id": _bounded_observation_text(item.get("id"), 120),
+ "selector": _bounded_observation_text(item.get("selector"), 160),
+ }
+ )
+ if containers:
+ return containers
+ raw_rows = observed_data.get("rows")
+ if isinstance(raw_rows, list) and raw_rows:
+ sample_rows = [text for row in raw_rows[:5] if (text := _observed_row_text(row))]
+ return [
+ {
+ "tag": "table",
+ "id": _bounded_observation_text(observed_data.get("id"), 120),
+ "selector": _bounded_observation_text(observed_data.get("selector"), 160),
+ "row_count": len(raw_rows),
+ "sample_rows": sample_rows,
+ }
+ ]
+ table = observed_data.get("table")
+ if table:
+ return [{"tag": "table", "id": "", "selector": ""}]
+ return []
+
+
+def _normalize_evaluate_challenge_state(
+ observed_data: dict[str, Any],
+ anti_bot_indicators: list[str],
+) -> dict[str, Any]:
+ detected = bool(anti_bot_indicators)
+ gated_controls: list[dict[str, Any]] = []
+ for key, value in observed_data.items():
+ normalized_key = str(key).strip().lower().replace("_", " ").replace("-", " ")
+ if "disabled" not in normalized_key or value is not True:
+ continue
+ if any(token in normalized_key for token in ("submit", "search", "button", "btn")):
+ gated_controls.append(
+ {
+ "text": _bounded_observation_text(str(key), 120),
+ "id": "",
+ "name": _bounded_observation_text(str(key), 120),
+ "selector": "",
+ "disabled": True,
+ }
+ )
+ indicator_text = " ".join(anti_bot_indicators).lower()
+ if "access denied" in indicator_text:
+ kind = "access_denied"
+ elif "turnstile" in indicator_text or "captcha" in indicator_text or "are you a robot" in indicator_text:
+ kind = "captcha"
+ elif detected:
+ kind = "human_verification"
+ else:
+ kind = "none"
+ return {
+ "detected": detected,
+ "kind": kind,
+ "source": "mcp_evaluate" if detected else "",
+ "indicators": anti_bot_indicators[:8],
+ "requires_human_verification": detected,
+ "visual_location": "",
+ "gates_submit_controls": bool(detected and gated_controls),
+ "gated_submit_controls": gated_controls[:5] if detected else [],
+ }
+
+
+def _evaluate_anti_bot_indicators(observed_data: dict[str, Any], text: str) -> list[str]:
+ key_text = " ".join(str(key) for key in observed_data.keys()).lower()
+ combined = f"{text} {key_text}"
+ return [pattern for pattern in _DISCOVERY_ANTI_BOT_PATTERNS if pattern in combined]
+
+
+def _normalize_evaluate_page_schema(observed_data: object) -> dict[str, Any]:
+ observed_data = _evaluate_observed_payload(observed_data)
+ if not observed_data:
+ return {}
+ text_parts = [
+ observed_data.get("body"),
+ observed_data.get("bodyText"),
+ observed_data.get("text"),
+ observed_data.get("html"),
+ ]
+ text = " ".join(part for part in text_parts if isinstance(part, str)).lower()[:4096]
+ anti_bot = _evaluate_anti_bot_indicators(observed_data, text)
+ forms = _normalize_observed_forms(observed_data)
+ result_containers = _normalize_observed_result_containers(observed_data)
+ if not forms and not result_containers and not anti_bot:
+ return {}
+ return {
+ "forms": forms,
+ "result_containers": result_containers,
+ "anti_bot_indicators": anti_bot,
+ "challenge_state": _normalize_evaluate_challenge_state(observed_data, anti_bot),
+ "evidence_sources": ["mcp_evaluate"],
+ "evidence_confidence": (
+ _EVALUATE_EVIDENCE_CONFIDENCE_WITH_SCHEMA
+ if forms or result_containers
+ else _EVALUATE_EVIDENCE_CONFIDENCE_ANTIBOT_ONLY
+ ),
+ }
+
+
+def _record_composition_page_observation(
+ ctx: AgentContext,
+ *,
+ source_tool: str,
+ url: str,
+ title: str = "",
+ observed_data: object | None = None,
+ append_to_flow: bool = False,
+ reached_via: str = "current_page",
+) -> int | None:
+ if not url:
+ return None
+ _mark_post_run_page_observed(ctx, source_tool=source_tool, url=url)
+ if title:
+ _workflow_verification_evidence(ctx).page_title = title[:160]
+ existing = ctx.composition_page_evidence
+
+ evidence: dict[str, Any] = {
+ "inspected_url": url,
+ "current_url": url,
+ "page_title": title[:240],
+ "forms": [],
+ "navigation_targets": [],
+ "result_containers": [],
+ "anti_bot_indicators": [],
+ "evidence_confidence": 0.0,
+ "source_tool": source_tool,
+ "observed_after_workflow_run": False,
+ }
+ if isinstance(observed_data, dict):
+ observed_title = observed_data.get("title")
+ if isinstance(observed_title, str) and observed_title and not evidence["page_title"]:
+ evidence["page_title"] = observed_title[:240]
+ if source_tool == "get_browser_screenshot":
+ evidence.update(
+ {
+ "evidence_sources": ["screenshot"],
+ "screenshot_used": True,
+ }
+ )
+ elif source_tool == "evaluate":
+ evidence.update(_normalize_evaluate_page_schema(observed_data))
+ run_id = ctx.last_run_blocks_workflow_run_id
+ if isinstance(run_id, str) and run_id:
+ evidence["workflow_run_id"] = run_id
+ evidence["observed_after_workflow_run"] = True
+
+ observation_step: int | None = None
+ if append_to_flow and has_bounded_page_schema(evidence):
+ actual_reached_via = reached_via
+ if reached_via == "auto":
+ if isinstance(run_id, str) and run_id:
+ actual_reached_via = "post_run"
+ elif _consume_pending_browser_interaction_observation(ctx, current_url=url, evidence=evidence):
+ actual_reached_via = "interaction"
+ else:
+ actual_reached_via = "current_page"
+ observation_step = _append_flow_evidence(ctx, evidence, reached_via=actual_reached_via)
+
+ if not _should_keep_existing_composition_page_evidence(existing, evidence):
+ ctx.composition_page_evidence = evidence
+ return observation_step
+
+
+def _should_keep_existing_composition_page_evidence(
+ existing: dict[str, Any] | None,
+ incoming: dict[str, Any],
+) -> bool:
+ if not isinstance(existing, dict) or existing.get("source_tool") != "inspect_page_for_composition":
+ return False
+ if incoming.get("source_tool") != "evaluate":
+ return True
+ if not has_bounded_page_schema(incoming):
+ return True
+ if not has_bounded_page_schema(existing):
+ return False
+ existing_url = _composition_evidence_page_url(existing)
+ incoming_url = _composition_evidence_page_url(incoming)
+ if existing_url and incoming_url and not _same_page_ignoring_fragment(existing_url, incoming_url):
+ return False
+ return True
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
new file mode 100644
index 000000000..2e89f4287
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -0,0 +1,1627 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import time
+from collections import defaultdict
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Any, Literal
+from urllib.parse import urlparse
+
+import structlog
+
+from skyvern.forge import app
+from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ CopilotToolBlockerSignal,
+ clear_blocker_signal_for_reason_codes,
+ stash_blocker_signal,
+)
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisRepairContract,
+ build_diagnosis_repair_contract,
+)
+from skyvern.forge.sdk.copilot.failure_tracking import (
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ PER_TOOL_BUDGET_FAILURE_CATEGORY,
+ compute_action_sequence_fingerprint,
+ update_repeated_failure_state,
+)
+from skyvern.forge.sdk.copilot.narration import NarratorState
+from skyvern.forge.sdk.copilot.narration import handler_available as narration_handler_available
+from skyvern.forge.sdk.copilot.narration import narrator_poll_tick
+from skyvern.forge.sdk.copilot.outcome_verification_trace import record_completion_verification
+from skyvern.forge.sdk.copilot.output_utils import (
+ _INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
+ build_run_blocks_response,
+ iter_failure_reasons,
+ truncate_output,
+)
+from skyvern.forge.sdk.copilot.runtime import (
+ AgentContext,
+ ensure_browser_session,
+)
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.settings_manager import SettingsManager
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+from skyvern.schemas.workflows import BlockType
+from skyvern.webeye.navigation import is_skip_inner_retry_error
+from skyvern.webeye.utils.page import SkyvernFrame
+
+from ._shared import (
+ _FAILED_BLOCK_STATUSES,
+ RUN_BLOCKS_SAFETY_CEILING_SECONDS,
+ _completed_run_block_labels,
+ _failed_run_block_labels,
+ _fallback_page_info,
+ _unverified_current_workflow_labels,
+ _valid_runtime_anchor_url,
+ _workflow_verification_evidence,
+)
+from .blockers import (
+ _active_block_run_budget_seconds,
+ _active_run_terminal_evidence_detected,
+ _active_run_terminal_evidence_signal,
+ _analyze_run_blocks,
+ _looks_like_anti_bot_blocker,
+ _pending_reconciliation_requires_input_signal,
+ _run_blocks_structured_blocker_message,
+ _safe_read_workflow_run,
+ _trusted_post_drain_status,
+)
+from .completion import (
+ _emit_completion_verification_trace,
+ _outcome_failure_warrants_repair,
+ _outcome_unverified_reason,
+)
+from .composition_capture import (
+ ActiveRunTerminalEvidenceSample,
+ _active_run_terminal_evidence_result,
+ _active_run_terminal_evidence_sample,
+ _active_run_terminal_monitor_enabled,
+)
+from .credentials import (
+ _credential_ids_validation_error,
+ _extract_credential_ids_from_tool_value,
+ _extract_credential_ids_from_workflow_definition,
+)
+from .frontier import (
+ _MAX_INCREMENTAL_PAGE_FRONTIER_LABELS,
+ _blocks_by_label,
+ _workflow_with_runtime_block_goal_context,
+ _workflow_with_runtime_frontier_anchor,
+ _workflow_with_runtime_frontier_starter_url_seed,
+)
+from .guardrails import (
+ _parameter_binding_invariant_error,
+ _placeholder_for_parameter_type,
+)
+from .scouting import _mark_page_inspected
+
+LOG = structlog.get_logger()
+
+_ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS = 30.0
+_ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS = 30.0
+_ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES = 8
+
+# Primary exit condition: seconds of no observed progress across the combined
+# run / block / step heartbeat. Sized to accommodate the slowest single LLM
+# round-trip (~30-60 s in practice) with headroom; going tighter risks
+# false-positives on healthy runs.
+RUN_BLOCKS_STAGNATION_WINDOW_SECONDS = 90
+
+
+# 5 s balances responsiveness (18 samples inside the stagnation window) against
+# DB load (240 polls worst case at the safety ceiling).
+RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0
+
+# Detached cleanup tasks held here so the garbage collector does not drop them
+# while they still have work to do, and so the "task exception was never
+# retrieved" warning cannot fire — each task adds a done-callback that logs
+# exceptions and removes itself from this set.
+_DETACHED_CLEANUP_TASKS: set[asyncio.Task] = set()
+
+
+async def _cancel_run_task_if_not_final(
+ run_task: asyncio.Task,
+ workflow_run_id: str,
+) -> None:
+ """Cancel ``run_task`` and reconcile the workflow run row to a terminal
+ state.
+
+ ``run_task.cancel()`` is synchronous — it just flips the cancel flag. We
+ then wait briefly for ``execute_workflow``'s outer ``finally`` to drain
+ its shielded ``_finalize_workflow_run_status`` call, which restores the
+ real terminal status (``failed``/``terminated``/``timed_out``) even when
+ we cancel mid-flight. After that we issue a conditional DB cancel that
+ is a no-op when the row is already terminal — so a run whose finally
+ block produced a proper terminal status keeps it, and a run that truly
+ never finalized (e.g. cancel landed before block execution captured a
+ ``pre_finally_status``) lands as ``canceled``. All awaits are
+ exception-contained so teardown of the enclosing tool task doesn't
+ surface a secondary error over the original cancellation.
+ """
+ run_task.cancel()
+ try:
+ # Shield run_task so OUR wait timeout does not send another cancel
+ # through to it — the cancel we want is already pending.
+ await asyncio.wait_for(asyncio.shield(run_task), timeout=5.0)
+ except (asyncio.CancelledError, asyncio.TimeoutError):
+ pass
+ except Exception:
+ LOG.warning(
+ "Run task raised during cancellation grace window",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+ try:
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled_if_not_final(
+ workflow_run_id=workflow_run_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Conditional cancel write failed",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+
+
+def _log_detached_cleanup_failure(task: asyncio.Task) -> None:
+ exc = task.exception() if task.done() and not task.cancelled() else None
+ if exc is not None:
+ LOG.warning("Detached cancel fallback failed", exc_info=exc)
+
+
+def _maybe_clear_reconciliation_flag(copilot_ctx: Any, result: Any) -> None:
+ """Clear ``pending_reconciliation_run_id`` iff the matching resolved run
+ landed in a status the caller can move past: any ``is_final_excluding_canceled``
+ status, or any status (including ``canceled``) when the prior exit was an
+ internal per-tool-budget cancel.
+ """
+ pending_run_id = getattr(copilot_ctx, "pending_reconciliation_run_id", None)
+ if not isinstance(pending_run_id, str) or not pending_run_id:
+ return
+ if not isinstance(result, dict):
+ return
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return
+ resolved_run_id = data.get("workflow_run_id")
+ resolved_status = data.get("overall_status")
+ if not isinstance(resolved_run_id, str) or resolved_run_id != pending_run_id:
+ return
+ if not isinstance(resolved_status, str):
+ return
+ is_trusted_final = WorkflowRunStatus(resolved_status).is_final_excluding_canceled()
+ # ``last_failure_category_top`` reflects the prior block-running tool's outcome —
+ # only ``_record_run_blocks_result`` writes it, and the reconciliation guard
+ # prevents another block-running call from clobbering it before this read.
+ internal_watchdog_cancel_category = getattr(copilot_ctx, "last_failure_category_top", None) in {
+ PER_TOOL_BUDGET_FAILURE_CATEGORY,
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ }
+ if is_trusted_final or internal_watchdog_cancel_category:
+ copilot_ctx.pending_reconciliation_run_id = None
+ copilot_ctx.pending_reconciliation_requires_user_input = False
+ clear_blocker_signal_for_reason_codes(
+ copilot_ctx,
+ frozenset(
+ {
+ "tool_error_pending_reconciliation_no_input",
+ "tool_error_pending_reconciliation_requires_input",
+ }
+ ),
+ )
+ return
+ if resolved_status == WorkflowRunStatus.canceled.value:
+ copilot_ctx.pending_reconciliation_requires_user_input = True
+ # Replace the no_input blocker with the requires-input one; unrelated
+ # blockers (e.g. `loop_detected`) survive the targeted clear.
+ existing_blocker = getattr(copilot_ctx, "blocker_signal", None)
+ # Preserve the original blocked_tool so trace queries filtering on it correlate the no_input → requires_input transition.
+ original_blocked_tool = (
+ existing_blocker.blocked_tool
+ if (
+ isinstance(existing_blocker, CopilotToolBlockerSignal)
+ and existing_blocker.internal_reason_code == "tool_error_pending_reconciliation_no_input"
+ and existing_blocker.blocked_tool
+ )
+ else "get_run_results"
+ )
+ clear_blocker_signal_for_reason_codes(
+ copilot_ctx,
+ frozenset({"tool_error_pending_reconciliation_no_input"}),
+ )
+ stash_blocker_signal(
+ copilot_ctx,
+ _pending_reconciliation_requires_input_signal(
+ pending_run_id=pending_run_id,
+ blocked_tool=original_blocked_tool,
+ ),
+ )
+
+
+def _mark_pending_reconciliation_run(copilot_ctx: Any, workflow_run_id: str) -> None:
+ copilot_ctx.pending_reconciliation_run_id = workflow_run_id
+ copilot_ctx.pending_reconciliation_requires_user_input = False
+
+
+async def _attach_action_traces(
+ blocks: list,
+ results: list[dict[str, Any]],
+ organization_id: str,
+) -> None:
+ """For non-success blocks with a task_id, fetch and attach a compact action trace."""
+ failed_task_ids = [
+ b.task_id for b, r in zip(blocks, results) if b.task_id and r.get("status") in _FAILED_BLOCK_STATUSES
+ ]
+ if not failed_task_ids:
+ return
+
+ rows = await app.DATABASE.tasks.get_recent_actions_for_tasks(
+ task_ids=failed_task_ids,
+ organization_id=organization_id,
+ )
+
+ actions_by_task: dict[str, list] = defaultdict(list)
+ for row in rows:
+ if row.task_id is not None:
+ actions_by_task[row.task_id].append(row)
+
+ for block, block_result in zip(blocks, results):
+ if block_result.get("status") not in _FAILED_BLOCK_STATUSES or not block.task_id:
+ continue
+ task_actions = actions_by_task.get(block.task_id, [])
+ block_result["action_trace"] = [
+ {
+ "action": a.action_type,
+ "status": a.status,
+ "reasoning": a.reasoning[:150] if a.reasoning else None,
+ "element": a.element_id,
+ }
+ for a in task_actions
+ ]
+
+
+async def _fetch_last_screenshot_b64(task_id: str, organization_id: str) -> str | None:
+ try:
+ artifacts = await app.DATABASE.artifacts.get_artifacts_for_task_v2(
+ task_v2_id=task_id,
+ organization_id=organization_id,
+ artifact_types=[ArtifactType.SCREENSHOT_LLM],
+ )
+ if not artifacts:
+ return None
+ # The last artifact is the one captured closest to the failure.
+ artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifacts[-1])
+ if not artifact_bytes:
+ return None
+ return base64.b64encode(artifact_bytes).decode("utf-8")
+ except Exception:
+ LOG.debug("Failed to fetch screenshot for failed block", task_id=task_id, exc_info=True)
+ return None
+
+
+async def _attach_failed_block_screenshots(
+ blocks: list,
+ results: list[dict[str, Any]],
+ organization_id: str,
+) -> None:
+ """For failed blocks with a task_id, fetch the last SCREENSHOT_LLM artifact."""
+ task_id_to_block: dict[str, dict] = {
+ block.task_id: block_result
+ for block, block_result in zip(blocks, results)
+ if block.task_id and block_result.get("status") in _FAILED_BLOCK_STATUSES
+ }
+ if not task_id_to_block:
+ return
+
+ task_ids = list(task_id_to_block.keys())
+ screenshots = await asyncio.gather(
+ *(_fetch_last_screenshot_b64(task_id, organization_id) for task_id in task_ids),
+ )
+ for task_id, b64 in zip(task_ids, screenshots):
+ if b64 is not None:
+ task_id_to_block[task_id]["screenshot_b64"] = b64
+
+
+# Block types that establish browser state (loaded page / authenticated
+# session / navigation target). These are valid upstream anchors to walk back
+# to when a downstream edit invalidates part of the chain.
+#
+# We intentionally do NOT maintain a companion "rerunnable from current
+# browser state" set. We have no signal that the persistent browser session
+# is actually anchored at the frontier boundary — after a successful
+# [A, B, C] the browser is at post-C state, not pre-C — so rerunning only
+# an edited block is unsafe even for read-only types. Every edit walks back
+# to an upstream state-establisher, or falls back to the full requested list.
+def _summarize_action_trace(action_trace: list[dict[str, Any]] | None) -> list[str]:
+ """Compact, stringified summary of action entries for the compact packet."""
+ if not action_trace:
+ return []
+ summary: list[str] = []
+ for entry in action_trace[-6:]:
+ if not isinstance(entry, dict):
+ continue
+ action = entry.get("action") or "?"
+ status = entry.get("status") or ""
+ element = entry.get("element")
+ bits = [str(action)]
+ if element:
+ bits.append(str(element))
+ if status:
+ bits.append(str(status))
+ summary.append(" ".join(bits).strip())
+ return summary
+
+
+# Watchdog exit reasons. ``success`` means the run reached a trustworthy
+# terminal status inside the poll loop OR after the post-drain reconcile.
+# The three non-success reasons share the reconcile path but produce distinct
+# error messages: ``stagnation`` is the primary trip (no progress signals
+# for ``RUN_BLOCKS_STAGNATION_WINDOW_SECONDS`` seconds), ``ceiling`` is the
+# last-resort budget-exhausted branch, and ``task_exit_unfinalized`` is the
+# rare race where ``execute_workflow`` naturally exits before writing a
+# terminal row.
+WatchdogExitReason = Literal[
+ "success",
+ "stagnation",
+ "ceiling",
+ "per_tool_budget",
+ "task_exit_unfinalized",
+ "active_run_terminal_evidence",
+]
+
+
+def _watchdog_exit_allows_terminal_promotion(exit_reason: WatchdogExitReason | None) -> bool:
+ return exit_reason != "active_run_terminal_evidence"
+
+
+# Block types that legitimately execute long silent periods: one DB write on
+# entry, work done without intermediate writes (sleep / LLM call / await human
+# input / browser download wait), one write on finish. The watchdog can't
+# distinguish these from "stuck", so any invocation that includes one disables
+# stagnation for the whole run and relies on the safety ceiling alone.
+_QUIET_BLOCK_TYPES: frozenset[str] = frozenset(
+ {
+ BlockType.WAIT.value,
+ BlockType.TEXT_PROMPT.value,
+ BlockType.HUMAN_INTERACTION.value,
+ BlockType.FILE_DOWNLOAD.value,
+ }
+)
+
+
+def _any_quiet_block_requested(
+ copilot_ctx: CopilotContext,
+ labels: list[str] | None,
+) -> bool:
+ """Return True if any of ``labels`` refers to a block whose type is in
+ ``_QUIET_BLOCK_TYPES``. Reuses ``_blocks_by_label`` on the already-loaded
+ workflow definition — no DB call.
+ """
+ if not labels:
+ return False
+ last_workflow = getattr(copilot_ctx, "last_workflow", None)
+ if last_workflow is None:
+ return False
+ by_label = _blocks_by_label(getattr(last_workflow, "workflow_definition", None))
+ for label in labels:
+ block = by_label.get(label)
+ if block is None:
+ continue
+ block_type = getattr(block, "block_type", None)
+ if block_type is None:
+ continue
+ block_type_str = block_type.value if hasattr(block_type, "value") else str(block_type)
+ if block_type_str in _QUIET_BLOCK_TYPES:
+ return True
+ return False
+
+
+async def _read_progress_sources(
+ ctx: CopilotContext,
+ workflow_run_id: str,
+) -> tuple[WorkflowRun | None, datetime | None, datetime | None]:
+ """Read one ``workflow_runs`` row + the two progress aggregates needed
+ by the watchdog marker. Three cheap indexed queries; no row hydration
+ on the aggregate side. The two repo calls run concurrently — they open
+ separate async sessions and hit different tables.
+ """
+
+ async def _read_timestamps() -> tuple[datetime | None, datetime | None]:
+ try:
+ return await app.DATABASE.tasks.get_workflow_run_progress_timestamps(
+ workflow_run_id=workflow_run_id,
+ organization_id=ctx.organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Workflow run progress timestamps read failed",
+ workflow_run_id=workflow_run_id,
+ exc_info=True,
+ )
+ return None, None
+
+ run, (step_ts, block_ts) = await asyncio.gather(
+ _safe_read_workflow_run(workflow_run_id, ctx.organization_id, context="watchdog-poll"),
+ _read_timestamps(),
+ )
+ return run, step_ts, block_ts
+
+
+def _progress_marker(
+ run: WorkflowRun | None,
+ step_ts: datetime | None,
+ block_ts: datetime | None,
+) -> tuple[Any, ...]:
+ """Hashable scalar snapshot. Changes iff any observable progress has
+ occurred at the run, step, or block level since the last poll. Every
+ ``update_step`` fires during action execution (including incremental
+ token/cost accumulators at ``forge/agent.py:1449``), so
+ ``max(step.modified_at)`` is the per-LLM-call heartbeat. Non-task blocks
+ (CODE, TEXT_PROMPT) don't create step rows — ``max(workflow_run_block.modified_at)``
+ covers that case. ``run.modified_at`` and ``run.status`` catch the
+ run-level transitions that happen outside those two tables.
+ """
+ return (
+ run.status if run else None,
+ run.modified_at if run is not None else None,
+ step_ts,
+ block_ts,
+ )
+
+
+async def _watchdog_error_message(
+ exit_reason: WatchdogExitReason,
+ ctx: AgentContext,
+ workflow_run_id: str,
+ run: WorkflowRun | None,
+ budget_seconds: int,
+) -> str:
+ """LLM-facing error string for a non-success watchdog exit. No variant uses
+ "timed out" or other retry-inviting phrasing — those are SKY-9163 traps.
+ """
+ if exit_reason == "stagnation":
+ body = (
+ f"The run has not made progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s. "
+ f"No step, block, or workflow-run row updates were observed in that window. "
+ f"The page is most likely blocked by a captcha, popup, anti-bot challenge, "
+ f"hidden validation error, or an infinite-retry loop on an action the agent "
+ f"cannot detect is failing."
+ )
+ elif exit_reason == "per_tool_budget":
+ message = (
+ f"The run exceeded the {budget_seconds}s per-tool-call budget while still "
+ f"making progress. This budget exists so a single in-flight call cannot "
+ f"consume the whole copilot session.\n"
+ f"Run ID: {workflow_run_id}.\n"
+ f"Next step: call get_run_results with this workflow_run_id to inspect what "
+ f"the cancelled run actually completed (the in-flight block was cancelled "
+ f"mid-execution and may have left partial side effects). If the result "
+ f"includes a current_url, inspect that current page before any further "
+ f'block-running call with inspect_page_for_composition(target_url="current_page"). '
+ f"Generic screenshot/evaluate reads can help answer the user, but they do not "
+ f"satisfy the bounded page-evidence contract for workflow mutations. Use the "
+ f"bounded evidence to decide whether the answer is already visible, whether "
+ f"a challenge-gated submit/search control is still disabled, or which page-state "
+ f"change is still missing. If challenge_state.gates_submit_controls=true and "
+ f"the requested answer is not visible, stop and report the observed anti-bot "
+ f"blocker instead of retrying the same solve/wait/submit chain. Only then call "
+ f"update_and_run_blocks with a smaller chain — the first 1-2 unverified blocks. "
+ f"Verified-prefix state is preserved, so the next call only re-runs from the new frontier. "
+ f"Do NOT retry the same chain unchanged — a longer "
+ f"run won't fit either."
+ )
+ current_url, _ = await _fallback_page_info(ctx)
+ if current_url:
+ message += f" Browser was on: {current_url}"
+ return message
+ elif exit_reason == "active_run_terminal_evidence":
+ message = (
+ "The active run was interrupted because bounded current-page evidence matched the requested "
+ "browser terminal state while the workflow run was still in progress.\n"
+ f"Run ID: {workflow_run_id}.\n"
+ "This is NOT full workflow verification: the requested browser state was observed, but the durable "
+ "workflow chain still needs diagnosis/repair and a clean verification run. Next step: call "
+ "get_run_results with this workflow_run_id to inspect the active run boundary, preserve the observed "
+ "current-page evidence, and update only the block(s) that overshot or kept running after the state "
+ "was reached. Do NOT report end-to-end success unless a corrected workflow run verifies cleanly."
+ )
+ current_url, _ = await _fallback_page_info(ctx)
+ if current_url:
+ message += f" Browser was on: {current_url}"
+ return message
+ elif exit_reason == "ceiling":
+ body = (
+ f"The run exceeded the {budget_seconds}s absolute ceiling "
+ f"while still showing progress. The workflow is too long to fit in a single "
+ f"tool invocation — split it into smaller block groups."
+ )
+ else: # task_exit_unfinalized
+ last_observed = f"last observed status: {run.status}" if run is not None else "workflow run row was unreadable"
+ body = (
+ f"The run ended but did not record a trustworthy terminal status in the "
+ f"cancellation grace window ({last_observed})."
+ )
+
+ message = (
+ f"{body} Run ID: {workflow_run_id}. Outcome is uncertain. "
+ f"Do NOT re-invoke block-running tools in this session without first calling "
+ f"`get_run_results` with this workflow_run_id and reporting the result to the user."
+ )
+ current_url, _ = await _fallback_page_info(ctx)
+ if current_url:
+ message += f" Browser was on: {current_url}"
+ return message
+
+
+def _watchdog_user_failure_reason(
+ exit_reason: WatchdogExitReason,
+ workflow_run_id: str,
+ budget_seconds: int,
+ run: WorkflowRun | None,
+) -> str:
+ if exit_reason == "stagnation":
+ body = f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."
+ elif exit_reason == "per_tool_budget":
+ body = f"The run exceeded the {budget_seconds}s per-tool-call budget while still making progress."
+ elif exit_reason == "active_run_terminal_evidence":
+ body = (
+ "The active run reached the requested browser state before the workflow finished, "
+ "so it was interrupted for diagnosis/repair. Full workflow verification is still required."
+ )
+ elif exit_reason == "ceiling":
+ body = f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."
+ else:
+ status = f" Last observed status: {run.status}." if run is not None else ""
+ body = "The run ended before recording a trustworthy terminal status." + status
+ return f"{body} Run ID: {workflow_run_id}. Outcome is uncertain."
+
+
+def _watchdog_user_facing_summary(
+ exit_reason: WatchdogExitReason,
+ budget_seconds: int,
+ run: WorkflowRun | None,
+) -> str:
+ if exit_reason == "stagnation":
+ return f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."
+ if exit_reason == "per_tool_budget":
+ return f"The run exceeded the {budget_seconds}s per-tool-call budget while still making progress."
+ if exit_reason == "ceiling":
+ return f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."
+ if run is not None:
+ return f"The run ended before recording a trustworthy terminal status. Last observed status: {run.status}."
+ return "The run ended before recording a trustworthy terminal status."
+
+
+async def _run_blocks_and_collect_debug(
+ params: dict[str, Any],
+ ctx: CopilotContext,
+ *,
+ labels_to_execute: list[str] | None = None,
+ block_outputs_to_seed: dict[str, Any] | None = None,
+ frontier_start_label: str | None = None,
+) -> dict[str, Any]:
+ block_labels = params["block_labels"]
+ if not block_labels:
+ return {"ok": False, "error": "block_labels must not be empty"}
+
+ labels_to_execute = list(labels_to_execute) if labels_to_execute else list(block_labels)
+ block_outputs_to_seed = block_outputs_to_seed or {}
+ if frontier_start_label is None:
+ frontier_start_label = labels_to_execute[0] if labels_to_execute else None
+
+ ctx.last_requested_block_labels = list(block_labels)
+ ctx.last_executed_block_labels = list(labels_to_execute)
+ ctx.last_frontier_start_label = frontier_start_label
+
+ # Verified state is NOT invalidated pre-run. On a failed / partial run we
+ # want the prior verified prefix preserved so the next edit can still use
+ # the optimization. YAML-diff-based invalidation for edited/downstream
+ # labels happens in update_and_run_blocks_tool at edit time, which is the
+ # right moment to drop stale outputs. Full success at the end of this
+ # function updates verified state in place (overwriting re-run labels).
+
+ # Common-case staging leaves the canonical row stale; prefer the staged copy.
+ workflow = ctx.staged_workflow
+ if workflow is None:
+ workflow = await app.DATABASE.workflows.get_workflow_by_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ )
+ if not workflow:
+ return {"ok": False, "error": f"Workflow not found: {ctx.workflow_permanent_id}"}
+
+ credential_ids = list(
+ dict.fromkeys(
+ _extract_credential_ids_from_tool_value(params.get("parameters") or {})
+ + _extract_credential_ids_from_workflow_definition(workflow.workflow_definition)
+ )
+ )
+ credential_error = await _credential_ids_validation_error(credential_ids, ctx)
+ if credential_error is not None:
+ return {"ok": False, "error": credential_error}
+
+ for label in block_labels:
+ if not workflow.get_output_parameter(label):
+ return {"ok": False, "error": f"Block label not found in saved workflow: {label!r}"}
+
+ from skyvern.forge.sdk.schemas.organizations import Organization
+ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody
+ from skyvern.services import workflow_service
+
+ org = await app.DATABASE.organizations.get_organization(organization_id=ctx.organization_id)
+ if not org:
+ return {"ok": False, "error": "Organization not found"}
+
+ organization = Organization.model_validate(org)
+ runtime_workflow = _workflow_with_runtime_block_goal_context(workflow, ctx)
+ runtime_workflow, runtime_frontier_anchor_url = _workflow_with_runtime_frontier_anchor(
+ runtime_workflow,
+ ctx,
+ labels_to_execute=labels_to_execute,
+ frontier_start_label=frontier_start_label,
+ block_outputs_to_seed=block_outputs_to_seed,
+ )
+ runtime_frontier_starter_url_seeded = False
+
+ user_params: dict[str, Any] = params.get("parameters") or {}
+ all_workflow_params, all_output_params = await asyncio.gather(
+ app.WORKFLOW_SERVICE.get_workflow_parameters(workflow_id=workflow.workflow_id),
+ app.DATABASE.workflow_params.get_workflow_output_parameters(workflow_id=workflow.workflow_id),
+ )
+
+ # Short-circuit before a wasted workflow execution when the definition
+ # JSON has drifted from the persisted parameter rows that runtime reads.
+ invariant_error = _parameter_binding_invariant_error(workflow, all_workflow_params, all_output_params)
+ if invariant_error is not None:
+ summary, missing_persisted, missing_from_definition = invariant_error
+ return {
+ "ok": False,
+ "error": summary,
+ "data": {
+ "workflow_run_id": None,
+ "overall_status": "failed",
+ "failure_reason": summary,
+ "requested_block_labels": list(block_labels),
+ "executed_block_labels": [],
+ "frontier_start_label": None,
+ "blocks": [],
+ "failure_categories": [
+ {
+ "category": "PARAMETER_BINDING_ERROR",
+ "confidence_float": 0.99,
+ "reasoning": "Pre-run invariant: workflow_definition and persisted parameter rows disagree",
+ }
+ ],
+ "missing_persisted": missing_persisted,
+ "missing_from_definition": missing_from_definition,
+ },
+ }
+
+ data: dict[str, Any] = {}
+ for wp in all_workflow_params:
+ if wp.key in user_params:
+ data[wp.key] = user_params[wp.key]
+ elif wp.default_value is not None:
+ data[wp.key] = wp.default_value
+ else:
+ placeholder = _placeholder_for_parameter_type(wp.workflow_parameter_type)
+ if placeholder is not None:
+ data[wp.key] = placeholder
+ LOG.info(
+ "Auto-filled missing workflow parameter for copilot test run",
+ parameter_key=wp.key,
+ parameter_type=str(wp.workflow_parameter_type),
+ )
+
+ # Without a session, the workflow service launches the browser in-process,
+ # which only works in worker pods (cloakbrowser isn't in the API image).
+ session_err = await ensure_browser_session(ctx)
+ if session_err is not None:
+ return session_err
+
+ seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(
+ runtime_workflow,
+ ctx,
+ labels_to_execute=labels_to_execute,
+ runtime_frontier_anchor_url=runtime_frontier_anchor_url,
+ )
+ runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
+ runtime_workflow = seeded_runtime_workflow
+
+ workflow_request = WorkflowRequestBody(
+ data=data if data else None,
+ browser_session_id=ctx.browser_session_id,
+ # Copilot test runs don't need scrolling post-action screenshots;
+ # the ForgeAgent's split screenshots (used for LLM context) are unaffected.
+ max_screenshot_scrolls=0,
+ )
+
+ workflow_run = await workflow_service.prepare_workflow(
+ workflow_id=ctx.workflow_permanent_id,
+ organization=organization,
+ workflow_request=workflow_request,
+ template=False,
+ version=None,
+ max_steps=None,
+ request_id=None,
+ copilot_session_id=ctx.workflow_copilot_chat_id,
+ )
+
+ from skyvern.utils.files import initialize_skyvern_state_file
+
+ await initialize_skyvern_state_file(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=ctx.organization_id,
+ )
+
+ run_task = asyncio.create_task(
+ app.WORKFLOW_SERVICE.execute_workflow(
+ workflow_run_id=workflow_run.workflow_run_id,
+ api_key="copilot-agent",
+ organization=organization,
+ browser_session_id=ctx.browser_session_id,
+ block_labels=labels_to_execute,
+ block_outputs=block_outputs_to_seed or None,
+ workflow_override=runtime_workflow,
+ )
+ )
+
+ # The OpenAI Agents SDK wraps this tool in
+ # ``asyncio.wait_for(..., timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS)``, so
+ # the inner budget leaves 10 s of headroom for the cancel-drain and
+ # post-drain reconcile to finish before the SDK's own cancel fires.
+ #
+ # Do NOT short-circuit on client disconnect: the agent loop runs to
+ # completion after the SSE stream is gone so its reply persists
+ # (SKY-8986); aborting mid-block would strand the run without debug
+ # output for the final chat message.
+ initial_run, initial_step_ts, initial_block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
+ progress_marker = _progress_marker(initial_run, initial_step_ts, initial_block_ts)
+ last_progress_monotonic = time.monotonic()
+ started_monotonic = last_progress_monotonic
+ final_status: str | None = None
+ run: Any = initial_run
+ exit_reason: WatchdogExitReason | None = None
+ run_cancelled_by_watchdog = False
+ active_run_terminal_evidence: ActiveRunTerminalEvidenceSample | None = None
+ # Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have
+ # DB-silent periods; disable stagnation for any invocation that includes
+ # one. Safety ceiling still applies.
+ stagnation_enabled = not _any_quiet_block_requested(ctx, labels_to_execute)
+ # Active block runs use the tighter per-tool budget so a single in-flight
+ # call cannot consume the whole copilot session. Quiet-block runs keep the
+ # long safety ceiling because HumanInteractionBlock can legitimately pause
+ # indefinitely.
+ budget_exit_reason: WatchdogExitReason
+ if stagnation_enabled:
+ budget_seconds = _active_block_run_budget_seconds(ctx)
+ budget_exit_reason = "per_tool_budget"
+ else:
+ budget_seconds = max(1, RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10)
+ budget_exit_reason = "ceiling"
+
+ # Mid-tool narrator bridge: feed block-status changes and step-level
+ # heartbeats into NarratorState so the narration ticker keeps emitting
+ # while a long workflow run is in flight.
+ narrator_state: NarratorState | None = getattr(ctx, "narrator_state", None)
+ narrator_enabled = narrator_state is not None and narration_handler_available()
+ seen_block_states: dict[str, str] = {}
+ prior_block_ts: datetime | None = initial_block_ts
+ last_block_fetch_monotonic = 0.0
+ active_terminal_monitor_enabled = await _active_run_terminal_monitor_enabled(ctx)
+ next_active_terminal_monitor_monotonic = (
+ started_monotonic + _ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS
+ if active_terminal_monitor_enabled
+ else float("inf")
+ )
+ active_terminal_monitor_samples = 0
+
+ try:
+ while True:
+ await asyncio.sleep(RUN_BLOCKS_POLL_INTERVAL_SECONDS)
+
+ run, step_ts, block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
+
+ if narrator_enabled:
+ assert narrator_state is not None # narrator_enabled implies non-None
+ tick_result = await narrator_poll_tick(
+ narrator_state,
+ current_block_ts=block_ts,
+ prior_block_ts=prior_block_ts,
+ last_block_fetch_monotonic=last_block_fetch_monotonic,
+ seen_block_states=seen_block_states,
+ fetch_block_statuses=lambda: app.DATABASE.observer.get_workflow_run_blocks(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=ctx.organization_id,
+ ),
+ stream=ctx.stream,
+ block_state_map=ctx.block_state_map,
+ block_started_at_map=ctx.block_started_at_map,
+ block_ended_at_map=ctx.block_ended_at_map,
+ )
+ prior_block_ts = tick_result.prior_block_ts
+ last_block_fetch_monotonic = tick_result.last_block_fetch_monotonic
+
+ if run and WorkflowRunStatus(run.status).is_final():
+ final_status = run.status
+ exit_reason = "success"
+ break
+
+ if run_task.done():
+ # Row not terminal yet — shared reconcile path below flips
+ # most of these back to success after post-drain reread.
+ exit_reason = "task_exit_unfinalized"
+ break
+
+ now = time.monotonic()
+ new_marker = _progress_marker(run, step_ts, block_ts)
+ # A run in ``paused`` status (e.g. HumanInteractionBlock) is a
+ # user-driven wait, not stagnation — never trip.
+ is_paused = run is not None and run.status == WorkflowRunStatus.paused.value
+ stagnation_active = stagnation_enabled and not is_paused
+
+ if (
+ active_terminal_monitor_enabled
+ and not is_paused
+ and active_terminal_monitor_samples < _ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES
+ and now >= next_active_terminal_monitor_monotonic
+ ):
+ active_terminal_monitor_samples += 1
+ next_active_terminal_monitor_monotonic += _ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS
+ sample = await _active_run_terminal_evidence_sample(
+ ctx,
+ workflow_run_id=workflow_run.workflow_run_id,
+ labels_to_execute=labels_to_execute,
+ sample_index=active_terminal_monitor_samples,
+ )
+ if sample is not None:
+ post_sample_run = await _safe_read_workflow_run(
+ workflow_run.workflow_run_id,
+ ctx.organization_id,
+ context="active-terminal-post-sample",
+ )
+ if post_sample_run is not None:
+ run = post_sample_run
+ active_run_terminal_evidence = sample
+ exit_reason = "active_run_terminal_evidence"
+ break
+
+ if new_marker != progress_marker:
+ progress_marker = new_marker
+ last_progress_monotonic = now
+ elif stagnation_active and now - last_progress_monotonic >= RUN_BLOCKS_STAGNATION_WINDOW_SECONDS:
+ exit_reason = "stagnation"
+ break
+
+ if now - started_monotonic >= budget_seconds:
+ exit_reason = budget_exit_reason
+ break
+
+ if exit_reason is not None and exit_reason != "success":
+ # Pre-cancel read first: a legitimate self-finalize (user/block
+ # cancel, or any terminal the run wrote itself) can land between
+ # the last poll and here, and trusting it avoids the
+ # synthetic-``canceled`` ambiguity that the post-drain reread
+ # has to exclude. Then cancel + reread +
+ # ``_trusted_post_drain_status`` applies SKY-9167's success-race
+ # recovery uniformly to all three non-success exit reasons.
+ pre_cancel_run = await _safe_read_workflow_run(
+ workflow_run.workflow_run_id, ctx.organization_id, context="pre-cancel"
+ )
+ if (
+ _watchdog_exit_allows_terminal_promotion(exit_reason)
+ and pre_cancel_run is not None
+ and WorkflowRunStatus(pre_cancel_run.status).is_final()
+ ):
+ final_status = pre_cancel_run.status
+ run = pre_cancel_run
+ exit_reason = "success"
+ else:
+ if pre_cancel_run is not None:
+ run = pre_cancel_run
+ if run is None or not WorkflowRunStatus(run.status).is_final():
+ await _cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id)
+ run_cancelled_by_watchdog = True
+ run = await _safe_read_workflow_run(
+ workflow_run.workflow_run_id, ctx.organization_id, context="post-drain"
+ )
+ if _watchdog_exit_allows_terminal_promotion(exit_reason):
+ trusted = _trusted_post_drain_status(run)
+ if trusted is not None:
+ final_status = trusted
+ exit_reason = "success"
+
+ if exit_reason != "success":
+ assert exit_reason is not None # narrows for mypy; outer check excludes "success" but not None
+ _mark_pending_reconciliation_run(ctx, workflow_run.workflow_run_id)
+ error_msg = await _watchdog_error_message(
+ exit_reason, ctx, workflow_run.workflow_run_id, run, budget_seconds
+ )
+ user_failure_reason = _watchdog_user_failure_reason(
+ exit_reason, workflow_run.workflow_run_id, budget_seconds, run
+ )
+ user_facing_summary = _watchdog_user_facing_summary(exit_reason, budget_seconds, run)
+ current_url, page_title = await _fallback_page_info(ctx)
+ if exit_reason == "active_run_terminal_evidence" and active_run_terminal_evidence is not None:
+ result: dict[str, Any] = _active_run_terminal_evidence_result(
+ workflow_run_id=workflow_run.workflow_run_id,
+ run_status=run.status if run is not None else None,
+ sample=active_run_terminal_evidence,
+ requested_block_labels=list(block_labels),
+ executed_block_labels=list(labels_to_execute),
+ current_url=current_url,
+ page_title=page_title,
+ )
+ result["error"] = error_msg
+ result["data"]["failure_reason"] = user_failure_reason
+ else:
+ result = {
+ "ok": False,
+ "error": error_msg,
+ "data": {
+ "workflow_run_id": workflow_run.workflow_run_id,
+ "overall_status": run.status if run is not None else None,
+ "failure_reason": user_failure_reason,
+ "current_url": current_url,
+ "page_title": page_title,
+ },
+ }
+ result["data"]["control_signal"] = {
+ "kind": f"watchdog_{exit_reason}",
+ "user_facing_summary": user_facing_summary,
+ }
+ result["data"]["user_facing_summary"] = user_facing_summary
+ if exit_reason == "per_tool_budget":
+ # Stable failure_categories entry so consecutive budget trips
+ # hash to the same streak signature; without it the run_id in
+ # ``error_msg`` would make every trip unique.
+ result["data"]["failure_categories"] = [
+ {
+ "category": PER_TOOL_BUDGET_FAILURE_CATEGORY,
+ "confidence_float": 1.0,
+ "reasoning": (
+ f"Per-tool-call budget of {budget_seconds}s exceeded; "
+ "the run was making progress but cannot fit in a single tool call."
+ ),
+ }
+ ]
+ if run_cancelled_by_watchdog:
+ result[_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY] = True
+ return result
+ except asyncio.CancelledError:
+ # The SDK's @function_tool(timeout=...) cancelled us mid-poll. Shield
+ # the cleanup so the parent cancellation can't interrupt it mid-await.
+ # If the shield itself is cancelled, fall back to a detached task
+ # that outlives tool teardown and still reconciles workflow state.
+ try:
+ await asyncio.shield(_cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id))
+ except asyncio.CancelledError:
+ fallback = asyncio.ensure_future(_cancel_run_task_if_not_final(run_task, workflow_run.workflow_run_id))
+ _DETACHED_CLEANUP_TASKS.add(fallback)
+ fallback.add_done_callback(_DETACHED_CLEANUP_TASKS.discard)
+ fallback.add_done_callback(_log_detached_cleanup_failure)
+ raise
+ finally:
+ # Belt and braces. If any exit path above missed a cancel — e.g. an
+ # unexpected exception bubbling out of the poll loop — make sure the
+ # run_task is at least signaled to cancel so we don't leak it.
+ if not run_task.done():
+ run_task.cancel()
+
+ if run and run.browser_session_id:
+ ctx.browser_session_id = run.browser_session_id
+
+ blocks = await app.DATABASE.observer.get_workflow_run_blocks(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=ctx.organization_id,
+ )
+
+ results = []
+ block_outputs_by_label: dict[str, Any] = {}
+ for block in blocks:
+ block_result: dict[str, Any] = {
+ "label": block.label,
+ "block_type": block.block_type.name if hasattr(block.block_type, "name") else str(block.block_type),
+ "status": block.status,
+ }
+ if block.failure_reason:
+ block_result["failure_reason"] = block.failure_reason
+ if hasattr(block, "output") and block.output:
+ block_result["extracted_data"] = block.output
+ if block.label is not None:
+ block_outputs_by_label[block.label] = block.output
+ results.append(block_result)
+
+ await _attach_action_traces(blocks, results, ctx.organization_id)
+
+ # final_status is guaranteed set here: every non-success exit returns
+ # above, and the success path always populates final_status.
+ assert final_status is not None
+ run_ok = WorkflowRunStatus(final_status) == WorkflowRunStatus.completed
+
+ action_trace_summary: list[str] = []
+ first_failed = next(
+ (r for r in results if r.get("status") in _FAILED_BLOCK_STATUSES and r.get("action_trace")),
+ None,
+ )
+ if first_failed is not None:
+ action_trace_summary = _summarize_action_trace(first_failed.get("action_trace"))
+
+ # Compute the action-sequence fingerprint BEFORE we strip action_trace.
+ # Stash it on a pending ctx field so update_repeated_failure_state can
+ # compare the NEW fingerprint against ctx.last_action_sequence_fingerprint
+ # (the PRIOR value) and increment the streak. Never enters the LLM-visible
+ # packet. Drives the repeated-action streak that hard-aborts a stuck
+ # fill→click→re-fill loop in _tool_loop_error.
+ ctx.pending_action_sequence_fingerprint = compute_action_sequence_fingerprint(results)
+
+ # Per-block action_trace is for derivation only — keep it out of the
+ # compact packet. get_run_results remains the heavier inspection path.
+ for entry in results:
+ entry.pop("action_trace", None)
+
+ current_url, page_title = await _fallback_page_info(ctx)
+
+ screenshot_b64: str | None = None
+ if not run_ok and ctx.browser_session_id:
+ try:
+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
+ session_id=ctx.browser_session_id,
+ organization_id=ctx.organization_id,
+ )
+ if browser_state:
+ page = await browser_state.get_or_create_page()
+ if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
+ try:
+ await SkyvernFrame.hide_cursor_overlay(page)
+ except Exception:
+ pass
+ try:
+ screenshot_bytes = await page.screenshot(type="png")
+ finally:
+ if SettingsManager.get_settings().BROWSER_CURSOR_VISUALIZATION:
+ try:
+ await SkyvernFrame.show_cursor_overlay(page)
+ except Exception:
+ pass
+ screenshot_b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+ except Exception:
+ LOG.debug("Failed to capture post-run screenshot", exc_info=True)
+
+ result_data: dict[str, Any] = {
+ "workflow_run_id": workflow_run.workflow_run_id,
+ "browser_session_id": ctx.browser_session_id,
+ "overall_status": final_status,
+ "requested_block_labels": list(block_labels),
+ "executed_block_labels": list(labels_to_execute),
+ "frontier_start_label": frontier_start_label,
+ "blocks": results,
+ "current_url": current_url,
+ "page_title": page_title,
+ "action_trace_summary": action_trace_summary,
+ }
+ if runtime_frontier_anchor_url is not None:
+ result_data["runtime_frontier_anchor_url"] = runtime_frontier_anchor_url
+ if runtime_frontier_starter_url_seeded:
+ result_data["runtime_frontier_starter_url_seeded"] = True
+ if screenshot_b64 is not None:
+ result_data["screenshot_base64"] = screenshot_b64
+ if not run_ok and run and getattr(run, "failure_reason", None):
+ result_data["failure_reason"] = run.failure_reason
+
+ # Update verified prefix state ONLY on a fully-successful run. A failed
+ # suffix run leaves the browser in post-failure state, so we must not
+ # trust blocks that individually succeeded inside it.
+ if run_ok and all(r.get("status") == "completed" for r in results):
+ for label, output in block_outputs_by_label.items():
+ ctx.verified_block_outputs[label] = output
+ existing_prefix = list(getattr(ctx, "verified_prefix_labels", []) or [])
+ existing_set = set(existing_prefix)
+ for label in labels_to_execute:
+ if label not in existing_set:
+ existing_prefix.append(label)
+ existing_set.add(label)
+ ctx.verified_prefix_labels = existing_prefix
+ verified_current_url = _valid_runtime_anchor_url(current_url)
+ if verified_current_url is not None:
+ ctx.verified_prefix_current_url = verified_current_url
+
+ return build_run_blocks_response(run_ok, result_data)
+
+
+async def _get_run_results(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
+ workflow_run_id = params.get("workflow_run_id")
+ pending_run_id = getattr(ctx, "pending_reconciliation_run_id", None)
+ if isinstance(pending_run_id, str) and pending_run_id:
+ if workflow_run_id and workflow_run_id != pending_run_id:
+ return {
+ "ok": False,
+ "error": (
+ f"Run inspection is pending for {pending_run_id}; "
+ "call get_run_results with that workflow_run_id first."
+ ),
+ }
+ workflow_run_id = pending_run_id
+ if not workflow_run_id:
+ same_turn_run_id = getattr(ctx, "last_successful_run_blocks_workflow_run_id", None)
+ if not isinstance(same_turn_run_id, str) or not same_turn_run_id:
+ same_turn_run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
+ if isinstance(same_turn_run_id, str) and same_turn_run_id:
+ workflow_run_id = same_turn_run_id
+
+ if not workflow_run_id:
+ # Include every final state so the agent can inspect failures via the
+ # fallback. Non-final states (created/queued/running/paused) remain
+ # excluded — reading block records from an in-flight run is unsafe.
+ runs = await app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id(
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ page=1,
+ page_size=1,
+ status=[
+ WorkflowRunStatus.completed,
+ WorkflowRunStatus.failed,
+ WorkflowRunStatus.terminated,
+ WorkflowRunStatus.canceled,
+ WorkflowRunStatus.timed_out,
+ ],
+ )
+ if not runs:
+ return {"ok": False, "error": "No runs found for this workflow."}
+ workflow_run_id = runs[0].workflow_run_id
+
+ run = await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=ctx.organization_id,
+ )
+ if not run:
+ return {"ok": False, "error": f"Workflow run not found: {workflow_run_id}"}
+ if getattr(run, "workflow_permanent_id", None) != ctx.workflow_permanent_id:
+ return {"ok": False, "error": f"Workflow run not found for this workflow: {workflow_run_id}"}
+
+ blocks = await app.DATABASE.observer.get_workflow_run_blocks(
+ workflow_run_id=workflow_run_id,
+ organization_id=ctx.organization_id,
+ )
+
+ results = []
+ for block in blocks:
+ block_result: dict[str, Any] = {
+ "label": block.label,
+ "block_type": block.block_type.name if hasattr(block.block_type, "name") else str(block.block_type),
+ "status": block.status,
+ }
+ if block.failure_reason:
+ block_result["failure_reason"] = block.failure_reason
+ output = truncate_output(getattr(block, "output", None))
+ if output:
+ block_result["output"] = output
+ results.append(block_result)
+
+ await _attach_action_traces(blocks, results, ctx.organization_id)
+ await _attach_failed_block_screenshots(blocks, results, ctx.organization_id)
+
+ result_data: dict[str, Any] = {
+ "workflow_run_id": workflow_run_id,
+ "overall_status": run.status,
+ "blocks": results,
+ }
+ current_url, page_title = await _fallback_page_info(ctx)
+ if current_url:
+ result_data["current_url"] = current_url
+ result_data["page_title"] = page_title
+ if getattr(run, "failure_reason", None):
+ result_data["failure_reason"] = run.failure_reason
+
+ return {
+ "ok": True,
+ "data": result_data,
+ }
+
+
+def _composition_anti_bot_reason(copilot_ctx: object) -> str | None:
+ evidence = getattr(copilot_ctx, "composition_page_evidence", None)
+ if not isinstance(evidence, dict):
+ return None
+ indicators = evidence.get("anti_bot_indicators")
+ challenge_controls = evidence.get("challenge_controls")
+ challenge_state = evidence.get("challenge_state")
+ normalized_indicators = (
+ [str(item) for item in indicators if isinstance(item, str)] if isinstance(indicators, list) else []
+ )
+ control_count = len(challenge_controls) if isinstance(challenge_controls, list) else 0
+ challenge_detected = isinstance(challenge_state, dict) and challenge_state.get("detected") is True
+ if not normalized_indicators and control_count == 0 and not challenge_detected:
+ return None
+ detail_parts = normalized_indicators[:4]
+ if isinstance(challenge_state, dict):
+ state_indicators = challenge_state.get("indicators")
+ if isinstance(state_indicators, list):
+ detail_parts.extend(str(item) for item in state_indicators if isinstance(item, str))
+ challenge_kind = challenge_state.get("kind")
+ if isinstance(challenge_kind, str) and challenge_kind and challenge_kind != "none":
+ detail_parts.append(challenge_kind)
+ gated_controls = challenge_state.get("gated_submit_controls")
+ if challenge_state.get("gates_submit_controls") is True:
+ gated_control_items = gated_controls if isinstance(gated_controls, list) else []
+ control_labels = [
+ str(item.get("text") or item.get("value") or item.get("id") or item.get("name") or item.get("selector"))
+ for item in gated_control_items
+ if isinstance(item, dict)
+ and (
+ item.get("text") or item.get("value") or item.get("id") or item.get("name") or item.get("selector")
+ )
+ ]
+ labels = ", ".join(list(dict.fromkeys(control_labels))[:3]) or "submit/search control"
+ detail_parts.append(f"challenge-gated disabled submit/search control: {labels}")
+ if control_count:
+ detail_parts.append(f"{control_count} challenge control(s)")
+ details = ", ".join(list(dict.fromkeys(detail_parts))[:6])
+ return f"Observed anti-bot challenge evidence before the run: {details}"
+
+
+# Generic failure-reason template emitted by the shared agent when the
+# browser-side scraper catches ScrapingFailed / NoElementFound. Matching on
+# the template (not the shared classifier) lets the copilot notice a repeated
+# site-block/unreadable-page pattern even though the classifier routes it to
+# DATA_EXTRACTION_FAILURE, not ANTI_BOT_DETECTION.
+# Coupling note: these substrings come from the run-level failure_reason
+# produced when the shared scraper raises ScrapingFailed. If the template
+# wording changes, update this tuple and the test that locks it in
+# (tests/unit/test_copilot_probable_site_block.py).
+_PROBABLE_SITE_BLOCK_FAILURE_REASON_SUBSTRINGS = (
+ "failed to load the website",
+ "page may have navigated unexpectedly",
+)
+
+
+def _detect_probable_site_block_wall(result: dict[str, Any]) -> bool:
+ """True when a block failed with the site-load template and the failure is
+ not a non-retriable nav error (DNS / SSL / invalid URL are owned by
+ :func:`_detect_non_retriable_nav_error`)."""
+ if bool(result.get("ok", False)):
+ return False
+ if _detect_non_retriable_nav_error(result):
+ return False
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return False
+ blocks = data.get("blocks")
+ if not isinstance(blocks, list):
+ return False
+
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ reason = block.get("failure_reason")
+ if isinstance(reason, str):
+ lowered = reason.lower()
+ if any(sub in lowered for sub in _PROBABLE_SITE_BLOCK_FAILURE_REASON_SUBSTRINGS):
+ return True
+ return False
+
+
+def _detect_non_retriable_nav_error(result: dict[str, Any]) -> str | None:
+ """Return the first failure_reason that matches SKIP_INNER_NAV_RETRY_ERRORS
+ (DNS / cert / SSL / invalid URL), preferring run-level over block-level.
+ Same set is_skip_inner_retry_error uses at the browser layer, so the copilot
+ classifies on exactly the patterns that already short-circuit retries in
+ navigate_with_retry (skyvern/webeye/navigation.py)."""
+ return next((reason for reason in iter_failure_reasons(result) if is_skip_inner_retry_error(reason)), None)
+
+
+def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, result: Mapping[str, object]) -> None:
+ evidence = _workflow_verification_evidence(copilot_ctx)
+ data_value = result.get("data")
+ data: Mapping[str, object] = data_value if isinstance(data_value, dict) else {}
+ run_ok = bool(result.get("ok", False))
+ full_workflow_verified = run_ok and copilot_ctx.last_full_workflow_test_ok is True
+ evidence.full_workflow_verified = full_workflow_verified
+ evidence.test_attempted_but_incomplete = not full_workflow_verified
+
+ run_id = data.get("workflow_run_id")
+ if isinstance(run_id, str) and run_id.strip():
+ evidence.workflow_run_id = run_id.strip()
+ if _active_run_terminal_evidence_detected(result):
+ evidence.active_run_terminal_evidence_detected = True
+ evidence.live_page_state_verified = True
+ evidence.verified_from_current_browser_state = True
+ evidence.full_workflow_verified = False
+ evidence.test_attempted_but_incomplete = True
+ if isinstance(run_id, str) and run_id.strip():
+ evidence.active_run_terminal_evidence_workflow_run_id = run_id.strip()
+ sample_index = data.get("active_run_terminal_evidence_sample_index")
+ if isinstance(sample_index, int):
+ evidence.active_run_terminal_evidence_sample_index = sample_index
+ current_url = _valid_runtime_anchor_url(data.get("current_url"))
+ if current_url is not None:
+ evidence.current_url = current_url
+ evidence.live_page_state_verified = True
+ if data.get("observed_after_workflow_run") is True:
+ evidence.current_url_observed_after_workflow_run = True
+ evidence.current_url_may_encode_runtime_state = bool(urlparse(current_url).query)
+ page_title = data.get("page_title")
+ if isinstance(page_title, str) and page_title.strip():
+ evidence.page_title = " ".join(page_title.split())[:160]
+
+ if run_ok:
+ evidence.merge_verified_blocks(_completed_run_block_labels(data))
+ unverified = list(copilot_ctx.last_unverified_block_labels or [])
+ evidence.unverified_block_labels = list(dict.fromkeys(str(label) for label in unverified if str(label)))
+ evidence.failed_block_labels = []
+ # A completed-but-suspicious run (outcome unverified, null data, blocker)
+ # keeps its failure reason so the evidence stays consistent with
+ # test_attempted_but_incomplete instead of reading as a clean success.
+ suspicious_reason = copilot_ctx.last_test_failure_reason if copilot_ctx.last_test_suspicious_success else None
+ evidence.failure_reason = (
+ " ".join(suspicious_reason.split())[:240]
+ if isinstance(suspicious_reason, str) and suspicious_reason.strip()
+ else None
+ )
+ if evidence.unverified_block_labels:
+ evidence.verified_from_current_browser_state = True
+ return
+
+ failed_labels = _failed_run_block_labels(data)
+ evidence.failed_block_labels = failed_labels
+ if copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY:
+ evidence.merge_per_tool_budget_blocks(failed_labels)
+ failure_reason = copilot_ctx.last_test_failure_reason or result.get("error")
+ if isinstance(failure_reason, str) and failure_reason.strip():
+ evidence.failure_reason = " ".join(failure_reason.split())[:240]
+
+
+def _record_run_blocks_result(
+ copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None
+) -> None:
+ run_ok = bool(result.get("ok", False))
+ data = result.get("data")
+ run_id = data.get("workflow_run_id") if isinstance(data, dict) else None
+ copilot_ctx.completion_verification_result = completion_verification
+ record_completion_verification(copilot_ctx, completion_verification)
+ if completion_verification is not None and completion_verification.status == "evaluated":
+ _emit_completion_verification_trace(copilot_ctx, completion_verification)
+ copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None
+ copilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else None
+ # Watchdog cancels normally count as ok=False; only a coincident total
+ # timeout softens to ``None`` to keep the unvalidated WIP rescue open.
+ cancelled_by_watchdog = result.get(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY) is True
+ timeout_latched = bool(copilot_ctx.copilot_total_timeout_exceeded)
+ copilot_ctx.last_test_ok = None if (cancelled_by_watchdog and timeout_latched) else run_ok
+ copilot_ctx.last_full_workflow_test_ok = False
+ copilot_ctx.last_unverified_block_labels = _unverified_current_workflow_labels(copilot_ctx)
+ copilot_ctx.last_test_failure_reason = None
+ copilot_ctx.last_test_suspicious_success = False
+ copilot_ctx.suspicious_success_nudge_count = 0
+ copilot_ctx.last_test_anti_bot = None
+ prior_budget_flag = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY
+ copilot_ctx.last_failure_category_top = None
+ copilot_ctx.last_test_non_retriable_nav_error = None
+ copilot_ctx.post_run_page_observation_tool = None
+ copilot_ctx.post_run_page_observation_url = None
+ copilot_ctx.post_run_page_observation_workflow_run_id = None
+ copilot_ctx.post_run_page_observation_after_failed_test = False
+ copilot_ctx.post_run_current_page_inspection_workflow_run_id = None
+
+ structured_blocker = _run_blocks_structured_blocker_message(result)
+ anti_bot_match, empty_data_blocks, failure_categories = _analyze_run_blocks(result)
+ if not anti_bot_match:
+ anti_bot_match = _composition_anti_bot_reason(copilot_ctx)
+ if anti_bot_match:
+ copilot_ctx.last_test_anti_bot = anti_bot_match
+ if failure_categories:
+ top = failure_categories[0]
+ if isinstance(top, dict):
+ top_category = top.get("category")
+ if isinstance(top_category, str):
+ copilot_ctx.last_failure_category_top = top_category
+
+ if copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY and isinstance(data, dict):
+ current_url = _valid_runtime_anchor_url(data.get("current_url"))
+ if current_url is not None:
+ copilot_ctx.post_budget_page_inspection_required = True
+ copilot_ctx.post_budget_page_inspection_url = current_url
+ copilot_ctx.post_budget_page_inspection_run_id = run_id if isinstance(run_id, str) else None
+
+ # A fresh budget trip on a different chain should get the dedicated split
+ # nudge again rather than falling through to the generic failed-test path,
+ # so reset the cap when the latest run is not itself a budget trip.
+ if prior_budget_flag and copilot_ctx.last_failure_category_top != PER_TOOL_BUDGET_FAILURE_CATEGORY:
+ copilot_ctx.per_tool_budget_nudge_count = 0
+
+ # Expose full failure classification in tool output for agent reasoning
+ if failure_categories:
+ data = result.get("data")
+ if isinstance(data, dict):
+ data["failure_categories"] = failure_categories
+
+ if _active_run_terminal_evidence_detected(result):
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+ signal = _active_run_terminal_evidence_signal(copilot_ctx, "update_and_run_blocks")
+ if signal is not None:
+ stash_blocker_signal(copilot_ctx, signal)
+
+ if run_ok:
+ _mark_page_inspected(copilot_ctx)
+ if structured_blocker:
+ failure_reason = f"Run completed, but extracted data reported a blocker: {structured_blocker}"
+ copilot_ctx.last_test_ok = False
+ copilot_ctx.last_test_suspicious_success = True
+ copilot_ctx.last_test_failure_reason = failure_reason
+ copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
+ if not copilot_ctx.last_test_anti_bot and _looks_like_anti_bot_blocker(structured_blocker):
+ copilot_ctx.last_test_anti_bot = f"Extracted data reported anti-bot blocker: {structured_blocker[:160]}"
+ data = result.get("data")
+ if isinstance(data, dict):
+ data.setdefault("failure_reason", failure_reason)
+ if copilot_ctx.last_test_anti_bot and not failure_categories:
+ anti_bot_category = {
+ "category": "ANTI_BOT_DETECTION",
+ "confidence_float": 0.7,
+ "reasoning": "Structured extracted data reported an anti-bot blocker.",
+ }
+ data["failure_categories"] = [anti_bot_category]
+ update_repeated_failure_state(copilot_ctx, result)
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+ return
+ if empty_data_blocks:
+ copilot_ctx.last_test_ok = None
+ copilot_ctx.last_test_suspicious_success = True
+ copilot_ctx.null_data_streak_count = getattr(copilot_ctx, "null_data_streak_count", 0) + 1
+ copilot_ctx.last_test_failure_reason = (
+ "All blocks completed but data-producing blocks "
+ "(extraction/text_prompt) produced no meaningful output "
+ "(missing, empty, or all-null fields). "
+ "The workflow may not be working correctly."
+ )
+ # Clean-ish success (no scrape-fail pattern): reset the streak.
+ copilot_ctx.probable_site_block_streak_count = 0
+ update_repeated_failure_state(copilot_ctx, result)
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+ return
+ copilot_ctx.failed_test_nudge_count = 0
+ copilot_ctx.null_data_streak_count = 0
+ copilot_ctx.probable_site_block_streak_count = 0
+ copilot_ctx.last_failed_workflow_yaml = None
+ # Real success: clear the signature latch so a subsequent bad URL in
+ # the same session can re-fire the stop nudge.
+ copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
+ unverified = _unverified_current_workflow_labels(copilot_ctx)
+ copilot_ctx.last_unverified_block_labels = unverified
+ outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ if outcome_unverified_reason is not None and _outcome_failure_warrants_repair(
+ copilot_ctx, completion_verification
+ ):
+ # The workflow already has a confirmation block, yet the produced
+ # evidence does not demonstrate the outcome (or contradicts it). Treat
+ # it as a suspicious success so the existing repair/partial machinery
+ # fires. A mid-build run with no confirmation block yet falls through to
+ # keep-building below; terminal success stays withheld either way via
+ # the verification result.
+ copilot_ctx.last_test_suspicious_success = True
+ copilot_ctx.last_test_failure_reason = outcome_unverified_reason
+ if isinstance(data, dict):
+ data.setdefault("failure_reason", outcome_unverified_reason)
+ elif not unverified:
+ copilot_ctx.last_full_workflow_test_ok = True
+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
+ else:
+ copilot_ctx.last_test_failure_reason = (
+ "The last run verified only the current browser frontier; unverified workflow blocks remain: "
+ + ", ".join(unverified[:8])
+ )
+ update_repeated_failure_state(copilot_ctx, result)
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+ return
+
+ copilot_ctx.last_failed_workflow_yaml = getattr(copilot_ctx, "workflow_yaml", None)
+ copilot_ctx.last_test_non_retriable_nav_error = _detect_non_retriable_nav_error(result)
+ if _detect_probable_site_block_wall(result):
+ copilot_ctx.probable_site_block_streak_count += 1
+ else:
+ copilot_ctx.probable_site_block_streak_count = 0
+
+ data = result.get("data")
+ if isinstance(data, dict):
+ blocks = data.get("blocks")
+ if isinstance(blocks, list):
+ for block in blocks:
+ if isinstance(block, dict) and block.get("failure_reason"):
+ copilot_ctx.last_test_failure_reason = str(block["failure_reason"])
+ break
+ if copilot_ctx.last_test_failure_reason is None:
+ copilot_ctx.last_test_failure_reason = next(iter_failure_reasons(result), None)
+ if result.get("error") and copilot_ctx.last_test_failure_reason is None:
+ copilot_ctx.last_test_failure_reason = str(result["error"])
+ update_repeated_failure_state(copilot_ctx, result)
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+
+
+def _record_diagnosis_repair_contract(
+ copilot_ctx: Any,
+ *,
+ source_tool: str,
+ result: dict[str, Any],
+ workflow_updated: bool = False,
+) -> DiagnosisRepairContract:
+ contract = build_diagnosis_repair_contract(
+ source_tool=source_tool,
+ result=result,
+ ctx=copilot_ctx,
+ workflow_updated=workflow_updated,
+ )
+ copilot_ctx.latest_diagnosis_repair_contract = contract
+ trace_data = contract.to_trace_data()
+ LOG.info(
+ "copilot diagnosis repair contract shadow",
+ **{f"diagnosis_repair_{key}": value for key, value in trace_data.items()},
+ )
+ with copilot_span("diagnosis_repair_contract", data=trace_data):
+ pass
+ return contract
+
+
+def _diagnosis_repair_tool_error(copilot_ctx: Any, source_tool: str, error: str) -> str:
+ result = {"ok": False, "error": error}
+ _record_diagnosis_repair_contract(copilot_ctx, source_tool=source_tool, result=result)
+ return json.dumps(result)
+
+
+def _run_blocks_span_data(
+ block_labels: list[str],
+ labels_to_execute: list[str],
+ frontier_start_label: str | None,
+ seeded_outputs: dict[str, Any],
+ ctx: object,
+) -> dict[str, Any]:
+ return {
+ "requested_block_labels": block_labels,
+ "executed_block_labels": labels_to_execute,
+ "frontier_start_label": frontier_start_label,
+ "seeded_output_count": len(seeded_outputs or {}),
+ "repeated_failure_streak_count": int(getattr(ctx, "repeated_failure_streak_count", 0) or 0),
+ "block_count": len(block_labels),
+ }
+
+
+def _frontier_run_size_result(
+ error: str,
+ block_labels: list[str],
+ labels_to_execute: list[str],
+) -> dict[str, Any]:
+ suggested_labels = list(labels_to_execute[:_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS])
+ user_facing_summary = (
+ "Workflow draft saved; I still need to test the next smaller browser frontier before continuing."
+ )
+ return {
+ "ok": False,
+ "error": error,
+ "data": {
+ "workflow_run_id": None,
+ "overall_status": "skipped",
+ "workflow_run_skipped": True,
+ "requested_block_labels": list(block_labels),
+ "executed_block_labels": [],
+ "planned_block_labels": list(labels_to_execute),
+ "suggested_block_labels": suggested_labels,
+ "deferred_block_labels": list(labels_to_execute[_MAX_INCREMENTAL_PAGE_FRONTIER_LABELS:]),
+ "control_signal": {
+ "kind": "intermediate_success",
+ "user_facing_summary": user_facing_summary,
+ "next_tool": "run_blocks_and_collect_debug",
+ "next_block_labels": suggested_labels,
+ "preserve_workflow_yaml": True,
+ },
+ "user_facing_summary": user_facing_summary,
+ },
+ }
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
new file mode 100644
index 000000000..87cf75cc9
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any, cast
+from urllib.parse import urlparse
+
+import structlog
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ scout_accessible_role_name_expression as _scout_accessible_role_name_expression,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ SCOUT_INTERACTION_EVIDENCE_TOOL,
+ has_bounded_page_schema,
+)
+from skyvern.forge.sdk.copilot.runtime import (
+ AgentContext,
+ PendingBrowserInteractionObservation,
+ ScoutedInteraction,
+)
+
+from ._shared import (
+ _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ _append_flow_evidence,
+ _same_page_ignoring_fragment,
+ _workflow_verification_evidence,
+)
+
+LOG = structlog.get_logger()
+
+
+def _mark_page_inspected(ctx: AgentContext) -> None:
+ ctx.post_budget_page_inspection_required = False
+ ctx.post_budget_page_inspection_url = None
+ ctx.post_budget_page_inspection_run_id = None
+
+
+def _clear_pending_browser_interaction_observation(ctx: AgentContext) -> None:
+ ctx.pending_browser_interaction_observation = None
+
+
+def _mark_pending_browser_interaction_observation(ctx: AgentContext, *, tool_name: str, url: str) -> None:
+ if not url.strip():
+ _clear_pending_browser_interaction_observation(ctx)
+ return
+ ctx.pending_browser_interaction_observation = PendingBrowserInteractionObservation(
+ tool_name=tool_name,
+ url=url.strip(),
+ )
+
+
+def _consume_pending_browser_interaction_observation(
+ ctx: AgentContext,
+ *,
+ current_url: str,
+ evidence: dict[str, Any],
+) -> bool:
+ pending = ctx.pending_browser_interaction_observation
+ if pending is None:
+ return False
+ _clear_pending_browser_interaction_observation(ctx)
+ if not has_bounded_page_schema(evidence):
+ return False
+ if not _same_page_ignoring_fragment(pending.url, current_url):
+ LOG.warning(
+ "copilot_pending_browser_interaction_observation_page_mismatch",
+ tool_name=pending.tool_name,
+ pending_url=pending.url,
+ current_url=current_url,
+ )
+ return False
+ return True
+
+
+_MAX_SCOUTED_INTERACTIONS = 20
+
+
+async def _live_working_page_url(ctx: AgentContext) -> str | None:
+ if not ctx.browser_session_id:
+ return None
+ try:
+ browser_state = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_state(
+ session_id=ctx.browser_session_id,
+ organization_id=ctx.organization_id,
+ )
+ if not browser_state:
+ return None
+ page = await browser_state.get_or_create_page()
+ return page.url if page else None
+ except Exception:
+ return None
+
+
+async def _capture_scout_source_url(ctx: AgentContext) -> None:
+ # Pre-action: a navigating click/Enter would leave only the destination URL, not the page the selector acted on.
+ ctx.pending_scout_source_url = await _live_working_page_url(ctx)
+
+
+def _consume_scout_source_url(ctx: AgentContext) -> str | None:
+ source_url = ctx.pending_scout_source_url
+ # Cleared unconditionally so a non-recording action can't bleed its source page into a later interaction.
+ ctx.pending_scout_source_url = None
+ return source_url
+
+
+_ROLE_NAME_SELECTOR_RE = re.compile(r'^role=([a-zA-Z]+)(?:\[name="((?:[^"\\]|\\.)*)"\])?(.*)$')
+
+
+def _role_name_from_selector(selector: str) -> tuple[str, str] | None:
+ """Parse the ``role=<role>[name="<name>"]`` form (ref_to_selector) — TIER 1, no browser read.
+
+ Returns (role, accessible_name) when the selector is a plain role/name locator;
+ None for bare CSS/xpath or when an engine chain (`>> nth=`) trails the role/name.
+ """
+ selector = selector.strip()
+ match = _ROLE_NAME_SELECTOR_RE.match(selector)
+ if not match:
+ return None
+ role, raw_name, suffix = match.group(1), match.group(2), match.group(3)
+ if suffix.strip():
+ return None
+ name = raw_name.replace('\\"', '"') if raw_name is not None else ""
+ return role, name
+
+
+async def _capture_accessible_role_name(ctx: AgentContext, selector: str) -> tuple[str, str] | None:
+ """TIER 2: read the element's role/accessible name for a bare CSS/xpath selector.
+
+ A failed read degrades gracefully to None so the selector-only auto-credit
+ path (SKY-10712) stays intact.
+ """
+ selector = selector.strip()
+ if not selector:
+ return None
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return None
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool(
+ "skyvern_evaluate",
+ {"expression": _scout_accessible_role_name_expression(selector)},
+ ),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return None
+ if not isinstance(result, dict) or not result.get("ok"):
+ return None
+ value = (result.get("data") or {}).get("result")
+ if not isinstance(value, dict):
+ return None
+ role = str(value.get("role") or "").strip()
+ name = str(value.get("accessible_name") or "").strip()
+ if not role and not name:
+ return None
+ return role, name
+
+
+async def _resolve_scout_role_name(
+ ctx: AgentContext, selector: str, *, allow_browser_read: bool = True
+) -> tuple[str, str]:
+ """Resolve (role, accessible_name) for a scouted selector. TIER 1 parse first;
+ TIER 2 browser read only for bare CSS/xpath. Always degrades to ("", "").
+
+ ``allow_browser_read=False`` skips TIER 2 when the action navigated: a post-action
+ read against the landing page would capture the wrong element's name, so the bare
+ selector is kept verbatim (the synthesizer prefers it anyway)."""
+ selector = selector.strip()
+ if not selector:
+ return "", ""
+ parsed = _role_name_from_selector(selector)
+ if parsed is not None:
+ return parsed
+ if not allow_browser_read:
+ return "", ""
+ captured = await _capture_accessible_role_name(ctx, selector)
+ if captured is not None:
+ return captured
+ return "", ""
+
+
+def _record_scouted_interaction(
+ ctx: AgentContext,
+ *,
+ tool_name: str,
+ selector: str = "",
+ source_url: str | None = None,
+ value: str = "",
+ key: str = "",
+ typed_length: int = 0,
+ role: str = "",
+ accessible_name: str = "",
+) -> None:
+ selector = selector.strip()
+ # press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
+ if tool_name != "press_key" and not selector:
+ return
+ artifact: ScoutedInteraction = {"tool_name": tool_name}
+ if selector:
+ artifact["selector"] = selector
+ if source_url and source_url.strip():
+ artifact["source_url"] = source_url.strip()
+ if value:
+ artifact["value"] = value
+ if key:
+ artifact["key"] = key
+ if typed_length:
+ artifact["typed_length"] = typed_length
+ if role:
+ artifact["role"] = role
+ if accessible_name:
+ artifact["accessible_name"] = accessible_name
+ interactions = [
+ item
+ for item in ctx.scouted_interactions
+ if not (
+ item.get("tool_name") == artifact["tool_name"]
+ and item.get("selector") == artifact.get("selector")
+ and item.get("source_url") == artifact.get("source_url")
+ )
+ ]
+ interactions.append(artifact)
+ ctx.scouted_interactions = interactions[-_MAX_SCOUTED_INTERACTIONS:]
+
+ trajectory = list(ctx.scout_trajectory)
+ trajectory_artifact = cast(ScoutedInteraction, artifact.copy())
+ trajectory_artifact["trajectory_index"] = len(trajectory)
+ trajectory.append(trajectory_artifact)
+ ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]
+
+ LOG.info(
+ "copilot_scout_interaction_captured",
+ tool_name=tool_name,
+ selector=selector or None,
+ source_url=artifact.get("source_url"),
+ role=role or None,
+ total_scouted_interactions=len(ctx.scouted_interactions),
+ total_scout_trajectory=len(ctx.scout_trajectory),
+ )
+
+
+def _register_scout_interaction_observation(
+ ctx: AgentContext, *, tool_name: str, selector: str, source_url: str | None, url: str
+) -> int | None:
+ # A successful scout interaction reaches the post-action page; record it as an
+ # interaction-reached observation so a click-reached block can be authored
+ # against it without a separate inspect_page_for_composition.
+ selector = selector.strip()
+ if not selector or not url:
+ return None
+ evidence: dict[str, Any] = {
+ "inspected_url": url,
+ "current_url": url,
+ "source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,
+ "interaction_tool": tool_name,
+ "interaction_selector": selector,
+ }
+ if source_url and source_url.strip():
+ evidence["interaction_source_url"] = source_url.strip()
+ return _append_flow_evidence(ctx, evidence, reached_via="interaction")
+
+
+def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:
+ run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
+ if not isinstance(run_id, str) or not run_id:
+ return
+ ctx.post_run_page_observation_tool = source_tool
+ ctx.post_run_page_observation_url = url
+ ctx.post_run_page_observation_workflow_run_id = run_id
+ ctx.post_run_page_observation_after_failed_test = getattr(ctx, "last_test_ok", None) is False
+ evidence = _workflow_verification_evidence(ctx)
+ evidence.live_page_state_verified = True
+ evidence.verified_from_current_browser_state = True
+ evidence.workflow_run_id = run_id
+ if url:
+ evidence.current_url = url
+ evidence.current_url_observed_after_workflow_run = True
+ evidence.current_url_may_encode_runtime_state = bool(urlparse(url).query)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
new file mode 100644
index 000000000..6417b4be8
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -0,0 +1,634 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from typing import Annotated, Any, Literal
+
+import structlog
+import yaml
+from pydantic import AliasChoices, BaseModel, Field, ValidationError
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ composition_page_evidence_error,
+ normalize_block_observation_refs,
+ workflow_target_url,
+)
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.enforcement import (
+ POST_INTERMEDIATE_SUCCESS_NUDGE,
+ _goal_likely_needs_more_blocks,
+)
+from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
+from skyvern.forge.sdk.copilot.output_policy import (
+ evaluate_output_policy,
+ format_output_policy_tool_error,
+ output_policy_verdict_to_trace_data,
+)
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
+from skyvern.forge.sdk.workflow.models.workflow import Workflow
+from skyvern.schemas.proxy_location import ProxyLocation
+from skyvern.schemas.workflows import BlockType
+
+from ._shared import (
+ BLOCK_RUNNING_TOOLS,
+ _enum_or_string_name,
+ _proxy_location_trace_value,
+ _raw_yaml_proxy_location,
+ _workflow_definition_as_dict,
+ _workflow_yaml_blocks_by_label,
+)
+from .banned_blocks import (
+ _banned_block_reject_message,
+ _challenge_http_request_reject_message,
+ _copilot_banned_block_types,
+ _detect_new_banned_blocks,
+ _record_banned_block_reject_span,
+ _timing_only_challenge_wait_reject_message,
+)
+from .blockers import _clear_resolved_per_tool_budget_problem_labels
+from .credentials import (
+ _credential_id_misbinding_error_message,
+ _credential_id_misbinding_findings,
+ _credential_reference_validation_error,
+)
+from .frontier import (
+ _detect_stale_block_metadata,
+ _get_prior_workflow,
+ _invalidate_verified_state_on_edit,
+ _stale_block_metadata_message,
+ _workflow_requires_canonical_persist,
+)
+from .guardrails import _authority_tool_error
+
+LOG = structlog.get_logger()
+
+
+class BlockObservationRef(BaseModel):
+ label: str
+ observation_step: Annotated[int, Field(ge=0)]
+
+
+ArtifactEvidenceStatus = Literal["satisfied", "missing", "diagnostic_only", "observed_not_verified"]
+
+
+class CodeArtifactClaimedOutcome(BaseModel):
+ id: str
+ scope: str
+ text: str
+ status: ArtifactEvidenceStatus
+ depends_on: list[str] = Field(default_factory=list)
+ covered_criteria: list[str] = Field(default_factory=list)
+ criteria_ids: list[str] = Field(default_factory=list)
+ evidence_refs: list[str] = Field(default_factory=list)
+ observation_refs: list[str] = Field(default_factory=list)
+ required_tokens: list[str] = Field(default_factory=list)
+ entities: list[str] = Field(default_factory=list)
+
+
+class CodeArtifactPageDependency(BaseModel):
+ id: str
+ scope: str
+ status: ArtifactEvidenceStatus
+ url_hint: str | None = None
+ page_state_hint: str | None = None
+ required_affordances: list[str] = Field(default_factory=list)
+ required_outcomes: list[str] = Field(default_factory=list)
+ evidence_refs: list[str] = Field(default_factory=list, description="Dependency-scoped evidence_ref ids.")
+ observation_refs: list[str] = Field(default_factory=list, description="Dependency-scoped observation_ref ids.")
+
+
+class CodeArtifactCompletionCriterion(BaseModel):
+ id: str
+ text: str
+ level: Literal["terminal", "outcome", "prefix", "method"]
+ outcome: str | None = None
+ terminal: bool | None = None
+
+
+class CodeArtifactScopedRef(BaseModel):
+ claim_id: str | None = None
+ dependency_id: str | None = None
+ criterion_id: str | None = None
+ evidence_ref: str | None = None
+ observation_ref: str | None = None
+ status: ArtifactEvidenceStatus = Field(validation_alias=AliasChoices("status", "evidence_status"))
+ source_tool: str | None = None
+ observation_step: Annotated[int, Field(ge=0)] | None = None
+ run_sample_id: str | None = None
+ current_url: str | None = None
+ source_label: str | None = None
+ checkpoint_next_mode: Literal["advance", "stop"] | None = None
+
+
+class CodeArtifactTerminalVerifierExpectation(BaseModel):
+ id: str
+ text: str
+ criteria_ids: list[str] = Field(default_factory=list)
+ claimed_outcome_ids: list[str] = Field(default_factory=list)
+
+
+class CodeArtifactExplorationObservation(BaseModel):
+ id: str
+ text: str
+ status: Literal["observed_not_verified"] = Field(
+ default="observed_not_verified",
+ validation_alias=AliasChoices("status", "evidence_status"),
+ )
+ observation_ref: str | None = None
+ source_tool: str | None = None
+ observation_step: Annotated[int, Field(ge=0)] | None = None
+ current_url: str | None = None
+ source_label: str | None = None
+ checkpoint_next_mode: Literal["stop"] | None = None
+
+
+class CodeArtifactMetadata(BaseModel):
+ artifact_id: str | None = None
+ block_label: str | None = None
+ block_id: str | None = None
+ declared_goal: str
+ claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(default_factory=list)
+ page_dependencies: list[CodeArtifactPageDependency] = Field(default_factory=list)
+ completion_criteria: list[CodeArtifactCompletionCriterion] = Field(default_factory=list)
+ evidence_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)
+ observation_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)
+ terminal_verifier_expectations: list[CodeArtifactTerminalVerifierExpectation] = Field(default_factory=list)
+ exploration_observations: list[CodeArtifactExplorationObservation] = Field(default_factory=list)
+
+
+_CODE_ARTIFACT_REQUIRED_LIST_FIELDS = (
+ "claimed_outcomes",
+ "page_dependencies",
+ "completion_criteria",
+ "terminal_verifier_expectations",
+)
+
+
+def _code_artifact_metadata_as_tool_argument(
+ metadata: list[CodeArtifactMetadata] | None,
+) -> list[dict[str, Any]]:
+ if not metadata:
+ return []
+ return [item.model_dump(mode="json", exclude_none=True) for item in metadata]
+
+
+def _format_code_artifact_violations(violations: list[str]) -> str:
+ # Surface every contract violation at once so the agent fixes them in a single
+ # update instead of round-tripping one error per `update_and_run_blocks` call.
+ if len(violations) == 1:
+ return violations[0]
+ numbered = "\n".join(f"{index}. {message}" for index, message in enumerate(violations, start=1))
+ return f"Artifact metadata has {len(violations)} contract violations; fix all of them in one update:\n{numbered}"
+
+
+def _normalize_code_artifact_metadata(
+ raw_metadata: Any,
+ workflow_yaml: str,
+) -> tuple[dict[str, dict[str, Any]], str | None]:
+ if raw_metadata in (None, [], {}):
+ return {}, None
+ items = _code_artifact_metadata_items(raw_metadata)
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ normalized: dict[str, dict[str, Any]] = {}
+ violations: list[str] = []
+ seen_labels: set[str] = set()
+ for raw_item in items:
+ try:
+ metadata = (
+ raw_item
+ if isinstance(raw_item, CodeArtifactMetadata)
+ else CodeArtifactMetadata.model_validate(raw_item)
+ )
+ except ValidationError as exc:
+ violations.append(f"Artifact metadata is malformed: {exc}")
+ continue
+ dumped = metadata.model_dump(mode="json", exclude_none=True)
+ label = str(dumped.get("block_label") or "").strip()
+ if not label:
+ violations.append("Artifact metadata requires a non-empty `block_label` for each artifact.")
+ continue
+ if label not in code_blocks:
+ violations.append(
+ f"Artifact metadata for `{label}` does not reference an existing code block label. "
+ "Attach metadata only to authored code blocks."
+ )
+ continue
+ if label in seen_labels:
+ violations.append(f"Artifact metadata contains duplicate entries for `{label}`.")
+ continue
+ seen_labels.add(label)
+ dumped["block_label"] = label
+ item_violations: list[str] = []
+ artifact_id = str(dumped.get("artifact_id") or "").strip()
+ if artifact_id and not artifact_id.startswith("code_artifact:"):
+ item_violations.append(
+ f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."
+ )
+ dumped["artifact_id"] = artifact_id or _artifact_id_for_block_label(label)
+ block_id = str(
+ dumped.get("block_id") or code_blocks[label].get("block_id") or code_blocks[label].get("id") or ""
+ ).strip()
+ if block_id:
+ dumped["block_id"] = block_id
+ declared_goal = str(dumped.get("declared_goal") or "").strip()
+ if not declared_goal:
+ item_violations.append(f"Artifact metadata for `{label}` requires a non-empty `declared_goal`.")
+ for field_name in _CODE_ARTIFACT_REQUIRED_LIST_FIELDS:
+ value = dumped.get(field_name)
+ if not isinstance(value, list) or not value:
+ item_violations.append(f"Artifact metadata for `{label}` requires non-empty `{field_name}`.")
+ if not dumped.get("evidence_refs") and not dumped.get("observation_refs"):
+ item_violations.append(f"Artifact metadata for `{label}` requires `evidence_refs` or `observation_refs`.")
+ item_violations.extend(_code_artifact_metadata_shape_errors(label, dumped))
+ if item_violations:
+ violations.extend(item_violations)
+ continue
+ normalized[label] = dumped
+ if violations:
+ return {}, _format_code_artifact_violations(violations)
+ return normalized, None
+
+
+def _code_artifact_metadata_items(raw_metadata: Any) -> list[Any]:
+ if isinstance(raw_metadata, Mapping):
+ items: list[Any] = []
+ for block_label, value in raw_metadata.items():
+ if isinstance(value, Mapping) and "block_label" not in value:
+ items.append({"block_label": block_label, **value})
+ else:
+ items.append(value)
+ return items
+ if isinstance(raw_metadata, list):
+ return raw_metadata
+ return [raw_metadata]
+
+
+def _workflow_yaml_code_blocks_by_label(workflow_yaml: str | None) -> dict[str, Mapping[str, Any]]:
+ blocks: dict[str, Mapping[str, Any]] = {}
+ for label, block in _workflow_yaml_blocks_by_label(workflow_yaml).items():
+ if _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value:
+ blocks[label] = block
+ return blocks
+
+
+def _artifact_id_for_block_label(label: str) -> str:
+ fragment = re.sub(r"[^A-Za-z0-9_.-]+", "_", label).strip("_") or "artifact"
+ return f"code_artifact:{fragment}"
+
+
+def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:
+ """Return every shape violation for one artifact; the caller aggregates them."""
+ errors: list[str] = []
+ for field_name, ref_key in (("evidence_refs", "evidence_ref"), ("observation_refs", "observation_ref")):
+ for index, ref in enumerate(_artifact_rows(artifact.get(field_name))):
+ if not str(ref.get(ref_key) or "").strip():
+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `{ref_key}`.")
+ if not any(str(ref.get(key) or "").strip() for key in ("claim_id", "dependency_id", "criterion_id")):
+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires a scoped id.")
+ status = str(ref.get("status") or "").strip()
+ if ref.get("checkpoint_next_mode") == "advance" and status != "diagnostic_only":
+ errors.append(
+ f"Artifact metadata for `{label}` `{field_name}` entry {index} has "
+ "`checkpoint_next_mode=advance`; it must stay `diagnostic_only`."
+ )
+ if ref.get("checkpoint_next_mode") == "stop" and status not in {"observed_not_verified", "diagnostic_only"}:
+ errors.append(
+ f"Artifact metadata for `{label}` `{field_name}` entry {index} has "
+ "`checkpoint_next_mode=stop`; it must remain `observed_not_verified` or `diagnostic_only`."
+ )
+ if status != "missing" and not str(ref.get("source_tool") or "").strip():
+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `source_tool`.")
+
+ for index, claim in enumerate(_artifact_rows(artifact.get("claimed_outcomes"))):
+ claim_id = str(claim.get("id") or "").strip()
+ if not _artifact_string_list(claim.get("depends_on")):
+ errors.append(f"Artifact metadata claim `{claim_id or index}` for `{label}` requires `depends_on`.")
+ claim_criteria = _artifact_string_list(claim.get("covered_criteria")) or _artifact_string_list(
+ claim.get("criteria_ids")
+ )
+ if not claim_criteria:
+ errors.append(f"Artifact metadata claim `{claim_id}` for `{label}` requires covered criterion ids.")
+ claim_evidence_refs = _artifact_string_list(claim.get("evidence_refs"))
+ claim_observation_refs = _artifact_string_list(claim.get("observation_refs"))
+ if claim.get("status") == "satisfied" and not claim_evidence_refs:
+ errors.append(
+ f"Artifact metadata claim `{claim_id}` for `{label}` is `satisfied` but has no "
+ "claim-scoped `evidence_refs`."
+ )
+ if claim.get("status") != "missing" and not claim_evidence_refs and not claim_observation_refs:
+ errors.append(
+ f"Artifact metadata claim `{claim_id}` for `{label}` requires claim-scoped "
+ "`evidence_refs` or `observation_refs` unless status is `missing`."
+ )
+
+ for dependency in _artifact_rows(artifact.get("page_dependencies")):
+ dependency_id = str(dependency.get("id") or "").strip()
+ dependency_evidence_refs = _artifact_string_list(dependency.get("evidence_refs"))
+ dependency_observation_refs = _artifact_string_list(dependency.get("observation_refs"))
+ if dependency.get("status") == "satisfied" and not dependency_evidence_refs:
+ errors.append(
+ f"Artifact metadata dependency `{dependency_id}` for `{label}` is `satisfied` but has no "
+ "dependency-scoped `evidence_refs`."
+ )
+ if dependency.get("status") != "missing" and not dependency_evidence_refs and not dependency_observation_refs:
+ errors.append(
+ f"Artifact metadata dependency `{dependency_id}` for `{label}` requires scoped "
+ "`evidence_refs` or `observation_refs` unless status is `missing`."
+ )
+
+ for index, expectation in enumerate(_artifact_rows(artifact.get("terminal_verifier_expectations"))):
+ expectation_id = str(expectation.get("id") or "").strip()
+ if not _artifact_string_list(expectation.get("criteria_ids")) and not _artifact_string_list(
+ expectation.get("claimed_outcome_ids")
+ ):
+ errors.append(
+ f"Artifact metadata terminal verifier expectation `{expectation_id or index}` for `{label}` "
+ "requires `criteria_ids` or `claimed_outcome_ids`."
+ )
+
+ for index, observation in enumerate(_artifact_rows(artifact.get("exploration_observations"))):
+ if observation.get("status") != "observed_not_verified":
+ errors.append(
+ f"Artifact metadata for `{label}` exploration observation {index} must be marked "
+ "`observed_not_verified` until authored execution and terminal verification pass."
+ )
+ if observation.get("checkpoint_next_mode") == "advance":
+ errors.append(
+ f"Artifact metadata for `{label}` exploration observation {index} cannot carry "
+ "`checkpoint_next_mode=advance`; record that as `diagnostic_only` evidence instead."
+ )
+ return errors
+
+
+def _artifact_rows(value: Any) -> list[Mapping[str, Any]]:
+ return [row for row in value if isinstance(row, Mapping)] if isinstance(value, list) else []
+
+
+def _artifact_string_list(value: Any) -> list[str]:
+ if not isinstance(value, list):
+ return []
+ return [str(item).strip() for item in value if str(item).strip()]
+
+
+async def _update_workflow(
+ params: dict[str, Any],
+ ctx: AgentContext,
+ *,
+ allow_missing_credentials: bool | None = None,
+) -> dict[str, Any]:
+ authority_error = _authority_tool_error(ctx, "update_workflow")
+ if authority_error is not None:
+ return {"ok": False, "error": authority_error}
+
+ workflow_yaml = params["workflow_yaml"]
+ # Tool wrappers run authority/loop guards before calling here. The composition
+ # gate below consumes these refs, so they must be visible before validation.
+ ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
+ ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
+ ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
+ code_artifact_metadata, code_artifact_metadata_error = _normalize_code_artifact_metadata(
+ params.get("code_artifact_metadata"),
+ workflow_yaml,
+ )
+ if code_artifact_metadata_error is not None:
+ return {"ok": False, "error": code_artifact_metadata_error}
+ if code_artifact_metadata:
+ ctx.code_artifact_metadata = code_artifact_metadata
+ ctx.workflow_verification_evidence.code_artifact_metadata = code_artifact_metadata
+ params["code_artifact_metadata"] = code_artifact_metadata
+ if allow_missing_credentials is None:
+ allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
+ if not allow_missing_credentials:
+ credential_error = await _credential_reference_validation_error(workflow_yaml, ctx)
+ if credential_error is not None:
+ return {"ok": False, "error": credential_error}
+
+ misbinding_findings = _credential_id_misbinding_findings(workflow_yaml)
+ if misbinding_findings:
+ LOG.info(
+ "copilot credential id misbinding rejected",
+ organization_id=ctx.organization_id,
+ workflow_id=ctx.workflow_id,
+ findings=misbinding_findings,
+ )
+ return {"ok": False, "error": _credential_id_misbinding_error_message(misbinding_findings)}
+
+ output_policy_verdict = evaluate_output_policy(
+ request_policy=getattr(ctx, "request_policy", None),
+ workflow_yaml=workflow_yaml,
+ tool_arguments=params,
+ )
+ if not output_policy_verdict.allowed:
+ LOG.info(
+ "copilot output policy tool body verdict",
+ **output_policy_verdict_to_trace_data(
+ output_policy_verdict,
+ surface="tool_body",
+ tool_name="update_workflow",
+ ),
+ )
+ return {"ok": False, "error": format_output_policy_tool_error(output_policy_verdict)}
+
+ # Prefer the most-recent in-turn emission so cross-path flows (inline
+ # REPLACE_WORKFLOW followed by update_workflow) compare against what the
+ # LLM actually saw, not the turn-start persisted state.
+ last_yaml = getattr(ctx, "last_workflow_yaml", None)
+ prior_yaml = last_yaml if isinstance(last_yaml, str) and last_yaml else ctx.workflow_yaml
+ stale_metadata = _detect_stale_block_metadata(workflow_yaml, prior_yaml)
+ if stale_metadata:
+ return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}
+
+ wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
+ if wait_block_error:
+ return {"ok": False, "error": wait_block_error}
+
+ challenge_http_error = _challenge_http_request_reject_message(ctx, workflow_yaml, ctx.workflow_yaml)
+ if challenge_http_error:
+ return {"ok": False, "error": challenge_http_error}
+
+ # Post-emission reject of copilot-v2 writes that introduce a banned
+ # block type. The schema pre_hook only fires when the LLM consults the
+ # schema; this safety net fires regardless of emission path. Label-based
+ # diff preserves legacy workflows — only NEW banned labels trip the reject.
+ banned_items = _detect_new_banned_blocks(
+ workflow_yaml,
+ ctx.workflow_yaml,
+ banned_types=_copilot_banned_block_types(ctx),
+ )
+ if banned_items:
+ _record_banned_block_reject_span("_update_workflow", banned_items)
+ return {"ok": False, "error": _banned_block_reject_message(banned_items, ctx)}
+
+ composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)
+ if composition_evidence_error:
+ LOG.info(
+ "copilot composition page evidence rejected workflow",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ target_url=workflow_target_url(workflow_yaml),
+ )
+ return {"ok": False, "error": composition_evidence_error}
+
+ try:
+ workflow = _process_workflow_yaml(
+ workflow_id=ctx.workflow_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ organization_id=ctx.organization_id,
+ workflow_yaml=workflow_yaml,
+ )
+ _record_workflow_proxy_location_span(workflow_yaml, workflow)
+
+ # Param / top-level setting changes go through canonical because
+ # prepare_workflow and the runtime parameter-row read consume canonical
+ # values; terminal handlers roll back on non-auto-accept.
+ prior_workflow = await _get_prior_workflow(ctx)
+ requires_canonical_persist = _workflow_requires_canonical_persist(prior_workflow, workflow)
+ if requires_canonical_persist:
+ created_by_stamp = await resolve_copilot_created_by_stamp(ctx.workflow_id, ctx.organization_id)
+ await app.WORKFLOW_SERVICE.update_workflow_definition(
+ workflow_id=ctx.workflow_id,
+ organization_id=ctx.organization_id,
+ title=workflow.title,
+ description=workflow.description,
+ workflow_definition=workflow.workflow_definition,
+ proxy_location=workflow.proxy_location,
+ webhook_callback_url=workflow.webhook_callback_url,
+ totp_verification_url=workflow.totp_verification_url,
+ totp_identifier=workflow.totp_identifier,
+ persist_browser_session=workflow.persist_browser_session,
+ browser_profile_id=workflow.browser_profile_id,
+ model=workflow.model,
+ max_screenshot_scrolling_times=workflow.max_screenshot_scrolls,
+ extra_http_headers=workflow.extra_http_headers,
+ cdp_connect_headers=workflow.cdp_connect_headers,
+ run_with=workflow.run_with,
+ ai_fallback=workflow.ai_fallback,
+ cache_key=workflow.cache_key,
+ adaptive_caching=workflow.adaptive_caching,
+ code_version=workflow.code_version,
+ run_sequentially=workflow.run_sequentially,
+ sequential_key=workflow.sequential_key,
+ created_by=created_by_stamp,
+ edited_by="copilot",
+ )
+ ctx.canonical_was_persisted_due_to_param_change = True
+ ctx.staged_workflow_yaml = workflow_yaml
+ ctx.staged_workflow = workflow
+ ctx.has_staged_proposal = True
+ ctx.workflow_yaml = workflow_yaml
+ # Best-effort — narrative emit failures must never abort an
+ # otherwise-successful update_workflow tool call. ``isinstance``
+ # narrows the parameter's declared ``AgentContext`` to the
+ # envelope-aware ``CopilotContext`` for mypy.
+ if isinstance(ctx, CopilotContext) and ctx.stream is not None:
+ try:
+ await maybe_emit_design_end(ctx.stream, ctx)
+ await emit_workflow_draft(ctx.stream, ctx, workflow)
+ except Exception as emit_err:
+ LOG.warning("copilot_narrative_workflow_draft_emit_failed", error=str(emit_err))
+ return {
+ "ok": True,
+ "data": {
+ "message": "Workflow updated successfully.",
+ "block_count": len(workflow.workflow_definition.blocks) if workflow.workflow_definition else 0,
+ },
+ "_workflow": workflow,
+ }
+ except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:
+ return {
+ "ok": False,
+ "error": f"Workflow validation failed: {e}",
+ }
+
+
+def _record_workflow_proxy_location_span(workflow_yaml: str, workflow: Workflow) -> None:
+ input_present, input_proxy_location = _raw_yaml_proxy_location(workflow_yaml)
+ effective_proxy_location = _proxy_location_trace_value(workflow.proxy_location)
+ with copilot_span(
+ "workflow_proxy_location_normalized",
+ data={
+ "input_proxy_location_present": input_present,
+ "input_proxy_location": input_proxy_location,
+ "effective_proxy_location": effective_proxy_location,
+ },
+ ):
+ pass
+
+
+def _record_workflow_update_result(
+ copilot_ctx: Any, result: dict[str, Any], prior_definition: object | None = None
+) -> None:
+ if not (result.get("ok") and "_workflow" in result):
+ return
+
+ wf = result["_workflow"]
+ copilot_ctx.last_workflow = wf
+ _clear_resolved_per_tool_budget_problem_labels(copilot_ctx, wf)
+ copilot_ctx.last_workflow_yaml = copilot_ctx.workflow_yaml or None
+ copilot_ctx.effective_workflow_proxy_location = getattr(wf, "proxy_location", None) or ProxyLocation.RESIDENTIAL
+ data = result.get("data")
+ if isinstance(data, dict):
+ block_count = data.get("block_count")
+ if isinstance(block_count, int):
+ copilot_ctx.last_update_block_count = block_count
+ copilot_ctx.last_test_ok = None
+ copilot_ctx.last_test_failure_reason = None
+ # A fresh workflow edit invalidates the prior test's failure state —
+ # otherwise an exhausted POST_UPDATE_NUDGE on the new draft would raise
+ # CopilotNonRetriableNavError with the old run's error, telling the user
+ # to "verify the URL" for a URL they just corrected in the new draft.
+ copilot_ctx.last_test_non_retriable_nav_error = None
+ copilot_ctx.non_retriable_nav_error_last_emitted_signature = None
+
+ # Block-running failures keyed off (labels, parameters) go stale once the
+ # workflow itself changes — without this clear, a user who fixes the bug
+ # via update_workflow gets a LOOP DETECTED on the next legitimate run.
+ clear_failed_step_tracker_for_tools_in_ctx(copilot_ctx, BLOCK_RUNNING_TOOLS)
+
+ _invalidate_verified_state_on_edit(copilot_ctx, prior_definition, getattr(wf, "workflow_definition", None))
+
+
+def _last_update_is_single_goto_bootstrap(copilot_ctx: CopilotContext) -> bool:
+ last_workflow = copilot_ctx.last_workflow
+ definition = _workflow_definition_as_dict(last_workflow.workflow_definition if last_workflow is not None else None)
+ blocks = definition.get("blocks")
+ if not isinstance(blocks, list) or len(blocks) != 1:
+ return False
+ block = blocks[0]
+ if not isinstance(block, dict):
+ return False
+ return str(block.get("block_type") or "").strip().lower() == "goto_url"
+
+
+def _pre_run_workflow_coverage_error(copilot_ctx: Any) -> str | None:
+ block_count = getattr(copilot_ctx, "last_update_block_count", None)
+ if not isinstance(block_count, int):
+ return None
+ if block_count == 1 and _last_update_is_single_goto_bootstrap(copilot_ctx):
+ return None
+
+ user_message = getattr(copilot_ctx, "user_message", "")
+ request_policy = getattr(copilot_ctx, "request_policy", None)
+ completion_contract = getattr(request_policy, "completion_contract", None)
+ if isinstance(completion_contract, str):
+ completion_contract = completion_contract.strip() or None
+ else:
+ completion_contract = None
+
+ if not _goal_likely_needs_more_blocks(user_message, block_count, completion_contract):
+ return None
+
+ nudge_count = getattr(copilot_ctx, "coverage_nudge_count", 0)
+ if nudge_count >= 1:
+ return None
+ copilot_ctx.coverage_nudge_count = nudge_count + 1
+ return (
+ f"{POST_INTERMEDIATE_SUCCESS_NUDGE} The workflow was saved with {block_count} block"
+ f"{'' if block_count == 1 else 's'}, but it has not been run because the request-policy "
+ "completion contract still leaves distinct requested actions uncovered."
+ )
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.py
index 33bea4a8c..9b4d04d8b 100644
--- a/skyvern/forge/sdk/db/agent_db.py
+++ b/skyvern/forge/sdk/db/agent_db.py
@@ -40,7 +40,7 @@ from skyvern.forge.sdk.trace import traced
LOG = structlog.get_logger()
-def _build_engine(database_string: str) -> AsyncEngine:
+def _build_engine(database_string: str, disable_pool: bool | None = None) -> AsyncEngine:
"""
Build a SQLAlchemy async engine.
@@ -53,6 +53,9 @@ def _build_engine(database_string: str) -> AsyncEngine:
and do not set statement_timeout - set at role level in the database,
since the transaction pooler does not maintain session-level settings.
+ Pass disable_pool to override the global DISABLE_CONNECTION_POOL setting for a
+ single engine.
+
SQLite behaviour:
For :memory: databases, uses StaticPool to keep the single connection alive.
For file-backed databases, enables WAL mode for concurrent read support.
@@ -81,8 +84,9 @@ def _build_engine(database_string: str) -> AsyncEngine:
return engine
+ use_null_pool = settings.DISABLE_CONNECTION_POOL if disable_pool is None else disable_pool
connect_args: dict[str, Any] = {}
- if settings.DISABLE_CONNECTION_POOL:
+ if use_null_pool:
if "postgresql+psycopg" in database_string:
connect_args["options"] = f"-c statement_timeout={settings.DATABASE_STATEMENT_TIMEOUT_MS}"
if "postgresql+asyncpg" in database_string:
@@ -113,8 +117,16 @@ __all__ = ["AgentDB", "ScheduleLimitExceededError"]
class AgentDB(BaseAlchemyDB):
- def __init__(self, database_string: str, debug_enabled: bool = False, db_engine: AsyncEngine | None = None) -> None:
- super().__init__(db_engine or _build_engine(database_string))
+ def __init__(
+ self,
+ database_string: str,
+ debug_enabled: bool = False,
+ db_engine: AsyncEngine | None = None,
+ disable_pool: bool | None = None,
+ ) -> None:
+ if db_engine is not None and disable_pool is not None:
+ raise ValueError("Pass either db_engine or disable_pool, not both")
+ super().__init__(db_engine or _build_engine(database_string, disable_pool=disable_pool))
self.debug_enabled = debug_enabled
# Global lock for SQLite schedule serialization. Unlike Postgres advisory locks
# (which are scoped per org:workflow via hashtext(key)), this serializes ALL
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index e6b6270c2..493223ccb 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -1143,6 +1143,7 @@ class PersistentBrowserSessionModel(Base):
extensions = Column(JSON, nullable=True)
browser_type = Column(String, nullable=True)
browser_profile_id = Column(String, nullable=True, index=True)
+ generate_browser_profile = Column(Boolean, default=False, nullable=False, server_default=sqlalchemy.false())
instance_type = Column(String, nullable=True)
vcpu_millicores = Column(Integer, nullable=True)
memory_mb = Column(Integer, nullable=True)
diff --git a/skyvern/forge/sdk/db/repositories/browser_sessions.py b/skyvern/forge/sdk/db/repositories/browser_sessions.py
index dd5dd2c30..4abbbda98 100644
--- a/skyvern/forge/sdk/db/repositories/browser_sessions.py
+++ b/skyvern/forge/sdk/db/repositories/browser_sessions.py
@@ -267,6 +267,7 @@ class BrowserSessionsRepository(BaseRepository):
extensions: list[Extensions] | None = None,
browser_type: PersistentBrowserType | None = None,
browser_profile_id: str | None = None,
+ generate_browser_profile: bool = False,
) -> PersistentBrowserSession:
"""Create a new persistent browser session."""
extensions_str: list[str] | None = (
@@ -282,6 +283,7 @@ class BrowserSessionsRepository(BaseRepository):
extensions=extensions_str,
browser_type=browser_type.value if browser_type else None,
browser_profile_id=browser_profile_id,
+ generate_browser_profile=generate_browser_profile,
)
session.add(browser_session)
await session.commit()
@@ -298,6 +300,7 @@ class BrowserSessionsRepository(BaseRepository):
organization_id: str | None = None,
completed_at: datetime | None = None,
started_at: datetime | None = None,
+ generate_browser_profile: bool | None = None,
) -> PersistentBrowserSession:
async with self.Session() as session:
persistent_browser_session = (
@@ -319,6 +322,8 @@ class BrowserSessionsRepository(BaseRepository):
persistent_browser_session.completed_at = to_naive_utc(completed_at)
if started_at:
persistent_browser_session.started_at = to_naive_utc(started_at)
+ if generate_browser_profile is not None:
+ persistent_browser_session.generate_browser_profile = generate_browser_profile
await session.commit()
await session.refresh(persistent_browser_session)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 6630f2a24..fe2cc0d89 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -307,6 +307,35 @@ class WorkflowRunsRepository(BaseRepository):
else:
raise WorkflowRunNotFound(workflow_run_id)
+ @db_operation("claim_workflow_run_webhook_delivery")
+ async def claim_workflow_run_webhook_delivery(
+ self,
+ *,
+ workflow_run_id: str,
+ in_progress_reason: str,
+ claim_older_than: datetime | None = None,
+ ) -> bool:
+ delivery_state_filter = WorkflowRunModel.webhook_failure_reason.is_(None)
+ if claim_older_than is not None:
+ stale_claim_cutoff = to_naive_utc(claim_older_than)
+ delivery_state_filter = or_(
+ delivery_state_filter,
+ and_(
+ WorkflowRunModel.webhook_failure_reason == in_progress_reason,
+ WorkflowRunModel.modified_at <= stale_claim_cutoff,
+ ),
+ )
+
+ async with self.Session() as session:
+ result = await session.execute(
+ update(WorkflowRunModel)
+ .where(WorkflowRunModel.workflow_run_id == workflow_run_id)
+ .where(delivery_state_filter)
+ .values(webhook_failure_reason=in_progress_reason, modified_at=naive_utc_now())
+ )
+ await session.commit()
+ return result.rowcount == 1
+
@db_operation("increment_workflow_run_credits")
async def increment_workflow_run_credits(
self,
@@ -758,6 +787,48 @@ class WorkflowRunsRepository(BaseRepository):
workflow_runs = (await session.scalars(query)).all()
return [convert_to_workflow_run(workflow_run) for workflow_run in workflow_runs]
+ @db_operation("get_workflow_runs_with_pending_webhook_delivery")
+ async def get_workflow_runs_with_pending_webhook_delivery(
+ self,
+ *,
+ older_than: datetime,
+ limit: int = 100,
+ in_progress_reason: str | None = None,
+ ) -> list[WorkflowRun]:
+ terminal_statuses = [status.value for status in WorkflowRunStatus if status.is_final()]
+ cutoff = to_naive_utc(older_than)
+ run_age_filter = or_(
+ WorkflowRunModel.finished_at <= cutoff,
+ and_(
+ WorkflowRunModel.finished_at.is_(None),
+ WorkflowRunModel.modified_at <= cutoff,
+ ),
+ )
+ delivery_state_filter = WorkflowRunModel.webhook_failure_reason.is_(None)
+ if in_progress_reason is not None:
+ delivery_state_filter = or_(
+ delivery_state_filter,
+ and_(
+ WorkflowRunModel.webhook_failure_reason == in_progress_reason,
+ WorkflowRunModel.modified_at <= cutoff,
+ ),
+ )
+
+ async with self.Session() as session:
+ query = (
+ select(WorkflowRunModel)
+ .filter(WorkflowRunModel.status.in_(terminal_statuses))
+ .filter(WorkflowRunModel.webhook_callback_url.isnot(None))
+ # TODO: add a partial index for the finalized pending-webhook sweep if production scan cost grows.
+ .filter(func.length(func.trim(WorkflowRunModel.webhook_callback_url)) > 0)
+ .filter(delivery_state_filter)
+ .filter(run_age_filter)
+ .order_by(func.coalesce(WorkflowRunModel.finished_at, WorkflowRunModel.modified_at).asc())
+ .limit(limit)
+ )
+ workflow_runs = (await session.scalars(query)).all()
+ return [convert_to_workflow_run(workflow_run) for workflow_run in workflow_runs]
+
@staticmethod
def _apply_search_key_filter(query, search_key: str | None): # type: ignore[no-untyped-def]
if not search_key:
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index b93c551be..95074b0e0 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -695,6 +695,7 @@ async def create_workflow_legacy(
request: Request,
folder_id: str | None = Query(None, description="Optional folder ID to assign the workflow to"),
current_org: Organization = Depends(org_auth_service.get_current_org),
+ user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),
) -> Workflow:
analytics.capture("skyvern-oss-agent-workflow-create-legacy")
raw_yaml = await request.body()
@@ -712,7 +713,10 @@ async def create_workflow_legacy(
if folder_id is not None:
workflow_create_request.folder_id = folder_id
return await app.WORKFLOW_SERVICE.create_workflow_from_request(
- organization=current_org, request=workflow_create_request
+ organization=current_org,
+ request=workflow_create_request,
+ created_by=user_id,
+ edited_by=user_id,
)
except WorkflowDefinitionValidationException as e:
raise e
@@ -754,6 +758,7 @@ async def create_workflow(
data: WorkflowRequest,
folder_id: str | None = Query(None, description="Optional folder ID to assign the workflow to"),
current_org: Organization = Depends(org_auth_service.get_current_org),
+ user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),
) -> Workflow:
analytics.capture("skyvern-oss-agent-workflow-create")
try:
@@ -775,6 +780,8 @@ async def create_workflow(
return await app.WORKFLOW_SERVICE.create_workflow_from_request(
organization=current_org,
request=workflow_definition,
+ created_by=user_id,
+ edited_by=user_id,
)
except yaml.YAMLError as exc:
raise HTTPException(status_code=422, detail=format_yaml_error(exc))
@@ -1100,6 +1107,7 @@ async def update_workflow_legacy(
..., description="The ID of the workflow to update. Workflow ID starts with `wpid_`.", examples=["wpid_123"]
),
current_org: Organization = Depends(org_auth_service.get_current_org),
+ user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),
) -> Workflow:
analytics.capture("skyvern-oss-agent-workflow-update")
# validate the workflow
@@ -1115,6 +1123,8 @@ async def update_workflow_legacy(
organization=current_org,
request=workflow_create_request,
workflow_permanent_id=workflow_id,
+ created_by=user_id,
+ edited_by=user_id,
)
except WorkflowDefinitionValidationException as e:
raise e
@@ -1173,6 +1183,7 @@ async def update_workflow(
..., description="The ID of the agent to update. Starts with `wpid_`.", examples=["wpid_123"]
),
current_org: Organization = Depends(org_auth_service.get_current_org),
+ user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),
) -> Workflow:
analytics.capture("skyvern-oss-agent-workflow-update")
try:
@@ -1192,6 +1203,8 @@ async def update_workflow(
organization=current_org,
request=workflow_definition,
workflow_permanent_id=workflow_id,
+ created_by=user_id,
+ edited_by=user_id,
)
except yaml.YAMLError as exc:
raise HTTPException(status_code=422, detail=format_yaml_error(exc))
diff --git a/skyvern/forge/sdk/routes/browser_profiles.py b/skyvern/forge/sdk/routes/browser_profiles.py
index b3ec61381..4004b3ca7 100644
--- a/skyvern/forge/sdk/routes/browser_profiles.py
+++ b/skyvern/forge/sdk/routes/browser_profiles.py
@@ -401,6 +401,21 @@ async def _create_profile_from_session(
profile_id=browser_session_id,
)
if not session_dir:
+ # An opted-out session never uploads an archive, so retrying can't help — fail fast with a
+ # distinct message the client can tell apart from the transient "upload not finished yet" case.
+ if not browser_session.should_export_profile():
+ LOG.info(
+ "Browser session was not configured to generate a browser profile",
+ organization_id=organization_id,
+ browser_session_id=browser_session_id,
+ )
+ raise HTTPException(
+ status_code=status.HTTP_400_BAD_REQUEST,
+ detail=(
+ "This browser session was not configured to generate a browser profile. "
+ "Start a session with generate_browser_profile enabled to capture a profile."
+ ),
+ )
LOG.warning(
"Browser session archive not found for profile creation",
organization_id=organization_id,
diff --git a/skyvern/forge/sdk/routes/browser_sessions.py b/skyvern/forge/sdk/routes/browser_sessions.py
index 5b0a0ff2c..2ebf14825 100644
--- a/skyvern/forge/sdk/routes/browser_sessions.py
+++ b/skyvern/forge/sdk/routes/browser_sessions.py
@@ -17,12 +17,14 @@ from skyvern.forge.sdk.routes.code_samples import (
)
from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
from skyvern.forge.sdk.services import org_auth_service
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun
from skyvern.schemas.browser_sessions import (
CreateBrowserSessionRequest,
ProcessBrowserSessionRecordingRequest,
ProcessBrowserSessionRecordingResponse,
+ UpdateBrowserSessionRequest,
)
from skyvern.webeye.schemas import BrowserSessionResponse
@@ -109,6 +111,7 @@ async def create_browser_session(
extensions=browser_session_request.extensions,
browser_type=browser_session_request.browser_type,
browser_profile_id=browser_session_request.browser_profile_id,
+ generate_browser_profile=browser_session_request.generate_browser_profile,
)
return await BrowserSessionResponse.from_browser_session(browser_session)
@@ -154,6 +157,54 @@ async def close_browser_session(
)
+@base_router.patch(
+ "/browser_sessions/{browser_session_id}",
+ response_model=BrowserSessionResponse,
+ tags=["Browser Sessions"],
+ openapi_extra={
+ "x-fern-sdk-method-name": "update_browser_session",
+ },
+ description=(
+ "Update a live browser session. Currently supports toggling generate_browser_profile, which is read "
+ "when the session ends to decide whether to save its browser profile."
+ ),
+ summary="Update a session",
+ responses={
+ 200: {"description": "Successfully updated browser session"},
+ 404: {"description": "Browser session not found"},
+ 403: {"description": "Unauthorized - Invalid or missing authentication"},
+ },
+)
+@base_router.patch(
+ "/browser_sessions/{browser_session_id}/",
+ response_model=BrowserSessionResponse,
+ include_in_schema=False,
+)
+async def update_browser_session(
+ request: UpdateBrowserSessionRequest,
+ browser_session_id: str = Path(
+ ..., description="The ID of the browser session. browser_session_id starts with `pbs_`", examples=["pbs_123456"]
+ ),
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> BrowserSessionResponse:
+ existing = await app.PERSISTENT_SESSIONS_MANAGER.get_session(browser_session_id, current_org.organization_id)
+ if not existing:
+ raise HTTPException(status_code=404, detail=f"Browser session {browser_session_id} not found")
+ # The flag is read at teardown, so toggling it after the session has ended has no effect and
+ # would make a later profile-creation attempt treat the missing archive as a transient upload.
+ if is_final_status(existing.status):
+ raise HTTPException(
+ status_code=409,
+ detail=f"Browser session {browser_session_id} has already ended and can no longer be updated.",
+ )
+ updated = await app.DATABASE.browser_sessions.update_persistent_browser_session(
+ browser_session_id,
+ organization_id=current_org.organization_id,
+ generate_browser_profile=request.generate_browser_profile,
+ )
+ return await BrowserSessionResponse.from_browser_session(updated, app.STORAGE)
+
+
@base_router.get(
"/browser_sessions/{browser_session_id}",
response_model=BrowserSessionResponse,
diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.py
index 2aa3a37c5..6de0dcdf0 100644
--- a/skyvern/forge/sdk/routes/credentials.py
+++ b/skyvern/forge/sdk/routes/credentials.py
@@ -83,6 +83,7 @@ from skyvern.forge.sdk.schemas.organizations import (
BitwardenCredentialSafe,
BitwardenOrganizationAuthToken,
BitwardenOrganizationAuthTokenSafe,
+ ClearOrganizationAuthTokenResponse,
CreateAzureClientSecretCredentialRequest,
CreateBitwardenCredentialRequest,
CreateCustomCredentialServiceConfigRequest,
@@ -128,6 +129,13 @@ _background_tasks: set[asyncio.Task] = set()
# profile task waits; the status grace period derives from these to stay aligned.
_SESSION_PERSIST_MAX_RETRIES = 20
_SESSION_PERSIST_RETRY_INTERVAL_SECONDS = 3
+
+_ORG_AUTH_CREDENTIAL_TOKEN_TYPES = {
+ "onepassword": OrganizationAuthTokenType.onepassword_service_account,
+ "bitwarden": OrganizationAuthTokenType.bitwarden_credential,
+ "azure_credential": OrganizationAuthTokenType.azure_client_secret_credential,
+ "custom_credential": OrganizationAuthTokenType.custom_credential_service,
+}
# -1 because no sleep follows the final attempt.
_SESSION_PERSIST_MAX_WAIT_SECONDS = (_SESSION_PERSIST_MAX_RETRIES - 1) * _SESSION_PERSIST_RETRY_INTERVAL_SECONDS
# Buffer over the max wait so the status endpoint doesn't misreport while the task still retries.
@@ -1886,6 +1894,47 @@ async def update_onepassword_token(
)
+@base_router.delete(
+ "/credentials/{credential_provider}",
+ response_model=ClearOrganizationAuthTokenResponse,
+ summary="Clear organization auth credential",
+ description="Clears the current organization auth credential for the current organization.",
+ include_in_schema=False,
+)
+@base_router.delete(
+ "/credentials/{credential_provider}/",
+ response_model=ClearOrganizationAuthTokenResponse,
+ include_in_schema=False,
+)
+async def clear_org_auth_credential(
+ credential_provider: str = Path(..., description="The organization auth credential provider to clear."),
+ current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> ClearOrganizationAuthTokenResponse:
+ """
+ Clear the current organization auth credential for the organization.
+
+ This endpoint is idempotent; it succeeds even when no valid token exists.
+ """
+ token_type = _ORG_AUTH_CREDENTIAL_TOKEN_TYPES.get(credential_provider)
+ if not token_type:
+ raise HTTPException(status_code=404, detail="Unsupported organization auth credential provider")
+ try:
+ await app.DATABASE.organizations.invalidate_org_auth_tokens(
+ organization_id=current_org.organization_id,
+ token_type=token_type,
+ )
+ return ClearOrganizationAuthTokenResponse(success=True)
+ except Exception as e:
+ LOG.error(
+ "Failed to clear organization auth token",
+ organization_id=current_org.organization_id,
+ token_type=token_type.value,
+ error=str(e),
+ exc_info=True,
+ )
+ raise HTTPException(status_code=500, detail="Failed to clear organization auth credential") from e
+
+
def _to_safe_bitwarden_response(auth_token: BitwardenOrganizationAuthToken) -> BitwardenCredentialResponse:
"""Strip master_password from the response for security."""
safe_token = BitwardenOrganizationAuthTokenSafe(
diff --git a/skyvern/forge/sdk/schemas/credentials.py b/skyvern/forge/sdk/schemas/credentials.py
index 2abea0d4a..5fe1b3d26 100644
--- a/skyvern/forge/sdk/schemas/credentials.py
+++ b/skyvern/forge/sdk/schemas/credentials.py
@@ -12,6 +12,7 @@ from skyvern.utils.url_validators import validate_url
class CredentialVaultType(StrEnum):
BITWARDEN = "bitwarden"
AZURE_VAULT = "azure_vault"
+ GCP = "gcp"
CUSTOM = "custom"
diff --git a/skyvern/forge/sdk/schemas/organizations.py b/skyvern/forge/sdk/schemas/organizations.py
index 31cc86b59..dd371678f 100644
--- a/skyvern/forge/sdk/schemas/organizations.py
+++ b/skyvern/forge/sdk/schemas/organizations.py
@@ -117,6 +117,12 @@ class CreateOnePasswordTokenResponse(BaseModel):
)
+class ClearOrganizationAuthTokenResponse(BaseModel):
+ """Response model for clearing an organization auth token."""
+
+ success: bool = Field(..., description="Whether the token was cleared successfully")
+
+
class AzureClientSecretCredentialResponse(BaseModel):
"""Response model for Azure ClientSecretCredential operations."""
diff --git a/skyvern/forge/sdk/schemas/persistent_browser_sessions.py b/skyvern/forge/sdk/schemas/persistent_browser_sessions.py
index 4ae1c8cd3..a2448d66b 100644
--- a/skyvern/forge/sdk/schemas/persistent_browser_sessions.py
+++ b/skyvern/forge/sdk/schemas/persistent_browser_sessions.py
@@ -71,6 +71,7 @@ class PersistentBrowserSession(BaseModel):
extensions: list[Extensions] | None = None
browser_type: PersistentBrowserType | None = None
browser_profile_id: str | None = None
+ generate_browser_profile: bool = False
@field_validator("proxy_location", mode="before")
@classmethod
@@ -79,6 +80,14 @@ class PersistentBrowserSession(BaseModel):
return deserialize_proxy_location(value, raise_on_invalid_geo_target=True)
return value
+ def should_export_profile(self) -> bool:
+ """A session persists its profile at teardown only when it opted in or is reusing a saved profile.
+
+ A reuse session (browser_profile_id set) must always re-export so the updated session-cookie
+ sidecar survives; gating it off would silently log the profile out on the next reuse.
+ """
+ return bool(self.generate_browser_profile or self.browser_profile_id)
+
class AddressablePersistentBrowserSession(PersistentBrowserSession):
browser_address: str
diff --git a/skyvern/forge/sdk/services/credential/gcp_credential_vault_service.py b/skyvern/forge/sdk/services/credential/gcp_credential_vault_service.py
new file mode 100644
index 000000000..708d8d048
--- /dev/null
+++ b/skyvern/forge/sdk/services/credential/gcp_credential_vault_service.py
@@ -0,0 +1,217 @@
+import re
+import uuid
+from typing import Annotated, Any, ClassVar, Literal, Union
+
+import structlog
+from pydantic import BaseModel, Field, TypeAdapter
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.api.gcp import AsyncGcpSecretManagerClient
+from skyvern.forge.sdk.schemas.credentials import (
+ CreateCredentialRequest,
+ Credential,
+ CredentialItem,
+ CredentialType,
+ CredentialVaultType,
+ CreditCardCredential,
+ PasswordCredential,
+ SecretCredential,
+)
+from skyvern.forge.sdk.services.credential.credential_vault_service import CredentialVaultService
+
+LOG = structlog.get_logger()
+
+
+class GcpCredentialVaultService(CredentialVaultService):
+ class _PasswordCredentialDataImage(BaseModel):
+ type: Literal["password"]
+ password: str
+ username: str
+ totp: str | None = None
+
+ class _CreditCardCredentialDataImage(BaseModel):
+ type: Literal["credit_card"]
+ card_number: str
+ card_cvv: str
+ card_exp_month: str
+ card_exp_year: str
+ card_brand: str
+ card_holder_name: str
+
+ class _SecretCredentialDataImage(BaseModel):
+ type: Literal["secret"]
+ secret_value: str
+ secret_label: str | None = None
+
+ _CredentialDataImage = Annotated[
+ Union[_PasswordCredentialDataImage, _CreditCardCredentialDataImage, _SecretCredentialDataImage],
+ Field(discriminator="type"),
+ ]
+
+ # Built once: TypeAdapter rebuilds the discriminated-union validator on every
+ # construction, so hoisting it off the per-read path matters on the credential
+ # hot path.
+ _CREDENTIAL_DATA_ADAPTER: ClassVar[TypeAdapter[Any]] = TypeAdapter(_CredentialDataImage)
+
+ # GCP Secret Manager secret IDs accept only [A-Za-z0-9_-] (max 255 chars).
+ _SECRET_ID_PATTERN: ClassVar[re.Pattern[str]] = re.compile(r"^[A-Za-z0-9_-]{1,255}$")
+
+ def __init__(self, client: AsyncGcpSecretManagerClient, project_id: str):
+ self._client = client
+ self._project_id = project_id
+
+ async def create_credential(self, organization_id: str, data: CreateCredentialRequest) -> Credential:
+ item_id = await self._create_gcp_secret_item(
+ organization_id=organization_id,
+ credential=data.credential,
+ )
+
+ credential = await self._create_db_credential(
+ organization_id=organization_id,
+ data=data,
+ item_id=item_id,
+ vault_type=CredentialVaultType.GCP,
+ )
+
+ return credential
+
+ async def update_credential(self, credential: Credential, data: CreateCredentialRequest) -> Credential:
+ # Updating a Secret Manager secret adds a new version under the same
+ # secret id, so we reuse item_id. NOTE: if the DB update below fails the
+ # vault holds the new value while DB metadata (name, type, username) is
+ # stale; the credential data itself is correct and a retry reconciles
+ # the metadata.
+ await self._update_gcp_secret_item(
+ item_id=credential.item_id,
+ credential=data.credential,
+ )
+
+ try:
+ updated_credential = await self._update_db_credential(
+ credential=credential,
+ data=data,
+ item_id=credential.item_id,
+ )
+ except Exception:
+ LOG.error(
+ "DB update failed after GCP Secret Manager secret was already updated. "
+ "Vault data is updated but DB metadata may be stale.",
+ organization_id=credential.organization_id,
+ credential_id=credential.credential_id,
+ item_id=credential.item_id,
+ )
+ raise
+
+ return updated_credential
+
+ async def delete_credential(self, credential: Credential) -> None:
+ # Deliberate deviation from Azure (which empties the value, then deletes
+ # the secret in a background task): GCP can't cheaply "empty" a secret,
+ # so we delete it synchronously to guarantee the value is gone the moment
+ # DELETE returns. Delete the secret BEFORE the DB row so a transient
+ # Secret Manager error leaves a consistent, retryable state instead of an
+ # orphaned secret. (The route still schedules the base no-op
+ # post_delete_credential_item afterwards, which is harmless here.)
+ await self._client.delete_secret(secret_id=credential.item_id, project_id=self._project_id)
+ await app.DATABASE.credentials.delete_credential(credential.credential_id, credential.organization_id)
+
+ async def get_credential_item(self, db_credential: Credential) -> CredentialItem:
+ secret_json_str = await self._client.get_secret(secret_id=db_credential.item_id, project_id=self._project_id)
+ if secret_json_str is None:
+ raise ValueError(f"GCP Credential Vault secret not found for {db_credential.item_id}")
+
+ data = GcpCredentialVaultService._CREDENTIAL_DATA_ADAPTER.validate_json(secret_json_str)
+ if isinstance(data, GcpCredentialVaultService._PasswordCredentialDataImage):
+ return CredentialItem(
+ item_id=db_credential.item_id,
+ credential=PasswordCredential(
+ username=data.username,
+ password=data.password,
+ totp=data.totp,
+ totp_type=db_credential.totp_type,
+ ),
+ name=db_credential.name,
+ credential_type=CredentialType.PASSWORD,
+ )
+ elif isinstance(data, GcpCredentialVaultService._CreditCardCredentialDataImage):
+ return CredentialItem(
+ item_id=db_credential.item_id,
+ credential=CreditCardCredential(
+ card_holder_name=data.card_holder_name,
+ card_number=data.card_number,
+ card_exp_month=data.card_exp_month,
+ card_exp_year=data.card_exp_year,
+ card_cvv=data.card_cvv,
+ card_brand=data.card_brand,
+ ),
+ name=db_credential.name,
+ credential_type=CredentialType.CREDIT_CARD,
+ )
+ elif isinstance(data, GcpCredentialVaultService._SecretCredentialDataImage):
+ return CredentialItem(
+ item_id=db_credential.item_id,
+ credential=SecretCredential(secret_value=data.secret_value, secret_label=data.secret_label),
+ name=db_credential.name,
+ credential_type=CredentialType.SECRET,
+ )
+ else:
+ raise TypeError(f"Invalid credential type: {type(data)}")
+
+ def _build_data_image(self, credential: PasswordCredential | CreditCardCredential | SecretCredential) -> BaseModel:
+ if isinstance(credential, PasswordCredential):
+ return GcpCredentialVaultService._PasswordCredentialDataImage(
+ type="password",
+ username=credential.username,
+ password=credential.password,
+ totp=credential.totp,
+ )
+ elif isinstance(credential, CreditCardCredential):
+ return GcpCredentialVaultService._CreditCardCredentialDataImage(
+ type="credit_card",
+ card_number=credential.card_number,
+ card_cvv=credential.card_cvv,
+ card_exp_month=credential.card_exp_month,
+ card_exp_year=credential.card_exp_year,
+ card_brand=credential.card_brand,
+ card_holder_name=credential.card_holder_name,
+ )
+ elif isinstance(credential, SecretCredential):
+ return GcpCredentialVaultService._SecretCredentialDataImage(
+ type="secret",
+ secret_value=credential.secret_value,
+ secret_label=credential.secret_label,
+ )
+ else:
+ raise TypeError(f"Invalid credential type: {type(credential)}")
+
+ async def _create_gcp_secret_item(
+ self,
+ organization_id: str,
+ credential: PasswordCredential | CreditCardCredential | SecretCredential,
+ ) -> str:
+ data = self._build_data_image(credential)
+ secret_id = f"{settings.GCP_CREDENTIAL_VAULT_PREFIX}{organization_id}-{uuid.uuid4()}"
+ if not self._SECRET_ID_PATTERN.match(secret_id):
+ raise ValueError(f"Constructed GCP secret id is not valid for Secret Manager: {secret_id!r}")
+ secret_value = data.model_dump_json(exclude_none=True)
+
+ return await self._client.create_or_update_secret(
+ secret_id=secret_id,
+ project_id=self._project_id,
+ value=secret_value,
+ )
+
+ async def _update_gcp_secret_item(
+ self,
+ item_id: str,
+ credential: PasswordCredential | CreditCardCredential | SecretCredential,
+ ) -> None:
+ data = self._build_data_image(credential)
+ secret_value = data.model_dump_json(exclude_none=True)
+
+ await self._client.create_or_update_secret(
+ secret_id=item_id,
+ project_id=self._project_id,
+ value=secret_value,
+ )
diff --git a/skyvern/forge/sdk/services/org_auth_service.py b/skyvern/forge/sdk/services/org_auth_service.py
index 14ff721b7..03b49a8d1 100644
--- a/skyvern/forge/sdk/services/org_auth_service.py
+++ b/skyvern/forge/sdk/services/org_auth_service.py
@@ -274,6 +274,41 @@ async def get_current_user_id(
)
+async def get_current_user_id_or_none(
+ authorization: Annotated[str | None, Header(include_in_schema=False)] = None,
+ x_api_key: Annotated[str | None, Header(include_in_schema=False)] = None,
+ x_user_agent: Annotated[str | None, Header(include_in_schema=False)] = None,
+) -> str | None:
+ """Best-effort caller resolution for write attribution; never rejects the request."""
+ try:
+ user_id = await get_current_user_id(
+ authorization=authorization,
+ x_api_key=x_api_key,
+ x_user_agent=x_user_agent,
+ )
+ # Org auth prefers x-api-key while the user comes from the bearer; only stamp verified members of the key org.
+ if user_id and authorization and x_api_key and app.authenticate_user_function:
+ key_org = await get_current_org_cached(x_api_key, app.DATABASE)
+ bearer_parts = authorization.split(" ", 1)
+ is_member = await app.AGENT_FUNCTION.validate_user_organization_membership(
+ user_id=user_id,
+ organization_id=key_org.organization_id,
+ bearer_token=bearer_parts[1] if len(bearer_parts) == 2 else None,
+ )
+ if not is_member:
+ LOG.warning(
+ "Skipping write attribution: bearer user membership in the api key org is not verified",
+ organization_id=key_org.organization_id,
+ )
+ return None
+ return user_id
+ except HTTPException:
+ return None
+ except Exception:
+ LOG.warning("Failed to resolve user for write attribution", exc_info=True)
+ return None
+
+
async def get_current_user_id_with_authentication(
authorization: Annotated[str | None, Header()] = None,
) -> str:
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 420db8714..4e5f92d35 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -179,24 +179,19 @@ def warn_if_file_download_max_steps_low(
)
-BLOCK_BASELINE_MARKER = "_set_by_block"
-
-
async def capture_block_download_baseline(
context: SkyvernContext,
organization_id: str,
workflow_run_id: str,
block_label: str,
) -> None:
- """Snapshot downloaded files before a task block runs.
+ """Snapshot the files already downloaded before this block runs.
- Sets ``loop_internal_state`` so ``filter_downloaded_files_for_current_iteration``
- scopes the block's output to only files it downloaded. Skips capture when the
- baseline was already set by a ForLoopBlock (no marker).
+ Recorded in ``loop_internal_state`` so ``filter_downloaded_files_for_current_iteration``
+ scopes the block's output to only the files it produced. Captured fresh for every
+ block — including each block inside a loop iteration — so sibling download-producing
+ blocks don't inherit one another's files. Best-effort: cleared on timeout/error.
"""
- existing = context.loop_internal_state
- if existing and BLOCK_BASELINE_MARKER not in existing:
- return
try:
async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
baseline_files = await app.STORAGE.get_downloaded_files(
@@ -205,7 +200,6 @@ async def capture_block_download_baseline(
)
context.loop_internal_state = {
DOWNLOADED_FILE_SIGS_KEY: [to_downloaded_file_signature(fi) for fi in baseline_files],
- BLOCK_BASELINE_MARKER: True,
}
LOG.debug(
"Captured block download baseline",
@@ -2246,7 +2240,9 @@ class ForLoopBlock(Block):
)
LOG.info("Starting loop iteration", loop_idx=loop_idx, loop_over_value=loop_over_value_repr)
- # Capture baseline downloaded files for per-iteration scoping (SKY-7005)
+ # Capture baseline downloaded files for per-iteration scoping (SKY-7005).
+ # Download-producing child blocks re-capture their own per-block baseline
+ # at start; this seed only covers filtering before the first such capture.
loop_context = skyvern_context.current()
if loop_context:
downloaded_file_sigs_before: list[tuple[str | None, str | None, str | None]] = []
@@ -2971,7 +2967,9 @@ class WhileLoopBlock(Block):
last_block=current_block,
)
- # Capture baseline downloaded files for per-iteration scoping (SKY-7005)
+ # Capture baseline downloaded files for per-iteration scoping (SKY-7005).
+ # Download-producing child blocks re-capture their own per-block baseline
+ # at start; this seed only covers filtering before the first such capture.
loop_context = skyvern_context.current()
if loop_context:
downloaded_file_sigs_before: list[tuple[str | None, str | None, str | None]] = []
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index a50abbc91..9be4f69ae 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -1,6 +1,6 @@
from datetime import datetime
from enum import StrEnum
-from typing import Any, List
+from typing import Any, List, Literal
from pydantic import BaseModel, Field, computed_field, field_serializer, field_validator
from typing_extensions import deprecated
@@ -342,6 +342,7 @@ class WorkflowRunResponseBase(BaseModel):
failure_category: list[dict[str, Any]] | None = None
proxy_location: ProxyLocationInput = None
webhook_callback_url: str | None = None
+ webhook_delivery_status: Literal["pending", "failed"] | None = None
webhook_failure_reason: str | None = None
totp_verification_url: str | None = None
totp_identifier: str | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index d9c8a70c1..1d821ba05 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -11,7 +11,7 @@ from collections.abc import Sequence
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from hashlib import sha256
-from typing import Any, Literal, cast
+from typing import Any, Literal, TypeAlias, cast
import structlog
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
@@ -155,7 +155,16 @@ from skyvern.services.script_review_cap import (
)
from skyvern.services.script_reviewer_v3.cohort import is_v3_cohort
from skyvern.services.script_reviewer_v3.postrun import v3_review_post_run
-from skyvern.services.webhook_delivery import PreparedWorkflowWebhook, deliver_webhook_with_retries
+from skyvern.services.webhook_delivery import (
+ WEBHOOK_DELIVERY_HTTP_TIMEOUT_SECONDS,
+ WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON,
+ WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON,
+ WEBHOOK_DELIVERY_SUCCESS_REASON,
+ PreparedWorkflowWebhook,
+ deliver_webhook_with_retries,
+ scrub_internal_webhook_delivery_state,
+)
from skyvern.utils.css_selector import build_action_summaries_with_timing # shared with script_service
from skyvern.utils.secret_headers import merge_masked_headers
from skyvern.utils.url_validators import validate_url as validate_url_with_blocked_host_check
@@ -171,6 +180,7 @@ DEFAULT_WORKFLOW_TITLE = "New Workflow"
RECORDING_WINDOW_END_BUFFER = timedelta(minutes=15)
# Skip post-run work when only a sub-millisecond budget remains; asyncio.timeout would fire on the first await.
POST_RUN_TIMEOUT_EXHAUSTED_THRESHOLD_SECONDS = 0.001
+WorkflowWebhookDeliveryStatus: TypeAlias = Literal["pending", "failed"]
def _as_utc(dt: datetime) -> datetime:
@@ -189,6 +199,31 @@ def _format_workflow_run_elapsed_timeout_failure_reason(effective_minutes: int)
return f"Workflow run exceeded max elapsed runtime limit of {effective_minutes} {minute_label}."
+def _get_workflow_webhook_delivery_status(
+ *,
+ webhook_callback_url: str | None,
+ webhook_failure_reason: str | None,
+ workflow_run_status: WorkflowRunStatus,
+) -> WorkflowWebhookDeliveryStatus | None:
+ if not webhook_callback_url or not workflow_run_status.is_final():
+ return None
+ # Empty string is the persisted webhook delivery success sentinel.
+ if webhook_failure_reason == WEBHOOK_DELIVERY_SUCCESS_REASON:
+ return None
+ if webhook_failure_reason is None or webhook_failure_reason == WEBHOOK_DELIVERY_IN_PROGRESS_REASON:
+ return "pending"
+ return "failed"
+
+
+def _get_public_workflow_webhook_failure_reason(webhook_failure_reason: str | None) -> str | None:
+ if webhook_failure_reason in {
+ WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ WEBHOOK_DELIVERY_SUCCESS_REASON,
+ }:
+ return None
+ return webhook_failure_reason
+
+
def _collect_enterprise_gated_workflow_features(
workflow: Workflow,
*,
@@ -5422,13 +5457,35 @@ class WorkflowService:
task_v2: Any | None,
organization_id: str | None,
) -> tuple[list[str], bool]:
- """Fetch recording URLs, preferring browser-session recordings with artifact-store fallback.
+ """Fetch recording URLs, preferring precise run-scoped recordings over the shared
+ browser-session recording.
Returns (recording_urls, recording_archived).
"""
recording_urls: list[str] = []
recording_archived = False
- if workflow_run.browser_session_id:
+
+ # Prefer precise per-run clips over the shared browser-session recording (for a persistent
+ # session the latter is one continuous, often multi-hour video spanning every run). A
+ # non-clip run recording may be an in-progress per-step snapshot, so it does NOT pre-empt
+ # the finalized session recording below — only run_recordings/ clips do.
+ run_id = task_v2.observer_cruise_id if task_v2 else workflow_run.workflow_run_id
+ run_recording_artifacts = await app.DATABASE.artifacts.list_artifacts_for_run_by_type(
+ run_id=run_id,
+ artifact_type=ArtifactType.RECORDING,
+ organization_id=workflow_run.organization_id,
+ )
+ # "/run_recordings/" mirrors run_recording_clips.RUN_RECORDING_PATH_SEGMENT.
+ clip_artifacts = [a for a in run_recording_artifacts if a.uri and "/run_recordings/" in a.uri]
+ if clip_artifacts:
+ recording_archived = await app.ARTIFACT_MANAGER.is_recording_archived(clip_artifacts[0])
+ if not recording_archived:
+ urls = await app.ARTIFACT_MANAGER.get_share_links_with_bundle_support(clip_artifacts)
+ recording_urls = [u for u in urls if u is not None]
+
+ # Fall back to the shared browser-session recording (windowed, then unbounded for a
+ # persistent session whose continuous recording finalizes long after the run ends).
+ if not recording_urls and not recording_archived and workflow_run.browser_session_id:
if workflow_run.started_at is None:
LOG.warning(
"Skipping recording fan-out: workflow run has browser_session_id but no started_at",
@@ -5454,17 +5511,13 @@ class WorkflowService:
except asyncio.TimeoutError:
LOG.warning("Timeout getting recordings", browser_session_id=workflow_run.browser_session_id)
- if not recording_urls:
- recording_artifacts = await app.DATABASE.artifacts.list_artifacts_for_run_by_type(
- run_id=task_v2.observer_cruise_id if task_v2 else workflow_run.workflow_run_id,
- artifact_type=ArtifactType.RECORDING,
- organization_id=workflow_run.organization_id,
- )
- if recording_artifacts:
- recording_archived = await app.ARTIFACT_MANAGER.is_recording_archived(recording_artifacts[0])
- if not recording_archived:
- urls = await app.ARTIFACT_MANAGER.get_share_links_with_bundle_support(recording_artifacts)
- recording_urls = [u for u in urls if u is not None]
+ # Last resort: a run's own recording — a non-session task recording, or a session run
+ # whose clip and finalized session recording are both unavailable.
+ if not recording_urls and not recording_archived and run_recording_artifacts and not clip_artifacts:
+ recording_archived = await app.ARTIFACT_MANAGER.is_recording_archived(run_recording_artifacts[0])
+ if not recording_archived:
+ urls = await app.ARTIFACT_MANAGER.get_share_links_with_bundle_support(run_recording_artifacts)
+ recording_urls = [u for u in urls if u is not None]
return recording_urls, recording_archived
@@ -5624,7 +5677,12 @@ class WorkflowService:
failure_category=workflow_run.failure_category,
proxy_location=workflow_run.proxy_location,
webhook_callback_url=workflow_run.webhook_callback_url,
- webhook_failure_reason=workflow_run.webhook_failure_reason,
+ webhook_delivery_status=_get_workflow_webhook_delivery_status(
+ webhook_callback_url=workflow_run.webhook_callback_url,
+ webhook_failure_reason=workflow_run.webhook_failure_reason,
+ workflow_run_status=workflow_run.status,
+ ),
+ webhook_failure_reason=_get_public_workflow_webhook_failure_reason(workflow_run.webhook_failure_reason),
totp_verification_url=workflow_run.totp_verification_url,
totp_identifier=workflow_run.totp_identifier,
extra_http_headers=workflow_run.extra_http_headers,
@@ -5816,6 +5874,19 @@ class WorkflowService:
workflow_run_id=workflow_run.workflow_run_id,
workflow_permanent_id=workflow_run.workflow_permanent_id,
)
+ if workflow_run.webhook_callback_url and workflow_run.webhook_callback_url.strip():
+ try:
+ await app.DATABASE.workflow_runs.update_workflow_run(
+ workflow_run_id=workflow_run.workflow_run_id,
+ webhook_failure_reason=WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to record skipped workflow webhook delivery for missing workflow",
+ workflow_id=workflow_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ exc_info=True,
+ )
return None
if not workflow_run.webhook_callback_url:
LOG.warning(
@@ -5844,6 +5915,18 @@ class WorkflowService:
workflow_run_id=workflow_run.workflow_run_id,
organization_id=workflow_run.organization_id,
)
+ try:
+ await app.DATABASE.workflow_runs.update_workflow_run(
+ workflow_run_id=workflow_run.workflow_run_id,
+ webhook_failure_reason=WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to record skipped workflow webhook delivery",
+ workflow_id=workflow_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ exc_info=True,
+ )
return None
# build new schema for backward compatible webhook payload
@@ -5877,7 +5960,9 @@ class WorkflowService:
errors=workflow_run_status_response.errors,
step_count=workflow_run_status_response.total_steps,
)
- payload_dict: dict = json.loads(workflow_run_status_response.model_dump_json())
+ payload_dict: dict = scrub_internal_webhook_delivery_state(
+ json.loads(workflow_run_status_response.model_dump_json())
+ )
workflow_run_response_dict = json.loads(workflow_run_response.model_dump_json())
payload_dict.update(workflow_run_response_dict)
signed_data = generate_skyvern_webhook_signature(
@@ -5902,7 +5987,25 @@ class WorkflowService:
payload_for_log=signed_data.payload_for_log,
)
- async def deliver_prepared_workflow_webhook(self, webhook: PreparedWorkflowWebhook) -> None:
+ async def deliver_prepared_workflow_webhook(
+ self,
+ webhook: PreparedWorkflowWebhook,
+ *,
+ retry_claim_older_than: datetime | None = None,
+ ) -> None:
+ claimed = await app.DATABASE.workflow_runs.claim_workflow_run_webhook_delivery(
+ workflow_run_id=webhook.workflow_run_id,
+ in_progress_reason=WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ claim_older_than=retry_claim_older_than,
+ )
+ if not claimed:
+ LOG.info(
+ "Skipping workflow webhook delivery because it was already claimed or recorded",
+ workflow_id=webhook.workflow_id,
+ workflow_run_id=webhook.workflow_run_id,
+ )
+ return
+
LOG.info(
"Sending webhook run status to webhook callback url",
workflow_id=webhook.workflow_id,
@@ -5916,7 +6019,7 @@ class WorkflowService:
url=webhook.webhook_callback_url,
payload=webhook.signed_payload,
headers=webhook.headers,
- timeout_seconds=30.0,
+ timeout_seconds=WEBHOOK_DELIVERY_HTTP_TIMEOUT_SECONDS,
organization_id=webhook.organization_id,
run_id=webhook.workflow_run_id,
)
@@ -5953,7 +6056,8 @@ class WorkflowService:
try:
await app.DATABASE.workflow_runs.update_workflow_run(
workflow_run_id=webhook.workflow_run_id,
- webhook_failure_reason="",
+ # Empty string is the persisted successful-delivery sentinel.
+ webhook_failure_reason=WEBHOOK_DELIVERY_SUCCESS_REASON,
)
except Exception:
LOG.warning(
@@ -5989,11 +6093,12 @@ class WorkflowService:
self,
workflow_run: WorkflowRun,
api_key: str | None = None,
+ retry_claim_older_than: datetime | None = None,
) -> None:
webhook = await self.prepare_workflow_webhook(workflow_run, api_key)
if webhook is None:
return
- await self.deliver_prepared_workflow_webhook(webhook)
+ await self.deliver_prepared_workflow_webhook(webhook, retry_claim_older_than=retry_claim_older_than)
async def persist_video_data(
self,
@@ -6301,6 +6406,7 @@ class WorkflowService:
title=title,
description=request.description,
workflow_definition=workflow_definition,
+ edited_by=edited_by,
)
await self.maybe_delete_cached_code(
diff --git a/skyvern/schemas/browser_sessions.py b/skyvern/schemas/browser_sessions.py
index dc0f7561c..fcee70b7d 100644
--- a/skyvern/schemas/browser_sessions.py
+++ b/skyvern/schemas/browser_sessions.py
@@ -53,6 +53,21 @@ class CreateBrowserSessionRequest(BaseModel):
pattern=r"^bp_",
)
+ generate_browser_profile: bool = Field(
+ default=False,
+ description="When true, the session's browser profile (cookies, localStorage, etc.) is saved to storage "
+ "when the session ends so it can be turned into a reusable browser profile. Defaults to false to avoid "
+ "storing profiles for sessions that never need them. Sessions started with a browser_profile_id always "
+ "persist their profile regardless of this flag.",
+ )
+
+
+class UpdateBrowserSessionRequest(BaseModel):
+ generate_browser_profile: bool = Field(
+ description="Enable or disable saving this session's browser profile when it ends. Can be toggled while "
+ "the session is still alive; the value is read at session teardown.",
+ )
+
class ProcessBrowserSessionRecordingRequest(BaseModel):
compressed_chunks: list[str] = Field(
diff --git a/skyvern/services/webhook_delivery.py b/skyvern/services/webhook_delivery.py
index 5ed3d7b53..49622876c 100644
--- a/skyvern/services/webhook_delivery.py
+++ b/skyvern/services/webhook_delivery.py
@@ -7,6 +7,7 @@ import random
from dataclasses import dataclass
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
+from typing import Any
import httpx
import structlog
@@ -25,8 +26,27 @@ NON_5XX_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset(
)
WEBHOOK_DELIVERY_MAX_ATTEMPTS = 3
+WEBHOOK_DELIVERY_HTTP_TIMEOUT_SECONDS = 30.0
WEBHOOK_DELIVERY_RETRY_BASE_DELAY_SECONDS = 1.0
WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS = 30.0
+# Persisted in workflow_runs.webhook_failure_reason; changing requires migrating in-progress rows.
+WEBHOOK_DELIVERY_IN_PROGRESS_REASON = "Webhook delivery is in progress"
+# Persisted in workflow_runs.webhook_failure_reason for successful deliveries; changing requires
+# migrating historical success rows.
+WEBHOOK_DELIVERY_SUCCESS_REASON = ""
+WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON = "Workflow webhook delivery skipped because no API key was available"
+WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON = "Workflow webhook delivery skipped because the workflow was unavailable"
+
+
+def scrub_internal_webhook_delivery_state(payload: dict[str, Any]) -> dict[str, Any]:
+ scrubbed = dict(payload)
+ scrubbed.pop("webhook_delivery_status", None)
+ if scrubbed.get("webhook_failure_reason") in {
+ WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ WEBHOOK_DELIVERY_SUCCESS_REASON,
+ }:
+ scrubbed["webhook_failure_reason"] = None
+ return scrubbed
@dataclass(frozen=True)
diff --git a/skyvern/services/webhook_service.py b/skyvern/services/webhook_service.py
index bec51b1f5..96b3a88dc 100644
--- a/skyvern/services/webhook_service.py
+++ b/skyvern/services/webhook_service.py
@@ -40,6 +40,7 @@ from skyvern.schemas.runs import (
)
from skyvern.schemas.webhooks import RunWebhookPreviewResponse, RunWebhookReplayResponse
from skyvern.services import run_service, task_v2_service
+from skyvern.services.webhook_delivery import scrub_internal_webhook_delivery_state
from skyvern.utils.url_validators import validate_url
LOG = structlog.get_logger()
@@ -439,14 +440,16 @@ async def _build_workflow_payload(
errors=status_response.errors,
)
- payload_dict = json.loads(
- status_response.model_dump_json(
- exclude={
- "webhook_callback_url",
- "totp_verification_url",
- "totp_identifier",
- "extra_http_headers",
- }
+ payload_dict = scrub_internal_webhook_delivery_state(
+ json.loads(
+ status_response.model_dump_json(
+ exclude={
+ "webhook_callback_url",
+ "totp_verification_url",
+ "totp_identifier",
+ "extra_http_headers",
+ }
+ )
)
)
payload_dict.update(json.loads(run_response.model_dump_json(exclude={"run_request"})))
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index 01ab9e779..165ccbe3e 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -308,6 +308,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
browser_type: PersistentBrowserType | None = None,
is_high_priority: bool = False,
browser_profile_id: str | None = None,
+ generate_browser_profile: bool = False,
wait_for_startup: bool = True,
) -> PersistentBrowserSession:
"""Create a new browser session for an organization and return its ID with the browser state."""
@@ -324,6 +325,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
extensions=extensions,
browser_type=browser_type,
browser_profile_id=browser_profile_id,
+ generate_browser_profile=generate_browser_profile,
)
# Launch the browser immediately for standalone sessions so the
@@ -460,24 +462,50 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
browser_artifacts = browser_session.browser_state.browser_artifacts
if browser_artifacts and browser_artifacts.browser_session_dir:
# Snapshot session cookies before store_browser_profile copies the dir; the later
- # close() persist runs after this export, too late to land in the archive.
+ # close() persist runs after this export, too late to land in the archive. The
+ # snapshot stays unconditional even when the export is skipped so the sidecar is
+ # ready if the session opted in.
await persist_session_cookies(
browser_session.browser_state.browser_context, browser_artifacts.browser_session_dir
)
+ # Re-read the persisted row so an update-while-alive opt-in toggle is honored.
+ # Isolated and fail-open: a lookup error must neither block the close cleanup below
+ # nor drop an opted-in / profile-reuse session's profile, so skip only on a confirmed
+ # opted-out row.
+ should_export = True
try:
- await app.STORAGE.store_browser_profile(
- organization_id=organization_id,
- profile_id=browser_session_id,
- directory=browser_artifacts.browser_session_dir,
+ persisted_session = await self.database.browser_sessions.get_persistent_browser_session(
+ browser_session_id, organization_id
)
- LOG.info(
- "Exported browser session profile",
+ should_export = persisted_session is None or persisted_session.should_export_profile()
+ except Exception:
+ LOG.warning(
+ "Failed to read browser session opt-in flag; exporting profile to be safe",
browser_session_id=browser_session_id,
organization_id=organization_id,
+ exc_info=True,
)
- except Exception:
- LOG.exception(
- "Failed to export browser session profile",
+ if should_export:
+ try:
+ await app.STORAGE.store_browser_profile(
+ organization_id=organization_id,
+ profile_id=browser_session_id,
+ directory=browser_artifacts.browser_session_dir,
+ )
+ LOG.info(
+ "Exported browser session profile",
+ browser_session_id=browser_session_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to export browser session profile",
+ browser_session_id=browser_session_id,
+ organization_id=organization_id,
+ )
+ else:
+ LOG.info(
+ "Skipping browser profile export; session did not opt into profile generation",
browser_session_id=browser_session_id,
organization_id=organization_id,
)
diff --git a/skyvern/webeye/persistent_sessions_manager.py b/skyvern/webeye/persistent_sessions_manager.py
index 7ebf0113c..2211e4876 100644
--- a/skyvern/webeye/persistent_sessions_manager.py
+++ b/skyvern/webeye/persistent_sessions_manager.py
@@ -82,6 +82,7 @@ class PersistentSessionsManager(Protocol):
browser_type: PersistentBrowserType | None = None,
is_high_priority: bool = False,
browser_profile_id: str | None = None,
+ generate_browser_profile: bool = False,
wait_for_startup: bool = True,
) -> PersistentBrowserSession:
"""Create a new browser session."""
diff --git a/skyvern/webeye/schemas.py b/skyvern/webeye/schemas.py
index acb69b790..2208b6c8d 100644
--- a/skyvern/webeye/schemas.py
+++ b/skyvern/webeye/schemas.py
@@ -67,6 +67,10 @@ class BrowserSessionResponse(BaseModel):
default=None,
description="ID of the browser profile loaded into this session, if any. browser_profile_id starts with `bp_`.",
)
+ generate_browser_profile: bool = Field(
+ default=False,
+ description="Whether this session's browser profile will be saved when it ends so it can become a reusable browser profile.",
+ )
vnc_streaming_supported: bool = Field(False, description="Whether the browser session supports VNC streaming")
download_path: str | None = Field(None, description="The path where the browser session downloads files")
downloaded_files: list[FileInfo] | None = Field(
@@ -153,4 +157,5 @@ class BrowserSessionResponse(BaseModel):
extensions=browser_session.extensions,
browser_type=browser_session.browser_type,
browser_profile_id=browser_session.browser_profile_id,
+ generate_browser_profile=browser_session.generate_browser_profile,
)
diff --git a/skyvern/webeye/video_utils.py b/skyvern/webeye/video_utils.py
index ed6a89106..5cd0cfc5e 100644
--- a/skyvern/webeye/video_utils.py
+++ b/skyvern/webeye/video_utils.py
@@ -7,6 +7,7 @@ import tempfile
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
import structlog
@@ -15,7 +16,12 @@ from skyvern.config import settings
LOG = structlog.get_logger()
FFMPEG_BINARY = "ffmpeg"
+FFPROBE_BINARY = "ffprobe"
FFMPEG_REMUX_TIMEOUT_SECONDS = 30
+FFPROBE_TIMEOUT_SECONDS = 15
+# Clips are re-encoded (libx264), not stream-copied; size the timeout like the full-recording
+# compression since a clip is a subset of what already compresses within that budget.
+FFMPEG_CUT_TIMEOUT_SECONDS = 300
@dataclass(frozen=True)
@@ -45,6 +51,7 @@ async def _run_ffmpeg_to_temp(
output_args: list[str],
timeout_seconds: float,
operation: str,
+ input_args: list[str] | None = None,
) -> str | None:
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as dst_tmp:
dst_path = dst_tmp.name
@@ -58,6 +65,7 @@ async def _run_ffmpeg_to_temp(
"-loglevel",
"error",
"-y",
+ *(input_args or []),
"-i",
src_path,
*output_args,
@@ -233,3 +241,137 @@ async def finalize_webm(src_path: str) -> bytes:
"""
async with prepare_webm_for_upload(src_path) as upload_path:
return await asyncio.to_thread(_read_file, upload_path)
+
+
+async def probe_media_duration_seconds(src_path: str) -> float | None:
+ """Return the media duration in seconds via ffprobe, or None if unavailable.
+
+ Used to anchor a session recording on the wall-clock timeline: a recording that
+ finished at close time T spans ``[T - duration, T]``, which is how per-run clips
+ are located within it.
+ """
+ if not os.path.exists(src_path):
+ return None
+ if shutil.which(FFPROBE_BINARY) is None:
+ LOG.warning("ffprobe binary not found on PATH", src=src_path)
+ return None
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ FFPROBE_BINARY,
+ "-v",
+ "error",
+ "-show_entries",
+ "format=duration",
+ "-of",
+ "default=noprint_wrappers=1:nokey=1",
+ src_path,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ except Exception:
+ LOG.warning("ffprobe failed to start", src=src_path, exc_info=True)
+ return None
+ try:
+ stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=FFPROBE_TIMEOUT_SECONDS)
+ except asyncio.TimeoutError:
+ await _kill_and_wait_for_process(proc)
+ LOG.warning("ffprobe timed out", src=src_path, timeout_seconds=FFPROBE_TIMEOUT_SECONDS)
+ return None
+ if proc.returncode != 0:
+ return None
+ try:
+ duration = float(stdout.decode(errors="replace").strip())
+ except ValueError:
+ return None
+ return duration if duration > 0 else None
+
+
+@asynccontextmanager
+async def cut_recording_segment(
+ src_path: str, *, start_seconds: float, duration_seconds: float
+) -> AsyncIterator[str | None]:
+ """Yield a path to a frame-accurate ``[start, start+duration]`` mp4 clip of ``src_path``.
+
+ Re-encodes rather than stream-copying so the clip starts exactly at ``start_seconds``: a
+ stream copy can only cut on a keyframe, and with libx264's default GOP that pulls in seconds
+ of pre-roll — for a session recording shared across runs, the *previous* run's frames. Input
+ ``-ss`` keeps the seek fast while the re-encode makes it accurate. Yields None (not raising)
+ when ffmpeg is unavailable or the cut fails, so the caller skips one run without aborting the
+ rest; the temp clip is removed on exit.
+ """
+ if not os.path.exists(src_path):
+ raise FileNotFoundError(src_path)
+ cut_path: str | None = None
+ if shutil.which(FFMPEG_BINARY) is None:
+ LOG.warning("ffmpeg binary not found on PATH; skipping recording cut", src=src_path)
+ elif duration_seconds <= 0:
+ LOG.warning("non-positive cut duration; skipping recording cut", src=src_path, duration=duration_seconds)
+ else:
+ cut_path = await _run_ffmpeg_to_temp(
+ src_path,
+ suffix=".mp4",
+ input_args=["-ss", f"{max(0.0, start_seconds):.3f}"],
+ output_args=[
+ "-t",
+ f"{duration_seconds:.3f}",
+ "-map",
+ "0:v:0",
+ "-an",
+ "-c:v",
+ "libx264",
+ "-preset",
+ settings.VIDEO_COMPRESSION_PRESET,
+ "-crf",
+ str(settings.VIDEO_COMPRESSION_CRF),
+ "-pix_fmt",
+ "yuv420p",
+ "-movflags",
+ "+faststart",
+ ],
+ timeout_seconds=FFMPEG_CUT_TIMEOUT_SECONDS,
+ operation="ffmpeg recording cut",
+ )
+ try:
+ yield cut_path
+ finally:
+ if cut_path and os.path.exists(cut_path):
+ try:
+ os.unlink(cut_path)
+ except OSError:
+ LOG.debug("failed to cleanup recording cut temp output", path=cut_path, exc_info=True)
+
+
+def _as_utc(dt: datetime) -> datetime:
+ # Naive datetimes in this codebase are UTC (see workflow.service._as_utc).
+ return dt if dt.tzinfo is not None else dt.replace(tzinfo=UTC)
+
+
+def plan_run_segment(
+ run_started_at: datetime,
+ run_finished_at: datetime | None,
+ video_start: datetime,
+ video_duration_seconds: float,
+ *,
+ min_segment_seconds: float = 1.0,
+) -> tuple[float, float] | None:
+ """Map a run's wall-clock ``[started_at, finished_at]`` onto offsets within a session
+ recording that began at ``video_start`` and ran ``video_duration_seconds``.
+
+ Returns ``(start_seconds, duration_seconds)`` clamped to the recording, or None when the
+ run does not overlap it or the overlap is shorter than ``min_segment_seconds`` (e.g. a run
+ that finished before this video started, or one whose clip would be too short to be useful).
+ """
+ if video_duration_seconds <= 0:
+ return None
+ start = _as_utc(run_started_at)
+ end = (
+ _as_utc(run_finished_at)
+ if run_finished_at is not None
+ else video_start + timedelta(seconds=video_duration_seconds)
+ )
+ anchor = _as_utc(video_start)
+ start_offset = min(max((start - anchor).total_seconds(), 0.0), video_duration_seconds)
+ end_offset = min(max((end - anchor).total_seconds(), 0.0), video_duration_seconds)
+ if end_offset - start_offset < min_segment_seconds:
+ return None
+ return start_offset, end_offset - start_offset
diff --git a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
index 55b4d64ad..5cd6d5c8a 100644
--- a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
+++ b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py
@@ -3,6 +3,7 @@
from __future__ import annotations
from datetime import datetime, timezone
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -203,6 +204,97 @@ async def test_get_all_runs_v2_search_key_matches_run_id_and_workflow_permanent_
assert "abc123" in where_clause
+async def test_get_workflow_runs_with_pending_webhook_delivery_filters_finalized_null_webhook_state() -> None:
+ captured: dict[str, Any] = {}
+
+ async def _scalars(query):
+ captured["query"] = query
+ return _EmptyExecuteResult()
+
+ session = MagicMock()
+ session.scalars = AsyncMock(side_effect=_scalars)
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ await repo.get_workflow_runs_with_pending_webhook_delivery(
+ older_than=datetime(2026, 6, 1, 13, 20, tzinfo=timezone.utc),
+ limit=17,
+ in_progress_reason="Webhook delivery is in progress",
+ )
+
+ rendered = str(captured["query"].compile(compile_kwargs={"literal_binds": True}))
+ where_clause = _where_clause_sql(captured["query"])
+ assert "workflow_runs.status IN ('failed', 'terminated', 'canceled', 'timed_out', 'completed')" in where_clause
+ for expected in [
+ "workflow_runs.webhook_callback_url IS NOT NULL",
+ "length(trim(workflow_runs.webhook_callback_url)) > 0",
+ # SQLAlchemy omits parentheses in the rendered string; SQL operator
+ # precedence keeps the modified_at cutoff bound to only the sentinel branch.
+ "workflow_runs.webhook_failure_reason IS NULL OR workflow_runs.webhook_failure_reason = 'Webhook delivery is in progress' AND workflow_runs.modified_at <= '2026-06-01 13:20:00'",
+ "workflow_runs.finished_at <= '2026-06-01 13:20:00' OR workflow_runs.finished_at IS NULL AND workflow_runs.modified_at <= '2026-06-01 13:20:00'",
+ ]:
+ assert expected in where_clause
+ assert "ORDER BY coalesce(workflow_runs.finished_at, workflow_runs.modified_at) ASC" in rendered
+ assert "LIMIT 17" in rendered
+
+
+async def test_claim_workflow_run_webhook_delivery_only_claims_null_state() -> None:
+ captured: dict[str, Any] = {}
+
+ async def _execute(query):
+ captured["query"] = query
+ return SimpleNamespace(rowcount=1)
+
+ session = MagicMock()
+ session.execute = AsyncMock(side_effect=_execute)
+ session.commit = AsyncMock()
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ claimed = await repo.claim_workflow_run_webhook_delivery(
+ workflow_run_id="wr_abc",
+ in_progress_reason="Webhook delivery is in progress",
+ )
+
+ rendered = str(captured["query"].compile(compile_kwargs={"literal_binds": True}))
+ assert claimed is True
+ assert "UPDATE workflow_runs SET" in rendered
+ assert "webhook_failure_reason='Webhook delivery is in progress'" in rendered
+ assert "modified_at=" in rendered
+ assert "workflow_runs.workflow_run_id = 'wr_abc'" in rendered
+ assert "workflow_runs.webhook_failure_reason IS NULL" in rendered
+ session.commit.assert_awaited_once()
+
+
+async def test_claim_workflow_run_webhook_delivery_can_reclaim_stale_in_progress_state() -> None:
+ captured: dict[str, Any] = {}
+
+ async def _execute(query):
+ captured["query"] = query
+ return SimpleNamespace(rowcount=1)
+
+ session = MagicMock()
+ session.execute = AsyncMock(side_effect=_execute)
+ session.commit = AsyncMock()
+ repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)
+
+ claimed = await repo.claim_workflow_run_webhook_delivery(
+ workflow_run_id="wr_abc",
+ in_progress_reason="Webhook delivery is in progress",
+ claim_older_than=datetime(2026, 6, 1, 13, 20, tzinfo=timezone.utc),
+ )
+
+ where_clause = _where_clause_sql(captured["query"])
+ assert claimed is True
+ assert "workflow_runs.workflow_run_id = 'wr_abc'" in where_clause
+ assert (
+ "workflow_runs.webhook_failure_reason IS NULL OR workflow_runs.webhook_failure_reason = "
+ "'Webhook delivery is in progress' AND workflow_runs.modified_at <= '2026-06-01 13:20:00'"
+ ) in where_clause
+ session.commit.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_get_all_runs_v2_selects_workflow_deleted_flag() -> None:
captured: dict[str, Any] = {}
diff --git a/tests/unit/services/test_org_auth_optional_user.py b/tests/unit/services/test_org_auth_optional_user.py
new file mode 100644
index 000000000..662bfe62f
--- /dev/null
+++ b/tests/unit/services/test_org_auth_optional_user.py
@@ -0,0 +1,144 @@
+"""Tests for the fail-open user-resolution dependency used for write attribution."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.services import org_auth_service
+
+
+async def test_returns_user_id_for_valid_bearer() -> None:
+ with patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app:
+ mock_app.authenticate_user_function = AsyncMock(return_value="user_abc")
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization="Bearer good-token",
+ x_api_key=None,
+ x_user_agent=None,
+ )
+ assert result == "user_abc"
+
+
+async def test_returns_none_without_credentials() -> None:
+ with patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app:
+ mock_app.authenticate_user_function = None
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization=None,
+ x_api_key=None,
+ x_user_agent=None,
+ )
+ assert result is None
+
+
+async def test_returns_none_for_invalid_bearer() -> None:
+ with patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app:
+ mock_app.authenticate_user_function = AsyncMock(return_value=None)
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization="Bearer bad-token",
+ x_api_key=None,
+ x_user_agent=None,
+ )
+ assert result is None
+
+
+async def test_returns_synthetic_id_for_api_key_with_ui_agent() -> None:
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+ with (
+ patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app,
+ patch(
+ "skyvern.forge.sdk.services.org_auth_service.get_current_org_cached",
+ new=AsyncMock(return_value=organization),
+ ),
+ ):
+ mock_app.authenticate_user_function = None
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization=None,
+ x_api_key="some-api-key",
+ x_user_agent="skyvern-ui",
+ )
+ assert result == "o_123_user"
+
+
+async def test_returns_none_for_api_key_without_ui_agent() -> None:
+ with patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app:
+ mock_app.authenticate_user_function = None
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization=None,
+ x_api_key="some-api-key",
+ x_user_agent=None,
+ )
+ assert result is None
+
+
+async def test_returns_user_id_when_bearer_user_is_member_of_api_key_org() -> None:
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+ with (
+ patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app,
+ patch(
+ "skyvern.forge.sdk.services.org_auth_service.get_current_org_cached",
+ new=AsyncMock(return_value=organization),
+ ),
+ ):
+ mock_app.authenticate_user_function = AsyncMock(return_value="user_abc")
+ mock_app.authentication_function = AsyncMock(return_value=organization)
+ mock_app.AGENT_FUNCTION.validate_user_organization_membership = AsyncMock(return_value=True)
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization="Bearer good-token",
+ x_api_key="some-api-key",
+ x_user_agent="skyvern-ui",
+ )
+ assert result == "user_abc"
+ mock_app.AGENT_FUNCTION.validate_user_organization_membership.assert_awaited_once_with(
+ user_id="user_abc",
+ organization_id="o_123",
+ bearer_token="good-token",
+ )
+ # Bearer org auth is side-effectful in overrides; the guard must never invoke it.
+ mock_app.authentication_function.assert_not_called()
+
+
[email protected]("membership_verdict", [False, None])
+async def test_returns_none_unless_bearer_user_membership_verified(membership_verdict: bool | None) -> None:
+ # Org auth wins on x-api-key while the user comes from the bearer; a stale
+ # cross-org key must not stamp the bearer user into the api-key org.
+ key_org = MagicMock()
+ key_org.organization_id = "o_key"
+ with (
+ patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app,
+ patch(
+ "skyvern.forge.sdk.services.org_auth_service.get_current_org_cached",
+ new=AsyncMock(return_value=key_org),
+ ),
+ ):
+ mock_app.authenticate_user_function = AsyncMock(return_value="user_abc")
+ mock_app.authentication_function = AsyncMock(return_value=key_org)
+ mock_app.AGENT_FUNCTION.validate_user_organization_membership = AsyncMock(return_value=membership_verdict)
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization="Bearer good-token",
+ x_api_key="some-api-key",
+ x_user_agent="skyvern-ui",
+ )
+ assert result is None
+ mock_app.authentication_function.assert_not_called()
+
+
+async def test_returns_none_on_unexpected_auth_error() -> None:
+ with patch("skyvern.forge.sdk.services.org_auth_service.app") as mock_app:
+ mock_app.authenticate_user_function = AsyncMock(side_effect=RuntimeError("auth backend down"))
+ result = await org_auth_service.get_current_user_id_or_none(
+ authorization="Bearer token",
+ x_api_key=None,
+ x_user_agent=None,
+ )
+ assert result is None
diff --git a/tests/unit/test_block_downloaded_file_scoping.py b/tests/unit/test_block_downloaded_file_scoping.py
index 1ad74bc78..ceb05395e 100644
--- a/tests/unit/test_block_downloaded_file_scoping.py
+++ b/tests/unit/test_block_downloaded_file_scoping.py
@@ -19,7 +19,6 @@ from skyvern.forge.sdk.workflow.loop_download_filter import (
to_downloaded_file_signature,
)
from skyvern.forge.sdk.workflow.models.block import (
- BLOCK_BASELINE_MARKER,
capture_block_download_baseline,
)
@@ -106,12 +105,12 @@ def test_duplicate_files_handled_correctly():
assert len(filtered) == 1
-# --- Tests for capture_block_download_baseline marker logic ---
+# --- Tests for capture_block_download_baseline per-block capture ---
@pytest.mark.asyncio
async def test_baseline_captured_when_loop_internal_state_is_none():
- """When loop_internal_state is None, the helper creates a new baseline with the marker."""
+ """When loop_internal_state is None, the helper creates a new baseline."""
file1 = _make_file_info("s3://bucket/file1.pdf", "file1.pdf", "checksum1")
context = SkyvernContext(run_id="wr_test", loop_internal_state=None)
@@ -123,13 +122,13 @@ async def test_baseline_captured_when_loop_internal_state_is_none():
await capture_block_download_baseline(context, "org_1", "wr_test", "block_1")
assert context.loop_internal_state is not None
- assert BLOCK_BASELINE_MARKER in context.loop_internal_state
assert len(context.loop_internal_state[DOWNLOADED_FILE_SIGS_KEY]) == 1
@pytest.mark.asyncio
async def test_baseline_recaptured_when_set_by_previous_block():
- """When loop_internal_state was set by a previous block (has marker), it gets overwritten."""
+ """When loop_internal_state was set by a previous block, it gets overwritten with the
+ files that exist now — so the current block is scoped to only what it adds next."""
file1 = _make_file_info("s3://bucket/file1.pdf", "file1.pdf", "checksum1")
file2 = _make_file_info("s3://bucket/file2.pdf", "file2.pdf", "checksum2")
@@ -138,7 +137,6 @@ async def test_baseline_recaptured_when_set_by_previous_block():
run_id="wr_test",
loop_internal_state={
DOWNLOADED_FILE_SIGS_KEY: [to_downloaded_file_signature(file1)],
- BLOCK_BASELINE_MARKER: True,
},
)
@@ -150,29 +148,36 @@ async def test_baseline_recaptured_when_set_by_previous_block():
await capture_block_download_baseline(context, "org_1", "wr_test", "block_2")
assert context.loop_internal_state is not None
- assert BLOCK_BASELINE_MARKER in context.loop_internal_state
# New baseline includes both files
assert len(context.loop_internal_state[DOWNLOADED_FILE_SIGS_KEY]) == 2
@pytest.mark.asyncio
-async def test_loop_baseline_preserved_when_no_marker():
- """When loop_internal_state was set by ForLoopBlock (no marker), it is NOT overwritten."""
+async def test_baseline_recaptured_even_when_loop_set_it():
+ """SKY-10784: a block inside a loop must re-capture its own baseline rather than
+ defer to the loop's per-iteration baseline. Otherwise sibling download-producing
+ blocks in the same iteration accumulate each other's files."""
loop_baseline = {
DOWNLOADED_FILE_SIGS_KEY: [("a.pdf", "abc", "s3://a.pdf")],
}
context = SkyvernContext(run_id="wr_test", loop_internal_state=loop_baseline)
+ # A sibling block earlier in this iteration already produced b.pdf, so it now
+ # exists alongside the loop's pre-iteration a.pdf.
+ sibling_file = _make_file_info("s3://bucket/b.pdf", "b.pdf", "checksum_b")
mock_storage = AsyncMock()
- mock_storage.get_downloaded_files = AsyncMock(return_value=[])
+ mock_storage.get_downloaded_files = AsyncMock(
+ return_value=[_make_file_info("s3://a.pdf", "a.pdf", "abc"), sibling_file]
+ )
with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:
mock_app.STORAGE = mock_storage
- await capture_block_download_baseline(context, "org_1", "wr_test", "block_1")
+ await capture_block_download_baseline(context, "org_1", "wr_test", "block_2")
- # Should be unchanged — loop baseline preserved
- assert context.loop_internal_state is loop_baseline
- mock_storage.get_downloaded_files.assert_not_called()
+ # Re-captured: the sibling's file is now part of this block's baseline, so it
+ # will be filtered out of this block's own output.
+ mock_storage.get_downloaded_files.assert_called_once()
+ assert len(context.loop_internal_state[DOWNLOADED_FILE_SIGS_KEY]) == 2
@pytest.mark.asyncio
@@ -191,10 +196,9 @@ async def test_baseline_capture_degrades_on_timeout():
@pytest.mark.asyncio
-async def test_stale_loop_baseline_blocks_fresh_capture_without_restore():
- """Regression guard: a loop-set baseline (no marker) left in context
- causes capture_block_download_baseline to skip capture. ForLoopBlock
- must clear/restore its state so subsequent blocks can re-capture."""
+async def test_stale_loop_baseline_overwritten_by_fresh_capture():
+ """A stale baseline left in context (e.g. from a prior loop iteration) is
+ overwritten by the current files, so it can never cause stale filtering."""
stale_loop_state = {
DOWNLOADED_FILE_SIGS_KEY: [("a.pdf", "abc", "s3://a.pdf")],
}
@@ -206,10 +210,9 @@ async def test_stale_loop_baseline_blocks_fresh_capture_without_restore():
mock_app.STORAGE = mock_storage
await capture_block_download_baseline(context, "org_1", "wr_test", "block_1")
- # Capture was skipped because the loop baseline has no marker — this is
- # why ForLoopBlock.execute() must save/restore loop_internal_state.
- assert context.loop_internal_state is stale_loop_state
- mock_storage.get_downloaded_files.assert_not_called()
+ mock_storage.get_downloaded_files.assert_called_once()
+ assert context.loop_internal_state is not stale_loop_state
+ assert context.loop_internal_state[DOWNLOADED_FILE_SIGS_KEY] == []
@pytest.mark.asyncio
@@ -264,3 +267,47 @@ async def test_forloopblock_execute_restores_outer_loop_state():
# Outer state must be restored even though _run_loop mutated and raised.
assert context.loop_internal_state is outer_state
+
+
+async def test_sibling_download_blocks_in_loop_iteration_scope_to_own_files():
+ """SKY-10784 regression: multiple download-producing blocks inside one loop
+ iteration must each surface only the file they produced — not their siblings'.
+ Mirrors the reported production run, where three Print Page blocks in a single loop
+ iteration showed 1, then 2, then 3 PDFs instead of 1 each.
+ """
+ pp1 = _make_file_info("https://api/v1/artifacts/a_pp1/content", "page_1.pdf", "c1")
+ pp2 = _make_file_info("https://api/v1/artifacts/a_pp2/content", "page_2.pdf", "c2")
+ pp3 = _make_file_info("https://api/v1/artifacts/a_pp3/content", "page_3.pdf", "c3")
+ pp4 = _make_file_info("https://api/v1/artifacts/a_pp4/content", "page_4.pdf", "c4")
+
+ # The run's download directory, shared across blocks and growing as each produces a file.
+ run_files: list[FileInfo] = []
+
+ # ForLoopBlock sets the per-iteration baseline (no marker) at iteration start.
+ context = SkyvernContext(run_id="wr_test", loop_internal_state={DOWNLOADED_FILE_SIGS_KEY: []})
+
+ mock_storage = AsyncMock()
+ # Each baseline capture reads a snapshot of the files that exist at that moment.
+ mock_storage.get_downloaded_files = AsyncMock(side_effect=lambda **_: list(run_files))
+
+ async def run_download_block(produced: FileInfo) -> list[FileInfo]:
+ with patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app:
+ mock_app.STORAGE = mock_storage
+ await capture_block_download_baseline(context, "org_1", "wr_test", "print")
+ run_files.append(produced)
+ return filter_downloaded_files_for_current_iteration(list(run_files), context.loop_internal_state)
+
+ out1 = await run_download_block(pp1)
+ out2 = await run_download_block(pp2)
+ out3 = await run_download_block(pp3)
+
+ assert [f.filename for f in out1] == ["page_1.pdf"]
+ assert [f.filename for f in out2] == ["page_2.pdf"]
+ assert [f.filename for f in out3] == ["page_3.pdf"]
+
+ # Next loop iteration: ForLoopBlock re-captures the per-iteration baseline, and the
+ # first block of the new iteration must still exclude every prior iteration's file.
+ context.loop_internal_state = {DOWNLOADED_FILE_SIGS_KEY: [to_downloaded_file_signature(f) for f in run_files]}
+ out4 = await run_download_block(pp4)
+ assert [f.filename for f in out4] == ["page_4.pdf"]
diff --git a/tests/unit/test_browser_session_profile_export_gate.py b/tests/unit/test_browser_session_profile_export_gate.py
new file mode 100644
index 000000000..21d2bb744
--- /dev/null
+++ b/tests/unit/test_browser_session_profile_export_gate.py
@@ -0,0 +1,40 @@
+from datetime import datetime
+
+from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
+from skyvern.schemas.browser_sessions import CreateBrowserSessionRequest, UpdateBrowserSessionRequest
+
+
+def _session(**kwargs: object) -> PersistentBrowserSession:
+ base: dict[str, object] = {
+ "persistent_browser_session_id": "pbs_1",
+ "organization_id": "o_1",
+ "created_at": datetime(2026, 1, 1),
+ "modified_at": datetime(2026, 1, 1),
+ }
+ base.update(kwargs)
+ return PersistentBrowserSession(**base)
+
+
+def test_should_export_profile_opted_out_by_default() -> None:
+ assert _session().should_export_profile() is False
+ assert _session(generate_browser_profile=False).should_export_profile() is False
+
+
+def test_should_export_profile_when_opted_in() -> None:
+ assert _session(generate_browser_profile=True).should_export_profile() is True
+
+
+def test_should_export_profile_always_true_when_reusing_a_profile() -> None:
+ # A session reusing a saved profile must always re-export so the updated session-cookie
+ # sidecar survives — gating it off would silently log the profile out on the next reuse.
+ assert _session(browser_profile_id="bp_1").should_export_profile() is True
+ assert _session(browser_profile_id="bp_1", generate_browser_profile=False).should_export_profile() is True
+
+
+def test_create_request_defaults_to_opt_out() -> None:
+ assert CreateBrowserSessionRequest().generate_browser_profile is False
+
+
+def test_update_request_carries_flag() -> None:
+ assert UpdateBrowserSessionRequest(generate_browser_profile=True).generate_browser_profile is True
+ assert UpdateBrowserSessionRequest(generate_browser_profile=False).generate_browser_profile is False
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 8fb3cb18d..76a3e1875 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -3052,11 +3052,11 @@ workflow_definition:
workflow.sequential_key = None
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml",
lambda **kwargs: workflow,
)
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools.resolve_copilot_created_by_stamp",
+ "skyvern.forge.sdk.copilot.tools.workflow_update.resolve_copilot_created_by_stamp",
AsyncMock(return_value="copilot"),
)
workflow_service = MagicMock()
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index bbb3ccfd3..a99f89d2e 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -563,7 +563,7 @@ async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatc
return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -584,7 +584,7 @@ async def test_maybe_run_completion_verification_skips_active_terminal_evidence(
return handler
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
fake_completion_verification_handler,
)
ctx = _run_ctx()
@@ -632,12 +632,14 @@ async def test_active_run_terminal_evidence_sample_matches_current_page(
return handler
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
fake_completion_verification_handler,
)
- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._fallback_page_info", fake_fallback_page_info)
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._capture_composition_evidence",
+ "skyvern.forge.sdk.copilot.tools.composition_capture._fallback_page_info", fake_fallback_page_info
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._capture_composition_evidence",
fake_capture_composition_evidence,
)
ctx = _run_ctx()
@@ -664,7 +666,9 @@ async def test_active_run_terminal_evidence_sample_skips_method_only_criteria(
async def handler(**_: object) -> dict:
raise AssertionError("method-mandated criteria cannot be verified from page state")
- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", lambda: handler)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler", lambda: handler
+ )
ctx = _run_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[_criterion("c0", "must use website search", method_mandated=True)]
@@ -815,7 +819,7 @@ async def test_page_observation_verification_recognizes_budgeted_outcome(
return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -861,7 +865,7 @@ async def test_page_observation_verification_does_not_overwrite_satisfied_verdic
raise AssertionError("handler should not be called once the outcome is verified")
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -898,7 +902,7 @@ async def test_page_observation_verification_preserves_existing_unsatisfied_verd
return {"verdicts": [{"criterion_id": "c0", "satisfied": False, "reason_code": "no_evidence"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -932,7 +936,7 @@ async def test_page_observation_verification_can_upgrade_unsatisfied_verdict(
return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -1038,7 +1042,7 @@ async def test_maybe_run_completion_verification_runs_on_unverified_prefix(monke
return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _ctx_unverified_prefix()
@@ -1068,7 +1072,7 @@ async def test_method_mandated_criteria_excluded_from_verification(monkeypatch:
return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -1201,7 +1205,7 @@ async def test_completion_verification_receives_verified_context_labels(monkeypa
}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
monkeypatch.setattr(settings, "COPILOT_OUTCOME_VERIFICATION_ENABLED", True)
@@ -1271,7 +1275,7 @@ async def test_maybe_run_completion_verification_unavailable_on_low_budget(monke
return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(handler),
)
ctx = _run_ctx()
@@ -1285,7 +1289,7 @@ async def test_maybe_run_completion_verification_unavailable_on_low_budget(monke
# A missing judge handler stays a soft fallback (None), not a hard block.
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._completion_verification_handler",
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
_completion_handler_lookup(None),
)
assert await _maybe_run_completion_verification(ctx, _clean_success_result(), time.monotonic()) is None
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index e98d68a6c..dd2dd66fd 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -2415,7 +2415,7 @@ async def test_navigation_failure_uses_structured_evidence_when_bounded() -> Non
@pytest.mark.asyncio
async def test_capture_prefers_html_parse_over_hollow_structured_on_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
# When both extractor and get_html are unbounded, the get_html parse wins (carries the fallback signals).
- monkeypatch.setattr(tools_module, "_COMPOSITION_HOLLOW_RECAPTURE_RETRIES", 0)
+ monkeypatch.setattr(tools_module.composition_capture, "_COMPOSITION_HOLLOW_RECAPTURE_RETRIES", 0)
hollow = json.dumps(
{
"page_title": "",
diff --git a/tests/unit/test_copilot_composition_scrape.py b/tests/unit/test_copilot_composition_scrape.py
index 2966c7384..3a2ee64ef 100644
--- a/tests/unit/test_copilot_composition_scrape.py
+++ b/tests/unit/test_copilot_composition_scrape.py
@@ -56,9 +56,11 @@ async def test_recapture_skips_raw_get_html_after_cap_drop(monkeypatch: pytest.M
async def identity(ctx: object, evidence: dict) -> dict:
return evidence
- monkeypatch.setattr(tools, "_discovery_get_html", fake_raw)
- monkeypatch.setattr(tools, "_composition_get_stripped_html", fake_stripped)
- monkeypatch.setattr(tools, "_augment_composition_evidence_with_computed_obstruction_candidates", identity)
+ monkeypatch.setattr(tools._shared, "_discovery_get_html", fake_raw)
+ monkeypatch.setattr(tools._shared, "_composition_get_stripped_html", fake_stripped)
+ monkeypatch.setattr(
+ tools.composition_capture, "_augment_composition_evidence_with_computed_obstruction_candidates", identity
+ )
evidence, html_error = await tools._capture_composition_evidence(
SimpleNamespace(), inspected_url="https://example.com/s", current_url="https://example.com/s"
diff --git a/tests/unit/test_copilot_discovery.py b/tests/unit/test_copilot_discovery.py
index 485aed846..94868694e 100644
--- a/tests/unit/test_copilot_discovery.py
+++ b/tests/unit/test_copilot_discovery.py
@@ -370,7 +370,7 @@ async def test_inspect_current_page_uses_existing_browser_page(monkeypatch: pyte
async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
return "https://www.example.com/results", "Results"
- monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_fallback_page_info)
result = await _inspect_page_for_composition_impl(ctx, "current_page")
@@ -397,7 +397,7 @@ async def test_post_run_current_page_inspection_budget_bypass_does_not_consume_c
async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
return "https://www.example.com/results", "Results"
- monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_fallback_page_info)
result = await _inspect_page_for_composition_impl(ctx, "current_page")
@@ -418,7 +418,7 @@ async def test_current_page_inspection_without_earned_interaction_is_not_click_r
async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
return "https://www.example.com/results", "Results"
- monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_fallback_page_info)
result = await _inspect_page_for_composition_impl(ctx, "current_page")
@@ -441,7 +441,7 @@ async def test_current_page_inspection_after_browser_action_is_click_reached_onc
async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
return "https://www.example.com/results", "Results"
- monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_fallback_page_info)
result = await _inspect_page_for_composition_impl(ctx, "current_page")
@@ -499,7 +499,7 @@ async def test_target_url_inspection_uses_visual_summary_for_generic_obstruction
"omissions": [],
}, None
- monkeypatch.setattr(tools_module, "_composition_summarize_screenshot", fake_visual_summary)
+ monkeypatch.setattr(tools_module.composition_capture, "_composition_summarize_screenshot", fake_visual_summary)
result = await _inspect_page_for_composition_impl(ctx, "https://www.example.com/search")
@@ -538,7 +538,7 @@ async def test_target_url_inspection_clears_pending_interaction_credit(
async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
return "https://www.example.com/results", "Results"
- monkeypatch.setattr(tools_module, "_fallback_page_info", fake_fallback_page_info)
+ monkeypatch.setattr(tools_module.composition_capture, "_fallback_page_info", fake_fallback_page_info)
current_result = await _inspect_page_for_composition_impl(ctx, "current_page")
diff --git a/tests/unit/test_copilot_evaluate_page_evidence.py b/tests/unit/test_copilot_evaluate_page_evidence.py
index 003028851..ce0a2cea0 100644
--- a/tests/unit/test_copilot_evaluate_page_evidence.py
+++ b/tests/unit/test_copilot_evaluate_page_evidence.py
@@ -136,7 +136,7 @@ async def test_target_url_inspection_does_not_navigate_away_from_interaction_evi
async def unexpected_navigate(*_: object, **__: object) -> dict[str, object]:
raise AssertionError("target_url inspection should not navigate away from reached evidence")
- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._discovery_navigate", unexpected_navigate)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.composition_capture._discovery_navigate", unexpected_navigate)
result = await _inspect_page_for_composition_impl(ctx, "https://example.test/")
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index ea6aec0ca..c46987a81 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.tools import (
_record_workflow_update_result,
_referenced_output_labels,
)
+from skyvern.forge.sdk.copilot.tools import frontier as frontier_module
from skyvern.forge.sdk.workflow.models.parameter import RESERVED_PARAMETER_KEYS
@@ -688,12 +689,10 @@ def test_plan_frontier_cold_start_no_old_definition_uses_first_requested() -> No
def test_plan_frontier_ambiguous_diff_falls_back_on_exception(monkeypatch: pytest.MonkeyPatch) -> None:
- from skyvern.forge.sdk.copilot import tools
-
def _blow_up(*args: object, **kwargs: object) -> set[str]:
raise RuntimeError("parse failure in diff")
- monkeypatch.setattr(tools, "_find_invalidated_labels", _blow_up)
+ monkeypatch.setattr(frontier_module, "_find_invalidated_labels", _blow_up)
old = _FakeDefinition([_FakeBlock("a", "navigation")])
new = _FakeDefinition([_FakeBlock("a", "navigation")])
@@ -1685,7 +1684,7 @@ def test_differ_exception_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None:
def _boom(*_args: object, **_kwargs: object) -> set[str]:
raise RuntimeError("differ blew up")
- monkeypatch.setattr(tools, "_find_invalidated_labels", _boom)
+ monkeypatch.setattr(frontier_module, "_find_invalidated_labels", _boom)
_invalidate_verified_state_on_edit(ctx, prior, new)
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 168f3ca73..967a87bc7 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -815,7 +815,7 @@ class TestBrowserInteractionObservationHooks:
async def fake_verify(*_args: object, **_kwargs: object) -> dict[str, object]:
return {"ok": False, "error": "field is still empty"}
- monkeypatch.setattr(tools_module, "_verify_scout_type_landed", fake_verify)
+ monkeypatch.setattr(tools_module.mcp_hooks, "_verify_scout_type_landed", fake_verify)
ctx = SimpleNamespace(
pending_browser_interaction_observation=None,
discovery_mcp_server=None,
@@ -983,7 +983,7 @@ class TestScoutedInteractionCapture:
async def passes(*_a: object, **_k: object) -> None:
return None
- monkeypatch.setattr(tools_module, "_verify_scout_type_landed", passes)
+ monkeypatch.setattr(tools_module.mcp_hooks, "_verify_scout_type_landed", passes)
ctx = self._ctx()
await tools_module._type_text_post_hook(
{"ok": True, "data": {"selector": "#q", "text_length": 8}},
@@ -1001,7 +1001,7 @@ class TestScoutedInteractionCapture:
async def fails(*_a: object, **_k: object) -> dict[str, object]:
return {"ok": False, "error": "field is still empty"}
- monkeypatch.setattr(tools_module, "_verify_scout_type_landed", fails)
+ monkeypatch.setattr(tools_module.mcp_hooks, "_verify_scout_type_landed", fails)
ctx = self._ctx()
await tools_module._type_text_post_hook(
{"ok": True, "data": {"selector": "#q", "text_length": 8}},
@@ -1037,7 +1037,7 @@ class TestScoutedInteractionCapture:
async def passes(*_a: object, **_k: object) -> None:
return None
- monkeypatch.setattr(tools_module, "_verify_scout_type_landed", passes)
+ monkeypatch.setattr(tools_module.mcp_hooks, "_verify_scout_type_landed", passes)
ctx = self._ctx()
await tools_module._type_text_post_hook(
{"ok": True, "data": {"selector": "#q", "text_length": 8}},
@@ -1064,11 +1064,12 @@ class TestScoutedInteractionCapture:
@pytest.mark.asyncio
async def test_capture_scout_source_url_reads_live_url(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.forge.sdk.copilot import tools as tools_module
+ from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
async def fake_url(_ctx: object) -> str:
return "https://example.com/product"
- monkeypatch.setattr(tools_module, "_live_working_page_url", fake_url)
+ monkeypatch.setattr(scouting_module, "_live_working_page_url", fake_url)
ctx = self._ctx()
await tools_module._capture_scout_source_url(ctx)
assert ctx.pending_scout_source_url == "https://example.com/product"
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 909f7b8b0..0a8bbabb5 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -1433,7 +1433,7 @@ async def test_update_workflow_rejects_raw_secret_before_processing(monkeypatch)
from skyvern.forge.sdk.copilot.tools import _update_workflow
process_mock = MagicMock()
- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", process_mock)
result = await _update_workflow(
{
@@ -1456,7 +1456,7 @@ workflow_definition:
def test_inline_replace_workflow_rejects_raw_secret_before_processing(monkeypatch) -> None:
process_mock = MagicMock()
- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", process_mock)
result = _fake_run_result(
{
"type": "REPLACE_WORKFLOW",
diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.py
index a2cf88ec5..bc114bc17 100644
--- a/tests/unit/test_copilot_screenshot_handling.py
+++ b/tests/unit/test_copilot_screenshot_handling.py
@@ -9,19 +9,19 @@ import pytest
def _install_mock_database(monkeypatch: pytest.MonkeyPatch, mock_db: Any) -> None:
- """Replace `skyvern.forge.sdk.copilot.tools.app` with a stub whose
+ """Replace `skyvern.forge.sdk.copilot.tools.run_execution.app` with a stub whose
DATABASE attribute is the provided mock.
`app` is an AppHolder that raises RuntimeError on attribute access until
start_forge_app() runs, so monkeypatching `app.DATABASE` directly fails at
resolve-time in uninitialized test environments.
"""
- import skyvern.forge.sdk.copilot.tools as tools_module
+ import skyvern.forge.sdk.copilot.tools.run_execution as run_execution_module
class _AppStub:
DATABASE = mock_db
- monkeypatch.setattr(tools_module, "app", _AppStub())
+ monkeypatch.setattr(run_execution_module, "app", _AppStub())
class TestIsValidPngBase64:
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
index f2e52e8b1..d3599b8ad 100644
--- a/tests/unit/test_copilot_session_injection.py
+++ b/tests/unit/test_copilot_session_injection.py
@@ -300,7 +300,9 @@ class TestScreenshotAdapter:
async def handler_lookup(_: object) -> object:
return handler
- monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._completion_verification_handler", handler_lookup)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler", handler_lookup
+ )
ctx = _make_ctx(
request_policy=RequestPolicy(
completion_criteria=[CompletionCriterion(id="c0", outcome="the requested item is visible")]
@@ -489,7 +491,7 @@ class TestUpdateWorkflowDirect:
mock_workflow.workflow_definition.blocks = [MagicMock(), MagicMock()]
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml",
lambda **kwargs: mock_workflow,
)
@@ -521,7 +523,7 @@ class TestUpdateWorkflowDirect:
mock_workflow.workflow_definition.blocks = []
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml",
lambda **kwargs: mock_workflow,
)
@@ -548,7 +550,7 @@ class TestUpdateWorkflowDirect:
raise _yaml.YAMLError("bad yaml")
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml",
raise_yaml_error,
)
@@ -576,7 +578,7 @@ class TestUpdateWorkflowDirect:
)
monkeypatch.setattr(
- "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml",
raise_validation_error,
)
diff --git a/tests/unit/test_copilot_task_block_rejection.py b/tests/unit/test_copilot_task_block_rejection.py
index 5f0dcd1c5..7fa4c7ade 100644
--- a/tests/unit/test_copilot_task_block_rejection.py
+++ b/tests/unit/test_copilot_task_block_rejection.py
@@ -276,7 +276,7 @@ async def test_update_workflow_rejects_new_task_block_and_emits_span() -> None:
submitted = _yaml({"block_type": "task", "label": "fill_contact_form", "navigation_goal": "do"})
ctx = _ctx(prior_yaml=None)
- with patch("skyvern.forge.sdk.copilot.tools._record_banned_block_reject_span") as mock_span:
+ with patch("skyvern.forge.sdk.copilot.tools.workflow_update._record_banned_block_reject_span") as mock_span:
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
assert result["ok"] is False
@@ -319,9 +319,11 @@ async def test_update_workflow_preserves_legacy_task_block_under_unchanged_label
fake_workflow.sequential_key = None
with (
- patch("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", return_value=fake_workflow),
- patch("skyvern.forge.sdk.copilot.tools._record_workflow_proxy_location_span") as mock_proxy_span,
- patch("skyvern.forge.sdk.copilot.tools.app") as mock_app,
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", return_value=fake_workflow),
+ patch(
+ "skyvern.forge.sdk.copilot.tools.workflow_update._record_workflow_proxy_location_span"
+ ) as mock_proxy_span,
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update.app") as mock_app,
):
mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=None)
mock_app.WORKFLOW_SERVICE.update_workflow_definition = AsyncMock()
@@ -362,8 +364,8 @@ async def test_update_workflow_allows_all_allowed_block_types() -> None:
setattr(fake_workflow, attr, None)
with (
- patch("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", return_value=fake_workflow),
- patch("skyvern.forge.sdk.copilot.tools.app") as mock_app,
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", return_value=fake_workflow),
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update.app") as mock_app,
):
mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=None)
mock_app.WORKFLOW_SERVICE.update_workflow_definition = AsyncMock()
diff --git a/tests/unit/test_copilot_tools_credential_id_misbinding.py b/tests/unit/test_copilot_tools_credential_id_misbinding.py
index 23b3cc5e5..bed098d1e 100644
--- a/tests/unit/test_copilot_tools_credential_id_misbinding.py
+++ b/tests/unit/test_copilot_tools_credential_id_misbinding.py
@@ -365,10 +365,10 @@ async def test_update_workflow_rejects_credential_id_in_navigation_goal() -> Non
with (
patch(
- "skyvern.forge.sdk.copilot.tools._credential_reference_validation_error",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._credential_reference_validation_error",
new=AsyncMock(return_value=None),
),
- patch("skyvern.forge.sdk.copilot.tools.app") as mock_app,
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update.app") as mock_app,
):
mock_app.WORKFLOW_SERVICE.update_workflow_definition = AsyncMock()
result = await _update_workflow({"workflow_yaml": submitted}, _ctx())
@@ -402,11 +402,11 @@ async def test_update_workflow_allows_credential_id_in_credential_parameter_slot
with (
patch(
- "skyvern.forge.sdk.copilot.tools._credential_reference_validation_error",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._credential_reference_validation_error",
new=AsyncMock(return_value=None),
),
patch(
- "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ "skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml",
return_value=MagicMock(
title="Sign in",
description=None,
@@ -426,10 +426,11 @@ async def test_update_workflow_allows_credential_id_in_credential_parameter_slot
),
),
patch(
- "skyvern.forge.sdk.copilot.tools.resolve_copilot_created_by_stamp", new=AsyncMock(return_value="copilot")
+ "skyvern.forge.sdk.copilot.tools.workflow_update.resolve_copilot_created_by_stamp",
+ new=AsyncMock(return_value="copilot"),
),
- patch("skyvern.forge.sdk.copilot.tools._record_workflow_proxy_location_span"),
- patch("skyvern.forge.sdk.copilot.tools.app") as mock_app,
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update._record_workflow_proxy_location_span"),
+ patch("skyvern.forge.sdk.copilot.tools.workflow_update.app") as mock_app,
):
mock_app.WORKFLOW_SERVICE.update_workflow_definition = AsyncMock()
mock_app.DATABASE = MagicMock()
diff --git a/tests/unit/test_copilot_turn_intent_tool_gate.py b/tests/unit/test_copilot_turn_intent_tool_gate.py
index f1c2a8159..8aeb0f345 100644
--- a/tests/unit/test_copilot_turn_intent_tool_gate.py
+++ b/tests/unit/test_copilot_turn_intent_tool_gate.py
@@ -305,7 +305,7 @@ async def test_update_workflow_stops_before_persisting_for_answer_only_intent()
)
ctx = _ctx(intent)
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.workflow_update.app") as mock_app:
mock_app.WORKFLOW_SERVICE.update_workflow_definition = AsyncMock()
result = await _update_workflow({"workflow_yaml": ctx.workflow_yaml}, ctx)
@@ -327,7 +327,7 @@ async def test_request_policy_refusal_wins_even_when_turn_intent_allows_update()
)
ctx = _ctx(intent, RequestPolicy(allow_update_workflow=False, allow_run_blocks=False))
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.workflow_update.app") as mock_app:
mock_app.WORKFLOW_SERVICE.update_workflow_definition = AsyncMock()
result = await _update_workflow({"workflow_yaml": ctx.workflow_yaml}, ctx)
@@ -515,7 +515,7 @@ async def test_get_run_results_defaults_to_successful_same_turn_run() -> None:
)
run = SimpleNamespace(workflow_run_id="wr_completed_this_turn", workflow_permanent_id="wfp-1", status="completed")
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.run_execution.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=run)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[])
mock_app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id = AsyncMock()
@@ -535,7 +535,7 @@ async def test_get_run_results_defaults_to_latest_same_turn_run_when_no_success(
)
run = SimpleNamespace(workflow_run_id="wr_failed_this_turn", workflow_permanent_id="wfp-1", status="canceled")
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.run_execution.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=run)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[])
mock_app.WORKFLOW_SERVICE.get_workflow_runs_for_workflow_permanent_id = AsyncMock()
@@ -571,7 +571,7 @@ async def test_get_run_results_rejects_explicit_run_from_other_workflow() -> Non
)
run = SimpleNamespace(workflow_run_id="wr_other", workflow_permanent_id="wfp-other", status="failed")
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.run_execution.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=run)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock()
result = await _get_run_results({"workflow_run_id": "wr_other"}, ctx)
@@ -585,7 +585,7 @@ async def test_get_run_results_uses_pending_reconciliation_run_when_id_omitted()
ctx = _ctx(TurnIntent(mode=TurnIntentMode.UNKNOWN), pending_reconciliation_run_id="wr_pending")
run = SimpleNamespace(workflow_run_id="wr_pending", workflow_permanent_id="wfp-1", status="failed")
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.run_execution.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=run)
mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(return_value=[])
result = await _get_run_results({}, ctx)
@@ -602,7 +602,7 @@ async def test_get_run_results_uses_pending_reconciliation_run_when_id_omitted()
async def test_get_run_results_rejects_different_run_while_reconciliation_pending() -> None:
ctx = _ctx(TurnIntent(mode=TurnIntentMode.UNKNOWN), pending_reconciliation_run_id="wr_pending")
- with patch("skyvern.forge.sdk.copilot.tools.app") as mock_app:
+ with patch("skyvern.forge.sdk.copilot.tools.run_execution.app") as mock_app:
mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock()
result = await _get_run_results({"workflow_run_id": "wr_other"}, ctx)
diff --git a/tests/unit/test_fetch_recording_urls.py b/tests/unit/test_fetch_recording_urls.py
new file mode 100644
index 000000000..e376da90b
--- /dev/null
+++ b/tests/unit/test_fetch_recording_urls.py
@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.schemas.files import FileInfo
+from skyvern.forge.sdk.workflow.service import WorkflowService
+
+
+def _workflow_run() -> SimpleNamespace:
+ return SimpleNamespace(
+ browser_session_id="pbs_1",
+ started_at=datetime(2026, 6, 4, 16, 0, tzinfo=UTC),
+ finished_at=datetime(2026, 6, 4, 16, 30, tzinfo=UTC),
+ organization_id="o_1",
+ workflow_run_id="wr_1",
+ )
+
+
+def _clip_artifact() -> SimpleNamespace:
+ return SimpleNamespace(artifact_id="a_clip", uri="s3://b/browser_sessions/pbs_1/run_recordings/wr_1/wr_1.mp4")
+
+
+async def _fetch(workflow_run: SimpleNamespace, task_v2: object | None = None) -> tuple[list[str], bool]:
+ # _fetch_recording_urls does not use self; a bare object stands in.
+ return await WorkflowService._fetch_recording_urls(object(), workflow_run, task_v2, "o_1")
+
+
+async def test_prefers_run_scoped_clip_over_session_recording(monkeypatch) -> None:
+ list_clips = AsyncMock(return_value=[_clip_artifact()])
+ get_session = AsyncMock(
+ return_value=[FileInfo(url="session_url", checksum=None, filename=None, modified_at=_workflow_run().started_at)]
+ )
+ monkeypatch.setattr(app.DATABASE.artifacts, "list_artifacts_for_run_by_type", list_clips, raising=False)
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "is_recording_archived", AsyncMock(return_value=False), raising=False)
+ monkeypatch.setattr(
+ app.ARTIFACT_MANAGER, "get_share_links_with_bundle_support", AsyncMock(return_value=["clip_url"]), raising=False
+ )
+ monkeypatch.setattr(app.STORAGE, "get_shared_recordings_in_browser_session", get_session, raising=False)
+
+ urls, archived = await _fetch(_workflow_run())
+
+ assert urls == ["clip_url"]
+ assert archived is False
+ get_session.assert_not_awaited() # the multi-hour session recording is not consulted when a clip exists
+
+
+async def test_falls_back_to_session_window_when_no_clip(monkeypatch) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.artifacts, "list_artifacts_for_run_by_type", AsyncMock(return_value=[]), raising=False
+ )
+ in_window = FileInfo(
+ url="session_url", checksum=None, filename=None, modified_at=datetime(2026, 6, 4, 16, 20, tzinfo=UTC)
+ )
+ monkeypatch.setattr(
+ app.STORAGE, "get_shared_recordings_in_browser_session", AsyncMock(return_value=[in_window]), raising=False
+ )
+
+ urls, archived = await _fetch(_workflow_run())
+
+ assert urls == ["session_url"]
+ assert archived is False
+
+
+async def test_unbounded_fallback_serves_late_session_recording(monkeypatch) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.artifacts, "list_artifacts_for_run_by_type", AsyncMock(return_value=[]), raising=False
+ )
+ # Finalized ~2h after the window upper bound (16:45) — bounded window misses it, unbounded recovers it.
+ late = FileInfo(url="late_url", checksum=None, filename=None, modified_at=datetime(2026, 6, 4, 18, 45, tzinfo=UTC))
+ monkeypatch.setattr(
+ app.STORAGE, "get_shared_recordings_in_browser_session", AsyncMock(return_value=[late]), raising=False
+ )
+
+ urls, _archived = await _fetch(_workflow_run())
+
+ assert urls == ["late_url"]
+
+
+async def test_archived_run_clip_reports_archived_without_session_fallback(monkeypatch) -> None:
+ monkeypatch.setattr(
+ app.DATABASE.artifacts,
+ "list_artifacts_for_run_by_type",
+ AsyncMock(return_value=[_clip_artifact()]),
+ raising=False,
+ )
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "is_recording_archived", AsyncMock(return_value=True), raising=False)
+ get_session = AsyncMock(return_value=[])
+ monkeypatch.setattr(app.STORAGE, "get_shared_recordings_in_browser_session", get_session, raising=False)
+
+ urls, archived = await _fetch(_workflow_run())
+
+ assert urls == []
+ assert archived is True
+ get_session.assert_not_awaited()
+
+
+def _legacy_artifact() -> SimpleNamespace:
+ # A run's own (non-clip) recording — no /run_recordings/ segment.
+ return SimpleNamespace(artifact_id="a_legacy", uri="s3://b/v1/prod/o_1/wr_1/step/recording.webm")
+
+
+async def test_legacy_run_recording_does_not_preempt_session_recording(monkeypatch) -> None:
+ # A non-clip run recording (e.g. an in-progress per-step snapshot) must NOT be served over
+ # the finalized session recording for a browser-session run (Codex P2).
+ monkeypatch.setattr(
+ app.DATABASE.artifacts,
+ "list_artifacts_for_run_by_type",
+ AsyncMock(return_value=[_legacy_artifact()]),
+ raising=False,
+ )
+ in_window = FileInfo(
+ url="session_url", checksum=None, filename=None, modified_at=datetime(2026, 6, 4, 16, 20, tzinfo=UTC)
+ )
+ monkeypatch.setattr(
+ app.STORAGE, "get_shared_recordings_in_browser_session", AsyncMock(return_value=[in_window]), raising=False
+ )
+ get_links = AsyncMock(return_value=["legacy_url"])
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "get_share_links_with_bundle_support", get_links, raising=False)
+
+ urls, _archived = await _fetch(_workflow_run())
+
+ assert urls == ["session_url"]
+ get_links.assert_not_awaited() # the legacy run recording is not served
+
+
+async def test_legacy_run_recording_served_for_non_session_run(monkeypatch) -> None:
+ # A non-session task run's own recording is the last resort (keeps normal task recordings working).
+ monkeypatch.setattr(
+ app.DATABASE.artifacts,
+ "list_artifacts_for_run_by_type",
+ AsyncMock(return_value=[_legacy_artifact()]),
+ raising=False,
+ )
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "is_recording_archived", AsyncMock(return_value=False), raising=False)
+ monkeypatch.setattr(
+ app.ARTIFACT_MANAGER,
+ "get_share_links_with_bundle_support",
+ AsyncMock(return_value=["legacy_url"]),
+ raising=False,
+ )
+ get_session = AsyncMock(return_value=[])
+ monkeypatch.setattr(app.STORAGE, "get_shared_recordings_in_browser_session", get_session, raising=False)
+
+ run = _workflow_run()
+ run.browser_session_id = None # non-session run
+
+ urls, _archived = await _fetch(run)
+
+ assert urls == ["legacy_url"]
+ get_session.assert_not_awaited() # no browser session to fall back to
diff --git a/tests/unit/test_gcp_credential_vault_service.py b/tests/unit/test_gcp_credential_vault_service.py
new file mode 100644
index 000000000..42e004cf8
--- /dev/null
+++ b/tests/unit/test_gcp_credential_vault_service.py
@@ -0,0 +1,343 @@
+import json
+from types import SimpleNamespace
+from typing import cast
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from google.api_core.exceptions import AlreadyExists, Conflict, NotFound
+from google.cloud import secretmanager_v1
+
+from skyvern.config import settings
+from skyvern.forge.sdk.api.real_gcp import RealAsyncGcpSecretManagerClient
+from skyvern.forge.sdk.schemas.credentials import (
+ CreateCredentialRequest,
+ Credential,
+ CredentialType,
+ CredentialVaultType,
+ CreditCardCredential,
+ PasswordCredential,
+ SecretCredential,
+ TotpType,
+)
+from skyvern.forge.sdk.services.credential.gcp_credential_vault_service import GcpCredentialVaultService
+
+TEST_PROJECT = "test-project"
+TEST_ORG = "o_123"
+
+
+def _service() -> tuple[GcpCredentialVaultService, AsyncMock]:
+ """Return a service plus the mock vault client it delegates to.
+
+ The client is an AsyncMock (dynamic attributes) so its methods can be
+ stubbed without tripping mypy's method-assign check on the Protocol type.
+ """
+ client = AsyncMock()
+ return GcpCredentialVaultService(client=client, project_id=TEST_PROJECT), client
+
+
+def _db_credential(**fields: object) -> Credential:
+ # Only a handful of attributes are read by the service; a SimpleNamespace
+ # stands in for a full Credential row.
+ return cast(Credential, SimpleNamespace(**fields))
+
+
+class TestGcpSecretItemCreation:
+ async def test_create_password_secret_item(self) -> None:
+ svc, client = _service()
+ client.create_or_update_secret = AsyncMock(return_value="ret-id")
+
+ item_id = await svc._create_gcp_secret_item(
+ organization_id=TEST_ORG,
+ credential=PasswordCredential(username="[email protected]", password="pw", totp="JBSWY3DPEHPK3PXP"),
+ )
+
+ assert item_id == "ret-id"
+ call = client.create_or_update_secret.call_args.kwargs
+ assert call["project_id"] == TEST_PROJECT
+ assert call["secret_id"].startswith(f"{settings.GCP_CREDENTIAL_VAULT_PREFIX}{TEST_ORG}-")
+ assert json.loads(call["value"]) == {
+ "type": "password",
+ "password": "pw",
+ "username": "[email protected]",
+ "totp": "JBSWY3DPEHPK3PXP",
+ }
+
+ async def test_create_credit_card_secret_item(self) -> None:
+ svc, client = _service()
+ client.create_or_update_secret = AsyncMock(return_value="ret-id")
+
+ await svc._create_gcp_secret_item(
+ organization_id=TEST_ORG,
+ credential=CreditCardCredential(
+ card_number="4111111111111111",
+ card_cvv="123",
+ card_exp_month="12",
+ card_exp_year="2030",
+ card_brand="visa",
+ card_holder_name="John Doe",
+ ),
+ )
+
+ payload = json.loads(client.create_or_update_secret.call_args.kwargs["value"])
+ assert payload["type"] == "credit_card"
+ assert payload["card_number"] == "4111111111111111"
+ assert payload["card_holder_name"] == "John Doe"
+
+ async def test_create_secret_item_excludes_none_label(self) -> None:
+ svc, client = _service()
+ client.create_or_update_secret = AsyncMock(return_value="ret-id")
+
+ await svc._create_gcp_secret_item(
+ organization_id=TEST_ORG,
+ credential=SecretCredential(secret_value="sk-abc123"),
+ )
+
+ payload = json.loads(client.create_or_update_secret.call_args.kwargs["value"])
+ assert payload == {"type": "secret", "secret_value": "sk-abc123"} # secret_label dropped by exclude_none
+
+ async def test_create_rejects_unsafe_organization_id(self) -> None:
+ svc, client = _service()
+ client.create_or_update_secret = AsyncMock(return_value="ret-id")
+
+ with pytest.raises(ValueError, match="not valid for Secret Manager"):
+ await svc._create_gcp_secret_item(
+ organization_id="org with spaces/and+slashes",
+ credential=SecretCredential(secret_value="sk-abc123"),
+ )
+
+ client.create_or_update_secret.assert_not_called()
+
+ async def test_update_reuses_item_id(self) -> None:
+ svc, client = _service()
+ client.create_or_update_secret = AsyncMock(return_value="sid")
+
+ await svc._update_gcp_secret_item(
+ item_id="existing-sid",
+ credential=PasswordCredential(username="u", password="p"),
+ )
+
+ assert client.create_or_update_secret.call_args.kwargs["secret_id"] == "existing-sid"
+
+
+class TestGetCredentialItem:
+ async def test_get_password_credential(self) -> None:
+ svc, client = _service()
+ client.get_secret = AsyncMock(
+ return_value=json.dumps({"type": "password", "username": "u", "password": "p", "totp": "T"})
+ )
+ db_cred = _db_credential(item_id="sid", totp_type=TotpType.NONE, name="My Login")
+
+ item = await svc.get_credential_item(db_cred)
+
+ assert item.credential_type == CredentialType.PASSWORD
+ assert item.item_id == "sid"
+ assert item.name == "My Login"
+ assert isinstance(item.credential, PasswordCredential)
+ assert item.credential.username == "u"
+ assert item.credential.password == "p"
+
+ async def test_get_credit_card_credential(self) -> None:
+ svc, client = _service()
+ client.get_secret = AsyncMock(
+ return_value=json.dumps(
+ {
+ "type": "credit_card",
+ "card_number": "4111111111111111",
+ "card_cvv": "123",
+ "card_exp_month": "12",
+ "card_exp_year": "2030",
+ "card_brand": "visa",
+ "card_holder_name": "John Doe",
+ }
+ )
+ )
+ db_cred = _db_credential(item_id="sid", totp_type=TotpType.NONE, name="My Card")
+
+ item = await svc.get_credential_item(db_cred)
+
+ assert item.credential_type == CredentialType.CREDIT_CARD
+ assert isinstance(item.credential, CreditCardCredential)
+ assert item.credential.card_number == "4111111111111111"
+
+ async def test_get_secret_credential(self) -> None:
+ svc, client = _service()
+ client.get_secret = AsyncMock(
+ return_value=json.dumps({"type": "secret", "secret_value": "sk-abc123", "secret_label": "API key"})
+ )
+ db_cred = _db_credential(item_id="sid", totp_type=TotpType.NONE, name="My Secret")
+
+ item = await svc.get_credential_item(db_cred)
+
+ assert item.credential_type == CredentialType.SECRET
+ assert isinstance(item.credential, SecretCredential)
+ assert item.credential.secret_value == "sk-abc123"
+ assert item.credential.secret_label == "API key"
+
+ async def test_get_nonexistent_credential_raises(self) -> None:
+ svc, client = _service()
+ client.get_secret = AsyncMock(return_value=None)
+ db_cred = _db_credential(item_id="missing", totp_type=TotpType.NONE, name="x")
+
+ with pytest.raises(ValueError, match="GCP Credential Vault secret not found"):
+ await svc.get_credential_item(db_cred)
+
+
+class TestCreateAndDeleteCredential:
+ async def test_create_credential_uses_gcp_vault_type(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ import skyvern.forge.sdk.services.credential.credential_vault_service as base_mod
+
+ fake_app = MagicMock()
+ fake_app.DATABASE.credentials.create_credential = AsyncMock(return_value="DB_CRED")
+ monkeypatch.setattr(base_mod, "app", fake_app)
+
+ svc, client = _service()
+ client.create_or_update_secret = AsyncMock(return_value="sid-xyz")
+ req = CreateCredentialRequest(
+ name="My Login",
+ credential_type=CredentialType.PASSWORD,
+ credential={"username": "u", "password": "p"},
+ )
+
+ result = await svc.create_credential(TEST_ORG, req)
+
+ assert result == "DB_CRED"
+ client.create_or_update_secret.assert_awaited_once()
+ kwargs = fake_app.DATABASE.credentials.create_credential.call_args.kwargs
+ assert kwargs["vault_type"] == CredentialVaultType.GCP
+ assert kwargs["item_id"] == "sid-xyz"
+
+ async def test_delete_credential_deletes_db_and_secret(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ import skyvern.forge.sdk.services.credential.gcp_credential_vault_service as gcp_mod
+
+ fake_app = MagicMock()
+ fake_app.DATABASE.credentials.delete_credential = AsyncMock()
+ monkeypatch.setattr(gcp_mod, "app", fake_app)
+
+ order: list[str] = []
+ fake_app.DATABASE.credentials.delete_credential = AsyncMock(side_effect=lambda *a: order.append("db"))
+ svc, client = _service()
+ client.delete_secret = AsyncMock(side_effect=lambda **k: order.append("secret"))
+ cred = _db_credential(credential_id="c1", organization_id="o_1", item_id="sid")
+
+ await svc.delete_credential(cred)
+
+ fake_app.DATABASE.credentials.delete_credential.assert_awaited_once_with("c1", "o_1")
+ client.delete_secret.assert_awaited_once_with(secret_id="sid", project_id=TEST_PROJECT)
+ # Secret must be deleted BEFORE the DB row so a vault error can't orphan it.
+ assert order == ["secret", "db"]
+
+
+class TestRealSecretManagerClient:
+ async def test_create_or_update_creates_then_adds_version(self) -> None:
+ mock_client = MagicMock()
+ mock_client.list_secret_versions.return_value = []
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+
+ ret = await c.create_or_update_secret(secret_id="sid", project_id="proj", value="hello")
+
+ assert ret == "sid"
+ create_req = mock_client.create_secret.call_args.kwargs["request"]
+ assert create_req["parent"] == "projects/proj"
+ assert create_req["secret_id"] == "sid"
+ add_req = mock_client.add_secret_version.call_args.kwargs["request"]
+ assert add_req["parent"] == "projects/proj/secrets/sid"
+ assert add_req["payload"]["data"] == b"hello"
+
+ @pytest.mark.parametrize("existing_exc", [Conflict("exists"), AlreadyExists("exists")])
+ async def test_create_or_update_ignores_existing_secret(self, existing_exc: Exception) -> None:
+ # REST transport raises generic Conflict (409); gRPC raises AlreadyExists.
+ # Both mean "secret already exists" and must fall through to add a version.
+ mock_client = MagicMock()
+ mock_client.create_secret.side_effect = existing_exc
+ mock_client.list_secret_versions.return_value = []
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+
+ ret = await c.create_or_update_secret(secret_id="sid", project_id="proj", value="v")
+
+ assert ret == "sid"
+ mock_client.add_secret_version.assert_called_once() # still adds a new version
+
+ async def test_create_or_update_revokes_prior_versions(self) -> None:
+ # After adding the new version, every other non-destroyed version must be
+ # destroyed so a rotated/leaked credential can't be read back by version id.
+ mock_client = MagicMock()
+ added = MagicMock()
+ added.name = "projects/proj/secrets/sid/versions/2"
+ mock_client.add_secret_version.return_value = added
+
+ prior_enabled = MagicMock()
+ prior_enabled.name = "projects/proj/secrets/sid/versions/1"
+ prior_enabled.state = secretmanager_v1.SecretVersion.State.ENABLED
+ already_destroyed = MagicMock()
+ already_destroyed.name = "projects/proj/secrets/sid/versions/0"
+ already_destroyed.state = secretmanager_v1.SecretVersion.State.DESTROYED
+ mock_client.list_secret_versions.return_value = [prior_enabled, already_destroyed, added]
+
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+ await c.create_or_update_secret(secret_id="sid", project_id="proj", value="v2")
+
+ # Only the prior ENABLED version is destroyed; the just-added and the
+ # already-destroyed versions are skipped.
+ destroyed_names = [call.kwargs["request"]["name"] for call in mock_client.destroy_secret_version.call_args_list]
+ assert destroyed_names == ["projects/proj/secrets/sid/versions/1"]
+
+ async def test_create_or_update_keeps_concurrently_added_newer_versions(self) -> None:
+ # If a concurrent update added a newer version between our add and our
+ # list, destroying it would race both writers into destroying each
+ # other's version, leaving every version destroyed. Only versions
+ # numerically older than ours may be destroyed (last-writer-wins).
+ mock_client = MagicMock()
+ added = MagicMock()
+ added.name = "projects/proj/secrets/sid/versions/2"
+ mock_client.add_secret_version.return_value = added
+
+ prior_enabled = MagicMock()
+ prior_enabled.name = "projects/proj/secrets/sid/versions/1"
+ prior_enabled.state = secretmanager_v1.SecretVersion.State.ENABLED
+ concurrent_newer = MagicMock()
+ concurrent_newer.name = "projects/proj/secrets/sid/versions/3"
+ concurrent_newer.state = secretmanager_v1.SecretVersion.State.ENABLED
+ mock_client.list_secret_versions.return_value = [prior_enabled, added, concurrent_newer]
+
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+ await c.create_or_update_secret(secret_id="sid", project_id="proj", value="v2")
+
+ destroyed_names = [call.kwargs["request"]["name"] for call in mock_client.destroy_secret_version.call_args_list]
+ assert destroyed_names == ["projects/proj/secrets/sid/versions/1"]
+
+ async def test_get_secret_returns_decoded(self) -> None:
+ mock_client = MagicMock()
+ resp = MagicMock()
+ resp.payload.data = b"secret-bytes"
+ mock_client.access_secret_version.return_value = resp
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+
+ assert await c.get_secret("sid", "proj") == "secret-bytes"
+ name = mock_client.access_secret_version.call_args.kwargs["request"]["name"]
+ assert name == "projects/proj/secrets/sid/versions/latest"
+
+ async def test_get_secret_not_found_returns_none(self) -> None:
+ mock_client = MagicMock()
+ mock_client.access_secret_version.side_effect = NotFound("nope")
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+
+ assert await c.get_secret("sid", "proj") is None
+
+ async def test_delete_secret(self) -> None:
+ mock_client = MagicMock()
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+
+ await c.delete_secret("sid", "proj")
+
+ assert mock_client.delete_secret.call_args.kwargs["request"]["name"] == "projects/proj/secrets/sid"
+
+ async def test_delete_secret_not_found_swallowed(self) -> None:
+ mock_client = MagicMock()
+ mock_client.delete_secret.side_effect = NotFound("nope")
+ c = RealAsyncGcpSecretManagerClient(client=mock_client)
+
+ await c.delete_secret("sid", "proj") # must not raise
diff --git a/tests/unit/test_onboarding_milestone_hooks.py b/tests/unit/test_onboarding_milestone_hooks.py
index 958f187ff..20e20f2d6 100644
--- a/tests/unit/test_onboarding_milestone_hooks.py
+++ b/tests/unit/test_onboarding_milestone_hooks.py
@@ -3,12 +3,16 @@
from __future__ import annotations
import asyncio
+import inspect
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from fastapi import params as fastapi_params
from skyvern.forge.agent_functions import AgentFunction
+from skyvern.forge.sdk.services import org_auth_service
+from skyvern.schemas.workflows import WorkflowCreateYAMLRequest, WorkflowDefinitionYAML, WorkflowRequest
@pytest.fixture()
@@ -35,6 +39,14 @@ class TestBaseAgentFunctionNoOps:
)
assert result is None
+ @pytest.mark.asyncio
+ async def test_validate_user_organization_membership_unknown(self, base_agent_fn: AgentFunction) -> None:
+ result = await base_agent_fn.validate_user_organization_membership(
+ user_id="u_test",
+ organization_id="o_test",
+ )
+ assert result is None
+
class TestWorkflowSaveHookFires:
"""update_workflow_definition fires on_workflow_saved as a background task."""
@@ -106,6 +118,189 @@ class TestWorkflowSaveHookFires:
)
+def _yaml_request(title: str = "Funnel Workflow") -> WorkflowCreateYAMLRequest:
+ return WorkflowCreateYAMLRequest(
+ title=title,
+ workflow_definition=WorkflowDefinitionYAML(parameters=[], blocks=[]),
+ )
+
+
+def _stubbed_workflow_service() -> tuple[object, MagicMock]:
+ from skyvern.forge.sdk.workflow.service import WorkflowService
+
+ saved_workflow = MagicMock()
+ saved_workflow.workflow_id = "wf_new"
+
+ svc = WorkflowService.__new__(WorkflowService)
+ svc.create_workflow = AsyncMock(return_value=saved_workflow)
+ svc.make_workflow_definition = AsyncMock(return_value=MagicMock())
+ svc.validate_workflow_block_graph = MagicMock()
+ svc._validate_payload_templates = MagicMock()
+ svc.update_workflow_definition = AsyncMock(return_value=saved_workflow)
+ svc.maybe_delete_cached_code = AsyncMock()
+ return svc, saved_workflow
+
+
+class TestCreateWorkflowFromRequestThreadsAttribution:
+ """create_workflow_from_request must forward the actor so on_workflow_saved sees it."""
+
+ @pytest.mark.asyncio
+ async def test_create_flow_threads_attribution(self) -> None:
+ svc, _ = _stubbed_workflow_service()
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+
+ await svc.create_workflow_from_request(
+ organization=organization,
+ request=_yaml_request(),
+ created_by="u_456",
+ edited_by="u_456",
+ )
+
+ create_kwargs = svc.create_workflow.await_args.kwargs
+ assert create_kwargs.get("created_by") == "u_456"
+ assert create_kwargs.get("edited_by") == "u_456"
+ # on_workflow_saved fires inside update_workflow_definition and needs the actor.
+ update_kwargs = svc.update_workflow_definition.await_args.kwargs
+ assert update_kwargs.get("edited_by") == "u_456"
+
+ @pytest.mark.asyncio
+ async def test_update_flow_threads_attribution(self) -> None:
+ svc, _ = _stubbed_workflow_service()
+ existing = MagicMock()
+ existing.version = 1
+ existing.cdp_connect_headers = None
+ existing.max_elapsed_time_minutes = None
+ existing.folder_id = None
+ existing.code_version = None
+ existing.workflow_permanent_id = "wpid_1"
+ svc.get_workflow_by_permanent_id = AsyncMock(return_value=existing)
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+
+ await svc.create_workflow_from_request(
+ organization=organization,
+ request=_yaml_request(),
+ workflow_permanent_id="wpid_1",
+ created_by="u_456",
+ edited_by="u_456",
+ )
+
+ create_kwargs = svc.create_workflow.await_args.kwargs
+ assert create_kwargs.get("created_by") == "u_456"
+ assert create_kwargs.get("edited_by") == "u_456"
+ update_kwargs = svc.update_workflow_definition.await_args.kwargs
+ assert update_kwargs.get("edited_by") == "u_456"
+
+
+class TestWorkflowRoutesThreadUser:
+ """The UI create/update routes must resolve the caller and stamp created_by/edited_by."""
+
+ @pytest.mark.asyncio
+ async def test_create_workflow_route_passes_user(self) -> None:
+ from skyvern.forge.sdk.routes.agent_protocol import create_workflow
+
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+ data = WorkflowRequest(json_definition=_yaml_request())
+
+ with patch("skyvern.forge.sdk.routes.agent_protocol.app") as mock_app:
+ mock_app.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=MagicMock())
+ await create_workflow(
+ data=data,
+ folder_id=None,
+ current_org=organization,
+ user_id="u_456",
+ )
+ kwargs = mock_app.WORKFLOW_SERVICE.create_workflow_from_request.await_args.kwargs
+
+ assert kwargs.get("created_by") == "u_456"
+ assert kwargs.get("edited_by") == "u_456"
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_route_passes_user(self) -> None:
+ from skyvern.forge.sdk.routes.agent_protocol import update_workflow
+
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+ data = WorkflowRequest(json_definition=_yaml_request())
+
+ with patch("skyvern.forge.sdk.routes.agent_protocol.app") as mock_app:
+ mock_app.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=MagicMock())
+ await update_workflow(
+ data=data,
+ workflow_id="wpid_1",
+ current_org=organization,
+ user_id="u_456",
+ )
+ kwargs = mock_app.WORKFLOW_SERVICE.create_workflow_from_request.await_args.kwargs
+
+ assert kwargs.get("created_by") == "u_456"
+ assert kwargs.get("edited_by") == "u_456"
+
+ @pytest.mark.asyncio
+ async def test_create_workflow_legacy_route_passes_user(self) -> None:
+ from skyvern.forge.sdk.routes.agent_protocol import create_workflow_legacy
+
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+ raw_request = MagicMock()
+ raw_request.body = AsyncMock(
+ return_value=b"title: Funnel Workflow\nworkflow_definition:\n parameters: []\n blocks: []\n"
+ )
+
+ with patch("skyvern.forge.sdk.routes.agent_protocol.app") as mock_app:
+ mock_app.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=MagicMock())
+ await create_workflow_legacy(
+ request=raw_request,
+ folder_id=None,
+ current_org=organization,
+ user_id="u_456",
+ )
+ kwargs = mock_app.WORKFLOW_SERVICE.create_workflow_from_request.await_args.kwargs
+
+ assert kwargs.get("created_by") == "u_456"
+ assert kwargs.get("edited_by") == "u_456"
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_legacy_route_passes_user(self) -> None:
+ from skyvern.forge.sdk.routes.agent_protocol import update_workflow_legacy
+
+ organization = MagicMock()
+ organization.organization_id = "o_123"
+ raw_request = MagicMock()
+ raw_request.body = AsyncMock(
+ return_value=b"title: Funnel Workflow\nworkflow_definition:\n parameters: []\n blocks: []\n"
+ )
+
+ with patch("skyvern.forge.sdk.routes.agent_protocol.app") as mock_app:
+ mock_app.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=MagicMock())
+ await update_workflow_legacy(
+ request=raw_request,
+ workflow_id="wpid_1",
+ current_org=organization,
+ user_id="u_456",
+ )
+ kwargs = mock_app.WORKFLOW_SERVICE.create_workflow_from_request.await_args.kwargs
+
+ assert kwargs.get("created_by") == "u_456"
+ assert kwargs.get("edited_by") == "u_456"
+
+ def test_routes_wire_fail_open_user_dependency(self) -> None:
+ from skyvern.forge.sdk.routes import agent_protocol
+
+ for route_fn in (
+ agent_protocol.create_workflow,
+ agent_protocol.create_workflow_legacy,
+ agent_protocol.update_workflow,
+ agent_protocol.update_workflow_legacy,
+ ):
+ user_param = inspect.signature(route_fn).parameters.get("user_id")
+ assert user_param is not None, f"{route_fn.__name__} is missing the user_id dependency"
+ assert isinstance(user_param.default, fastapi_params.Depends)
+ assert user_param.default.dependency is org_auth_service.get_current_user_id_or_none
+
+
class TestWorkflowRunCompleteHookFires:
"""_update_workflow_run_status fires on_workflow_run_completed for final statuses."""
diff --git a/tests/unit/test_org_auth_credential_clear_routes.py b/tests/unit/test_org_auth_credential_clear_routes.py
new file mode 100644
index 000000000..50e6e9d52
--- /dev/null
+++ b/tests/unit/test_org_auth_credential_clear_routes.py
@@ -0,0 +1,103 @@
+from collections.abc import Awaitable, Callable
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.routes import credentials as routes
+from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken
+
+
+class FakeOrganizationsRepository:
+ def __init__(self) -> None:
+ self.tokens: list[OrganizationAuthToken] = []
+ self.invalidate_calls: list[tuple[str, OrganizationAuthTokenType]] = []
+
+ async def invalidate_org_auth_tokens(self, organization_id: str, token_type: OrganizationAuthTokenType) -> None:
+ self.invalidate_calls.append((organization_id, token_type))
+ for token in self.tokens:
+ if token.organization_id == organization_id and token.token_type == token_type:
+ token.valid = False
+
+ async def get_valid_org_auth_token(self, organization_id: str, token_type: str) -> OrganizationAuthToken | None:
+ return next(
+ (
+ t
+ for t in self.tokens
+ if t.organization_id == organization_id and t.token_type.value == token_type and t.valid
+ ),
+ None,
+ )
+
+
+def _org() -> Organization:
+ now = datetime.now(timezone.utc)
+ return Organization(organization_id="o_test", organization_name="Test Org", created_at=now, modified_at=now)
+
+
+GetRoute = Callable[[Organization], Awaitable[object]]
+
+
+PROVIDERS: list[tuple[str, OrganizationAuthTokenType, GetRoute]] = [
+ ("onepassword", OrganizationAuthTokenType.onepassword_service_account, routes.get_onepassword_token),
+ ("bitwarden", OrganizationAuthTokenType.bitwarden_credential, routes.get_bitwarden_credential),
+ (
+ "azure_credential",
+ OrganizationAuthTokenType.azure_client_secret_credential,
+ routes.get_azure_client_secret_credential,
+ ),
+ (
+ "custom_credential",
+ OrganizationAuthTokenType.custom_credential_service,
+ routes.get_custom_credential_service_config,
+ ),
+]
+
+
+def fake_organizations(monkeypatch: pytest.MonkeyPatch) -> FakeOrganizationsRepository:
+ organizations = FakeOrganizationsRepository()
+ monkeypatch.setattr(routes.app, "DATABASE", SimpleNamespace(organizations=organizations))
+ return organizations
+
+
[email protected]("provider_path, token_type, get_route", PROVIDERS, ids=[provider[0] for provider in PROVIDERS])
+async def test_clear_org_auth_credential_invalidates_existing_and_get_404s(
+ fake_organizations: FakeOrganizationsRepository,
+ provider_path: str,
+ token_type: OrganizationAuthTokenType,
+ get_route: GetRoute,
+) -> None:
+ org = _org()
+ now = datetime.now(timezone.utc)
+ fake_organizations.tokens.append(
+ OrganizationAuthToken(
+ id="oat_test",
+ organization_id=org.organization_id,
+ token_type=token_type,
+ token="secret",
+ valid=True,
+ created_at=now,
+ modified_at=now,
+ )
+ )
+
+ response = await routes.clear_org_auth_credential(provider_path, org)
+
+ assert response.success is True
+ assert fake_organizations.invalidate_calls == [(org.organization_id, token_type)]
+ assert fake_organizations.tokens[0].valid is False
+ with pytest.raises(HTTPException) as exc_info:
+ await get_route(org)
+ assert exc_info.value.status_code == 404
+
+
+async def test_clear_org_auth_credential_succeeds_without_existing_token(
+ fake_organizations: FakeOrganizationsRepository,
+) -> None:
+ assert (await routes.clear_org_auth_credential("onepassword", _org())).success is True
+ assert len(fake_organizations.invalidate_calls) == 1
diff --git a/tests/unit/test_print_page_block_downloaded_files.py b/tests/unit/test_print_page_block_downloaded_files.py
index 002a35b2e..f14321bdc 100644
--- a/tests/unit/test_print_page_block_downloaded_files.py
+++ b/tests/unit/test_print_page_block_downloaded_files.py
@@ -170,10 +170,12 @@ async def test_print_page_block_filters_downloads_to_current_loop_iteration(
artifact_id="a_new",
)
+ # Baseline capture (block start) sees only the earlier iteration's file; the
+ # post-PDF read sees both. The block must scope its output to just the new one.
fake_app = SimpleNamespace(
STORAGE=SimpleNamespace(
save_downloaded_files=AsyncMock(),
- get_downloaded_files=AsyncMock(return_value=[prev_file, new_file]),
+ get_downloaded_files=AsyncMock(side_effect=[[prev_file], [prev_file, new_file]]),
),
)
monkeypatch.setattr(block_module, "app", fake_app)
diff --git a/tests/unit/test_run_recording_clips.py b/tests/unit/test_run_recording_clips.py
new file mode 100644
index 000000000..c6fd79016
--- /dev/null
+++ b/tests/unit/test_run_recording_clips.py
@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.artifact.storage import run_recording_clips
+from skyvern.forge.sdk.artifact.storage.run_recording_clips import sync_run_recording_clips
+
+# close time 19:00 with a 3h recording => the video spans 16:00 - 19:00 UTC.
+NOW = datetime(2026, 6, 4, 19, 0, 0, tzinfo=UTC)
+DURATION = 3 * 60 * 60
+
+
+def _run(run_id: str, started: datetime, finished: datetime | None) -> SimpleNamespace:
+ return SimpleNamespace(workflow_run_id=run_id, started_at=started, finished_at=finished)
+
+
+def _artifact(uri: str) -> SimpleNamespace:
+ return SimpleNamespace(artifact_id="a_x", uri=uri)
+
+
+@asynccontextmanager
+async def _fake_cut(src_path, *, start_seconds, duration_seconds):
+ yield "/tmp/clip.mp4"
+
+
+def _setup(monkeypatch, runs, existing_by_run=None, duration=DURATION, task_v2_by_run=None):
+ existing_by_run = existing_by_run or {}
+ task_v2_by_run = task_v2_by_run or {}
+ get_runs = AsyncMock(return_value=runs)
+ list_existing = AsyncMock(
+ side_effect=lambda run_id, organization_id, artifact_type: existing_by_run.get(run_id, [])
+ )
+ get_task_v2 = AsyncMock(side_effect=lambda workflow_run_id, organization_id: task_v2_by_run.get(workflow_run_id))
+ create_clip = AsyncMock(return_value="a_clip")
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_runs_for_browser_session", get_runs, raising=False)
+ monkeypatch.setattr(app.DATABASE.artifacts, "list_artifacts_for_run_by_type", list_existing, raising=False)
+ monkeypatch.setattr(app.DATABASE.observer, "get_task_v2_by_workflow_run_id", get_task_v2, raising=False)
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "create_run_recording_artifact", create_clip, raising=False)
+ monkeypatch.setattr(
+ run_recording_clips, "probe_media_duration_seconds", AsyncMock(return_value=duration), raising=False
+ )
+ monkeypatch.setattr(run_recording_clips, "cut_recording_segment", _fake_cut, raising=False)
+ monkeypatch.setattr(run_recording_clips, "calculate_sha256_for_file", lambda p: "sha", raising=False)
+ return get_runs, list_existing, create_clip
+
+
+async def _run_clips(upload_clip) -> None:
+ await sync_run_recording_clips(
+ organization_id="o_1",
+ browser_session_id="pbs_1",
+ source_path="/tmp/session.mp4",
+ upload_clip=upload_clip,
+ now=NOW,
+ )
+
+
+def _recording_upload():
+ uploaded: list[tuple[str, str, str]] = []
+
+ async def upload_clip(run_id: str, clip_path: str, filename: str) -> str:
+ uploaded.append((run_id, clip_path, filename))
+ return f"s3://b/run_recordings/{run_id}/{filename}"
+
+ return uploaded, upload_clip
+
+
+async def test_creates_run_scoped_clip_per_overlapping_run(monkeypatch) -> None:
+ runs = [
+ _run("wr_1", datetime(2026, 6, 4, 16, 5, tzinfo=UTC), datetime(2026, 6, 4, 16, 35, tzinfo=UTC)),
+ _run("wr_2", datetime(2026, 6, 4, 17, 0, tzinfo=UTC), datetime(2026, 6, 4, 17, 10, tzinfo=UTC)),
+ ]
+ _get, _list, create_clip = _setup(monkeypatch, runs)
+ uploaded, upload_clip = _recording_upload()
+
+ await _run_clips(upload_clip)
+
+ assert [u[0] for u in uploaded] == ["wr_1", "wr_2"]
+ assert create_clip.await_count == 2
+ created = {c.kwargs["run_id"]: c.kwargs for c in create_clip.await_args_list}
+ # Clip filename is stamped with the source-video stem ("session" from /tmp/session.mp4).
+ assert created["wr_1"]["uri"] == "s3://b/run_recordings/wr_1/wr_1-session.mp4"
+ assert created["wr_1"]["workflow_run_id"] == "wr_1"
+ assert created["wr_1"]["checksum"] == "sha"
+
+
+async def test_skips_run_that_already_has_a_clip(monkeypatch) -> None:
+ runs = [
+ _run("wr_1", datetime(2026, 6, 4, 16, 5, tzinfo=UTC), datetime(2026, 6, 4, 16, 35, tzinfo=UTC)),
+ _run("wr_2", datetime(2026, 6, 4, 17, 0, tzinfo=UTC), datetime(2026, 6, 4, 17, 10, tzinfo=UTC)),
+ ]
+ existing = {"wr_1": [_artifact("s3://b/browser_sessions/pbs_1/run_recordings/2026/wr_1/wr_1-session.mp4")]}
+ _get, _list, create_clip = _setup(monkeypatch, runs, existing)
+ uploaded, upload_clip = _recording_upload()
+
+ await _run_clips(upload_clip)
+
+ assert [u[0] for u in uploaded] == ["wr_2"]
+ assert create_clip.await_count == 1
+
+
+async def test_second_source_video_for_same_run_still_clipped(monkeypatch) -> None:
+ # A popup/secondary video is synced separately; a clip from the first source must not
+ # suppress clipping the second source for the same run (Codex P2).
+ runs = [_run("wr_1", datetime(2026, 6, 4, 16, 5, tzinfo=UTC), datetime(2026, 6, 4, 16, 35, tzinfo=UTC))]
+ existing = {"wr_1": [_artifact("s3://b/browser_sessions/pbs_1/run_recordings/2026/wr_1/wr_1-session.mp4")]}
+ _get, _list, create_clip = _setup(monkeypatch, runs, existing)
+ uploaded, upload_clip = _recording_upload()
+
+ # Second source video (a popup), distinct stem. Clips are always re-encoded to mp4.
+ await sync_run_recording_clips(
+ organization_id="o_1",
+ browser_session_id="pbs_1",
+ source_path="/tmp/popup.webm",
+ upload_clip=upload_clip,
+ now=NOW,
+ )
+
+ assert uploaded == [("wr_1", "/tmp/clip.mp4", "wr_1-popup.mp4")]
+ assert create_clip.await_count == 1
+
+
+async def test_does_not_skip_when_only_full_recording_exists(monkeypatch) -> None:
+ # A run's own (non-clip) RECORDING artifact must not suppress clip creation (issue 4).
+ runs = [_run("wr_1", datetime(2026, 6, 4, 16, 5, tzinfo=UTC), datetime(2026, 6, 4, 16, 35, tzinfo=UTC))]
+ existing = {"wr_1": [_artifact("s3://b/v1/prod/o_1/wr_1/step/recording.webm")]} # not a run_recordings/ clip
+ _get, _list, create_clip = _setup(monkeypatch, runs, existing)
+ uploaded, upload_clip = _recording_upload()
+
+ await _run_clips(upload_clip)
+
+ assert uploaded == [("wr_1", "/tmp/clip.mp4", "wr_1-session.mp4")]
+ assert create_clip.await_count == 1
+
+
+async def test_task_v2_clip_written_under_observer_cruise_id(monkeypatch) -> None:
+ # task_v2 runs are read by observer_cruise_id, so the clip must be scoped to it (issue 3).
+ runs = [_run("wr_1", datetime(2026, 6, 4, 16, 5, tzinfo=UTC), datetime(2026, 6, 4, 16, 35, tzinfo=UTC))]
+ task_v2 = {"wr_1": SimpleNamespace(observer_cruise_id="oc_1")}
+ _get, _list, create_clip = _setup(monkeypatch, runs, task_v2_by_run=task_v2)
+ _uploaded, upload_clip = _recording_upload()
+
+ await _run_clips(upload_clip)
+
+ create_clip.assert_awaited_once()
+ kwargs = create_clip.await_args.kwargs
+ assert kwargs["run_id"] == "oc_1" # read-path id
+ assert kwargs["workflow_run_id"] == "wr_1" # original run preserved
+
+
+async def test_skips_run_with_no_overlap(monkeypatch) -> None:
+ # Finished before the video (16:00) started — belongs to an earlier recording.
+ runs = [_run("wr_old", datetime(2026, 6, 4, 15, 0, tzinfo=UTC), datetime(2026, 6, 4, 15, 30, tzinfo=UTC))]
+ _get, _list, create_clip = _setup(monkeypatch, runs)
+ uploaded, upload_clip = _recording_upload()
+
+ await _run_clips(upload_clip)
+
+ assert uploaded == []
+ assert create_clip.await_count == 0
+
+
+async def test_no_duration_returns_early_without_enumerating_runs(monkeypatch) -> None:
+ get_runs, _list, create_clip = _setup(monkeypatch, [], duration=None)
+ await _run_clips(AsyncMock())
+ get_runs.assert_not_awaited()
+ create_clip.assert_not_awaited()
+
+
+async def test_one_run_failure_does_not_block_others(monkeypatch) -> None:
+ runs = [
+ _run("wr_boom", datetime(2026, 6, 4, 16, 5, tzinfo=UTC), datetime(2026, 6, 4, 16, 35, tzinfo=UTC)),
+ _run("wr_ok", datetime(2026, 6, 4, 17, 0, tzinfo=UTC), datetime(2026, 6, 4, 17, 10, tzinfo=UTC)),
+ ]
+ _get, _list, create_clip = _setup(monkeypatch, runs)
+ uploaded: list[str] = []
+
+ async def upload_clip(run_id: str, clip_path: str, filename: str) -> str:
+ if run_id == "wr_boom":
+ raise RuntimeError("upload exploded")
+ uploaded.append(run_id)
+ return f"s3://b/{run_id}.mp4"
+
+ await _run_clips(upload_clip)
+
+ assert uploaded == ["wr_ok"]
+ assert create_clip.await_count == 1
diff --git a/tests/unit/test_taskv2_block_downloaded_files.py b/tests/unit/test_taskv2_block_downloaded_files.py
index 24e3a4963..cffe4dc45 100644
--- a/tests/unit/test_taskv2_block_downloaded_files.py
+++ b/tests/unit/test_taskv2_block_downloaded_files.py
@@ -183,10 +183,8 @@ async def test_taskv2_block_uses_pre_run_loop_state_for_download_filtering(
outer_workflow_run = SimpleNamespace(proxy_location=None, max_screenshot_scrolls=None)
child_workflow_run = SimpleNamespace(failure_reason=None)
organization = SimpleNamespace(organization_id="org_1", organization_name="Org 1")
- downloaded_files = [
- _file("file:///app/downloads/wr_parent/a.zip", "a.zip", "abc"),
- _file("file:///app/downloads/wr_parent/b.zip", "b.zip", "def"),
- ]
+ a_zip = _file("file:///app/downloads/wr_parent/a.zip", "a.zip", "abc")
+ b_zip = _file("file:///app/downloads/wr_parent/b.zip", "b.zip", "def")
fake_app = SimpleNamespace(
DATABASE=SimpleNamespace(
@@ -204,7 +202,9 @@ async def test_taskv2_block_uses_pre_run_loop_state_for_download_filtering(
get_recent_task_screenshot_artifacts=AsyncMock(return_value=[]),
get_recent_workflow_screenshot_artifacts=AsyncMock(return_value=[]),
),
- STORAGE=SimpleNamespace(get_downloaded_files=AsyncMock(return_value=downloaded_files)),
+ # First read is the block's own baseline capture (only a.zip exists yet); the
+ # second is the post-run read (a.zip from before + b.zip downloaded this block).
+ STORAGE=SimpleNamespace(get_downloaded_files=AsyncMock(side_effect=[[a_zip], [a_zip, b_zip]])),
)
monkeypatch.setattr(block_module, "app", fake_app)
diff --git a/tests/unit/test_video_utils.py b/tests/unit/test_video_utils.py
index 6eb8bf92d..52661347e 100644
--- a/tests/unit/test_video_utils.py
+++ b/tests/unit/test_video_utils.py
@@ -3,13 +3,20 @@ from __future__ import annotations
import asyncio
import os
import struct
+from datetime import UTC, datetime
from unittest.mock import AsyncMock, patch
import pytest
from skyvern.config import settings
from skyvern.webeye import video_utils
-from skyvern.webeye.video_utils import finalize_webm, prepare_recording_for_upload
+from skyvern.webeye.video_utils import (
+ cut_recording_segment,
+ finalize_webm,
+ plan_run_segment,
+ prepare_recording_for_upload,
+ probe_media_duration_seconds,
+)
def _write_unfinalized_webm(path: str) -> bytes:
@@ -298,3 +305,163 @@ async def test_finalize_webm_end_to_end_sets_duration(tmp_path) -> None:
output = await finalize_webm(src)
# Duration element tag is 0x4489 — must be present after finalization.
assert b"\x44\x89" in output[:4096]
+
+
+def test_plan_run_segment_clamps_leading_run_in_long_session() -> None:
+ # The production incident: a 29-min run is the first slice of a ~3h session recording.
+ video_start = datetime(2026, 6, 4, 15, 59, 0, tzinfo=UTC)
+ duration = 3 * 60 * 60 # 15:59 -> 18:59
+ run_started = datetime(2026, 6, 4, 15, 58, 42, tzinfo=UTC) # 18s before the video began
+ run_finished = datetime(2026, 6, 4, 16, 27, 29, tzinfo=UTC)
+ segment = plan_run_segment(run_started, run_finished, video_start, duration)
+ assert segment is not None
+ start, clip_duration = segment
+ assert start == 0.0 # negative offset clamped to start of video
+ assert clip_duration == pytest.approx((run_finished - video_start).total_seconds())
+
+
+def test_plan_run_segment_run_fully_inside() -> None:
+ video_start = datetime(2026, 1, 1, 10, 0, tzinfo=UTC)
+ segment = plan_run_segment(
+ datetime(2026, 1, 1, 10, 10, tzinfo=UTC), datetime(2026, 1, 1, 10, 20, tzinfo=UTC), video_start, 3600
+ )
+ assert segment == (600.0, 600.0)
+
+
+def test_plan_run_segment_no_overlap_returns_none() -> None:
+ video_start = datetime(2026, 1, 1, 10, 0, tzinfo=UTC)
+ # Run finished before this video started — belongs to an earlier recording.
+ assert (
+ plan_run_segment(
+ datetime(2026, 1, 1, 9, 0, tzinfo=UTC), datetime(2026, 1, 1, 9, 30, tzinfo=UTC), video_start, 3600
+ )
+ is None
+ )
+
+
+def test_plan_run_segment_unfinished_run_extends_to_video_end() -> None:
+ video_start = datetime(2026, 1, 1, 10, 0, tzinfo=UTC)
+ segment = plan_run_segment(datetime(2026, 1, 1, 10, 30, tzinfo=UTC), None, video_start, 3600)
+ assert segment == (1800.0, 1800.0)
+
+
+def test_plan_run_segment_too_short_returns_none() -> None:
+ video_start = datetime(2026, 1, 1, 10, 0, tzinfo=UTC)
+ segment = plan_run_segment(
+ datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC),
+ datetime(2026, 1, 1, 10, 0, 0, 500000, tzinfo=UTC), # 0.5s
+ video_start,
+ 3600,
+ )
+ assert segment is None
+
+
+def test_plan_run_segment_zero_duration_video_returns_none() -> None:
+ video_start = datetime(2026, 1, 1, 10, 0, tzinfo=UTC)
+ assert plan_run_segment(datetime(2026, 1, 1, 10, 5, tzinfo=UTC), None, video_start, 0) is None
+
+
+def test_plan_run_segment_treats_naive_as_utc() -> None:
+ video_start = datetime(2026, 1, 1, 10, 0) # naive
+ segment = plan_run_segment(datetime(2026, 1, 1, 10, 10), datetime(2026, 1, 1, 10, 20), video_start, 3600)
+ assert segment == (600.0, 600.0)
+
+
+async def test_probe_media_duration_parses_seconds(tmp_path) -> None:
+ src = str(tmp_path / "v.mp4")
+ with open(src, "wb") as f:
+ f.write(b"x")
+
+ async def fake_exec(*args, **kwargs):
+ proc = AsyncMock()
+ proc.returncode = 0
+ proc.communicate = AsyncMock(return_value=(b"12.5\n", b""))
+ return proc
+
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffprobe"):
+ with patch.object(video_utils.asyncio, "create_subprocess_exec", side_effect=fake_exec):
+ assert await probe_media_duration_seconds(src) == 12.5
+
+
+async def test_probe_media_duration_without_ffprobe_returns_none(tmp_path) -> None:
+ src = str(tmp_path / "v.mp4")
+ with open(src, "wb") as f:
+ f.write(b"x")
+ with patch.object(video_utils.shutil, "which", return_value=None):
+ assert await probe_media_duration_seconds(src) is None
+
+
+async def test_probe_media_duration_unparseable_returns_none(tmp_path) -> None:
+ src = str(tmp_path / "v.mp4")
+ with open(src, "wb") as f:
+ f.write(b"x")
+
+ async def fake_exec(*args, **kwargs):
+ proc = AsyncMock()
+ proc.returncode = 0
+ proc.communicate = AsyncMock(return_value=(b"N/A\n", b""))
+ return proc
+
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffprobe"):
+ with patch.object(video_utils.asyncio, "create_subprocess_exec", side_effect=fake_exec):
+ assert await probe_media_duration_seconds(src) is None
+
+
+async def test_cut_recording_segment_reencodes_with_accurate_seek(tmp_path) -> None:
+ src = str(tmp_path / "session.mp4")
+ with open(src, "wb") as f:
+ f.write(b"src")
+
+ captured_args: list[str] = []
+
+ async def fake_exec(*args, **kwargs):
+ captured_args.extend(args)
+ with open(args[-1], "wb") as f:
+ f.write(b"CLIP")
+ proc = AsyncMock()
+ proc.returncode = 0
+ proc.communicate = AsyncMock(return_value=(b"", b""))
+ proc.wait = AsyncMock(return_value=0)
+ return proc
+
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffmpeg"):
+ with patch.object(video_utils.asyncio, "create_subprocess_exec", side_effect=fake_exec):
+ async with cut_recording_segment(src, start_seconds=12.0, duration_seconds=30.0) as clip_path:
+ assert clip_path is not None
+ temp_path = clip_path
+ with open(clip_path, "rb") as f:
+ assert f.read() == b"CLIP"
+ assert clip_path.endswith(".mp4")
+
+ assert not os.path.exists(temp_path) # cleaned up on exit
+ assert "-ss" in captured_args and captured_args[captured_args.index("-ss") + 1] == "12.000"
+ assert "-t" in captured_args and captured_args[captured_args.index("-t") + 1] == "30.000"
+ # Re-encode (not -c copy) so the cut is frame-accurate and never pulls in prior-run pre-roll.
+ assert "-c:v" in captured_args and captured_args[captured_args.index("-c:v") + 1] == "libx264"
+ assert "-an" in captured_args
+ # input-side seek: -ss must come before -i
+ assert captured_args.index("-ss") < captured_args.index("-i")
+
+
+async def test_cut_recording_segment_without_ffmpeg_yields_none(tmp_path) -> None:
+ src = str(tmp_path / "session.mp4")
+ with open(src, "wb") as f:
+ f.write(b"src")
+ with patch.object(video_utils.shutil, "which", return_value=None):
+ async with cut_recording_segment(src, start_seconds=0.0, duration_seconds=10.0) as clip_path:
+ assert clip_path is None
+
+
+async def test_cut_recording_segment_nonpositive_duration_yields_none(tmp_path) -> None:
+ src = str(tmp_path / "session.mp4")
+ with open(src, "wb") as f:
+ f.write(b"src")
+ with patch.object(video_utils.shutil, "which", return_value="/usr/bin/ffmpeg"):
+ async with cut_recording_segment(src, start_seconds=0.0, duration_seconds=0.0) as clip_path:
+ assert clip_path is None
diff --git a/tests/unit/test_webhook_timestamps.py b/tests/unit/test_webhook_timestamps.py
index e5e36f293..b6453bd5c 100644
--- a/tests/unit/test_webhook_timestamps.py
+++ b/tests/unit/test_webhook_timestamps.py
@@ -12,6 +12,11 @@ from datetime import datetime, timezone
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunResponseBase, WorkflowRunStatus
from skyvern.schemas.runs import RunStatus, RunType, WorkflowRunRequest, WorkflowRunResponse
+from skyvern.services.webhook_delivery import (
+ WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ WEBHOOK_DELIVERY_SUCCESS_REASON,
+ scrub_internal_webhook_delivery_state,
+)
from skyvern.services.webhook_service import build_sample_workflow_run_payload
@@ -31,6 +36,19 @@ def _make_status_response() -> WorkflowRunResponseBase:
)
+def _make_status_payload(status_response: WorkflowRunResponseBase) -> dict:
+ return json.loads(
+ status_response.model_dump_json(
+ exclude={
+ "webhook_callback_url",
+ "totp_verification_url",
+ "totp_identifier",
+ "extra_http_headers",
+ }
+ )
+ )
+
+
def test_workflow_webhook_payload_includes_timestamps() -> None:
"""Reproduce the payload_dict.update() overwrite pattern from execute_workflow_webhook
and verify timestamps are preserved."""
@@ -99,16 +117,7 @@ def test_webhook_replay_payload_includes_timestamps() -> None:
errors=status_response.errors,
)
- payload_dict = json.loads(
- status_response.model_dump_json(
- exclude={
- "webhook_callback_url",
- "totp_verification_url",
- "totp_identifier",
- "extra_http_headers",
- }
- )
- )
+ payload_dict = _make_status_payload(status_response)
payload_dict.update(json.loads(run_response.model_dump_json(exclude={"run_request"})))
assert payload_dict["queued_at"] is not None, "queued_at should not be null in webhook replay payload"
@@ -116,6 +125,29 @@ def test_webhook_replay_payload_includes_timestamps() -> None:
assert payload_dict["finished_at"] is not None, "finished_at should not be null in webhook replay payload"
+def test_webhook_replay_payload_scrubs_internal_delivery_state() -> None:
+ status_response = _make_status_response()
+ status_response.webhook_delivery_status = "pending"
+ status_response.webhook_failure_reason = WEBHOOK_DELIVERY_IN_PROGRESS_REASON
+
+ payload = _make_status_payload(status_response)
+ payload_dict = scrub_internal_webhook_delivery_state(payload)
+
+ assert "webhook_delivery_status" not in payload_dict
+ assert payload_dict["webhook_failure_reason"] is None
+ assert payload["webhook_delivery_status"] == "pending"
+ assert payload["webhook_failure_reason"] == WEBHOOK_DELIVERY_IN_PROGRESS_REASON
+
+
+def test_webhook_replay_payload_scrubs_success_sentinel() -> None:
+ status_response = _make_status_response()
+ status_response.webhook_failure_reason = WEBHOOK_DELIVERY_SUCCESS_REASON
+
+ payload_dict = scrub_internal_webhook_delivery_state(_make_status_payload(status_response))
+
+ assert payload_dict["webhook_failure_reason"] is None
+
+
def test_sample_workflow_webhook_payload_uses_public_credit_fields() -> None:
payload = build_sample_workflow_run_payload("wr_sample_test")
diff --git a/tests/unit/test_workflow_copilot_llm_config.py b/tests/unit/test_workflow_copilot_llm_config.py
index 37c200d1a..782446d59 100644
--- a/tests/unit/test_workflow_copilot_llm_config.py
+++ b/tests/unit/test_workflow_copilot_llm_config.py
@@ -353,7 +353,10 @@ async def test_completion_verification_handler_uses_main_copilot_lane(
assert organization_id == "org_1"
return main_handler
- monkeypatch.setattr(copilot_tools, "resolve_main_copilot_handler", _main_lookup)
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.completion.resolve_main_copilot_handler",
+ _main_lookup,
+ )
ctx: Any = SimpleNamespace(workflow_permanent_id="wpid_1", organization_id="org_1")
handler = await copilot_tools._completion_verification_handler(ctx)
@@ -371,7 +374,7 @@ async def test_composition_visual_handler_uses_fast_copilot_lane(
assert organization_id == "org_1"
return fast_handler
- monkeypatch.setattr(copilot_tools, "resolve_fast_copilot_handler", _fast_lookup)
+ monkeypatch.setattr(copilot_tools.composition_capture, "resolve_fast_copilot_handler", _fast_lookup)
ctx: Any = SimpleNamespace(workflow_permanent_id="wpid_1", organization_id="org_1")
handler = await copilot_tools._composition_visual_handler(ctx)
diff --git a/tests/unit/test_workflow_webhook_delivery.py b/tests/unit/test_workflow_webhook_delivery.py
index 81fe3a3e7..66a9e93ec 100644
--- a/tests/unit/test_workflow_webhook_delivery.py
+++ b/tests/unit/test_workflow_webhook_delivery.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -8,13 +9,20 @@ import httpx
import pytest
from skyvern.forge.sdk.workflow import service as service_module
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunResponseBase
from skyvern.forge.sdk.workflow.service import WorkflowService
class _StatusResponse:
- def __init__(self) -> None:
+ def __init__(
+ self,
+ webhook_failure_reason: str | None = None,
+ webhook_delivery_status: str | None = None,
+ ) -> None:
now = datetime.now(timezone.utc)
self.status = "completed"
+ self.webhook_delivery_status = webhook_delivery_status
+ self.webhook_failure_reason = webhook_failure_reason
self.outputs: dict = {}
self.downloaded_files: list = []
self.recording_url = None
@@ -32,7 +40,14 @@ class _StatusResponse:
self.finished_at = now
def model_dump_json(self) -> str:
- return '{"workflow_run_id":"wr_abc","status":"completed"}'
+ return json.dumps(
+ {
+ "workflow_run_id": "wr_abc",
+ "status": "completed",
+ "webhook_delivery_status": self.webhook_delivery_status,
+ "webhook_failure_reason": self.webhook_failure_reason,
+ }
+ )
class _WebhookRunResponse:
@@ -60,6 +75,62 @@ def _response(status_code: int, body: str = "") -> httpx.Response:
return httpx.Response(status_code=status_code, content=body.encode("utf-8"))
+ ("webhook_callback_url", "webhook_failure_reason", "expected"),
+ [
+ ("https://example.com/hook", None, "pending"),
+ ("https://example.com/hook", service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON, "pending"),
+ ("https://example.com/hook", "Webhook failed with status code 400", "failed"),
+ ("https://example.com/hook", service_module.WEBHOOK_DELIVERY_SUCCESS_REASON, None),
+ (None, None, None),
+ ],
+)
+def test_get_workflow_webhook_delivery_status(
+ webhook_callback_url: str | None,
+ webhook_failure_reason: str | None,
+ expected: str | None,
+) -> None:
+ assert (
+ service_module._get_workflow_webhook_delivery_status(
+ webhook_callback_url=webhook_callback_url,
+ webhook_failure_reason=webhook_failure_reason,
+ workflow_run_status=service_module.WorkflowRunStatus.completed,
+ )
+ == expected
+ )
+
+
+ ("webhook_failure_reason", "expected"),
+ [
+ (service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON, None),
+ ("Webhook failed with status code 400", "Webhook failed with status code 400"),
+ (service_module.WEBHOOK_DELIVERY_SUCCESS_REASON, None),
+ (None, None),
+ ],
+)
+def test_get_public_workflow_webhook_failure_reason(
+ webhook_failure_reason: str | None,
+ expected: str | None,
+) -> None:
+ assert service_module._get_public_workflow_webhook_failure_reason(webhook_failure_reason) == expected
+
+
+def test_workflow_run_response_base_serializes_webhook_delivery_status() -> None:
+ now = datetime.now(timezone.utc)
+ response = WorkflowRunResponseBase(
+ workflow_id="wpid_abc",
+ workflow_run_id="wr_abc",
+ status=service_module.WorkflowRunStatus.completed,
+ webhook_delivery_status="pending",
+ created_at=now,
+ modified_at=now,
+ parameters={},
+ )
+
+ assert json.loads(response.model_dump_json())["webhook_delivery_status"] == "pending"
+
+
@pytest.fixture
def webhook_service(monkeypatch: pytest.MonkeyPatch) -> tuple[WorkflowService, AsyncMock, AsyncMock]:
svc = WorkflowService()
@@ -83,6 +154,11 @@ def webhook_service(monkeypatch: pytest.MonkeyPatch) -> tuple[WorkflowService, A
AsyncMock(return_value=SimpleNamespace(token="api-key")),
)
monkeypatch.setattr(service_module.app.DATABASE.workflow_runs, "update_workflow_run", update_run)
+ monkeypatch.setattr(
+ service_module.app.DATABASE.workflow_runs,
+ "claim_workflow_run_webhook_delivery",
+ AsyncMock(return_value=True),
+ )
return svc, build_response, update_run
@@ -107,6 +183,73 @@ async def test_prepare_workflow_webhook_builds_request_without_delivery(
deliver.assert_not_awaited()
+async def test_prepare_workflow_webhook_scrubs_internal_delivery_state_from_payload(
+ webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ svc, build_response, _update_run = webhook_service
+ signed_payload: dict[str, object] = {}
+
+ build_response.return_value = _StatusResponse(
+ webhook_delivery_status="pending",
+ webhook_failure_reason=service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ )
+
+ def capture_signature(payload: dict[str, object], api_key: str) -> SimpleNamespace:
+ signed_payload.update(payload)
+ return SimpleNamespace(
+ payload_for_log=json.dumps(payload),
+ headers={"x-skyvern-signature": "sig"},
+ signed_payload=json.dumps(payload),
+ )
+
+ monkeypatch.setattr(service_module, "generate_skyvern_webhook_signature", capture_signature)
+
+ webhook = await svc.prepare_workflow_webhook(_workflow_run())
+
+ assert webhook is not None
+ assert "webhook_delivery_status" not in signed_payload
+ assert signed_payload["webhook_failure_reason"] is None
+
+
+async def test_prepare_workflow_webhook_records_missing_signing_key_as_failure(
+ webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ svc, _build_response, update_run = webhook_service
+ monkeypatch.setattr(
+ service_module.app.DATABASE.organizations,
+ "get_valid_org_auth_token",
+ AsyncMock(return_value=None),
+ )
+
+ webhook = await svc.prepare_workflow_webhook(_workflow_run())
+
+ assert webhook is None
+ update_run.assert_awaited_once_with(
+ workflow_run_id="wr_abc",
+ webhook_failure_reason=service_module.WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON,
+ )
+
+
+async def test_prepare_workflow_webhook_records_missing_workflow_as_failure(
+ webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
+) -> None:
+ svc, build_response, update_run = webhook_service
+ build_response.side_effect = service_module.WorkflowNotFound(workflow_permanent_id="wpid_abc")
+
+ webhook = await svc.prepare_workflow_webhook(_workflow_run())
+
+ assert webhook is None
+ update_run.assert_awaited_once_with(
+ workflow_run_id="wr_abc",
+ webhook_failure_reason=service_module.WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON,
+ )
+
+
@pytest.mark.asyncio
async def test_execute_workflow_webhook_records_customer_failure_without_raising(
webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
@@ -125,6 +268,52 @@ async def test_execute_workflow_webhook_records_customer_failure_without_raising
)
+async def test_execute_workflow_webhook_passes_retry_claim_cutoff(
+ webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ svc, _build_response, _update_run = webhook_service
+ retry_claim_older_than = datetime(2026, 6, 1, 13, 20, tzinfo=timezone.utc)
+ deliver = AsyncMock(return_value=_response(200, "ok"))
+ claim = AsyncMock(return_value=True)
+ monkeypatch.setattr(service_module, "deliver_webhook_with_retries", deliver)
+ monkeypatch.setattr(
+ service_module.app.DATABASE.workflow_runs,
+ "claim_workflow_run_webhook_delivery",
+ claim,
+ )
+
+ await svc.execute_workflow_webhook(_workflow_run(), retry_claim_older_than=retry_claim_older_than)
+
+ claim.assert_awaited_once_with(
+ workflow_run_id="wr_abc",
+ in_progress_reason=service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON,
+ claim_older_than=retry_claim_older_than,
+ )
+ deliver.assert_awaited_once()
+
+
+async def test_execute_workflow_webhook_skips_delivery_when_claim_lost(
+ webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ svc, _build_response, update_run = webhook_service
+ deliver = AsyncMock()
+ monkeypatch.setattr(service_module, "deliver_webhook_with_retries", deliver)
+ monkeypatch.setattr(
+ service_module.app.DATABASE.workflow_runs,
+ "claim_workflow_run_webhook_delivery",
+ AsyncMock(return_value=False),
+ )
+
+ await svc.execute_workflow_webhook(_workflow_run())
+
+ deliver.assert_not_awaited()
+ update_run.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_execute_workflow_webhook_records_delivery_exception_without_raising(
webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],
@@ -154,7 +343,10 @@ async def test_execute_workflow_webhook_does_not_raise_if_post_delivery_recordin
await svc.execute_workflow_webhook(_workflow_run())
deliver.assert_awaited_once()
- update_run.assert_awaited_once_with(workflow_run_id="wr_abc", webhook_failure_reason="")
+ update_run.assert_awaited_once_with(
+ workflow_run_id="wr_abc",
+ webhook_failure_reason=service_module.WEBHOOK_DELIVERY_SUCCESS_REASON,
+ )
@pytest.mark.asyncio
diff --git a/uv.lock b/uv.lock
index 3a6230b82..aa9d32290 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1546,6 +1546,23 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/3d/f7/661d7a9023e877a226b5683429c3662f75a29ef45cb1464cf39adb689218/google_cloud_resource_manager-1.17.0-py3-none-any.whl", hash = "sha256:e479baf4b014a57f298e01b8279e3290b032e3476d69c8e5e1427af8f82739a5", size = 404403, upload-time = "2026-03-26T22:15:26.57Z" },
]
+[[package]]
+name = "google-cloud-secret-manager"
+version = "2.29.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "google-api-core", extra = ["grpc"] },
+ { name = "google-auth" },
+ { name = "grpc-google-iam-v1" },
+ { name = "grpcio" },
+ { name = "proto-plus" },
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d2/7c/5c88cdde9664f6c75fb68aa11e0af4309a92bef38dd38df0456ffb0f469b/google_cloud_secret_manager-2.29.0.tar.gz", hash = "sha256:ee64133af8fdb3780affb65ec6ccf10ab15a0113d8edeba388665f4be87ce1be", size = 278437, upload-time = "2026-06-03T16:13:43.149Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b6/c2/fc3275bc42a522757cb5141d7dae51f048b93d2f5fe4574fcee5392cef03/google_cloud_secret_manager-2.29.0-py3-none-any.whl", hash = "sha256:21bac2d0adb0bb3c13c346d7223832f197c2266534528a1bf1402774e06395a3", size = 225042, upload-time = "2026-06-03T16:12:20.162Z" },
+]
+
[[package]]
name = "google-cloud-storage"
version = "3.10.1"
@@ -5752,6 +5769,7 @@ local = [
{ name = "google-auth" },
{ name = "google-auth-oauthlib" },
{ name = "google-cloud-aiplatform" },
+ { name = "google-cloud-secret-manager" },
{ name = "google-cloud-storage" },
{ name = "greenlet", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "greenlet", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
@@ -5815,6 +5833,7 @@ server = [
{ name = "google-auth" },
{ name = "google-auth-oauthlib" },
{ name = "google-cloud-aiplatform" },
+ { name = "google-cloud-secret-manager" },
{ name = "google-cloud-storage" },
{ name = "greenlet", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "greenlet", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
@@ -5962,6 +5981,8 @@ requires-dist = [
{ name = "google-auth-oauthlib", marker = "extra == 'server'", specifier = ">=1.3.1,<2" },
{ name = "google-cloud-aiplatform", marker = "extra == 'local'", specifier = ">=1.149.0,<2" },
{ name = "google-cloud-aiplatform", marker = "extra == 'server'", specifier = ">=1.149.0,<2" },
+ { name = "google-cloud-secret-manager", marker = "extra == 'local'", specifier = ">=2.29,<3" },
+ { name = "google-cloud-secret-manager", marker = "extra == 'server'", specifier = ">=2.29,<3" },
{ name = "google-cloud-storage", marker = "extra == 'local'", specifier = ">=2.18,<4" },
{ name = "google-cloud-storage", marker = "extra == 'server'", specifier = ">=2.18,<4" },
{ name = "greenlet", marker = "python_full_version == '3.11.*' and extra == 'local'", specifier = "==3.0.3" },