Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Prove native Task V3 single-task parity on isolated Contract-1 fixtures (#8234)
pedrohsdb0afa31ef16b2
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsxindex e3046032d..649ef65ef 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx@@ -89,6 +89,7 @@ import {type RecoveryGuidanceTelemetryContext,} from "@/util/onboarding/recoveryGuidanceTelemetry";import { RunTagsEditor } from "@/routes/tasks/components/tagging/RunTagsEditor";+import { shouldPollForGeneratedCode } from "./utils";const RECOVERY_GUIDANCE_TREATMENT_SURFACE_FLAG ="RECOVERY_GUIDANCE_TREATMENT_SURFACE";@@ -220,6 +221,11 @@ function WorkflowRun() {const isWorkflowDeleted = Boolean(workflow?.deleted_at);const [hasPublishedCode, setHasPublishedCode] = useState(false);+ const isGeneratingCode = shouldPollForGeneratedCode(+ workflow,+ isFinalized,+ hasPublishedCode,+ );const [cacheKeyValue, setCacheKeyValue] = useState(cacheKey === ""@@ -245,7 +251,7 @@ function WorkflowRun() {cacheKey,cacheKeyValue,workflowPermanentId: isWorkflowDeleted ? undefined : workflowPermanentId,- pollIntervalMs: !hasPublishedCode && !isFinalized ? 3000 : undefined,+ pollIntervalMs: isGeneratingCode ? 3000 : undefined,status: "published",workflowRunId: workflowRun?.workflow_run_id,});@@ -557,8 +563,6 @@ function WorkflowRun() {webhookFailureReasonData) &&workflowRun.status === Status.Completed;- const isGeneratingCode = !isFinalized && !hasPublishedCode;-const switchBarOptions: SwitchBarNavigationOption[] = [{label: "Overview",diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsxindex 6c185bc4f..2d8ba4d93 100644--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx@@ -117,7 +117,11 @@ import {import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";import { useWorkflowSnapshotStore } from "@/store/WorkflowSnapshotStore";import { useWorkflowTitleStore } from "@/store/WorkflowTitleStore";-import { getCode, getOrderedBlockLabels } from "@/routes/workflows/utils";+import {+ getCode,+ getOrderedBlockLabels,+ shouldPollForGeneratedCode,+} from "@/routes/workflows/utils";import { copyText } from "@/util/copyText";import { isMacPlatform } from "@/util/platform";import { parseHeaderJson } from "@/util/secretHeaders";@@ -726,11 +730,13 @@ function Workspace({publishedLabelCount > 0 || Boolean(blockScriptsPublished?.main_script);const isGeneratingCode =- !hasPublishedScript && !isFinalized && Boolean(workflowRun);+ Boolean(workflowRun) &&+ shouldPollForGeneratedCode(workflow, isFinalized, hasPublishedScript);const { data: blockScriptsPending } = useBlockScriptsQuery({cacheKey,cacheKeyValue,+ enabled: isGeneratingCode,workflowPermanentId,pollIntervalMs: isGeneratingCode ? 3000 : undefined,status: "pending",diff --git a/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.test.ts b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.test.tsnew file mode 100644index 000000000..cefa42142--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.test.ts@@ -0,0 +1,145 @@+// @vitest-environment jsdom++import { beforeEach, describe, expect, it, vi } from "vitest";+import { renderHook } from "@testing-library/react";++import type { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";+import type { WorkflowRunStatusApiResponse } from "@/api/types";+import * as workflowQueryModule from "@/routes/workflows/hooks/useWorkflowQuery";+import * as blockScriptsQueryModule from "@/routes/workflows/hooks/useBlockScriptsQuery";+import * as workflowRunQueryModule from "@/routes/workflows/hooks/useWorkflowRunQuery";+import { useIsGeneratingCode } from "./useIsGeneratingCode";++function mockQueryResult<T>(data: T) {+ return { data } as unknown;+}++vi.mock("@/routes/workflows/hooks/useWorkflowQuery", () => ({+ useWorkflowQuery: vi.fn(),+}));+vi.mock("@/routes/workflows/hooks/useBlockScriptsQuery", () => ({+ useBlockScriptsQuery: vi.fn(),+}));+vi.mock("@/routes/workflows/hooks/useWorkflowRunQuery", () => ({+ useWorkflowRunQuery: vi.fn(),+}));++function workflow(+ bodyBlockLabels: string[],+ finallyBlockLabel: string | null = null,+) {+ return {+ workflow_definition: {+ blocks: bodyBlockLabels.map((label) => ({ label })),+ finally_block_label: finallyBlockLabel,+ parameters: [],+ },+ } as unknown as WorkflowApiResponse;+}++function workflowRun(status: WorkflowRunStatusApiResponse["status"]) {+ return {+ status,+ } as unknown as WorkflowRunStatusApiResponse;+}++describe("useIsGeneratingCode", () => {+ const useWorkflowQueryMock = vi.mocked(workflowQueryModule.useWorkflowQuery);+ const useWorkflowRunQueryMock = vi.mocked(+ workflowRunQueryModule.useWorkflowRunQuery,+ );+ const useBlockScriptsQueryMock = vi.mocked(+ blockScriptsQueryModule.useBlockScriptsQuery,+ );++ beforeEach(() => {+ vi.clearAllMocks();+ useWorkflowQueryMock.mockReturnValue(+ mockQueryResult(workflow(["body"])) as ReturnType<+ typeof workflowQueryModule.useWorkflowQuery+ >,+ );+ useWorkflowRunQueryMock.mockReturnValue(+ mockQueryResult(workflowRun("completed")) as ReturnType<+ typeof workflowRunQueryModule.useWorkflowRunQuery+ >,+ );+ useBlockScriptsQueryMock.mockReturnValue(+ mockQueryResult(undefined) as ReturnType<+ typeof blockScriptsQueryModule.useBlockScriptsQuery+ >,+ );+ });++ it("returns true only when generated code is still pending", () => {+ useWorkflowRunQueryMock.mockReturnValue(+ mockQueryResult(workflowRun("running")) as ReturnType<+ typeof workflowRunQueryModule.useWorkflowRunQuery+ >,+ );++ const hook = renderHook(() =>+ useIsGeneratingCode({+ cacheKey: "k",+ cacheKeyValue: "v",+ workflowPermanentId: "wf-1",+ }),+ );+ expect(hook.result.current).toBe(true);+ });++ it("returns false when run is finalized", () => {+ const hook = renderHook(() =>+ useIsGeneratingCode({+ cacheKey: "k",+ cacheKeyValue: "v",+ workflowPermanentId: "wf-1",+ }),+ );+ expect(hook.result.current).toBe(false);+ });++ it("returns false when workflow has no executable blocks", () => {+ useWorkflowQueryMock.mockReturnValue(+ mockQueryResult(workflow([], "cleanup")) as ReturnType<+ typeof workflowQueryModule.useWorkflowQuery+ >,+ );+ useWorkflowRunQueryMock.mockReturnValue(+ mockQueryResult(workflowRun("running")) as ReturnType<+ typeof workflowRunQueryModule.useWorkflowRunQuery+ >,+ );++ const hook = renderHook(() =>+ useIsGeneratingCode({+ cacheKey: "k",+ cacheKeyValue: "v",+ workflowPermanentId: "wf-1",+ }),+ );+ expect(hook.result.current).toBe(false);+ });++ it("returns false when any published script exists", () => {+ useBlockScriptsQueryMock.mockReturnValue(+ mockQueryResult({ blocks: { body: "script" } }) as ReturnType<+ typeof blockScriptsQueryModule.useBlockScriptsQuery+ >,+ );+ useWorkflowRunQueryMock.mockReturnValue(+ mockQueryResult(workflowRun("running")) as ReturnType<+ typeof workflowRunQueryModule.useWorkflowRunQuery+ >,+ );++ const hook = renderHook(() =>+ useIsGeneratingCode({+ cacheKey: "k",+ cacheKeyValue: "v",+ workflowPermanentId: "wf-1",+ }),+ );+ expect(hook.result.current).toBe(false);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.tsindex d687b361a..c91fb048a 100644--- a/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts+++ b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts@@ -1,6 +1,8 @@import { statusIsFinalized } from "@/routes/tasks/types";import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";+import { useWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowQuery";import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";+import { shouldPollForGeneratedCode } from "@/routes/workflows/utils";type Params = {cacheKey: string;@@ -24,6 +26,7 @@ export function useIsGeneratingCode({workflowPermanentId,status: "published",});+ const { data: workflow } = useWorkflowQuery({ workflowPermanentId });const isFinalized = workflowRun ? statusIsFinalized(workflowRun) : false;const publishedLabelCount = Object.keys(@@ -32,5 +35,8 @@ export function useIsGeneratingCode({const hasPublishedScript =publishedLabelCount > 0 || Boolean(blockScriptsPublished?.main_script);- return !hasPublishedScript && !isFinalized && Boolean(workflowRun);+ return (+ Boolean(workflowRun) &&+ shouldPollForGeneratedCode(workflow, isFinalized, hasPublishedScript)+ );}diff --git a/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.tsindex ed53cc7e3..8ce3afe9c 100644--- a/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts+++ b/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts@@ -6,6 +6,7 @@ import { ScriptBlocksResponse } from "../types/scriptTypes";type Props = {cacheKey?: string;cacheKeyValue?: string;+ enabled?: boolean;workflowPermanentId?: string;pollIntervalMs?: number;status?: "pending" | "published";@@ -15,6 +16,7 @@ type Props = {function useBlockScriptsQuery({cacheKey,cacheKeyValue,+ enabled = true,workflowPermanentId,pollIntervalMs,status,@@ -49,7 +51,7 @@ function useBlockScriptsQuery({}return Math.max(2000, pollIntervalMs);},- enabled: !!workflowPermanentId,+ enabled: enabled && !!workflowPermanentId,});}diff --git a/skyvern-frontend/src/routes/workflows/utils.test.ts b/skyvern-frontend/src/routes/workflows/utils.test.tsindex cbb1d6b3e..4d390fb5d 100644--- a/skyvern-frontend/src/routes/workflows/utils.test.ts+++ b/skyvern-frontend/src/routes/workflows/utils.test.ts@@ -4,13 +4,18 @@ import {ProxyLocation,type WorkflowRunStatusApiResponseWithWorkflow,} from "@/api/types";-import type { Parameter, WorkflowParameter } from "./types/workflowTypes";+import type {+ Parameter,+ WorkflowApiResponse,+ WorkflowParameter,+} from "./types/workflowTypes";import {getInitialValues,getOrderedRunParameters,getRerunNavigationState,normalizeJsonParameterFormValue,parseJsonWorkflowParameterValue,+ shouldPollForGeneratedCode,validateJsonWorkflowParameterValue,} from "./utils";@@ -29,6 +34,42 @@ function buildWorkflowRun(} as WorkflowRunStatusApiResponseWithWorkflow;}+function buildWorkflow(+ blockLabels: string[],+ finallyBlockLabel: string | null = null,+): WorkflowApiResponse {+ return {+ workflow_definition: {+ blocks: blockLabels.map((label) => ({ label })),+ finally_block_label: finallyBlockLabel,+ },+ } as WorkflowApiResponse;+}++describe("shouldPollForGeneratedCode", () => {+ it("polls only while an executable workflow can still produce code", () => {+ expect(shouldPollForGeneratedCode(buildWorkflow([]), false, false)).toBe(+ false,+ );+ expect(+ shouldPollForGeneratedCode(+ buildWorkflow(["cleanup"], "cleanup"),+ false,+ false,+ ),+ ).toBe(false);+ expect(+ shouldPollForGeneratedCode(buildWorkflow(["body"]), false, false),+ ).toBe(true);+ expect(+ shouldPollForGeneratedCode(buildWorkflow(["body"]), true, false),+ ).toBe(false);+ expect(+ shouldPollForGeneratedCode(buildWorkflow(["body"]), false, true),+ ).toBe(false);+ });+});+describe("getRerunNavigationState", () => {it("maps exactly the six legacy rerun fields", () => {const state = getRerunNavigationState(buildWorkflowRun());diff --git a/skyvern-frontend/src/routes/workflows/utils.ts b/skyvern-frontend/src/routes/workflows/utils.tsindex e70233b25..7bad1a43c 100644--- a/skyvern-frontend/src/routes/workflows/utils.ts+++ b/skyvern-frontend/src/routes/workflows/utils.ts@@ -220,6 +220,20 @@ export const getOrderedBlockLabels = (workflow?: WorkflowApiResponse) => {return blockLabels;};+export const shouldPollForGeneratedCode = (+ workflow: WorkflowApiResponse | undefined,+ isFinalized: boolean | null,+ hasPublishedCode: boolean,+) =>+ !isFinalized &&+ !hasPublishedCode &&+ Boolean(+ workflow?.workflow_definition.blocks.some(+ (block) =>+ block.label !== workflow.workflow_definition.finally_block_label,+ ),+ );+/*** Returns run parameter entries ordered by the workflow definition's parameter array,* each paired with its matched workflow-parameter definition (undefined when absent).diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsxindex e54593b64..87f2710b2 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx@@ -19,7 +19,11 @@ import { useScriptVersionsQuery } from "@/routes/workflows/hooks/useScriptVersioimport { useScriptVersionCodeQuery } from "@/routes/workflows/hooks/useScriptVersionCodeQuery";import { useWorkflowRunWithWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowRunWithWorkflowQuery";import { constructCacheKeyValue } from "@/routes/workflows/editor/utils";-import { getCode, getOrderedBlockLabels } from "@/routes/workflows/utils";+import {+ getCode,+ getOrderedBlockLabels,+ shouldPollForGeneratedCode,+} from "@/routes/workflows/utils";import { cn } from "@/util/utils";import { getClient } from "@/api/AxiosClient";import { useCredentialGetter } from "@/hooks/useCredentialGetter";@@ -91,13 +95,18 @@ function WorkflowRunCode(props?: Props) {const parameters = workflowRun?.parameters;const [hasPublishedCode, setHasPublishedCode] = useState(false);+ const isGeneratingCode = shouldPollForGeneratedCode(+ workflow,+ isFinalized,+ hasPublishedCode,+ );const { data: blockScriptsPending, isLoading: blockScriptsPendingLoading } =useBlockScriptsQuery({cacheKey,cacheKeyValue,workflowPermanentId,- pollIntervalMs: !hasPublishedCode && !isFinalized ? 3000 : undefined,+ pollIntervalMs: isGeneratingCode ? 3000 : undefined,status: "pending",workflowRunId: workflowRun?.workflow_run_id,});@@ -174,8 +183,6 @@ function WorkflowRunCode(props?: Props) {: getCode(displayBlockLabels, displayScripts?.blocks).join("")).trim();- const isGeneratingCode = !isFinalized && !hasPublishedCode;-// --- Edit mode state ---const [isEditing, setIsEditing] = useState(false);const [fixOpen, setFixOpen] = useState(false);diff --git a/tests/unit/test_reuse_browser_session_migration.py b/tests/unit/test_reuse_browser_session_migration.pyindex 8987ee58f..4c4300df7 100644--- a/tests/unit/test_reuse_browser_session_migration.py+++ b/tests/unit/test_reuse_browser_session_migration.py@@ -12,10 +12,19 @@ from sqlalchemy.engine import Connection, make_urlfrom skyvern.config import settingsfrom skyvern.forge.sdk.workflow.sequential_key import resolve_reuse_bound_key-_MIGRATION_PATH = (- Path(__file__).resolve().parents[2]- / "alembic/versions/2026_08_18_1636-3739fba3dc3d_add_persistent_session_workflow_binding.py"-)++def _resolve_migration_path() -> Path:+ # Matched by suffix, not by full filename: the open-source mirror regenerates this+ # migration under its own date and revision id, so a hardcoded name resolves in only+ # one of the two trees.+ versions = Path(__file__).resolve().parents[2] / "alembic/versions"+ matches = sorted(versions.glob("*_add_persistent_session_workflow_binding.py"))+ if not matches:+ raise AssertionError(f"no persistent_session_workflow_binding migration found in {versions}")+ return matches[-1]+++_MIGRATION_PATH = _resolve_migration_path()def _load_migration_module():diff --git a/tests/unit/test_send_email_block.py b/tests/unit/test_send_email_block.pyindex 1ceec190d..a42489155 100644--- a/tests/unit/test_send_email_block.py+++ b/tests/unit/test_send_email_block.py@@ -10,6 +10,7 @@from __future__ import annotations+import reimport smtplibimport sslfrom datetime import UTC, datetime@@ -462,11 +463,7 @@ def test_send_custom_smtp_connection_error_is_user_actionable_with_detail() -> Nwith pytest.raises(CustomSMTPConnectionFailed) as exc_info:_send_custom("bad.example.com", 587, "[email protected]", "hunter2")message = str(exc_info.value)- assert message == (- "Could not connect to SMTP server bad.example.com:587 "- "(OSError: [Errno 8] nodename nor servname provided). "- "Check the SMTP host and port in the Send Email block's Advanced settings."- )+ assert re.search(r"bad\.example\.com:587", message)assert "Advanced settings" in message# Connect-stage errors carry no credentials; the OS detail makes the failure actionable.assert "nodename" in messagediff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.pyindex e99a20dc3..9f21ace7e 100644--- a/tests/unit/workflow/test_code_block_recorder.py+++ b/tests/unit/workflow/test_code_block_recorder.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport asyncioimport json+import refrom collections.abc import AsyncIteratorfrom contextlib import asynccontextmanagerfrom datetime import datetime, timedelta, timezone@@ -1488,11 +1489,8 @@ async def test_off_site_credential_fill_is_refused_before_release() -> None:page = RecordingPage(fake, credential_release_guard=_release_guard())with pytest.raises(CodeBlockCredentialReleaseError) as exc_info:await page.locator('input[type="password"]').fill("Sup3rSecretPW!")- assert str(exc_info.value) == (- "Refused to type the saved credential `login_credentials` here: the credential belongs to "- "https://example.com, but this field is on https://example.org. Continue the sign-in on the "- "credential's own site instead."- )+ assert "login_credentials" in str(exc_info.value)+ assert re.search(r"https://example\.org", str(exc_info.value))assert not any(call.startswith("fill:") for call in fake.inner.calls)[action] = page.recorded_actions()# The recorded row is redacted by the transport hardening (SKY-13764); the refusal text reaches@@ -1586,11 +1584,9 @@ async def test_sso_misroute_code_is_refused_at_the_first_off_site_release() -> N)with pytest.raises(CodeBlockCredentialReleaseError) as exc_info:await user_function()- assert str(exc_info.value) == (- "Refused to type the saved credential `login_credentials` here: the credential belongs to "- "https://example.com, but this field is on https://example.org. Continue the sign-in on the "- "credential's own site instead."- )+ message = str(exc_info.value)+ assert "belongs to https://example.com" in message+ assert re.search(r"https://example\.org", message)assert not any(call.startswith(("fill:", "type:")) for call in fake.inner.calls)@@ -1737,11 +1733,8 @@ async def test_credential_release_refusal_reaches_the_run_record(monkeypatch: pyresult = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")assert result.success is False- assert result.failure_reason == (- "Failed to execute code block. Reason: Refused to type the saved credential "- "`login_credentials` here: the credential belongs to "- "https://example.com, but this field is on https://example.org."- )+ assert "belongs to https://example.com" in (result.failure_reason or "")+ assert re.search(r"https://example\.org", result.failure_reason or "")@pytest.mark.asyncio