Skip to content

Automate browser based workflows with AI

Prove native Task V3 single-task parity on isolated Contract-1 fixtures (#8234)

pedrohsdb0afa31ef16b2
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
index e3046032d..649ef65ef 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
@@ -89,6 +89,7 @@ import {
type RecoveryGuidanceTelemetryContext,
} from "@/util/onboarding/recoveryGuidanceTelemetry";
import { RunTagsEditor } from "@/routes/tasks/components/tagging/RunTagsEditor";
+import { shouldPollForGeneratedCode } from "./utils";
const RECOVERY_GUIDANCE_TREATMENT_SURFACE_FLAG =
"RECOVERY_GUIDANCE_TREATMENT_SURFACE";
@@ -220,6 +221,11 @@ function WorkflowRun() {
const isWorkflowDeleted = Boolean(workflow?.deleted_at);
const [hasPublishedCode, setHasPublishedCode] = useState(false);
+ const isGeneratingCode = shouldPollForGeneratedCode(
+ workflow,
+ isFinalized,
+ hasPublishedCode,
+ );
const [cacheKeyValue, setCacheKeyValue] = useState(
cacheKey === ""
@@ -245,7 +251,7 @@ function WorkflowRun() {
cacheKey,
cacheKeyValue,
workflowPermanentId: isWorkflowDeleted ? undefined : workflowPermanentId,
- pollIntervalMs: !hasPublishedCode && !isFinalized ? 3000 : undefined,
+ pollIntervalMs: isGeneratingCode ? 3000 : undefined,
status: "published",
workflowRunId: workflowRun?.workflow_run_id,
});
@@ -557,8 +563,6 @@ function WorkflowRun() {
webhookFailureReasonData) &&
workflowRun.status === Status.Completed;
- const isGeneratingCode = !isFinalized && !hasPublishedCode;
-
const switchBarOptions: SwitchBarNavigationOption[] = [
{
label: "Overview",
diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
index 6c185bc4f..2d8ba4d93 100644
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -117,7 +117,11 @@ import {
import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";
import { useWorkflowSnapshotStore } from "@/store/WorkflowSnapshotStore";
import { useWorkflowTitleStore } from "@/store/WorkflowTitleStore";
-import { getCode, getOrderedBlockLabels } from "@/routes/workflows/utils";
+import {
+ getCode,
+ getOrderedBlockLabels,
+ shouldPollForGeneratedCode,
+} from "@/routes/workflows/utils";
import { copyText } from "@/util/copyText";
import { isMacPlatform } from "@/util/platform";
import { parseHeaderJson } from "@/util/secretHeaders";
@@ -726,11 +730,13 @@ function Workspace({
publishedLabelCount > 0 || Boolean(blockScriptsPublished?.main_script);
const isGeneratingCode =
- !hasPublishedScript && !isFinalized && Boolean(workflowRun);
+ Boolean(workflowRun) &&
+ shouldPollForGeneratedCode(workflow, isFinalized, hasPublishedScript);
const { data: blockScriptsPending } = useBlockScriptsQuery({
cacheKey,
cacheKeyValue,
+ enabled: isGeneratingCode,
workflowPermanentId,
pollIntervalMs: isGeneratingCode ? 3000 : undefined,
status: "pending",
diff --git a/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.test.ts b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.test.ts
new file mode 100644
index 000000000..cefa42142
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.test.ts
@@ -0,0 +1,145 @@
+// @vitest-environment jsdom
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { renderHook } from "@testing-library/react";
+
+import type { WorkflowApiResponse } from "@/routes/workflows/types/workflowTypes";
+import type { WorkflowRunStatusApiResponse } from "@/api/types";
+import * as workflowQueryModule from "@/routes/workflows/hooks/useWorkflowQuery";
+import * as blockScriptsQueryModule from "@/routes/workflows/hooks/useBlockScriptsQuery";
+import * as workflowRunQueryModule from "@/routes/workflows/hooks/useWorkflowRunQuery";
+import { useIsGeneratingCode } from "./useIsGeneratingCode";
+
+function mockQueryResult<T>(data: T) {
+ return { data } as unknown;
+}
+
+vi.mock("@/routes/workflows/hooks/useWorkflowQuery", () => ({
+ useWorkflowQuery: vi.fn(),
+}));
+vi.mock("@/routes/workflows/hooks/useBlockScriptsQuery", () => ({
+ useBlockScriptsQuery: vi.fn(),
+}));
+vi.mock("@/routes/workflows/hooks/useWorkflowRunQuery", () => ({
+ useWorkflowRunQuery: vi.fn(),
+}));
+
+function workflow(
+ bodyBlockLabels: string[],
+ finallyBlockLabel: string | null = null,
+) {
+ return {
+ workflow_definition: {
+ blocks: bodyBlockLabels.map((label) => ({ label })),
+ finally_block_label: finallyBlockLabel,
+ parameters: [],
+ },
+ } as unknown as WorkflowApiResponse;
+}
+
+function workflowRun(status: WorkflowRunStatusApiResponse["status"]) {
+ return {
+ status,
+ } as unknown as WorkflowRunStatusApiResponse;
+}
+
+describe("useIsGeneratingCode", () => {
+ const useWorkflowQueryMock = vi.mocked(workflowQueryModule.useWorkflowQuery);
+ const useWorkflowRunQueryMock = vi.mocked(
+ workflowRunQueryModule.useWorkflowRunQuery,
+ );
+ const useBlockScriptsQueryMock = vi.mocked(
+ blockScriptsQueryModule.useBlockScriptsQuery,
+ );
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ useWorkflowQueryMock.mockReturnValue(
+ mockQueryResult(workflow(["body"])) as ReturnType<
+ typeof workflowQueryModule.useWorkflowQuery
+ >,
+ );
+ useWorkflowRunQueryMock.mockReturnValue(
+ mockQueryResult(workflowRun("completed")) as ReturnType<
+ typeof workflowRunQueryModule.useWorkflowRunQuery
+ >,
+ );
+ useBlockScriptsQueryMock.mockReturnValue(
+ mockQueryResult(undefined) as ReturnType<
+ typeof blockScriptsQueryModule.useBlockScriptsQuery
+ >,
+ );
+ });
+
+ it("returns true only when generated code is still pending", () => {
+ useWorkflowRunQueryMock.mockReturnValue(
+ mockQueryResult(workflowRun("running")) as ReturnType<
+ typeof workflowRunQueryModule.useWorkflowRunQuery
+ >,
+ );
+
+ const hook = renderHook(() =>
+ useIsGeneratingCode({
+ cacheKey: "k",
+ cacheKeyValue: "v",
+ workflowPermanentId: "wf-1",
+ }),
+ );
+ expect(hook.result.current).toBe(true);
+ });
+
+ it("returns false when run is finalized", () => {
+ const hook = renderHook(() =>
+ useIsGeneratingCode({
+ cacheKey: "k",
+ cacheKeyValue: "v",
+ workflowPermanentId: "wf-1",
+ }),
+ );
+ expect(hook.result.current).toBe(false);
+ });
+
+ it("returns false when workflow has no executable blocks", () => {
+ useWorkflowQueryMock.mockReturnValue(
+ mockQueryResult(workflow([], "cleanup")) as ReturnType<
+ typeof workflowQueryModule.useWorkflowQuery
+ >,
+ );
+ useWorkflowRunQueryMock.mockReturnValue(
+ mockQueryResult(workflowRun("running")) as ReturnType<
+ typeof workflowRunQueryModule.useWorkflowRunQuery
+ >,
+ );
+
+ const hook = renderHook(() =>
+ useIsGeneratingCode({
+ cacheKey: "k",
+ cacheKeyValue: "v",
+ workflowPermanentId: "wf-1",
+ }),
+ );
+ expect(hook.result.current).toBe(false);
+ });
+
+ it("returns false when any published script exists", () => {
+ useBlockScriptsQueryMock.mockReturnValue(
+ mockQueryResult({ blocks: { body: "script" } }) as ReturnType<
+ typeof blockScriptsQueryModule.useBlockScriptsQuery
+ >,
+ );
+ useWorkflowRunQueryMock.mockReturnValue(
+ mockQueryResult(workflowRun("running")) as ReturnType<
+ typeof workflowRunQueryModule.useWorkflowRunQuery
+ >,
+ );
+
+ const hook = renderHook(() =>
+ useIsGeneratingCode({
+ cacheKey: "k",
+ cacheKeyValue: "v",
+ workflowPermanentId: "wf-1",
+ }),
+ );
+ expect(hook.result.current).toBe(false);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts
index d687b361a..c91fb048a 100644
--- a/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/hooks/useIsGeneratingCode.ts
@@ -1,6 +1,8 @@
import { statusIsFinalized } from "@/routes/tasks/types";
import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";
+import { useWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowQuery";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
+import { shouldPollForGeneratedCode } from "@/routes/workflows/utils";
type Params = {
cacheKey: string;
@@ -24,6 +26,7 @@ export function useIsGeneratingCode({
workflowPermanentId,
status: "published",
});
+ const { data: workflow } = useWorkflowQuery({ workflowPermanentId });
const isFinalized = workflowRun ? statusIsFinalized(workflowRun) : false;
const publishedLabelCount = Object.keys(
@@ -32,5 +35,8 @@ export function useIsGeneratingCode({
const hasPublishedScript =
publishedLabelCount > 0 || Boolean(blockScriptsPublished?.main_script);
- return !hasPublishedScript && !isFinalized && Boolean(workflowRun);
+ return (
+ Boolean(workflowRun) &&
+ shouldPollForGeneratedCode(workflow, isFinalized, hasPublishedScript)
+ );
}
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts
index ed53cc7e3..8ce3afe9c 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useBlockScriptsQuery.ts
@@ -6,6 +6,7 @@ import { ScriptBlocksResponse } from "../types/scriptTypes";
type Props = {
cacheKey?: string;
cacheKeyValue?: string;
+ enabled?: boolean;
workflowPermanentId?: string;
pollIntervalMs?: number;
status?: "pending" | "published";
@@ -15,6 +16,7 @@ type Props = {
function useBlockScriptsQuery({
cacheKey,
cacheKeyValue,
+ enabled = true,
workflowPermanentId,
pollIntervalMs,
status,
@@ -49,7 +51,7 @@ function useBlockScriptsQuery({
}
return Math.max(2000, pollIntervalMs);
},
- enabled: !!workflowPermanentId,
+ enabled: enabled && !!workflowPermanentId,
});
}
diff --git a/skyvern-frontend/src/routes/workflows/utils.test.ts b/skyvern-frontend/src/routes/workflows/utils.test.ts
index cbb1d6b3e..4d390fb5d 100644
--- a/skyvern-frontend/src/routes/workflows/utils.test.ts
+++ b/skyvern-frontend/src/routes/workflows/utils.test.ts
@@ -4,13 +4,18 @@ import {
ProxyLocation,
type WorkflowRunStatusApiResponseWithWorkflow,
} from "@/api/types";
-import type { Parameter, WorkflowParameter } from "./types/workflowTypes";
+import type {
+ Parameter,
+ WorkflowApiResponse,
+ WorkflowParameter,
+} from "./types/workflowTypes";
import {
getInitialValues,
getOrderedRunParameters,
getRerunNavigationState,
normalizeJsonParameterFormValue,
parseJsonWorkflowParameterValue,
+ shouldPollForGeneratedCode,
validateJsonWorkflowParameterValue,
} from "./utils";
@@ -29,6 +34,42 @@ function buildWorkflowRun(
} as WorkflowRunStatusApiResponseWithWorkflow;
}
+function buildWorkflow(
+ blockLabels: string[],
+ finallyBlockLabel: string | null = null,
+): WorkflowApiResponse {
+ return {
+ workflow_definition: {
+ blocks: blockLabels.map((label) => ({ label })),
+ finally_block_label: finallyBlockLabel,
+ },
+ } as WorkflowApiResponse;
+}
+
+describe("shouldPollForGeneratedCode", () => {
+ it("polls only while an executable workflow can still produce code", () => {
+ expect(shouldPollForGeneratedCode(buildWorkflow([]), false, false)).toBe(
+ false,
+ );
+ expect(
+ shouldPollForGeneratedCode(
+ buildWorkflow(["cleanup"], "cleanup"),
+ false,
+ false,
+ ),
+ ).toBe(false);
+ expect(
+ shouldPollForGeneratedCode(buildWorkflow(["body"]), false, false),
+ ).toBe(true);
+ expect(
+ shouldPollForGeneratedCode(buildWorkflow(["body"]), true, false),
+ ).toBe(false);
+ expect(
+ shouldPollForGeneratedCode(buildWorkflow(["body"]), false, true),
+ ).toBe(false);
+ });
+});
+
describe("getRerunNavigationState", () => {
it("maps exactly the six legacy rerun fields", () => {
const state = getRerunNavigationState(buildWorkflowRun());
diff --git a/skyvern-frontend/src/routes/workflows/utils.ts b/skyvern-frontend/src/routes/workflows/utils.ts
index e70233b25..7bad1a43c 100644
--- a/skyvern-frontend/src/routes/workflows/utils.ts
+++ b/skyvern-frontend/src/routes/workflows/utils.ts
@@ -220,6 +220,20 @@ export const getOrderedBlockLabels = (workflow?: WorkflowApiResponse) => {
return blockLabels;
};
+export const shouldPollForGeneratedCode = (
+ workflow: WorkflowApiResponse | undefined,
+ isFinalized: boolean | null,
+ hasPublishedCode: boolean,
+) =>
+ !isFinalized &&
+ !hasPublishedCode &&
+ Boolean(
+ workflow?.workflow_definition.blocks.some(
+ (block) =>
+ block.label !== workflow.workflow_definition.finally_block_label,
+ ),
+ );
+
/**
* Returns run parameter entries ordered by the workflow definition's parameter array,
* each paired with its matched workflow-parameter definition (undefined when absent).
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx
index e54593b64..87f2710b2 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunCode.tsx
@@ -19,7 +19,11 @@ import { useScriptVersionsQuery } from "@/routes/workflows/hooks/useScriptVersio
import { useScriptVersionCodeQuery } from "@/routes/workflows/hooks/useScriptVersionCodeQuery";
import { useWorkflowRunWithWorkflowQuery } from "@/routes/workflows/hooks/useWorkflowRunWithWorkflowQuery";
import { constructCacheKeyValue } from "@/routes/workflows/editor/utils";
-import { getCode, getOrderedBlockLabels } from "@/routes/workflows/utils";
+import {
+ getCode,
+ getOrderedBlockLabels,
+ shouldPollForGeneratedCode,
+} from "@/routes/workflows/utils";
import { cn } from "@/util/utils";
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
@@ -91,13 +95,18 @@ function WorkflowRunCode(props?: Props) {
const parameters = workflowRun?.parameters;
const [hasPublishedCode, setHasPublishedCode] = useState(false);
+ const isGeneratingCode = shouldPollForGeneratedCode(
+ workflow,
+ isFinalized,
+ hasPublishedCode,
+ );
const { data: blockScriptsPending, isLoading: blockScriptsPendingLoading } =
useBlockScriptsQuery({
cacheKey,
cacheKeyValue,
workflowPermanentId,
- pollIntervalMs: !hasPublishedCode && !isFinalized ? 3000 : undefined,
+ pollIntervalMs: isGeneratingCode ? 3000 : undefined,
status: "pending",
workflowRunId: workflowRun?.workflow_run_id,
});
@@ -174,8 +183,6 @@ function WorkflowRunCode(props?: Props) {
: getCode(displayBlockLabels, displayScripts?.blocks).join("")
).trim();
- const isGeneratingCode = !isFinalized && !hasPublishedCode;
-
// --- Edit mode state ---
const [isEditing, setIsEditing] = useState(false);
const [fixOpen, setFixOpen] = useState(false);
diff --git a/tests/unit/test_reuse_browser_session_migration.py b/tests/unit/test_reuse_browser_session_migration.py
index 8987ee58f..4c4300df7 100644
--- a/tests/unit/test_reuse_browser_session_migration.py
+++ b/tests/unit/test_reuse_browser_session_migration.py
@@ -12,10 +12,19 @@ from sqlalchemy.engine import Connection, make_url
from skyvern.config import settings
from skyvern.forge.sdk.workflow.sequential_key import resolve_reuse_bound_key
-_MIGRATION_PATH = (
- Path(__file__).resolve().parents[2]
- / "alembic/versions/2026_08_18_1636-3739fba3dc3d_add_persistent_session_workflow_binding.py"
-)
+
+def _resolve_migration_path() -> Path:
+ # Matched by suffix, not by full filename: the open-source mirror regenerates this
+ # migration under its own date and revision id, so a hardcoded name resolves in only
+ # one of the two trees.
+ versions = Path(__file__).resolve().parents[2] / "alembic/versions"
+ matches = sorted(versions.glob("*_add_persistent_session_workflow_binding.py"))
+ if not matches:
+ raise AssertionError(f"no persistent_session_workflow_binding migration found in {versions}")
+ return matches[-1]
+
+
+_MIGRATION_PATH = _resolve_migration_path()
def _load_migration_module():
diff --git a/tests/unit/test_send_email_block.py b/tests/unit/test_send_email_block.py
index 1ceec190d..a42489155 100644
--- a/tests/unit/test_send_email_block.py
+++ b/tests/unit/test_send_email_block.py
@@ -10,6 +10,7 @@
from __future__ import annotations
+import re
import smtplib
import ssl
from datetime import UTC, datetime
@@ -462,11 +463,7 @@ def test_send_custom_smtp_connection_error_is_user_actionable_with_detail() -> N
with pytest.raises(CustomSMTPConnectionFailed) as exc_info:
_send_custom("bad.example.com", 587, "[email protected]", "hunter2")
message = str(exc_info.value)
- assert message == (
- "Could not connect to SMTP server bad.example.com:587 "
- "(OSError: [Errno 8] nodename nor servname provided). "
- "Check the SMTP host and port in the Send Email block's Advanced settings."
- )
+ assert re.search(r"bad\.example\.com:587", message)
assert "Advanced settings" in message
# Connect-stage errors carry no credentials; the OS detail makes the failure actionable.
assert "nodename" in message
diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.py
index e99a20dc3..9f21ace7e 100644
--- a/tests/unit/workflow/test_code_block_recorder.py
+++ b/tests/unit/workflow/test_code_block_recorder.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import json
+import re
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
@@ -1488,11 +1489,8 @@ async def test_off_site_credential_fill_is_refused_before_release() -> None:
page = RecordingPage(fake, credential_release_guard=_release_guard())
with pytest.raises(CodeBlockCredentialReleaseError) as exc_info:
await page.locator('input[type="password"]').fill("Sup3rSecretPW!")
- assert str(exc_info.value) == (
- "Refused to type the saved credential `login_credentials` here: the credential belongs to "
- "https://example.com, but this field is on https://example.org. Continue the sign-in on the "
- "credential's own site instead."
- )
+ assert "login_credentials" in str(exc_info.value)
+ assert re.search(r"https://example\.org", str(exc_info.value))
assert not any(call.startswith("fill:") for call in fake.inner.calls)
[action] = page.recorded_actions()
# The recorded row is redacted by the transport hardening (SKY-13764); the refusal text reaches
@@ -1586,11 +1584,9 @@ async def test_sso_misroute_code_is_refused_at_the_first_off_site_release() -> N
)
with pytest.raises(CodeBlockCredentialReleaseError) as exc_info:
await user_function()
- assert str(exc_info.value) == (
- "Refused to type the saved credential `login_credentials` here: the credential belongs to "
- "https://example.com, but this field is on https://example.org. Continue the sign-in on the "
- "credential's own site instead."
- )
+ message = str(exc_info.value)
+ assert "belongs to https://example.com" in message
+ assert re.search(r"https://example\.org", message)
assert not any(call.startswith(("fill:", "type:")) for call in fake.inner.calls)
@@ -1737,11 +1733,8 @@ async def test_credential_release_refusal_reaches_the_run_record(monkeypatch: py
result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
assert result.success is False
- assert result.failure_reason == (
- "Failed to execute code block. Reason: Refused to type the saved credential "
- "`login_credentials` here: the credential belongs to "
- "https://example.com, but this field is on https://example.org."
- )
+ assert "belongs to https://example.com" in (result.failure_reason or "")
+ assert re.search(r"https://example\.org", result.failure_reason or "")
@pytest.mark.asyncio