Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add PII-safe lineage telemetry for download filename naming (SKY-12516) (#7478)
LawyZhengc55be07dc0c1
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsxnew file mode 100644index 000000000..9b70cb5ee--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.historyRace.test.tsx@@ -0,0 +1,370 @@+import {+ act,+ cleanup,+ fireEvent,+ render,+ screen,+ waitFor,+} from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import {+ FeatureFlagContext,+ FeatureFlagValueContext,+} from "@/hooks/useFeatureFlag";+import { COPILOT_UX_V1_FLAG } from "@/util/featureFlags";++type HistoryData = {+ workflow_copilot_chat_id: string | null;+ chat_history: unknown[];+ proposed_workflow: Record<string, unknown> | null;+ auto_accept: boolean;+};++// Only chat-history GETs are deferred (held here so a test controls+// isLoadingHistory); every other GET resolves immediately.+const { postStreaming, cancelPost, historyQueue, flagMap, boolFlags } =+ vi.hoisted(() => ({+ postStreaming: vi.fn(() => new Promise<void>(() => {})),+ cancelPost: vi.fn().mockResolvedValue({}),+ historyQueue: [] as Array<(resp: { data: HistoryData }) => void>,+ flagMap: { current: {} as Record<string, boolean> },+ boolFlags: { current: {} as Record<string, boolean> },+ }));++vi.mock("@/api/sse", () => ({+ getSseClient: vi.fn().mockResolvedValue({ postStreaming }),+}));++vi.mock("@/api/AxiosClient", () => ({+ getClient: vi.fn().mockResolvedValue({+ get: vi.fn((path: string) => {+ if (path === "/workflow/copilot/chat-history") {+ return new Promise((resolve) => {+ historyQueue.push(resolve as (resp: { data: HistoryData }) => void);+ });+ }+ return Promise.resolve({ data: [] });+ }),+ post: cancelPost,+ }),+}));++vi.mock("@/hooks/useCredentialGetter", () => ({+ useCredentialGetter: () => null,+}));++vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));++vi.mock("react-router-dom", async (importOriginal) => {+ const actual = await importOriginal<typeof import("react-router-dom")>();+ return {+ ...actual,+ useParams: () => ({+ workflowPermanentId: "wpid_1",+ workflowRunId: undefined,+ }),+ useSearchParams: () => [new URLSearchParams(), vi.fn()],+ };+});++vi.mock("posthog-js/react", () => ({+ useFeatureFlagEnabled: (flag: string) => flagMap.current[flag] ?? false,+}));++vi.mock("@/store/WorkflowHasChangesStore", () => ({+ useWorkflowHasChangesStore: () => ({+ getSaveData: () => ({+ title: "Test WF",+ workflow: {+ workflow_id: "wf_1",+ workflow_permanent_id: "wpid_1",+ description: "",+ totp_verification_url: null,+ is_saved_task: false,+ status: "published",+ },+ settings: {},+ parameters: [],+ blocks: [],+ workflowDefinitionVersion: 1,+ }),+ }),+}));++vi.mock("@/routes/workflows/hooks/useWorkflowRunQuery", () => ({+ useWorkflowRunQuery: () => ({ data: undefined }),+}));++// The real selector needs an infinite-query + debounced Popover; a plain button+// standing in for "pick a different past chat" is enough to drive the switch.+vi.mock("./WorkflowCopilotHistory", () => ({+ WorkflowCopilotHistory: ({+ onSelect,+ }: {+ onSelect: (chat: { workflow_copilot_chat_id: string }) => void;+ }) => (+ <button+ onClick={() => onSelect({ workflow_copilot_chat_id: "chat_other" })}+ >+ mock-select-history-chat+ </button>+ ),+}));++import { WorkflowCopilotChat } from "./WorkflowCopilotChat";++const narrativePayload = (+ overrides: Record<string, unknown> = {},+): Record<string, unknown> => ({+ turnId: "turn-hist",+ turnIndex: 0,+ mode: "build",+ responseType: "REPLY",+ cancelled: false,+ proposalDisposition: "no_proposal",+ designStarted: true,+ designEnded: true,+ draft: null,+ blocks: [],+ terminal: "response",+ terminalMessage: "All set.",+ narrativeSummary: "All set.",+ priorBlockCount: 0,+ designActivity: [],+ startedAt: "2026-07-15T00:00:00Z",+ endedAt: "2026-07-15T00:00:05Z",+ ...overrides,+});++const aiHistoryMessage = (+ narrative_payload: Record<string, unknown> | null,+ content = "prior turn",+) => ({+ sender: "ai" as const,+ content,+ created_at: "2026-07-15T00:00:00Z",+ narrative_payload,+ turn_outcome: null,+});++const historyData = (overrides: Partial<HistoryData> = {}): HistoryData => ({+ workflow_copilot_chat_id: "chat-1",+ chat_history: [],+ proposed_workflow: null,+ auto_accept: false,+ ...overrides,+});++const portalTargets: HTMLElement[] = [];++function chatUi(props: {+ docked?: boolean;+ requiresLiveBrowser?: boolean;+ isLiveBrowserReady?: boolean;+ portalTarget?: HTMLElement | null;+}) {+ return (+ <FeatureFlagContext.Provider value={(name) => boolFlags.current[name]}>+ <FeatureFlagValueContext.Provider value={() => undefined}>+ <WorkflowCopilotChat+ docked={props.docked ?? false}+ portalTarget={props.portalTarget}+ requiresLiveBrowser={props.requiresLiveBrowser}+ isLiveBrowserReady={props.isLiveBrowserReady}+ />+ </FeatureFlagValueContext.Provider>+ </FeatureFlagContext.Provider>+ );+}++async function renderChat(+ props: Parameters<typeof chatUi>[0] = {},+): Promise<void> {+ let portalTarget: HTMLElement | undefined;+ if (props.docked) {+ portalTarget = document.createElement("div");+ document.body.appendChild(portalTarget);+ portalTargets.push(portalTarget);+ }+ render(chatUi({ ...props, portalTarget }));+ await waitFor(() => expect(screen.getByRole("textbox")).toBeTruthy());+}++async function flushHistory(data: HistoryData): Promise<void> {+ await waitFor(() => expect(historyQueue.length).toBeGreaterThan(0));+ const resolve = historyQueue.shift()!;+ await act(async () => {+ resolve({ data });+ await Promise.resolve();+ });+}++function textarea(): HTMLTextAreaElement {+ return screen.getByRole("textbox") as HTMLTextAreaElement;+}++async function submit(value: string): Promise<void> {+ fireEvent.change(textarea(), { target: { value } });+ await act(async () => {+ fireEvent.keyDown(textarea(), { key: "Enter" });+ });+}++beforeEach(() => {+ HTMLElement.prototype.scrollIntoView = vi.fn();+ HTMLElement.prototype.scrollTo = vi.fn();+ postStreaming.mockClear();+ cancelPost.mockClear();+ cancelPost.mockResolvedValue({});+ historyQueue.length = 0;+ flagMap.current = {};+ boolFlags.current = {};+});++afterEach(() => {+ cleanup();+ portalTargets.splice(0).forEach((el) => el.remove());+});++// Item 1 (SKY-12384): during a chat-history SWITCH, a prior chat's action card+// must not stay actionable — its action would post into the OUTGOING chat.+describe("WorkflowCopilotChat — history-race action-card gating (item 1)", () => {+ it("hides the review gate's Accept while a chat switch is loading (flag-on)", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ await renderChat();+ await flushHistory(+ historyData({+ proposed_workflow: { workflow_id: "wf_p", _copilot_unvalidated: true },+ chat_history: [aiHistoryMessage(null, "Here is a draft.")],+ }),+ );++ await waitFor(() =>+ expect(screen.getByRole("button", { name: "Accept" })).toBeTruthy(),+ );++ fireEvent.click(screen.getByText("mock-select-history-chat"));++ // Switch GET is in flight (isLoadingHistory=true): the outgoing gate's+ // actions are gone, so no Accept can apply into the outgoing chat.+ await waitFor(() =>+ expect(screen.queryByRole("button", { name: "Accept" })).toBeNull(),+ );+ expect(cancelPost).not.toHaveBeenCalledWith(+ "/workflow/copilot/apply-proposed-workflow",+ expect.anything(),+ );++ await flushHistory(historyData({ workflow_copilot_chat_id: "chat_other" }));+ });++ it("hides the Confirm chip while a chat switch is loading (flag-on)", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ await renderChat();+ await flushHistory(+ historyData({+ chat_history: [+ aiHistoryMessage(+ narrativePayload({+ terminalMessage: "Want me to confirm and I'll apply the change?",+ narrativeSummary: "Want me to confirm and I'll apply the change?",+ }),+ "Want me to confirm and I'll apply the change?",+ ),+ ],+ }),+ );++ await waitFor(() =>+ expect(screen.getByRole("button", { name: "Confirm" })).toBeTruthy(),+ );++ fireEvent.click(screen.getByText("mock-select-history-chat"));++ await waitFor(() =>+ expect(screen.queryByRole("button", { name: "Confirm" })).toBeNull(),+ );++ await flushHistory(historyData({ workflow_copilot_chat_id: "chat_other" }));+ });++ it("hides the docked FixCard while a chat switch is loading (flag-off)", async () => {+ await renderChat({ docked: true });+ await flushHistory(+ historyData({+ chat_history: [+ aiHistoryMessage(+ narrativePayload({+ terminal: "error",+ terminalMessage: "The last run hit an error.",+ narrativeSummary: "The last run hit an error.",+ }),+ "The last run hit an error.",+ ),+ ],+ }),+ );++ await waitFor(() =>+ expect(+ screen.getByRole("button", { name: "Fix with Copilot" }),+ ).toBeTruthy(),+ );++ fireEvent.click(screen.getByText("mock-select-history-chat"));++ await waitFor(() =>+ expect(+ screen.queryByRole("button", { name: "Fix with Copilot" }),+ ).toBeNull(),+ );++ await flushHistory(historyData({ workflow_copilot_chat_id: "chat_other" }));+ });+});++// Item 2 (SKY-12384): a live_browser prompt queued before the initial history+// load must keep one owner — footer while the bubble exists, else the chip.+describe("WorkflowCopilotChat — queued prompt survives initial history load (item 2)", () => {+ it("keeps the queued status + Cancel visible after the history load lands", async () => {+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };+ boolFlags.current = { ENABLE_WORKFLOW_COPILOT_V2: true };+ await renderChat({ requiresLiveBrowser: true, isLiveBrowserReady: false });++ // Queue while the initial history GET is still pending (isLoadingHistory).+ // The bubble owns the footer here.+ await submit("log into the portal");+ await waitFor(() =>+ expect(+ screen.getByText("Prompt queued. Waiting for live browser..."),+ ).toBeTruthy(),+ );+ expect(+ screen.getByRole("button", { name: "Cancel queued message" }),+ ).toBeTruthy();++ // History resolves and replaces messages, dropping the queued bubble — the+ // composer chip must pick up the status/Cancel (old code shows neither).+ await flushHistory(+ historyData({ chat_history: [aiHistoryMessage(null, "Earlier chat.")] }),+ );++ expect(+ screen.getByText("Prompt queued. Waiting for live browser..."),+ ).toBeTruthy();+ const cancel = screen.getByRole("button", {+ name: "Cancel queued message",+ });++ await act(async () => {+ fireEvent.click(cancel);+ });++ await waitFor(() =>+ expect(+ screen.queryByText("Prompt queued. Waiting for live browser..."),+ ).toBeNull(),+ );+ });+});diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex 1a21ea789..d8f1a53a5 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -2745,12 +2745,20 @@ export function WorkflowCopilotChat({const gateIndex = gateOwnerIndex >= 0 ? gateOwnerIndex : lastTurnIndex;// Mid-turn Accept would be clobbered by the in-flight turn's terminal// restore, so gate actions wait for idle.- const gateActionable = Boolean(proposedWorkflow) && !isLoading;+ const gateActionable =+ Boolean(proposedWorkflow) && !isLoading && !isLoadingHistory;const hasComposerText = inputValue.trim().length > 0;// A live_browser-reason queued prompt parks with no active turn to stop and// a disabled, emptied textarea — the morph button would otherwise render as// a live "Send" that's a guaranteed no-op; only the Queued chip's ✕ acts.const waitingOnQueueOnly = queuedPrompt?.reason === "live_browser";+ // When the initial history load drops the queued bubble, the composer chip+ // takes over its live_browser status/Cancel (footer-else-chip).+ const queuedBubbleOrphaned = Boolean(+ queuedPrompt &&+ queuedPrompt.reason === "live_browser" &&+ !messages.some((message) => message.id === queuedPrompt.id),+ );const morphButtonLabel = waitingOnQueueOnly? "Send disabled — waiting for live browser": !isLoading@@ -3066,6 +3074,7 @@ export function WorkflowCopilotChat({</>) : null}{copilotUxV1Enabled &&+ !isLoadingHistory &&isLastMessage &&shouldShowConfirmCard(message.narrative) ? (<ConfirmCard@@ -3077,6 +3086,7 @@ export function WorkflowCopilotChat({/>) : null}{docked &&+ !isLoadingHistory &&isLastMessage &&shouldShowFixCard(message.narrative) ? (<FixCard@@ -3089,7 +3099,12 @@ export function WorkflowCopilotChat({/>) : null}{(() => {- if (!copilotUxV1Enabled || turnId === null) return null;+ if (+ isLoadingHistory ||+ !copilotUxV1Enabled ||+ turnId === null+ )+ return null;const credFrame = credentialCardFrameFor(message.narrative,);@@ -3231,6 +3246,7 @@ export function WorkflowCopilotChat({uxV1={copilotUxV1Enabled}/>{copilotUxV1Enabled &&+ !isLoadingHistory &&livePauseFrame &&livePauseFrame.turn_id === narrative.turnId ? (<CredentialCard@@ -3340,7 +3356,7 @@ export function WorkflowCopilotChat({{copilotUxV1Enabled &&copilotV2Enabled &&queuedPrompt &&- queuedPrompt.reason === "working" ? (+ (queuedPrompt.reason === "working" || queuedBubbleOrphaned) ? (<div className="mb-2 flex items-center gap-2 rounded-lg border border-border bg-slate-elevation2 px-2.5 py-1.5 text-xs text-muted-foreground"><ReloadIcon className="h-3 w-3 shrink-0 animate-spin" /><span className="shrink-0 font-medium text-foreground">Queued</span>diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex fd14bd8e1..8520f2683 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -96,6 +96,7 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.cache import extraction_cache, extraction_shadowfrom skyvern.forge.sdk.copilot.block_goal_wrapping import unwrap_goal_fieldsfrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import TaskType@@ -562,6 +563,21 @@ class ForgeAgent:if Path(f).name != local_basename}desired_name = download_filename_from_suffix(download_suffix, file_extension, existing_names)+ # finalize_* keys avoid the forge_log processor, which overwrites bare task_id/+ # workflow_run_id with the ambient context's values; under a stale/shared context those+ # would otherwise mask the task actually being finalized (the divergence to diagnose).+ LOG.info(+ "download_suffix_finalize_rename",+ execution_path="agent_finalize",+ finalize_task_id=task.task_id,+ finalize_workflow_run_id=task.workflow_run_id,+ context_task_id=context.task_id if context else None,+ pre_rename_filename_fp=diagnostic_fingerprint(local_basename),+ passed_download_suffix_fp=diagnostic_fingerprint(download_suffix),+ context_download_suffix_fp=diagnostic_fingerprint(context.download_suffix if context else None),+ desired_name_fp=diagnostic_fingerprint(desired_name),+ will_rename=local_basename != desired_name,+ )if local_basename != desired_name:rename_file(os.path.join(workflow_download_directory, local_file_name), desired_name)continuediff --git a/skyvern/forge/sdk/core/hashing.py b/skyvern/forge/sdk/core/hashing.pyindex 4323ca2f4..60198c58a 100644--- a/skyvern/forge/sdk/core/hashing.py+++ b/skyvern/forge/sdk/core/hashing.py@@ -1,5 +1,46 @@import hashlib+import hmac++from skyvern.config import settings++# Domain separator so this fingerprint can never collide with any other use of SECRET_KEY (e.g. auth+# token signing). Bump the version suffix if the construction ever changes.+_FP_DOMAIN = b"skyvern.download_suffix.diagnostic_fingerprint.v1"def generate_url_hash(url: str) -> str:return hashlib.sha256(url.encode()).hexdigest()+++def _fingerprint_key() -> bytes | None:+ """The server-side HMAC key, or None when no real secret is configured (fail closed).++ Compares against the ``SECRET_KEY`` field default (rather than a hardcoded literal) so it keeps+ fail-closing if that placeholder default ever changes.+ """+ key = settings.SECRET_KEY+ if not key or key == type(settings).model_fields["SECRET_KEY"].default:+ return None+ return key.encode("utf-8")+++def diagnostic_fingerprint(value: str | None) -> str:+ """Keyed, non-reversible tag for correlating a possibly-sensitive string across logs.++ Returns ``none`` for None and ``empty:0`` for the empty string. Otherwise returns+ ``<hmac_sha256(SECRET_KEY, value)[:12]>:<len>`` — keyed by the deployment secret so equal values map+ to equal tags for correlation, while the raw value (a download_suffix is often a low-entropy,+ account-number-bearing filename) cannot be recovered by brute force from log access alone; a bare+ unsalted hash of a low-entropy value is trivially reversible. Fails closed to ``unkeyed`` when no+ server-side key is configured rather than emitting a brute-forceable bare hash. The raw value and the+ key never appear in the output. ``surrogatepass`` keeps invalid-UTF-8 on-disk filenames from raising.+ """+ if value is None:+ return "none"+ if not value:+ return "empty:0"+ key = _fingerprint_key()+ if key is None:+ return "unkeyed"+ digest = hmac.new(key, _FP_DOMAIN + b"\x00" + value.encode("utf-8", "surrogatepass"), hashlib.sha256)+ return f"{digest.hexdigest()[:12]}:{len(value)}"diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 5ed2e47b2..5a5d0e64d 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -106,6 +106,7 @@ from skyvern.forge.sdk.copilot.self_heal_recovery import SelfHealRecoveryResult,from skyvern.forge.sdk.copilot.turn_origin import HealAdoptionFailed, TurnOriginfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request+from skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.db.exceptions import NotFoundError@@ -1122,6 +1123,12 @@ class BaseTaskBlock(Block):)# encode the suffix to prevent invalid path styleself.download_suffix = quote(string=self.download_suffix, safe="")+ LOG.info(+ "download_suffix_rendered",+ block_label=self.label,+ current_index=workflow_run_context.get_block_metadata(self.label).get("current_index"),+ download_suffix_fp=diagnostic_fingerprint(self.download_suffix),+ )if self.navigation_goal:self.navigation_goal = self.format_block_parameter_template_from_workflow_run_context(diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 8e3e70c48..d103d351e 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -46,6 +46,7 @@ from skyvern.forge.sdk.api.files import ()from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -2338,6 +2339,22 @@ async def download(if Path(f).name != local_basename}desired_name = download_filename_from_suffix(download_suffix, file_extension, existing_names)+ # context suffix fields are omitted: cached-script mode bakes the suffix into the+ # generated script (no per-step contextvar stamping), so there is no task_block-vs-+ # context divergence to attribute; task_id/block_label give per-download attribution.+ # finalize_* keys avoid the forge_log processor overwriting bare task_id/+ # workflow_run_id with the ambient context's values (see agent finalize path).+ LOG.info(+ "download_suffix_finalize_rename",+ execution_path="cached_script",+ finalize_workflow_run_id=run_id,+ finalize_task_id=task_id,+ block_label=cache_key,+ pre_rename_filename_fp=diagnostic_fingerprint(local_basename),+ passed_download_suffix_fp=diagnostic_fingerprint(download_suffix),+ desired_name_fp=diagnostic_fingerprint(desired_name),+ will_rename=local_basename != desired_name,+ )if local_basename != desired_name:file_path = rename_file(file_path, desired_name)newly_downloaded_files.append(file_path)diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 22e5ab4ee..bafe29c64 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -84,6 +84,7 @@ from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extractfrom skyvern.forge.sdk.cache import extraction_cache, extraction_shadowfrom skyvern.forge.sdk.copilot.block_goal_wrapping import unwrap_goal_fieldsfrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, ensure_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler@@ -642,7 +643,15 @@ def _download_target_path(download_dir: Path, suggested_filename: str | None) -># Name the file by the block-configured download_suffix so the watcher syncs the# request-based name instead of the site's suggested name.existing = {p.name for p in download_dir.iterdir()} if download_dir.exists() else set()- return download_dir / download_filename_from_suffix(download_suffix, suffix, existing)+ target_name = download_filename_from_suffix(download_suffix, suffix, existing)+ LOG.info(+ "download_suffix_target_named",+ context_task_id=context.task_id if context else None,+ context_download_suffix_fp=diagnostic_fingerprint(download_suffix),+ suggested_filename_fp=diagnostic_fingerprint(suggested_filename),+ desired_name_fp=diagnostic_fingerprint(target_name),+ )+ return download_dir / target_namereturn download_dir / f"{uuid.uuid4()}-{stem or 'download'}{suffix}"diff --git a/tests/unit/_fingerprint_expectations.py b/tests/unit/_fingerprint_expectations.pynew file mode 100644index 000000000..88582ce8d--- /dev/null+++ b/tests/unit/_fingerprint_expectations.py@@ -0,0 +1,36 @@+"""Shared expected-value helpers for the keyed ``diagnostic_fingerprint`` contract.++``expected_fingerprint`` re-implements the keyed HMAC independently (it does NOT call the production+helper) so the tests catch any drift in the algorithm while still asserting concrete expected values.+Pair it with the ``fingerprint_secret_key`` fixture (in ``conftest.py``), which pins ``SECRET_KEY`` to+``FINGERPRINT_TEST_SECRET_KEY`` so production and these helpers agree.+"""++from __future__ import annotations++import hashlib+import hmac++# Must stay byte-identical to skyvern.forge.sdk.core.hashing._FP_DOMAIN. The keyed contract tests assert+# full production output against expected_fingerprint(), so any divergence here surfaces as a failure.+_FP_DOMAIN = b"skyvern.download_suffix.diagnostic_fingerprint.v1"++FINGERPRINT_TEST_SECRET_KEY = "sky-unit-test-fingerprint-secret-v1"+++def expected_fingerprint(value: str | None, *, key: str = FINGERPRINT_TEST_SECRET_KEY) -> str:+ if value is None:+ return "none"+ if not value:+ return "empty:0"+ digest = hmac.new(+ key.encode("utf-8"),+ _FP_DOMAIN + b"\x00" + value.encode("utf-8", "surrogatepass"),+ hashlib.sha256,+ )+ return f"{digest.hexdigest()[:12]}:{len(value)}"+++def bare_sha256_fingerprint(value: str) -> str:+ """The OLD insecure format (unsalted sha256). Used only to prove production no longer emits it."""+ return f"{hashlib.sha256(value.encode()).hexdigest()[:12]}:{len(value)}"diff --git a/tests/unit/conftest.py b/tests/unit/conftest.pyindex 1c0967e24..b263a0075 100644--- a/tests/unit/conftest.py+++ b/tests/unit/conftest.py@@ -23,8 +23,22 @@ from sqlalchemy.ext.asyncio import AsyncEngine, create_async_enginefrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.db.models import Base+from tests.unit._fingerprint_expectations import FINGERPRINT_TEST_SECRET_KEYfrom tests.unit.force_stub_app import start_forge_stub_app++def fingerprint_secret_key(monkeypatch: pytest.MonkeyPatch) -> str:+ """Pin ``SECRET_KEY`` so ``diagnostic_fingerprint`` produces stable, keyed output in tests.++ Patches the shared ``settings`` singleton, so it is seen wherever the helper reads it.+ """+ from skyvern.config import settings++ monkeypatch.setattr(settings, "SECRET_KEY", FINGERPRINT_TEST_SECRET_KEY)+ return FINGERPRINT_TEST_SECRET_KEY++# Wire structlog through stdlib so caplog can capture log records in tests.structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),diff --git a/tests/unit/fake_workflow_run_context.py b/tests/unit/fake_workflow_run_context.pyindex 71184ee3d..e76531941 100644--- a/tests/unit/fake_workflow_run_context.py+++ b/tests/unit/fake_workflow_run_context.py@@ -45,6 +45,12 @@ class FakeWorkflowRunContext:else:self._blocks_metadata[label] = dict(metadata)+ def resolve_effective_workflow_system_prompt(self) -> str | None:+ return None++ def record_block_workflow_system_prompt(self, label: str, value: str | None) -> None:+ return None+def has_value(self, key: str) -> bool:return key in self.valuesdiff --git a/tests/unit/test_agent_cleanup_download_suffix.py b/tests/unit/test_agent_cleanup_download_suffix.pyindex 2be7e43f8..b30911c66 100644--- a/tests/unit/test_agent_cleanup_download_suffix.py+++ b/tests/unit/test_agent_cleanup_download_suffix.py@@ -5,10 +5,22 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from structlog.testing import capture_logsfrom skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.schemas.runs import RunEnginefrom skyvern.webeye.actions.models import DetailedAgentStepOutput+from tests.unit._fingerprint_expectations import expected_fingerprint++[email protected](autouse=True)+def _keyed_fingerprint(fingerprint_secret_key: str) -> str:+ return fingerprint_secret_key+++def _finalize_events(cap: list[dict]) -> list[dict]:+ return [e for e in cap if e.get("event") == "download_suffix_finalize_rename"]def _make_task(@@ -417,3 +429,183 @@ async def test_execute_step_reuses_initial_download_baseline_across_recursive_stassert (download_dir / "req-123.zip").exists()assert not (download_dir / "uuid-file.zip").exists()+++async def test_finalize_emits_lineage_with_correlation_fields(tmp_path) -> None:+ agent = ForgeAgent()+ task = _make_task(task_id="task-A", workflow_run_id="wr-A")+ download_dir = tmp_path / "downloads"+ download_dir.mkdir()+ stale_ctx = SkyvernContext(task_id="task-0", download_suffix="AllDataExport_ACCT_STALE")++ with (+ patch("skyvern.forge.agent.get_path_for_workflow_download_directory", return_value=download_dir),+ patch("skyvern.forge.agent.list_files_in_directory", return_value=["uuid-file.zip"]),+ patch("skyvern.forge.agent.rename_file", MagicMock()),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=stale_ctx),+ capture_logs() as cap,+ ):+ await agent._finalize_downloaded_files_for_task(+ task,+ organization_id=task.organization_id,+ download_suffix="AllDataExport_ACCT_CURRENT",+ list_files_before=[],+ randomize_if_missing=False,+ )++ events = _finalize_events(cap)+ assert len(events) == 1+ event = events[0]+ assert event["finalize_task_id"] == "task-A"+ assert event["finalize_workflow_run_id"] == "wr-A"+ assert event["pre_rename_filename_fp"] == expected_fingerprint("uuid-file.zip")+ assert event["passed_download_suffix_fp"] == expected_fingerprint("AllDataExport_ACCT_CURRENT")+ assert event["desired_name_fp"] == expected_fingerprint("AllDataExport_ACCT_CURRENT.zip")+ assert event["will_rename"] is True+ # The contextvar suffix is captured alongside the task_block suffix so a divergence (stale context vs+ # late-download) is attributable; here they intentionally differ.+ assert event["context_download_suffix_fp"] == expected_fingerprint("AllDataExport_ACCT_STALE")+ assert event["context_task_id"] == "task-0"+ # Bare task_id/workflow_run_id must NOT be emitted: the forge_log processor overwrites them with the+ # ambient context's values, which under a stale/shared context would mask the finalize target.+ assert "task_id" not in event+ assert "workflow_run_id" not in event+++async def test_finalize_lineage_distinguishes_two_tasks(tmp_path) -> None:+ agent = ForgeAgent()+ records: list[dict] = []+ for account, task_id in (("ACCT_AAA", "task-A"), ("ACCT_BBB", "task-B")):+ download_dir = tmp_path / task_id+ download_dir.mkdir()+ task = _make_task(task_id=task_id, workflow_run_id=f"wr-{task_id}")+ with (+ patch("skyvern.forge.agent.get_path_for_workflow_download_directory", return_value=download_dir),+ patch("skyvern.forge.agent.list_files_in_directory", return_value=["uuid-file.zip"]),+ patch("skyvern.forge.agent.rename_file", MagicMock()),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),+ capture_logs() as cap,+ ):+ await agent._finalize_downloaded_files_for_task(+ task,+ organization_id=task.organization_id,+ download_suffix=f"Export_{account}",+ list_files_before=[],+ randomize_if_missing=False,+ )+ records.extend(_finalize_events(cap))++ assert len(records) == 2+ assert {r["finalize_task_id"] for r in records} == {"task-A", "task-B"}+ assert len({r["passed_download_suffix_fp"] for r in records}) == 2 # two iterations distinguishable+++async def test_execute_step_complete_on_download_emits_finalize_lineage(tmp_path) -> None:+ agent = ForgeAgent()+ download_dir = tmp_path / "downloads"+ download_dir.mkdir()+ suffix = "AllDataExport_ACCT_ZZZ"+ real_ctx = SkyvernContext(task_id="task-1", workflow_run_id="wr-1")++ task = _make_task()+ task.status = SimpleNamespace(value="running")+ task.navigation_goal = "Download invoice"+ task.data_extraction_goal = None+ task.complete_criterion = None+ task.terminate_criterion = None+ task.browser_address = None+ task.max_steps_per_run = None+ task.url = "https://example.com"+ task.proxy_location = None+ task.llm_key = None+ task.task_type = "general"++ step = MagicMock()+ step.step_id = "step-1"+ step.order = 0+ step.retry_index = 0+ step.status = "created"++ organization = MagicMock()+ organization.organization_id = task.organization_id+ organization.max_steps_per_run = None++ task_block = MagicMock()+ task_block.label = "bill_usage_download"+ task_block.complete_on_download = True+ task_block.download_timeout = None+ task_block.download_suffix = suffix++ browser_state = MagicMock()+ browser_state.get_working_page = AsyncMock(return_value=None)++ async def agent_step_side_effect(*args, **kwargs):+ (download_dir / "uuid-file.zip").write_text("dummy")+ return step, DetailedAgentStepOutput(+ scraped_page=None,+ extract_action_prompt=None,+ llm_response=None,+ actions=None,+ action_results=None,+ actions_and_results=None,+ cua_response=None,+ )++ async def update_step_side_effect(step_obj, *args, **kwargs):+ if "status" in kwargs:+ step_obj.status = kwargs["status"]+ if "is_last" in kwargs:+ step_obj.is_last = kwargs["is_last"]+ return step_obj++ async def update_task_side_effect(task_obj, *args, **kwargs):+ return task_obj++ with (+ patch("skyvern.forge.agent.analytics.capture"),+ patch("skyvern.forge.agent.otel_trace.get_current_span", return_value=MagicMock()),+ patch("skyvern.forge.agent.skyvern_context.ensure_context", return_value=real_ctx),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=real_ctx),+ patch("skyvern.forge.agent.get_path_for_workflow_download_directory", return_value=download_dir),+ patch("skyvern.forge.agent.list_downloading_files_in_directory", return_value=[]),+ patch("skyvern.forge.agent.app") as mock_app,+ patch.object(agent, "initialize_execution_state", AsyncMock(return_value=(step, browser_state, None))),+ patch.object(agent, "agent_step", AsyncMock(side_effect=agent_step_side_effect)),+ patch.object(agent, "update_step", AsyncMock(side_effect=update_step_side_effect)),+ patch.object(agent, "update_task", AsyncMock(side_effect=update_task_side_effect)),+ patch.object(agent, "update_task_errors_from_detailed_output", AsyncMock(return_value=task)),+ capture_logs() as cap,+ ):+ mock_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=None)+ mock_app.DATABASE.tasks.get_task = AsyncMock(return_value=task)+ mock_app.DATABASE.tasks.update_task = AsyncMock(return_value=task)+ mock_app.AGENT_FUNCTION.validate_step_execution = AsyncMock()+ mock_app.AGENT_FUNCTION.post_step_execution = AsyncMock()+ mock_app.ARTIFACT_MANAGER.flush_step_archive = AsyncMock()+ mock_app.BROWSER_MANAGER.get_for_task = MagicMock(return_value=None)+ mock_app.STORAGE.save_downloaded_files = AsyncMock()+ mock_app.STORAGE.list_downloaded_files_in_browser_session = AsyncMock(return_value=[])++ await agent.execute_step(+ organization=organization,+ task=task,+ step=step,+ task_block=task_block,+ close_browser_on_completion=True,+ complete_verification=True,+ engine=RunEngine.skyvern_v1,+ )++ finalize_events = _finalize_events(cap)+ assert finalize_events, "finalize lineage was not emitted through the real complete_on_download path"++ event = finalize_events[0]+ assert event["finalize_task_id"] == "task-1"+ # The finalize lineage consumes the task_block-derived suffix and records the contextvar value too,+ # so a task_block-vs-context divergence is attributable at the point the filename is decided.+ assert event["passed_download_suffix_fp"] == expected_fingerprint(suffix)+ assert event["context_download_suffix_fp"] == expected_fingerprint(suffix)+ assert (download_dir / f"{suffix}.zip").exists()diff --git a/tests/unit/test_cached_download_s3_upload.py b/tests/unit/test_cached_download_s3_upload.pyindex 55c68641d..e1265d59a 100644--- a/tests/unit/test_cached_download_s3_upload.py+++ b/tests/unit/test_cached_download_s3_upload.py@@ -16,10 +16,18 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from structlog.testing import capture_logs++from tests.unit._fingerprint_expectations import expected_fingerprintMODULE = "skyvern.services.script_service"[email protected](autouse=True)+def _keyed_fingerprint(fingerprint_secret_key: str) -> str:+ return fingerprint_secret_key++def _make_mock_app(storage):"""Create a mock that replaces the `app` module-level reference in script_service."""mock_app = MagicMock()@@ -529,6 +537,45 @@ async def test_download_suffix_rename_uses_file_path_directly(setup, tmp_path):_cleanup(refs)+async def test_cached_download_rename_emits_finalize_lineage(setup, tmp_path):+ """The cached-script (run_with=code) rename path must emit the same download_suffix_finalize_rename+ lineage as the agent path, tagged execution_path=cached_script, so a code-mode freeze is attributable."""+ download_dir = tmp_path / "downloads"+ abs_path = str(download_dir / "abc123.pdf")++ rename_mock = MagicMock(return_value=str(download_dir / "invoice.pdf"))+ refs = setup(+ get_side_effect=[[], ["invoice.pdf"]],+ list_files_side_effect=[[], [abs_path], [abs_path], [abs_path]],+ rename_mock=rename_mock,+ )+ try:+ from skyvern.services.script_service import download++ with capture_logs() as cap:+ await download(+ prompt="Download invoice",+ download_suffix="invoice",+ label="test_block",+ cache_key="bill_usage_download",+ )++ events = [e for e in cap if e.get("event") == "download_suffix_finalize_rename"]+ assert len(events) == 1, "cached rename path did not emit download_suffix_finalize_rename"+ event = events[0]+ assert event["execution_path"] == "cached_script"+ assert event["finalize_workflow_run_id"] == "wr_test_run"+ assert event["finalize_task_id"] == "tsk_1" # per-download-task attribution in cached mode+ assert event["block_label"] == "bill_usage_download" # persisted block label (= cache_key)+ assert event["passed_download_suffix_fp"] == expected_fingerprint("invoice")+ assert event["pre_rename_filename_fp"] == expected_fingerprint("abc123.pdf")+ assert event["desired_name_fp"] == expected_fingerprint("invoice.pdf")+ assert event["will_rename"] is True+ finally:+ _cleanup(refs)++@pytest.mark.asyncioasync def test_verification_raises_when_no_new_file(setup):"""When no new file appears on local filesystem, the poll loop should exhaustdiff --git a/tests/unit/test_download_suffix_lineage.py b/tests/unit/test_download_suffix_lineage.pynew file mode 100644index 000000000..e013cac09--- /dev/null+++ b/tests/unit/test_download_suffix_lineage.py@@ -0,0 +1,403 @@+"""SKY-12516 download_suffix lineage observability.++The naming freeze (all loop iterations' downloads named by iteration-0's account) is a post-render+runtime phenomenon: render is provably correct per iteration, yet the finalized artifact freezes.+These tests pin the diagnostic contract that lets the next production occurrence be attributed:++ (a) render -> ``download_suffix_rendered`` (block label, current index, suffix fp)+ (c) finalize path -> ``download_suffix_finalize_rename`` (task_block suffix vs context suffix vs desired)+ (c) at-save path -> ``download_suffix_target_named`` (context suffix, context task id, desired)++plus a 4-case harness driving the REAL render/copy + REAL finalize + REAL contextvar naming to locate+exactly where an expected suffix A becomes a consumed suffix B. Account values here are synthetic.+"""++from __future__ import annotations++import asyncio+import json+from datetime import UTC, datetime+from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest+from structlog.testing import capture_logs++from skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.workflow.models.block import FileDownloadBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter+from skyvern.webeye.actions.handler import _download_target_path+from tests.unit._fingerprint_expectations import (+ FINGERPRINT_TEST_SECRET_KEY,+ bare_sha256_fingerprint,+ expected_fingerprint,+)+from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext++SUFFIX_TEMPLATE = "AllDataExport_UsageDetail_{{current_value.account_number}}"+# Synthetic, obviously-fake account numbers (never real customer values).+ACCOUNTS = ["ACCT_AAA_1001", "ACCT_BBB_2002", "ACCT_CCC_3003"]+SITE_FILENAME = "detail_report.csv"++[email protected](autouse=True)+def _keyed_fingerprint(fingerprint_secret_key: str) -> str:+ """Every test here asserts against the keyed fingerprint, so pin the test key module-wide."""+ return fingerprint_secret_key+++def _rendered_suffix(account: str) -> str:+ return f"AllDataExport_UsageDetail_{account}"+++def _make_output_param(label: str) -> OutputParameter:+ now = datetime.now(UTC)+ return OutputParameter(+ output_parameter_id=f"op_{label}",+ key=f"{label}_output",+ workflow_id="wf_test",+ created_at=now,+ modified_at=now,+ )+++def _make_download_block() -> FileDownloadBlock:+ return FileDownloadBlock(+ label="bill_usage_download",+ url="https://billing.example.com/usage",+ navigation_goal="Select account {{current_value.account_number}} then export",+ download_suffix=SUFFIX_TEMPLATE,+ output_parameter=_make_output_param("bill_usage_download"),+ )+++def _loop_context(block_label: str, index: int, account: str) -> FakeWorkflowRunContext:+ return FakeWorkflowRunContext(+ values={},+ block_metadata={+ block_label: {+ "current_index": index,+ "current_value": {"account_number": account},+ "current_item": {"account_number": account},+ }+ },+ )+++def _events(cap: list[dict], name: str) -> list[dict]:+ return [e for e in cap if e.get("event") == name]+++def _emit_artifact(case: str, records: list[dict]) -> None:+ """Print lineage as JSONL so a ``pytest -s`` runner can tee it to the Hermes artifact path."""+ for record in records:+ print(f"LINEAGE {case} {json.dumps(record, sort_keys=True, default=str)}")+++def _make_task(task_id: str, workflow_run_id: str = "wr-harness") -> MagicMock:+ task = MagicMock()+ task.task_id = task_id+ task.organization_id = "org-harness"+ task.workflow_run_id = workflow_run_id+ task.browser_session_id = None+ return task+++# --------------------------------------------------------------------------------------------------+# diagnostic_fingerprint keyed contract (must NOT be a bare/unsalted hash of low-entropy PII values)+# --------------------------------------------------------------------------------------------------+++def test_diagnostic_fingerprint_none_and_empty() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ assert diagnostic_fingerprint(None) == "none"+ assert diagnostic_fingerprint("") == "empty:0"+++def test_diagnostic_fingerprint_is_keyed_not_bare_sha256() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ value = _rendered_suffix(ACCOUNTS[0])+ fp = diagnostic_fingerprint(value)+ assert fp == expected_fingerprint(value) # keyed HMAC of the value+ assert fp != bare_sha256_fingerprint(value) # NOT the offline-brute-forceable unsalted sha256+ assert fp.endswith(f":{len(value)}")+ assert len(fp.split(":")[0]) == 12+++def test_diagnostic_fingerprint_stable_for_same_key_and_value() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ value = _rendered_suffix(ACCOUNTS[1])+ assert diagnostic_fingerprint(value) == diagnostic_fingerprint(value)+++def test_diagnostic_fingerprint_changes_with_key() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ value = _rendered_suffix(ACCOUNTS[0])+ with patch("skyvern.forge.sdk.core.hashing.settings.SECRET_KEY", "key-alpha"):+ fp_a = diagnostic_fingerprint(value)+ with patch("skyvern.forge.sdk.core.hashing.settings.SECRET_KEY", "key-beta"):+ fp_b = diagnostic_fingerprint(value)+ assert fp_a != fp_b # keyed: a different secret yields a different tag+ assert fp_a == expected_fingerprint(value, key="key-alpha")+ assert fp_b == expected_fingerprint(value, key="key-beta")+++def test_diagnostic_fingerprint_leaks_neither_value_nor_key() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ value = _rendered_suffix(ACCOUNTS[0])+ fp = diagnostic_fingerprint(value)+ assert value not in fp+ assert ACCOUNTS[0] not in fp+ assert FINGERPRINT_TEST_SECRET_KEY not in fp+++def test_diagnostic_fingerprint_fails_closed_without_key() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ value = _rendered_suffix(ACCOUNTS[0])+ for missing_key in ("PLACEHOLDER", ""):+ with patch("skyvern.forge.sdk.core.hashing.settings.SECRET_KEY", missing_key):+ fp = diagnostic_fingerprint(value)+ assert fp == "unkeyed" # fail closed: never emit a brute-forceable bare hash+ assert fp != bare_sha256_fingerprint(value)+ assert value not in fp+++def test_diagnostic_fingerprint_handles_surrogate_filenames() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ # An invalid-UTF-8 on-disk filename surfaces via surrogateescape; the helper must not raise.+ surrogate = "invoice_\udc80.pdf"+ fp = diagnostic_fingerprint(surrogate)+ assert fp == expected_fingerprint(surrogate)+ assert fp.endswith(f":{len(surrogate)}")+++def test_diagnostic_fingerprint_distinguishes_values() -> None:+ from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint++ fps = {diagnostic_fingerprint(_rendered_suffix(a)) for a in ACCOUNTS}+ assert len(fps) == len(ACCOUNTS)+++# --------------------------------------------------------------------------------------------------+# (a) render boundary+# --------------------------------------------------------------------------------------------------+++def test_render_boundary_emits_lineage_per_iteration_and_leaves_template_untouched() -> None:+ block = _make_download_block()+ records: list[dict] = []+ for index, account in enumerate(ACCOUNTS):+ ctx = _loop_context(block.label, index, account)+ copy = block.model_copy(deep=True)+ with capture_logs() as cap:+ copy.format_potential_template_parameters(ctx)+ events = _events(cap, "download_suffix_rendered")+ assert len(events) == 1, f"expected one render lineage event for iteration {index}"+ event = events[0]+ assert event["block_label"] == "bill_usage_download"+ assert event["current_index"] == index+ assert event["download_suffix_fp"] == expected_fingerprint(_rendered_suffix(account))+ assert copy.download_suffix == _rendered_suffix(account) # render is correct per iteration+ records.append(event)++ _emit_artifact("render", records)+ assert len({r["download_suffix_fp"] for r in records}) == len(ACCOUNTS) # iterations distinguishable+ # The shared original template must never be mutated (freeze is not a render/mutate bug).+ assert block.download_suffix == SUFFIX_TEMPLATE+++# --------------------------------------------------------------------------------------------------+# 4-case reproduction harness (real render/copy + real finalize + real contextvar naming)+# --------------------------------------------------------------------------------------------------+++async def test_harness_case1_finalize_names_per_iteration_no_freeze(tmp_path: Path) -> None:+ """Deterministic path prod actually used: constant site filename, per-iteration task_block suffix.++ Finalize names each file by the ITERATION's own suffix -> three distinct account names, no freeze.+ """+ agent = ForgeAgent()+ block = _make_download_block()+ download_dir = tmp_path / "downloads"+ download_dir.mkdir()+ records: list[dict] = []++ for index, account in enumerate(ACCOUNTS):+ ctx = _loop_context(block.label, index, account)+ copy = block.model_copy(deep=True)+ copy.format_potential_template_parameters(ctx) # real per-iteration render+ before = sorted(str(p) for p in download_dir.iterdir())+ (download_dir / SITE_FILENAME).write_text(f"bytes-{account}") # constant site name each iteration+ task = _make_task(task_id=f"task-{index}")+ with (+ patch("skyvern.forge.agent.get_path_for_workflow_download_directory", return_value=download_dir),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),+ capture_logs() as cap,+ ):+ await agent._finalize_downloaded_files_for_task(+ task,+ organization_id=task.organization_id,+ download_suffix=copy.download_suffix,+ list_files_before=before,+ randomize_if_missing=False,+ )+ records.extend(_events(cap, "download_suffix_finalize_rename"))++ _emit_artifact("case1_finalize", records)+ names = {p.name for p in download_dir.iterdir()}+ assert names == {f"{_rendered_suffix(a)}.csv" for a in ACCOUNTS}, names # 3 distinct -> NO freeze+ assert len({r["passed_download_suffix_fp"] for r in records}) == len(ACCOUNTS)+++async def test_harness_case2_frozen_contextvar_reproduces_freeze_on_atsave_path(tmp_path: Path) -> None:+ """At-save naming via a contextvar that was stamped at iteration 0 and never re-stamped.++ All three downloads collapse onto iteration-0's suffix (+ dedup) -> the observed freeze signature.+ """+ download_dir = tmp_path / "downloads"+ download_dir.mkdir()+ frozen = SkyvernContext(task_id="task-0", download_suffix=_rendered_suffix(ACCOUNTS[0]))+ produced: list[str] = []+ records: list[dict] = []++ for account in ACCOUNTS: # later iterations, but context never advanced+ with skyvern_context.scoped(frozen), capture_logs() as cap:+ target = _download_target_path(download_dir, SITE_FILENAME)+ target.write_text(f"bytes-{account}")+ produced.append(target.name)+ records.extend(_events(cap, "download_suffix_target_named"))++ _emit_artifact("case2_frozen_context", records)+ base = _rendered_suffix(ACCOUNTS[0])+ assert produced == [f"{base}.csv", f"{base}_1.csv", f"{base}_2.csv"], produced # FREEZE reproduced+ assert len({r["context_download_suffix_fp"] for r in records}) == 1 # frozen suffix fp+++async def test_harness_case3_shared_page_callback_captures_stale_context(tmp_path: Path) -> None:+ """A download whose naming runs under a context captured at iteration 0 while the loop is on a later+ iteration. ``context_task_id`` in the lineage is the discriminator: it stays task-0 (stale)."""+ download_dir = tmp_path / "downloads"+ download_dir.mkdir()+ captured_iter0_ctx = SkyvernContext(task_id="task-0", download_suffix=_rendered_suffix(ACCOUNTS[0]))+ records: list[dict] = []++ # Loop is on iteration 2 (task-2), but the late download names under the captured iteration-0 context.+ current_iteration_task_id = "task-2"+ with skyvern_context.scoped(captured_iter0_ctx), capture_logs() as cap:+ target = _download_target_path(download_dir, SITE_FILENAME)+ target.write_text("late-bytes")+ records.extend(_events(cap, "download_suffix_target_named"))++ _emit_artifact("case3_stale_capture", records)+ assert target.name == f"{_rendered_suffix(ACCOUNTS[0])}.csv" # named by stale iteration-0 suffix+ assert len(records) == 1+ assert records[0]["context_task_id"] == "task-0"+ assert records[0]["context_task_id"] != current_iteration_task_id # stale-context attribution caught+++async def test_harness_case4_finalize_taskblock_path_diverges_from_contextvar_path(tmp_path: Path) -> None:+ """Same downloaded file, two naming paths. finalize uses the PASSED task_block suffix (current+ iteration); the at-save path uses the contextvar (possibly stale). The lineage exposes the divergence,+ which is the field that will attribute the next production occurrence to one path or the other."""+ agent = ForgeAgent()+ current_account, stale_account = ACCOUNTS[2], ACCOUNTS[0]++ finalize_dir = tmp_path / "finalize"+ finalize_dir.mkdir()+ (finalize_dir / SITE_FILENAME).write_text("bytes")+ task = _make_task(task_id="task-2")+ with (+ patch("skyvern.forge.agent.get_path_for_workflow_download_directory", return_value=finalize_dir),+ patch(+ "skyvern.forge.agent.skyvern_context.current",+ return_value=SkyvernContext(task_id="task-0", download_suffix=_rendered_suffix(stale_account)),+ ),+ capture_logs() as cap_finalize,+ ):+ await agent._finalize_downloaded_files_for_task(+ task,+ organization_id=task.organization_id,+ download_suffix=_rendered_suffix(current_account), # task_block-derived, current iteration+ list_files_before=[],+ randomize_if_missing=False,+ )+ finalize_events = _events(cap_finalize, "download_suffix_finalize_rename")++ atsave_dir = tmp_path / "atsave"+ atsave_dir.mkdir()+ with (+ skyvern_context.scoped(SkyvernContext(task_id="task-0", download_suffix=_rendered_suffix(stale_account))),+ capture_logs() as cap_atsave,+ ):+ atsave_target = _download_target_path(atsave_dir, SITE_FILENAME)+ atsave_events = _events(cap_atsave, "download_suffix_target_named")++ _emit_artifact("case4_finalize", finalize_events)+ _emit_artifact("case4_atsave", atsave_events)++ assert {p.name for p in finalize_dir.iterdir()} == {f"{_rendered_suffix(current_account)}.csv"}+ assert atsave_target.name == f"{_rendered_suffix(stale_account)}.csv"++ assert len(finalize_events) == 1+ finalize_event = finalize_events[0]+ # finalize path: names by task_block (current), and the lineage still records the divergent context.+ assert finalize_event["passed_download_suffix_fp"] == expected_fingerprint(_rendered_suffix(current_account))+ assert finalize_event["context_download_suffix_fp"] == expected_fingerprint(_rendered_suffix(stale_account))+ assert finalize_event["desired_name_fp"] == expected_fingerprint(f"{_rendered_suffix(current_account)}.csv")+ assert finalize_event["context_task_id"] == "task-0"+ assert finalize_event["finalize_task_id"] == "task-2"++ assert len(atsave_events) == 1+ assert atsave_events[0]["context_download_suffix_fp"] == expected_fingerprint(_rendered_suffix(stale_account))+++async def test_harness_case5_async_delayed_callback_reads_context_at_fire_time(tmp_path: Path) -> None:+ """A download callback registered during iteration 0 but firing (async) during iteration 1, against a+ single run-scoped context mutated in place (the re-stamp at agent.py: ``context.download_suffix = ...``).++ The late callback names under the context value AT FIRE TIME -> the LATEST suffix, not iteration 0's.+ That is the opposite of the observed earliest-freeze, so this rules out "naive in-place shared-context+ mutation + late callback" as the mechanism and points root cause at a captured iteration-0 snapshot or a+ frozen task_block. The lineage's ``context_task_id`` is what tells the two apart in production.+ """+ download_dir = tmp_path / "downloads"+ download_dir.mkdir()+ shared = SkyvernContext(task_id="task-0", download_suffix=_rendered_suffix(ACCOUNTS[0]))+ produced: list[str] = []+ records: list[dict] = []+ gate = asyncio.Event()++ async def deferred_download() -> None:+ await gate.wait() # the download event lands only after the loop advanced to iteration 1+ with skyvern_context.scoped(shared), capture_logs() as cap:+ target = _download_target_path(download_dir, SITE_FILENAME)+ target.write_text("late-bytes")+ produced.append(target.name)+ records.extend(_events(cap, "download_suffix_target_named"))++ pending = asyncio.create_task(deferred_download())+ # Loop advances to iteration 1 and re-stamps the SAME run-scoped context in place.+ shared.task_id = "task-1"+ shared.download_suffix = _rendered_suffix(ACCOUNTS[1])+ gate.set()+ await pending++ _emit_artifact("case5_async_delay", records)+ assert produced == [f"{_rendered_suffix(ACCOUNTS[1])}.csv"] # latest-wins, NOT iteration-0 freeze+ assert len(records) == 1+ assert records[0]["context_task_id"] == "task-1"+ assert records[0]["context_download_suffix_fp"] == expected_fingerprint(_rendered_suffix(ACCOUNTS[1]))