Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
revert(SKY-11055): remove workflow webhook-delivery hardening (#11731) (#6758)
Shuchang Zheng2842f713c052
diff --git a/docs/pr-screenshots/webhook-delivery-pending.png b/docs/pr-screenshots/webhook-delivery-pending.pngdeleted file mode 100644index 7725ea646..000000000Binary files a/docs/pr-screenshots/webhook-delivery-pending.png and /dev/null differdiff --git a/docs/pr-screenshots/webhook-failure-reason.png b/docs/pr-screenshots/webhook-failure-reason.pngdeleted file mode 100644index d83f4108c..000000000Binary files a/docs/pr-screenshots/webhook-failure-reason.png and /dev/null differdiff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex e50d93b42..0013f30e6 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -164,8 +164,6 @@ export type FailureCategory = {reasoning: string;};-export type WebhookDeliveryStatus = "pending" | "failed";-export type TaskApiResponse = {request: CreateTaskRequest;task_id: string;@@ -660,7 +658,6 @@ export type WorkflowRunStatusApiResponse = {outputs: Record<string, unknown> | null;failure_reason: string | null;failure_category: Array<FailureCategory> | null;- webhook_delivery_status?: WebhookDeliveryStatus | null;webhook_failure_reason: string | null;downloaded_file_urls: Array<string> | null;downloaded_files: Array<DownloadedFileInfo> | null;@@ -698,7 +695,6 @@ export type WorkflowRunStatusApiResponseWithWorkflow = {outputs: Record<string, unknown> | null;failure_reason: string | null;failure_category: Array<FailureCategory> | null;- webhook_delivery_status?: WebhookDeliveryStatus | null;webhook_failure_reason: string | null;downloaded_file_urls: Array<string> | null;downloaded_files: Array<DownloadedFileInfo> | null;diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsxindex 46ad174f4..ffd1c6507 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx@@ -66,7 +66,6 @@ import { WorkflowRunStatusAlert } from "@/routes/workflows/workflowRun/WorkflowRimport { WorkflowRunVerificationCodeForm } from "@/routes/workflows/workflowRun/WorkflowRunVerificationCodeForm";import { ScriptUpdateCard } from "@/routes/workflows/workflowRun/ScriptUpdateCard";import { useFallbackEpisodesQuery } from "@/routes/workflows/hooks/useFallbackEpisodesQuery";-import { WebhookDeliveryStatus } from "@/routes/workflows/workflowRun/WebhookDeliveryStatus";import { useRunsQuery } from "@/hooks/useRunsQuery";import { useOnboardingStateOptional } from "@/store/onboarding/useOnboardingState";import { FirstRunRecoveryGuidance } from "@/components/onboarding/FirstRunRecoveryGuidance";@@ -430,25 +429,18 @@ function WorkflowRun() {const isTaskv2Run = workflowRun && workflowRun.task_v2 !== null;- const taskWebhookFailureReason =- workflowRun?.task_v2?.webhook_failure_reason ?? null;- const workflowWebhookFailureReason =- workflowRun?.webhook_failure_reason ?? null;const webhookFailureReasonData =- taskWebhookFailureReason ?? workflowWebhookFailureReason;- // TaskV2 webhook state is legacy failure-only data; workflow-level pending- // delivery status is surfaced separately by webhook_delivery_status.- const webhookDeliveryStatusData = taskWebhookFailureReason- ? "failed"- : (workflowRun?.webhook_delivery_status ??- (workflowWebhookFailureReason ? "failed" : null));-- const webhookDeliveryStatus = (- <WebhookDeliveryStatus- webhookDeliveryStatus={webhookDeliveryStatusData}- webhookFailureReason={webhookFailureReasonData}- />- );+ workflowRun?.task_v2?.webhook_failure_reason ??+ workflowRun?.webhook_failure_reason;++ const webhookFailureReason = webhookFailureReasonData ? (+ <div className="space-y-4">+ <Label>Webhook Failure Reason</Label>+ <div className="rounded-md border border-yellow-600 p-4 text-sm">+ {webhookFailureReasonData}+ </div>+ </div>+ ) : null;const outputs = workflowRun?.outputs;const extractedInformation =@@ -751,9 +743,9 @@ function WorkflowRun() {</ScrollArea></div>)}+ {webhookFailureReason}</div>)}- {webhookDeliveryStatus}{workflowFailureReason}{fallbackEpisodes && fallbackEpisodes.episodes.length > 0 && (<ScriptUpdateCarddiff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WebhookDeliveryStatus.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WebhookDeliveryStatus.tsxdeleted file mode 100644index 2f1b6d3a6..000000000--- a/skyvern-frontend/src/routes/workflows/workflowRun/WebhookDeliveryStatus.tsx+++ /dev/null@@ -1,48 +0,0 @@-import type { WebhookDeliveryStatus as WebhookDeliveryStatusValue } from "@/api/types";-import { cn } from "@/util/utils";--type Props = {- webhookDeliveryStatus?: WebhookDeliveryStatusValue | null;- webhookFailureReason?: string | null;-};--function WebhookDeliveryStatus({- webhookDeliveryStatus,- webhookFailureReason,-}: Props) {- const isFailed = webhookDeliveryStatus === "failed";- const isPending = webhookDeliveryStatus === "pending";-- if (!isFailed && !isPending) {- return null;- }-- const title = isFailed- ? "Webhook Failure Reason"- : "Webhook Delivery Pending";- const message = isFailed- ? (webhookFailureReason ?? "Skyvern recorded a webhook delivery failure.")- : "Skyvern has started webhook delivery or has not recorded an automatic webhook delivery attempt yet. Delivery will retry automatically.";-- return (- <div- role="status"- className={cn("rounded border bg-slate-elevation2 p-6", {- "border-yellow-600/50": isFailed,- "border-sky-500/50": isPending,- })}- >- <h3 className="mb-4 text-lg font-bold">{title}</h3>- <div- className={cn("text-sm", {- "text-yellow-600": isFailed,- "text-slate-200": isPending,- })}- >- {message}- </div>- </div>- );-}--export { WebhookDeliveryStatus };diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsxindex d8785675b..aaecc086a 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOutput.tsx@@ -175,6 +175,9 @@ function WorkflowRunOutput() {? getBlockDownloadedFileUrls(activeBlock.output, allFileUrls): allFileUrls;const observerOutput = workflowRun?.task_v2?.output;+ const webhookFailureReasonData =+ workflowRun?.task_v2?.webhook_failure_reason ??+ workflowRun?.webhook_failure_reason;const blockContextKey = activeBlock? `block:${activeBlock.workflow_run_id}:${activeBlock.workflow_run_block_id}`@@ -193,6 +196,16 @@ function WorkflowRunOutput() {return (<div className="space-y-5">+ {webhookFailureReasonData ? (+ <div className="rounded bg-slate-elevation2 p-6">+ <div className="space-y-4">+ <h1 className="text-lg font-bold">Webhook Failure Reason</h1>+ <div className="space-y-2 text-yellow-600">+ {webhookFailureReasonData}+ </div>+ </div>+ </div>+ ) : null}{activeBlock ? (<div className="rounded bg-slate-elevation2 p-6"><div className="space-y-4">diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex 7f4c0cdbf..865ad2403 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -307,35 +307,6 @@ class WorkflowRunsRepository(BaseRepository):else:raise WorkflowRunNotFound(workflow_run_id)- @db_operation("claim_workflow_run_webhook_delivery")- async def claim_workflow_run_webhook_delivery(- self,- *,- workflow_run_id: str,- in_progress_reason: str,- claim_older_than: datetime | None = None,- ) -> bool:- delivery_state_filter = WorkflowRunModel.webhook_failure_reason.is_(None)- if claim_older_than is not None:- stale_claim_cutoff = to_naive_utc(claim_older_than)- delivery_state_filter = or_(- delivery_state_filter,- and_(- WorkflowRunModel.webhook_failure_reason == in_progress_reason,- WorkflowRunModel.modified_at <= stale_claim_cutoff,- ),- )-- async with self.Session() as session:- result = await session.execute(- update(WorkflowRunModel)- .where(WorkflowRunModel.workflow_run_id == workflow_run_id)- .where(delivery_state_filter)- .values(webhook_failure_reason=in_progress_reason, modified_at=naive_utc_now())- )- await session.commit()- return result.rowcount == 1-@db_operation("increment_workflow_run_credits")async def increment_workflow_run_credits(self,@@ -855,48 +826,6 @@ class WorkflowRunsRepository(BaseRepository):workflow_runs = (await session.scalars(query)).all()return [convert_to_workflow_run(workflow_run) for workflow_run in workflow_runs]- @db_operation("get_workflow_runs_with_pending_webhook_delivery")- async def get_workflow_runs_with_pending_webhook_delivery(- self,- *,- older_than: datetime,- limit: int = 100,- in_progress_reason: str | None = None,- ) -> list[WorkflowRun]:- terminal_statuses = [status.value for status in WorkflowRunStatus if status.is_final()]- cutoff = to_naive_utc(older_than)- run_age_filter = or_(- WorkflowRunModel.finished_at <= cutoff,- and_(- WorkflowRunModel.finished_at.is_(None),- WorkflowRunModel.modified_at <= cutoff,- ),- )- delivery_state_filter = WorkflowRunModel.webhook_failure_reason.is_(None)- if in_progress_reason is not None:- delivery_state_filter = or_(- delivery_state_filter,- and_(- WorkflowRunModel.webhook_failure_reason == in_progress_reason,- WorkflowRunModel.modified_at <= cutoff,- ),- )-- async with self.Session() as session:- query = (- select(WorkflowRunModel)- .filter(WorkflowRunModel.status.in_(terminal_statuses))- .filter(WorkflowRunModel.webhook_callback_url.isnot(None))- # TODO: add a partial index for the finalized pending-webhook sweep if production scan cost grows.- .filter(func.length(func.trim(WorkflowRunModel.webhook_callback_url)) > 0)- .filter(delivery_state_filter)- .filter(run_age_filter)- .order_by(func.coalesce(WorkflowRunModel.finished_at, WorkflowRunModel.modified_at).asc())- .limit(limit)- )- workflow_runs = (await session.scalars(query)).all()- return [convert_to_workflow_run(workflow_run) for workflow_run in workflow_runs]-@staticmethoddef _apply_search_key_filter(query, search_key: str | None): # type: ignore[no-untyped-def]if not search_key:diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex 36ace66cf..59862aa49 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -1,6 +1,6 @@from datetime import datetimefrom enum import StrEnum-from typing import Any, List, Literal+from typing import Any, Listfrom pydantic import BaseModel, Field, computed_field, field_serializer, field_validatorfrom typing_extensions import deprecated@@ -342,7 +342,6 @@ class WorkflowRunResponseBase(BaseModel):failure_category: list[dict[str, Any]] | None = Noneproxy_location: ProxyLocationInput = Nonewebhook_callback_url: str | None = None- webhook_delivery_status: Literal["pending", "failed"] | None = Nonewebhook_failure_reason: str | None = Nonetotp_verification_url: str | None = Nonetotp_identifier: str | None = Nonediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 0d8a0fa3d..3ceb96eca 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -11,7 +11,7 @@ from collections.abc import Sequencefrom dataclasses import dataclass, fieldfrom datetime import UTC, datetime, timedeltafrom hashlib import sha256-from typing import Any, Literal, TypeAlias, cast+from typing import Any, Literal, castimport structlogfrom sqlalchemy.exc import IntegrityError, SQLAlchemyError@@ -156,16 +156,7 @@ from skyvern.services.script_review_cap import ()from skyvern.services.script_reviewer_v3.cohort import is_v3_cohortfrom skyvern.services.script_reviewer_v3.postrun import v3_review_post_run-from skyvern.services.webhook_delivery import (- WEBHOOK_DELIVERY_HTTP_TIMEOUT_SECONDS,- WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON,- WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON,- WEBHOOK_DELIVERY_SUCCESS_REASON,- PreparedWorkflowWebhook,- deliver_webhook_with_retries,- scrub_internal_webhook_delivery_state,-)+from skyvern.services.webhook_delivery import PreparedWorkflowWebhook, deliver_webhook_with_retriesfrom skyvern.utils.css_selector import build_action_summaries_with_timing # shared with script_servicefrom skyvern.utils.secret_headers import merge_masked_headersfrom skyvern.utils.url_validators import validate_url as validate_url_with_blocked_host_check@@ -181,7 +172,6 @@ DEFAULT_WORKFLOW_TITLE = "New Workflow"RECORDING_WINDOW_END_BUFFER = timedelta(minutes=15)# Skip post-run work when only a sub-millisecond budget remains; asyncio.timeout would fire on the first await.POST_RUN_TIMEOUT_EXHAUSTED_THRESHOLD_SECONDS = 0.001-WorkflowWebhookDeliveryStatus: TypeAlias = Literal["pending", "failed"]# Structured warning emitted when a debug-session run's visible PBS profile is# incompatible with the LoginBlock credential's saved profile. Observability@@ -233,31 +223,6 @@ def _require_elapsed_timeout_failure_reason(timeout_failure_reason: str | None)return timeout_failure_reason-def _get_workflow_webhook_delivery_status(- *,- webhook_callback_url: str | None,- webhook_failure_reason: str | None,- workflow_run_status: WorkflowRunStatus,-) -> WorkflowWebhookDeliveryStatus | None:- if not webhook_callback_url or not workflow_run_status.is_final():- return None- # Empty string is the persisted webhook delivery success sentinel.- if webhook_failure_reason == WEBHOOK_DELIVERY_SUCCESS_REASON:- return None- if webhook_failure_reason is None or webhook_failure_reason == WEBHOOK_DELIVERY_IN_PROGRESS_REASON:- return "pending"- return "failed"---def _get_public_workflow_webhook_failure_reason(webhook_failure_reason: str | None) -> str | None:- if webhook_failure_reason in {- WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- WEBHOOK_DELIVERY_SUCCESS_REASON,- }:- return None- return webhook_failure_reason--def _collect_enterprise_gated_workflow_features(workflow: Workflow,*,@@ -5851,12 +5816,7 @@ class WorkflowService:failure_category=workflow_run.failure_category,proxy_location=workflow_run.proxy_location,webhook_callback_url=workflow_run.webhook_callback_url,- webhook_delivery_status=_get_workflow_webhook_delivery_status(- webhook_callback_url=workflow_run.webhook_callback_url,- webhook_failure_reason=workflow_run.webhook_failure_reason,- workflow_run_status=workflow_run.status,- ),- webhook_failure_reason=_get_public_workflow_webhook_failure_reason(workflow_run.webhook_failure_reason),+ webhook_failure_reason=workflow_run.webhook_failure_reason,totp_verification_url=workflow_run.totp_verification_url,totp_identifier=workflow_run.totp_identifier,extra_http_headers=workflow_run.extra_http_headers,@@ -6048,19 +6008,6 @@ class WorkflowService:workflow_run_id=workflow_run.workflow_run_id,workflow_permanent_id=workflow_run.workflow_permanent_id,)- if workflow_run.webhook_callback_url and workflow_run.webhook_callback_url.strip():- try:- await app.DATABASE.workflow_runs.update_workflow_run(- workflow_run_id=workflow_run.workflow_run_id,- webhook_failure_reason=WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON,- )- except Exception:- LOG.warning(- "Failed to record skipped workflow webhook delivery for missing workflow",- workflow_id=workflow_id,- workflow_run_id=workflow_run.workflow_run_id,- exc_info=True,- )return Noneif not workflow_run.webhook_callback_url:LOG.warning(@@ -6089,18 +6036,6 @@ class WorkflowService:workflow_run_id=workflow_run.workflow_run_id,organization_id=workflow_run.organization_id,)- try:- await app.DATABASE.workflow_runs.update_workflow_run(- workflow_run_id=workflow_run.workflow_run_id,- webhook_failure_reason=WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON,- )- except Exception:- LOG.warning(- "Failed to record skipped workflow webhook delivery",- workflow_id=workflow_id,- workflow_run_id=workflow_run.workflow_run_id,- exc_info=True,- )return None# build new schema for backward compatible webhook payload@@ -6134,9 +6069,7 @@ class WorkflowService:errors=workflow_run_status_response.errors,step_count=workflow_run_status_response.total_steps,)- payload_dict: dict = scrub_internal_webhook_delivery_state(- json.loads(workflow_run_status_response.model_dump_json())- )+ payload_dict: dict = json.loads(workflow_run_status_response.model_dump_json())workflow_run_response_dict = json.loads(workflow_run_response.model_dump_json())payload_dict.update(workflow_run_response_dict)signed_data = generate_skyvern_webhook_signature(@@ -6162,25 +6095,7 @@ class WorkflowService:payload_for_log=signed_data.payload_for_log,)- async def deliver_prepared_workflow_webhook(- self,- webhook: PreparedWorkflowWebhook,- *,- retry_claim_older_than: datetime | None = None,- ) -> None:- claimed = await app.DATABASE.workflow_runs.claim_workflow_run_webhook_delivery(- workflow_run_id=webhook.workflow_run_id,- in_progress_reason=WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- claim_older_than=retry_claim_older_than,- )- if not claimed:- LOG.info(- "Skipping workflow webhook delivery because it was already claimed or recorded",- workflow_id=webhook.workflow_id,- workflow_run_id=webhook.workflow_run_id,- )- return-+ async def deliver_prepared_workflow_webhook(self, webhook: PreparedWorkflowWebhook) -> None:LOG.info("Sending webhook run status to webhook callback url",sampling=True,@@ -6195,7 +6110,7 @@ class WorkflowService:url=webhook.webhook_callback_url,payload=webhook.signed_payload,headers=webhook.headers,- timeout_seconds=WEBHOOK_DELIVERY_HTTP_TIMEOUT_SECONDS,+ timeout_seconds=30.0,organization_id=webhook.organization_id,run_id=webhook.workflow_run_id,)@@ -6233,8 +6148,7 @@ class WorkflowService:try:await app.DATABASE.workflow_runs.update_workflow_run(workflow_run_id=webhook.workflow_run_id,- # Empty string is the persisted successful-delivery sentinel.- webhook_failure_reason=WEBHOOK_DELIVERY_SUCCESS_REASON,+ webhook_failure_reason="",)except Exception:LOG.warning(@@ -6270,12 +6184,11 @@ class WorkflowService:self,workflow_run: WorkflowRun,api_key: str | None = None,- retry_claim_older_than: datetime | None = None,) -> None:webhook = await self.prepare_workflow_webhook(workflow_run, api_key)if webhook is None:return- await self.deliver_prepared_workflow_webhook(webhook, retry_claim_older_than=retry_claim_older_than)+ await self.deliver_prepared_workflow_webhook(webhook)async def persist_video_data(self,diff --git a/skyvern/services/webhook_delivery.py b/skyvern/services/webhook_delivery.pyindex 49622876c..5ed3d7b53 100644--- a/skyvern/services/webhook_delivery.py+++ b/skyvern/services/webhook_delivery.py@@ -7,7 +7,6 @@ import randomfrom dataclasses import dataclassfrom datetime import datetime, timezonefrom email.utils import parsedate_to_datetime-from typing import Anyimport httpximport structlog@@ -26,27 +25,8 @@ NON_5XX_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset()WEBHOOK_DELIVERY_MAX_ATTEMPTS = 3-WEBHOOK_DELIVERY_HTTP_TIMEOUT_SECONDS = 30.0WEBHOOK_DELIVERY_RETRY_BASE_DELAY_SECONDS = 1.0WEBHOOK_DELIVERY_MAX_RETRY_AFTER_SECONDS = 30.0-# Persisted in workflow_runs.webhook_failure_reason; changing requires migrating in-progress rows.-WEBHOOK_DELIVERY_IN_PROGRESS_REASON = "Webhook delivery is in progress"-# Persisted in workflow_runs.webhook_failure_reason for successful deliveries; changing requires-# migrating historical success rows.-WEBHOOK_DELIVERY_SUCCESS_REASON = ""-WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON = "Workflow webhook delivery skipped because no API key was available"-WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON = "Workflow webhook delivery skipped because the workflow was unavailable"---def scrub_internal_webhook_delivery_state(payload: dict[str, Any]) -> dict[str, Any]:- scrubbed = dict(payload)- scrubbed.pop("webhook_delivery_status", None)- if scrubbed.get("webhook_failure_reason") in {- WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- WEBHOOK_DELIVERY_SUCCESS_REASON,- }:- scrubbed["webhook_failure_reason"] = None- return scrubbed@dataclass(frozen=True)diff --git a/skyvern/services/webhook_service.py b/skyvern/services/webhook_service.pyindex 96b3a88dc..bec51b1f5 100644--- a/skyvern/services/webhook_service.py+++ b/skyvern/services/webhook_service.py@@ -40,7 +40,6 @@ from skyvern.schemas.runs import ()from skyvern.schemas.webhooks import RunWebhookPreviewResponse, RunWebhookReplayResponsefrom skyvern.services import run_service, task_v2_service-from skyvern.services.webhook_delivery import scrub_internal_webhook_delivery_statefrom skyvern.utils.url_validators import validate_urlLOG = structlog.get_logger()@@ -440,16 +439,14 @@ async def _build_workflow_payload(errors=status_response.errors,)- payload_dict = scrub_internal_webhook_delivery_state(- json.loads(- status_response.model_dump_json(- exclude={- "webhook_callback_url",- "totp_verification_url",- "totp_identifier",- "extra_http_headers",- }- )+ payload_dict = json.loads(+ status_response.model_dump_json(+ exclude={+ "webhook_callback_url",+ "totp_verification_url",+ "totp_identifier",+ "extra_http_headers",+ }))payload_dict.update(json.loads(run_response.model_dump_json(exclude={"run_request"})))diff --git a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py b/tests/unit/forge/sdk/db/test_workflow_runs_repository.pyindex 20a6d5706..3d20a7008 100644--- a/tests/unit/forge/sdk/db/test_workflow_runs_repository.py+++ b/tests/unit/forge/sdk/db/test_workflow_runs_repository.py@@ -3,7 +3,6 @@from __future__ import annotationsfrom datetime import datetime, timezone-from types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -212,97 +211,6 @@ async def test_get_all_runs_v2_search_key_matches_run_id_and_workflow_permanent_assert "abc123" in where_clause-async def test_get_workflow_runs_with_pending_webhook_delivery_filters_finalized_null_webhook_state() -> None:- captured: dict[str, Any] = {}-- async def _scalars(query):- captured["query"] = query- return _EmptyExecuteResult()-- session = MagicMock()- session.scalars = AsyncMock(side_effect=_scalars)- repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)-- await repo.get_workflow_runs_with_pending_webhook_delivery(- older_than=datetime(2026, 6, 1, 13, 20, tzinfo=timezone.utc),- limit=17,- in_progress_reason="Webhook delivery is in progress",- )-- rendered = str(captured["query"].compile(compile_kwargs={"literal_binds": True}))- where_clause = _where_clause_sql(captured["query"])- assert "workflow_runs.status IN ('failed', 'terminated', 'canceled', 'timed_out', 'completed')" in where_clause- for expected in [- "workflow_runs.webhook_callback_url IS NOT NULL",- "length(trim(workflow_runs.webhook_callback_url)) > 0",- # SQLAlchemy omits parentheses in the rendered string; SQL operator- # precedence keeps the modified_at cutoff bound to only the sentinel branch.- "workflow_runs.webhook_failure_reason IS NULL OR workflow_runs.webhook_failure_reason = 'Webhook delivery is in progress' AND workflow_runs.modified_at <= '2026-06-01 13:20:00'",- "workflow_runs.finished_at <= '2026-06-01 13:20:00' OR workflow_runs.finished_at IS NULL AND workflow_runs.modified_at <= '2026-06-01 13:20:00'",- ]:- assert expected in where_clause- assert "ORDER BY coalesce(workflow_runs.finished_at, workflow_runs.modified_at) ASC" in rendered- assert "LIMIT 17" in rendered---async def test_claim_workflow_run_webhook_delivery_only_claims_null_state() -> None:- captured: dict[str, Any] = {}-- async def _execute(query):- captured["query"] = query- return SimpleNamespace(rowcount=1)-- session = MagicMock()- session.execute = AsyncMock(side_effect=_execute)- session.commit = AsyncMock()- repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)-- claimed = await repo.claim_workflow_run_webhook_delivery(- workflow_run_id="wr_abc",- in_progress_reason="Webhook delivery is in progress",- )-- rendered = str(captured["query"].compile(compile_kwargs={"literal_binds": True}))- assert claimed is True- assert "UPDATE workflow_runs SET" in rendered- assert "webhook_failure_reason='Webhook delivery is in progress'" in rendered- assert "modified_at=" in rendered- assert "workflow_runs.workflow_run_id = 'wr_abc'" in rendered- assert "workflow_runs.webhook_failure_reason IS NULL" in rendered- session.commit.assert_awaited_once()---async def test_claim_workflow_run_webhook_delivery_can_reclaim_stale_in_progress_state() -> None:- captured: dict[str, Any] = {}-- async def _execute(query):- captured["query"] = query- return SimpleNamespace(rowcount=1)-- session = MagicMock()- session.execute = AsyncMock(side_effect=_execute)- session.commit = AsyncMock()- repo = WorkflowRunsRepository(session_factory=lambda: _SessionContext(session), debug_enabled=False)-- claimed = await repo.claim_workflow_run_webhook_delivery(- workflow_run_id="wr_abc",- in_progress_reason="Webhook delivery is in progress",- claim_older_than=datetime(2026, 6, 1, 13, 20, tzinfo=timezone.utc),- )-- where_clause = _where_clause_sql(captured["query"])- assert claimed is True- assert "workflow_runs.workflow_run_id = 'wr_abc'" in where_clause- assert (- "workflow_runs.webhook_failure_reason IS NULL OR workflow_runs.webhook_failure_reason = "- "'Webhook delivery is in progress' AND workflow_runs.modified_at <= '2026-06-01 13:20:00'"- ) in where_clause- session.commit.assert_awaited_once()--@pytest.mark.asyncioasync def test_get_all_runs_v2_selects_workflow_deleted_flag() -> None:captured: dict[str, Any] = {}diff --git a/tests/unit/test_webhook_timestamps.py b/tests/unit/test_webhook_timestamps.pyindex b6453bd5c..e5e36f293 100644--- a/tests/unit/test_webhook_timestamps.py+++ b/tests/unit/test_webhook_timestamps.py@@ -12,11 +12,6 @@ from datetime import datetime, timezonefrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunResponseBase, WorkflowRunStatusfrom skyvern.schemas.runs import RunStatus, RunType, WorkflowRunRequest, WorkflowRunResponse-from skyvern.services.webhook_delivery import (- WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- WEBHOOK_DELIVERY_SUCCESS_REASON,- scrub_internal_webhook_delivery_state,-)from skyvern.services.webhook_service import build_sample_workflow_run_payload@@ -36,19 +31,6 @@ def _make_status_response() -> WorkflowRunResponseBase:)-def _make_status_payload(status_response: WorkflowRunResponseBase) -> dict:- return json.loads(- status_response.model_dump_json(- exclude={- "webhook_callback_url",- "totp_verification_url",- "totp_identifier",- "extra_http_headers",- }- )- )--def test_workflow_webhook_payload_includes_timestamps() -> None:"""Reproduce the payload_dict.update() overwrite pattern from execute_workflow_webhookand verify timestamps are preserved."""@@ -117,7 +99,16 @@ def test_webhook_replay_payload_includes_timestamps() -> None:errors=status_response.errors,)- payload_dict = _make_status_payload(status_response)+ payload_dict = json.loads(+ status_response.model_dump_json(+ exclude={+ "webhook_callback_url",+ "totp_verification_url",+ "totp_identifier",+ "extra_http_headers",+ }+ )+ )payload_dict.update(json.loads(run_response.model_dump_json(exclude={"run_request"})))assert payload_dict["queued_at"] is not None, "queued_at should not be null in webhook replay payload"@@ -125,29 +116,6 @@ def test_webhook_replay_payload_includes_timestamps() -> None:assert payload_dict["finished_at"] is not None, "finished_at should not be null in webhook replay payload"-def test_webhook_replay_payload_scrubs_internal_delivery_state() -> None:- status_response = _make_status_response()- status_response.webhook_delivery_status = "pending"- status_response.webhook_failure_reason = WEBHOOK_DELIVERY_IN_PROGRESS_REASON-- payload = _make_status_payload(status_response)- payload_dict = scrub_internal_webhook_delivery_state(payload)-- assert "webhook_delivery_status" not in payload_dict- assert payload_dict["webhook_failure_reason"] is None- assert payload["webhook_delivery_status"] == "pending"- assert payload["webhook_failure_reason"] == WEBHOOK_DELIVERY_IN_PROGRESS_REASON---def test_webhook_replay_payload_scrubs_success_sentinel() -> None:- status_response = _make_status_response()- status_response.webhook_failure_reason = WEBHOOK_DELIVERY_SUCCESS_REASON-- payload_dict = scrub_internal_webhook_delivery_state(_make_status_payload(status_response))-- assert payload_dict["webhook_failure_reason"] is None--def test_sample_workflow_webhook_payload_uses_public_credit_fields() -> None:payload = build_sample_workflow_run_payload("wr_sample_test")diff --git a/tests/unit/test_workflow_webhook_delivery.py b/tests/unit/test_workflow_webhook_delivery.pyindex 66a9e93ec..81fe3a3e7 100644--- a/tests/unit/test_workflow_webhook_delivery.py+++ b/tests/unit/test_workflow_webhook_delivery.py@@ -1,6 +1,5 @@from __future__ import annotations-import jsonfrom datetime import datetime, timezonefrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock@@ -9,20 +8,13 @@ import httpximport pytestfrom skyvern.forge.sdk.workflow import service as service_module-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunResponseBasefrom skyvern.forge.sdk.workflow.service import WorkflowServiceclass _StatusResponse:- def __init__(- self,- webhook_failure_reason: str | None = None,- webhook_delivery_status: str | None = None,- ) -> None:+ def __init__(self) -> None:now = datetime.now(timezone.utc)self.status = "completed"- self.webhook_delivery_status = webhook_delivery_status- self.webhook_failure_reason = webhook_failure_reasonself.outputs: dict = {}self.downloaded_files: list = []self.recording_url = None@@ -40,14 +32,7 @@ class _StatusResponse:self.finished_at = nowdef model_dump_json(self) -> str:- return json.dumps(- {- "workflow_run_id": "wr_abc",- "status": "completed",- "webhook_delivery_status": self.webhook_delivery_status,- "webhook_failure_reason": self.webhook_failure_reason,- }- )+ return '{"workflow_run_id":"wr_abc","status":"completed"}'class _WebhookRunResponse:@@ -75,62 +60,6 @@ def _response(status_code: int, body: str = "") -> httpx.Response:return httpx.Response(status_code=status_code, content=body.encode("utf-8"))- ("webhook_callback_url", "webhook_failure_reason", "expected"),- [- ("https://example.com/hook", None, "pending"),- ("https://example.com/hook", service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON, "pending"),- ("https://example.com/hook", "Webhook failed with status code 400", "failed"),- ("https://example.com/hook", service_module.WEBHOOK_DELIVERY_SUCCESS_REASON, None),- (None, None, None),- ],-)-def test_get_workflow_webhook_delivery_status(- webhook_callback_url: str | None,- webhook_failure_reason: str | None,- expected: str | None,-) -> None:- assert (- service_module._get_workflow_webhook_delivery_status(- webhook_callback_url=webhook_callback_url,- webhook_failure_reason=webhook_failure_reason,- workflow_run_status=service_module.WorkflowRunStatus.completed,- )- == expected- )--- ("webhook_failure_reason", "expected"),- [- (service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON, None),- ("Webhook failed with status code 400", "Webhook failed with status code 400"),- (service_module.WEBHOOK_DELIVERY_SUCCESS_REASON, None),- (None, None),- ],-)-def test_get_public_workflow_webhook_failure_reason(- webhook_failure_reason: str | None,- expected: str | None,-) -> None:- assert service_module._get_public_workflow_webhook_failure_reason(webhook_failure_reason) == expected---def test_workflow_run_response_base_serializes_webhook_delivery_status() -> None:- now = datetime.now(timezone.utc)- response = WorkflowRunResponseBase(- workflow_id="wpid_abc",- workflow_run_id="wr_abc",- status=service_module.WorkflowRunStatus.completed,- webhook_delivery_status="pending",- created_at=now,- modified_at=now,- parameters={},- )-- assert json.loads(response.model_dump_json())["webhook_delivery_status"] == "pending"--@pytest.fixturedef webhook_service(monkeypatch: pytest.MonkeyPatch) -> tuple[WorkflowService, AsyncMock, AsyncMock]:svc = WorkflowService()@@ -154,11 +83,6 @@ def webhook_service(monkeypatch: pytest.MonkeyPatch) -> tuple[WorkflowService, AAsyncMock(return_value=SimpleNamespace(token="api-key")),)monkeypatch.setattr(service_module.app.DATABASE.workflow_runs, "update_workflow_run", update_run)- monkeypatch.setattr(- service_module.app.DATABASE.workflow_runs,- "claim_workflow_run_webhook_delivery",- AsyncMock(return_value=True),- )return svc, build_response, update_run@@ -183,73 +107,6 @@ async def test_prepare_workflow_webhook_builds_request_without_delivery(deliver.assert_not_awaited()-async def test_prepare_workflow_webhook_scrubs_internal_delivery_state_from_payload(- webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],- monkeypatch: pytest.MonkeyPatch,-) -> None:- svc, build_response, _update_run = webhook_service- signed_payload: dict[str, object] = {}-- build_response.return_value = _StatusResponse(- webhook_delivery_status="pending",- webhook_failure_reason=service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- )-- def capture_signature(payload: dict[str, object], api_key: str) -> SimpleNamespace:- signed_payload.update(payload)- return SimpleNamespace(- payload_for_log=json.dumps(payload),- headers={"x-skyvern-signature": "sig"},- signed_payload=json.dumps(payload),- )-- monkeypatch.setattr(service_module, "generate_skyvern_webhook_signature", capture_signature)-- webhook = await svc.prepare_workflow_webhook(_workflow_run())-- assert webhook is not None- assert "webhook_delivery_status" not in signed_payload- assert signed_payload["webhook_failure_reason"] is None---async def test_prepare_workflow_webhook_records_missing_signing_key_as_failure(- webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],- monkeypatch: pytest.MonkeyPatch,-) -> None:- svc, _build_response, update_run = webhook_service- monkeypatch.setattr(- service_module.app.DATABASE.organizations,- "get_valid_org_auth_token",- AsyncMock(return_value=None),- )-- webhook = await svc.prepare_workflow_webhook(_workflow_run())-- assert webhook is None- update_run.assert_awaited_once_with(- workflow_run_id="wr_abc",- webhook_failure_reason=service_module.WEBHOOK_DELIVERY_MISSING_SIGNING_KEY_REASON,- )---async def test_prepare_workflow_webhook_records_missing_workflow_as_failure(- webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],-) -> None:- svc, build_response, update_run = webhook_service- build_response.side_effect = service_module.WorkflowNotFound(workflow_permanent_id="wpid_abc")-- webhook = await svc.prepare_workflow_webhook(_workflow_run())-- assert webhook is None- update_run.assert_awaited_once_with(- workflow_run_id="wr_abc",- webhook_failure_reason=service_module.WEBHOOK_DELIVERY_MISSING_WORKFLOW_REASON,- )--@pytest.mark.asyncioasync def test_execute_workflow_webhook_records_customer_failure_without_raising(webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],@@ -268,52 +125,6 @@ async def test_execute_workflow_webhook_records_customer_failure_without_raising)-async def test_execute_workflow_webhook_passes_retry_claim_cutoff(- webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],- monkeypatch: pytest.MonkeyPatch,-) -> None:- svc, _build_response, _update_run = webhook_service- retry_claim_older_than = datetime(2026, 6, 1, 13, 20, tzinfo=timezone.utc)- deliver = AsyncMock(return_value=_response(200, "ok"))- claim = AsyncMock(return_value=True)- monkeypatch.setattr(service_module, "deliver_webhook_with_retries", deliver)- monkeypatch.setattr(- service_module.app.DATABASE.workflow_runs,- "claim_workflow_run_webhook_delivery",- claim,- )-- await svc.execute_workflow_webhook(_workflow_run(), retry_claim_older_than=retry_claim_older_than)-- claim.assert_awaited_once_with(- workflow_run_id="wr_abc",- in_progress_reason=service_module.WEBHOOK_DELIVERY_IN_PROGRESS_REASON,- claim_older_than=retry_claim_older_than,- )- deliver.assert_awaited_once()---async def test_execute_workflow_webhook_skips_delivery_when_claim_lost(- webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],- monkeypatch: pytest.MonkeyPatch,-) -> None:- svc, _build_response, update_run = webhook_service- deliver = AsyncMock()- monkeypatch.setattr(service_module, "deliver_webhook_with_retries", deliver)- monkeypatch.setattr(- service_module.app.DATABASE.workflow_runs,- "claim_workflow_run_webhook_delivery",- AsyncMock(return_value=False),- )-- await svc.execute_workflow_webhook(_workflow_run())-- deliver.assert_not_awaited()- update_run.assert_not_awaited()--@pytest.mark.asyncioasync def test_execute_workflow_webhook_records_delivery_exception_without_raising(webhook_service: tuple[WorkflowService, AsyncMock, AsyncMock],@@ -343,10 +154,7 @@ async def test_execute_workflow_webhook_does_not_raise_if_post_delivery_recordinawait svc.execute_workflow_webhook(_workflow_run())deliver.assert_awaited_once()- update_run.assert_awaited_once_with(- workflow_run_id="wr_abc",- webhook_failure_reason=service_module.WEBHOOK_DELIVERY_SUCCESS_REASON,- )+ update_run.assert_awaited_once_with(workflow_run_id="wr_abc", webhook_failure_reason="")@pytest.mark.asyncio