Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Harden cached script deploy cleanup (#6018)
pedrohsdbc12482f5b313
diff --git a/skyvern/forge/sdk/db/repositories/scripts.py b/skyvern/forge/sdk/db/repositories/scripts.pyindex 9af26c053..bdb1e869b 100644--- a/skyvern/forge/sdk/db/repositories/scripts.py+++ b/skyvern/forge/sdk/db/repositories/scripts.py@@ -26,6 +26,7 @@ from skyvern.forge.sdk.db.utils import (convert_to_script,convert_to_script_block,convert_to_script_file,+ nullable_column_equals,)from skyvern.forge.sdk.utils.sanitization import sanitize_postgres_textfrom skyvern.schemas.scripts import (@@ -41,6 +42,28 @@ from skyvern.schemas.scripts import (LOG = structlog.get_logger()+def _script_status_value(status: ScriptStatus | str) -> str:+ return status.value if isinstance(status, ScriptStatus) else status+++def _workflow_script_matches_snapshot(workflow_script: WorkflowScript) -> list[Any]:+ return [+ WorkflowScriptModel.organization_id == workflow_script.organization_id,+ WorkflowScriptModel.workflow_script_id == workflow_script.workflow_script_id,+ WorkflowScriptModel.workflow_permanent_id == workflow_script.workflow_permanent_id,+ WorkflowScriptModel.script_id == workflow_script.script_id,+ nullable_column_equals(WorkflowScriptModel.workflow_id, workflow_script.workflow_id),+ nullable_column_equals(WorkflowScriptModel.workflow_run_id, workflow_script.workflow_run_id),+ WorkflowScriptModel.cache_key == workflow_script.cache_key,+ WorkflowScriptModel.cache_key_value == workflow_script.cache_key_value,+ WorkflowScriptModel.status == _script_status_value(workflow_script.status),+ WorkflowScriptModel.is_pinned == workflow_script.is_pinned,+ nullable_column_equals(WorkflowScriptModel.pinned_at, workflow_script.pinned_at),+ nullable_column_equals(WorkflowScriptModel.pinned_by, workflow_script.pinned_by),+ nullable_column_equals(WorkflowScriptModel.modified_at, workflow_script.modified_at),+ ]++class WorkflowScriptWriterIntent(StrEnum):deploy = "deploy"auto_regen = "auto_regen"@@ -57,6 +80,7 @@ class WorkflowScriptUpsertStatus(StrEnum):class WorkflowScriptUpsertResult:status: WorkflowScriptUpsertStatusworkflow_script: WorkflowScript+ previous_workflow_script: WorkflowScript | None = Noneclass ScriptsRepository(BaseRepository):@@ -715,6 +739,7 @@ class ScriptsRepository(BaseRepository):workflow_script=workflow_script,)+ previous_workflow_script = WorkflowScript.model_validate(existing)existing.script_id = script_idexisting.workflow_id = workflow_idexisting.workflow_run_id = workflow_run_id@@ -731,6 +756,7 @@ class ScriptsRepository(BaseRepository):return WorkflowScriptUpsertResult(status=WorkflowScriptUpsertStatus.updated,workflow_script=WorkflowScript.model_validate(existing),+ previous_workflow_script=previous_workflow_script,)@db_operation("get_workflow_script")@@ -922,6 +948,74 @@ class ScriptsRepository(BaseRepository):return result.rowcount > 0+ @db_operation("soft_delete_workflow_script_if_matches")+ async def soft_delete_workflow_script_if_matches(+ self,+ *,+ workflow_script: WorkflowScript,+ ) -> bool:+ """Soft delete one workflow_scripts row if it still matches a snapshot.++ Failed manual deploy cleanup uses this to avoid deleting a row that a+ concurrent successful deploy already changed.+ """+ async with self.Session() as session:+ stmt = (+ update(WorkflowScriptModel)+ .where(+ and_(+ *_workflow_script_matches_snapshot(workflow_script),+ WorkflowScriptModel.deleted_at.is_(None),+ )+ )+ .values(deleted_at=datetime.now(timezone.utc))+ )+ result = await session.execute(stmt)+ await session.commit()++ return result.rowcount > 0++ @db_operation("restore_workflow_script_if_matches")+ async def restore_workflow_script_if_matches(+ self,+ *,+ current_workflow_script: WorkflowScript,+ restore_workflow_script: WorkflowScript,+ ) -> bool:+ """Restore one workflow_scripts row if it still matches a written snapshot.++ The live-row predicate prevents cleanup from resurrecting a row another+ actor soft-deleted after this deploy's forward write.+ """+ async with self.Session() as session:+ status = _script_status_value(restore_workflow_script.status)+ stmt = (+ update(WorkflowScriptModel)+ .where(+ and_(+ *_workflow_script_matches_snapshot(current_workflow_script),+ WorkflowScriptModel.deleted_at.is_(None),+ )+ )+ .values(+ script_id=restore_workflow_script.script_id,+ workflow_id=restore_workflow_script.workflow_id,+ workflow_run_id=restore_workflow_script.workflow_run_id,+ cache_key=restore_workflow_script.cache_key,+ cache_key_value=restore_workflow_script.cache_key_value,+ status=status,+ is_pinned=restore_workflow_script.is_pinned,+ pinned_at=restore_workflow_script.pinned_at,+ pinned_by=restore_workflow_script.pinned_by,+ modified_at=restore_workflow_script.modified_at,+ deleted_at=restore_workflow_script.deleted_at,+ )+ )+ result = await session.execute(stmt)+ await session.commit()++ return result.rowcount > 0+@db_operation("delete_workflow_scripts_by_permanent_id")async def delete_workflow_scripts_by_permanent_id(self,diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.pyindex bb19a3949..51791cb7d 100644--- a/skyvern/forge/sdk/db/repositories/workflows.py+++ b/skyvern/forge/sdk/db/repositories/workflows.py@@ -1,5 +1,6 @@from __future__ import annotations+from dataclasses import dataclassfrom datetime import datetime, timezonefrom typing import Any, cast@@ -29,7 +30,7 @@ from skyvern.forge.sdk.db.models import (WorkflowTemplateModel,)from skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepository-from skyvern.forge.sdk.db.utils import convert_to_workflow, serialize_proxy_location+from skyvern.forge.sdk.db.utils import convert_to_workflow, nullable_column_equals, serialize_proxy_locationfrom skyvern.forge.sdk.workflow.models.block import Block, ForLoopBlock, WhileLoopBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameterfrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition@@ -39,6 +40,19 @@ from skyvern.schemas.workflows import WorkflowStatusLOG = structlog.get_logger()+@dataclass(frozen=True)+class WorkflowDispatchState:+ run_with: str | None+ cache_key: str | None+ code_version: int | None+++@dataclass(frozen=True)+class WorkflowDispatchUpdateResult:+ workflow: Workflow+ previous_dispatch_state: WorkflowDispatchState++def _align_block_output_parameters(workflow_definition: WorkflowDefinition) -> None:"""Rebind each block's ``output_parameter`` to the reconciled instancefrom ``workflow_definition.parameters`` by key, recursing into nested@@ -716,6 +730,151 @@ class WorkflowsRepository(BaseRepository):if updated_workflow_id is None:raise NotFoundError("Workflow not found or no longer latest")+ await session.commit()+ workflow = (+ await session.scalars(+ exclude_deleted(+ select(WorkflowModel).filter_by(+ workflow_id=workflow_id,+ organization_id=organization_id,+ ),+ WorkflowModel,+ )+ )+ ).one()+ is_template = await self.is_workflow_template(+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=workflow.organization_id,+ )+ return convert_to_workflow(+ workflow,+ self.debug_enabled,+ is_template=is_template,+ )++ @db_operation("update_workflow_dispatch_state_if_latest_with_previous")+ async def update_workflow_dispatch_state_if_latest_with_previous(+ self,+ *,+ workflow_id: str,+ workflow_permanent_id: str,+ organization_id: str,+ expected_version: int,+ run_with: str,+ cache_key: str,+ code_version: int,+ ) -> WorkflowDispatchUpdateResult:+ async with self.Session() as session:+ previous_workflow = (+ await session.scalars(+ exclude_deleted(+ select(WorkflowModel)+ .filter_by(+ workflow_id=workflow_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ version=expected_version,+ )+ .with_for_update(),+ WorkflowModel,+ )+ )+ ).first()+ if previous_workflow is None:+ raise NotFoundError("Workflow not found")+ previous_dispatch_state = WorkflowDispatchState(+ run_with=previous_workflow.run_with,+ cache_key=previous_workflow.cache_key,+ code_version=previous_workflow.code_version,+ )++ newer_version_exists = (+ select(WorkflowModel.workflow_id)+ .where(WorkflowModel.workflow_permanent_id == workflow_permanent_id)+ .where(WorkflowModel.organization_id == organization_id)+ .where(WorkflowModel.version > expected_version)+ .where(WorkflowModel.deleted_at.is_(None))+ .exists()+ )+ update_workflow_query = (+ update(WorkflowModel)+ .where(WorkflowModel.workflow_id == workflow_id)+ .where(WorkflowModel.organization_id == organization_id)+ .where(WorkflowModel.workflow_permanent_id == workflow_permanent_id)+ .where(WorkflowModel.version == expected_version)+ .where(WorkflowModel.deleted_at.is_(None))+ .where(~newer_version_exists)+ .values(+ run_with=run_with,+ cache_key=cache_key,+ code_version=code_version,+ )+ .returning(WorkflowModel.workflow_id)+ )+ updated_workflow_id = (await session.execute(update_workflow_query)).scalar_one_or_none()+ if updated_workflow_id is None:+ raise NotFoundError("Workflow not found or no longer latest")++ await session.commit()+ workflow = (+ await session.scalars(+ exclude_deleted(+ select(WorkflowModel).filter_by(+ workflow_id=workflow_id,+ organization_id=organization_id,+ ),+ WorkflowModel,+ )+ )+ ).one()+ is_template = await self.is_workflow_template(+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=workflow.organization_id,+ )+ return WorkflowDispatchUpdateResult(+ workflow=convert_to_workflow(+ workflow,+ self.debug_enabled,+ is_template=is_template,+ ),+ previous_dispatch_state=previous_dispatch_state,+ )++ @db_operation("restore_workflow_script_dispatch_if_matches")+ async def restore_workflow_script_dispatch_if_matches(+ self,+ *,+ workflow_id: str,+ organization_id: str,+ run_with: str | None,+ cache_key: str | None,+ code_version: int | None,+ current_run_with: str | None,+ current_cache_key: str | None,+ current_code_version: int | None,+ ) -> Workflow | None:+ """Restore script dispatch only if it still matches the deploy write."""+ async with self.Session() as session:+ update_workflow_query = (+ update(WorkflowModel)+ .where(WorkflowModel.workflow_id == workflow_id)+ .where(WorkflowModel.organization_id == organization_id)+ .where(WorkflowModel.deleted_at.is_(None))+ .where(nullable_column_equals(WorkflowModel.run_with, current_run_with))+ .where(nullable_column_equals(WorkflowModel.cache_key, current_cache_key))+ .where(nullable_column_equals(WorkflowModel.code_version, current_code_version))+ .values(+ run_with=run_with,+ cache_key=cache_key,+ code_version=code_version,+ )+ .returning(WorkflowModel.workflow_id)+ )+ updated_workflow_id = (await session.execute(update_workflow_query)).scalar_one_or_none()+ if updated_workflow_id is None:+ await session.commit()+ return None+await session.commit()workflow = (await session.scalars(diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 839ac145b..4a8e00444 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -1,5 +1,6 @@import jsonimport typing+from typing import Anyimport pydanticimport pydantic.json@@ -98,6 +99,10 @@ from skyvern.webeye.actions.actions import (LOG = structlog.get_logger()+def nullable_column_equals(column: Any, value: Any) -> Any:+ return column.is_(None) if value is None else column == value++def _safe_trigger_type(raw: str | None) -> WorkflowRunTriggerType | None:if not raw:return Nonediff --git a/skyvern/services/cached_script_deploy_service.py b/skyvern/services/cached_script_deploy_service.pyindex 149253faa..1c6619bcb 100644--- a/skyvern/services/cached_script_deploy_service.py+++ b/skyvern/services/cached_script_deploy_service.py@@ -14,7 +14,7 @@ from skyvern.core.script_generations.script_block_extractor import ()from skyvern.forge import appfrom skyvern.forge.sdk.db.exceptions import NotFoundError-from skyvern.forge.sdk.db.repositories.scripts import WorkflowScriptWriterIntent+from skyvern.forge.sdk.db.repositories.scripts import WorkflowScriptUpsertStatus, WorkflowScriptWriterIntentfrom skyvern.forge.sdk.workflow.models.workflow import Workflowfrom skyvern.schemas.scripts import (DeployCachedScriptBlockPlan,@@ -23,6 +23,7 @@ from skyvern.schemas.scripts import (FileEncoding,ScriptFileCreate,ScriptStatus,+ WorkflowScript,)from skyvern.services.workflow_script_service import CacheKeyResolutionError, resolve_cache_key_value@@ -51,6 +52,31 @@ class _ValidatedScriptFile:content_bytes: bytes+@dataclass+class _CachedScriptDeployUndoState:+ script_revision_id: str | None = None+ workflow_script_created_snapshot: WorkflowScript | None = None+ workflow_script_written_snapshot: WorkflowScript | None = None+ workflow_script_restore_snapshot: WorkflowScript | None = None+ workflow_dispatch_updated: bool = False+ workflow_dispatch_restore_run_with: str | None = None+ workflow_dispatch_restore_cache_key: str | None = None+ workflow_dispatch_restore_code_version: int | None = None+ workflow_dispatch_written_run_with: str | None = None+ workflow_dispatch_written_cache_key: str | None = None+ workflow_dispatch_written_code_version: int | None = None++ @property+ def has_writes(self) -> bool:+ return (+ self.script_revision_id is not None+ or self.workflow_script_created_snapshot is not None+ or self.workflow_script_written_snapshot is not None+ or self.workflow_script_restore_snapshot is not None+ or self.workflow_dispatch_updated+ )++def _decode_script_file_bytes(file: ScriptFileCreate) -> bytes:if file.encoding == FileEncoding.BASE64:try:@@ -262,6 +288,107 @@ def _response_from_plan()+async def _cleanup_failed_cached_script_deploy(+ *,+ organization_id: str,+ plan: _CachedScriptDeployPlan,+ undo_state: _CachedScriptDeployUndoState,+) -> None:+ cleanup_errors: list[str] = []+ if undo_state.workflow_dispatch_updated:+ try:+ restored_workflow = await app.DATABASE.workflows.restore_workflow_script_dispatch_if_matches(+ workflow_id=plan.workflow.workflow_id,+ organization_id=organization_id,+ run_with=undo_state.workflow_dispatch_restore_run_with,+ cache_key=undo_state.workflow_dispatch_restore_cache_key,+ code_version=undo_state.workflow_dispatch_restore_code_version,+ current_run_with=undo_state.workflow_dispatch_written_run_with,+ current_cache_key=undo_state.workflow_dispatch_written_cache_key,+ current_code_version=undo_state.workflow_dispatch_written_code_version,+ )+ if restored_workflow is None:+ LOG.warning(+ "Skipped restoring workflow dispatch after cached script deploy failure because it changed",+ workflow_id=plan.workflow.workflow_id,+ organization_id=organization_id,+ )+ except Exception:+ cleanup_errors.append("workflow_dispatch")+ LOG.warning(+ "Failed to restore workflow dispatch state after cached script deploy failure",+ workflow_id=plan.workflow.workflow_id,+ organization_id=organization_id,+ exc_info=True,+ )++ if (+ undo_state.workflow_script_restore_snapshot is not None+ and undo_state.workflow_script_written_snapshot is not None+ ):+ try:+ restored_workflow_script = await app.DATABASE.scripts.restore_workflow_script_if_matches(+ current_workflow_script=undo_state.workflow_script_written_snapshot,+ restore_workflow_script=undo_state.workflow_script_restore_snapshot,+ )+ if not restored_workflow_script:+ LOG.warning(+ "Skipped restoring workflow script after cached script deploy failure because it changed",+ workflow_script_id=undo_state.workflow_script_written_snapshot.workflow_script_id,+ organization_id=organization_id,+ )+ except Exception:+ cleanup_errors.append("workflow_script_restore")+ LOG.warning(+ "Failed to restore workflow script after cached script deploy failure",+ workflow_script_id=undo_state.workflow_script_restore_snapshot.workflow_script_id,+ organization_id=organization_id,+ exc_info=True,+ )+ elif undo_state.workflow_script_created_snapshot is not None:+ try:+ deleted_workflow_script = await app.DATABASE.scripts.soft_delete_workflow_script_if_matches(+ workflow_script=undo_state.workflow_script_created_snapshot,+ )+ if not deleted_workflow_script:+ LOG.warning(+ "Skipped soft-deleting workflow script after cached script deploy failure because it changed",+ workflow_script_id=undo_state.workflow_script_created_snapshot.workflow_script_id,+ organization_id=organization_id,+ )+ except Exception:+ cleanup_errors.append("workflow_script")+ LOG.warning(+ "Failed to soft-delete workflow script after cached script deploy failure",+ workflow_script_id=undo_state.workflow_script_created_snapshot.workflow_script_id,+ organization_id=organization_id,+ exc_info=True,+ )++ if undo_state.script_revision_id is not None:+ try:+ await app.DATABASE.scripts.soft_delete_script_by_revision(+ script_revision_id=undo_state.script_revision_id,+ organization_id=organization_id,+ )+ except Exception:+ cleanup_errors.append("script_revision")+ LOG.warning(+ "Failed to soft-delete script revision after cached script deploy failure",+ script_revision_id=undo_state.script_revision_id,+ organization_id=organization_id,+ exc_info=True,+ )++ if cleanup_errors:+ LOG.error(+ "Cached script deploy failed and cleanup was incomplete",+ workflow_id=plan.workflow.workflow_id,+ organization_id=organization_id,+ cleanup_errors=cleanup_errors,+ )++async def deploy_cached_script(*,organization_id: str,@@ -279,11 +406,13 @@ async def deploy_cached_script(script = Nonecommit_stage = "create_script"+ undo_state = _CachedScriptDeployUndoState()try:script = await app.DATABASE.scripts.create_script(organization_id=organization_id,run_id=request.source_workflow_run_id,)+ undo_state.script_revision_id = script.script_revision_idcommit_stage = "persist_script_files"await _persist_script_files(files=plan.validated_files,@@ -317,16 +446,49 @@ async def deploy_cached_script(is_pinned=True,writer_intent=WorkflowScriptWriterIntent.deploy,)+ if workflow_script_result.status == WorkflowScriptUpsertStatus.created:+ undo_state.workflow_script_created_snapshot = workflow_script_result.workflow_script+ elif workflow_script_result.status == WorkflowScriptUpsertStatus.updated:+ undo_state.workflow_script_written_snapshot = workflow_script_result.workflow_script+ undo_state.workflow_script_restore_snapshot = workflow_script_result.previous_workflow_script+ if undo_state.workflow_script_restore_snapshot is None:+ LOG.warning(+ "Cached script deploy updated workflow script without a previous snapshot",+ workflow_script_id=workflow_script_result.workflow_script.workflow_script_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ cache_key_value=plan.cache_key_value,+ )commit_stage = "update_workflow_dispatch_state"- await app.DATABASE.workflows.update_workflow_dispatch_state_if_latest(+ code_version = _code_version_for_cache_context(request)+ dispatch_update_result = await app.DATABASE.workflows.update_workflow_dispatch_state_if_latest_with_previous(workflow_id=plan.workflow.workflow_id,workflow_permanent_id=workflow_permanent_id,organization_id=organization_id,expected_version=plan.workflow.version,run_with="code",cache_key=cache_key,- code_version=_code_version_for_cache_context(request),+ code_version=code_version,+ )+ previous_dispatch_state = dispatch_update_result.previous_dispatch_state+ undo_state.workflow_dispatch_updated = True+ undo_state.workflow_dispatch_restore_run_with = previous_dispatch_state.run_with+ undo_state.workflow_dispatch_restore_cache_key = previous_dispatch_state.cache_key+ undo_state.workflow_dispatch_restore_code_version = previous_dispatch_state.code_version+ undo_state.workflow_dispatch_written_run_with = "code"+ undo_state.workflow_dispatch_written_cache_key = cache_key+ undo_state.workflow_dispatch_written_code_version = code_version++ return _response_from_plan(+ plan=plan,+ dry_run=False,+ script_id=script.script_id,+ script_revision_id=script.script_revision_id,+ script_version=script.version,+ workflow_script_id=workflow_script_result.workflow_script.workflow_script_id,+ workflow_script_upsert_status=workflow_script_result.status.value,+ script_was_created=True,)except NotFoundError as exc:LOG.exception(@@ -339,6 +501,12 @@ async def deploy_cached_script(script_id=script.script_id if script else None,commit_stage=commit_stage,)+ if undo_state.has_writes:+ await _cleanup_failed_cached_script_deploy(+ organization_id=organization_id,+ plan=plan,+ undo_state=undo_state,+ )raise HTTPException(status_code=409,detail=(@@ -357,15 +525,10 @@ async def deploy_cached_script(script_id=script.script_id if script else None,commit_stage=commit_stage,)+ if undo_state.has_writes:+ await _cleanup_failed_cached_script_deploy(+ organization_id=organization_id,+ plan=plan,+ undo_state=undo_state,+ )raise-- return _response_from_plan(- plan=plan,- dry_run=False,- script_id=script.script_id,- script_revision_id=script.script_revision_id,- script_version=script.version,- workflow_script_id=workflow_script_result.workflow_script.workflow_script_id,- workflow_script_upsert_status=workflow_script_result.status.value,- script_was_created=True,- )diff --git a/tests/unit/forge/sdk/db/test_repositories.py b/tests/unit/forge/sdk/db/test_repositories.pyindex 67171225d..874f61b5e 100644--- a/tests/unit/forge/sdk/db/test_repositories.py+++ b/tests/unit/forge/sdk/db/test_repositories.py@@ -69,6 +69,8 @@ def test_scripts_repository_instantiation():assert repo.Session is mock_sessionassert hasattr(repo, "create_script")assert hasattr(repo, "get_scripts")+ assert hasattr(repo, "soft_delete_workflow_script_if_matches")+ assert hasattr(repo, "restore_workflow_script_if_matches")def test_workflow_parameters_repository_instantiation():@@ -101,6 +103,8 @@ def test_workflows_repository_instantiation():assert hasattr(repo, "get_workflow")assert hasattr(repo, "create_workflow")assert hasattr(repo, "get_workflow_by_permanent_id")+ assert hasattr(repo, "update_workflow_dispatch_state_if_latest_with_previous")+ assert hasattr(repo, "restore_workflow_script_dispatch_if_matches")def test_browser_sessions_repository_instantiation():diff --git a/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py b/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.pyindex a47dc4431..b84759b46 100644--- a/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py+++ b/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py@@ -121,6 +121,40 @@ async def test_update_workflow_dispatch_state_if_latest_updates_current_version(assert updated.code_version == 2+async def test_update_workflow_dispatch_state_with_previous_returns_pre_update_state(agent_db: AgentDB) -> None:+ org = await agent_db.organizations.create_organization(+ organization_name="Dispatch Previous Org",+ domain="dispatch-previous.test",+ )+ workflow = await agent_db.workflows.create_workflow(+ title="dispatch-v1",+ workflow_definition={"parameters": [], "blocks": []},+ organization_id=org.organization_id,+ workflow_permanent_id="wpid_dispatch_previous",+ version=1,+ run_with="agent",+ cache_key="default",+ code_version=1,+ )++ result = await agent_db.workflows.update_workflow_dispatch_state_if_latest_with_previous(+ workflow_id=workflow.workflow_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ organization_id=org.organization_id,+ expected_version=1,+ run_with="code",+ cache_key="custom",+ code_version=2,+ )++ assert result.previous_dispatch_state.run_with == "agent"+ assert result.previous_dispatch_state.cache_key == "default"+ assert result.previous_dispatch_state.code_version == 1+ assert result.workflow.run_with == "code"+ assert result.workflow.cache_key == "custom"+ assert result.workflow.code_version == 2++async def test_update_workflow_dispatch_state_if_latest_rejects_stale_version(agent_db: AgentDB) -> None:org = await agent_db.organizations.create_organization(organization_name="Dispatch Stale Org",diff --git a/tests/unit/test_cached_script_deploy_service.py b/tests/unit/test_cached_script_deploy_service.pyindex d62ae51ad..c4a19aeb8 100644--- a/tests/unit/test_cached_script_deploy_service.py+++ b/tests/unit/test_cached_script_deploy_service.py@@ -6,12 +6,15 @@ import pytestfrom fastapi import HTTPExceptionfrom skyvern.forge.sdk.db.exceptions import NotFoundError+from skyvern.forge.sdk.db.repositories.scripts import WorkflowScriptUpsertStatusfrom skyvern.forge.sdk.workflow.models.workflow import Workflowfrom skyvern.schemas.scripts import (DeployCachedScriptCacheContext,DeployCachedScriptRequest,FileEncoding,ScriptFileCreate,+ ScriptStatus,+ WorkflowScript,)from skyvern.services import cached_script_deploy_service@@ -20,7 +23,13 @@ def _b64(text: str) -> str:return base64.b64encode(text.encode("utf-8")).decode("utf-8")-def _workflow(*, cache_key: str | None = "default", version: int = 3) -> Workflow:+def _workflow(+ *,+ cache_key: str | None = "default",+ version: int = 3,+ run_with: str = "code",+ code_version: int | None = 2,+) -> Workflow:return Workflow(workflow_id="wf_latest",organization_id="org_test",@@ -61,14 +70,49 @@ def _workflow(*, cache_key: str | None = "default", version: int = 3) -> Workflo},],},- run_with="code",+ run_with=run_with,cache_key=cache_key,- code_version=2,+ code_version=code_version,created_at=datetime.now(timezone.utc),modified_at=datetime.now(timezone.utc),)+def _workflow_script(+ *,+ workflow_script_id: str = "ws_created",+ script_id: str = "s_created",+ workflow_id: str | None = "wf_latest",+ workflow_run_id: str | None = "wr_source",+ cache_key: str = "default",+ cache_key_value: str = "default:example.com:v2",+ is_pinned: bool = True,+ pinned_at: datetime | None = None,+ pinned_by: str | None = None,+ created_at: datetime | None = None,+ modified_at: datetime | None = None,+ deleted_at: datetime | None = None,+) -> WorkflowScript:+ now = datetime.now(timezone.utc)+ return WorkflowScript(+ workflow_script_id=workflow_script_id,+ organization_id="org_test",+ script_id=script_id,+ workflow_permanent_id="wpid_test",+ workflow_id=workflow_id,+ workflow_run_id=workflow_run_id,+ cache_key=cache_key,+ cache_key_value=cache_key_value,+ status=ScriptStatus.published,+ is_pinned=is_pinned,+ pinned_at=pinned_at,+ pinned_by=pinned_by,+ created_at=created_at or now,+ modified_at=modified_at or now,+ deleted_at=deleted_at,+ )++def _request(source: str,*,@@ -109,9 +153,19 @@ def _stub_app(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:created_files=[],created_blocks=[],workflow_script_upserts=[],+ workflow_script_upsert_status=WorkflowScriptUpsertStatus.created,+ previous_workflow_script=None,workflow_updates=[],+ previous_dispatch_state=None,+ dispatch_restore_matches=True,+ workflow_script_soft_delete_matches=True,+ workflow_script_restore_matches=True,+ soft_deleted_workflow_scripts=[],+ restored_workflow_scripts=[],+ soft_deleted_script_revisions=[],artifacts=[],fail_dispatch_update=False,+ fail_create_artifact=False,)class Workflows:@@ -122,12 +176,27 @@ def _stub_app(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:state.workflow_updates.append(kwargs)return state.workflow+ async def restore_workflow_script_dispatch_if_matches(self, **kwargs: object) -> Workflow | None:+ state.workflow_updates.append(kwargs)+ return state.workflow if state.dispatch_restore_matches else None+async def update_workflow_dispatch_state_if_latest(self, **kwargs: object) -> Workflow:state.workflow_updates.append(kwargs)if state.fail_dispatch_update:raise NotFoundError("Workflow not found or no longer latest")return state.workflow+ async def update_workflow_dispatch_state_if_latest_with_previous(self, **kwargs: object) -> SimpleNamespace:+ state.workflow_updates.append(kwargs)+ if state.fail_dispatch_update:+ raise NotFoundError("Workflow not found or no longer latest")+ previous_dispatch_state = state.previous_dispatch_state or SimpleNamespace(+ run_with=state.workflow.run_with,+ cache_key=state.workflow.cache_key,+ code_version=state.workflow.code_version,+ )+ return SimpleNamespace(workflow=state.workflow, previous_dispatch_state=previous_dispatch_state)+class Scripts:async def create_script(self, **kwargs: object) -> SimpleNamespace:state.created_scripts.append(kwargs)@@ -148,13 +217,35 @@ def _stub_app(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:async def upsert_workflow_script(self, **kwargs: object) -> SimpleNamespace:state.workflow_script_upserts.append(kwargs)+ workflow_script = _workflow_script(+ script_id=str(kwargs["script_id"]),+ workflow_id=kwargs.get("workflow_id"),+ workflow_run_id=kwargs.get("workflow_run_id"),+ cache_key=str(kwargs["cache_key"]),+ cache_key_value=str(kwargs["cache_key_value"]),+ is_pinned=bool(kwargs["is_pinned"]),+ )return SimpleNamespace(- status=SimpleNamespace(value="created"),- workflow_script=SimpleNamespace(workflow_script_id="ws_created"),+ status=state.workflow_script_upsert_status,+ workflow_script=workflow_script,+ previous_workflow_script=state.previous_workflow_script,)+ async def soft_delete_workflow_script_if_matches(self, **kwargs: object) -> bool:+ state.soft_deleted_workflow_scripts.append(kwargs["workflow_script"])+ return state.workflow_script_soft_delete_matches++ async def restore_workflow_script_if_matches(self, **kwargs: object) -> bool:+ state.restored_workflow_scripts.append(kwargs)+ return state.workflow_script_restore_matches++ async def soft_delete_script_by_revision(self, **kwargs: object) -> None:+ state.soft_deleted_script_revisions.append(kwargs["script_revision_id"])+class ArtifactManager:async def create_script_file_artifact(self, **kwargs: object) -> str:+ if state.fail_create_artifact:+ raise RuntimeError("artifact write failed")state.artifacts.append(kwargs)return f"artifact_{len(state.artifacts)}"@@ -301,6 +392,337 @@ async def run(parameters):assert _stub_app.created_blocks[1]["requires_agent"] is False+async def test_commit_mode_cleans_up_script_when_file_persist_fails(+ _stub_app: SimpleNamespace,+) -> None:+ _stub_app.fail_create_artifact = True+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ with pytest.raises(RuntimeError, match="artifact write failed"):+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ resolved_cache_key_value="default:example.com:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+ assert _stub_app.soft_deleted_workflow_scripts == []+ assert _stub_app.workflow_updates == []+++async def test_commit_mode_cleans_up_mapping_when_workflow_update_fails(+ _stub_app: SimpleNamespace,+) -> None:+ _stub_app.fail_dispatch_update = True+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ with pytest.raises(HTTPException) as exc:+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ resolved_cache_key_value="default:example.com:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert exc.value.status_code == 409+ assert "became stale" in str(exc.value.detail)+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+ assert _stub_app.workflow_updates == [+ {+ "workflow_id": "wf_latest",+ "workflow_permanent_id": "wpid_test",+ "organization_id": "org_test",+ "expected_version": 3,+ "run_with": "code",+ "cache_key": "default",+ "code_version": 2,+ }+ ]+++async def test_commit_mode_restores_updated_mapping_when_workflow_update_fails(+ _stub_app: SimpleNamespace,+) -> None:+ previous_workflow_script = _workflow_script(+ script_id="s_previous",+ workflow_id="wf_previous",+ workflow_run_id="wr_previous",+ is_pinned=False,+ )+ _stub_app.workflow_script_upsert_status = WorkflowScriptUpsertStatus.updated+ _stub_app.previous_workflow_script = previous_workflow_script+ _stub_app.fail_dispatch_update = True+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ with pytest.raises(HTTPException) as exc:+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ resolved_cache_key_value="default:example.com:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert exc.value.status_code == 409+ assert _stub_app.soft_deleted_workflow_scripts == []+ assert _stub_app.restored_workflow_scripts[0]["restore_workflow_script"] == previous_workflow_script+ assert _stub_app.restored_workflow_scripts[0]["current_workflow_script"].script_id == "s_created"+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+++async def test_commit_mode_restores_workflow_dispatch_after_late_failure(+ _stub_app: SimpleNamespace,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_app.workflow = _workflow(cache_key="old-cache", run_with="agent", code_version=None)+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ def fail_response(**_: object) -> None:+ raise RuntimeError("response construction failed")++ monkeypatch.setattr(cached_script_deploy_service, "_response_from_plan", fail_response)++ with pytest.raises(RuntimeError, match="response construction failed"):+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ cache_key="new-cache",+ resolved_cache_key_value="new-cache:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert _stub_app.workflow_updates == [+ {+ "workflow_id": "wf_latest",+ "workflow_permanent_id": "wpid_test",+ "organization_id": "org_test",+ "expected_version": 3,+ "run_with": "code",+ "cache_key": "new-cache",+ "code_version": 2,+ },+ {+ "workflow_id": "wf_latest",+ "organization_id": "org_test",+ "run_with": "agent",+ "cache_key": "old-cache",+ "code_version": None,+ "current_run_with": "code",+ "current_cache_key": "new-cache",+ "current_code_version": 2,+ },+ ]+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+++async def test_commit_mode_restores_dispatch_state_captured_during_write(+ _stub_app: SimpleNamespace,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_app.workflow = _workflow(cache_key="old-plan-cache", run_with="agent", code_version=None)+ _stub_app.previous_dispatch_state = SimpleNamespace(+ run_with="code",+ cache_key="concurrent-cache",+ code_version=2,+ )+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ def fail_response(**_: object) -> None:+ raise RuntimeError("response construction failed")++ monkeypatch.setattr(cached_script_deploy_service, "_response_from_plan", fail_response)++ with pytest.raises(RuntimeError, match="response construction failed"):+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ cache_key="new-cache",+ resolved_cache_key_value="new-cache:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert _stub_app.workflow_updates[-1] == {+ "workflow_id": "wf_latest",+ "organization_id": "org_test",+ "run_with": "code",+ "cache_key": "concurrent-cache",+ "code_version": 2,+ "current_run_with": "code",+ "current_cache_key": "new-cache",+ "current_code_version": 2,+ }+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+++async def test_commit_mode_skips_dispatch_restore_when_current_state_changed(+ _stub_app: SimpleNamespace,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _stub_app.workflow = _workflow(cache_key="old-cache", run_with="agent", code_version=None)+ _stub_app.dispatch_restore_matches = False+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ def fail_response(**_: object) -> None:+ raise RuntimeError("response construction failed")++ monkeypatch.setattr(cached_script_deploy_service, "_response_from_plan", fail_response)++ with pytest.raises(RuntimeError, match="response construction failed"):+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ cache_key="new-cache",+ resolved_cache_key_value="new-cache:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert _stub_app.workflow_updates[-1] == {+ "workflow_id": "wf_latest",+ "organization_id": "org_test",+ "run_with": "agent",+ "cache_key": "old-cache",+ "code_version": None,+ "current_run_with": "code",+ "current_cache_key": "new-cache",+ "current_code_version": 2,+ }+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+++async def test_commit_mode_skips_created_mapping_delete_when_current_state_changed(+ _stub_app: SimpleNamespace,+) -> None:+ _stub_app.fail_dispatch_update = True+ _stub_app.workflow_script_soft_delete_matches = False+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ with pytest.raises(HTTPException) as exc:+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ resolved_cache_key_value="default:example.com:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert exc.value.status_code == 409+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]+++async def test_commit_mode_skips_mapping_restore_when_current_state_changed(+ _stub_app: SimpleNamespace,+) -> None:+ previous_workflow_script = _workflow_script(+ script_id="s_previous",+ workflow_id="wf_previous",+ workflow_run_id="wr_previous",+ is_pinned=False,+ )+ _stub_app.workflow_script_upsert_status = WorkflowScriptUpsertStatus.updated+ _stub_app.previous_workflow_script = previous_workflow_script+ _stub_app.fail_dispatch_update = True+ _stub_app.workflow_script_restore_matches = False+ source = """+import skyvern++async def run(parameters):+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")+"""++ with pytest.raises(HTTPException) as exc:+ await cached_script_deploy_service.deploy_cached_script(+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ request=_request(+ source,+ resolved_cache_key_value="default:example.com:v2",+ dry_run=False,+ source_workflow_run_id="wr_source",+ ),+ )++ assert exc.value.status_code == 409+ assert _stub_app.soft_deleted_workflow_scripts == []+ assert _stub_app.restored_workflow_scripts[0]["restore_workflow_script"] == previous_workflow_script+ assert _stub_app.restored_workflow_scripts[0]["current_workflow_script"].script_id == "s_created"+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]++@pytest.mark.asyncio@pytest.mark.parametrize("file_path",