Skip to content

Automate browser based workflows with AI

Harden cached script deploy cleanup (#6018)

pedrohsdbc12482f5b313
diff --git a/skyvern/forge/sdk/db/repositories/scripts.py b/skyvern/forge/sdk/db/repositories/scripts.py
index 9af26c053..bdb1e869b 100644
--- a/skyvern/forge/sdk/db/repositories/scripts.py
+++ b/skyvern/forge/sdk/db/repositories/scripts.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.db.utils import (
convert_to_script,
convert_to_script_block,
convert_to_script_file,
+ nullable_column_equals,
)
from skyvern.forge.sdk.utils.sanitization import sanitize_postgres_text
from skyvern.schemas.scripts import (
@@ -41,6 +42,28 @@ from skyvern.schemas.scripts import (
LOG = structlog.get_logger()
+def _script_status_value(status: ScriptStatus | str) -> str:
+ return status.value if isinstance(status, ScriptStatus) else status
+
+
+def _workflow_script_matches_snapshot(workflow_script: WorkflowScript) -> list[Any]:
+ return [
+ WorkflowScriptModel.organization_id == workflow_script.organization_id,
+ WorkflowScriptModel.workflow_script_id == workflow_script.workflow_script_id,
+ WorkflowScriptModel.workflow_permanent_id == workflow_script.workflow_permanent_id,
+ WorkflowScriptModel.script_id == workflow_script.script_id,
+ nullable_column_equals(WorkflowScriptModel.workflow_id, workflow_script.workflow_id),
+ nullable_column_equals(WorkflowScriptModel.workflow_run_id, workflow_script.workflow_run_id),
+ WorkflowScriptModel.cache_key == workflow_script.cache_key,
+ WorkflowScriptModel.cache_key_value == workflow_script.cache_key_value,
+ WorkflowScriptModel.status == _script_status_value(workflow_script.status),
+ WorkflowScriptModel.is_pinned == workflow_script.is_pinned,
+ nullable_column_equals(WorkflowScriptModel.pinned_at, workflow_script.pinned_at),
+ nullable_column_equals(WorkflowScriptModel.pinned_by, workflow_script.pinned_by),
+ nullable_column_equals(WorkflowScriptModel.modified_at, workflow_script.modified_at),
+ ]
+
+
class WorkflowScriptWriterIntent(StrEnum):
deploy = "deploy"
auto_regen = "auto_regen"
@@ -57,6 +80,7 @@ class WorkflowScriptUpsertStatus(StrEnum):
class WorkflowScriptUpsertResult:
status: WorkflowScriptUpsertStatus
workflow_script: WorkflowScript
+ previous_workflow_script: WorkflowScript | None = None
class ScriptsRepository(BaseRepository):
@@ -715,6 +739,7 @@ class ScriptsRepository(BaseRepository):
workflow_script=workflow_script,
)
+ previous_workflow_script = WorkflowScript.model_validate(existing)
existing.script_id = script_id
existing.workflow_id = workflow_id
existing.workflow_run_id = workflow_run_id
@@ -731,6 +756,7 @@ class ScriptsRepository(BaseRepository):
return WorkflowScriptUpsertResult(
status=WorkflowScriptUpsertStatus.updated,
workflow_script=WorkflowScript.model_validate(existing),
+ previous_workflow_script=previous_workflow_script,
)
@db_operation("get_workflow_script")
@@ -922,6 +948,74 @@ class ScriptsRepository(BaseRepository):
return result.rowcount > 0
+ @db_operation("soft_delete_workflow_script_if_matches")
+ async def soft_delete_workflow_script_if_matches(
+ self,
+ *,
+ workflow_script: WorkflowScript,
+ ) -> bool:
+ """Soft delete one workflow_scripts row if it still matches a snapshot.
+
+ Failed manual deploy cleanup uses this to avoid deleting a row that a
+ concurrent successful deploy already changed.
+ """
+ async with self.Session() as session:
+ stmt = (
+ update(WorkflowScriptModel)
+ .where(
+ and_(
+ *_workflow_script_matches_snapshot(workflow_script),
+ WorkflowScriptModel.deleted_at.is_(None),
+ )
+ )
+ .values(deleted_at=datetime.now(timezone.utc))
+ )
+ result = await session.execute(stmt)
+ await session.commit()
+
+ return result.rowcount > 0
+
+ @db_operation("restore_workflow_script_if_matches")
+ async def restore_workflow_script_if_matches(
+ self,
+ *,
+ current_workflow_script: WorkflowScript,
+ restore_workflow_script: WorkflowScript,
+ ) -> bool:
+ """Restore one workflow_scripts row if it still matches a written snapshot.
+
+ The live-row predicate prevents cleanup from resurrecting a row another
+ actor soft-deleted after this deploy's forward write.
+ """
+ async with self.Session() as session:
+ status = _script_status_value(restore_workflow_script.status)
+ stmt = (
+ update(WorkflowScriptModel)
+ .where(
+ and_(
+ *_workflow_script_matches_snapshot(current_workflow_script),
+ WorkflowScriptModel.deleted_at.is_(None),
+ )
+ )
+ .values(
+ script_id=restore_workflow_script.script_id,
+ workflow_id=restore_workflow_script.workflow_id,
+ workflow_run_id=restore_workflow_script.workflow_run_id,
+ cache_key=restore_workflow_script.cache_key,
+ cache_key_value=restore_workflow_script.cache_key_value,
+ status=status,
+ is_pinned=restore_workflow_script.is_pinned,
+ pinned_at=restore_workflow_script.pinned_at,
+ pinned_by=restore_workflow_script.pinned_by,
+ modified_at=restore_workflow_script.modified_at,
+ deleted_at=restore_workflow_script.deleted_at,
+ )
+ )
+ result = await session.execute(stmt)
+ await session.commit()
+
+ return result.rowcount > 0
+
@db_operation("delete_workflow_scripts_by_permanent_id")
async def delete_workflow_scripts_by_permanent_id(
self,
diff --git a/skyvern/forge/sdk/db/repositories/workflows.py b/skyvern/forge/sdk/db/repositories/workflows.py
index bb19a3949..51791cb7d 100644
--- a/skyvern/forge/sdk/db/repositories/workflows.py
+++ b/skyvern/forge/sdk/db/repositories/workflows.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, cast
@@ -29,7 +30,7 @@ from skyvern.forge.sdk.db.models import (
WorkflowTemplateModel,
)
from skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepository
-from skyvern.forge.sdk.db.utils import convert_to_workflow, serialize_proxy_location
+from skyvern.forge.sdk.db.utils import convert_to_workflow, nullable_column_equals, serialize_proxy_location
from skyvern.forge.sdk.workflow.models.block import Block, ForLoopBlock, WhileLoopBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
@@ -39,6 +40,19 @@ from skyvern.schemas.workflows import WorkflowStatus
LOG = structlog.get_logger()
+@dataclass(frozen=True)
+class WorkflowDispatchState:
+ run_with: str | None
+ cache_key: str | None
+ code_version: int | None
+
+
+@dataclass(frozen=True)
+class WorkflowDispatchUpdateResult:
+ workflow: Workflow
+ previous_dispatch_state: WorkflowDispatchState
+
+
def _align_block_output_parameters(workflow_definition: WorkflowDefinition) -> None:
"""Rebind each block's ``output_parameter`` to the reconciled instance
from ``workflow_definition.parameters`` by key, recursing into nested
@@ -716,6 +730,151 @@ class WorkflowsRepository(BaseRepository):
if updated_workflow_id is None:
raise NotFoundError("Workflow not found or no longer latest")
+ await session.commit()
+ workflow = (
+ await session.scalars(
+ exclude_deleted(
+ select(WorkflowModel).filter_by(
+ workflow_id=workflow_id,
+ organization_id=organization_id,
+ ),
+ WorkflowModel,
+ )
+ )
+ ).one()
+ is_template = await self.is_workflow_template(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=workflow.organization_id,
+ )
+ return convert_to_workflow(
+ workflow,
+ self.debug_enabled,
+ is_template=is_template,
+ )
+
+ @db_operation("update_workflow_dispatch_state_if_latest_with_previous")
+ async def update_workflow_dispatch_state_if_latest_with_previous(
+ self,
+ *,
+ workflow_id: str,
+ workflow_permanent_id: str,
+ organization_id: str,
+ expected_version: int,
+ run_with: str,
+ cache_key: str,
+ code_version: int,
+ ) -> WorkflowDispatchUpdateResult:
+ async with self.Session() as session:
+ previous_workflow = (
+ await session.scalars(
+ exclude_deleted(
+ select(WorkflowModel)
+ .filter_by(
+ workflow_id=workflow_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ version=expected_version,
+ )
+ .with_for_update(),
+ WorkflowModel,
+ )
+ )
+ ).first()
+ if previous_workflow is None:
+ raise NotFoundError("Workflow not found")
+ previous_dispatch_state = WorkflowDispatchState(
+ run_with=previous_workflow.run_with,
+ cache_key=previous_workflow.cache_key,
+ code_version=previous_workflow.code_version,
+ )
+
+ newer_version_exists = (
+ select(WorkflowModel.workflow_id)
+ .where(WorkflowModel.workflow_permanent_id == workflow_permanent_id)
+ .where(WorkflowModel.organization_id == organization_id)
+ .where(WorkflowModel.version > expected_version)
+ .where(WorkflowModel.deleted_at.is_(None))
+ .exists()
+ )
+ update_workflow_query = (
+ update(WorkflowModel)
+ .where(WorkflowModel.workflow_id == workflow_id)
+ .where(WorkflowModel.organization_id == organization_id)
+ .where(WorkflowModel.workflow_permanent_id == workflow_permanent_id)
+ .where(WorkflowModel.version == expected_version)
+ .where(WorkflowModel.deleted_at.is_(None))
+ .where(~newer_version_exists)
+ .values(
+ run_with=run_with,
+ cache_key=cache_key,
+ code_version=code_version,
+ )
+ .returning(WorkflowModel.workflow_id)
+ )
+ updated_workflow_id = (await session.execute(update_workflow_query)).scalar_one_or_none()
+ if updated_workflow_id is None:
+ raise NotFoundError("Workflow not found or no longer latest")
+
+ await session.commit()
+ workflow = (
+ await session.scalars(
+ exclude_deleted(
+ select(WorkflowModel).filter_by(
+ workflow_id=workflow_id,
+ organization_id=organization_id,
+ ),
+ WorkflowModel,
+ )
+ )
+ ).one()
+ is_template = await self.is_workflow_template(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=workflow.organization_id,
+ )
+ return WorkflowDispatchUpdateResult(
+ workflow=convert_to_workflow(
+ workflow,
+ self.debug_enabled,
+ is_template=is_template,
+ ),
+ previous_dispatch_state=previous_dispatch_state,
+ )
+
+ @db_operation("restore_workflow_script_dispatch_if_matches")
+ async def restore_workflow_script_dispatch_if_matches(
+ self,
+ *,
+ workflow_id: str,
+ organization_id: str,
+ run_with: str | None,
+ cache_key: str | None,
+ code_version: int | None,
+ current_run_with: str | None,
+ current_cache_key: str | None,
+ current_code_version: int | None,
+ ) -> Workflow | None:
+ """Restore script dispatch only if it still matches the deploy write."""
+ async with self.Session() as session:
+ update_workflow_query = (
+ update(WorkflowModel)
+ .where(WorkflowModel.workflow_id == workflow_id)
+ .where(WorkflowModel.organization_id == organization_id)
+ .where(WorkflowModel.deleted_at.is_(None))
+ .where(nullable_column_equals(WorkflowModel.run_with, current_run_with))
+ .where(nullable_column_equals(WorkflowModel.cache_key, current_cache_key))
+ .where(nullable_column_equals(WorkflowModel.code_version, current_code_version))
+ .values(
+ run_with=run_with,
+ cache_key=cache_key,
+ code_version=code_version,
+ )
+ .returning(WorkflowModel.workflow_id)
+ )
+ updated_workflow_id = (await session.execute(update_workflow_query)).scalar_one_or_none()
+ if updated_workflow_id is None:
+ await session.commit()
+ return None
+
await session.commit()
workflow = (
await session.scalars(
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 839ac145b..4a8e00444 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -1,5 +1,6 @@
import json
import typing
+from typing import Any
import pydantic
import pydantic.json
@@ -98,6 +99,10 @@ from skyvern.webeye.actions.actions import (
LOG = structlog.get_logger()
+def nullable_column_equals(column: Any, value: Any) -> Any:
+ return column.is_(None) if value is None else column == value
+
+
def _safe_trigger_type(raw: str | None) -> WorkflowRunTriggerType | None:
if not raw:
return None
diff --git a/skyvern/services/cached_script_deploy_service.py b/skyvern/services/cached_script_deploy_service.py
index 149253faa..1c6619bcb 100644
--- a/skyvern/services/cached_script_deploy_service.py
+++ b/skyvern/services/cached_script_deploy_service.py
@@ -14,7 +14,7 @@ from skyvern.core.script_generations.script_block_extractor import (
)
from skyvern.forge import app
from skyvern.forge.sdk.db.exceptions import NotFoundError
-from skyvern.forge.sdk.db.repositories.scripts import WorkflowScriptWriterIntent
+from skyvern.forge.sdk.db.repositories.scripts import WorkflowScriptUpsertStatus, WorkflowScriptWriterIntent
from skyvern.forge.sdk.workflow.models.workflow import Workflow
from skyvern.schemas.scripts import (
DeployCachedScriptBlockPlan,
@@ -23,6 +23,7 @@ from skyvern.schemas.scripts import (
FileEncoding,
ScriptFileCreate,
ScriptStatus,
+ WorkflowScript,
)
from skyvern.services.workflow_script_service import CacheKeyResolutionError, resolve_cache_key_value
@@ -51,6 +52,31 @@ class _ValidatedScriptFile:
content_bytes: bytes
+@dataclass
+class _CachedScriptDeployUndoState:
+ script_revision_id: str | None = None
+ workflow_script_created_snapshot: WorkflowScript | None = None
+ workflow_script_written_snapshot: WorkflowScript | None = None
+ workflow_script_restore_snapshot: WorkflowScript | None = None
+ workflow_dispatch_updated: bool = False
+ workflow_dispatch_restore_run_with: str | None = None
+ workflow_dispatch_restore_cache_key: str | None = None
+ workflow_dispatch_restore_code_version: int | None = None
+ workflow_dispatch_written_run_with: str | None = None
+ workflow_dispatch_written_cache_key: str | None = None
+ workflow_dispatch_written_code_version: int | None = None
+
+ @property
+ def has_writes(self) -> bool:
+ return (
+ self.script_revision_id is not None
+ or self.workflow_script_created_snapshot is not None
+ or self.workflow_script_written_snapshot is not None
+ or self.workflow_script_restore_snapshot is not None
+ or self.workflow_dispatch_updated
+ )
+
+
def _decode_script_file_bytes(file: ScriptFileCreate) -> bytes:
if file.encoding == FileEncoding.BASE64:
try:
@@ -262,6 +288,107 @@ def _response_from_plan(
)
+async def _cleanup_failed_cached_script_deploy(
+ *,
+ organization_id: str,
+ plan: _CachedScriptDeployPlan,
+ undo_state: _CachedScriptDeployUndoState,
+) -> None:
+ cleanup_errors: list[str] = []
+ if undo_state.workflow_dispatch_updated:
+ try:
+ restored_workflow = await app.DATABASE.workflows.restore_workflow_script_dispatch_if_matches(
+ workflow_id=plan.workflow.workflow_id,
+ organization_id=organization_id,
+ run_with=undo_state.workflow_dispatch_restore_run_with,
+ cache_key=undo_state.workflow_dispatch_restore_cache_key,
+ code_version=undo_state.workflow_dispatch_restore_code_version,
+ current_run_with=undo_state.workflow_dispatch_written_run_with,
+ current_cache_key=undo_state.workflow_dispatch_written_cache_key,
+ current_code_version=undo_state.workflow_dispatch_written_code_version,
+ )
+ if restored_workflow is None:
+ LOG.warning(
+ "Skipped restoring workflow dispatch after cached script deploy failure because it changed",
+ workflow_id=plan.workflow.workflow_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ cleanup_errors.append("workflow_dispatch")
+ LOG.warning(
+ "Failed to restore workflow dispatch state after cached script deploy failure",
+ workflow_id=plan.workflow.workflow_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+
+ if (
+ undo_state.workflow_script_restore_snapshot is not None
+ and undo_state.workflow_script_written_snapshot is not None
+ ):
+ try:
+ restored_workflow_script = await app.DATABASE.scripts.restore_workflow_script_if_matches(
+ current_workflow_script=undo_state.workflow_script_written_snapshot,
+ restore_workflow_script=undo_state.workflow_script_restore_snapshot,
+ )
+ if not restored_workflow_script:
+ LOG.warning(
+ "Skipped restoring workflow script after cached script deploy failure because it changed",
+ workflow_script_id=undo_state.workflow_script_written_snapshot.workflow_script_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ cleanup_errors.append("workflow_script_restore")
+ LOG.warning(
+ "Failed to restore workflow script after cached script deploy failure",
+ workflow_script_id=undo_state.workflow_script_restore_snapshot.workflow_script_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ elif undo_state.workflow_script_created_snapshot is not None:
+ try:
+ deleted_workflow_script = await app.DATABASE.scripts.soft_delete_workflow_script_if_matches(
+ workflow_script=undo_state.workflow_script_created_snapshot,
+ )
+ if not deleted_workflow_script:
+ LOG.warning(
+ "Skipped soft-deleting workflow script after cached script deploy failure because it changed",
+ workflow_script_id=undo_state.workflow_script_created_snapshot.workflow_script_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ cleanup_errors.append("workflow_script")
+ LOG.warning(
+ "Failed to soft-delete workflow script after cached script deploy failure",
+ workflow_script_id=undo_state.workflow_script_created_snapshot.workflow_script_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+
+ if undo_state.script_revision_id is not None:
+ try:
+ await app.DATABASE.scripts.soft_delete_script_by_revision(
+ script_revision_id=undo_state.script_revision_id,
+ organization_id=organization_id,
+ )
+ except Exception:
+ cleanup_errors.append("script_revision")
+ LOG.warning(
+ "Failed to soft-delete script revision after cached script deploy failure",
+ script_revision_id=undo_state.script_revision_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+
+ if cleanup_errors:
+ LOG.error(
+ "Cached script deploy failed and cleanup was incomplete",
+ workflow_id=plan.workflow.workflow_id,
+ organization_id=organization_id,
+ cleanup_errors=cleanup_errors,
+ )
+
+
async def deploy_cached_script(
*,
organization_id: str,
@@ -279,11 +406,13 @@ async def deploy_cached_script(
script = None
commit_stage = "create_script"
+ undo_state = _CachedScriptDeployUndoState()
try:
script = await app.DATABASE.scripts.create_script(
organization_id=organization_id,
run_id=request.source_workflow_run_id,
)
+ undo_state.script_revision_id = script.script_revision_id
commit_stage = "persist_script_files"
await _persist_script_files(
files=plan.validated_files,
@@ -317,16 +446,49 @@ async def deploy_cached_script(
is_pinned=True,
writer_intent=WorkflowScriptWriterIntent.deploy,
)
+ if workflow_script_result.status == WorkflowScriptUpsertStatus.created:
+ undo_state.workflow_script_created_snapshot = workflow_script_result.workflow_script
+ elif workflow_script_result.status == WorkflowScriptUpsertStatus.updated:
+ undo_state.workflow_script_written_snapshot = workflow_script_result.workflow_script
+ undo_state.workflow_script_restore_snapshot = workflow_script_result.previous_workflow_script
+ if undo_state.workflow_script_restore_snapshot is None:
+ LOG.warning(
+ "Cached script deploy updated workflow script without a previous snapshot",
+ workflow_script_id=workflow_script_result.workflow_script.workflow_script_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ cache_key_value=plan.cache_key_value,
+ )
commit_stage = "update_workflow_dispatch_state"
- await app.DATABASE.workflows.update_workflow_dispatch_state_if_latest(
+ code_version = _code_version_for_cache_context(request)
+ dispatch_update_result = await app.DATABASE.workflows.update_workflow_dispatch_state_if_latest_with_previous(
workflow_id=plan.workflow.workflow_id,
workflow_permanent_id=workflow_permanent_id,
organization_id=organization_id,
expected_version=plan.workflow.version,
run_with="code",
cache_key=cache_key,
- code_version=_code_version_for_cache_context(request),
+ code_version=code_version,
+ )
+ previous_dispatch_state = dispatch_update_result.previous_dispatch_state
+ undo_state.workflow_dispatch_updated = True
+ undo_state.workflow_dispatch_restore_run_with = previous_dispatch_state.run_with
+ undo_state.workflow_dispatch_restore_cache_key = previous_dispatch_state.cache_key
+ undo_state.workflow_dispatch_restore_code_version = previous_dispatch_state.code_version
+ undo_state.workflow_dispatch_written_run_with = "code"
+ undo_state.workflow_dispatch_written_cache_key = cache_key
+ undo_state.workflow_dispatch_written_code_version = code_version
+
+ return _response_from_plan(
+ plan=plan,
+ dry_run=False,
+ script_id=script.script_id,
+ script_revision_id=script.script_revision_id,
+ script_version=script.version,
+ workflow_script_id=workflow_script_result.workflow_script.workflow_script_id,
+ workflow_script_upsert_status=workflow_script_result.status.value,
+ script_was_created=True,
)
except NotFoundError as exc:
LOG.exception(
@@ -339,6 +501,12 @@ async def deploy_cached_script(
script_id=script.script_id if script else None,
commit_stage=commit_stage,
)
+ if undo_state.has_writes:
+ await _cleanup_failed_cached_script_deploy(
+ organization_id=organization_id,
+ plan=plan,
+ undo_state=undo_state,
+ )
raise HTTPException(
status_code=409,
detail=(
@@ -357,15 +525,10 @@ async def deploy_cached_script(
script_id=script.script_id if script else None,
commit_stage=commit_stage,
)
+ if undo_state.has_writes:
+ await _cleanup_failed_cached_script_deploy(
+ organization_id=organization_id,
+ plan=plan,
+ undo_state=undo_state,
+ )
raise
-
- return _response_from_plan(
- plan=plan,
- dry_run=False,
- script_id=script.script_id,
- script_revision_id=script.script_revision_id,
- script_version=script.version,
- workflow_script_id=workflow_script_result.workflow_script.workflow_script_id,
- workflow_script_upsert_status=workflow_script_result.status.value,
- script_was_created=True,
- )
diff --git a/tests/unit/forge/sdk/db/test_repositories.py b/tests/unit/forge/sdk/db/test_repositories.py
index 67171225d..874f61b5e 100644
--- a/tests/unit/forge/sdk/db/test_repositories.py
+++ b/tests/unit/forge/sdk/db/test_repositories.py
@@ -69,6 +69,8 @@ def test_scripts_repository_instantiation():
assert repo.Session is mock_session
assert hasattr(repo, "create_script")
assert hasattr(repo, "get_scripts")
+ assert hasattr(repo, "soft_delete_workflow_script_if_matches")
+ assert hasattr(repo, "restore_workflow_script_if_matches")
def test_workflow_parameters_repository_instantiation():
@@ -101,6 +103,8 @@ def test_workflows_repository_instantiation():
assert hasattr(repo, "get_workflow")
assert hasattr(repo, "create_workflow")
assert hasattr(repo, "get_workflow_by_permanent_id")
+ assert hasattr(repo, "update_workflow_dispatch_state_if_latest_with_previous")
+ assert hasattr(repo, "restore_workflow_script_dispatch_if_matches")
def test_browser_sessions_repository_instantiation():
diff --git a/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py b/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py
index a47dc4431..b84759b46 100644
--- a/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py
+++ b/tests/unit/forge/sdk/db/test_update_workflow_definition_fields.py
@@ -121,6 +121,40 @@ async def test_update_workflow_dispatch_state_if_latest_updates_current_version(
assert updated.code_version == 2
+async def test_update_workflow_dispatch_state_with_previous_returns_pre_update_state(agent_db: AgentDB) -> None:
+ org = await agent_db.organizations.create_organization(
+ organization_name="Dispatch Previous Org",
+ domain="dispatch-previous.test",
+ )
+ workflow = await agent_db.workflows.create_workflow(
+ title="dispatch-v1",
+ workflow_definition={"parameters": [], "blocks": []},
+ organization_id=org.organization_id,
+ workflow_permanent_id="wpid_dispatch_previous",
+ version=1,
+ run_with="agent",
+ cache_key="default",
+ code_version=1,
+ )
+
+ result = await agent_db.workflows.update_workflow_dispatch_state_if_latest_with_previous(
+ workflow_id=workflow.workflow_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ organization_id=org.organization_id,
+ expected_version=1,
+ run_with="code",
+ cache_key="custom",
+ code_version=2,
+ )
+
+ assert result.previous_dispatch_state.run_with == "agent"
+ assert result.previous_dispatch_state.cache_key == "default"
+ assert result.previous_dispatch_state.code_version == 1
+ assert result.workflow.run_with == "code"
+ assert result.workflow.cache_key == "custom"
+ assert result.workflow.code_version == 2
+
+
async def test_update_workflow_dispatch_state_if_latest_rejects_stale_version(agent_db: AgentDB) -> None:
org = await agent_db.organizations.create_organization(
organization_name="Dispatch Stale Org",
diff --git a/tests/unit/test_cached_script_deploy_service.py b/tests/unit/test_cached_script_deploy_service.py
index d62ae51ad..c4a19aeb8 100644
--- a/tests/unit/test_cached_script_deploy_service.py
+++ b/tests/unit/test_cached_script_deploy_service.py
@@ -6,12 +6,15 @@ import pytest
from fastapi import HTTPException
from skyvern.forge.sdk.db.exceptions import NotFoundError
+from skyvern.forge.sdk.db.repositories.scripts import WorkflowScriptUpsertStatus
from skyvern.forge.sdk.workflow.models.workflow import Workflow
from skyvern.schemas.scripts import (
DeployCachedScriptCacheContext,
DeployCachedScriptRequest,
FileEncoding,
ScriptFileCreate,
+ ScriptStatus,
+ WorkflowScript,
)
from skyvern.services import cached_script_deploy_service
@@ -20,7 +23,13 @@ def _b64(text: str) -> str:
return base64.b64encode(text.encode("utf-8")).decode("utf-8")
-def _workflow(*, cache_key: str | None = "default", version: int = 3) -> Workflow:
+def _workflow(
+ *,
+ cache_key: str | None = "default",
+ version: int = 3,
+ run_with: str = "code",
+ code_version: int | None = 2,
+) -> Workflow:
return Workflow(
workflow_id="wf_latest",
organization_id="org_test",
@@ -61,14 +70,49 @@ def _workflow(*, cache_key: str | None = "default", version: int = 3) -> Workflo
},
],
},
- run_with="code",
+ run_with=run_with,
cache_key=cache_key,
- code_version=2,
+ code_version=code_version,
created_at=datetime.now(timezone.utc),
modified_at=datetime.now(timezone.utc),
)
+def _workflow_script(
+ *,
+ workflow_script_id: str = "ws_created",
+ script_id: str = "s_created",
+ workflow_id: str | None = "wf_latest",
+ workflow_run_id: str | None = "wr_source",
+ cache_key: str = "default",
+ cache_key_value: str = "default:example.com:v2",
+ is_pinned: bool = True,
+ pinned_at: datetime | None = None,
+ pinned_by: str | None = None,
+ created_at: datetime | None = None,
+ modified_at: datetime | None = None,
+ deleted_at: datetime | None = None,
+) -> WorkflowScript:
+ now = datetime.now(timezone.utc)
+ return WorkflowScript(
+ workflow_script_id=workflow_script_id,
+ organization_id="org_test",
+ script_id=script_id,
+ workflow_permanent_id="wpid_test",
+ workflow_id=workflow_id,
+ workflow_run_id=workflow_run_id,
+ cache_key=cache_key,
+ cache_key_value=cache_key_value,
+ status=ScriptStatus.published,
+ is_pinned=is_pinned,
+ pinned_at=pinned_at,
+ pinned_by=pinned_by,
+ created_at=created_at or now,
+ modified_at=modified_at or now,
+ deleted_at=deleted_at,
+ )
+
+
def _request(
source: str,
*,
@@ -109,9 +153,19 @@ def _stub_app(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
created_files=[],
created_blocks=[],
workflow_script_upserts=[],
+ workflow_script_upsert_status=WorkflowScriptUpsertStatus.created,
+ previous_workflow_script=None,
workflow_updates=[],
+ previous_dispatch_state=None,
+ dispatch_restore_matches=True,
+ workflow_script_soft_delete_matches=True,
+ workflow_script_restore_matches=True,
+ soft_deleted_workflow_scripts=[],
+ restored_workflow_scripts=[],
+ soft_deleted_script_revisions=[],
artifacts=[],
fail_dispatch_update=False,
+ fail_create_artifact=False,
)
class Workflows:
@@ -122,12 +176,27 @@ def _stub_app(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
state.workflow_updates.append(kwargs)
return state.workflow
+ async def restore_workflow_script_dispatch_if_matches(self, **kwargs: object) -> Workflow | None:
+ state.workflow_updates.append(kwargs)
+ return state.workflow if state.dispatch_restore_matches else None
+
async def update_workflow_dispatch_state_if_latest(self, **kwargs: object) -> Workflow:
state.workflow_updates.append(kwargs)
if state.fail_dispatch_update:
raise NotFoundError("Workflow not found or no longer latest")
return state.workflow
+ async def update_workflow_dispatch_state_if_latest_with_previous(self, **kwargs: object) -> SimpleNamespace:
+ state.workflow_updates.append(kwargs)
+ if state.fail_dispatch_update:
+ raise NotFoundError("Workflow not found or no longer latest")
+ previous_dispatch_state = state.previous_dispatch_state or SimpleNamespace(
+ run_with=state.workflow.run_with,
+ cache_key=state.workflow.cache_key,
+ code_version=state.workflow.code_version,
+ )
+ return SimpleNamespace(workflow=state.workflow, previous_dispatch_state=previous_dispatch_state)
+
class Scripts:
async def create_script(self, **kwargs: object) -> SimpleNamespace:
state.created_scripts.append(kwargs)
@@ -148,13 +217,35 @@ def _stub_app(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
async def upsert_workflow_script(self, **kwargs: object) -> SimpleNamespace:
state.workflow_script_upserts.append(kwargs)
+ workflow_script = _workflow_script(
+ script_id=str(kwargs["script_id"]),
+ workflow_id=kwargs.get("workflow_id"),
+ workflow_run_id=kwargs.get("workflow_run_id"),
+ cache_key=str(kwargs["cache_key"]),
+ cache_key_value=str(kwargs["cache_key_value"]),
+ is_pinned=bool(kwargs["is_pinned"]),
+ )
return SimpleNamespace(
- status=SimpleNamespace(value="created"),
- workflow_script=SimpleNamespace(workflow_script_id="ws_created"),
+ status=state.workflow_script_upsert_status,
+ workflow_script=workflow_script,
+ previous_workflow_script=state.previous_workflow_script,
)
+ async def soft_delete_workflow_script_if_matches(self, **kwargs: object) -> bool:
+ state.soft_deleted_workflow_scripts.append(kwargs["workflow_script"])
+ return state.workflow_script_soft_delete_matches
+
+ async def restore_workflow_script_if_matches(self, **kwargs: object) -> bool:
+ state.restored_workflow_scripts.append(kwargs)
+ return state.workflow_script_restore_matches
+
+ async def soft_delete_script_by_revision(self, **kwargs: object) -> None:
+ state.soft_deleted_script_revisions.append(kwargs["script_revision_id"])
+
class ArtifactManager:
async def create_script_file_artifact(self, **kwargs: object) -> str:
+ if state.fail_create_artifact:
+ raise RuntimeError("artifact write failed")
state.artifacts.append(kwargs)
return f"artifact_{len(state.artifacts)}"
@@ -301,6 +392,337 @@ async def run(parameters):
assert _stub_app.created_blocks[1]["requires_agent"] is False
+async def test_commit_mode_cleans_up_script_when_file_persist_fails(
+ _stub_app: SimpleNamespace,
+) -> None:
+ _stub_app.fail_create_artifact = True
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ with pytest.raises(RuntimeError, match="artifact write failed"):
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ resolved_cache_key_value="default:example.com:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+ assert _stub_app.soft_deleted_workflow_scripts == []
+ assert _stub_app.workflow_updates == []
+
+
+async def test_commit_mode_cleans_up_mapping_when_workflow_update_fails(
+ _stub_app: SimpleNamespace,
+) -> None:
+ _stub_app.fail_dispatch_update = True
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ with pytest.raises(HTTPException) as exc:
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ resolved_cache_key_value="default:example.com:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert exc.value.status_code == 409
+ assert "became stale" in str(exc.value.detail)
+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+ assert _stub_app.workflow_updates == [
+ {
+ "workflow_id": "wf_latest",
+ "workflow_permanent_id": "wpid_test",
+ "organization_id": "org_test",
+ "expected_version": 3,
+ "run_with": "code",
+ "cache_key": "default",
+ "code_version": 2,
+ }
+ ]
+
+
+async def test_commit_mode_restores_updated_mapping_when_workflow_update_fails(
+ _stub_app: SimpleNamespace,
+) -> None:
+ previous_workflow_script = _workflow_script(
+ script_id="s_previous",
+ workflow_id="wf_previous",
+ workflow_run_id="wr_previous",
+ is_pinned=False,
+ )
+ _stub_app.workflow_script_upsert_status = WorkflowScriptUpsertStatus.updated
+ _stub_app.previous_workflow_script = previous_workflow_script
+ _stub_app.fail_dispatch_update = True
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ with pytest.raises(HTTPException) as exc:
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ resolved_cache_key_value="default:example.com:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert exc.value.status_code == 409
+ assert _stub_app.soft_deleted_workflow_scripts == []
+ assert _stub_app.restored_workflow_scripts[0]["restore_workflow_script"] == previous_workflow_script
+ assert _stub_app.restored_workflow_scripts[0]["current_workflow_script"].script_id == "s_created"
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+
+
+async def test_commit_mode_restores_workflow_dispatch_after_late_failure(
+ _stub_app: SimpleNamespace,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_app.workflow = _workflow(cache_key="old-cache", run_with="agent", code_version=None)
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ def fail_response(**_: object) -> None:
+ raise RuntimeError("response construction failed")
+
+ monkeypatch.setattr(cached_script_deploy_service, "_response_from_plan", fail_response)
+
+ with pytest.raises(RuntimeError, match="response construction failed"):
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ cache_key="new-cache",
+ resolved_cache_key_value="new-cache:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert _stub_app.workflow_updates == [
+ {
+ "workflow_id": "wf_latest",
+ "workflow_permanent_id": "wpid_test",
+ "organization_id": "org_test",
+ "expected_version": 3,
+ "run_with": "code",
+ "cache_key": "new-cache",
+ "code_version": 2,
+ },
+ {
+ "workflow_id": "wf_latest",
+ "organization_id": "org_test",
+ "run_with": "agent",
+ "cache_key": "old-cache",
+ "code_version": None,
+ "current_run_with": "code",
+ "current_cache_key": "new-cache",
+ "current_code_version": 2,
+ },
+ ]
+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+
+
+async def test_commit_mode_restores_dispatch_state_captured_during_write(
+ _stub_app: SimpleNamespace,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_app.workflow = _workflow(cache_key="old-plan-cache", run_with="agent", code_version=None)
+ _stub_app.previous_dispatch_state = SimpleNamespace(
+ run_with="code",
+ cache_key="concurrent-cache",
+ code_version=2,
+ )
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ def fail_response(**_: object) -> None:
+ raise RuntimeError("response construction failed")
+
+ monkeypatch.setattr(cached_script_deploy_service, "_response_from_plan", fail_response)
+
+ with pytest.raises(RuntimeError, match="response construction failed"):
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ cache_key="new-cache",
+ resolved_cache_key_value="new-cache:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert _stub_app.workflow_updates[-1] == {
+ "workflow_id": "wf_latest",
+ "organization_id": "org_test",
+ "run_with": "code",
+ "cache_key": "concurrent-cache",
+ "code_version": 2,
+ "current_run_with": "code",
+ "current_cache_key": "new-cache",
+ "current_code_version": 2,
+ }
+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+
+
+async def test_commit_mode_skips_dispatch_restore_when_current_state_changed(
+ _stub_app: SimpleNamespace,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _stub_app.workflow = _workflow(cache_key="old-cache", run_with="agent", code_version=None)
+ _stub_app.dispatch_restore_matches = False
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ def fail_response(**_: object) -> None:
+ raise RuntimeError("response construction failed")
+
+ monkeypatch.setattr(cached_script_deploy_service, "_response_from_plan", fail_response)
+
+ with pytest.raises(RuntimeError, match="response construction failed"):
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ cache_key="new-cache",
+ resolved_cache_key_value="new-cache:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert _stub_app.workflow_updates[-1] == {
+ "workflow_id": "wf_latest",
+ "organization_id": "org_test",
+ "run_with": "agent",
+ "cache_key": "old-cache",
+ "code_version": None,
+ "current_run_with": "code",
+ "current_cache_key": "new-cache",
+ "current_code_version": 2,
+ }
+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+
+
+async def test_commit_mode_skips_created_mapping_delete_when_current_state_changed(
+ _stub_app: SimpleNamespace,
+) -> None:
+ _stub_app.fail_dispatch_update = True
+ _stub_app.workflow_script_soft_delete_matches = False
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ with pytest.raises(HTTPException) as exc:
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ resolved_cache_key_value="default:example.com:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert exc.value.status_code == 409
+ assert [row.workflow_script_id for row in _stub_app.soft_deleted_workflow_scripts] == ["ws_created"]
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+
+
+async def test_commit_mode_skips_mapping_restore_when_current_state_changed(
+ _stub_app: SimpleNamespace,
+) -> None:
+ previous_workflow_script = _workflow_script(
+ script_id="s_previous",
+ workflow_id="wf_previous",
+ workflow_run_id="wr_previous",
+ is_pinned=False,
+ )
+ _stub_app.workflow_script_upsert_status = WorkflowScriptUpsertStatus.updated
+ _stub_app.previous_workflow_script = previous_workflow_script
+ _stub_app.fail_dispatch_update = True
+ _stub_app.workflow_script_restore_matches = False
+ source = """
+import skyvern
+
+async def run(parameters):
+ await skyvern.run_task(prompt="...", label="step_a", cache_key="step_a")
+"""
+
+ with pytest.raises(HTTPException) as exc:
+ await cached_script_deploy_service.deploy_cached_script(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ request=_request(
+ source,
+ resolved_cache_key_value="default:example.com:v2",
+ dry_run=False,
+ source_workflow_run_id="wr_source",
+ ),
+ )
+
+ assert exc.value.status_code == 409
+ assert _stub_app.soft_deleted_workflow_scripts == []
+ assert _stub_app.restored_workflow_scripts[0]["restore_workflow_script"] == previous_workflow_script
+ assert _stub_app.restored_workflow_scripts[0]["current_workflow_script"].script_id == "s_created"
+ assert _stub_app.soft_deleted_script_revisions == ["sr_created"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"file_path",