Skip to content

Automate browser based workflows with AI

Fail runs orphaned by a crashed self-hosted server (SKY-7819) (#8925)

Shuchang Zheng8e73b4dbbc37
diff --git a/.env.example b/.env.example
index c94b1f5ef..cc0e1658f 100644
--- a/.env.example
+++ b/.env.example
@@ -146,6 +146,10 @@ BROWSER_ACTION_TIMEOUT_MS=5000
# Agent run configuration:
# MAX_STEPS_PER_RUN: Maximum number of steps to execute per run unless the agent finishes with a terminal state (last step or error).
MAX_STEPS_PER_RUN=50
+# RUN_HEARTBEAT_TIMEOUT_MINUTES: Minutes without a heartbeat before a server fails a run and sends its webhook or starts a policy retry.
+# Applies when the API server executes runs itself (the default). The same server after restart or another server can act.
+# Allowed values: 0 (off) or 2 to 1440. Servers that share a database must use the same value.
+RUN_HEARTBEAT_TIMEOUT_MINUTES=30
# Logging and database configuration:
# LOG_LEVEL: Control log level (e.g., INFO, DEBUG).
diff --git a/alembic/versions/2026_10_16_0738-50c59227239d_add_execution_leases_for_in_process_runs.py b/alembic/versions/2026_10_16_0738-50c59227239d_add_execution_leases_for_in_process_runs.py
new file mode 100644
index 000000000..333a18b4f
--- /dev/null
+++ b/alembic/versions/2026_10_16_0738-50c59227239d_add_execution_leases_for_in_process_runs.py
@@ -0,0 +1,34 @@
+"""Add execution leases for in-process runs.
+
+Revision ID: 50c59227239d
+Revises: 4452dbf24863
+Create Date: 2026-10-16T07:38:57.828780+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "50c59227239d"
+down_revision: Union[str, None] = "4452dbf24863"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "run_execution_leases",
+ sa.Column("run_id", sa.String(), primary_key=True),
+ sa.Column("run_type", sa.String(), nullable=False),
+ sa.Column("owner_id", sa.String(), nullable=False),
+ sa.Column("attempt_number", sa.Integer(), nullable=True),
+ sa.Column("heartbeat_at", sa.DateTime(), nullable=False),
+ )
+
+
+def downgrade() -> None:
+ op.drop_table("run_execution_leases")
diff --git a/docs/reference/configuration.mdx b/docs/reference/configuration.mdx
index b456f7936..1268c963b 100644
--- a/docs/reference/configuration.mdx
+++ b/docs/reference/configuration.mdx
@@ -726,8 +726,11 @@ Never change `ENCRYPTOR_AES_SECRET_KEY`, `ENCRYPTOR_AES_SALT` or `ENCRYPTOR_AES_
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR` or `CRITICAL`. Any other value means `INFO`. |
| `CODE_BLOCK_MODE` | `enabled` | `disabled` rejects [Code blocks](/reference/blocks) in runs |
| `TOTP_LIFESPAN_MINUTES` | `10` | How long a 2FA code sent to Skyvern stays usable. Older codes are ignored. |
+| `RUN_HEARTBEAT_TIMEOUT_MINUTES` | `30` | Minutes without a heartbeat from the server process that executes a run. After this time, a server marks the run `failed` and sends its failure webhook, or starts a retry under the workflow's retry policy. This can be the same server after a restart, or another server on the same database. This check applies when the API server executes runs itself (the default). |
| `ENABLE_URL_VALIDATION` | `false` | `true` makes Skyvern refuse URLs that point at loopback, private-network or cloud-metadata addresses. Off by default. See [URL validation](#url-validation). |
+Set `RUN_HEARTBEAT_TIMEOUT_MINUTES` to `0` to turn off this check. Allowed values are `0` or `2` to `1440`. The check never fails a run that has no execution lease: runs from older versions, scripts, embedded clients, or servers with the check off, and runs that were already stuck before the upgrade. All server processes that share a database must use the same value and keep their clocks synchronized (NTP). When other servers share the database, they fail a server's runs if it stalls for longer than this time. They also fail its runs if it cannot reach the database for longer than this time. A single server keeps its runs and continues them when it resumes. The check does not fail runs that have not started (`created` or `queued`). When a server shuts down, it waits up to 15 seconds for failure webhooks that its reaper has started. A webhook that does not finish in that time is not sent. Webhook receivers should deduplicate deliveries by run ID, attempt, and status.
+
### URL validation
URL validation is off by default, so Skyvern can open, fetch and call internal hosts without extra setup. That includes an intranet portal, a service on your network and `localhost`. Set `ENABLE_URL_VALIDATION=true` if people you don't trust can start tasks or workflows on your server, or if it runs next to internal services or a cloud metadata endpoint.
diff --git a/skyvern/config.py b/skyvern/config.py
index e3d9d622a..6d4a20780 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -306,6 +306,7 @@ class Settings(BaseSettings):
COPILOT_ALLOW_INLINE_CODE_EXECUTION: bool = False
# Default code_only for MCP block/workflow tools. Off = permissive.
MCP_CODE_ONLY_MODE: bool = False
+ RUN_HEARTBEAT_TIMEOUT_MINUTES: int = 30
PORT: int = 8000
# uvicorn answers 503 without dispatching to ASGI once *either* open connections or in-flight
# requests reach this. Open connections is the binding term -- idle keep-alives and long-lived
@@ -1087,6 +1088,13 @@ class Settings(BaseSettings):
return self
+ @field_validator("RUN_HEARTBEAT_TIMEOUT_MINUTES")
+ @classmethod
+ def _run_heartbeat_timeout_minutes_bounds(cls, value: int) -> int:
+ if value != 0 and not 2 <= value <= 1440:
+ raise ValueError("RUN_HEARTBEAT_TIMEOUT_MINUTES must be 0 (off) or between 2 and 1440 minutes")
+ return value
+
@field_validator("API_LIMIT_CONCURRENCY", mode="before")
@classmethod
def _api_limit_concurrency_unlimited_sentinels(cls, value: Any) -> Any:
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 7854034ce..f162bb616 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -157,6 +157,7 @@ from skyvern.forge.sdk.core.skyvern_context import (
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.exceptions import NotFoundError
+from skyvern.forge.sdk.executor.run_liveness import note_run_finalized, run_lease_lost
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
@@ -2199,11 +2200,14 @@ class ForgeAgent:
task_retry=task_retry,
)
# Update task status to running
- task = await app.DATABASE.tasks.update_task(
- task_id=task.task_id,
- organization_id=task.organization_id,
- status=TaskStatus.running,
- )
+ if await run_lease_lost(workflow_run.workflow_run_id):
+ task = await app.DATABASE.tasks.get_task(task.task_id, task.organization_id) or task
+ else:
+ task = await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=task.organization_id,
+ status=TaskStatus.running,
+ )
step = await app.DATABASE.tasks.create_step(
task.task_id,
@@ -2244,11 +2248,14 @@ class ForgeAgent:
order=task_order,
retry=task_retry,
)
- task = await app.DATABASE.tasks.update_task(
- task_id=task.task_id,
- organization_id=organization_id,
- status=TaskStatus.running,
- )
+ if await run_lease_lost(workflow_run_id):
+ task = await app.DATABASE.tasks.get_task(task.task_id, organization_id) or task
+ else:
+ task = await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.running,
+ )
step = await app.DATABASE.tasks.create_step(
task.task_id,
order=0,
@@ -9820,11 +9827,21 @@ class ForgeAgent:
diff=skyvern_context.mask_multi_field_totp_data(update_comparison, task_id=task.task_id),
sampling=True,
)
+ registered_run_id = task.workflow_run_id or task.task_id
+ lease_lost = await run_lease_lost(registered_run_id)
+ if lease_lost and not status.is_final():
+ return await app.DATABASE.tasks.get_task(task_id=task.task_id, organization_id=task.organization_id) or task
updated_task, finish_claimed = await app.DATABASE.tasks.update_task_and_claim_finish(
task.task_id,
organization_id=task.organization_id,
+ keep_final_status=lease_lost,
**updates,
)
+ if lease_lost and updated_task.status != status:
+ note_run_finalized(task.task_id)
+ return updated_task
+ if status.is_final():
+ note_run_finalized(task.task_id)
# Minutes emit only when THIS write atomically flipped finished_at from NULL
# -- the shared claim every task_v1 finalizer rides -- so a concurrent
# finalizer (the stuck-task sweep's CAS, the temporal timeout activity, a
diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.py
index 54f8b5014..94d077bda 100644
--- a/skyvern/forge/api_app.py
+++ b/skyvern/forge/api_app.py
@@ -47,6 +47,7 @@ from skyvern.forge.sdk.db.exceptions import DatabaseConnectionUnavailableError,
from skyvern.forge.sdk.db.models import Base
from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutor
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
+from skyvern.forge.sdk.executor.run_liveness import start_run_lease_tasks, stop_run_lease_tasks
from skyvern.forge.sdk.routes import internal_auth, internal_llms
from skyvern.forge.sdk.routes.google_oauth import google_oauth_router
from skyvern.forge.sdk.routes.google_sheets import google_sheets_router
@@ -68,6 +69,7 @@ from skyvern.services.cleanup_service import (
stop_cleanup_scheduler,
stop_temp_artifact_sweep,
)
+from skyvern.services.orphaned_run_reaper import reap_orphaned_runs_once
from skyvern.services.workflow_run_group_service import run_workflow_run_group_recovery_loop
from skyvern.services.workflow_schedule_service import (
start_workflow_schedule_scheduler,
@@ -431,6 +433,11 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:
if isinstance(AsyncExecutorFactory.get_executor(), BackgroundTaskExecutor)
else None
)
+ if (
+ isinstance(AsyncExecutorFactory.get_executor(), BackgroundTaskExecutor)
+ and settings.RUN_HEARTBEAT_TIMEOUT_MINUTES > 0
+ ):
+ start_run_lease_tasks(reap_orphaned_runs_once)
try:
if mcp_app:
async with mcp_app.lifespan(mcp_app):
@@ -446,6 +453,7 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:
await asyncio.gather(
retry_recovery_task, *([group_recovery_task] if group_recovery_task else []), return_exceptions=True
)
+ await stop_run_lease_tasks()
# The initial pass starts the periodic sweep even when cancelled; stop it so the loop can drain.
await AsyncExecutorFactory.get_executor().stop_retry_recovery()
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.py
index 0536e9715..2f2421c60 100644
--- a/skyvern/forge/sdk/db/agent_db.py
+++ b/skyvern/forge/sdk/db/agent_db.py
@@ -36,6 +36,7 @@ from skyvern.forge.sdk.db.repositories.microsoft_oauth import MicrosoftOAuthRepo
from skyvern.forge.sdk.db.repositories.observer import ObserverRepository
from skyvern.forge.sdk.db.repositories.organizations import OrganizationsRepository
from skyvern.forge.sdk.db.repositories.otp import OTPRepository
+from skyvern.forge.sdk.db.repositories.run_execution_leases import RunExecutionLeasesRepository
from skyvern.forge.sdk.db.repositories.run_feedback import RunFeedbackRepository
from skyvern.forge.sdk.db.repositories.schedules import SchedulesRepository
from skyvern.forge.sdk.db.repositories.scripts import ScriptsRepository
@@ -398,6 +399,7 @@ class AgentDB(BaseAlchemyDB):
)
# -- Zero-dependency repositories --
+ self.run_execution_leases = RunExecutionLeasesRepository(self.Session, debug_enabled, self.is_retryable_error)
self.tasks = TasksRepository(self.Session, debug_enabled, self.is_retryable_error)
self.workflows = WorkflowsRepository(
self.Session,
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index c210b6797..59e32bfec 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -95,6 +95,16 @@ class Base(AsyncAttrs, DeclarativeBase):
pass
+class RunExecutionLeaseModel(Base):
+ __tablename__ = "run_execution_leases"
+
+ run_id = Column(String, primary_key=True)
+ run_type = Column(String, nullable=False)
+ owner_id = Column(String, nullable=False)
+ attempt_number = Column(Integer, nullable=True)
+ heartbeat_at = Column(DateTime, nullable=False)
+
+
class TaskModel(Base):
__tablename__ = "tasks"
__table_args__ = (
diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.py
index 169c7259d..237bb6c33 100644
--- a/skyvern/forge/sdk/db/repositories/observer.py
+++ b/skyvern/forge/sdk/db/repositories/observer.py
@@ -348,6 +348,36 @@ class ObserverRepository(BaseRepository):
return Thought.model_validate(thought_obj)
raise NotFoundError(f"Thought {thought_id}")
+ @db_operation("update_task_v2_status_if_not_final")
+ async def update_task_v2_status_if_not_final(
+ self,
+ task_v2_id: str,
+ organization_id: str,
+ status: TaskV2Status,
+ *,
+ failure_category: list[dict[str, Any]] | None = None,
+ ) -> bool:
+ now = naive_utc_now()
+ values: dict[str, Any] = {"status": status.value, "modified_at": now}
+ if status.is_final():
+ values["finished_at"] = func.coalesce(TaskV2Model.finished_at, now)
+ if failure_category is not None:
+ values["failure_category"] = failure_category
+ async with self.Session() as session:
+ result = await session.execute(
+ update(TaskV2Model)
+ .where(
+ TaskV2Model.observer_cruise_id == task_v2_id,
+ TaskV2Model.organization_id == organization_id,
+ TaskV2Model.status.in_([s.value for s in TaskV2Status if not s.is_final()]),
+ )
+ .values(**values)
+ .returning(TaskV2Model.observer_cruise_id)
+ )
+ won = result.scalar_one_or_none() is not None
+ await session.commit()
+ return won
+
@db_operation("update_task_v2")
async def update_task_v2(
self,
@@ -363,16 +393,19 @@ class ObserverRepository(BaseRepository):
organization_id: str | None = None,
webhook_failure_reason: str | None = None,
failure_category: list[dict[str, Any]] | None = None,
+ *,
+ keep_final_status: bool = False,
) -> TaskV2:
async with self.Session() as session:
- task_v2 = (
- await session.scalars(
- select(TaskV2Model)
- .filter_by(observer_cruise_id=task_v2_id)
- .filter_by(organization_id=organization_id)
- )
- ).first()
+ query = (
+ select(TaskV2Model).filter_by(observer_cruise_id=task_v2_id).filter_by(organization_id=organization_id)
+ )
+ if keep_final_status:
+ query = query.with_for_update()
+ task_v2 = (await session.scalars(query)).first()
if task_v2:
+ if keep_final_status and TaskV2Status(task_v2.status).is_final():
+ return convert_to_task_v2(task_v2, debug_enabled=self.debug_enabled)
if status:
task_v2.status = status
if status == TaskV2Status.queued and task_v2.queued_at is None:
diff --git a/skyvern/forge/sdk/db/repositories/run_execution_leases.py b/skyvern/forge/sdk/db/repositories/run_execution_leases.py
new file mode 100644
index 000000000..4928dd4a2
--- /dev/null
+++ b/skyvern/forge/sdk/db/repositories/run_execution_leases.py
@@ -0,0 +1,142 @@
+from collections.abc import Collection, Mapping
+from dataclasses import dataclass
+from datetime import datetime
+
+from sqlalchemy import delete, or_, select, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+
+from skyvern.forge.sdk.db._error_handling import db_operation
+from skyvern.forge.sdk.db.base_repository import BaseRepository
+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
+from skyvern.forge.sdk.db.models import RunExecutionLeaseModel
+
+
+@dataclass(frozen=True)
+class RunExecutionLease:
+ run_id: str
+ run_type: str
+ owner_id: str
+ attempt_number: int | None
+ heartbeat_at: datetime
+
+
+class RunExecutionLeasesRepository(BaseRepository):
+ @db_operation("acquire_run_execution_leases")
+ async def acquire(
+ self, owner_id: str, runs: Mapping[str, tuple[str, int | None]], stale_before: datetime
+ ) -> set[str]:
+ if not runs:
+ return set()
+ now = naive_utc_now()
+ acquired: set[str] = set()
+ async with self.Session() as session:
+ insert = sqlite_insert if session.bind.dialect.name == "sqlite" else pg_insert
+ # One row per statement also stays below SQLite's bind parameter limit for large registries.
+ for run_id, (run_type, attempt_number) in runs.items():
+ stmt = insert(RunExecutionLeaseModel).values(
+ run_id=run_id,
+ run_type=run_type,
+ owner_id=owner_id,
+ attempt_number=attempt_number,
+ heartbeat_at=now,
+ )
+ result = await session.execute(
+ stmt.on_conflict_do_update(
+ index_elements=[RunExecutionLeaseModel.run_id],
+ set_={
+ key: stmt.excluded[key]
+ for key in ("run_type", "owner_id", "attempt_number", "heartbeat_at")
+ },
+ where=or_(
+ RunExecutionLeaseModel.owner_id == owner_id,
+ RunExecutionLeaseModel.heartbeat_at < stale_before,
+ ),
+ ).returning(RunExecutionLeaseModel.run_id)
+ )
+ acquired.update(result.scalars().all())
+ await session.commit()
+ return acquired
+
+ @db_operation("renew_run_execution_leases")
+ async def renew(self, owner_id: str, run_ids: Collection[str]) -> set[str]:
+ if not run_ids:
+ return set()
+ async with self.Session() as session:
+ result = await session.execute(
+ update(RunExecutionLeaseModel)
+ .where(RunExecutionLeaseModel.owner_id == owner_id, RunExecutionLeaseModel.run_id.in_(run_ids))
+ .values(heartbeat_at=naive_utc_now())
+ .returning(RunExecutionLeaseModel.run_id)
+ )
+ renewed = set(result.scalars().all())
+ await session.commit()
+ return renewed
+
+ @db_operation("release_run_execution_leases")
+ async def release(self, owner_id: str, run_ids: Collection[str]) -> None:
+ if not run_ids:
+ return
+ async with self.Session() as session:
+ await session.execute(
+ delete(RunExecutionLeaseModel).where(
+ RunExecutionLeaseModel.owner_id == owner_id,
+ RunExecutionLeaseModel.run_id.in_(run_ids),
+ )
+ )
+ await session.commit()
+
+ @db_operation("get_expired_run_execution_leases")
+ async def get_expired(self, heartbeat_before: datetime, limit: int = 500) -> list[RunExecutionLease]:
+ async with self.Session() as session:
+ rows = (
+ await session.scalars(
+ select(RunExecutionLeaseModel)
+ .where(RunExecutionLeaseModel.heartbeat_at < heartbeat_before)
+ .order_by(RunExecutionLeaseModel.heartbeat_at)
+ .limit(limit)
+ )
+ ).all()
+ return [
+ RunExecutionLease(row.run_id, row.run_type, row.owner_id, row.attempt_number, row.heartbeat_at)
+ for row in rows
+ ]
+
+ @db_operation("claim_run_execution_lease")
+ async def claim(
+ self,
+ run_id: str,
+ observed_owner_id: str,
+ observed_heartbeat_at: datetime,
+ reaper_owner_id: str,
+ ) -> bool:
+ async with self.Session() as session:
+ result = await session.execute(
+ update(RunExecutionLeaseModel)
+ .where(
+ RunExecutionLeaseModel.run_id == run_id,
+ RunExecutionLeaseModel.owner_id == observed_owner_id,
+ RunExecutionLeaseModel.heartbeat_at == observed_heartbeat_at,
+ )
+ .values(owner_id=reaper_owner_id, heartbeat_at=naive_utc_now())
+ .returning(RunExecutionLeaseModel.run_id)
+ )
+ won = result.scalar_one_or_none() is not None
+ await session.commit()
+ return won
+
+ @db_operation("unclaim_run_execution_lease")
+ async def unclaim(self, run_id: str, reaper_owner_id: str, observed_heartbeat_at: datetime) -> bool:
+ async with self.Session() as session:
+ result = await session.execute(
+ update(RunExecutionLeaseModel)
+ .where(
+ RunExecutionLeaseModel.run_id == run_id,
+ RunExecutionLeaseModel.owner_id == reaper_owner_id,
+ )
+ .values(heartbeat_at=observed_heartbeat_at)
+ .returning(RunExecutionLeaseModel.run_id)
+ )
+ unclaimed = result.scalar_one_or_none() is not None
+ await session.commit()
+ return unclaimed
diff --git a/skyvern/forge/sdk/db/repositories/tasks.py b/skyvern/forge/sdk/db/repositories/tasks.py
index 0e373007d..fde2b8223 100644
--- a/skyvern/forge/sdk/db/repositories/tasks.py
+++ b/skyvern/forge/sdk/db/repositories/tasks.py
@@ -603,6 +603,8 @@ class TasksRepository(BaseRepository):
max_steps_per_run: int | None = None,
organization_id: str | None = None,
failure_category: list[dict[str, Any]] | None = None,
+ *,
+ keep_final_status: bool = False,
) -> Task:
updated_task, _ = await self._update_task_with_finish_claim(
task_id,
@@ -614,6 +616,7 @@ class TasksRepository(BaseRepository):
max_steps_per_run=max_steps_per_run,
organization_id=organization_id,
failure_category=failure_category,
+ keep_final_status=keep_final_status,
)
return updated_task
@@ -630,6 +633,8 @@ class TasksRepository(BaseRepository):
max_steps_per_run: int | None = None,
organization_id: str | None = None,
failure_category: list[dict[str, Any]] | None = None,
+ *,
+ keep_final_status: bool = False,
) -> tuple[Task, bool]:
"""``update_task``, plus whether THIS write flipped ``finished_at`` from NULL.
@@ -648,6 +653,7 @@ class TasksRepository(BaseRepository):
max_steps_per_run=max_steps_per_run,
organization_id=organization_id,
failure_category=failure_category,
+ keep_final_status=keep_final_status,
)
async def _update_task_with_finish_claim(
@@ -661,6 +667,8 @@ class TasksRepository(BaseRepository):
max_steps_per_run: int | None = None,
organization_id: str | None = None,
failure_category: list[dict[str, Any]] | None = None,
+ *,
+ keep_final_status: bool = False,
) -> tuple[Task, bool]:
if (
status is None
@@ -687,6 +695,8 @@ class TasksRepository(BaseRepository):
.with_for_update()
)
).first():
+ if keep_final_status and TaskStatus(task.status).is_final():
+ return convert_to_task(task, debug_enabled=self.debug_enabled), False
if status is not None:
task.status = status
if status == TaskStatus.queued and task.queued_at is None:
@@ -909,6 +919,30 @@ class TasksRepository(BaseRepository):
await session.commit()
return result.rowcount or 0
+ @db_operation("bulk_update_steps_by_task_ids")
+ async def bulk_update_steps_by_task_ids(
+ self,
+ task_ids: list[str],
+ new_status: StepStatus,
+ only_if_status_in: list[StepStatus],
+ ) -> int:
+ if not task_ids or not only_if_status_in:
+ return 0
+
+ update_values: dict[str, Any] = {"status": new_status.value}
+ if new_status.is_terminal():
+ update_values["finished_at"] = func.coalesce(StepModel.finished_at, naive_utc_now())
+ async with self.Session() as session:
+ stmt = (
+ update(StepModel)
+ .where(StepModel.task_id.in_(task_ids))
+ .where(StepModel.status.in_([s.value for s in only_if_status_in]))
+ .values(**update_values)
+ )
+ result = await session.execute(stmt)
+ await session.commit()
+ return result.rowcount or 0
+
@db_operation("get_tasks")
async def get_tasks(
self,
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index bc45454c1..1fb7b8d8e 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -1303,6 +1303,7 @@ class WorkflowRunsRepository(BaseRepository):
job_id: str | None = None,
depends_on_workflow_run_id: str | None = None,
workflow_definition_sha256: str | None = None,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun | None:
"""One conditional UPDATE to ``status`` from a non-terminal state (``only_from`` narrows it), so a late
cancel cannot clobber a finalization; None when the row was terminal or missing. Timestamps follow
@@ -1361,9 +1362,16 @@ class WorkflowRunsRepository(BaseRepository):
values["queued_at"] = func.coalesce(
WorkflowRunModel.queued_at, await _first_queued_at(session, workflow_run)
)
+ statement = update(WorkflowRunModel)
+ if require_latest_attempt_number is not None:
+ statement = statement.where(
+ ~exists().where(
+ WorkflowRunAttemptModel.workflow_run_id == WorkflowRunModel.workflow_run_id,
+ WorkflowRunAttemptModel.attempt_number > require_latest_attempt_number,
+ )
+ )
result = await session.execute(
- update(WorkflowRunModel)
- .where(
+ statement.where(
WorkflowRunModel.workflow_run_id == workflow_run_id,
WorkflowRunModel.status.in_(non_terminal),
)
diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.py
index 044cce7e9..da04528f2 100644
--- a/skyvern/forge/sdk/executor/background_task_executor.py
+++ b/skyvern/forge/sdk/executor/background_task_executor.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.db.repositories.workflow_run_attempts import (
ATTEMPT_RECOVERY_MAX_PAGES,
)
from skyvern.forge.sdk.executor.async_executor import AsyncExecutor
+from skyvern.forge.sdk.executor.run_liveness import acquire_run_lease, hold_run, track_run
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
@@ -442,6 +443,14 @@ class BackgroundTaskExecutor(AsyncExecutor):
await asyncio.sleep(min(backoff_seconds, remaining_seconds))
backoff_seconds = min(backoff_seconds * 2, 30.0)
+ async def schedule_recorded_retry(self, workflow_run_id: str, attempt_number: int) -> bool:
+ attempts = await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id)
+ attempt = next((row for row in attempts if row.attempt_number == attempt_number), None)
+ if attempt is None or attempt.retry_decision != "retry":
+ return False
+ self._schedule_retry_resume(attempt)
+ return True
+
def _schedule_retry_resume(self, attempt: Any) -> None:
key = (attempt.workflow_run_id, attempt.attempt_number)
if key in self._scheduled_retry_resumes:
@@ -717,47 +726,59 @@ class BackgroundTaskExecutor(AsyncExecutor):
organization_id=organization_id,
)
- task = await app.DATABASE.tasks.update_task(
- task_id,
- status=TaskStatus.running,
- organization_id=organization_id,
- )
-
- close_browser_on_completion = browser_session_id is None and not task.browser_address
-
- run_obj = await app.DATABASE.tasks.get_run(run_id=task_id, organization_id=organization_id)
- engine = RunEngine.skyvern_v1
- if run_obj and run_obj.task_run_type == RunType.openai_cua:
- engine = RunEngine.openai_cua
- elif run_obj and run_obj.task_run_type == RunType.anthropic_cua:
- engine = RunEngine.anthropic_cua
- elif run_obj and run_obj.task_run_type == RunType.ui_tars:
- engine = RunEngine.ui_tars
- elif run_obj and run_obj.task_run_type == RunType.yutori_navigator:
- engine = RunEngine.yutori_navigator
- elif run_obj and run_obj.task_run_type == RunType.task_v3:
- engine = RunEngine.skyvern_v3
-
- context: SkyvernContext = skyvern_context.ensure_context()
- context.task_id = task.task_id
- context.run_id = context.run_id or task.task_id
- context.organization_id = organization_id
- context.max_steps_override = max_steps_override
- context.max_screenshot_scrolls = task.max_screenshot_scrolls
+ release_hold = hold_run(task_id, "task")
+ try:
+ await acquire_run_lease(task_id)
+ task = await app.DATABASE.tasks.update_task(
+ task_id,
+ status=TaskStatus.running,
+ organization_id=organization_id,
+ )
- await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)
- await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)
- self._schedule(
- background_tasks,
- app.agent.execute_step,
- organization,
- task,
- step,
- api_key,
- close_browser_on_completion=close_browser_on_completion,
- browser_session_id=browser_session_id,
- engine=engine,
- )
+ close_browser_on_completion = browser_session_id is None and not task.browser_address
+
+ run_obj = await app.DATABASE.tasks.get_run(run_id=task_id, organization_id=organization_id)
+ engine = RunEngine.skyvern_v1
+ if run_obj and run_obj.task_run_type == RunType.openai_cua:
+ engine = RunEngine.openai_cua
+ elif run_obj and run_obj.task_run_type == RunType.anthropic_cua:
+ engine = RunEngine.anthropic_cua
+ elif run_obj and run_obj.task_run_type == RunType.ui_tars:
+ engine = RunEngine.ui_tars
+ elif run_obj and run_obj.task_run_type == RunType.yutori_navigator:
+ engine = RunEngine.yutori_navigator
+ elif run_obj and run_obj.task_run_type == RunType.task_v3:
+ engine = RunEngine.skyvern_v3
+
+ context: SkyvernContext = skyvern_context.ensure_context()
+ context.task_id = task.task_id
+ context.run_id = context.run_id or task.task_id
+ context.organization_id = organization_id
+ context.max_steps_override = max_steps_override
+ context.max_screenshot_scrolls = task.max_screenshot_scrolls
+
+ await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)
+ await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)
+
+ async def execute_registered_step(*args: Any, **kwargs: Any) -> None:
+ with track_run(task_id, "task"):
+ release_hold()
+ await app.agent.execute_step(*args, **kwargs)
+
+ self._schedule(
+ background_tasks,
+ execute_registered_step,
+ organization,
+ task,
+ step,
+ api_key,
+ close_browser_on_completion=close_browser_on_completion,
+ browser_session_id=browser_session_id,
+ engine=engine,
+ )
+ except BaseException:
+ release_hold()
+ raise
async def execute_workflow(
self,
diff --git a/skyvern/forge/sdk/executor/run_liveness.py b/skyvern/forge/sdk/executor/run_liveness.py
new file mode 100644
index 000000000..45ee46a1a
--- /dev/null
+++ b/skyvern/forge/sdk/executor/run_liveness.py
@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import socket
+import time
+import uuid
+from collections.abc import Awaitable, Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
+
+if TYPE_CHECKING:
+ from skyvern.forge.sdk.db.repositories.run_execution_leases import RunExecutionLeasesRepository
+
+LOG = structlog.get_logger()
+INTERVAL_SECONDS = 60
+SHUTDOWN_DRAIN_SECONDS = 15
+_pid: int | None = None
+_owner_id: str | None = None
+_enabled = False
+_wake = asyncio.Event()
+_lease_write_lock = asyncio.Lock()
+_registry: dict[str, _Registration] = {}
+_kept_leases: set[str] = set()
+_held: set[str] = set()
+_tasks: set[asyncio.Task[None]] = set()
+_side_effect_tasks: set[asyncio.Task[None]] = set()
+_side_effect_limit = asyncio.Semaphore(10)
+
+
+@dataclass
+class _Registration:
+ run_type: str
+ attempt_number: int | None
+ count: int = 0
+ task: asyncio.Task[Any] | None = None
+ acquired: bool = False
+ lost: bool = False
+ finalized: bool = False
+
+
+def owner_id() -> str:
+ global _pid, _owner_id
+ pid = os.getpid()
+ if _pid != pid or _owner_id is None:
+ _pid = pid
+ _owner_id = f"{socket.gethostname()}:{pid}:{uuid.uuid4().hex[:8]}"
+ return _owner_id
+
+
+def run_lease_tasks_enabled() -> bool:
+ return _enabled
+
+
+def effective_timeout_minutes() -> int:
+ return settings.RUN_HEARTBEAT_TIMEOUT_MINUTES
+
+
+def _register(
+ run_id: str,
+ run_type: str,
+ attempt_number: int | None,
+ task: asyncio.Task[Any] | None,
+) -> Callable[[bool], None]:
+ if not _enabled:
+ return lambda cancelled: None
+ _kept_leases.discard(run_id)
+ entry = _registry.get(run_id)
+ if entry is None:
+ entry = _Registration(run_type, attempt_number)
+ _registry[run_id] = entry
+ elif entry.attempt_number != attempt_number:
+ entry.attempt_number = attempt_number
+ entry.acquired = False
+ entry.lost = False
+ entry.finalized = False
+ entry.count += 1
+ entry.task = task
+ _wake.set()
+ released = False
+
+ def release(cancelled: bool) -> None:
+ nonlocal released
+ if released:
+ return
+ released = True
+ entry.count -= 1
+ if entry.count == 0 and _registry.get(run_id) is entry:
+ if cancelled:
+ _kept_leases.add(run_id)
+ del _registry[run_id]
+ _wake.set()
+
+ return release
+
+
+@contextmanager
+def track_run(run_id: str, run_type: str, attempt_number: int | None = None) -> Iterator[None]:
+ try:
+ task = asyncio.current_task()
+ except RuntimeError:
+ task = None
+ release = _register(run_id, run_type, attempt_number, task)
+ cancelled = False
+ try:
+ yield
+ except asyncio.CancelledError:
+ cancelled = True
+ raise
+ finally:
+ release(cancelled)
+
+
+def hold_run(run_id: str, run_type: str) -> Callable[[], None]:
+ release = _register(run_id, run_type, None, None)
+ return lambda: release(False)
+
+
+def is_run_registered(run_id: str) -> bool:
+ return run_id in _registry
+
+
+def note_run_finalized(run_id: str) -> None:
+ entry = _registry.get(run_id)
+ if entry is not None:
+ entry.finalized = True
+
+
+async def acquire_run_lease(run_id: str) -> None:
+ if not _enabled:
+ return
+ entry = _registry.get(run_id)
+ if entry is None or entry.acquired or entry.lost:
+ return
+ attempt = entry.attempt_number
+ try:
+ async with _lease_write_lock:
+ if _registry.get(run_id) is not entry or entry.attempt_number != attempt or entry.acquired or entry.lost:
+ return
+ acquired = await app.DATABASE.run_execution_leases.acquire(
+ owner_id(),
+ {run_id: (entry.run_type, attempt)},
+ stale_before=naive_utc_now() - timedelta(minutes=effective_timeout_minutes()),
+ )
+ if run_id not in acquired:
+ LOG.info(
+ "Run lease held by another process",
+ run_id=run_id,
+ run_type=entry.run_type,
+ attempt_number=attempt,
+ owner_id=owner_id(),
+ )
+ return
+ _held.add(run_id)
+ if _registry.get(run_id) is entry and entry.attempt_number == attempt:
+ entry.acquired = True
+ except Exception:
+ LOG.warning("Run lease acquire failed before run start", run_id=run_id, exc_info=True)
+
+
+async def run_lease_lost(run_id: str) -> bool:
+ if not _enabled:
+ return False
+ entry = _registry.get(run_id)
+ if entry is None or (not entry.acquired and not entry.lost):
+ return False
+ if entry.lost:
+ return True
+ try:
+ renewed = await app.DATABASE.run_execution_leases.renew(owner_id(), [run_id])
+ except Exception:
+ LOG.warning("Run lease check failed before status write", run_id=run_id, exc_info=True)
+ return False
+ if run_id in renewed:
+ return False
+ LOG.warning(
+ "Run lease lost before status write",
+ run_id=run_id,
+ run_type=entry.run_type,
+ attempt_number=entry.attempt_number,
+ owner_id=owner_id(),
+ )
+ return True
+
+
+async def _heartbeat(repository: RunExecutionLeasesRepository) -> None:
+ last_renew = time.monotonic()
+ while True:
+ _wake.clear()
+ try:
+ # Snapshot the generation as well as the object: a retry can enter while an UPDATE awaits I/O.
+ renewing = {
+ key: (entry, entry.attempt_number)
+ for key, entry in _registry.items()
+ if entry.acquired and not entry.lost
+ }
+ # Serialize release and acquire so a release cannot delete a new registration's lease.
+ async with _lease_write_lock:
+ released = _held - _registry.keys()
+ await repository.release(owner_id(), released - _kept_leases)
+ _held.difference_update(released)
+ if time.monotonic() - last_renew >= INTERVAL_SECONDS:
+ last_renew = time.monotonic()
+ renewed = await repository.renew(owner_id(), renewing.keys())
+ newly_lost: list[_Registration] = []
+ for run_id, (entry, attempt) in renewing.items():
+ if run_id in renewed or _registry.get(run_id) is not entry or entry.attempt_number != attempt:
+ continue
+ entry.lost = True
+ entry.acquired = False
+ newly_lost.append(entry)
+ _held.discard(run_id)
+ LOG.warning(
+ "Run lease lost",
+ finalized=entry.finalized,
+ run_id=run_id,
+ run_type=entry.run_type,
+ attempt_number=attempt,
+ owner_id=owner_id(),
+ )
+ # Repeated cancellation would interrupt the cleanup started by the first cancellation.
+ lost_targets = {entry.task for entry in newly_lost if entry.task is not None and not entry.finalized}
+ for target in lost_targets:
+ if not target.done() and all(
+ entry.lost and not entry.finalized for entry in _registry.values() if entry.task is target
+ ):
+ target.cancel()
+ async with _lease_write_lock:
+ acquiring = {
+ key: (entry, entry.attempt_number)
+ for key, entry in _registry.items()
+ if not entry.acquired and not entry.lost
+ }
+ acquired = await repository.acquire(
+ owner_id(),
+ {key: (entry.run_type, attempt) for key, (entry, attempt) in acquiring.items()},
+ stale_before=naive_utc_now() - timedelta(minutes=effective_timeout_minutes()),
+ )
+ for run_id, (entry, attempt) in acquiring.items():
+ if run_id not in acquired:
+ LOG.info(
+ "Run lease held by another process",
+ run_id=run_id,
+ run_type=entry.run_type,
+ attempt_number=attempt,
+ owner_id=owner_id(),
+ )
+ continue
+ _held.add(run_id)
+ if _registry.get(run_id) is entry and entry.attempt_number == attempt:
+ entry.acquired = True
+ _kept_leases.intersection_update(_held)
+ except Exception:
+ LOG.exception("Run lease heartbeat failed", owner_id=owner_id())
+ try:
+ remaining = max(0.0, min(INTERVAL_SECONDS, INTERVAL_SECONDS - (time.monotonic() - last_renew)))
+ async with asyncio.timeout(remaining):
+ await _wake.wait()
+ except TimeoutError:
+ pass
+
+
+async def _reaper_loop(reap_once: Callable[[], Awaitable[None]]) -> None:
+ while True:
+ await asyncio.sleep(INTERVAL_SECONDS)
+ try:
+ await reap_once()
+ LOG.debug("Orphaned run reaper pass completed")
+ except Exception:
+ LOG.exception("Orphaned run reaper pass failed")
+
+
+def schedule_side_effects(effect: Callable[[], Awaitable[None]]) -> None:
+ async def run() -> None:
+ try:
+ async with _side_effect_limit, asyncio.timeout(300):
+ await effect()
+ except Exception:
+ LOG.exception("Orphaned run side effects failed")
+
+ task = asyncio.create_task(run(), name="orphaned-run-side-effects")
+ _side_effect_tasks.add(task)
+ task.add_done_callback(_side_effect_tasks.discard)
+
+
+def start_run_lease_tasks(reap_once: Callable[[], Awaitable[None]]) -> None:
+ global _enabled, _wake, _side_effect_limit, _lease_write_lock
+ if _enabled or settings.RUN_HEARTBEAT_TIMEOUT_MINUTES <= 0:
+ return
+ _enabled = True
+ _wake = asyncio.Event()
+ _lease_write_lock = asyncio.Lock()
+ _held.clear()
+ _side_effect_limit = asyncio.Semaphore(10)
+ _tasks.add(asyncio.create_task(_heartbeat(app.DATABASE.run_execution_leases), name="run-lease-heartbeat"))
+ _tasks.add(asyncio.create_task(_reaper_loop(reap_once), name="orphaned-run-reaper"))
+ LOG.info("Run lease tasks started", owner_id=owner_id(), timeout_minutes=effective_timeout_minutes())
+
+
+async def stop_run_lease_tasks() -> None:
+ global _enabled
+ _enabled = False
+ # Stop the producers before collecting effects so a pass cannot enqueue work during shutdown.
+ for task in _tasks:
+ task.cancel()
+ await asyncio.gather(*_tasks, return_exceptions=True)
+ _tasks.clear()
+ effects = list(_side_effect_tasks)
+ if not effects:
+ return
+ _, pending = await asyncio.wait(effects, timeout=SHUTDOWN_DRAIN_SECONDS)
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
+ if pending:
+ LOG.warning("Orphaned run side effects cancelled at shutdown", count=len(pending))
+ _side_effect_tasks.clear()
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 47fbb2674..1490d73f0 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -184,6 +184,7 @@ from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.exceptions import NotFoundError
from skyvern.forge.sdk.db.id import generate_action_id, generate_workflow_run_id
from skyvern.forge.sdk.db.repositories.google_oauth import DISPATCH_ACCEPTED, DISPATCH_FAILED, GmailSendDispatch
+from skyvern.forge.sdk.executor.run_liveness import run_lease_lost, run_lease_tasks_enabled
from skyvern.forge.sdk.experimentation.code_block_ai_fallback import code_block_ai_fallback_flag_enabled
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
@@ -1150,6 +1151,8 @@ class Block(BaseModel, abc.ABC):
value=value,
)
self._output_recorded_this_execution = True
+ if run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id):
+ return
await app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter(
workflow_run_id=workflow_run_id,
output_parameter_id=self.output_parameter.output_parameter_id,
@@ -8214,11 +8217,16 @@ async def wrapper({default_args}):
await record_task_nav_error_code(escalation_task.task_id, escalation_nav_error, escalation_url)
elif failed_nav_error_code and not escalation_navigated:
skyvern_context.ensure_context().task_nav_error_codes[escalation_task.task_id] = failed_nav_error_code
- escalation_task = await app.DATABASE.tasks.update_task(
- task_id=escalation_task.task_id,
- organization_id=organization_id,
- status=TaskStatus.running,
- )
+ if await run_lease_lost(workflow_run_id):
+ escalation_task = (
+ await app.DATABASE.tasks.get_task(escalation_task.task_id, organization_id) or escalation_task
+ )
+ else:
+ escalation_task = await app.DATABASE.tasks.update_task(
+ task_id=escalation_task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.running,
+ )
escalation_step = await app.DATABASE.tasks.create_step(
escalation_task.task_id,
order=0,
@@ -15587,6 +15595,15 @@ class HumanInteractionBlock(BaseTaskBlock):
browser_session_id=browser_session_id,
)
+ if await run_lease_lost(workflow_run_id):
+ return await self.build_block_result(
+ success=False,
+ failure_reason="Another server already finalized this run",
+ output_parameter_value=None,
+ status=BlockStatus.failed,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
await app.DATABASE.workflow_runs.update_workflow_run(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.paused,
diff --git a/skyvern/forge/sdk/workflow/models/code_block_recording.py b/skyvern/forge/sdk/workflow/models/code_block_recording.py
index 380ea8b67..7e0cfed2c 100644
--- a/skyvern/forge/sdk/workflow/models/code_block_recording.py
+++ b/skyvern/forge/sdk/workflow/models/code_block_recording.py
@@ -11,6 +11,7 @@ import structlog
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.executor.run_liveness import run_lease_lost
from skyvern.forge.sdk.models import StepStatus
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.code_block_recorder import (
@@ -347,12 +348,15 @@ class CodeBlockActionRecording:
return
self._finalized = True
try:
- await app.DATABASE.tasks.update_task(
+ status = TaskStatus.completed if success else TaskStatus.failed
+ lease_lost = await run_lease_lost(self._workflow_run_id)
+ task = await app.DATABASE.tasks.update_task(
task_id=self._task.task_id,
organization_id=self._organization_id,
- status=TaskStatus.completed if success else TaskStatus.failed,
+ status=status,
+ keep_final_status=lease_lost,
)
- if self._step is not None:
+ if self._step is not None and (not lease_lost or task.status == status):
await app.DATABASE.tasks.update_step(
task_id=self._task.task_id,
step_id=self._step.step_id,
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 5895cc52a..dc2daad4f 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -128,6 +128,13 @@ from skyvern.forge.sdk.db.repositories.workflow_runs import (
lock_workflow_run_for_dispatch,
)
from skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_features
+from skyvern.forge.sdk.executor.run_liveness import (
+ acquire_run_lease,
+ note_run_finalized,
+ run_lease_lost,
+ run_lease_tasks_enabled,
+ track_run,
+)
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_transient_ui_capture_arm
from skyvern.forge.sdk.experimentation.workflow_block_engine import (
@@ -997,6 +1004,10 @@ def _workflow_block_engine_attribution_for_duration_log(workflow_run_id: str) ->
return ARM_ATTRIBUTION_LOST
+def _run_lease_attempt_kwargs(key: str, attempt_number: int | None) -> dict[str, Any]:
+ return {key: attempt_number} if run_lease_tasks_enabled() else {}
+
+
def _get_workflow_run_max_elapsed_timeout_seconds(workflow_run: WorkflowRun) -> float:
effective_minutes = get_effective_workflow_run_max_elapsed_time_minutes(workflow_run.max_elapsed_time_minutes)
started_at = _as_utc(workflow_run.started_at or workflow_run.created_at)
@@ -6137,6 +6148,7 @@ class WorkflowService:
pre_finally_status: WorkflowRunStatus | None,
pre_finally_failure_reason: str | None,
timeout_failure_reason: str,
+ require_latest_attempt_number: int | None = None,
) -> tuple[WorkflowRun, WorkflowRunStatus | None, str | None]:
if pre_finally_status is None or not pre_finally_status.is_final():
if refreshed_workflow_run := await app.DATABASE.workflow_runs.get_workflow_run(
@@ -6158,6 +6170,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=timeout_failure_reason,
fallback_workflow_run=workflow_run,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
pre_finally_status = WorkflowRunStatus.timed_out
pre_finally_failure_reason = timeout_failure_reason
@@ -6172,6 +6185,7 @@ class WorkflowService:
pre_finally_status: WorkflowRunStatus | None,
pre_finally_failure_reason: str | None,
timeout_failure_reason: str,
+ require_latest_attempt_number: int | None = None,
) -> tuple[WorkflowRun, WorkflowRunStatus | None, str | None]:
handle_timeout_task = asyncio.create_task(
self._handle_post_run_elapsed_timeout(
@@ -6181,6 +6195,7 @@ class WorkflowService:
pre_finally_status=pre_finally_status,
pre_finally_failure_reason=pre_finally_failure_reason,
timeout_failure_reason=timeout_failure_reason,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
),
name=f"post_run_elapsed_timeout_{workflow_run_id}",
)
@@ -6203,6 +6218,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=timeout_failure_reason,
fallback_workflow_run=workflow_run,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
)
return fallback_workflow_run, WorkflowRunStatus.timed_out, timeout_failure_reason
@@ -6246,6 +6262,7 @@ class WorkflowService:
return await self.mark_workflow_run_as_timed_out(
workflow_run.workflow_run_id,
failure_reason="Workflow run exceeded its remaining runtime while waiting for sequential clearance.",
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", attempt_number),
)
async def _prepare_recorded_retry(
@@ -6707,28 +6724,62 @@ class WorkflowService:
) -> WorkflowRun:
if on_execution_authority is not None:
on_execution_authority(attempt_number, dispatch_claim_started_at)
+ with track_run(workflow_run_id, "workflow_run", attempt_number):
+ await acquire_run_lease(workflow_run_id)
+ try:
+ return await self._execute_workflow(
+ workflow_run_id=workflow_run_id,
+ api_key=api_key,
+ organization=organization,
+ block_labels=block_labels,
+ block_outputs=block_outputs,
+ browser_session_id=browser_session_id,
+ need_call_webhook=need_call_webhook,
+ workflow_override=workflow_override,
+ requested_completion_contract=requested_completion_contract,
+ attempt_number=attempt_number,
+ dispatch_claim_started_at=dispatch_claim_started_at,
+ on_finalization_denied=on_finalization_denied,
+ )
+ except BaseException as error:
+ if isinstance(error, WorkflowAttemptDispatchSuperseded) or BROWSER_RETIREMENT_DENIED_NOTE in getattr(
+ error, "__notes__", ()
+ ):
+ if on_finalization_denied is not None:
+ on_finalization_denied()
+ raise
+
+ async def _leave_if_attempt_superseded(
+ self,
+ workflow_run_id: str,
+ attempt_number: int,
+ on_finalization_denied: Callable[[], None] | None,
+ renewal_task: asyncio.Task[None] | None,
+ ) -> None:
+ if not run_lease_tasks_enabled():
+ return
try:
- return await self._execute_workflow(
+ attempts = await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id)
+ except Exception:
+ LOG.warning(
+ "Could not check for a later workflow attempt; finalizing this attempt",
workflow_run_id=workflow_run_id,
- api_key=api_key,
- organization=organization,
- block_labels=block_labels,
- block_outputs=block_outputs,
- browser_session_id=browser_session_id,
- need_call_webhook=need_call_webhook,
- workflow_override=workflow_override,
- requested_completion_contract=requested_completion_contract,
attempt_number=attempt_number,
- dispatch_claim_started_at=dispatch_claim_started_at,
- on_finalization_denied=on_finalization_denied,
+ exc_info=True,
)
- except BaseException as error:
- if isinstance(error, WorkflowAttemptDispatchSuperseded) or BROWSER_RETIREMENT_DENIED_NOTE in getattr(
- error, "__notes__", ()
- ):
- if on_finalization_denied is not None:
- on_finalization_denied()
- raise
+ return
+ if any(attempt.attempt_number > attempt_number for attempt in attempts):
+ if on_finalization_denied is not None:
+ on_finalization_denied()
+ if renewal_task is not None:
+ renewal_task.cancel()
+ with contained_effect("record superseded workflow attempt"):
+ LOG.info(
+ "Workflow attempt superseded; skipping finalization and cleanup",
+ workflow_run_id=workflow_run_id,
+ attempt_number=attempt_number,
+ )
+ raise WorkflowAttemptDispatchSuperseded(workflow_run_id, attempt_number)
async def _execute_workflow(
self,
@@ -6756,6 +6807,9 @@ class WorkflowService:
both sync and async (Temporal-dispatched) trigger modes.
"""
organization_id = organization.organization_id
+ attempt_fence_kwargs: dict[str, Any] = (
+ {"require_latest_attempt_number": attempt_number} if run_lease_tasks_enabled() else {}
+ )
LOG.info(
"Executing workflow",
@@ -6803,7 +6857,7 @@ class WorkflowService:
failure_reason=failure_reason,
)
return await self.mark_workflow_run_as_failed(
- workflow_run_id=workflow_run_id, failure_reason=failure_reason
+ workflow_run_id=workflow_run_id, failure_reason=failure_reason, **attempt_fence_kwargs
)
if not workflow.workflow_definition.blocks:
@@ -7196,7 +7250,7 @@ class WorkflowService:
workflow_run = failed
else:
workflow_run = await self.mark_workflow_run_as_failed(
- workflow_run_id=workflow_run_id, failure_reason=failure_reason
+ workflow_run_id=workflow_run_id, failure_reason=failure_reason, **attempt_fence_kwargs
)
await self.clean_up_workflow(
workflow=workflow,
@@ -7302,6 +7356,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=failure_reason,
failure_category=lease_failure_category,
+ **attempt_fence_kwargs,
)
await self.clean_up_workflow(
workflow=workflow,
@@ -7397,6 +7452,7 @@ class WorkflowService:
block_outputs=block_outputs,
script=workflow_script,
script_is_pinned=script_is_pinned,
+ **attempt_fence_kwargs,
)
if max_elapsed_timeout_seconds <= 0:
@@ -7407,6 +7463,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=pre_finally_failure_reason,
fallback_workflow_run=workflow_run,
+ **attempt_fence_kwargs,
)
pre_finally_status = WorkflowRunStatus.timed_out
pre_finally_browser_persistence_status = pre_finally_status
@@ -7429,6 +7486,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=pre_finally_failure_reason,
fallback_workflow_run=workflow_run,
+ **attempt_fence_kwargs,
)
pre_finally_status = WorkflowRunStatus.timed_out
pre_finally_browser_persistence_status = pre_finally_status
@@ -7447,6 +7505,7 @@ class WorkflowService:
pre_finally_status=pre_finally_status,
pre_finally_failure_reason=pre_finally_failure_reason,
timeout_failure_reason=_require_elapsed_timeout_failure_reason(timeout_failure_reason),
+ **attempt_fence_kwargs,
)
pre_finally_browser_persistence_status = pre_finally_status
return workflow_run
@@ -7633,6 +7692,7 @@ class WorkflowService:
pre_finally_status=pre_finally_status,
pre_finally_failure_reason=pre_finally_failure_reason,
timeout_failure_reason=_require_elapsed_timeout_failure_reason(timeout_failure_reason),
+ **attempt_fence_kwargs,
)
if pre_finally_browser_persistence_status is None:
# A timeout before the body outcome was captured must retain
@@ -7658,6 +7718,9 @@ class WorkflowService:
renewal_task.cancel()
assert escaped_error is not None
raise escaped_error
+ await self._leave_if_attempt_superseded(
+ workflow_run_id, attempt_number, on_finalization_denied, renewal_task
+ )
browser_cleanup_result: WorkflowBrowserCleanupResult | None = None
browser_persistence_status: WorkflowRunStatus | None = None
browser_write_back_exhausted = False
@@ -7755,6 +7818,7 @@ class WorkflowService:
pre_finally_failure_category=pre_finally_failure_category,
run_minutes_recorded_through=run_minutes_recorded_through,
lost_external_browser=run_lost_external_browser,
+ attempt_number=attempt_number if run_lease_tasks_enabled() else None,
)
)
except BaseException:
@@ -7814,6 +7878,7 @@ class WorkflowService:
failure_reason=failure_reason,
failure_category=failure_category,
cascade_children=True,
+ **attempt_fence_kwargs,
)
)
while not finalize_task.done():
@@ -7839,6 +7904,10 @@ class WorkflowService:
exc_info=True,
)
+ await self._leave_if_attempt_superseded(
+ workflow_run_id, attempt_number, on_finalization_denied, renewal_task
+ )
+
if getattr(workflow_run, "copilot_session_id", None):
with contained_effect(
"publish Copilot origin-run runtime secrets",
@@ -7937,9 +8006,13 @@ class WorkflowService:
block_outputs: dict[str, Any] | None = None,
script: Script | None = None,
script_is_pinned: bool = False,
+ require_latest_attempt_number: int | None = None,
) -> tuple[WorkflowRun, set[str]]:
organization_id = organization.organization_id
workflow_run_id = workflow_run.workflow_run_id
+ attempt_fence_kwargs: dict[str, Any] = (
+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}
+ )
top_level_blocks = workflow.workflow_definition.blocks
all_blocks = get_all_blocks(top_level_blocks)
# Load script blocks if script is provided
@@ -8325,6 +8398,7 @@ class WorkflowService:
is_script_run=is_script_run,
blocks_to_update=blocks_to_update,
denied_cached_labels=denied_cached_labels,
+ **attempt_fence_kwargs,
)
#
@@ -8352,6 +8426,7 @@ class WorkflowService:
is_script_run=is_script_run,
blocks_to_update=blocks_to_update,
denied_cached_labels=denied_cached_labels,
+ **attempt_fence_kwargs,
)
if should_stop:
@@ -8538,7 +8613,11 @@ class WorkflowService:
is_script_run: bool,
blocks_to_update: set[str],
denied_cached_labels: frozenset[str] = frozenset(),
+ require_latest_attempt_number: int | None = None,
) -> tuple[WorkflowRun, set[str]]:
+ attempt_fence_kwargs: dict[str, Any] = (
+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}
+ )
finally_block_label = workflow.workflow_definition.finally_block_label
dag_blocks = workflow.workflow_definition.blocks
if finally_block_label:
@@ -8559,6 +8638,7 @@ class WorkflowService:
workflow_run = await self.mark_workflow_run_as_failed(
workflow_run_id=workflow_run.workflow_run_id,
failure_reason=str(exc),
+ **attempt_fence_kwargs,
)
return workflow_run, blocks_to_update
@@ -8583,6 +8663,7 @@ class WorkflowService:
workflow_run = await self.mark_workflow_run_as_failed(
workflow_run_id=workflow_run.workflow_run_id,
failure_reason=f"Unable to find block with label {current_label}",
+ **attempt_fence_kwargs,
)
break
@@ -8616,6 +8697,7 @@ class WorkflowService:
blocks_to_update=blocks_to_update,
parent_workflow_run_block_id=parent_wrb_id,
denied_cached_labels=denied_cached_labels,
+ **attempt_fence_kwargs,
)
# Track conditional workflow_run_block_ids so branch targets
@@ -8666,6 +8748,7 @@ class WorkflowService:
workflow_run = await self.mark_workflow_run_as_failed(
workflow_run_id=workflow_run.workflow_run_id,
failure_reason=f"Next block label {next_label} not found in workflow definition",
+ **attempt_fence_kwargs,
)
break
@@ -8681,6 +8764,7 @@ class WorkflowService:
workflow_run = await self.mark_workflow_run_as_failed(
workflow_run_id=workflow_run.workflow_run_id,
failure_reason=f"Cycle detected while traversing workflow definition at block {next_label}",
+ **attempt_fence_kwargs,
)
break
@@ -8706,7 +8790,11 @@ class WorkflowService:
blocks_to_update: set[str],
parent_workflow_run_block_id: str | None = None,
denied_cached_labels: frozenset[str] = frozenset(),
+ require_latest_attempt_number: int | None = None,
) -> tuple[WorkflowRun, set[str], BlockResult | None, bool, dict[str, Any] | None]:
+ attempt_fence_kwargs: dict[str, Any] = (
+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}
+ )
organization_id = organization.organization_id
workflow_run_block_result: BlockResult | None = None
branch_metadata: dict[str, Any] | None = None
@@ -8844,7 +8932,7 @@ class WorkflowService:
else:
no_block_result_reason = "Block result is None"
updated_workflow_run = await self.mark_workflow_run_as_failed_if_not_final(
- workflow_run_id=workflow_run_id, failure_reason=no_block_result_reason
+ workflow_run_id=workflow_run_id, failure_reason=no_block_result_reason, **attempt_fence_kwargs
)
workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(
workflow_run_id, workflow_run
@@ -8870,6 +8958,7 @@ class WorkflowService:
block_result=workflow_run_block_result,
workflow_run=workflow_run,
workflow_run_id=workflow_run_id,
+ **attempt_fence_kwargs,
)
# Generate pending script after block completes successfully
@@ -8892,7 +8981,7 @@ class WorkflowService:
failure_reason = f"{block.block_type} block failed. failure reason: {exception_message}"
updated_workflow_run = await self.mark_workflow_run_as_failed_if_not_final(
- workflow_run_id=workflow_run_id, failure_reason=failure_reason
+ workflow_run_id=workflow_run_id, failure_reason=failure_reason, **attempt_fence_kwargs
)
workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(
workflow_run_id, workflow_run
@@ -10217,7 +10306,11 @@ class WorkflowService:
block_result: BlockResult,
workflow_run: WorkflowRun,
workflow_run_id: str,
+ require_latest_attempt_number: int | None = None,
) -> tuple[WorkflowRun, bool]:
+ attempt_fence_kwargs: dict[str, Any] = (
+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}
+ )
if block_result.status not in (
BlockStatus.canceled,
BlockStatus.failed,
@@ -10235,7 +10328,9 @@ class WorkflowService:
block_type_var=block.block_type,
block_label=block.label,
)
- workflow_run = await self.mark_workflow_run_as_canceled(workflow_run_id=workflow_run_id)
+ workflow_run = await self.mark_workflow_run_as_canceled(
+ workflow_run_id=workflow_run_id, **attempt_fence_kwargs
+ )
return workflow_run, True
target_status, failure_reason, task_failure_category = self._resolve_block_terminal_outcome(
block=block,
@@ -10320,6 +10415,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=failure_reason,
failure_category=task_failure_category,
+ **attempt_fence_kwargs,
)
workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(
workflow_run_id, workflow_run
@@ -10329,6 +10425,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
failure_reason=failure_reason,
failure_category=task_failure_category,
+ **attempt_fence_kwargs,
)
workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(
workflow_run_id, workflow_run
@@ -13122,11 +13219,15 @@ class WorkflowService:
run_with: str | None = None,
ai_fallback: bool | None = None,
failure_category: list[dict] | None = None,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun:
failure_category = self._with_saved_profile_signed_out(workflow_run_id, status, failure_category)
flipped_to_final = False
workflow_run: WorkflowRun | None = None
if status.is_final():
+ if run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id):
+ note_run_finalized(workflow_run_id)
+ return await self.get_workflow_run(workflow_run_id)
workflow_run = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id=workflow_run_id,
status=status,
@@ -13134,6 +13235,7 @@ class WorkflowService:
run_with=run_with,
ai_fallback=ai_fallback,
failure_category=failure_category,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
flipped_to_final = workflow_run is not None
if workflow_run is not None:
@@ -13144,14 +13246,21 @@ class WorkflowService:
status,
failure_reason,
failure_category,
+ **_run_lease_attempt_kwargs("attempt_number", require_latest_attempt_number),
)
finally:
await self._after_workflow_run_status_write(workflow_run, status, emit_run_minutes=True)
return workflow_run
+ if run_lease_tasks_enabled() and require_latest_attempt_number is not None:
+ return await self.get_workflow_run(workflow_run_id)
if await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id):
# Policy runs keep the winning terminal status and its frozen retry decision.
return await self.get_workflow_run(workflow_run_id)
if workflow_run is None:
+ if await run_lease_lost(workflow_run_id):
+ if status.is_final():
+ note_run_finalized(workflow_run_id)
+ return await self.get_workflow_run(workflow_run_id)
workflow_run = await app.DATABASE.workflow_runs.update_workflow_run(
workflow_run_id=workflow_run_id,
status=status,
@@ -13174,10 +13283,14 @@ class WorkflowService:
run_minutes_recorded_through: datetime | None = None,
only_from: Sequence[WorkflowRunStatus] | None = None,
worker_container_restarted: bool = False,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun | None:
""":meth:`_update_workflow_run_status` for writers that must lose a race against the
run's own finalizer. Returns ``None`` when the row was already terminal."""
failure_category = self._with_saved_profile_signed_out(workflow_run_id, status, failure_category)
+ if run_lease_tasks_enabled() and status.is_final() and await run_lease_lost(workflow_run_id):
+ note_run_finalized(workflow_run_id)
+ return None
workflow_run = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id=workflow_run_id,
status=status,
@@ -13185,6 +13298,7 @@ class WorkflowService:
run_with=run_with,
failure_category=failure_category,
only_from=only_from,
+ require_latest_attempt_number=require_latest_attempt_number,
)
if workflow_run is None:
return None
@@ -13195,6 +13309,7 @@ class WorkflowService:
failure_reason,
failure_category,
worker_container_restarted=worker_container_restarted,
+ attempt_number=require_latest_attempt_number,
)
finally:
await self._after_workflow_run_status_write(
@@ -13247,6 +13362,7 @@ class WorkflowService:
) -> None:
workflow_run_id = workflow_run.workflow_run_id
if status.is_final():
+ note_run_finalized(workflow_run_id)
# Free extraction-cache entries for this run.
extraction_cache.clear_workflow_run(workflow_run_id)
now = datetime.now(UTC)
@@ -13447,6 +13563,7 @@ class WorkflowService:
run_minutes_recorded_through: datetime | None = None,
workflow: Workflow | None = None,
lost_external_browser: bool = False,
+ attempt_number: int | None = None,
) -> WorkflowRun:
"""
Set final workflow run status based on pre-finally state.
@@ -13457,6 +13574,9 @@ class WorkflowService:
shielded write can land after the activity's own cancellation — an unconditional
write here would restore the parent while the children stay failed.
"""
+ attempt_fence_kwargs: dict[str, Any] = (
+ {"require_latest_attempt_number": attempt_number} if run_lease_tasks_enabled() else {}
+ )
if pre_finally_status not in (
WorkflowRunStatus.canceled,
WorkflowRunStatus.failed,
@@ -13471,6 +13591,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.failed,
failure_reason=registration_end,
+ **attempt_fence_kwargs,
)
if updated is None:
return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)
@@ -13486,6 +13607,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.terminated,
failure_reason=contract_verdict.reason,
+ **attempt_fence_kwargs,
)
if updated is None:
return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)
@@ -13493,6 +13615,7 @@ class WorkflowService:
updated = await self._update_workflow_run_status_if_not_final(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.completed,
+ **attempt_fence_kwargs,
)
if updated is not None:
otel_trace.get_current_span().set_attribute("task.completion_status", WorkflowRunStatus.completed)
@@ -13506,6 +13629,7 @@ class WorkflowService:
failure_reason=pre_finally_failure_reason,
failure_category=pre_finally_failure_category,
run_minutes_recorded_through=run_minutes_recorded_through,
+ **attempt_fence_kwargs,
)
if updated is None:
return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)
@@ -13702,6 +13826,7 @@ class WorkflowService:
run_with: str | None = None,
failure_category: list[dict] | None = None,
cascade_children: bool = False,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun:
LOG.info(
f"Marking workflow run {workflow_run_id} as failed",
@@ -13729,14 +13854,26 @@ class WorkflowService:
failure_category_source=failure_category_source,
)
- workflow_run = await self._update_workflow_run_status(
- workflow_run_id=workflow_run_id,
- status=WorkflowRunStatus.failed,
- failure_reason=failure_reason,
- run_with=run_with,
- failure_category=failure_category,
- )
- if cascade_children:
+ if run_lease_tasks_enabled() and require_latest_attempt_number is not None:
+ workflow_run = await self._update_workflow_run_status_if_not_final(
+ workflow_run_id=workflow_run_id,
+ status=WorkflowRunStatus.failed,
+ failure_reason=failure_reason,
+ run_with=run_with,
+ failure_category=failure_category,
+ require_latest_attempt_number=require_latest_attempt_number,
+ )
+ if workflow_run is None:
+ return await self.get_workflow_run(workflow_run_id)
+ else:
+ workflow_run = await self._update_workflow_run_status(
+ workflow_run_id=workflow_run_id,
+ status=WorkflowRunStatus.failed,
+ failure_reason=failure_reason,
+ run_with=run_with,
+ failure_category=failure_category,
+ )
+ if cascade_children and not (run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id)):
# Opt-in: run-level failures normally leave child rows to their own executors,
# but out-of-band finalization (an interrupted worker) has no executor left to do it.
try:
@@ -13848,6 +13985,7 @@ class WorkflowService:
cascade_children: bool = False,
only_from: Sequence[WorkflowRunStatus] | None = None,
worker_container_restarted: bool = False,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun | None:
"""Conditional failure finalize that no-ops when the run has already reached a terminal
state. Out-of-band finalizers (the interrupted-activity backstop) run concurrently with
@@ -13867,6 +14005,7 @@ class WorkflowService:
failure_reason=failure_reason,
failure_category=failure_category,
only_from=only_from,
+ require_latest_attempt_number=require_latest_attempt_number,
worker_container_restarted=worker_container_restarted,
)
if workflow_run is None:
@@ -13939,6 +14078,7 @@ class WorkflowService:
failure_reason: str | None,
run_with: str | None = None,
failure_category: list[dict] | None = None,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun:
LOG.info(
f"Marking workflow run {workflow_run_id} as terminated",
@@ -13974,6 +14114,7 @@ class WorkflowService:
failure_reason=failure_reason,
run_with=run_with,
failure_category=failure_category,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
return workflow_run
@@ -13982,6 +14123,7 @@ class WorkflowService:
workflow_run_id: str,
failure_reason: str | None,
failure_category: list[dict] | None = None,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun | None:
if failure_category is None:
failure_category = self._classify_workflow_terminal_failure(
@@ -13994,6 +14136,7 @@ class WorkflowService:
status=WorkflowRunStatus.terminated,
failure_reason=failure_reason,
failure_category=failure_category,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
if workflow_run is None:
return None
@@ -14006,7 +14149,9 @@ class WorkflowService:
)
return workflow_run
- async def mark_workflow_run_as_canceled(self, workflow_run_id: str) -> WorkflowRun:
+ async def mark_workflow_run_as_canceled(
+ self, workflow_run_id: str, require_latest_attempt_number: int | None = None
+ ) -> WorkflowRun:
"""Cancel a workflow run, rejecting the transition if the run has already
reached a terminal state.
@@ -14023,6 +14168,7 @@ class WorkflowService:
updated = await self.mark_workflow_run_as_canceled_if_not_final(
workflow_run_id=workflow_run_id,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
if updated is not None:
return updated
@@ -14033,6 +14179,11 @@ class WorkflowService:
if current is None:
raise WorkflowRunNotFound(workflow_run_id)
+ if run_lease_tasks_enabled() and (
+ require_latest_attempt_number is not None or await run_lease_lost(workflow_run_id)
+ ):
+ return current
+
if current.status.is_final():
attempt_rows = await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id)
if attempt_rows:
@@ -14080,16 +14231,21 @@ class WorkflowService:
self,
workflow_run_id: str,
failure_reason: str | None = None,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun | None:
"""Conditional cancel that is a no-op when the run has already reached a
terminal state. Safe to call from cancellation cleanup paths (e.g. the
copilot tool's timeout branch) that race with the run's own
``_finalize_workflow_run_status`` writes.
"""
+ if run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id):
+ note_run_finalized(workflow_run_id)
+ return None
updated = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.canceled,
failure_reason=failure_reason,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
if updated is None:
return None
@@ -14106,6 +14262,7 @@ class WorkflowService:
WorkflowRunStatus.canceled,
failure_reason,
None,
+ **_run_lease_attempt_kwargs("attempt_number", require_latest_attempt_number),
)
finally:
await self._after_workflow_run_status_write(updated, WorkflowRunStatus.canceled)
@@ -14118,6 +14275,7 @@ class WorkflowService:
run_with: str | None = None,
fallback_workflow_run: WorkflowRun | None = None,
finalized_by: str | None = None,
+ require_latest_attempt_number: int | None = None,
) -> WorkflowRun:
LOG.info(
f"Marking workflow run {workflow_run_id} as timed out",
@@ -14144,8 +14302,13 @@ class WorkflowService:
run_with=run_with,
failure_category=failure_category,
finalized_by=finalized_by,
+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),
)
if updated_workflow_run is None:
+ if run_lease_tasks_enabled() and (
+ require_latest_attempt_number is not None or await run_lease_lost(workflow_run_id)
+ ):
+ return await self.get_workflow_run(workflow_run_id=workflow_run_id)
# The CAS lost to a terminal writer, so only a fresh row can identify
# the winner. A supplied fallback is necessarily stale here; if the
# refresh fails, propagate so the timeout activity retries the repair
diff --git a/skyvern/services/orphaned_run_reaper.py b/skyvern/services/orphaned_run_reaper.py
new file mode 100644
index 000000000..10b404a13
--- /dev/null
+++ b/skyvern/services/orphaned_run_reaper.py
@@ -0,0 +1,274 @@
+from datetime import timedelta
+from functools import partial
+
+import structlog
+
+from skyvern.forge import app
+from skyvern.forge.failure_classifier import WORKER_CONTAINER_RESTARTED_REASON_CODE, classify_from_failure_reason
+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
+from skyvern.forge.sdk.db.repositories.run_execution_leases import RunExecutionLease
+from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutor
+from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
+from skyvern.forge.sdk.executor.run_liveness import (
+ effective_timeout_minutes,
+ is_run_registered,
+ owner_id,
+ schedule_side_effects,
+)
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status
+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.retry_policy import get_recorded_decision
+from skyvern.services import task_v2_service, uploaded_file_service
+
+LOG = structlog.get_logger()
+
+
+async def _api_key(organization_id: str) -> str | None:
+ token = await app.DATABASE.organizations.get_valid_org_auth_token(
+ organization_id,
+ OrganizationAuthTokenType.api.value,
+ )
+ if token is None:
+ LOG.warning("Orphaned run webhook skipped: no API key", organization_id=organization_id)
+ return None
+ return token.token
+
+
+async def _workflow_side_effects(run: WorkflowRun, attempt_number: int | None, has_attempts: bool) -> None:
+ if has_attempts:
+ decision = await get_recorded_decision(run.workflow_run_id, attempt_number=attempt_number)
+ if decision is None:
+ LOG.warning("Orphaned run has no recorded retry decision", workflow_run_id=run.workflow_run_id)
+ return
+ if decision.retry:
+ executor = AsyncExecutorFactory.get_executor()
+ if isinstance(executor, BackgroundTaskExecutor):
+ await executor.schedule_recorded_retry(run.workflow_run_id, decision.attempt_number)
+ return
+ await app.WORKFLOW_SERVICE._run_terminal_side_effects_with_retries(
+ run,
+ decision,
+ api_key=await _api_key(run.organization_id),
+ )
+ return
+ await uploaded_file_service.delete_files_attached_to_run(run_id=run.workflow_run_id)
+ api_key = await _api_key(run.organization_id)
+ if api_key:
+ await app.WORKFLOW_SERVICE.execute_workflow_webhook(run, api_key, claim_kind=None)
+
+
+async def _task_v2_side_effects(run: WorkflowRun, task: TaskV2) -> None:
+ try:
+ task = await app.DATABASE.observer.get_task_v2(task.observer_cruise_id, task.organization_id) or task
+ except Exception:
+ LOG.warning("Failed to reload reaped task v2", run_id=run.workflow_run_id, exc_info=True)
+ try:
+ await app.DATABASE.tasks.sync_task_run_status(
+ organization_id=run.organization_id,
+ run_id=task.observer_cruise_id,
+ status=task.status.value,
+ started_at=task.started_at,
+ finished_at=task.finished_at,
+ )
+ except Exception:
+ LOG.warning("Failed to sync reaped task v2", run_id=run.workflow_run_id, exc_info=True)
+ await uploaded_file_service.delete_files_attached_to_run(run_id=run.workflow_run_id)
+ await task_v2_service.send_task_v2_webhook(task)
+
+
+async def _task_side_effects(task: Task) -> None:
+ try:
+ task = await app.DATABASE.tasks.get_task(task.task_id, task.organization_id) or task
+ except Exception:
+ LOG.warning("Failed to reload reaped task", run_id=task.task_id, exc_info=True)
+ try:
+ await app.DATABASE.tasks.sync_task_run_status(
+ organization_id=task.organization_id,
+ run_id=task.task_id,
+ status=task.status.value,
+ started_at=task.started_at,
+ finished_at=task.finished_at,
+ )
+ except Exception:
+ LOG.warning("Failed to sync reaped task", run_id=task.task_id, exc_info=True)
+ await uploaded_file_service.delete_files_attached_to_run(run_id=task.task_id)
+ api_key = await _api_key(task.organization_id)
+ if api_key:
+ await app.agent.execute_task_webhook(task, api_key)
+
+
+async def _reap_workflow(lease: RunExecutionLease, reason: str, category: list[dict] | None) -> bool:
+ run = await app.DATABASE.workflow_runs.get_workflow_run(lease.run_id)
+ if run is None or run.status not in (WorkflowRunStatus.running, WorkflowRunStatus.paused):
+ return False
+ attempts = await app.DATABASE.workflow_run_attempts.get_attempts(lease.run_id)
+ if attempts and max(row.attempt_number for row in attempts) != lease.attempt_number:
+ return False
+ task_v2 = await app.DATABASE.observer.get_task_v2_by_workflow_run_id(lease.run_id, run.organization_id)
+ if task_v2 is not None:
+ won = await app.DATABASE.observer.update_task_v2_status_if_not_final(
+ task_v2.observer_cruise_id,
+ run.organization_id,
+ TaskV2Status.failed,
+ failure_category=category,
+ )
+ if not won:
+ task_v2 = await app.DATABASE.observer.get_task_v2(task_v2.observer_cruise_id, run.organization_id)
+ if task_v2 is None:
+ return False
+ if not task_v2.status.is_final():
+ return False
+ finalized = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(
+ lease.run_id,
+ WorkflowRunStatus(task_v2.status.value),
+ only_from=[WorkflowRunStatus.running, WorkflowRunStatus.paused],
+ require_latest_attempt_number=lease.attempt_number,
+ )
+ if finalized is None:
+ return False
+ else:
+ task_v2 = task_v2.model_copy(
+ update={
+ "status": TaskV2Status.failed,
+ "failure_category": category,
+ "finished_at": task_v2.finished_at or naive_utc_now(),
+ }
+ )
+ try:
+ finalized = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ lease.run_id,
+ reason,
+ category,
+ cascade_children=True,
+ only_from=[WorkflowRunStatus.running, WorkflowRunStatus.paused],
+ require_latest_attempt_number=lease.attempt_number,
+ worker_container_restarted=True,
+ )
+ except Exception:
+ finalized = await app.DATABASE.workflow_runs.get_workflow_run(lease.run_id)
+ if (
+ finalized is None
+ or finalized.status != WorkflowRunStatus.failed
+ or finalized.failure_reason != reason
+ ):
+ raise
+ LOG.warning(
+ "Reaped workflow status committed before finalizer failed", run_id=lease.run_id, exc_info=True
+ )
+ if finalized is None:
+ return False
+ schedule_side_effects(partial(_task_v2_side_effects, finalized, task_v2))
+ outcome = "task_v2_webhook"
+ else:
+ try:
+ finalized = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ lease.run_id,
+ reason,
+ category,
+ cascade_children=True,
+ only_from=[WorkflowRunStatus.running, WorkflowRunStatus.paused],
+ require_latest_attempt_number=lease.attempt_number,
+ worker_container_restarted=True,
+ )
+ except Exception:
+ finalized = await app.DATABASE.workflow_runs.get_workflow_run(lease.run_id)
+ if finalized is None or finalized.status != WorkflowRunStatus.failed or finalized.failure_reason != reason:
+ raise
+ LOG.warning("Reaped workflow status committed before finalizer failed", run_id=lease.run_id, exc_info=True)
+ if finalized is None:
+ return False
+ schedule_side_effects(partial(_workflow_side_effects, finalized, lease.attempt_number, bool(attempts)))
+ outcome = "retry_decision" if attempts else "webhook"
+ LOG.info(
+ "Reaped orphaned workflow run",
+ run_id=lease.run_id,
+ organization_id=run.organization_id,
+ attempt_number=lease.attempt_number,
+ outcome=outcome,
+ )
+ return True
+
+
+async def _reap_task(lease: RunExecutionLease, reason: str) -> bool:
+ task = await app.DATABASE.tasks.get_task(lease.run_id)
+ if task is None or task.workflow_run_id is not None or task.status != TaskStatus.running:
+ return False
+ claimed = await app.DATABASE.tasks.bulk_update_tasks(
+ [lease.run_id],
+ status=TaskStatus.failed,
+ failure_reason=reason,
+ only_if_status_in=[TaskStatus.running],
+ )
+ if not claimed:
+ return False
+ task = task.model_copy(
+ update={
+ "status": TaskStatus.failed,
+ "failure_reason": reason,
+ "finished_at": task.finished_at or naive_utc_now(),
+ }
+ )
+ try:
+ await app.DATABASE.tasks.bulk_update_steps_by_task_ids(
+ [lease.run_id],
+ new_status=StepStatus.failed,
+ only_if_status_in=[StepStatus.created, StepStatus.running],
+ )
+ except Exception:
+ LOG.warning("Failed to finalize steps for reaped task", run_id=lease.run_id, exc_info=True)
+ schedule_side_effects(partial(_task_side_effects, task))
+ LOG.info(
+ "Reaped orphaned task",
+ run_id=lease.run_id,
+ organization_id=task.organization_id,
+ attempt_number=lease.attempt_number,
+ outcome="webhook",
+ )
+ return True
+
+
+async def reap_orphaned_runs_once() -> None:
+ timeout = effective_timeout_minutes()
+ reason = (
+ "This run stopped because the Skyvern server process that ran it exited or stopped responding "
+ f"(no heartbeat for {timeout} minutes). Retry the run if needed."
+ )
+ category = classify_from_failure_reason(
+ reason,
+ exception_name=WORKER_CONTAINER_RESTARTED_REASON_CODE,
+ fallback_to_unknown=True,
+ )
+ repository = app.DATABASE.run_execution_leases
+ expired = await repository.get_expired(naive_utc_now() - timedelta(minutes=timeout), limit=500)
+ reaper_id = owner_id() + ":reaper"
+ for lease in expired:
+ if is_run_registered(lease.run_id):
+ continue
+ claimed = False
+ try:
+ if not await repository.claim(lease.run_id, lease.owner_id, lease.heartbeat_at, reaper_id):
+ continue
+ claimed = True
+ won = False
+ if lease.run_type == "workflow_run":
+ won = await _reap_workflow(lease, reason, category)
+ elif lease.run_type == "task":
+ won = await _reap_task(lease, reason)
+ await repository.release(reaper_id, [lease.run_id])
+ if not won:
+ LOG.info(
+ "Released stale run lease",
+ run_id=lease.run_id,
+ run_type=lease.run_type,
+ attempt_number=lease.attempt_number,
+ )
+ except Exception:
+ if claimed:
+ try:
+ await repository.unclaim(lease.run_id, reaper_id, lease.heartbeat_at)
+ except Exception:
+ LOG.exception("Failed to unclaim orphaned run lease", run_id=lease.run_id)
+ LOG.exception("Orphaned run reaper pass failed", run_id=lease.run_id)
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 010166bab..448bf7196 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -37,6 +37,7 @@ from skyvern.exceptions import (
ScriptNotFound,
ScriptTerminationException,
StepTerminationError,
+ TaskNotFound,
WorkflowRunNotFound,
)
from skyvern.forge import app
@@ -58,6 +59,7 @@ from skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_a
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint
from skyvern.forge.sdk.db.enums import TaskType, is_job_recipe_workflow_run_trigger_type
+from skyvern.forge.sdk.executor.run_liveness import run_lease_lost, run_lease_tasks_enabled
from skyvern.forge.sdk.experimentation.workflow_block_engine import run_honors_chosen_engine
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
@@ -998,6 +1000,7 @@ async def _update_workflow_block(
final_output = output
if task_id:
+ lease_lost = await run_lease_lost(context.workflow_run_id)
if step_id:
# Build step output from script actions similar to agent flow
step_output = None
@@ -1016,6 +1019,30 @@ async def _update_workflow_block(
errors=errors,
)
+ if not lease_lost:
+ await app.DATABASE.tasks.update_step(
+ step_id=step_id,
+ task_id=task_id,
+ organization_id=context.organization_id,
+ status=step_status,
+ is_last=is_last,
+ output=step_output,
+ )
+ if lease_lost and not task_status.is_final():
+ updated_task = await app.DATABASE.tasks.get_task(task_id, context.organization_id)
+ if updated_task is None:
+ raise TaskNotFound(task_id)
+ else:
+ updated_task = await app.DATABASE.tasks.update_task(
+ task_id=task_id,
+ organization_id=context.organization_id,
+ status=task_status,
+ failure_reason=failure_reason,
+ extracted_information=output,
+ errors=[error.model_dump() for error in user_defined_errors] if user_defined_errors else None,
+ keep_final_status=lease_lost,
+ )
+ if lease_lost and step_id and updated_task.status == task_status:
await app.DATABASE.tasks.update_step(
step_id=step_id,
task_id=task_id,
@@ -1024,14 +1051,8 @@ async def _update_workflow_block(
is_last=is_last,
output=step_output,
)
- updated_task = await app.DATABASE.tasks.update_task(
- task_id=task_id,
- organization_id=context.organization_id,
- status=task_status,
- failure_reason=failure_reason,
- extracted_information=output,
- errors=[error.model_dump() for error in user_defined_errors] if user_defined_errors else None,
- )
+ if run_lease_tasks_enabled() and lease_lost:
+ return
downloaded_files: list[FileInfo] = []
try:
async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
@@ -1095,6 +1116,8 @@ async def _update_workflow_block(
# final_output is already set to `output` at line 596.
pass
+ if run_lease_tasks_enabled() and await run_lease_lost(context.workflow_run_id):
+ return
updated_block = await app.DATABASE.observer.update_workflow_run_block(
workflow_run_block_id=workflow_run_block_id,
organization_id=context.organization_id if context else None,
@@ -1115,6 +1138,8 @@ async def _update_workflow_block(
"Failed to record cached block outcome", workflow_run_block_id=workflow_run_block_id, exc_info=True
)
+ if run_lease_tasks_enabled() and await run_lease_lost(context.workflow_run_id):
+ return
recorded_output_parameter = await _record_output_parameter_value(
context.workflow_run_id,
context.workflow_id,
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 4bd0d53fd..c1e22f8ab 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -4,6 +4,7 @@ import copy
import json
import string
import textwrap
+from contextlib import nullcontext
from datetime import UTC, datetime
from typing import Any
@@ -39,6 +40,13 @@ from skyvern.forge.sdk.core.hashing import generate_url_hash
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerType
+from skyvern.forge.sdk.executor.run_liveness import (
+ acquire_run_lease,
+ note_run_finalized,
+ run_lease_lost,
+ run_lease_tasks_enabled,
+ track_run,
+)
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import unusable_browser_session_error
from skyvern.forge.sdk.schemas.task_v2 import (
@@ -630,7 +638,12 @@ async def run_task_v2(
# SKY-7005: scoped() restores the parent context on exit, preserving
# loop_internal_state so per-iteration download filtering continues to
# work for subsequent blocks in the same loop iteration.
- with skyvern_context.scoped(context, propagate_captcha_timeout=True):
+ with (
+ skyvern_context.scoped(context, propagate_captcha_timeout=True),
+ track_run(task_v2.workflow_run_id, "workflow_run") if task_v2.workflow_run_id else nullcontext(),
+ ):
+ if task_v2.workflow_run_id:
+ await acquire_run_lease(task_v2.workflow_run_id)
try:
workflow, workflow_run, task_v2 = await run_task_v2_helper(
organization=organization,
@@ -829,6 +842,9 @@ async def run_task_v2_helper(
context.browser_session_id = browser_session_id
context.max_screenshot_scrolls = task_v2.max_screenshot_scrolls
+ if await run_lease_lost(workflow_run_id):
+ current = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)
+ return None, None, current or task_v2
task_v2 = await app.DATABASE.observer.update_task_v2(
task_v2_id=task_v2_id, organization_id=organization_id, status=TaskV2Status.running
)
@@ -2235,6 +2251,13 @@ async def _update_task_v2_status(
output: dict[str, Any] | None = None,
failure_category: list[dict] | None = None,
) -> TaskV2:
+ keep_final_status = False
+ if run_lease_tasks_enabled():
+ current = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)
+ if current is not None and current.workflow_run_id and await run_lease_lost(current.workflow_run_id):
+ if not status.is_final():
+ return await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id) or current
+ keep_final_status = True
task_v2 = await app.DATABASE.observer.update_task_v2(
task_v2_id,
organization_id=organization_id,
@@ -2242,7 +2265,12 @@ async def _update_task_v2_status(
summary=summary,
output=output,
failure_category=failure_category,
+ keep_final_status=keep_final_status,
)
+ if keep_final_status and task_v2.status != status:
+ if task_v2.workflow_run_id:
+ note_run_finalized(task_v2.workflow_run_id)
+ return task_v2
if status in [TaskV2Status.completed, TaskV2Status.failed, TaskV2Status.terminated]:
start_time = (
task_v2.started_at.replace(tzinfo=UTC) if task_v2.started_at else task_v2.created_at.replace(tzinfo=UTC)
@@ -2261,6 +2289,11 @@ async def _update_task_v2_status(
return task_v2
+def _task_v2_final_write_refused(task_v2: TaskV2, status: TaskV2Status) -> bool:
+ # Without leases, a concurrent writer's post-commit refresh must preserve the existing side effects.
+ return run_lease_tasks_enabled() and task_v2.status != status
+
+
async def mark_task_v2_as_failed(
task_v2_id: str,
workflow_run_id: str | None = None,
@@ -2313,6 +2346,8 @@ async def mark_task_v2_as_failed(
output=output,
failure_category=failure_category,
)
+ if _task_v2_final_write_refused(task_v2, TaskV2Status.failed):
+ return task_v2
if workflow_run_id:
await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(
workflow_run_id,
@@ -2341,6 +2376,8 @@ async def mark_task_v2_as_completed(
summary=summary,
output=output,
)
+ if _task_v2_final_write_refused(task_v2, TaskV2Status.completed):
+ return task_v2
if workflow_run_id:
await app.WORKFLOW_SERVICE.mark_workflow_run_as_completed(workflow_run_id)
@@ -2373,6 +2410,8 @@ async def mark_task_v2_as_canceled(
organization_id=organization_id,
status=TaskV2Status.canceled,
)
+ if _task_v2_final_write_refused(task_v2, TaskV2Status.canceled):
+ return task_v2
if workflow_run_id:
await app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled(workflow_run_id)
@@ -2396,6 +2435,8 @@ async def mark_task_v2_as_terminated(
status=TaskV2Status.terminated,
failure_category=failure_category,
)
+ if _task_v2_final_write_refused(task_v2, TaskV2Status.terminated):
+ return task_v2
if workflow_run_id:
await app.WORKFLOW_SERVICE.mark_workflow_run_as_terminated(
workflow_run_id,
@@ -2423,6 +2464,8 @@ async def mark_task_v2_as_timed_out(
organization_id=organization_id,
status=TaskV2Status.timed_out,
)
+ if _task_v2_final_write_refused(task_v2, TaskV2Status.timed_out):
+ return task_v2
async def repair_workflow_run() -> None:
nonlocal fallback_workflow_run