Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fail runs orphaned by a crashed self-hosted server (SKY-7819) (#8925)
Shuchang Zheng8e73b4dbbc37
diff --git a/.env.example b/.env.exampleindex c94b1f5ef..cc0e1658f 100644--- a/.env.example+++ b/.env.example@@ -146,6 +146,10 @@ BROWSER_ACTION_TIMEOUT_MS=5000# Agent run configuration:# MAX_STEPS_PER_RUN: Maximum number of steps to execute per run unless the agent finishes with a terminal state (last step or error).MAX_STEPS_PER_RUN=50+# RUN_HEARTBEAT_TIMEOUT_MINUTES: Minutes without a heartbeat before a server fails a run and sends its webhook or starts a policy retry.+# Applies when the API server executes runs itself (the default). The same server after restart or another server can act.+# Allowed values: 0 (off) or 2 to 1440. Servers that share a database must use the same value.+RUN_HEARTBEAT_TIMEOUT_MINUTES=30# Logging and database configuration:# LOG_LEVEL: Control log level (e.g., INFO, DEBUG).diff --git a/alembic/versions/2026_10_16_0738-50c59227239d_add_execution_leases_for_in_process_runs.py b/alembic/versions/2026_10_16_0738-50c59227239d_add_execution_leases_for_in_process_runs.pynew file mode 100644index 000000000..333a18b4f--- /dev/null+++ b/alembic/versions/2026_10_16_0738-50c59227239d_add_execution_leases_for_in_process_runs.py@@ -0,0 +1,34 @@+"""Add execution leases for in-process runs.++Revision ID: 50c59227239d+Revises: 4452dbf24863+Create Date: 2026-10-16T07:38:57.828780+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "50c59227239d"+down_revision: Union[str, None] = "4452dbf24863"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.create_table(+ "run_execution_leases",+ sa.Column("run_id", sa.String(), primary_key=True),+ sa.Column("run_type", sa.String(), nullable=False),+ sa.Column("owner_id", sa.String(), nullable=False),+ sa.Column("attempt_number", sa.Integer(), nullable=True),+ sa.Column("heartbeat_at", sa.DateTime(), nullable=False),+ )+++def downgrade() -> None:+ op.drop_table("run_execution_leases")diff --git a/docs/reference/configuration.mdx b/docs/reference/configuration.mdxindex b456f7936..1268c963b 100644--- a/docs/reference/configuration.mdx+++ b/docs/reference/configuration.mdx@@ -726,8 +726,11 @@ Never change `ENCRYPTOR_AES_SECRET_KEY`, `ENCRYPTOR_AES_SALT` or `ENCRYPTOR_AES_| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR` or `CRITICAL`. Any other value means `INFO`. || `CODE_BLOCK_MODE` | `enabled` | `disabled` rejects [Code blocks](/reference/blocks) in runs || `TOTP_LIFESPAN_MINUTES` | `10` | How long a 2FA code sent to Skyvern stays usable. Older codes are ignored. |+| `RUN_HEARTBEAT_TIMEOUT_MINUTES` | `30` | Minutes without a heartbeat from the server process that executes a run. After this time, a server marks the run `failed` and sends its failure webhook, or starts a retry under the workflow's retry policy. This can be the same server after a restart, or another server on the same database. This check applies when the API server executes runs itself (the default). || `ENABLE_URL_VALIDATION` | `false` | `true` makes Skyvern refuse URLs that point at loopback, private-network or cloud-metadata addresses. Off by default. See [URL validation](#url-validation). |+Set `RUN_HEARTBEAT_TIMEOUT_MINUTES` to `0` to turn off this check. Allowed values are `0` or `2` to `1440`. The check never fails a run that has no execution lease: runs from older versions, scripts, embedded clients, or servers with the check off, and runs that were already stuck before the upgrade. All server processes that share a database must use the same value and keep their clocks synchronized (NTP). When other servers share the database, they fail a server's runs if it stalls for longer than this time. They also fail its runs if it cannot reach the database for longer than this time. A single server keeps its runs and continues them when it resumes. The check does not fail runs that have not started (`created` or `queued`). When a server shuts down, it waits up to 15 seconds for failure webhooks that its reaper has started. A webhook that does not finish in that time is not sent. Webhook receivers should deduplicate deliveries by run ID, attempt, and status.+### URL validationURL validation is off by default, so Skyvern can open, fetch and call internal hosts without extra setup. That includes an intranet portal, a service on your network and `localhost`. Set `ENABLE_URL_VALIDATION=true` if people you don't trust can start tasks or workflows on your server, or if it runs next to internal services or a cloud metadata endpoint.diff --git a/skyvern/config.py b/skyvern/config.pyindex e3d9d622a..6d4a20780 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -306,6 +306,7 @@ class Settings(BaseSettings):COPILOT_ALLOW_INLINE_CODE_EXECUTION: bool = False# Default code_only for MCP block/workflow tools. Off = permissive.MCP_CODE_ONLY_MODE: bool = False+ RUN_HEARTBEAT_TIMEOUT_MINUTES: int = 30PORT: int = 8000# uvicorn answers 503 without dispatching to ASGI once *either* open connections or in-flight# requests reach this. Open connections is the binding term -- idle keep-alives and long-lived@@ -1087,6 +1088,13 @@ class Settings(BaseSettings):return self+ @field_validator("RUN_HEARTBEAT_TIMEOUT_MINUTES")+ @classmethod+ def _run_heartbeat_timeout_minutes_bounds(cls, value: int) -> int:+ if value != 0 and not 2 <= value <= 1440:+ raise ValueError("RUN_HEARTBEAT_TIMEOUT_MINUTES must be 0 (off) or between 2 and 1440 minutes")+ return value+@field_validator("API_LIMIT_CONCURRENCY", mode="before")@classmethoddef _api_limit_concurrency_unlimited_sentinels(cls, value: Any) -> Any:diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 7854034ce..f162bb616 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -157,6 +157,7 @@ from skyvern.forge.sdk.core.skyvern_context import (from skyvern.forge.sdk.db.datetime_utils import naive_utc_nowfrom skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.db.exceptions import NotFoundError+from skyvern.forge.sdk.executor.run_liveness import note_run_finalized, run_lease_lostfrom skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_contextfrom skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handlerfrom skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value@@ -2199,11 +2200,14 @@ class ForgeAgent:task_retry=task_retry,)# Update task status to running- task = await app.DATABASE.tasks.update_task(- task_id=task.task_id,- organization_id=task.organization_id,- status=TaskStatus.running,- )+ if await run_lease_lost(workflow_run.workflow_run_id):+ task = await app.DATABASE.tasks.get_task(task.task_id, task.organization_id) or task+ else:+ task = await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=task.organization_id,+ status=TaskStatus.running,+ )step = await app.DATABASE.tasks.create_step(task.task_id,@@ -2244,11 +2248,14 @@ class ForgeAgent:order=task_order,retry=task_retry,)- task = await app.DATABASE.tasks.update_task(- task_id=task.task_id,- organization_id=organization_id,- status=TaskStatus.running,- )+ if await run_lease_lost(workflow_run_id):+ task = await app.DATABASE.tasks.get_task(task.task_id, organization_id) or task+ else:+ task = await app.DATABASE.tasks.update_task(+ task_id=task.task_id,+ organization_id=organization_id,+ status=TaskStatus.running,+ )step = await app.DATABASE.tasks.create_step(task.task_id,order=0,@@ -9820,11 +9827,21 @@ class ForgeAgent:diff=skyvern_context.mask_multi_field_totp_data(update_comparison, task_id=task.task_id),sampling=True,)+ registered_run_id = task.workflow_run_id or task.task_id+ lease_lost = await run_lease_lost(registered_run_id)+ if lease_lost and not status.is_final():+ return await app.DATABASE.tasks.get_task(task_id=task.task_id, organization_id=task.organization_id) or taskupdated_task, finish_claimed = await app.DATABASE.tasks.update_task_and_claim_finish(task.task_id,organization_id=task.organization_id,+ keep_final_status=lease_lost,**updates,)+ if lease_lost and updated_task.status != status:+ note_run_finalized(task.task_id)+ return updated_task+ if status.is_final():+ note_run_finalized(task.task_id)# Minutes emit only when THIS write atomically flipped finished_at from NULL# -- the shared claim every task_v1 finalizer rides -- so a concurrent# finalizer (the stuck-task sweep's CAS, the temporal timeout activity, adiff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.pyindex 54f8b5014..94d077bda 100644--- a/skyvern/forge/api_app.py+++ b/skyvern/forge/api_app.py@@ -47,6 +47,7 @@ from skyvern.forge.sdk.db.exceptions import DatabaseConnectionUnavailableError,from skyvern.forge.sdk.db.models import Basefrom skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutorfrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactory+from skyvern.forge.sdk.executor.run_liveness import start_run_lease_tasks, stop_run_lease_tasksfrom skyvern.forge.sdk.routes import internal_auth, internal_llmsfrom skyvern.forge.sdk.routes.google_oauth import google_oauth_routerfrom skyvern.forge.sdk.routes.google_sheets import google_sheets_router@@ -68,6 +69,7 @@ from skyvern.services.cleanup_service import (stop_cleanup_scheduler,stop_temp_artifact_sweep,)+from skyvern.services.orphaned_run_reaper import reap_orphaned_runs_oncefrom skyvern.services.workflow_run_group_service import run_workflow_run_group_recovery_loopfrom skyvern.services.workflow_schedule_service import (start_workflow_schedule_scheduler,@@ -431,6 +433,11 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:if isinstance(AsyncExecutorFactory.get_executor(), BackgroundTaskExecutor)else None)+ if (+ isinstance(AsyncExecutorFactory.get_executor(), BackgroundTaskExecutor)+ and settings.RUN_HEARTBEAT_TIMEOUT_MINUTES > 0+ ):+ start_run_lease_tasks(reap_orphaned_runs_once)try:if mcp_app:async with mcp_app.lifespan(mcp_app):@@ -446,6 +453,7 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:await asyncio.gather(retry_recovery_task, *([group_recovery_task] if group_recovery_task else []), return_exceptions=True)+ await stop_run_lease_tasks()# The initial pass starts the periodic sweep even when cancelled; stop it so the loop can drain.await AsyncExecutorFactory.get_executor().stop_retry_recovery()diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex 0536e9715..2f2421c60 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -36,6 +36,7 @@ from skyvern.forge.sdk.db.repositories.microsoft_oauth import MicrosoftOAuthRepofrom skyvern.forge.sdk.db.repositories.observer import ObserverRepositoryfrom skyvern.forge.sdk.db.repositories.organizations import OrganizationsRepositoryfrom skyvern.forge.sdk.db.repositories.otp import OTPRepository+from skyvern.forge.sdk.db.repositories.run_execution_leases import RunExecutionLeasesRepositoryfrom skyvern.forge.sdk.db.repositories.run_feedback import RunFeedbackRepositoryfrom skyvern.forge.sdk.db.repositories.schedules import SchedulesRepositoryfrom skyvern.forge.sdk.db.repositories.scripts import ScriptsRepository@@ -398,6 +399,7 @@ class AgentDB(BaseAlchemyDB):)# -- Zero-dependency repositories --+ self.run_execution_leases = RunExecutionLeasesRepository(self.Session, debug_enabled, self.is_retryable_error)self.tasks = TasksRepository(self.Session, debug_enabled, self.is_retryable_error)self.workflows = WorkflowsRepository(self.Session,diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex c210b6797..59e32bfec 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -95,6 +95,16 @@ class Base(AsyncAttrs, DeclarativeBase):pass+class RunExecutionLeaseModel(Base):+ __tablename__ = "run_execution_leases"++ run_id = Column(String, primary_key=True)+ run_type = Column(String, nullable=False)+ owner_id = Column(String, nullable=False)+ attempt_number = Column(Integer, nullable=True)+ heartbeat_at = Column(DateTime, nullable=False)++class TaskModel(Base):__tablename__ = "tasks"__table_args__ = (diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.pyindex 169c7259d..237bb6c33 100644--- a/skyvern/forge/sdk/db/repositories/observer.py+++ b/skyvern/forge/sdk/db/repositories/observer.py@@ -348,6 +348,36 @@ class ObserverRepository(BaseRepository):return Thought.model_validate(thought_obj)raise NotFoundError(f"Thought {thought_id}")+ @db_operation("update_task_v2_status_if_not_final")+ async def update_task_v2_status_if_not_final(+ self,+ task_v2_id: str,+ organization_id: str,+ status: TaskV2Status,+ *,+ failure_category: list[dict[str, Any]] | None = None,+ ) -> bool:+ now = naive_utc_now()+ values: dict[str, Any] = {"status": status.value, "modified_at": now}+ if status.is_final():+ values["finished_at"] = func.coalesce(TaskV2Model.finished_at, now)+ if failure_category is not None:+ values["failure_category"] = failure_category+ async with self.Session() as session:+ result = await session.execute(+ update(TaskV2Model)+ .where(+ TaskV2Model.observer_cruise_id == task_v2_id,+ TaskV2Model.organization_id == organization_id,+ TaskV2Model.status.in_([s.value for s in TaskV2Status if not s.is_final()]),+ )+ .values(**values)+ .returning(TaskV2Model.observer_cruise_id)+ )+ won = result.scalar_one_or_none() is not None+ await session.commit()+ return won+@db_operation("update_task_v2")async def update_task_v2(self,@@ -363,16 +393,19 @@ class ObserverRepository(BaseRepository):organization_id: str | None = None,webhook_failure_reason: str | None = None,failure_category: list[dict[str, Any]] | None = None,+ *,+ keep_final_status: bool = False,) -> TaskV2:async with self.Session() as session:- task_v2 = (- await session.scalars(- select(TaskV2Model)- .filter_by(observer_cruise_id=task_v2_id)- .filter_by(organization_id=organization_id)- )- ).first()+ query = (+ select(TaskV2Model).filter_by(observer_cruise_id=task_v2_id).filter_by(organization_id=organization_id)+ )+ if keep_final_status:+ query = query.with_for_update()+ task_v2 = (await session.scalars(query)).first()if task_v2:+ if keep_final_status and TaskV2Status(task_v2.status).is_final():+ return convert_to_task_v2(task_v2, debug_enabled=self.debug_enabled)if status:task_v2.status = statusif status == TaskV2Status.queued and task_v2.queued_at is None:diff --git a/skyvern/forge/sdk/db/repositories/run_execution_leases.py b/skyvern/forge/sdk/db/repositories/run_execution_leases.pynew file mode 100644index 000000000..4928dd4a2--- /dev/null+++ b/skyvern/forge/sdk/db/repositories/run_execution_leases.py@@ -0,0 +1,142 @@+from collections.abc import Collection, Mapping+from dataclasses import dataclass+from datetime import datetime++from sqlalchemy import delete, or_, select, update+from sqlalchemy.dialects.postgresql import insert as pg_insert+from sqlalchemy.dialects.sqlite import insert as sqlite_insert++from skyvern.forge.sdk.db._error_handling import db_operation+from skyvern.forge.sdk.db.base_repository import BaseRepository+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now+from skyvern.forge.sdk.db.models import RunExecutionLeaseModel+++@dataclass(frozen=True)+class RunExecutionLease:+ run_id: str+ run_type: str+ owner_id: str+ attempt_number: int | None+ heartbeat_at: datetime+++class RunExecutionLeasesRepository(BaseRepository):+ @db_operation("acquire_run_execution_leases")+ async def acquire(+ self, owner_id: str, runs: Mapping[str, tuple[str, int | None]], stale_before: datetime+ ) -> set[str]:+ if not runs:+ return set()+ now = naive_utc_now()+ acquired: set[str] = set()+ async with self.Session() as session:+ insert = sqlite_insert if session.bind.dialect.name == "sqlite" else pg_insert+ # One row per statement also stays below SQLite's bind parameter limit for large registries.+ for run_id, (run_type, attempt_number) in runs.items():+ stmt = insert(RunExecutionLeaseModel).values(+ run_id=run_id,+ run_type=run_type,+ owner_id=owner_id,+ attempt_number=attempt_number,+ heartbeat_at=now,+ )+ result = await session.execute(+ stmt.on_conflict_do_update(+ index_elements=[RunExecutionLeaseModel.run_id],+ set_={+ key: stmt.excluded[key]+ for key in ("run_type", "owner_id", "attempt_number", "heartbeat_at")+ },+ where=or_(+ RunExecutionLeaseModel.owner_id == owner_id,+ RunExecutionLeaseModel.heartbeat_at < stale_before,+ ),+ ).returning(RunExecutionLeaseModel.run_id)+ )+ acquired.update(result.scalars().all())+ await session.commit()+ return acquired++ @db_operation("renew_run_execution_leases")+ async def renew(self, owner_id: str, run_ids: Collection[str]) -> set[str]:+ if not run_ids:+ return set()+ async with self.Session() as session:+ result = await session.execute(+ update(RunExecutionLeaseModel)+ .where(RunExecutionLeaseModel.owner_id == owner_id, RunExecutionLeaseModel.run_id.in_(run_ids))+ .values(heartbeat_at=naive_utc_now())+ .returning(RunExecutionLeaseModel.run_id)+ )+ renewed = set(result.scalars().all())+ await session.commit()+ return renewed++ @db_operation("release_run_execution_leases")+ async def release(self, owner_id: str, run_ids: Collection[str]) -> None:+ if not run_ids:+ return+ async with self.Session() as session:+ await session.execute(+ delete(RunExecutionLeaseModel).where(+ RunExecutionLeaseModel.owner_id == owner_id,+ RunExecutionLeaseModel.run_id.in_(run_ids),+ )+ )+ await session.commit()++ @db_operation("get_expired_run_execution_leases")+ async def get_expired(self, heartbeat_before: datetime, limit: int = 500) -> list[RunExecutionLease]:+ async with self.Session() as session:+ rows = (+ await session.scalars(+ select(RunExecutionLeaseModel)+ .where(RunExecutionLeaseModel.heartbeat_at < heartbeat_before)+ .order_by(RunExecutionLeaseModel.heartbeat_at)+ .limit(limit)+ )+ ).all()+ return [+ RunExecutionLease(row.run_id, row.run_type, row.owner_id, row.attempt_number, row.heartbeat_at)+ for row in rows+ ]++ @db_operation("claim_run_execution_lease")+ async def claim(+ self,+ run_id: str,+ observed_owner_id: str,+ observed_heartbeat_at: datetime,+ reaper_owner_id: str,+ ) -> bool:+ async with self.Session() as session:+ result = await session.execute(+ update(RunExecutionLeaseModel)+ .where(+ RunExecutionLeaseModel.run_id == run_id,+ RunExecutionLeaseModel.owner_id == observed_owner_id,+ RunExecutionLeaseModel.heartbeat_at == observed_heartbeat_at,+ )+ .values(owner_id=reaper_owner_id, heartbeat_at=naive_utc_now())+ .returning(RunExecutionLeaseModel.run_id)+ )+ won = result.scalar_one_or_none() is not None+ await session.commit()+ return won++ @db_operation("unclaim_run_execution_lease")+ async def unclaim(self, run_id: str, reaper_owner_id: str, observed_heartbeat_at: datetime) -> bool:+ async with self.Session() as session:+ result = await session.execute(+ update(RunExecutionLeaseModel)+ .where(+ RunExecutionLeaseModel.run_id == run_id,+ RunExecutionLeaseModel.owner_id == reaper_owner_id,+ )+ .values(heartbeat_at=observed_heartbeat_at)+ .returning(RunExecutionLeaseModel.run_id)+ )+ unclaimed = result.scalar_one_or_none() is not None+ await session.commit()+ return unclaimeddiff --git a/skyvern/forge/sdk/db/repositories/tasks.py b/skyvern/forge/sdk/db/repositories/tasks.pyindex 0e373007d..fde2b8223 100644--- a/skyvern/forge/sdk/db/repositories/tasks.py+++ b/skyvern/forge/sdk/db/repositories/tasks.py@@ -603,6 +603,8 @@ class TasksRepository(BaseRepository):max_steps_per_run: int | None = None,organization_id: str | None = None,failure_category: list[dict[str, Any]] | None = None,+ *,+ keep_final_status: bool = False,) -> Task:updated_task, _ = await self._update_task_with_finish_claim(task_id,@@ -614,6 +616,7 @@ class TasksRepository(BaseRepository):max_steps_per_run=max_steps_per_run,organization_id=organization_id,failure_category=failure_category,+ keep_final_status=keep_final_status,)return updated_task@@ -630,6 +633,8 @@ class TasksRepository(BaseRepository):max_steps_per_run: int | None = None,organization_id: str | None = None,failure_category: list[dict[str, Any]] | None = None,+ *,+ keep_final_status: bool = False,) -> tuple[Task, bool]:"""``update_task``, plus whether THIS write flipped ``finished_at`` from NULL.@@ -648,6 +653,7 @@ class TasksRepository(BaseRepository):max_steps_per_run=max_steps_per_run,organization_id=organization_id,failure_category=failure_category,+ keep_final_status=keep_final_status,)async def _update_task_with_finish_claim(@@ -661,6 +667,8 @@ class TasksRepository(BaseRepository):max_steps_per_run: int | None = None,organization_id: str | None = None,failure_category: list[dict[str, Any]] | None = None,+ *,+ keep_final_status: bool = False,) -> tuple[Task, bool]:if (status is None@@ -687,6 +695,8 @@ class TasksRepository(BaseRepository):.with_for_update())).first():+ if keep_final_status and TaskStatus(task.status).is_final():+ return convert_to_task(task, debug_enabled=self.debug_enabled), Falseif status is not None:task.status = statusif status == TaskStatus.queued and task.queued_at is None:@@ -909,6 +919,30 @@ class TasksRepository(BaseRepository):await session.commit()return result.rowcount or 0+ @db_operation("bulk_update_steps_by_task_ids")+ async def bulk_update_steps_by_task_ids(+ self,+ task_ids: list[str],+ new_status: StepStatus,+ only_if_status_in: list[StepStatus],+ ) -> int:+ if not task_ids or not only_if_status_in:+ return 0++ update_values: dict[str, Any] = {"status": new_status.value}+ if new_status.is_terminal():+ update_values["finished_at"] = func.coalesce(StepModel.finished_at, naive_utc_now())+ async with self.Session() as session:+ stmt = (+ update(StepModel)+ .where(StepModel.task_id.in_(task_ids))+ .where(StepModel.status.in_([s.value for s in only_if_status_in]))+ .values(**update_values)+ )+ result = await session.execute(stmt)+ await session.commit()+ return result.rowcount or 0+@db_operation("get_tasks")async def get_tasks(self,diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex bc45454c1..1fb7b8d8e 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -1303,6 +1303,7 @@ class WorkflowRunsRepository(BaseRepository):job_id: str | None = None,depends_on_workflow_run_id: str | None = None,workflow_definition_sha256: str | None = None,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun | None:"""One conditional UPDATE to ``status`` from a non-terminal state (``only_from`` narrows it), so a latecancel cannot clobber a finalization; None when the row was terminal or missing. Timestamps follow@@ -1361,9 +1362,16 @@ class WorkflowRunsRepository(BaseRepository):values["queued_at"] = func.coalesce(WorkflowRunModel.queued_at, await _first_queued_at(session, workflow_run))+ statement = update(WorkflowRunModel)+ if require_latest_attempt_number is not None:+ statement = statement.where(+ ~exists().where(+ WorkflowRunAttemptModel.workflow_run_id == WorkflowRunModel.workflow_run_id,+ WorkflowRunAttemptModel.attempt_number > require_latest_attempt_number,+ )+ )result = await session.execute(- update(WorkflowRunModel)- .where(+ statement.where(WorkflowRunModel.workflow_run_id == workflow_run_id,WorkflowRunModel.status.in_(non_terminal),)diff --git a/skyvern/forge/sdk/executor/background_task_executor.py b/skyvern/forge/sdk/executor/background_task_executor.pyindex 044cce7e9..da04528f2 100644--- a/skyvern/forge/sdk/executor/background_task_executor.py+++ b/skyvern/forge/sdk/executor/background_task_executor.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.db.repositories.workflow_run_attempts import (ATTEMPT_RECOVERY_MAX_PAGES,)from skyvern.forge.sdk.executor.async_executor import AsyncExecutor+from skyvern.forge.sdk.executor.run_liveness import acquire_run_lease, hold_run, track_runfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPEfrom skyvern.forge.sdk.schemas.task_v2 import TaskV2Status@@ -442,6 +443,14 @@ class BackgroundTaskExecutor(AsyncExecutor):await asyncio.sleep(min(backoff_seconds, remaining_seconds))backoff_seconds = min(backoff_seconds * 2, 30.0)+ async def schedule_recorded_retry(self, workflow_run_id: str, attempt_number: int) -> bool:+ attempts = await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id)+ attempt = next((row for row in attempts if row.attempt_number == attempt_number), None)+ if attempt is None or attempt.retry_decision != "retry":+ return False+ self._schedule_retry_resume(attempt)+ return True+def _schedule_retry_resume(self, attempt: Any) -> None:key = (attempt.workflow_run_id, attempt.attempt_number)if key in self._scheduled_retry_resumes:@@ -717,47 +726,59 @@ class BackgroundTaskExecutor(AsyncExecutor):organization_id=organization_id,)- task = await app.DATABASE.tasks.update_task(- task_id,- status=TaskStatus.running,- organization_id=organization_id,- )-- close_browser_on_completion = browser_session_id is None and not task.browser_address-- run_obj = await app.DATABASE.tasks.get_run(run_id=task_id, organization_id=organization_id)- engine = RunEngine.skyvern_v1- if run_obj and run_obj.task_run_type == RunType.openai_cua:- engine = RunEngine.openai_cua- elif run_obj and run_obj.task_run_type == RunType.anthropic_cua:- engine = RunEngine.anthropic_cua- elif run_obj and run_obj.task_run_type == RunType.ui_tars:- engine = RunEngine.ui_tars- elif run_obj and run_obj.task_run_type == RunType.yutori_navigator:- engine = RunEngine.yutori_navigator- elif run_obj and run_obj.task_run_type == RunType.task_v3:- engine = RunEngine.skyvern_v3-- context: SkyvernContext = skyvern_context.ensure_context()- context.task_id = task.task_id- context.run_id = context.run_id or task.task_id- context.organization_id = organization_id- context.max_steps_override = max_steps_override- context.max_screenshot_scrolls = task.max_screenshot_scrolls+ release_hold = hold_run(task_id, "task")+ try:+ await acquire_run_lease(task_id)+ task = await app.DATABASE.tasks.update_task(+ task_id,+ status=TaskStatus.running,+ organization_id=organization_id,+ )- await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)- await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)- self._schedule(- background_tasks,- app.agent.execute_step,- organization,- task,- step,- api_key,- close_browser_on_completion=close_browser_on_completion,- browser_session_id=browser_session_id,- engine=engine,- )+ close_browser_on_completion = browser_session_id is None and not task.browser_address++ run_obj = await app.DATABASE.tasks.get_run(run_id=task_id, organization_id=organization_id)+ engine = RunEngine.skyvern_v1+ if run_obj and run_obj.task_run_type == RunType.openai_cua:+ engine = RunEngine.openai_cua+ elif run_obj and run_obj.task_run_type == RunType.anthropic_cua:+ engine = RunEngine.anthropic_cua+ elif run_obj and run_obj.task_run_type == RunType.ui_tars:+ engine = RunEngine.ui_tars+ elif run_obj and run_obj.task_run_type == RunType.yutori_navigator:+ engine = RunEngine.yutori_navigator+ elif run_obj and run_obj.task_run_type == RunType.task_v3:+ engine = RunEngine.skyvern_v3++ context: SkyvernContext = skyvern_context.ensure_context()+ context.task_id = task.task_id+ context.run_id = context.run_id or task.task_id+ context.organization_id = organization_id+ context.max_steps_override = max_steps_override+ context.max_screenshot_scrolls = task.max_screenshot_scrolls++ await prepare_org_llm_runtime(app.DATABASE, organization_id, organization)+ await initialize_skyvern_state_file(task_id=task_id, organization_id=organization_id)++ async def execute_registered_step(*args: Any, **kwargs: Any) -> None:+ with track_run(task_id, "task"):+ release_hold()+ await app.agent.execute_step(*args, **kwargs)++ self._schedule(+ background_tasks,+ execute_registered_step,+ organization,+ task,+ step,+ api_key,+ close_browser_on_completion=close_browser_on_completion,+ browser_session_id=browser_session_id,+ engine=engine,+ )+ except BaseException:+ release_hold()+ raiseasync def execute_workflow(self,diff --git a/skyvern/forge/sdk/executor/run_liveness.py b/skyvern/forge/sdk/executor/run_liveness.pynew file mode 100644index 000000000..45ee46a1a--- /dev/null+++ b/skyvern/forge/sdk/executor/run_liveness.py@@ -0,0 +1,325 @@+from __future__ import annotations++import asyncio+import os+import socket+import time+import uuid+from collections.abc import Awaitable, Callable, Iterator+from contextlib import contextmanager+from dataclasses import dataclass+from datetime import timedelta+from typing import TYPE_CHECKING, Any++import structlog++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now++if TYPE_CHECKING:+ from skyvern.forge.sdk.db.repositories.run_execution_leases import RunExecutionLeasesRepository++LOG = structlog.get_logger()+INTERVAL_SECONDS = 60+SHUTDOWN_DRAIN_SECONDS = 15+_pid: int | None = None+_owner_id: str | None = None+_enabled = False+_wake = asyncio.Event()+_lease_write_lock = asyncio.Lock()+_registry: dict[str, _Registration] = {}+_kept_leases: set[str] = set()+_held: set[str] = set()+_tasks: set[asyncio.Task[None]] = set()+_side_effect_tasks: set[asyncio.Task[None]] = set()+_side_effect_limit = asyncio.Semaphore(10)+++@dataclass+class _Registration:+ run_type: str+ attempt_number: int | None+ count: int = 0+ task: asyncio.Task[Any] | None = None+ acquired: bool = False+ lost: bool = False+ finalized: bool = False+++def owner_id() -> str:+ global _pid, _owner_id+ pid = os.getpid()+ if _pid != pid or _owner_id is None:+ _pid = pid+ _owner_id = f"{socket.gethostname()}:{pid}:{uuid.uuid4().hex[:8]}"+ return _owner_id+++def run_lease_tasks_enabled() -> bool:+ return _enabled+++def effective_timeout_minutes() -> int:+ return settings.RUN_HEARTBEAT_TIMEOUT_MINUTES+++def _register(+ run_id: str,+ run_type: str,+ attempt_number: int | None,+ task: asyncio.Task[Any] | None,+) -> Callable[[bool], None]:+ if not _enabled:+ return lambda cancelled: None+ _kept_leases.discard(run_id)+ entry = _registry.get(run_id)+ if entry is None:+ entry = _Registration(run_type, attempt_number)+ _registry[run_id] = entry+ elif entry.attempt_number != attempt_number:+ entry.attempt_number = attempt_number+ entry.acquired = False+ entry.lost = False+ entry.finalized = False+ entry.count += 1+ entry.task = task+ _wake.set()+ released = False++ def release(cancelled: bool) -> None:+ nonlocal released+ if released:+ return+ released = True+ entry.count -= 1+ if entry.count == 0 and _registry.get(run_id) is entry:+ if cancelled:+ _kept_leases.add(run_id)+ del _registry[run_id]+ _wake.set()++ return release+++@contextmanager+def track_run(run_id: str, run_type: str, attempt_number: int | None = None) -> Iterator[None]:+ try:+ task = asyncio.current_task()+ except RuntimeError:+ task = None+ release = _register(run_id, run_type, attempt_number, task)+ cancelled = False+ try:+ yield+ except asyncio.CancelledError:+ cancelled = True+ raise+ finally:+ release(cancelled)+++def hold_run(run_id: str, run_type: str) -> Callable[[], None]:+ release = _register(run_id, run_type, None, None)+ return lambda: release(False)+++def is_run_registered(run_id: str) -> bool:+ return run_id in _registry+++def note_run_finalized(run_id: str) -> None:+ entry = _registry.get(run_id)+ if entry is not None:+ entry.finalized = True+++async def acquire_run_lease(run_id: str) -> None:+ if not _enabled:+ return+ entry = _registry.get(run_id)+ if entry is None or entry.acquired or entry.lost:+ return+ attempt = entry.attempt_number+ try:+ async with _lease_write_lock:+ if _registry.get(run_id) is not entry or entry.attempt_number != attempt or entry.acquired or entry.lost:+ return+ acquired = await app.DATABASE.run_execution_leases.acquire(+ owner_id(),+ {run_id: (entry.run_type, attempt)},+ stale_before=naive_utc_now() - timedelta(minutes=effective_timeout_minutes()),+ )+ if run_id not in acquired:+ LOG.info(+ "Run lease held by another process",+ run_id=run_id,+ run_type=entry.run_type,+ attempt_number=attempt,+ owner_id=owner_id(),+ )+ return+ _held.add(run_id)+ if _registry.get(run_id) is entry and entry.attempt_number == attempt:+ entry.acquired = True+ except Exception:+ LOG.warning("Run lease acquire failed before run start", run_id=run_id, exc_info=True)+++async def run_lease_lost(run_id: str) -> bool:+ if not _enabled:+ return False+ entry = _registry.get(run_id)+ if entry is None or (not entry.acquired and not entry.lost):+ return False+ if entry.lost:+ return True+ try:+ renewed = await app.DATABASE.run_execution_leases.renew(owner_id(), [run_id])+ except Exception:+ LOG.warning("Run lease check failed before status write", run_id=run_id, exc_info=True)+ return False+ if run_id in renewed:+ return False+ LOG.warning(+ "Run lease lost before status write",+ run_id=run_id,+ run_type=entry.run_type,+ attempt_number=entry.attempt_number,+ owner_id=owner_id(),+ )+ return True+++async def _heartbeat(repository: RunExecutionLeasesRepository) -> None:+ last_renew = time.monotonic()+ while True:+ _wake.clear()+ try:+ # Snapshot the generation as well as the object: a retry can enter while an UPDATE awaits I/O.+ renewing = {+ key: (entry, entry.attempt_number)+ for key, entry in _registry.items()+ if entry.acquired and not entry.lost+ }+ # Serialize release and acquire so a release cannot delete a new registration's lease.+ async with _lease_write_lock:+ released = _held - _registry.keys()+ await repository.release(owner_id(), released - _kept_leases)+ _held.difference_update(released)+ if time.monotonic() - last_renew >= INTERVAL_SECONDS:+ last_renew = time.monotonic()+ renewed = await repository.renew(owner_id(), renewing.keys())+ newly_lost: list[_Registration] = []+ for run_id, (entry, attempt) in renewing.items():+ if run_id in renewed or _registry.get(run_id) is not entry or entry.attempt_number != attempt:+ continue+ entry.lost = True+ entry.acquired = False+ newly_lost.append(entry)+ _held.discard(run_id)+ LOG.warning(+ "Run lease lost",+ finalized=entry.finalized,+ run_id=run_id,+ run_type=entry.run_type,+ attempt_number=attempt,+ owner_id=owner_id(),+ )+ # Repeated cancellation would interrupt the cleanup started by the first cancellation.+ lost_targets = {entry.task for entry in newly_lost if entry.task is not None and not entry.finalized}+ for target in lost_targets:+ if not target.done() and all(+ entry.lost and not entry.finalized for entry in _registry.values() if entry.task is target+ ):+ target.cancel()+ async with _lease_write_lock:+ acquiring = {+ key: (entry, entry.attempt_number)+ for key, entry in _registry.items()+ if not entry.acquired and not entry.lost+ }+ acquired = await repository.acquire(+ owner_id(),+ {key: (entry.run_type, attempt) for key, (entry, attempt) in acquiring.items()},+ stale_before=naive_utc_now() - timedelta(minutes=effective_timeout_minutes()),+ )+ for run_id, (entry, attempt) in acquiring.items():+ if run_id not in acquired:+ LOG.info(+ "Run lease held by another process",+ run_id=run_id,+ run_type=entry.run_type,+ attempt_number=attempt,+ owner_id=owner_id(),+ )+ continue+ _held.add(run_id)+ if _registry.get(run_id) is entry and entry.attempt_number == attempt:+ entry.acquired = True+ _kept_leases.intersection_update(_held)+ except Exception:+ LOG.exception("Run lease heartbeat failed", owner_id=owner_id())+ try:+ remaining = max(0.0, min(INTERVAL_SECONDS, INTERVAL_SECONDS - (time.monotonic() - last_renew)))+ async with asyncio.timeout(remaining):+ await _wake.wait()+ except TimeoutError:+ pass+++async def _reaper_loop(reap_once: Callable[[], Awaitable[None]]) -> None:+ while True:+ await asyncio.sleep(INTERVAL_SECONDS)+ try:+ await reap_once()+ LOG.debug("Orphaned run reaper pass completed")+ except Exception:+ LOG.exception("Orphaned run reaper pass failed")+++def schedule_side_effects(effect: Callable[[], Awaitable[None]]) -> None:+ async def run() -> None:+ try:+ async with _side_effect_limit, asyncio.timeout(300):+ await effect()+ except Exception:+ LOG.exception("Orphaned run side effects failed")++ task = asyncio.create_task(run(), name="orphaned-run-side-effects")+ _side_effect_tasks.add(task)+ task.add_done_callback(_side_effect_tasks.discard)+++def start_run_lease_tasks(reap_once: Callable[[], Awaitable[None]]) -> None:+ global _enabled, _wake, _side_effect_limit, _lease_write_lock+ if _enabled or settings.RUN_HEARTBEAT_TIMEOUT_MINUTES <= 0:+ return+ _enabled = True+ _wake = asyncio.Event()+ _lease_write_lock = asyncio.Lock()+ _held.clear()+ _side_effect_limit = asyncio.Semaphore(10)+ _tasks.add(asyncio.create_task(_heartbeat(app.DATABASE.run_execution_leases), name="run-lease-heartbeat"))+ _tasks.add(asyncio.create_task(_reaper_loop(reap_once), name="orphaned-run-reaper"))+ LOG.info("Run lease tasks started", owner_id=owner_id(), timeout_minutes=effective_timeout_minutes())+++async def stop_run_lease_tasks() -> None:+ global _enabled+ _enabled = False+ # Stop the producers before collecting effects so a pass cannot enqueue work during shutdown.+ for task in _tasks:+ task.cancel()+ await asyncio.gather(*_tasks, return_exceptions=True)+ _tasks.clear()+ effects = list(_side_effect_tasks)+ if not effects:+ return+ _, pending = await asyncio.wait(effects, timeout=SHUTDOWN_DRAIN_SECONDS)+ for task in pending:+ task.cancel()+ await asyncio.gather(*pending, return_exceptions=True)+ if pending:+ LOG.warning("Orphaned run side effects cancelled at shutdown", count=len(pending))+ _side_effect_tasks.clear()diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 47fbb2674..1490d73f0 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -184,6 +184,7 @@ from skyvern.forge.sdk.db.enums import TaskTypefrom skyvern.forge.sdk.db.exceptions import NotFoundErrorfrom skyvern.forge.sdk.db.id import generate_action_id, generate_workflow_run_idfrom skyvern.forge.sdk.db.repositories.google_oauth import DISPATCH_ACCEPTED, DISPATCH_FAILED, GmailSendDispatch+from skyvern.forge.sdk.executor.run_liveness import run_lease_lost, run_lease_tasks_enabledfrom skyvern.forge.sdk.experimentation.code_block_ai_fallback import code_block_ai_fallback_flag_enabledfrom skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_typefrom skyvern.forge.sdk.experimentation.workflow_block_engine import (@@ -1150,6 +1151,8 @@ class Block(BaseModel, abc.ABC):value=value,)self._output_recorded_this_execution = True+ if run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id):+ returnawait app.DATABASE.workflow_runs.create_or_update_workflow_run_output_parameter(workflow_run_id=workflow_run_id,output_parameter_id=self.output_parameter.output_parameter_id,@@ -8214,11 +8217,16 @@ async def wrapper({default_args}):await record_task_nav_error_code(escalation_task.task_id, escalation_nav_error, escalation_url)elif failed_nav_error_code and not escalation_navigated:skyvern_context.ensure_context().task_nav_error_codes[escalation_task.task_id] = failed_nav_error_code- escalation_task = await app.DATABASE.tasks.update_task(- task_id=escalation_task.task_id,- organization_id=organization_id,- status=TaskStatus.running,- )+ if await run_lease_lost(workflow_run_id):+ escalation_task = (+ await app.DATABASE.tasks.get_task(escalation_task.task_id, organization_id) or escalation_task+ )+ else:+ escalation_task = await app.DATABASE.tasks.update_task(+ task_id=escalation_task.task_id,+ organization_id=organization_id,+ status=TaskStatus.running,+ )escalation_step = await app.DATABASE.tasks.create_step(escalation_task.task_id,order=0,@@ -15587,6 +15595,15 @@ class HumanInteractionBlock(BaseTaskBlock):browser_session_id=browser_session_id,)+ if await run_lease_lost(workflow_run_id):+ return await self.build_block_result(+ success=False,+ failure_reason="Another server already finalized this run",+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ )await app.DATABASE.workflow_runs.update_workflow_run(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.paused,diff --git a/skyvern/forge/sdk/workflow/models/code_block_recording.py b/skyvern/forge/sdk/workflow/models/code_block_recording.pyindex 380ea8b67..7e0cfed2c 100644--- a/skyvern/forge/sdk/workflow/models/code_block_recording.py+++ b/skyvern/forge/sdk/workflow/models/code_block_recording.py@@ -11,6 +11,7 @@ import structlogfrom skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.executor.run_liveness import run_lease_lostfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.models.code_block_recorder import (@@ -347,12 +348,15 @@ class CodeBlockActionRecording:returnself._finalized = Truetry:- await app.DATABASE.tasks.update_task(+ status = TaskStatus.completed if success else TaskStatus.failed+ lease_lost = await run_lease_lost(self._workflow_run_id)+ task = await app.DATABASE.tasks.update_task(task_id=self._task.task_id,organization_id=self._organization_id,- status=TaskStatus.completed if success else TaskStatus.failed,+ status=status,+ keep_final_status=lease_lost,)- if self._step is not None:+ if self._step is not None and (not lease_lost or task.status == status):await app.DATABASE.tasks.update_step(task_id=self._task.task_id,step_id=self._step.step_id,diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 5895cc52a..dc2daad4f 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -128,6 +128,13 @@ from skyvern.forge.sdk.db.repositories.workflow_runs import (lock_workflow_run_for_dispatch,)from skyvern.forge.sdk.enterprise_features import collect_enterprise_gated_run_features+from skyvern.forge.sdk.executor.run_liveness import (+ acquire_run_lease,+ note_run_finalized,+ run_lease_lost,+ run_lease_tasks_enabled,+ track_run,+)from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_contextfrom skyvern.forge.sdk.experimentation.transient_ui_capture import resolve_transient_ui_capture_armfrom skyvern.forge.sdk.experimentation.workflow_block_engine import (@@ -997,6 +1004,10 @@ def _workflow_block_engine_attribution_for_duration_log(workflow_run_id: str) ->return ARM_ATTRIBUTION_LOST+def _run_lease_attempt_kwargs(key: str, attempt_number: int | None) -> dict[str, Any]:+ return {key: attempt_number} if run_lease_tasks_enabled() else {}++def _get_workflow_run_max_elapsed_timeout_seconds(workflow_run: WorkflowRun) -> float:effective_minutes = get_effective_workflow_run_max_elapsed_time_minutes(workflow_run.max_elapsed_time_minutes)started_at = _as_utc(workflow_run.started_at or workflow_run.created_at)@@ -6137,6 +6148,7 @@ class WorkflowService:pre_finally_status: WorkflowRunStatus | None,pre_finally_failure_reason: str | None,timeout_failure_reason: str,+ require_latest_attempt_number: int | None = None,) -> tuple[WorkflowRun, WorkflowRunStatus | None, str | None]:if pre_finally_status is None or not pre_finally_status.is_final():if refreshed_workflow_run := await app.DATABASE.workflow_runs.get_workflow_run(@@ -6158,6 +6170,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=timeout_failure_reason,fallback_workflow_run=workflow_run,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)pre_finally_status = WorkflowRunStatus.timed_outpre_finally_failure_reason = timeout_failure_reason@@ -6172,6 +6185,7 @@ class WorkflowService:pre_finally_status: WorkflowRunStatus | None,pre_finally_failure_reason: str | None,timeout_failure_reason: str,+ require_latest_attempt_number: int | None = None,) -> tuple[WorkflowRun, WorkflowRunStatus | None, str | None]:handle_timeout_task = asyncio.create_task(self._handle_post_run_elapsed_timeout(@@ -6181,6 +6195,7 @@ class WorkflowService:pre_finally_status=pre_finally_status,pre_finally_failure_reason=pre_finally_failure_reason,timeout_failure_reason=timeout_failure_reason,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),),name=f"post_run_elapsed_timeout_{workflow_run_id}",)@@ -6203,6 +6218,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=timeout_failure_reason,fallback_workflow_run=workflow_run,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),))return fallback_workflow_run, WorkflowRunStatus.timed_out, timeout_failure_reason@@ -6246,6 +6262,7 @@ class WorkflowService:return await self.mark_workflow_run_as_timed_out(workflow_run.workflow_run_id,failure_reason="Workflow run exceeded its remaining runtime while waiting for sequential clearance.",+ **_run_lease_attempt_kwargs("require_latest_attempt_number", attempt_number),)async def _prepare_recorded_retry(@@ -6707,28 +6724,62 @@ class WorkflowService:) -> WorkflowRun:if on_execution_authority is not None:on_execution_authority(attempt_number, dispatch_claim_started_at)+ with track_run(workflow_run_id, "workflow_run", attempt_number):+ await acquire_run_lease(workflow_run_id)+ try:+ return await self._execute_workflow(+ workflow_run_id=workflow_run_id,+ api_key=api_key,+ organization=organization,+ block_labels=block_labels,+ block_outputs=block_outputs,+ browser_session_id=browser_session_id,+ need_call_webhook=need_call_webhook,+ workflow_override=workflow_override,+ requested_completion_contract=requested_completion_contract,+ attempt_number=attempt_number,+ dispatch_claim_started_at=dispatch_claim_started_at,+ on_finalization_denied=on_finalization_denied,+ )+ except BaseException as error:+ if isinstance(error, WorkflowAttemptDispatchSuperseded) or BROWSER_RETIREMENT_DENIED_NOTE in getattr(+ error, "__notes__", ()+ ):+ if on_finalization_denied is not None:+ on_finalization_denied()+ raise++ async def _leave_if_attempt_superseded(+ self,+ workflow_run_id: str,+ attempt_number: int,+ on_finalization_denied: Callable[[], None] | None,+ renewal_task: asyncio.Task[None] | None,+ ) -> None:+ if not run_lease_tasks_enabled():+ returntry:- return await self._execute_workflow(+ attempts = await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id)+ except Exception:+ LOG.warning(+ "Could not check for a later workflow attempt; finalizing this attempt",workflow_run_id=workflow_run_id,- api_key=api_key,- organization=organization,- block_labels=block_labels,- block_outputs=block_outputs,- browser_session_id=browser_session_id,- need_call_webhook=need_call_webhook,- workflow_override=workflow_override,- requested_completion_contract=requested_completion_contract,attempt_number=attempt_number,- dispatch_claim_started_at=dispatch_claim_started_at,- on_finalization_denied=on_finalization_denied,+ exc_info=True,)- except BaseException as error:- if isinstance(error, WorkflowAttemptDispatchSuperseded) or BROWSER_RETIREMENT_DENIED_NOTE in getattr(- error, "__notes__", ()- ):- if on_finalization_denied is not None:- on_finalization_denied()- raise+ return+ if any(attempt.attempt_number > attempt_number for attempt in attempts):+ if on_finalization_denied is not None:+ on_finalization_denied()+ if renewal_task is not None:+ renewal_task.cancel()+ with contained_effect("record superseded workflow attempt"):+ LOG.info(+ "Workflow attempt superseded; skipping finalization and cleanup",+ workflow_run_id=workflow_run_id,+ attempt_number=attempt_number,+ )+ raise WorkflowAttemptDispatchSuperseded(workflow_run_id, attempt_number)async def _execute_workflow(self,@@ -6756,6 +6807,9 @@ class WorkflowService:both sync and async (Temporal-dispatched) trigger modes."""organization_id = organization.organization_id+ attempt_fence_kwargs: dict[str, Any] = (+ {"require_latest_attempt_number": attempt_number} if run_lease_tasks_enabled() else {}+ )LOG.info("Executing workflow",@@ -6803,7 +6857,7 @@ class WorkflowService:failure_reason=failure_reason,)return await self.mark_workflow_run_as_failed(- workflow_run_id=workflow_run_id, failure_reason=failure_reason+ workflow_run_id=workflow_run_id, failure_reason=failure_reason, **attempt_fence_kwargs)if not workflow.workflow_definition.blocks:@@ -7196,7 +7250,7 @@ class WorkflowService:workflow_run = failedelse:workflow_run = await self.mark_workflow_run_as_failed(- workflow_run_id=workflow_run_id, failure_reason=failure_reason+ workflow_run_id=workflow_run_id, failure_reason=failure_reason, **attempt_fence_kwargs)await self.clean_up_workflow(workflow=workflow,@@ -7302,6 +7356,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=failure_reason,failure_category=lease_failure_category,+ **attempt_fence_kwargs,)await self.clean_up_workflow(workflow=workflow,@@ -7397,6 +7452,7 @@ class WorkflowService:block_outputs=block_outputs,script=workflow_script,script_is_pinned=script_is_pinned,+ **attempt_fence_kwargs,)if max_elapsed_timeout_seconds <= 0:@@ -7407,6 +7463,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=pre_finally_failure_reason,fallback_workflow_run=workflow_run,+ **attempt_fence_kwargs,)pre_finally_status = WorkflowRunStatus.timed_outpre_finally_browser_persistence_status = pre_finally_status@@ -7429,6 +7486,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=pre_finally_failure_reason,fallback_workflow_run=workflow_run,+ **attempt_fence_kwargs,)pre_finally_status = WorkflowRunStatus.timed_outpre_finally_browser_persistence_status = pre_finally_status@@ -7447,6 +7505,7 @@ class WorkflowService:pre_finally_status=pre_finally_status,pre_finally_failure_reason=pre_finally_failure_reason,timeout_failure_reason=_require_elapsed_timeout_failure_reason(timeout_failure_reason),+ **attempt_fence_kwargs,)pre_finally_browser_persistence_status = pre_finally_statusreturn workflow_run@@ -7633,6 +7692,7 @@ class WorkflowService:pre_finally_status=pre_finally_status,pre_finally_failure_reason=pre_finally_failure_reason,timeout_failure_reason=_require_elapsed_timeout_failure_reason(timeout_failure_reason),+ **attempt_fence_kwargs,)if pre_finally_browser_persistence_status is None:# A timeout before the body outcome was captured must retain@@ -7658,6 +7718,9 @@ class WorkflowService:renewal_task.cancel()assert escaped_error is not Noneraise escaped_error+ await self._leave_if_attempt_superseded(+ workflow_run_id, attempt_number, on_finalization_denied, renewal_task+ )browser_cleanup_result: WorkflowBrowserCleanupResult | None = Nonebrowser_persistence_status: WorkflowRunStatus | None = Nonebrowser_write_back_exhausted = False@@ -7755,6 +7818,7 @@ class WorkflowService:pre_finally_failure_category=pre_finally_failure_category,run_minutes_recorded_through=run_minutes_recorded_through,lost_external_browser=run_lost_external_browser,+ attempt_number=attempt_number if run_lease_tasks_enabled() else None,))except BaseException:@@ -7814,6 +7878,7 @@ class WorkflowService:failure_reason=failure_reason,failure_category=failure_category,cascade_children=True,+ **attempt_fence_kwargs,))while not finalize_task.done():@@ -7839,6 +7904,10 @@ class WorkflowService:exc_info=True,)+ await self._leave_if_attempt_superseded(+ workflow_run_id, attempt_number, on_finalization_denied, renewal_task+ )+if getattr(workflow_run, "copilot_session_id", None):with contained_effect("publish Copilot origin-run runtime secrets",@@ -7937,9 +8006,13 @@ class WorkflowService:block_outputs: dict[str, Any] | None = None,script: Script | None = None,script_is_pinned: bool = False,+ require_latest_attempt_number: int | None = None,) -> tuple[WorkflowRun, set[str]]:organization_id = organization.organization_idworkflow_run_id = workflow_run.workflow_run_id+ attempt_fence_kwargs: dict[str, Any] = (+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}+ )top_level_blocks = workflow.workflow_definition.blocksall_blocks = get_all_blocks(top_level_blocks)# Load script blocks if script is provided@@ -8325,6 +8398,7 @@ class WorkflowService:is_script_run=is_script_run,blocks_to_update=blocks_to_update,denied_cached_labels=denied_cached_labels,+ **attempt_fence_kwargs,)#@@ -8352,6 +8426,7 @@ class WorkflowService:is_script_run=is_script_run,blocks_to_update=blocks_to_update,denied_cached_labels=denied_cached_labels,+ **attempt_fence_kwargs,)if should_stop:@@ -8538,7 +8613,11 @@ class WorkflowService:is_script_run: bool,blocks_to_update: set[str],denied_cached_labels: frozenset[str] = frozenset(),+ require_latest_attempt_number: int | None = None,) -> tuple[WorkflowRun, set[str]]:+ attempt_fence_kwargs: dict[str, Any] = (+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}+ )finally_block_label = workflow.workflow_definition.finally_block_labeldag_blocks = workflow.workflow_definition.blocksif finally_block_label:@@ -8559,6 +8638,7 @@ class WorkflowService:workflow_run = await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run.workflow_run_id,failure_reason=str(exc),+ **attempt_fence_kwargs,)return workflow_run, blocks_to_update@@ -8583,6 +8663,7 @@ class WorkflowService:workflow_run = await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run.workflow_run_id,failure_reason=f"Unable to find block with label {current_label}",+ **attempt_fence_kwargs,)break@@ -8616,6 +8697,7 @@ class WorkflowService:blocks_to_update=blocks_to_update,parent_workflow_run_block_id=parent_wrb_id,denied_cached_labels=denied_cached_labels,+ **attempt_fence_kwargs,)# Track conditional workflow_run_block_ids so branch targets@@ -8666,6 +8748,7 @@ class WorkflowService:workflow_run = await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run.workflow_run_id,failure_reason=f"Next block label {next_label} not found in workflow definition",+ **attempt_fence_kwargs,)break@@ -8681,6 +8764,7 @@ class WorkflowService:workflow_run = await self.mark_workflow_run_as_failed(workflow_run_id=workflow_run.workflow_run_id,failure_reason=f"Cycle detected while traversing workflow definition at block {next_label}",+ **attempt_fence_kwargs,)break@@ -8706,7 +8790,11 @@ class WorkflowService:blocks_to_update: set[str],parent_workflow_run_block_id: str | None = None,denied_cached_labels: frozenset[str] = frozenset(),+ require_latest_attempt_number: int | None = None,) -> tuple[WorkflowRun, set[str], BlockResult | None, bool, dict[str, Any] | None]:+ attempt_fence_kwargs: dict[str, Any] = (+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}+ )organization_id = organization.organization_idworkflow_run_block_result: BlockResult | None = Nonebranch_metadata: dict[str, Any] | None = None@@ -8844,7 +8932,7 @@ class WorkflowService:else:no_block_result_reason = "Block result is None"updated_workflow_run = await self.mark_workflow_run_as_failed_if_not_final(- workflow_run_id=workflow_run_id, failure_reason=no_block_result_reason+ workflow_run_id=workflow_run_id, failure_reason=no_block_result_reason, **attempt_fence_kwargs)workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(workflow_run_id, workflow_run@@ -8870,6 +8958,7 @@ class WorkflowService:block_result=workflow_run_block_result,workflow_run=workflow_run,workflow_run_id=workflow_run_id,+ **attempt_fence_kwargs,)# Generate pending script after block completes successfully@@ -8892,7 +8981,7 @@ class WorkflowService:failure_reason = f"{block.block_type} block failed. failure reason: {exception_message}"updated_workflow_run = await self.mark_workflow_run_as_failed_if_not_final(- workflow_run_id=workflow_run_id, failure_reason=failure_reason+ workflow_run_id=workflow_run_id, failure_reason=failure_reason, **attempt_fence_kwargs)workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(workflow_run_id, workflow_run@@ -10217,7 +10306,11 @@ class WorkflowService:block_result: BlockResult,workflow_run: WorkflowRun,workflow_run_id: str,+ require_latest_attempt_number: int | None = None,) -> tuple[WorkflowRun, bool]:+ attempt_fence_kwargs: dict[str, Any] = (+ {"require_latest_attempt_number": require_latest_attempt_number} if run_lease_tasks_enabled() else {}+ )if block_result.status not in (BlockStatus.canceled,BlockStatus.failed,@@ -10235,7 +10328,9 @@ class WorkflowService:block_type_var=block.block_type,block_label=block.label,)- workflow_run = await self.mark_workflow_run_as_canceled(workflow_run_id=workflow_run_id)+ workflow_run = await self.mark_workflow_run_as_canceled(+ workflow_run_id=workflow_run_id, **attempt_fence_kwargs+ )return workflow_run, Truetarget_status, failure_reason, task_failure_category = self._resolve_block_terminal_outcome(block=block,@@ -10320,6 +10415,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=failure_reason,failure_category=task_failure_category,+ **attempt_fence_kwargs,)workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(workflow_run_id, workflow_run@@ -10329,6 +10425,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,failure_reason=failure_reason,failure_category=task_failure_category,+ **attempt_fence_kwargs,)workflow_run = updated_workflow_run or await self._current_row_after_lost_finalize(workflow_run_id, workflow_run@@ -13122,11 +13219,15 @@ class WorkflowService:run_with: str | None = None,ai_fallback: bool | None = None,failure_category: list[dict] | None = None,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun:failure_category = self._with_saved_profile_signed_out(workflow_run_id, status, failure_category)flipped_to_final = Falseworkflow_run: WorkflowRun | None = Noneif status.is_final():+ if run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id):+ note_run_finalized(workflow_run_id)+ return await self.get_workflow_run(workflow_run_id)workflow_run = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(workflow_run_id=workflow_run_id,status=status,@@ -13134,6 +13235,7 @@ class WorkflowService:run_with=run_with,ai_fallback=ai_fallback,failure_category=failure_category,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)flipped_to_final = workflow_run is not Noneif workflow_run is not None:@@ -13144,14 +13246,21 @@ class WorkflowService:status,failure_reason,failure_category,+ **_run_lease_attempt_kwargs("attempt_number", require_latest_attempt_number),)finally:await self._after_workflow_run_status_write(workflow_run, status, emit_run_minutes=True)return workflow_run+ if run_lease_tasks_enabled() and require_latest_attempt_number is not None:+ return await self.get_workflow_run(workflow_run_id)if await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id):# Policy runs keep the winning terminal status and its frozen retry decision.return await self.get_workflow_run(workflow_run_id)if workflow_run is None:+ if await run_lease_lost(workflow_run_id):+ if status.is_final():+ note_run_finalized(workflow_run_id)+ return await self.get_workflow_run(workflow_run_id)workflow_run = await app.DATABASE.workflow_runs.update_workflow_run(workflow_run_id=workflow_run_id,status=status,@@ -13174,10 +13283,14 @@ class WorkflowService:run_minutes_recorded_through: datetime | None = None,only_from: Sequence[WorkflowRunStatus] | None = None,worker_container_restarted: bool = False,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun | None:""":meth:`_update_workflow_run_status` for writers that must lose a race against therun's own finalizer. Returns ``None`` when the row was already terminal."""failure_category = self._with_saved_profile_signed_out(workflow_run_id, status, failure_category)+ if run_lease_tasks_enabled() and status.is_final() and await run_lease_lost(workflow_run_id):+ note_run_finalized(workflow_run_id)+ return Noneworkflow_run = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(workflow_run_id=workflow_run_id,status=status,@@ -13185,6 +13298,7 @@ class WorkflowService:run_with=run_with,failure_category=failure_category,only_from=only_from,+ require_latest_attempt_number=require_latest_attempt_number,)if workflow_run is None:return None@@ -13195,6 +13309,7 @@ class WorkflowService:failure_reason,failure_category,worker_container_restarted=worker_container_restarted,+ attempt_number=require_latest_attempt_number,)finally:await self._after_workflow_run_status_write(@@ -13247,6 +13362,7 @@ class WorkflowService:) -> None:workflow_run_id = workflow_run.workflow_run_idif status.is_final():+ note_run_finalized(workflow_run_id)# Free extraction-cache entries for this run.extraction_cache.clear_workflow_run(workflow_run_id)now = datetime.now(UTC)@@ -13447,6 +13563,7 @@ class WorkflowService:run_minutes_recorded_through: datetime | None = None,workflow: Workflow | None = None,lost_external_browser: bool = False,+ attempt_number: int | None = None,) -> WorkflowRun:"""Set final workflow run status based on pre-finally state.@@ -13457,6 +13574,9 @@ class WorkflowService:shielded write can land after the activity's own cancellation — an unconditionalwrite here would restore the parent while the children stay failed."""+ attempt_fence_kwargs: dict[str, Any] = (+ {"require_latest_attempt_number": attempt_number} if run_lease_tasks_enabled() else {}+ )if pre_finally_status not in (WorkflowRunStatus.canceled,WorkflowRunStatus.failed,@@ -13471,6 +13591,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,status=WorkflowRunStatus.failed,failure_reason=registration_end,+ **attempt_fence_kwargs,)if updated is None:return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)@@ -13486,6 +13607,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,status=WorkflowRunStatus.terminated,failure_reason=contract_verdict.reason,+ **attempt_fence_kwargs,)if updated is None:return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)@@ -13493,6 +13615,7 @@ class WorkflowService:updated = await self._update_workflow_run_status_if_not_final(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.completed,+ **attempt_fence_kwargs,)if updated is not None:otel_trace.get_current_span().set_attribute("task.completion_status", WorkflowRunStatus.completed)@@ -13506,6 +13629,7 @@ class WorkflowService:failure_reason=pre_finally_failure_reason,failure_category=pre_finally_failure_category,run_minutes_recorded_through=run_minutes_recorded_through,+ **attempt_fence_kwargs,)if updated is None:return await self._current_row_after_lost_finalize(workflow_run_id, workflow_run)@@ -13702,6 +13826,7 @@ class WorkflowService:run_with: str | None = None,failure_category: list[dict] | None = None,cascade_children: bool = False,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun:LOG.info(f"Marking workflow run {workflow_run_id} as failed",@@ -13729,14 +13854,26 @@ class WorkflowService:failure_category_source=failure_category_source,)- workflow_run = await self._update_workflow_run_status(- workflow_run_id=workflow_run_id,- status=WorkflowRunStatus.failed,- failure_reason=failure_reason,- run_with=run_with,- failure_category=failure_category,- )- if cascade_children:+ if run_lease_tasks_enabled() and require_latest_attempt_number is not None:+ workflow_run = await self._update_workflow_run_status_if_not_final(+ workflow_run_id=workflow_run_id,+ status=WorkflowRunStatus.failed,+ failure_reason=failure_reason,+ run_with=run_with,+ failure_category=failure_category,+ require_latest_attempt_number=require_latest_attempt_number,+ )+ if workflow_run is None:+ return await self.get_workflow_run(workflow_run_id)+ else:+ workflow_run = await self._update_workflow_run_status(+ workflow_run_id=workflow_run_id,+ status=WorkflowRunStatus.failed,+ failure_reason=failure_reason,+ run_with=run_with,+ failure_category=failure_category,+ )+ if cascade_children and not (run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id)):# Opt-in: run-level failures normally leave child rows to their own executors,# but out-of-band finalization (an interrupted worker) has no executor left to do it.try:@@ -13848,6 +13985,7 @@ class WorkflowService:cascade_children: bool = False,only_from: Sequence[WorkflowRunStatus] | None = None,worker_container_restarted: bool = False,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun | None:"""Conditional failure finalize that no-ops when the run has already reached a terminalstate. Out-of-band finalizers (the interrupted-activity backstop) run concurrently with@@ -13867,6 +14005,7 @@ class WorkflowService:failure_reason=failure_reason,failure_category=failure_category,only_from=only_from,+ require_latest_attempt_number=require_latest_attempt_number,worker_container_restarted=worker_container_restarted,)if workflow_run is None:@@ -13939,6 +14078,7 @@ class WorkflowService:failure_reason: str | None,run_with: str | None = None,failure_category: list[dict] | None = None,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun:LOG.info(f"Marking workflow run {workflow_run_id} as terminated",@@ -13974,6 +14114,7 @@ class WorkflowService:failure_reason=failure_reason,run_with=run_with,failure_category=failure_category,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)return workflow_run@@ -13982,6 +14123,7 @@ class WorkflowService:workflow_run_id: str,failure_reason: str | None,failure_category: list[dict] | None = None,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun | None:if failure_category is None:failure_category = self._classify_workflow_terminal_failure(@@ -13994,6 +14136,7 @@ class WorkflowService:status=WorkflowRunStatus.terminated,failure_reason=failure_reason,failure_category=failure_category,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)if workflow_run is None:return None@@ -14006,7 +14149,9 @@ class WorkflowService:)return workflow_run- async def mark_workflow_run_as_canceled(self, workflow_run_id: str) -> WorkflowRun:+ async def mark_workflow_run_as_canceled(+ self, workflow_run_id: str, require_latest_attempt_number: int | None = None+ ) -> WorkflowRun:"""Cancel a workflow run, rejecting the transition if the run has alreadyreached a terminal state.@@ -14023,6 +14168,7 @@ class WorkflowService:updated = await self.mark_workflow_run_as_canceled_if_not_final(workflow_run_id=workflow_run_id,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)if updated is not None:return updated@@ -14033,6 +14179,11 @@ class WorkflowService:if current is None:raise WorkflowRunNotFound(workflow_run_id)+ if run_lease_tasks_enabled() and (+ require_latest_attempt_number is not None or await run_lease_lost(workflow_run_id)+ ):+ return current+if current.status.is_final():attempt_rows = await app.DATABASE.workflow_run_attempts.get_attempts(workflow_run_id)if attempt_rows:@@ -14080,16 +14231,21 @@ class WorkflowService:self,workflow_run_id: str,failure_reason: str | None = None,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun | None:"""Conditional cancel that is a no-op when the run has already reached aterminal state. Safe to call from cancellation cleanup paths (e.g. thecopilot tool's timeout branch) that race with the run's own``_finalize_workflow_run_status`` writes."""+ if run_lease_tasks_enabled() and await run_lease_lost(workflow_run_id):+ note_run_finalized(workflow_run_id)+ return Noneupdated = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.canceled,failure_reason=failure_reason,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)if updated is None:return None@@ -14106,6 +14262,7 @@ class WorkflowService:WorkflowRunStatus.canceled,failure_reason,None,+ **_run_lease_attempt_kwargs("attempt_number", require_latest_attempt_number),)finally:await self._after_workflow_run_status_write(updated, WorkflowRunStatus.canceled)@@ -14118,6 +14275,7 @@ class WorkflowService:run_with: str | None = None,fallback_workflow_run: WorkflowRun | None = None,finalized_by: str | None = None,+ require_latest_attempt_number: int | None = None,) -> WorkflowRun:LOG.info(f"Marking workflow run {workflow_run_id} as timed out",@@ -14144,8 +14302,13 @@ class WorkflowService:run_with=run_with,failure_category=failure_category,finalized_by=finalized_by,+ **_run_lease_attempt_kwargs("require_latest_attempt_number", require_latest_attempt_number),)if updated_workflow_run is None:+ if run_lease_tasks_enabled() and (+ require_latest_attempt_number is not None or await run_lease_lost(workflow_run_id)+ ):+ return await self.get_workflow_run(workflow_run_id=workflow_run_id)# The CAS lost to a terminal writer, so only a fresh row can identify# the winner. A supplied fallback is necessarily stale here; if the# refresh fails, propagate so the timeout activity retries the repairdiff --git a/skyvern/services/orphaned_run_reaper.py b/skyvern/services/orphaned_run_reaper.pynew file mode 100644index 000000000..10b404a13--- /dev/null+++ b/skyvern/services/orphaned_run_reaper.py@@ -0,0 +1,274 @@+from datetime import timedelta+from functools import partial++import structlog++from skyvern.forge import app+from skyvern.forge.failure_classifier import WORKER_CONTAINER_RESTARTED_REASON_CODE, classify_from_failure_reason+from skyvern.forge.sdk.db.datetime_utils import naive_utc_now+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.db.repositories.run_execution_leases import RunExecutionLease+from skyvern.forge.sdk.executor.background_task_executor import BackgroundTaskExecutor+from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory+from skyvern.forge.sdk.executor.run_liveness import (+ effective_timeout_minutes,+ is_run_registered,+ owner_id,+ schedule_side_effects,+)+from skyvern.forge.sdk.models import StepStatus+from skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus+from skyvern.forge.sdk.workflow.retry_policy import get_recorded_decision+from skyvern.services import task_v2_service, uploaded_file_service++LOG = structlog.get_logger()+++async def _api_key(organization_id: str) -> str | None:+ token = await app.DATABASE.organizations.get_valid_org_auth_token(+ organization_id,+ OrganizationAuthTokenType.api.value,+ )+ if token is None:+ LOG.warning("Orphaned run webhook skipped: no API key", organization_id=organization_id)+ return None+ return token.token+++async def _workflow_side_effects(run: WorkflowRun, attempt_number: int | None, has_attempts: bool) -> None:+ if has_attempts:+ decision = await get_recorded_decision(run.workflow_run_id, attempt_number=attempt_number)+ if decision is None:+ LOG.warning("Orphaned run has no recorded retry decision", workflow_run_id=run.workflow_run_id)+ return+ if decision.retry:+ executor = AsyncExecutorFactory.get_executor()+ if isinstance(executor, BackgroundTaskExecutor):+ await executor.schedule_recorded_retry(run.workflow_run_id, decision.attempt_number)+ return+ await app.WORKFLOW_SERVICE._run_terminal_side_effects_with_retries(+ run,+ decision,+ api_key=await _api_key(run.organization_id),+ )+ return+ await uploaded_file_service.delete_files_attached_to_run(run_id=run.workflow_run_id)+ api_key = await _api_key(run.organization_id)+ if api_key:+ await app.WORKFLOW_SERVICE.execute_workflow_webhook(run, api_key, claim_kind=None)+++async def _task_v2_side_effects(run: WorkflowRun, task: TaskV2) -> None:+ try:+ task = await app.DATABASE.observer.get_task_v2(task.observer_cruise_id, task.organization_id) or task+ except Exception:+ LOG.warning("Failed to reload reaped task v2", run_id=run.workflow_run_id, exc_info=True)+ try:+ await app.DATABASE.tasks.sync_task_run_status(+ organization_id=run.organization_id,+ run_id=task.observer_cruise_id,+ status=task.status.value,+ started_at=task.started_at,+ finished_at=task.finished_at,+ )+ except Exception:+ LOG.warning("Failed to sync reaped task v2", run_id=run.workflow_run_id, exc_info=True)+ await uploaded_file_service.delete_files_attached_to_run(run_id=run.workflow_run_id)+ await task_v2_service.send_task_v2_webhook(task)+++async def _task_side_effects(task: Task) -> None:+ try:+ task = await app.DATABASE.tasks.get_task(task.task_id, task.organization_id) or task+ except Exception:+ LOG.warning("Failed to reload reaped task", run_id=task.task_id, exc_info=True)+ try:+ await app.DATABASE.tasks.sync_task_run_status(+ organization_id=task.organization_id,+ run_id=task.task_id,+ status=task.status.value,+ started_at=task.started_at,+ finished_at=task.finished_at,+ )+ except Exception:+ LOG.warning("Failed to sync reaped task", run_id=task.task_id, exc_info=True)+ await uploaded_file_service.delete_files_attached_to_run(run_id=task.task_id)+ api_key = await _api_key(task.organization_id)+ if api_key:+ await app.agent.execute_task_webhook(task, api_key)+++async def _reap_workflow(lease: RunExecutionLease, reason: str, category: list[dict] | None) -> bool:+ run = await app.DATABASE.workflow_runs.get_workflow_run(lease.run_id)+ if run is None or run.status not in (WorkflowRunStatus.running, WorkflowRunStatus.paused):+ return False+ attempts = await app.DATABASE.workflow_run_attempts.get_attempts(lease.run_id)+ if attempts and max(row.attempt_number for row in attempts) != lease.attempt_number:+ return False+ task_v2 = await app.DATABASE.observer.get_task_v2_by_workflow_run_id(lease.run_id, run.organization_id)+ if task_v2 is not None:+ won = await app.DATABASE.observer.update_task_v2_status_if_not_final(+ task_v2.observer_cruise_id,+ run.organization_id,+ TaskV2Status.failed,+ failure_category=category,+ )+ if not won:+ task_v2 = await app.DATABASE.observer.get_task_v2(task_v2.observer_cruise_id, run.organization_id)+ if task_v2 is None:+ return False+ if not task_v2.status.is_final():+ return False+ finalized = await app.DATABASE.workflow_runs.update_workflow_run_if_not_final(+ lease.run_id,+ WorkflowRunStatus(task_v2.status.value),+ only_from=[WorkflowRunStatus.running, WorkflowRunStatus.paused],+ require_latest_attempt_number=lease.attempt_number,+ )+ if finalized is None:+ return False+ else:+ task_v2 = task_v2.model_copy(+ update={+ "status": TaskV2Status.failed,+ "failure_category": category,+ "finished_at": task_v2.finished_at or naive_utc_now(),+ }+ )+ try:+ finalized = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(+ lease.run_id,+ reason,+ category,+ cascade_children=True,+ only_from=[WorkflowRunStatus.running, WorkflowRunStatus.paused],+ require_latest_attempt_number=lease.attempt_number,+ worker_container_restarted=True,+ )+ except Exception:+ finalized = await app.DATABASE.workflow_runs.get_workflow_run(lease.run_id)+ if (+ finalized is None+ or finalized.status != WorkflowRunStatus.failed+ or finalized.failure_reason != reason+ ):+ raise+ LOG.warning(+ "Reaped workflow status committed before finalizer failed", run_id=lease.run_id, exc_info=True+ )+ if finalized is None:+ return False+ schedule_side_effects(partial(_task_v2_side_effects, finalized, task_v2))+ outcome = "task_v2_webhook"+ else:+ try:+ finalized = await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(+ lease.run_id,+ reason,+ category,+ cascade_children=True,+ only_from=[WorkflowRunStatus.running, WorkflowRunStatus.paused],+ require_latest_attempt_number=lease.attempt_number,+ worker_container_restarted=True,+ )+ except Exception:+ finalized = await app.DATABASE.workflow_runs.get_workflow_run(lease.run_id)+ if finalized is None or finalized.status != WorkflowRunStatus.failed or finalized.failure_reason != reason:+ raise+ LOG.warning("Reaped workflow status committed before finalizer failed", run_id=lease.run_id, exc_info=True)+ if finalized is None:+ return False+ schedule_side_effects(partial(_workflow_side_effects, finalized, lease.attempt_number, bool(attempts)))+ outcome = "retry_decision" if attempts else "webhook"+ LOG.info(+ "Reaped orphaned workflow run",+ run_id=lease.run_id,+ organization_id=run.organization_id,+ attempt_number=lease.attempt_number,+ outcome=outcome,+ )+ return True+++async def _reap_task(lease: RunExecutionLease, reason: str) -> bool:+ task = await app.DATABASE.tasks.get_task(lease.run_id)+ if task is None or task.workflow_run_id is not None or task.status != TaskStatus.running:+ return False+ claimed = await app.DATABASE.tasks.bulk_update_tasks(+ [lease.run_id],+ status=TaskStatus.failed,+ failure_reason=reason,+ only_if_status_in=[TaskStatus.running],+ )+ if not claimed:+ return False+ task = task.model_copy(+ update={+ "status": TaskStatus.failed,+ "failure_reason": reason,+ "finished_at": task.finished_at or naive_utc_now(),+ }+ )+ try:+ await app.DATABASE.tasks.bulk_update_steps_by_task_ids(+ [lease.run_id],+ new_status=StepStatus.failed,+ only_if_status_in=[StepStatus.created, StepStatus.running],+ )+ except Exception:+ LOG.warning("Failed to finalize steps for reaped task", run_id=lease.run_id, exc_info=True)+ schedule_side_effects(partial(_task_side_effects, task))+ LOG.info(+ "Reaped orphaned task",+ run_id=lease.run_id,+ organization_id=task.organization_id,+ attempt_number=lease.attempt_number,+ outcome="webhook",+ )+ return True+++async def reap_orphaned_runs_once() -> None:+ timeout = effective_timeout_minutes()+ reason = (+ "This run stopped because the Skyvern server process that ran it exited or stopped responding "+ f"(no heartbeat for {timeout} minutes). Retry the run if needed."+ )+ category = classify_from_failure_reason(+ reason,+ exception_name=WORKER_CONTAINER_RESTARTED_REASON_CODE,+ fallback_to_unknown=True,+ )+ repository = app.DATABASE.run_execution_leases+ expired = await repository.get_expired(naive_utc_now() - timedelta(minutes=timeout), limit=500)+ reaper_id = owner_id() + ":reaper"+ for lease in expired:+ if is_run_registered(lease.run_id):+ continue+ claimed = False+ try:+ if not await repository.claim(lease.run_id, lease.owner_id, lease.heartbeat_at, reaper_id):+ continue+ claimed = True+ won = False+ if lease.run_type == "workflow_run":+ won = await _reap_workflow(lease, reason, category)+ elif lease.run_type == "task":+ won = await _reap_task(lease, reason)+ await repository.release(reaper_id, [lease.run_id])+ if not won:+ LOG.info(+ "Released stale run lease",+ run_id=lease.run_id,+ run_type=lease.run_type,+ attempt_number=lease.attempt_number,+ )+ except Exception:+ if claimed:+ try:+ await repository.unclaim(lease.run_id, reaper_id, lease.heartbeat_at)+ except Exception:+ LOG.exception("Failed to unclaim orphaned run lease", run_id=lease.run_id)+ LOG.exception("Orphaned run reaper pass failed", run_id=lease.run_id)diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 010166bab..448bf7196 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -37,6 +37,7 @@ from skyvern.exceptions import (ScriptNotFound,ScriptTerminationException,StepTerminationError,+ TaskNotFound,WorkflowRunNotFound,)from skyvern.forge import app@@ -58,6 +59,7 @@ from skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_afrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.db.enums import TaskType, is_job_recipe_workflow_run_trigger_type+from skyvern.forge.sdk.executor.run_liveness import run_lease_lost, run_lease_tasks_enabledfrom skyvern.forge.sdk.experimentation.workflow_block_engine import run_honors_chosen_enginefrom skyvern.forge.sdk.models import Step, StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfo@@ -998,6 +1000,7 @@ async def _update_workflow_block(final_output = outputif task_id:+ lease_lost = await run_lease_lost(context.workflow_run_id)if step_id:# Build step output from script actions similar to agent flowstep_output = None@@ -1016,6 +1019,30 @@ async def _update_workflow_block(errors=errors,)+ if not lease_lost:+ await app.DATABASE.tasks.update_step(+ step_id=step_id,+ task_id=task_id,+ organization_id=context.organization_id,+ status=step_status,+ is_last=is_last,+ output=step_output,+ )+ if lease_lost and not task_status.is_final():+ updated_task = await app.DATABASE.tasks.get_task(task_id, context.organization_id)+ if updated_task is None:+ raise TaskNotFound(task_id)+ else:+ updated_task = await app.DATABASE.tasks.update_task(+ task_id=task_id,+ organization_id=context.organization_id,+ status=task_status,+ failure_reason=failure_reason,+ extracted_information=output,+ errors=[error.model_dump() for error in user_defined_errors] if user_defined_errors else None,+ keep_final_status=lease_lost,+ )+ if lease_lost and step_id and updated_task.status == task_status:await app.DATABASE.tasks.update_step(step_id=step_id,task_id=task_id,@@ -1024,14 +1051,8 @@ async def _update_workflow_block(is_last=is_last,output=step_output,)- updated_task = await app.DATABASE.tasks.update_task(- task_id=task_id,- organization_id=context.organization_id,- status=task_status,- failure_reason=failure_reason,- extracted_information=output,- errors=[error.model_dump() for error in user_defined_errors] if user_defined_errors else None,- )+ if run_lease_tasks_enabled() and lease_lost:+ returndownloaded_files: list[FileInfo] = []try:async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):@@ -1095,6 +1116,8 @@ async def _update_workflow_block(# final_output is already set to `output` at line 596.pass+ if run_lease_tasks_enabled() and await run_lease_lost(context.workflow_run_id):+ returnupdated_block = await app.DATABASE.observer.update_workflow_run_block(workflow_run_block_id=workflow_run_block_id,organization_id=context.organization_id if context else None,@@ -1115,6 +1138,8 @@ async def _update_workflow_block("Failed to record cached block outcome", workflow_run_block_id=workflow_run_block_id, exc_info=True)+ if run_lease_tasks_enabled() and await run_lease_lost(context.workflow_run_id):+ returnrecorded_output_parameter = await _record_output_parameter_value(context.workflow_run_id,context.workflow_id,diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex 4bd0d53fd..c1e22f8ab 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -4,6 +4,7 @@ import copyimport jsonimport stringimport textwrap+from contextlib import nullcontextfrom datetime import UTC, datetimefrom typing import Any@@ -39,6 +40,13 @@ from skyvern.forge.sdk.core.hashing import generate_url_hashfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerType+from skyvern.forge.sdk.executor.run_liveness import (+ acquire_run_lease,+ note_run_finalized,+ run_lease_lost,+ run_lease_tasks_enabled,+ track_run,+)from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import unusable_browser_session_errorfrom skyvern.forge.sdk.schemas.task_v2 import (@@ -630,7 +638,12 @@ async def run_task_v2(# SKY-7005: scoped() restores the parent context on exit, preserving# loop_internal_state so per-iteration download filtering continues to# work for subsequent blocks in the same loop iteration.- with skyvern_context.scoped(context, propagate_captcha_timeout=True):+ with (+ skyvern_context.scoped(context, propagate_captcha_timeout=True),+ track_run(task_v2.workflow_run_id, "workflow_run") if task_v2.workflow_run_id else nullcontext(),+ ):+ if task_v2.workflow_run_id:+ await acquire_run_lease(task_v2.workflow_run_id)try:workflow, workflow_run, task_v2 = await run_task_v2_helper(organization=organization,@@ -829,6 +842,9 @@ async def run_task_v2_helper(context.browser_session_id = browser_session_idcontext.max_screenshot_scrolls = task_v2.max_screenshot_scrolls+ if await run_lease_lost(workflow_run_id):+ current = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)+ return None, None, current or task_v2task_v2 = await app.DATABASE.observer.update_task_v2(task_v2_id=task_v2_id, organization_id=organization_id, status=TaskV2Status.running)@@ -2235,6 +2251,13 @@ async def _update_task_v2_status(output: dict[str, Any] | None = None,failure_category: list[dict] | None = None,) -> TaskV2:+ keep_final_status = False+ if run_lease_tasks_enabled():+ current = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)+ if current is not None and current.workflow_run_id and await run_lease_lost(current.workflow_run_id):+ if not status.is_final():+ return await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id) or current+ keep_final_status = Truetask_v2 = await app.DATABASE.observer.update_task_v2(task_v2_id,organization_id=organization_id,@@ -2242,7 +2265,12 @@ async def _update_task_v2_status(summary=summary,output=output,failure_category=failure_category,+ keep_final_status=keep_final_status,)+ if keep_final_status and task_v2.status != status:+ if task_v2.workflow_run_id:+ note_run_finalized(task_v2.workflow_run_id)+ return task_v2if status in [TaskV2Status.completed, TaskV2Status.failed, TaskV2Status.terminated]:start_time = (task_v2.started_at.replace(tzinfo=UTC) if task_v2.started_at else task_v2.created_at.replace(tzinfo=UTC)@@ -2261,6 +2289,11 @@ async def _update_task_v2_status(return task_v2+def _task_v2_final_write_refused(task_v2: TaskV2, status: TaskV2Status) -> bool:+ # Without leases, a concurrent writer's post-commit refresh must preserve the existing side effects.+ return run_lease_tasks_enabled() and task_v2.status != status++async def mark_task_v2_as_failed(task_v2_id: str,workflow_run_id: str | None = None,@@ -2313,6 +2346,8 @@ async def mark_task_v2_as_failed(output=output,failure_category=failure_category,)+ if _task_v2_final_write_refused(task_v2, TaskV2Status.failed):+ return task_v2if workflow_run_id:await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed(workflow_run_id,@@ -2341,6 +2376,8 @@ async def mark_task_v2_as_completed(summary=summary,output=output,)+ if _task_v2_final_write_refused(task_v2, TaskV2Status.completed):+ return task_v2if workflow_run_id:await app.WORKFLOW_SERVICE.mark_workflow_run_as_completed(workflow_run_id)@@ -2373,6 +2410,8 @@ async def mark_task_v2_as_canceled(organization_id=organization_id,status=TaskV2Status.canceled,)+ if _task_v2_final_write_refused(task_v2, TaskV2Status.canceled):+ return task_v2if workflow_run_id:await app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled(workflow_run_id)@@ -2396,6 +2435,8 @@ async def mark_task_v2_as_terminated(status=TaskV2Status.terminated,failure_category=failure_category,)+ if _task_v2_final_write_refused(task_v2, TaskV2Status.terminated):+ return task_v2if workflow_run_id:await app.WORKFLOW_SERVICE.mark_workflow_run_as_terminated(workflow_run_id,@@ -2423,6 +2464,8 @@ async def mark_task_v2_as_timed_out(organization_id=organization_id,status=TaskV2Status.timed_out,)+ if _task_v2_final_write_refused(task_v2, TaskV2Status.timed_out):+ return task_v2async def repair_workflow_run() -> None:nonlocal fallback_workflow_run