Skip to content

Automate browser based workflows with AI

Delete the Temporal schedules of deleted schedules in the orphan sweep, behind a flag (SKY-18564) (#9052)

Shuchang Zheng98aeb9c4e776
diff --git a/skyvern/forge/sdk/db/repositories/schedules.py b/skyvern/forge/sdk/db/repositories/schedules.py
index fc9783996..c76800c0a 100644
--- a/skyvern/forge/sdk/db/repositories/schedules.py
+++ b/skyvern/forge/sdk/db/repositories/schedules.py
@@ -613,33 +613,6 @@ class SchedulesRepository(BaseRepository):
await session.refresh(workflow_schedule)
return convert_to_workflow_schedule(workflow_schedule, self.debug_enabled)
- @db_operation("restore_workflow_schedule")
- async def restore_workflow_schedule(
- self,
- workflow_schedule_id: str,
- organization_id: str,
- ) -> WorkflowSchedule | None:
- async with self.Session() as session:
- workflow_schedule = (
- await session.scalars(
- select(WorkflowScheduleModel)
- .filter_by(
- workflow_schedule_id=workflow_schedule_id,
- organization_id=organization_id,
- )
- .filter(WorkflowScheduleModel.deleted_at.isnot(None))
- )
- ).first()
- if not workflow_schedule:
- return None
-
- _adopt_schedule(workflow_schedule, organization_id)
- workflow_schedule.deleted_at = None
- workflow_schedule.modified_at = datetime.now(UTC)
- await session.commit()
- await session.refresh(workflow_schedule)
- return convert_to_workflow_schedule(workflow_schedule, self.debug_enabled)
-
@db_operation("list_organization_schedules")
async def list_organization_schedules(
self,
@@ -822,8 +795,8 @@ class SchedulesRepository(BaseRepository):
return schedules, total_count
@db_operation("soft_delete_orphaned_schedules")
- async def soft_delete_orphaned_schedules(self, limit: int = 500) -> list[tuple[str, str]]:
- """Soft-delete orphaned schedules and return their identities.
+ async def soft_delete_orphaned_schedules(self, limit: int = 500) -> list[tuple[str, str, str | None]]:
+ """Soft-delete orphaned schedules; returns (schedule id, workflow permanent id, backend schedule id).
Uses a single UPDATE ... RETURNING statement so orphan detection and
soft-deletion happen atomically in one DB round-trip.
@@ -856,8 +829,53 @@ class SchedulesRepository(BaseRepository):
.returning(
WorkflowScheduleModel.workflow_schedule_id,
WorkflowScheduleModel.workflow_permanent_id,
+ WorkflowScheduleModel.backend_schedule_id,
)
)
result = await session.execute(update_query)
await session.commit()
- return [(row[0], row[1]) for row in result.all()]
+ return [(row[0], row[1], row[2]) for row in result.all()]
+
+ @db_operation("list_deleted_schedules_with_backend_schedules")
+ async def list_deleted_schedules_with_backend_schedules(self, limit: int = 500) -> list[tuple[str, str]]:
+ """(schedule id, backend schedule id) of soft-deleted rows whose backend schedule is not yet known gone."""
+ async with self.Session() as session:
+ rows = await session.execute(
+ select(WorkflowScheduleModel.workflow_schedule_id, WorkflowScheduleModel.backend_schedule_id)
+ .where(
+ WorkflowScheduleModel.deleted_at.is_not(None),
+ WorkflowScheduleModel.backend_schedule_id.is_not(None),
+ )
+ .order_by(WorkflowScheduleModel.deleted_at)
+ .limit(limit)
+ )
+ return [(row[0], row[1]) for row in rows.all()]
+
+ @db_operation("get_live_backend_schedule_ids")
+ async def get_live_backend_schedule_ids(self, backend_schedule_ids: list[str]) -> set[str]:
+ if not backend_schedule_ids:
+ return set()
+ async with self.Session() as session:
+ rows = await session.scalars(
+ select(WorkflowScheduleModel.backend_schedule_id).where(
+ WorkflowScheduleModel.deleted_at.is_(None),
+ WorkflowScheduleModel.backend_schedule_id.in_(backend_schedule_ids),
+ )
+ )
+ return {backend_id for backend_id in rows.all() if backend_id}
+
+ @db_operation("clear_deleted_schedule_backend_ids")
+ async def clear_deleted_schedule_backend_ids(self, workflow_schedule_ids: list[str]) -> None:
+ """Record that these soft-deleted rows' backend schedules are gone, so the sweep stops retrying them."""
+ if not workflow_schedule_ids:
+ return
+ async with self.Session() as session:
+ await session.execute(
+ update(WorkflowScheduleModel)
+ .where(
+ WorkflowScheduleModel.workflow_schedule_id.in_(workflow_schedule_ids),
+ WorkflowScheduleModel.deleted_at.is_not(None),
+ )
+ .values({WorkflowScheduleModel.backend_schedule_id: None})
+ )
+ await session.commit()
diff --git a/skyvern/forge/sdk/routes/workflow_schedules.py b/skyvern/forge/sdk/routes/workflow_schedules.py
index c65e6da72..ab9f6a289 100644
--- a/skyvern/forge/sdk/routes/workflow_schedules.py
+++ b/skyvern/forge/sdk/routes/workflow_schedules.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from datetime import UTC, datetime, timedelta
from typing import Any, Literal
@@ -32,6 +33,7 @@ from skyvern.forge.sdk.workflow.schedules import (
validate_cron_expression,
validate_timezone_name,
)
+from skyvern.forge.sdk.workflow.service import BACKEND_SCHEDULE_DELETE_TIMEOUT_SECONDS
LOG = structlog.get_logger()
SCHEDULE_SYNC_ERROR_DETAIL = "Failed to sync schedule with scheduling service"
@@ -867,7 +869,8 @@ async def delete_workflow_schedule_route(
)
else:
try:
- await app.AGENT_FUNCTION.delete_workflow_schedule(existing.backend_schedule_id)
+ async with asyncio.timeout(BACKEND_SCHEDULE_DELETE_TIMEOUT_SECONDS):
+ await app.AGENT_FUNCTION.delete_workflow_schedule(existing.backend_schedule_id)
except Exception as e:
LOG.exception(
"Failed to delete schedule on execution backend; leaving the schedule in place",
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 65af48c3d..2dd8bdfe4 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -12073,16 +12073,18 @@ class WorkflowService:
self,
deleted_schedules: list[tuple[str, str | None]],
organization_id: str | None,
- ) -> None:
- """Best effort: a failed backend delete is logged and left for the backend's own cleanup."""
+ ) -> list[str]:
+ """Best effort: returns the schedule ids whose backend schedule is gone; a failed delete is only logged."""
targets = [(schedule_id, backend_id) for schedule_id, backend_id in deleted_schedules if backend_id]
slots = asyncio.Semaphore(BACKEND_SCHEDULE_DELETE_CONCURRENCY)
outcomes = await asyncio.gather(
*(_delete_backend_schedule(backend_id, slots) for _, backend_id in targets), return_exceptions=True
)
+ gone_schedule_ids: list[str] = []
deleted_backend_ids: list[str] = []
for (workflow_schedule_id, backend_schedule_id), outcome in zip(targets, outcomes, strict=True):
if outcome is None:
+ gone_schedule_ids.append(workflow_schedule_id)
deleted_backend_ids.append(backend_schedule_id)
continue
LOG.warning(
@@ -12099,6 +12101,7 @@ class WorkflowService:
backend_schedule_ids=deleted_backend_ids,
count=len(deleted_backend_ids),
)
+ return gone_schedule_ids
async def delete_workflow_by_id(
self,
diff --git a/tests/unit/forge/sdk/test_workflow_schedules_routes.py b/tests/unit/forge/sdk/test_workflow_schedules_routes.py
index 8e8525474..fa3b63669 100644
--- a/tests/unit/forge/sdk/test_workflow_schedules_routes.py
+++ b/tests/unit/forge/sdk/test_workflow_schedules_routes.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
@@ -549,10 +550,16 @@ async def test_mcp_tools_create_edit_cancel_and_read_back_every_one_time_state(
("attempts", "statuses", "deleted"),
[
(["fails"], [502], False),
+ (["hangs"], [502], False),
(["deletes"], [200], True),
(["deletes, then times out", "deletes"], [502, 200], True),
],
- ids=["backend-delete-fails", "backend-delete-succeeds", "retry-after-a-backend-delete-that-landed"],
+ ids=[
+ "backend-delete-fails",
+ "backend-delete-hangs",
+ "backend-delete-succeeds",
+ "retry-after-a-backend-delete-that-landed",
+ ],
)
async def test_deleting_a_schedule_deletes_its_backend_schedule_before_its_row(
schedules_client: httpx.AsyncClient,
@@ -575,12 +582,16 @@ async def test_deleting_a_schedule_deletes_its_backend_schedule_before_its_row(
attempt = next(pending)
if attempt == "fails":
raise RuntimeError("scheduler unavailable")
+ if attempt == "hangs":
+ # Outlasts the route's timeout, so only a bounded route answers before this would land.
+ await asyncio.sleep(2)
# Like the cloud backend, deleting a schedule that is already gone succeeds.
live_backend.discard(backend_schedule_id)
if attempt == "deletes, then times out":
raise TimeoutError
monkeypatch.setattr(workflow_schedule_routes.app.AGENT_FUNCTION, "delete_workflow_schedule", delete_backend)
+ monkeypatch.setattr(workflow_schedule_routes, "BACKEND_SCHEDULE_DELETE_TIMEOUT_SECONDS", 0.05)
responses = [await _request(schedules_client, NOW, "DELETE", f"/{schedule_id}") for _ in attempts]