Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(cron): a Schedule left by a deleted workflow no longer blocks Start
trohitg45c504feed8d
diff --git a/CLAUDE.md b/CLAUDE.mdindex 7409d34e..1feb9eb7 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -1096,7 +1096,7 @@ Single source of truth: [`server/services/workflow_naming.py`](./server/services**Invariants** (locked by `tests/services/test_workflow_naming.py` + `test_workflow_rename.py` — 39 tests):- First creation always gets `_1` suffix (no bare-base slugs).-- Fill-gap: deleted `AI_Assistant_2` slot is reused on next "AI Assistant" creation.+- Fill-gap: deleted `AI_Assistant_2` slot is reused on next "AI Assistant" creation, unless a live generation of another saved workflow still names its Temporal ids by it (a workflow renamed while it runs keeps the slug its Start recorded, and `list_workflow_slugs` returns those too). A cron Schedule left by a deleted workflow under a reused slug is replaced at Start (`create_cron_schedule(owner_gone=...)`), not reported as `cron_schedule_ownership_conflict`.- Renaming `AI Assistant` → `AI Assistant!` (same slug base) keeps `_1` via `exclude_id` (no self-bump).- UNIQUE constraint on `slug` is the final collision guard; `IntegrityError` indicates a race the caller should retry.- Non-ASCII names transliterate via `text-unidecode` ("日本語" → "Ri_Ben_Yu"); fall back to `Workflow_N` only when slug is empty after sanitize.diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.mdindex dd3b2ca5..d907cc70 100644--- a/docs-internal/temporal-workflow-control.md+++ b/docs-internal/temporal-workflow-control.md@@ -201,7 +201,10 @@ the control plane is hardened against Temporal's per-run event-history ceilingstarts use `USE_EXISTING`, and cron schedule creation preserves server-ownedpause state. Listener and Schedule ids use the slug the generation's Startrecorded in its snapshot (`workflow_slug`), not the saved one, so a rename- while it runs cannot give the re-arm a second set of ids.+ while it runs cannot give the re-arm a second set of ids. That slug stays+ taken while the generation lives (`next_available_slug` skips it), and a+ Schedule whose workflow was deleted is replaced by the next workflow given+ its id instead of failing that workflow's Start.- **No lifetime caps.** Spawned graph runs, agent children, and delegated-taskrunners no longer carry 1-2h execution/run timeouts (Temporal's timers keepticking through a pause, so the caps silently terminated paused work).diff --git a/server/core/database.py b/server/core/database.pyindex fce71b34..9aa10709 100644--- a/server/core/database.py+++ b/server/core/database.py@@ -994,18 +994,35 @@ class Database:return []async def list_workflow_slugs(self) -> List[tuple]:- """Cheap projection of ``(id, slug)`` pairs.+ """``(id, slug)`` pairs of the slugs in use: each saved workflow's,+ and the one each live generation of a saved workflow names its+ Temporal ids by (``graph_snapshot.workflow_slug``). A rename leaves+ the latter as it was, and handed to another workflow it would give+ both the same cron Schedule id.Consumed by :func:`services.workflow_naming.next_available_slug`to find the lowest free ``_<N>`` suffix for a given slug base.Returns an empty list on error so the slug allocator fallsthrough to ``_1``."""+ from models.database import WORKFLOW_CONTROL_ACTIVE_STATES+try:async with self.get_session() as session:stmt = select(Workflow.id, Workflow.slug)result = await session.execute(stmt)- return list(result.all())+ rows = list(result.all())+ live = await session.execute(+ select(WorkflowControlExecution.workflow_id, WorkflowControlExecution.graph_snapshot).where(+ WorkflowControlExecution.status.in_(WORKFLOW_CONTROL_ACTIVE_STATES),+ WorkflowControlExecution.workflow_id.in_(select(Workflow.id)),+ )+ )+ for workflow_id, snapshot in live.all():+ slug = snapshot.get("workflow_slug") if isinstance(snapshot, dict) else None+ if isinstance(slug, str) and slug:+ rows.append((workflow_id, slug))+ return rowsexcept Exception as e:logger.error("Failed to list workflow slugs", error=str(e))return []diff --git a/server/services/deployment/manager.py b/server/services/deployment/manager.pyindex e33da6ad..ce00dd4b 100644--- a/server/services/deployment/manager.py+++ b/server/services/deployment/manager.py@@ -1062,6 +1062,7 @@ class DeploymentManager:cron_expression=cron_expr,timezone=timezone,listener_data=listener_data,+ owner_gone=self._workflow_deleted,)logger.info(@@ -1073,6 +1074,11 @@ class DeploymentManager:)return schedule_id+ async def _workflow_deleted(self, workflow_id: str) -> bool:+ """Whether a workflow's saved row is gone, so a Schedule it left is+ stale (``create_cron_schedule`` replaces it)."""+ return await self.database.get_workflow(workflow_id) is None+async def _cancel_canary_listeners(self, workflow_id: str) -> int:"""Cancel all canary listeners for this deployment.diff --git a/server/services/temporal/schedules.py b/server/services/temporal/schedules.pyindex 9b23287f..789fd79d 100644--- a/server/services/temporal/schedules.py+++ b/server/services/temporal/schedules.py@@ -22,7 +22,7 @@ from __future__ import annotationsfrom dataclasses import replacefrom datetime import timedelta-from typing import Any, Dict, List, Tuple+from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuplefrom temporalio.client import (Client,@@ -128,6 +128,14 @@ def _last_day_of_month(minute: int, hour: int) -> List[ScheduleCalendarSpec]:]+class _OwnedElsewhere(RuntimeError):+ """The Schedule id is held by another workflow's Schedule."""++ def __init__(self, schedule_id: str, owner: str) -> None:+ super().__init__(f"cron_schedule_ownership_conflict:{schedule_id}")+ self.owner = owner++async def create_cron_schedule(client: Client,*,@@ -140,6 +148,7 @@ async def create_cron_schedule(listener_data: Dict[str, Any],task_queue: str = "machina-tasks",overlap_policy: ScheduleOverlapPolicy = ScheduleOverlapPolicy.SKIP,+ owner_gone: Optional[Callable[[str], Awaitable[bool]]] = None,) -> str:"""Create a Temporal Schedule for a cron trigger, or update the one there.@@ -148,7 +157,10 @@ async def create_cron_schedule(boot re-arm), Temporal raises :exc:`ScheduleAlreadyRunningError` and theSchedule is updated in place: new spec, action args and SearchAttributes, same paused state. One owned by another workflow raises- ``cron_schedule_ownership_conflict`` instead. An updated ``CRON_ONCE``+ ``cron_schedule_ownership_conflict`` instead, unless ``owner_gone`` says+ that workflow was deleted: its Schedule is then left over (a workflow+ deleted before deleting stopped it first), and since slugs are handed+ out again, it is replaced. An updated ``CRON_ONCE``Schedule does not run again, since ``trigger_immediately`` applies onlywhen the Schedule is created.@@ -175,6 +187,8 @@ async def create_cron_schedule(overlap_policy: How concurrent firings interact. ``SKIP`` (default)drops a firing if the prior run is still going; mirrors thepre-Wave-12 APScheduler behaviour for slow workflows.+ owner_gone: Whether the workflow an existing Schedule names was+ deleted (the deployment manager asks its database)."""schedule_id = cron_schedule_id(workflow_slug, trigger_label)action_workflow_id = cron_action_workflow_id(workflow_slug, trigger_label)@@ -267,10 +281,7 @@ async def create_cron_schedule():existing_owner = existing_args[0].get("workflow_id")if str(existing_owner or "") != str(workflow_id):- raise RuntimeError(- "cron_schedule_ownership_conflict:"- f"{schedule_id}"- )+ raise _OwnedElsewhere(schedule_id, str(existing_owner or ""))updated_schedule = replace(schedule,state=description.schedule.state,@@ -280,7 +291,25 @@ async def create_cron_schedule(search_attributes=schedule_search_attributes,)- await handle.update(update_existing)+ try:+ await handle.update(update_existing)+ except _OwnedElsewhere as taken:+ if owner_gone is None or not taken.owner or not await owner_gone(taken.owner):+ raise+ await handle.delete()+ await client.create_schedule(+ schedule_id,+ schedule,+ search_attributes=schedule_search_attributes,+ trigger_immediately=cron_expression == CRON_ONCE,+ )+ logger.warning(+ "Replaced a cron Schedule left by a deleted workflow",+ schedule_id=schedule_id,+ previous_owner=taken.owner,+ workflow_id=workflow_id,+ )+ return schedule_idlogger.info("Updated existing Temporal cron Schedule",schedule_id=schedule_id,diff --git a/server/tests/services/test_workflow_control.py b/server/tests/services/test_workflow_control.pyindex 461ecc94..092e06fd 100644--- a/server/tests/services/test_workflow_control.py+++ b/server/tests/services/test_workflow_control.py@@ -80,6 +80,23 @@ async def test_generation_records_the_slug_its_temporal_ids_are_named_by():assert control.graph_snapshot["workflow_slug"] is None+async def test_a_live_generations_slug_is_not_handed_out_again(control_database):+ """Renamed while it runs, a workflow keeps the slug its Start recorded:+ another workflow given that slug would collide with its cron Schedule."""+ from services.workflow_naming import next_available_slug++ await control_database.save_workflow("wf-a", "Maya", "Maya_1", {"nodes": [], "edges": []})+ await WorkflowControlService(control_database).begin_generation(+ workflow_id="wf-a", nodes=[], edges=[], session_id="s", idempotency_key="start-a"+ )+ assert await control_database.rename_workflow("wf-a", "Maya Ortiz", "Maya_Ortiz_1")++ assert await next_available_slug("Maya", control_database) == "Maya_2"+ # The workflow itself may take its own slug back.+ assert await next_available_slug("Maya", control_database, exclude_id="wf-a") == "Maya_1"++@pytest.mark.asyncioasync def test_generation_atomically_creates_and_archives_isolated_data_scope(control_database):service = WorkflowControlService(control_database)diff --git a/server/tests/test_cron_canary.py b/server/tests/test_cron_canary.pyindex 548a3843..931dc8c4 100644--- a/server/tests/test_cron_canary.py+++ b/server/tests/test_cron_canary.py@@ -247,6 +247,54 @@ class TestCreateCronSchedule:))+ @pytest.mark.asyncio+ async def test_a_schedule_left_by_a_deleted_workflow_is_replaced(self):+ """A workflow deleted before deleting stopped it first left its+ Schedule, and a new workflow can get the same slug: its Start+ replaces the leftover instead of failing on every try."""+ from temporalio.client import ScheduleAlreadyRunningError++ from services.temporal.schedules import create_cron_schedule++ async def owned_by_six(updater):+ updater(+ types.SimpleNamespace(+ description=types.SimpleNamespace(+ schedule=types.SimpleNamespace(state=None, action=types.SimpleNamespace(args=())),+ typed_search_attributes=[+ types.SimpleNamespace(key=types.SimpleNamespace(name="EventWorkflowId"), value="6"),+ ],+ )+ )+ )++ args = dict(+ workflow_id="14",+ workflow_slug="Maya_1",+ node_id="cron-1",+ trigger_label="Schedule",+ cron_expression="0 * * * *",+ timezone="UTC",+ listener_data={},+ )+ client = MagicMock()+ client.create_schedule = AsyncMock(side_effect=[ScheduleAlreadyRunningError(), None])+ delete = AsyncMock()+ client.get_schedule_handle.return_value = MagicMock(update=owned_by_six, delete=delete)+ gone = AsyncMock(return_value=True)++ assert await create_cron_schedule(client, owner_gone=gone, **args) == "Maya_1-Schedule"+ gone.assert_awaited_once_with("6")+ delete.assert_awaited_once()+ assert client.create_schedule.await_count == 2++ # A workflow that still exists keeps its Schedule.+ client.create_schedule = AsyncMock(side_effect=ScheduleAlreadyRunningError())+ delete.reset_mock()+ with pytest.raises(RuntimeError, match="cron_schedule_ownership_conflict:Maya_1-Schedule"):+ await create_cron_schedule(client, owner_gone=AsyncMock(return_value=False), **args)+ delete.assert_not_awaited()+@pytest.mark.asyncioasync def test_cron_workflow_pause_resume_gate(self, monkeypatch):from nodes.scheduler.cron_scheduler import _workflow as cron_workflow@@ -767,6 +815,7 @@ class TestDeploymentCronCanaryRouting:"cron_expression": cron_expression,"timezone": timezone,"listener_data": listener_data,+ "owner_gone": kw.get("owner_gone"),})return f"{workflow_slug}-{trigger_label}"@@ -807,6 +856,11 @@ class TestDeploymentCronCanaryRouting:assert ld["trigger_node_id"] == "cron-1"assert ld["cron_expression"] == "*/5 * * * *"assert ld["schedule"] == "Every 5 minutes"+ # A Schedule another workflow left is stale only once that workflow is deleted.+ mgr.database.get_workflow = AsyncMock(return_value=None)+ assert await call["owner_gone"]("6") is True+ mgr.database.get_workflow = AsyncMock(return_value=MagicMock(slug="Maya_1"))+ assert await call["owner_gone"]("6") is False@pytest.mark.asyncioasync def test_start_returns_none_when_temporal_not_connected(self, monkeypatch):