Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(storage): stop a workflow before deleting it, and never write it back
trohitg6944d33cfd40
diff --git a/docs-internal/temporal-workflow-control.md b/docs-internal/temporal-workflow-control.mdindex d0774945..435debc4 100644--- a/docs-internal/temporal-workflow-control.md+++ b/docs-internal/temporal-workflow-control.md@@ -89,6 +89,14 @@ controller, and empty runtime namespace from the then-current saved canvas.Historical scopes therefore remain queryable without leaking state into thenew run.+Deleting a workflow uses Reset as its shutdown barrier+([deletion.py](../server/services/workflow_storage/deletion.py)): a generation+that is not `reset`, or a Workspace controller that exists, is Reset with the+latest control revision before the graph is removed, and a legacy local+deployment is cancelled. If shutdown fails, or a Start was admitted meanwhile,+the delete is refused (`workflow_shutdown_failed`) and the graph stays. A+workflow that never ran is deleted without connecting to Temporal.+Reset quiesces every producer before its final strict Visibility sweep: localadmission closes synchronously, the controller is told to close, cron Schedulesare deleted, and legacy local resources are cancelled. `EventWorkflowId` isdiff --git a/docs-internal/workflow_ops_protocol.md b/docs-internal/workflow_ops_protocol.mdindex 22d73046..b83015a8 100644--- a/docs-internal/workflow_ops_protocol.md+++ b/docs-internal/workflow_ops_protocol.md@@ -108,6 +108,9 @@ Unlike `save_workflow` this never replaces the graph, so it cannot drop whatan editor saved meanwhile. An editor holding unsaved changes made before thepush can still overwrite it on its next save (saves carry no revision check);adopting the batch into the editor's working copy narrows that window.+Such a save cannot bring back a workflow deleted meanwhile: saving an+existing workflow is an update only (`require_existing`), so it fails with+`workflow_not_found`.## Backend usagediff --git a/server/core/database.py b/server/core/database.pyindex 4a4f3e1b..ba954afb 100644--- a/server/core/database.py+++ b/server/core/database.py@@ -743,6 +743,8 @@ class Database:data: Dict[str, Any],description: Optional[str] = None,context_id_aliases: Optional[Dict[str, str]] = None,+ *,+ require_existing: bool = False,) -> bool:"""Save or update workflow.@@ -753,6 +755,11 @@ class Database::func:`services.workflow_naming.next_available_slug` beforecalling this method — the unique constraint on the column isthe final check against collision.++ ``require_existing`` makes this an update only: an editor save or+ read-time migration must not recreate a workflow deleted since its+ caller read the graph. The UPDATE checks the affected row count in+ the same transaction as the Context archive intents."""try:async with self.get_session() as session:@@ -762,11 +769,16 @@ class Database:previous_context_ids = _workflow_context_node_ids(existing.data if existing is not None else {})next_context_ids = _workflow_context_node_ids(data)- if existing:- existing.name = name- existing.slug = slug- existing.description = description- existing.data = data+ if existing is not None:+ updated = await session.execute(+ update(Workflow)+ .where(Workflow.id == workflow_id)+ .values(name=name, slug=slug, description=description, data=data)+ )+ if updated.rowcount != 1:+ return False+ elif require_existing:+ return Falseelse:existing = Workflow(id=workflow_id, name=name, slug=slug, description=description, data=data)session.add(existing)diff --git a/server/nodes/agent/vertex_managed_agent/_ops.py b/server/nodes/agent/vertex_managed_agent/_ops.pyindex 6065574c..f80ee2c7 100644--- a/server/nodes/agent/vertex_managed_agent/_ops.py+++ b/server/nodes/agent/vertex_managed_agent/_ops.py@@ -184,13 +184,17 @@ async def ensure_cloud_tool_nodes(data["edges"] = edges# Persist-then-broadcast: the DB write must land before any# consumer re-reads the canvas (agentBuilder invariant).- await database.save_workflow(+ saved = await database.save_workflow(workflow_id=workflow_id,name=workflow.name,slug=workflow.slug,data=data,description=workflow.description,+ require_existing=True,)+ if not saved:+ logger.warning("[Vertex Agent] cloud-tool graph save failed for %s", workflow_id)+ return {}await broadcaster.broadcast({"type": _WIRE_ROUTING_KEY,diff --git a/server/services/workflow_storage/deletion.py b/server/services/workflow_storage/deletion.pynew file mode 100644index 00000000..d4b043c3--- /dev/null+++ b/server/services/workflow_storage/deletion.py@@ -0,0 +1,87 @@+"""Stop an existing workflow runtime before its saved graph is deleted.++Deletion reuses the control plane's Reset barrier; it must not remove the+graph after a failed shutdown. This module neither deletes data nor starts+a runtime for an employee that has never run.+"""++from __future__ import annotations++from typing import Any, Dict, Optional++from core.logging import get_logger++logger = get_logger(__name__)+++def _failure(workflow_id: str, detail: str) -> Dict[str, Any]:+ return {+ "success": False,+ "workflow_id": workflow_id,+ "error": "workflow_shutdown_failed",+ "detail": detail,+ }+++async def stop_workflow_for_deletion(database: Any, workflow_id: str) -> Optional[Dict[str, Any]]:+ """Return a failure envelope, or ``None`` when deletion can proceed.++ Call this before ``database.delete_workflow`` and return any failure+ unchanged. Reset owns generation cancellation, cron removal, workspace+ task cleanup and retries of an interrupted ``resetting`` operation.+ Merely having a workspace-capable node does not mean tasks have run:+ with no live generation, only an existing workspace controller needs+ Reset. Looking it up uses an already-connected client, never connects+ or launches Temporal.+ """+ try:+ from core.container import container+ from services.deployment.handlers import handle_cancel_deployment, handle_reset_workflow+ from services.node_invocations import controller_status++ control = await database.get_latest_workflow_control(workflow_id)+ needs_reset = control is not None and control.status != "reset"+ if not needs_reset:+ wrapper = container.temporal_client()+ client = wrapper.client if wrapper is not None else None+ if client is not None:+ needs_reset = await controller_status(workflow_id, client=client) is not None++ if needs_reset:+ # The shared Reset handler obtains its DB from DI. Never silently+ # reset a different database from the one the caller will delete.+ if container.database() is not database:+ return _failure(workflow_id, "workflow_database_mismatch")+ reset = await handle_reset_workflow(+ {"workflow_id": workflow_id, "expected_revision": control.revision if control else 0},+ None,+ )+ if not reset.get("success"):+ return _failure(workflow_id, str(reset.get("error") or "workflow_reset_failed"))++ # Legacy deployments have no durable control row. Reset already+ # tears down controlled deployments; this also catches local-only+ # deployments without starting one for an untouched employee.+ status = container.workflow_service().get_deployment_status(workflow_id)+ if status.get("deployed") or status.get("active_runs", 0):+ if container.database() is not database:+ return _failure(workflow_id, "workflow_database_mismatch")+ cancelled = await handle_cancel_deployment({"workflow_id": workflow_id}, None)+ if not cancelled.get("success"):+ return _failure(+ workflow_id,+ str(cancelled.get("error") or cancelled.get("message") or "workflow_cancel_failed"),+ )++ # A Start admitted while shutdown yielded must not be deleted as if+ # the generation we just stopped were still the latest one.+ latest = await database.get_latest_workflow_control(workflow_id)+ if latest is not None and latest.status != "reset":+ return _failure(workflow_id, "control_revision_conflict")+ return None+ except Exception as exc:+ logger.warning("Workflow shutdown before deletion failed", workflow_id=workflow_id, exc_info=True)+ return _failure(workflow_id, str(exc) or type(exc).__name__)+++__all__ = ["stop_workflow_for_deletion"]diff --git a/server/services/workflow_storage/handlers.py b/server/services/workflow_storage/handlers.pyindex afc89270..ca20a8fa 100644--- a/server/services/workflow_storage/handlers.py+++ b/server/services/workflow_storage/handlers.py@@ -169,6 +169,16 @@ async def _broadcast_renamed(workflow_id: str, name: str, slug: str, old_slug: sawait _broadcast_lifecycle("renamed", workflow_id, name=name, slug=slug, old_slug=old_slug)+async def _failed_workflow_save(database: Any, workflow_id: str, *, require_existing: bool) -> Dict[str, Any]:+ """Report a vanished workflow separately from a storage failure."""+ missing = require_existing and await database.get_workflow(workflow_id) is None+ return {+ "success": False,+ "error": "workflow_not_found" if missing else "save_failed",+ "workflow_id": workflow_id,+ }++async def handle_save_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:"""Save workflow. Re-slugs + renames workspace dir when name changes.@@ -191,7 +201,7 @@ async def handle_save_workflow(data: Dict[str, Any], websocket: WebSocket) -> Diif is_new_marker:workflow_id = await database.allocate_workflow_id()elif existing is None:- return {"success": False, "error": "workflow_not_found"}+ return {"success": False, "error": "workflow_not_found", "workflow_id": requested_id}else:workflow_id = requested_idname = data["name"]@@ -261,41 +271,41 @@ async def handle_save_workflow(data: Dict[str, Any], websocket: WebSocket) -> Di}if _supports_context_archive_outbox(database):save_kwargs["context_id_aliases"] = normalization.aliases+ if existing is not None:+ save_kwargs["require_existing"] = Truesuccess = await database.save_workflow(**save_kwargs,)- if success:- await persist_parameter_aliases(+ if not success:+ return await _failed_workflow_save(database, workflow_id, require_existing=existing is not None)+ await persist_parameter_aliases(+ database,+ aliases=normalization.aliases,+ parameters=normalization.node_parameters,+ )+ if _supports_context_archive_outbox(database):+ context_archives_completed, context_archives_pending = await _drain_context_archive_outbox(database,- aliases=normalization.aliases,- parameters=normalization.node_parameters,+ workflow_id,)- if _supports_context_archive_outbox(database):- context_archives_completed, context_archives_pending = await _drain_context_archive_outbox(- database,- workflow_id,- )- else:- # Compatibility for isolated test doubles and older storage- # implementations. Production Database commits the outbox in the- # same transaction as the graph.- context_archives_completed = await archive_removed_contexts(- database,- workflow_id=workflow_id,- previous_nodes=((existing.data or {}).get("nodes") or [] if existing is not None else []),- normalized_nodes=normalization.nodes,- aliases=normalization.aliases,- )- context_archives_pending = 0- notify_graph_changed(workflow_id)else:- context_archives_completed = 0+ # Compatibility for isolated test doubles and older storage+ # implementations. Production Database commits the outbox in the+ # same transaction as the graph.+ context_archives_completed = await archive_removed_contexts(+ database,+ workflow_id=workflow_id,+ previous_nodes=((existing.data or {}).get("nodes") or [] if existing is not None else []),+ normalized_nodes=normalization.nodes,+ aliases=normalization.aliases,+ )context_archives_pending = 0+ notify_graph_changed(workflow_id)if existing and existing.slug and existing.slug != slug:_move_workspace(existing.slug, slug)await _broadcast_renamed(workflow_id, name, slug, existing.slug)- elif success and existing is None:+ elif existing is None:await _broadcast_lifecycle("created", workflow_id, name=name, slug=slug)return {@@ -389,6 +399,7 @@ async def handle_get_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dicif normalized_data != workflow_data:save_kwargs = {"workflow_id": workflow.id,+ "require_existing": True,"name": workflow.name,"slug": workflow.slug,"description": getattr(@@ -405,6 +416,8 @@ async def handle_get_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dicnormalization_persisted = (await database.save_workflow(**save_kwargs))+ if not normalization_persisted:+ return await _failed_workflow_save(database, workflow.id, require_existing=True)if normalization_persisted and _supports_context_archive_outbox(database):drained, pending_archives = await _drain_context_archive_outbox(database,@@ -419,10 +432,6 @@ async def handle_get_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dicnormalized_nodes=normalization.nodes,aliases=normalization.aliases,)- else:- # Keep the response aligned with the authoritative row.- # A later read retries normalization and outbox creation.- normalized_data = sanitize_workflow_graph(workflow_data)if normalization_persisted:await persist_parameter_aliases(database,@@ -445,7 +454,7 @@ async def handle_get_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dic"context_archives_completed": recovered_archives,"context_archives_pending": pending_archives,}- return {"success": False, "error": "Workflow not found"}+ return {"success": False, "error": "workflow_not_found", "workflow_id": workflow_id}async def handle_get_all_workflows(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:@@ -495,7 +504,13 @@ async def delete_workflow_with_context_archival(database: Any,workflow_id: str,) -> Dict[str, Any]:- """Delete a workflow through the durable Context lifecycle boundary."""+ """Stop runtime, then delete through the durable Context lifecycle boundary."""++ from services.workflow_storage.deletion import stop_workflow_for_deletion++ failure = await stop_workflow_for_deletion(database, workflow_id)+ if failure is not None:+ return failureif _supports_context_archive_outbox(database):# Production Database commits graph deletion and archive identities in@@ -530,7 +545,7 @@ async def delete_workflow_with_context_archival(async def handle_delete_workflow(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:- """Archive durable Context threads, then delete their workflow graph."""+ """Stop the workflow, archive its durable Context threads, then delete the graph."""database = container.database()workflow_id = str(data["workflow_id"])return await delete_workflow_with_context_archival(diff --git a/server/tests/nodes/test_vertex_agents.py b/server/tests/nodes/test_vertex_agents.pyindex f7d62566..31cda5e1 100644--- a/server/tests/nodes/test_vertex_agents.py+++ b/server/tests/nodes/test_vertex_agents.py@@ -1218,6 +1218,7 @@ class TestCloudToolMinting:# Persisted node + edge with the minted id.save_kwargs = database.save_workflow.await_args.kwargs+ assert save_kwargs["require_existing"] is Truepersisted_nodes = save_kwargs["data"]["nodes"]persisted_edges = save_kwargs["data"]["edges"]assert persisted_nodes[0]["id"] == minted_id@@ -1241,6 +1242,28 @@ class TestCloudToolMinting:# Pulsing is the caller's concern now — mint emits no node_status.broadcaster.update_node_status.assert_not_awaited()+ async def test_failed_graph_save_returns_no_nodes_and_does_not_broadcast(self):+ from nodes.agent.vertex_managed_agent import _ops++ database = MagicMock()+ database.get_workflow = AsyncMock(return_value=self._workflow())+ database.save_workflow = AsyncMock(return_value=False)+ database.save_node_parameters = AsyncMock(return_value=True)+ broadcaster = MagicMock()+ broadcaster.broadcast = AsyncMock()++ with (+ patch.object(_ops, "get_database", return_value=database),+ patch.object(_ops, "get_status_broadcaster", return_value=broadcaster),+ ):+ resolved = await _ops.ensure_cloud_tool_nodes(+ workflow_id="wf-1", agent_node_id="vx-1", used={"fn:run_command": "run_command"}+ )++ assert resolved == {}+ assert database.save_workflow.await_args.kwargs["require_existing"] is True+ broadcaster.broadcast.assert_not_awaited()+async def test_pulse_node_broadcasts_status(self):from nodes.agent.vertex_managed_agent import _opsdiff --git a/server/tests/services/employees/test_list_employees.py b/server/tests/services/employees/test_list_employees.pyindex 0dd7cf03..57b64f33 100644--- a/server/tests/services/employees/test_list_employees.py+++ b/server/tests/services/employees/test_list_employees.py@@ -263,8 +263,11 @@ async def test_list_is_newest_first_and_includes_every_workflow(real_database):async def test_deleting_the_workflow_removes_the_employee(real_database, monkeypatch):import services.employees as employees_package+ from unittest.mock import AsyncMock+ from services.workflow_storage import deletionfrom services.workflow_storage.handlers import delete_workflow_with_context_archival+ monkeypatch.setattr(deletion, "stop_workflow_for_deletion", AsyncMock(return_value=None))frames = []async def capture(stage, **kwargs):diff --git a/server/tests/services/test_workflow_context_archive_outbox.py b/server/tests/services/test_workflow_context_archive_outbox.pyindex ce8da859..4c8741de 100644--- a/server/tests/services/test_workflow_context_archive_outbox.py+++ b/server/tests/services/test_workflow_context_archive_outbox.py@@ -5,6 +5,7 @@ import sysimport uuidfrom pathlib import Pathfrom types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMockimport pytest@@ -17,7 +18,11 @@ from services.agent_context import (@pytest.fixture-async def outbox_database():+async def outbox_database(monkeypatch):+ from services.workflow_storage import deletion++ # These tests own only persistence; no workflow runtime is started.+ monkeypatch.setattr(deletion, "stop_workflow_for_deletion", AsyncMock(return_value=None))# The root test configuration replaces core.database for fast contract# tests. Load the real implementation privately for transaction coverage.module_name = f"tests._real_workflow_outbox_database_{uuid.uuid4().hex}"@@ -78,8 +83,10 @@ def _graph(*, with_context: bool) -> dict:@pytest.mark.asyncio[email protected]("require_existing", [False, True])async def test_graph_save_and_archive_intent_commit_atomically(outbox_database,+ require_existing,):database = outbox_databaseassert await database.save_workflow(@@ -102,6 +109,7 @@ async def test_graph_save_and_archive_intent_commit_atomically("First","second",_graph(with_context=False),+ require_existing=require_existing,)persisted = await database.get_workflow("1")assert persisted is not None@@ -113,12 +121,88 @@ async def test_graph_save_and_archive_intent_commit_atomically("First","first",_graph(with_context=False),+ require_existing=require_existing,)pending = await database.list_workflow_context_archive_outbox("1")assert len(pending) == 1assert pending[0]["context_node_id"] == "ctx"+async def test_update_existing_does_not_insert_a_deleted_workflow(outbox_database):+ database = outbox_database+ assert await database.save_workflow("1", "Workflow", "workflow", _graph(with_context=False))+ assert await database.delete_workflow("1")++ assert not await database.save_workflow(+ "1", "Stale workflow", "stale_workflow", _graph(with_context=False), require_existing=True+ )+ assert await database.get_workflow("1") is None+ assert await database.list_workflow_context_archive_outbox("1") == []++[email protected]("operation", ["save", "get"])[email protected]("deleted", [True, False], ids=["deleted-during-request", "storage-failure"])+async def test_failed_inflight_workflow_write_does_not_recreate_or_announce(+ outbox_database, monkeypatch, operation, deleted+):+ from services import workflow_context_migration+ from services.workflow_storage import handlers++ database = outbox_database+ # An old graph version makes get_workflow persist normalization, too.+ assert await database.save_workflow("1", "Maya", "Maya_1", {"nodes": [], "edges": []})+ monkeypatch.setattr(handlers.container, "database", lambda: database)++ async def interleave_delete(*args, **kwargs):+ # Both handlers have already read the workflow before this await.+ if deleted:+ assert await database.delete_workflow("1")+ return []++ monkeypatch.setattr(handlers, "_context_topology_errors", interleave_delete)+ if not deleted:+ monkeypatch.setattr(database, "save_workflow", AsyncMock(return_value=False))++ persist_parameters = AsyncMock()+ archive_contexts = AsyncMock()+ graph_changed = MagicMock()+ move_workspace = MagicMock()+ broadcast = AsyncMock()+ monkeypatch.setattr(workflow_context_migration, "persist_parameter_aliases", persist_parameters)+ monkeypatch.setattr(workflow_context_migration, "archive_removed_contexts", archive_contexts)+ monkeypatch.setattr(handlers, "notify_graph_changed", graph_changed)+ monkeypatch.setattr(handlers, "_move_workspace", move_workspace)+ monkeypatch.setattr(handlers, "_broadcast_lifecycle", broadcast)++ if operation == "save":+ result = await handlers.handle_save_workflow(+ {"workflow_id": "1", "name": "Untitled Workflow", "data": {"nodes": [], "edges": []}},+ websocket=None,+ )+ else:+ result = await handlers.handle_get_workflow({"workflow_id": "1"}, websocket=None)++ assert result == {+ "success": False,+ "error": "workflow_not_found" if deleted else "save_failed",+ "workflow_id": "1",+ }+ persisted = await database.get_workflow("1")+ if deleted:+ assert persisted is None+ assert await database.get_all_workflows() == []+ else:+ assert persisted.name == "Maya"+ assert persisted.data == {"nodes": [], "edges": []}+ persist_parameters.assert_not_awaited()+ archive_contexts.assert_not_awaited()+ graph_changed.assert_not_called()+ move_workspace.assert_not_called()+ broadcast.assert_not_awaited()++@pytest.mark.asyncioasync def test_context_id_canonicalization_is_not_a_deletion(outbox_database,diff --git a/server/tests/services/test_workflow_deletion_shutdown.py b/server/tests/services/test_workflow_deletion_shutdown.pynew file mode 100644index 00000000..afcf1107--- /dev/null+++ b/server/tests/services/test_workflow_deletion_shutdown.py@@ -0,0 +1,215 @@+"""Deletion's precondition delegates shutdown to the existing control plane."""++from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock++import pytest++import core.container as container_module+from services.deployment import handlers as deployment+from services import node_invocations+from services.workflow_storage.deletion import stop_workflow_for_deletion+++def runtime(monkeypatch):+ state = SimpleNamespace(control=None, calls=[])++ async def latest(_workflow_id):+ return state.control++ async def reset(_data, _socket):+ state.calls.append("reset")+ if state.control is not None:+ state.control = SimpleNamespace(status="reset", revision=state.control.revision + 1)+ state.status["deployed"] = False+ return {"success": True}++ async def cancel(_data, _socket):+ state.calls.append("cancel")+ return {"success": True}++ state.database = SimpleNamespace(get_latest_workflow_control=AsyncMock(side_effect=latest), delete_workflow=AsyncMock())+ state.status = {"deployed": False, "active_runs": 0}+ state.client = SimpleNamespace(client=None)+ state.service = SimpleNamespace(get_deployment_status=Mock(side_effect=lambda _: state.status))+ state.container = SimpleNamespace(+ database=lambda: state.database, temporal_client=lambda: state.client, workflow_service=lambda: state.service,+ )+ state.reset = AsyncMock(side_effect=reset)+ state.cancel = AsyncMock(side_effect=cancel)+ state.workspace = AsyncMock(return_value=None)+ monkeypatch.setattr(container_module, "container", state.container)+ monkeypatch.setattr(deployment, "handle_reset_workflow", state.reset)+ monkeypatch.setattr(deployment, "handle_cancel_deployment", state.cancel)+ monkeypatch.setattr(node_invocations, "controller_status", state.workspace)+ return state+++async def test_untouched_hire_needs_no_runtime_reset_or_connection(runtime):+ assert await stop_workflow_for_deletion(runtime.database, "7") is None+ runtime.reset.assert_not_awaited()+ runtime.cancel.assert_not_awaited()+ runtime.workspace.assert_not_awaited()+ runtime.database.delete_workflow.assert_not_awaited()++[email protected]("status", ["starting", "running", "pausing", "paused", "resuming", "failed", "resetting"])+async def test_live_or_interrupted_generation_uses_reset_barrier_with_latest_revision(runtime, status):+ runtime.control = SimpleNamespace(status=status, revision=9)+ assert await stop_workflow_for_deletion(runtime.database, "7") is None+ runtime.reset.assert_awaited_once_with({"workflow_id": "7", "expected_revision": 9}, None)+ runtime.database.delete_workflow.assert_not_awaited()++[email protected]("resetting", [False, True])+async def test_existing_workspace_controller_resets_even_without_a_generation(runtime, resetting):+ runtime.client.client = object()+ runtime.workspace.return_value = {"active_count": 0 if resetting else 1, "resetting": resetting}+ assert await stop_workflow_for_deletion(runtime.database, "7") is None+ runtime.workspace.assert_awaited_once_with("7", client=runtime.client.client)+ runtime.reset.assert_awaited_once_with({"workflow_id": "7", "expected_revision": 0}, None)+++async def test_absent_workspace_controller_does_not_create_one(runtime):+ runtime.client.client = object()+ assert await stop_workflow_for_deletion(runtime.database, "7") is None+ runtime.reset.assert_not_awaited()+++async def test_legacy_local_deployment_uses_existing_cancel(runtime):+ runtime.status["deployed"] = True+ assert await stop_workflow_for_deletion(runtime.database, "7") is None+ runtime.cancel.assert_awaited_once_with({"workflow_id": "7"}, None)+ runtime.reset.assert_not_awaited()++[email protected]("detail", ["control_revision_conflict", "workflow_local_cleanup_failed:busy", "temporal_unavailable"])+async def test_failed_shutdown_retains_graph_and_returns_actionable_failure(runtime, detail):+ runtime.control = SimpleNamespace(status="running", revision=3)+ runtime.reset.side_effect = None+ runtime.reset.return_value = {"success": False, "error": detail}+ result = await stop_workflow_for_deletion(runtime.database, "7")+ assert result == {"success": False, "workflow_id": "7", "error": "workflow_shutdown_failed", "detail": detail}+ runtime.database.delete_workflow.assert_not_awaited()+ runtime.cancel.assert_not_awaited()+++async def test_local_cancel_failure_also_refuses_deletion(runtime):+ runtime.status["deployed"] = True+ runtime.cancel.side_effect = RuntimeError("listener cleanup failed")+ result = await stop_workflow_for_deletion(runtime.database, "7")+ assert result["success"] is False+ assert result["detail"] == "listener cleanup failed"+ runtime.database.delete_workflow.assert_not_awaited()+++async def test_failed_workspace_status_query_is_not_treated_as_no_tasks(runtime):+ runtime.client.client = object()+ runtime.workspace.side_effect = RuntimeError("query unavailable")+ result = await stop_workflow_for_deletion(runtime.database, "7")+ assert result["detail"] == "query unavailable"+ runtime.database.delete_workflow.assert_not_awaited()+++async def test_reset_cannot_mutate_a_different_container_database(runtime):+ runtime.control = SimpleNamespace(status="running", revision=3)+ runtime.container.database = lambda: object()+ result = await stop_workflow_for_deletion(runtime.database, "7")+ assert result["detail"] == "workflow_database_mismatch"+ runtime.reset.assert_not_awaited()+++async def test_legacy_cancel_cannot_target_another_database_runtime(runtime):+ runtime.status["deployed"] = True+ runtime.container.database = lambda: object()+ result = await stop_workflow_for_deletion(runtime.database, "7")+ assert result["detail"] == "workflow_database_mismatch"+ runtime.cancel.assert_not_awaited()+++async def test_new_generation_admitted_during_cleanup_refuses_deletion(runtime):+ runtime.status["deployed"] = True++ async def start_during_cancel(*_):+ runtime.control = SimpleNamespace(status="starting", revision=12)+ return {"success": True}++ runtime.cancel.side_effect = start_during_cancel+ result = await stop_workflow_for_deletion(runtime.database, "7")+ assert result["detail"] == "control_revision_conflict"+ runtime.database.delete_workflow.assert_not_awaited()+++def deletion_boundary(runtime, monkeypatch):+ from services.workflow_storage import handlers, hooks++ runtime.graph_exists = True++ async def delete(_workflow_id):+ runtime.calls.append("delete")+ runtime.graph_exists = False+ return True++ async def archive(*_):+ runtime.calls.append("archive")+ return 1++ async def drain(*_):+ runtime.calls.append("archive")+ return 1, 0++ boundary = SimpleNamespace(+ handlers=handlers,+ supports_outbox=Mock(return_value=True),+ archive=AsyncMock(side_effect=archive),+ drain=AsyncMock(side_effect=drain),+ hooks=AsyncMock(side_effect=lambda *_: runtime.calls.append("hooks")),+ broadcast=AsyncMock(side_effect=lambda *_: runtime.calls.append("broadcast")),+ )+ runtime.database.delete_workflow.side_effect = delete+ monkeypatch.setattr(handlers, "_supports_context_archive_outbox", boundary.supports_outbox)+ monkeypatch.setattr(handlers, "_archive_workflow_contexts", boundary.archive)+ monkeypatch.setattr(handlers, "_drain_context_archive_outbox", boundary.drain)+ monkeypatch.setattr(hooks, "run_workflow_deleted_hooks", boundary.hooks)+ monkeypatch.setattr(handlers, "_broadcast_lifecycle", boundary.broadcast)+ return boundary++[email protected]("has_outbox", [True, False])+async def test_delete_handler_does_not_archive_or_delete_when_shutdown_fails(runtime, deletion_boundary, has_outbox):+ runtime.control = SimpleNamespace(status="running", revision=3)+ runtime.reset.side_effect = None+ runtime.reset.return_value = {"success": False, "error": "control_revision_conflict"}+ deletion_boundary.supports_outbox.return_value = has_outbox++ result = await deletion_boundary.handlers.delete_workflow_with_context_archival(runtime.database, "7")++ assert result == {+ "success": False, "workflow_id": "7", "error": "workflow_shutdown_failed", "detail": "control_revision_conflict",+ }+ assert runtime.graph_exists is True+ runtime.reset.assert_awaited_once()+ runtime.database.delete_workflow.assert_not_awaited()+ deletion_boundary.archive.assert_not_awaited()+ deletion_boundary.drain.assert_not_awaited()+ deletion_boundary.hooks.assert_not_awaited()+ deletion_boundary.broadcast.assert_not_awaited()++[email protected]("has_outbox", [True, False])+async def test_delete_handler_waits_for_shutdown_before_archival_and_deletion(runtime, deletion_boundary, has_outbox):+ runtime.control = SimpleNamespace(status="running", revision=3)+ deletion_boundary.supports_outbox.return_value = has_outbox++ result = await deletion_boundary.handlers.delete_workflow_with_context_archival(runtime.database, "7")++ assert result == {"success": True, "workflow_id": "7", "contexts_archived": 1, "context_archives_pending": 0}+ assert runtime.graph_exists is False+ assert runtime.calls == (+ ["reset", "delete", "archive", "hooks", "broadcast"]+ if has_outbox else ["reset", "archive", "delete", "hooks", "broadcast"]+ )+ deletion_boundary.hooks.assert_awaited_once_with(runtime.database, "7")+ deletion_boundary.broadcast.assert_awaited_once_with("deleted", "7")diff --git a/server/tests/services/test_workflow_parameter_preservation.py b/server/tests/services/test_workflow_parameter_preservation.pyindex 07ad5764..f0e54e80 100644--- a/server/tests/services/test_workflow_parameter_preservation.py+++ b/server/tests/services/test_workflow_parameter_preservation.py@@ -56,7 +56,11 @@ class _ParameterRecordingDatabase:data: Dict[str, Any],description: Optional[str] = None,context_id_aliases: Optional[Dict[str, str]] = None,+ *,+ require_existing: bool = False,) -> bool:+ if require_existing and workflow_id not in self._rows:+ return Falseself._rows[workflow_id] = SimpleNamespace(id=workflow_id,name=name,diff --git a/server/tests/services/test_workflow_rename.py b/server/tests/services/test_workflow_rename.pyindex c03d0970..52d45417 100644--- a/server/tests/services/test_workflow_rename.py+++ b/server/tests/services/test_workflow_rename.py@@ -44,7 +44,11 @@ class _FakeDatabase:slug: str,data: Dict[str, Any],description: Optional[str] = None,+ *,+ require_existing: bool = False,) -> bool:+ if require_existing and workflow_id not in self._rows:+ return Falseself._rows[workflow_id] = SimpleNamespace(id=workflow_id,name=name,diff --git a/server/tests/test_context_graph_migration.py b/server/tests/test_context_graph_migration.pyindex 56daf458..3be4a5ef 100644--- a/server/tests/test_context_graph_migration.py+++ b/server/tests/test_context_graph_migration.py@@ -362,7 +362,9 @@ def test_save_owner_resolution_preserves_existing_backend_owner():async def test_workflow_delete_archives_context_before_graph(monkeypatch,):- from services.workflow_storage import handlers+ from services.workflow_storage import deletion, handlers++ monkeypatch.setattr(deletion, "stop_workflow_for_deletion", AsyncMock(return_value=None))database = type("Database", (), {})()database.get_workflow = AsyncMock(