Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): tolerate workflow pause snapshot cleanup failure (#41581)
QuantumGhost324ee5f036c1
diff --git a/api/repositories/sqlalchemy_api_workflow_run_repository.py b/api/repositories/sqlalchemy_api_workflow_run_repository.pyindex ebdb9916540..bc0a494384d 100644--- a/api/repositories/sqlalchemy_api_workflow_run_repository.py+++ b/api/repositories/sqlalchemy_api_workflow_run_repository.py@@ -1229,9 +1229,9 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):"""Delete a workflow pause state.- Permanently removes the pause state for a workflow run, including- the stored state file. Used for cleanup operations when a paused- workflow is no longer needed.+ Removes the pause record for a workflow run and attempts to delete its+ stored state file. Used for cleanup operations when a paused workflow+ is no longer needed.Args:pause_entity: The pause entity to delete@@ -1241,8 +1241,8 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):_WorkflowRunError: If workflow is not pausedNote:- This operation is irreversible. The stored workflow state will be- permanently deleted along with the pause record.+ Storage deletion is best-effort. If it fails, the pause record is+ still deleted and the orphaned object key is logged for cleanup."""with self._session_maker() as session, session.begin():# Get the pause model by ID@@ -1252,8 +1252,18 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):self._delete_pause_model(session, pause_model)@staticmethod- def _delete_pause_model(session: Session, pause_model: WorkflowPause):- storage.delete(pause_model.state_object_key)+ def _delete_pause_model(session: Session, pause_model: WorkflowPause) -> None:+ try:+ storage.delete(pause_model.state_object_key)+ except Exception:+ # Keeping the database row would block the next pause because workflow_run_id is unique.+ logger.exception(+ "Failed to delete state object for workflow pause; continuing with pause record deletion, "+ "pause_id=%s, workflow_run_id=%s, object_key=%s",+ pause_model.id,+ pause_model.workflow_run_id,+ pause_model.state_object_key,+ )# Delete the pause recordsession.delete(pause_model)diff --git a/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py b/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.pyindex a11b19bf9b5..df11267842e 100644--- a/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py+++ b/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport secretsfrom dataclasses import dataclass, fieldfrom datetime import datetime, timedelta-from unittest.mock import Mock+from unittest.mock import Mock, patchfrom uuid import uuid4import pytest@@ -351,6 +351,75 @@ class TestCreateWorkflowPause:assert pause_entity.get_pause_reasons() == []assert pause_entity.get_state() == state.encode()+ def test_replaces_pause_when_previous_state_object_delete_fails(+ self,+ repository: DifyAPISQLAlchemyWorkflowRunRepository,+ db_session_with_containers: Session,+ test_scope: _TestScope,+ ) -> None:+ """Keep the run resumable when cleanup leaves an orphaned state object."""++ workflow_run = _create_workflow_run(+ db_session_with_containers,+ test_scope,+ status=WorkflowExecutionStatus.RUNNING,+ )+ previous_pause = repository.create_workflow_pause(+ workflow_run_id=workflow_run.id,+ state_owner_user_id=test_scope.user_id,+ state='{"pause": "previous"}',+ pause_reasons=[],+ )+ previous_pause_model = db_session_with_containers.get(WorkflowPause, previous_pause.id)+ assert previous_pause_model is not None+ previous_pause_model_id = previous_pause_model.id+ previous_state_object_key = previous_pause_model.state_object_key+ test_scope.state_keys.add(previous_state_object_key)++ repository.resume_workflow_pause(+ workflow_run_id=workflow_run.id,+ pause_entity=previous_pause,+ )++ with patch.object(+ storage,+ "delete",+ side_effect=PermissionError("DeleteObject denied"),+ ) as delete_state_object:+ current_pause = repository.create_workflow_pause(+ workflow_run_id=workflow_run.id,+ state_owner_user_id=test_scope.user_id,+ state='{"pause": "current"}',+ pause_reasons=[],+ )++ delete_state_object.assert_called_once_with(previous_state_object_key)+ db_session_with_containers.expire_all()+ pause_models = db_session_with_containers.scalars(+ select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run.id)+ ).all()+ assert len(pause_models) == 1+ current_pause_model = pause_models[0]+ test_scope.state_keys.add(current_pause_model.state_object_key)+ assert current_pause_model.id != previous_pause_model_id+ assert current_pause_model.id == current_pause.id+ assert current_pause_model.resumed_at is None+ assert current_pause.get_state() == b'{"pause": "current"}'+ assert storage.load(previous_state_object_key) == b'{"pause": "previous"}'++ db_session_with_containers.refresh(workflow_run)+ assert workflow_run.status == WorkflowExecutionStatus.PAUSED++ resumed_pause = repository.resume_workflow_pause(+ workflow_run_id=workflow_run.id,+ pause_entity=current_pause,+ )++ assert resumed_pause.id == current_pause.id+ assert resumed_pause.resumed_at is not None+ db_session_with_containers.refresh(workflow_run)+ assert workflow_run.status == WorkflowExecutionStatus.RUNNING+def test_create_workflow_pause_not_found(self,repository: DifyAPISQLAlchemyWorkflowRunRepository,diff --git a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.pyindex 67dfed2dc25..76383d88b9d 100644--- a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py+++ b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py@@ -1,6 +1,11 @@from __future__ import annotations+import loggingfrom datetime import UTC, datetime+from unittest.mock import Mock, patch++import pytest+from sqlalchemy.orm import Sessionfrom core.workflow.nodes.human_input.entities import FormDefinition, ParagraphInputConfig, UserActionConfigfrom core.workflow.nodes.human_input.enums import FormInputType@@ -9,6 +14,7 @@ from graphon.entities.pause_reason import HitlRequired, PauseReasonTypefrom models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientTypefrom models.workflow import WorkflowPause, WorkflowPauseReasonfrom repositories.sqlalchemy_api_workflow_run_repository import (+ DifyAPISQLAlchemyWorkflowRunRepository,_build_human_input_required_reason,_PrivateWorkflowPauseEntity,)@@ -151,3 +157,31 @@ def test_private_workflow_pause_entity_preserves_list_shaped_pause_reasons() ->assert isinstance(result, list)assert result == pause_reasons+++def test_delete_pause_model_deletes_record_when_state_object_delete_fails(+ caplog: pytest.LogCaptureFixture,+) -> None:+ pause_model = WorkflowPause(+ workflow_id="workflow-1",+ workflow_run_id="run-1",+ state_object_key="workflow-state.json",+ )+ pause_model.id = "pause-1"+ session = Mock(spec=Session)++ with (+ patch(+ "repositories.sqlalchemy_api_workflow_run_repository.storage.delete",+ side_effect=PermissionError("DeleteObject denied"),+ ) as delete_state_object,+ caplog.at_level(logging.ERROR, logger="repositories.sqlalchemy_api_workflow_run_repository"),+ ):+ DifyAPISQLAlchemyWorkflowRunRepository._delete_pause_model(session, pause_model)++ delete_state_object.assert_called_once_with(pause_model.state_object_key)+ session.delete.assert_called_once_with(pause_model)+ assert "pause_id=pause-1" in caplog.text+ assert "workflow_run_id=run-1" in caplog.text+ assert "object_key=workflow-state.json" in caplog.text+ assert caplog.records[-1].exc_info is not None