Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): tolerate workflow pause snapshot cleanup failure (#41581)

QuantumGhost324ee5f036c1
diff --git a/api/repositories/sqlalchemy_api_workflow_run_repository.py b/api/repositories/sqlalchemy_api_workflow_run_repository.py
index ebdb9916540..bc0a494384d 100644
--- a/api/repositories/sqlalchemy_api_workflow_run_repository.py
+++ b/api/repositories/sqlalchemy_api_workflow_run_repository.py
@@ -1229,9 +1229,9 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):
"""
Delete a workflow pause state.
- Permanently removes the pause state for a workflow run, including
- the stored state file. Used for cleanup operations when a paused
- workflow is no longer needed.
+ Removes the pause record for a workflow run and attempts to delete its
+ stored state file. Used for cleanup operations when a paused workflow
+ is no longer needed.
Args:
pause_entity: The pause entity to delete
@@ -1241,8 +1241,8 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):
_WorkflowRunError: If workflow is not paused
Note:
- This operation is irreversible. The stored workflow state will be
- permanently deleted along with the pause record.
+ Storage deletion is best-effort. If it fails, the pause record is
+ still deleted and the orphaned object key is logged for cleanup.
"""
with self._session_maker() as session, session.begin():
# Get the pause model by ID
@@ -1252,8 +1252,18 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):
self._delete_pause_model(session, pause_model)
@staticmethod
- def _delete_pause_model(session: Session, pause_model: WorkflowPause):
- storage.delete(pause_model.state_object_key)
+ def _delete_pause_model(session: Session, pause_model: WorkflowPause) -> None:
+ try:
+ storage.delete(pause_model.state_object_key)
+ except Exception:
+ # Keeping the database row would block the next pause because workflow_run_id is unique.
+ logger.exception(
+ "Failed to delete state object for workflow pause; continuing with pause record deletion, "
+ "pause_id=%s, workflow_run_id=%s, object_key=%s",
+ pause_model.id,
+ pause_model.workflow_run_id,
+ pause_model.state_object_key,
+ )
# Delete the pause record
session.delete(pause_model)
diff --git a/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py b/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py
index a11b19bf9b5..df11267842e 100644
--- a/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py
+++ b/api/tests/test_containers_integration_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import secrets
from dataclasses import dataclass, field
from datetime import datetime, timedelta
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
from uuid import uuid4
import pytest
@@ -351,6 +351,75 @@ class TestCreateWorkflowPause:
assert pause_entity.get_pause_reasons() == []
assert pause_entity.get_state() == state.encode()
+ def test_replaces_pause_when_previous_state_object_delete_fails(
+ self,
+ repository: DifyAPISQLAlchemyWorkflowRunRepository,
+ db_session_with_containers: Session,
+ test_scope: _TestScope,
+ ) -> None:
+ """Keep the run resumable when cleanup leaves an orphaned state object."""
+
+ workflow_run = _create_workflow_run(
+ db_session_with_containers,
+ test_scope,
+ status=WorkflowExecutionStatus.RUNNING,
+ )
+ previous_pause = repository.create_workflow_pause(
+ workflow_run_id=workflow_run.id,
+ state_owner_user_id=test_scope.user_id,
+ state='{"pause": "previous"}',
+ pause_reasons=[],
+ )
+ previous_pause_model = db_session_with_containers.get(WorkflowPause, previous_pause.id)
+ assert previous_pause_model is not None
+ previous_pause_model_id = previous_pause_model.id
+ previous_state_object_key = previous_pause_model.state_object_key
+ test_scope.state_keys.add(previous_state_object_key)
+
+ repository.resume_workflow_pause(
+ workflow_run_id=workflow_run.id,
+ pause_entity=previous_pause,
+ )
+
+ with patch.object(
+ storage,
+ "delete",
+ side_effect=PermissionError("DeleteObject denied"),
+ ) as delete_state_object:
+ current_pause = repository.create_workflow_pause(
+ workflow_run_id=workflow_run.id,
+ state_owner_user_id=test_scope.user_id,
+ state='{"pause": "current"}',
+ pause_reasons=[],
+ )
+
+ delete_state_object.assert_called_once_with(previous_state_object_key)
+ db_session_with_containers.expire_all()
+ pause_models = db_session_with_containers.scalars(
+ select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run.id)
+ ).all()
+ assert len(pause_models) == 1
+ current_pause_model = pause_models[0]
+ test_scope.state_keys.add(current_pause_model.state_object_key)
+ assert current_pause_model.id != previous_pause_model_id
+ assert current_pause_model.id == current_pause.id
+ assert current_pause_model.resumed_at is None
+ assert current_pause.get_state() == b'{"pause": "current"}'
+ assert storage.load(previous_state_object_key) == b'{"pause": "previous"}'
+
+ db_session_with_containers.refresh(workflow_run)
+ assert workflow_run.status == WorkflowExecutionStatus.PAUSED
+
+ resumed_pause = repository.resume_workflow_pause(
+ workflow_run_id=workflow_run.id,
+ pause_entity=current_pause,
+ )
+
+ assert resumed_pause.id == current_pause.id
+ assert resumed_pause.resumed_at is not None
+ db_session_with_containers.refresh(workflow_run)
+ assert workflow_run.status == WorkflowExecutionStatus.RUNNING
+
def test_create_workflow_pause_not_found(
self,
repository: DifyAPISQLAlchemyWorkflowRunRepository,
diff --git a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py
index 67dfed2dc25..76383d88b9d 100644
--- a/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py
+++ b/api/tests/unit_tests/repositories/test_sqlalchemy_api_workflow_run_repository.py
@@ -1,6 +1,11 @@
from __future__ import annotations
+import logging
from datetime import UTC, datetime
+from unittest.mock import Mock, patch
+
+import pytest
+from sqlalchemy.orm import Session
from core.workflow.nodes.human_input.entities import FormDefinition, ParagraphInputConfig, UserActionConfig
from core.workflow.nodes.human_input.enums import FormInputType
@@ -9,6 +14,7 @@ from graphon.entities.pause_reason import HitlRequired, PauseReasonType
from models.human_input import HumanInputForm, HumanInputFormRecipient, RecipientType
from models.workflow import WorkflowPause, WorkflowPauseReason
from repositories.sqlalchemy_api_workflow_run_repository import (
+ DifyAPISQLAlchemyWorkflowRunRepository,
_build_human_input_required_reason,
_PrivateWorkflowPauseEntity,
)
@@ -151,3 +157,31 @@ def test_private_workflow_pause_entity_preserves_list_shaped_pause_reasons() ->
assert isinstance(result, list)
assert result == pause_reasons
+
+
+def test_delete_pause_model_deletes_record_when_state_object_delete_fails(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ pause_model = WorkflowPause(
+ workflow_id="workflow-1",
+ workflow_run_id="run-1",
+ state_object_key="workflow-state.json",
+ )
+ pause_model.id = "pause-1"
+ session = Mock(spec=Session)
+
+ with (
+ patch(
+ "repositories.sqlalchemy_api_workflow_run_repository.storage.delete",
+ side_effect=PermissionError("DeleteObject denied"),
+ ) as delete_state_object,
+ caplog.at_level(logging.ERROR, logger="repositories.sqlalchemy_api_workflow_run_repository"),
+ ):
+ DifyAPISQLAlchemyWorkflowRunRepository._delete_pause_model(session, pause_model)
+
+ delete_state_object.assert_called_once_with(pause_model.state_object_key)
+ session.delete.assert_called_once_with(pause_model)
+ assert "pause_id=pause-1" in caplog.text
+ assert "workflow_run_id=run-1" in caplog.text
+ assert "object_key=workflow-state.json" in caplog.text
+ assert caplog.records[-1].exc_info is not None