Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: retry broke after #1623 started deleting staged temp files (#1709)

Wallgaub207a04f2a7d
diff --git a/src/services/task_service.py b/src/services/task_service.py
index bd147d594..d633aac47 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -388,7 +388,17 @@ class TaskService:
if temp_file_paths:
if upload_task.temp_file_paths is None:
upload_task.temp_file_paths = []
- upload_task.temp_file_paths.extend(temp_file_paths)
+ normalized_temp_paths = [str(path) for path in temp_file_paths]
+ unknown_temp_paths = [
+ path for path in normalized_temp_paths if path not in upload_task.file_tasks
+ ]
+ if unknown_temp_paths:
+ logger.warning(
+ "temp_file_paths do not match file_tasks keys; retry retention may fail",
+ task_id=task_id,
+ unknown_temp_paths=unknown_temp_paths,
+ )
+ upload_task.temp_file_paths.extend(normalized_temp_paths)
# Start background processing
background_task = asyncio.create_task(
@@ -594,20 +604,9 @@ class TaskService:
upload_task.status = TaskStatus.COMPLETED
upload_task.updated_at = time.time()
- # Clean up temp files after all processing is complete
- if hasattr(upload_task, "temp_file_paths") and upload_task.temp_file_paths:
- from utils.file_utils import safe_unlink
-
- for temp_path in upload_task.temp_file_paths:
- try:
- safe_unlink(temp_path)
- logger.debug("Cleaned up temp file", temp_path=temp_path)
- except Exception as cleanup_error:
- logger.warning(
- "Failed to clean up temp file after processing",
- temp_path=temp_path,
- error=str(cleanup_error),
- )
+ # Clean up upload temps that are not retryable; keep local RETRYABLE
+ # failures so retry can re-read the staged source file.
+ self._cleanup_upload_temp_files(upload_task)
status: str = "FAILED"
@@ -1295,6 +1294,9 @@ class TaskService:
task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]
and current_time - task.updated_at > max_age_seconds
):
+ # Task is leaving memory; reclaim any retained upload temps
+ # (including RETRYABLE locals kept for in-flight retry).
+ self._cleanup_upload_temp_files(task, force=True)
del self.task_store[user_id][task_id]
# Clean up the associated lock
self._task_locks.pop(task_id, None)
@@ -1368,8 +1370,80 @@ class TaskService:
file_task.error = "Task cancelled by user"
file_task.updated_at = time.time()
+ self._cleanup_upload_temp_files(upload_task, force=True)
+
return True
+ def _file_task_for_temp_path(self, upload_task: UploadTask, temp_path: str) -> FileTask | None:
+ """Resolve the FileTask for a staged upload temp path."""
+ file_task = upload_task.file_tasks.get(temp_path)
+ if file_task is not None:
+ return file_task
+ for candidate in upload_task.file_tasks.values():
+ if candidate.file_path == temp_path:
+ return candidate
+ return None
+
+ def _is_retryable_local_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
+ """True when a staged temp belongs to a failed local RETRYABLE upload.
+
+ Local uploads use the staged path as the file_tasks key and
+ FileTask.file_path. When those diverge, this falls back to matching
+ FileTask.file_path. Unmapped absolute temps are retained (see
+ _should_retain_upload_temp) rather than deleted silently.
+ """
+ if not os.path.isabs(temp_path):
+ return False
+ file_task = self._file_task_for_temp_path(upload_task, temp_path)
+ if file_task is None or file_task.status != TaskStatus.FAILED:
+ return False
+ metadata = self._infer_failure_metadata(file_task)
+ return bool(metadata and metadata.get("actionable_by") == "RETRYABLE")
+
+ def _should_retain_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
+ """Return True when an upload temp should be kept after processing."""
+ if self._is_retryable_local_upload_temp(upload_task, temp_path):
+ return True
+ if (
+ os.path.isabs(temp_path)
+ and self._file_task_for_temp_path(upload_task, temp_path) is None
+ ):
+ logger.warning(
+ "Upload temp path has no matching file task; retaining staged file",
+ temp_path=temp_path,
+ task_id=upload_task.task_id,
+ )
+ return True
+ return False
+
+ def _cleanup_upload_temp_files(self, upload_task: UploadTask, *, force: bool = False) -> None:
+ """Remove staged upload temp files that are not retryable.
+
+ Keeps temps for failed local uploads classified as RETRYABLE so retry
+ can reuse the original source path. Use *force* to delete all temps
+ (e.g. user cancelled the task).
+ """
+ if not getattr(upload_task, "temp_file_paths", None):
+ return
+
+ from utils.file_utils import safe_unlink
+
+ retained: list[str] = []
+ for temp_path in upload_task.temp_file_paths:
+ if not force and self._should_retain_upload_temp(upload_task, temp_path):
+ retained.append(temp_path)
+ continue
+ safe_unlink(temp_path)
+ if os.path.exists(temp_path):
+ retained.append(temp_path)
+ logger.warning(
+ "Failed to clean up temp file after processing",
+ temp_path=temp_path,
+ )
+ else:
+ logger.debug("Cleaned up temp file", temp_path=temp_path)
+ upload_task.temp_file_paths = retained
+
async def shutdown(self):
"""Cleanup process pool and cancel all background tasks
@@ -1377,7 +1451,8 @@ class TaskService:
1. Cancelling the periodic cleanup task
2. Cancelling all running background tasks
3. Waiting for cancellation to complete
- 4. Shutting down the process pool
+ 4. Force-cleaning staged upload temps for all tracked tasks
+ 5. Shutting down the process pool
"""
logger.info("Shutting down TaskService", background_tasks_count=len(self.background_tasks))
@@ -1403,3 +1478,7 @@ class TaskService:
logger.warning(
"Background task raised exception during shutdown", error=str(result)
)
+
+ for user_tasks in self.task_store.values():
+ for upload_task in user_tasks.values():
+ self._cleanup_upload_temp_files(upload_task, force=True)
diff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.py
index 3b346c48a..9a0711015 100644
--- a/tests/unit/test_task_service.py
+++ b/tests/unit/test_task_service.py
@@ -5,7 +5,7 @@ Tests timeout handling, cancellation, and cleanup functionality
import asyncio
import time
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
import pytest
@@ -100,6 +100,45 @@ async def test_cleanup_old_tasks(task_service):
assert "recent_task" in task_service._task_locks # Recent task lock should remain
+async def test_cleanup_old_tasks_force_cleans_retained_temp_files(task_service):
+ """Evicted tasks must reclaim temps kept for retryable local failures."""
+ temp_path = "/var/folders/tmp/retryable.tmp"
+ old_task = UploadTask(
+ task_id="old_task",
+ total_files=1,
+ file_tasks={"file1": FileTask(file_path="file1")},
+ temp_file_paths=[temp_path],
+ status=TaskStatus.FAILED,
+ )
+ old_task.updated_at = time.time() - 7200
+
+ task_service.task_store["user1"] = {"old_task": old_task}
+
+ with patch.object(task_service, "_cleanup_upload_temp_files") as mock_cleanup:
+ cleaned = await task_service.cleanup_old_tasks(max_age_seconds=3600)
+
+ assert cleaned == 1
+ mock_cleanup.assert_called_once_with(old_task, force=True)
+
+
+async def test_shutdown_force_cleans_upload_temp_files(task_service):
+ upload_task = UploadTask(
+ task_id="task1",
+ total_files=1,
+ file_tasks={},
+ temp_file_paths=["/var/folders/tmp/retryable.tmp"],
+ status=TaskStatus.FAILED,
+ )
+ task_service.task_store["user1"] = {"task1": upload_task}
+
+ with patch.object(task_service, "_cleanup_upload_temp_files") as mock_cleanup:
+ await task_service.shutdown()
+
+ mock_cleanup.assert_called_once_with(upload_task, force=True)
+
+
@pytest.mark.asyncio
async def test_shutdown_cancels_background_tasks(task_service):
"""Test that shutdown properly cancels background tasks"""
diff --git a/tests/unit/test_task_service_retry_failed_files.py b/tests/unit/test_task_service_retry_failed_files.py
index 811374efb..849d41f09 100644
--- a/tests/unit/test_task_service_retry_failed_files.py
+++ b/tests/unit/test_task_service_retry_failed_files.py
@@ -245,3 +245,143 @@ async def test_retry_connector_id_path_does_not_require_local_file(task_service)
assert result["status"] == "accepted"
assert ft.status == TaskStatus.PENDING
mock_bg.assert_called_once()
+
+
+def test_is_retryable_local_upload_temp_true_for_failed_local_timeout(task_service):
+ temp_path = "/var/folders/tmp/retryable.tmp"
+ ft = _retryable_failed_file(temp_path)
+ task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={temp_path: ft},
+ temp_file_paths=[temp_path],
+ )
+
+ assert task_service._is_retryable_local_upload_temp(task, temp_path) is True
+
+
+def test_is_retryable_local_upload_temp_resolves_file_task_by_file_path(task_service):
+ temp_path = "/var/folders/tmp/retryable.tmp"
+ ft = _retryable_failed_file(temp_path)
+ task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={"different-key": ft},
+ temp_file_paths=[temp_path],
+ )
+
+ assert task_service._is_retryable_local_upload_temp(task, temp_path) is True
+
+
+def test_is_retryable_local_upload_temp_false_for_connector_id(task_service):
+ file_path = "google-drive-file-id"
+ ft = FileTask(file_path=file_path, filename="doc.pdf")
+ ft.status = TaskStatus.FAILED
+ ft.error = "All connection attempts failed"
+ task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={file_path: ft},
+ temp_file_paths=[],
+ )
+
+ assert task_service._is_retryable_local_upload_temp(task, file_path) is False
+
+
+def test_cleanup_upload_temp_files_keeps_retryable_local_failure(task_service):
+ retryable_path = "/var/folders/tmp/retryable.tmp"
+ other_path = "/var/folders/tmp/success.tmp"
+ ft_retryable = _retryable_failed_file(retryable_path)
+ ft_success = FileTask(file_path=other_path, filename="ok.pdf")
+ ft_success.status = TaskStatus.COMPLETED
+ task = UploadTask(
+ task_id="task-1",
+ total_files=2,
+ file_tasks={retryable_path: ft_retryable, other_path: ft_success},
+ temp_file_paths=[retryable_path, other_path],
+ )
+
+ with (
+ patch("utils.file_utils.safe_unlink") as mock_unlink,
+ patch("os.path.exists", return_value=False),
+ ):
+ task_service._cleanup_upload_temp_files(task)
+
+ mock_unlink.assert_called_once_with(other_path)
+ assert task.temp_file_paths == [retryable_path]
+
+
+def test_cleanup_upload_temp_files_removes_non_retryable_failure(task_service):
+ temp_path = "/var/folders/tmp/corrupt.tmp"
+ ft = FileTask(file_path=temp_path, filename="bad.pdf")
+ ft.status = TaskStatus.FAILED
+ ft.error = "The file appears corrupted or invalid"
+ task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={temp_path: ft},
+ temp_file_paths=[temp_path],
+ )
+
+ with (
+ patch("utils.file_utils.safe_unlink") as mock_unlink,
+ patch("os.path.exists", return_value=False),
+ ):
+ task_service._cleanup_upload_temp_files(task)
+
+ mock_unlink.assert_called_once_with(temp_path)
+ assert task.temp_file_paths == []
+
+
+def test_cleanup_upload_temp_files_retains_path_when_unlink_fails(task_service):
+ temp_path = "/var/folders/tmp/still-there.tmp"
+ ft = FileTask(file_path=temp_path, filename="ok.pdf")
+ ft.status = TaskStatus.COMPLETED
+ task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={temp_path: ft},
+ temp_file_paths=[temp_path],
+ )
+
+ with (
+ patch("utils.file_utils.safe_unlink") as mock_unlink,
+ patch("os.path.exists", return_value=True),
+ ):
+ task_service._cleanup_upload_temp_files(task)
+
+ mock_unlink.assert_called_once_with(temp_path)
+ assert task.temp_file_paths == [temp_path]
+
+
+def test_cleanup_upload_temp_files_does_not_unlink_retryable_local_failure(task_service):
+ retryable_path = "/var/folders/tmp/retryable.tmp"
+ ft = _retryable_failed_file(retryable_path)
+ task = UploadTask(
+ task_id="task-1",
+ total_files=1,
+ file_tasks={retryable_path: ft},
+ temp_file_paths=[retryable_path],
+ )
+
+ with patch("utils.file_utils.safe_unlink") as mock_unlink:
+ task_service._cleanup_upload_temp_files(task)
+
+ mock_unlink.assert_not_called()
+ assert task.temp_file_paths == [retryable_path]
+
+
+def test_cleanup_upload_temp_files_retains_unmapped_absolute_temp(task_service):
+ temp_path = "/var/folders/tmp/unmapped.tmp"
+ task = UploadTask(
+ task_id="task-1",
+ total_files=0,
+ file_tasks={},
+ temp_file_paths=[temp_path],
+ )
+
+ with patch("utils.file_utils.safe_unlink") as mock_unlink:
+ task_service._cleanup_upload_temp_files(task)
+
+ mock_unlink.assert_not_called()
+ assert task.temp_file_paths == [temp_path]