OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: retry broke after #1623 started deleting staged temp files (#1709)
Wallgaub207a04f2a7d
diff --git a/src/services/task_service.py b/src/services/task_service.pyindex bd147d594..d633aac47 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -388,7 +388,17 @@ class TaskService:if temp_file_paths:if upload_task.temp_file_paths is None:upload_task.temp_file_paths = []- upload_task.temp_file_paths.extend(temp_file_paths)+ normalized_temp_paths = [str(path) for path in temp_file_paths]+ unknown_temp_paths = [+ path for path in normalized_temp_paths if path not in upload_task.file_tasks+ ]+ if unknown_temp_paths:+ logger.warning(+ "temp_file_paths do not match file_tasks keys; retry retention may fail",+ task_id=task_id,+ unknown_temp_paths=unknown_temp_paths,+ )+ upload_task.temp_file_paths.extend(normalized_temp_paths)# Start background processingbackground_task = asyncio.create_task(@@ -594,20 +604,9 @@ class TaskService:upload_task.status = TaskStatus.COMPLETEDupload_task.updated_at = time.time()- # Clean up temp files after all processing is complete- if hasattr(upload_task, "temp_file_paths") and upload_task.temp_file_paths:- from utils.file_utils import safe_unlink-- for temp_path in upload_task.temp_file_paths:- try:- safe_unlink(temp_path)- logger.debug("Cleaned up temp file", temp_path=temp_path)- except Exception as cleanup_error:- logger.warning(- "Failed to clean up temp file after processing",- temp_path=temp_path,- error=str(cleanup_error),- )+ # Clean up upload temps that are not retryable; keep local RETRYABLE+ # failures so retry can re-read the staged source file.+ self._cleanup_upload_temp_files(upload_task)status: str = "FAILED"@@ -1295,6 +1294,9 @@ class TaskService:task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]and current_time - task.updated_at > max_age_seconds):+ # Task is leaving memory; reclaim any retained upload temps+ # (including RETRYABLE locals kept for in-flight retry).+ self._cleanup_upload_temp_files(task, force=True)del self.task_store[user_id][task_id]# Clean up the associated lockself._task_locks.pop(task_id, None)@@ -1368,8 +1370,80 @@ class TaskService:file_task.error = "Task cancelled by user"file_task.updated_at = time.time()+ self._cleanup_upload_temp_files(upload_task, force=True)+return True+ def _file_task_for_temp_path(self, upload_task: UploadTask, temp_path: str) -> FileTask | None:+ """Resolve the FileTask for a staged upload temp path."""+ file_task = upload_task.file_tasks.get(temp_path)+ if file_task is not None:+ return file_task+ for candidate in upload_task.file_tasks.values():+ if candidate.file_path == temp_path:+ return candidate+ return None++ def _is_retryable_local_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:+ """True when a staged temp belongs to a failed local RETRYABLE upload.++ Local uploads use the staged path as the file_tasks key and+ FileTask.file_path. When those diverge, this falls back to matching+ FileTask.file_path. Unmapped absolute temps are retained (see+ _should_retain_upload_temp) rather than deleted silently.+ """+ if not os.path.isabs(temp_path):+ return False+ file_task = self._file_task_for_temp_path(upload_task, temp_path)+ if file_task is None or file_task.status != TaskStatus.FAILED:+ return False+ metadata = self._infer_failure_metadata(file_task)+ return bool(metadata and metadata.get("actionable_by") == "RETRYABLE")++ def _should_retain_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:+ """Return True when an upload temp should be kept after processing."""+ if self._is_retryable_local_upload_temp(upload_task, temp_path):+ return True+ if (+ os.path.isabs(temp_path)+ and self._file_task_for_temp_path(upload_task, temp_path) is None+ ):+ logger.warning(+ "Upload temp path has no matching file task; retaining staged file",+ temp_path=temp_path,+ task_id=upload_task.task_id,+ )+ return True+ return False++ def _cleanup_upload_temp_files(self, upload_task: UploadTask, *, force: bool = False) -> None:+ """Remove staged upload temp files that are not retryable.++ Keeps temps for failed local uploads classified as RETRYABLE so retry+ can reuse the original source path. Use *force* to delete all temps+ (e.g. user cancelled the task).+ """+ if not getattr(upload_task, "temp_file_paths", None):+ return++ from utils.file_utils import safe_unlink++ retained: list[str] = []+ for temp_path in upload_task.temp_file_paths:+ if not force and self._should_retain_upload_temp(upload_task, temp_path):+ retained.append(temp_path)+ continue+ safe_unlink(temp_path)+ if os.path.exists(temp_path):+ retained.append(temp_path)+ logger.warning(+ "Failed to clean up temp file after processing",+ temp_path=temp_path,+ )+ else:+ logger.debug("Cleaned up temp file", temp_path=temp_path)+ upload_task.temp_file_paths = retained+async def shutdown(self):"""Cleanup process pool and cancel all background tasks@@ -1377,7 +1451,8 @@ class TaskService:1. Cancelling the periodic cleanup task2. Cancelling all running background tasks3. Waiting for cancellation to complete- 4. Shutting down the process pool+ 4. Force-cleaning staged upload temps for all tracked tasks+ 5. Shutting down the process pool"""logger.info("Shutting down TaskService", background_tasks_count=len(self.background_tasks))@@ -1403,3 +1478,7 @@ class TaskService:logger.warning("Background task raised exception during shutdown", error=str(result))++ for user_tasks in self.task_store.values():+ for upload_task in user_tasks.values():+ self._cleanup_upload_temp_files(upload_task, force=True)diff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.pyindex 3b346c48a..9a0711015 100644--- a/tests/unit/test_task_service.py+++ b/tests/unit/test_task_service.py@@ -5,7 +5,7 @@ Tests timeout handling, cancellation, and cleanup functionalityimport asyncioimport time-from unittest.mock import Mock+from unittest.mock import Mock, patchimport pytest@@ -100,6 +100,45 @@ async def test_cleanup_old_tasks(task_service):assert "recent_task" in task_service._task_locks # Recent task lock should remain+async def test_cleanup_old_tasks_force_cleans_retained_temp_files(task_service):+ """Evicted tasks must reclaim temps kept for retryable local failures."""+ temp_path = "/var/folders/tmp/retryable.tmp"+ old_task = UploadTask(+ task_id="old_task",+ total_files=1,+ file_tasks={"file1": FileTask(file_path="file1")},+ temp_file_paths=[temp_path],+ status=TaskStatus.FAILED,+ )+ old_task.updated_at = time.time() - 7200++ task_service.task_store["user1"] = {"old_task": old_task}++ with patch.object(task_service, "_cleanup_upload_temp_files") as mock_cleanup:+ cleaned = await task_service.cleanup_old_tasks(max_age_seconds=3600)++ assert cleaned == 1+ mock_cleanup.assert_called_once_with(old_task, force=True)+++async def test_shutdown_force_cleans_upload_temp_files(task_service):+ upload_task = UploadTask(+ task_id="task1",+ total_files=1,+ file_tasks={},+ temp_file_paths=["/var/folders/tmp/retryable.tmp"],+ status=TaskStatus.FAILED,+ )+ task_service.task_store["user1"] = {"task1": upload_task}++ with patch.object(task_service, "_cleanup_upload_temp_files") as mock_cleanup:+ await task_service.shutdown()++ mock_cleanup.assert_called_once_with(upload_task, force=True)++@pytest.mark.asyncioasync def test_shutdown_cancels_background_tasks(task_service):"""Test that shutdown properly cancels background tasks"""diff --git a/tests/unit/test_task_service_retry_failed_files.py b/tests/unit/test_task_service_retry_failed_files.pyindex 811374efb..849d41f09 100644--- a/tests/unit/test_task_service_retry_failed_files.py+++ b/tests/unit/test_task_service_retry_failed_files.py@@ -245,3 +245,143 @@ async def test_retry_connector_id_path_does_not_require_local_file(task_service)assert result["status"] == "accepted"assert ft.status == TaskStatus.PENDINGmock_bg.assert_called_once()+++def test_is_retryable_local_upload_temp_true_for_failed_local_timeout(task_service):+ temp_path = "/var/folders/tmp/retryable.tmp"+ ft = _retryable_failed_file(temp_path)+ task = UploadTask(+ task_id="task-1",+ total_files=1,+ file_tasks={temp_path: ft},+ temp_file_paths=[temp_path],+ )++ assert task_service._is_retryable_local_upload_temp(task, temp_path) is True+++def test_is_retryable_local_upload_temp_resolves_file_task_by_file_path(task_service):+ temp_path = "/var/folders/tmp/retryable.tmp"+ ft = _retryable_failed_file(temp_path)+ task = UploadTask(+ task_id="task-1",+ total_files=1,+ file_tasks={"different-key": ft},+ temp_file_paths=[temp_path],+ )++ assert task_service._is_retryable_local_upload_temp(task, temp_path) is True+++def test_is_retryable_local_upload_temp_false_for_connector_id(task_service):+ file_path = "google-drive-file-id"+ ft = FileTask(file_path=file_path, filename="doc.pdf")+ ft.status = TaskStatus.FAILED+ ft.error = "All connection attempts failed"+ task = UploadTask(+ task_id="task-1",+ total_files=1,+ file_tasks={file_path: ft},+ temp_file_paths=[],+ )++ assert task_service._is_retryable_local_upload_temp(task, file_path) is False+++def test_cleanup_upload_temp_files_keeps_retryable_local_failure(task_service):+ retryable_path = "/var/folders/tmp/retryable.tmp"+ other_path = "/var/folders/tmp/success.tmp"+ ft_retryable = _retryable_failed_file(retryable_path)+ ft_success = FileTask(file_path=other_path, filename="ok.pdf")+ ft_success.status = TaskStatus.COMPLETED+ task = UploadTask(+ task_id="task-1",+ total_files=2,+ file_tasks={retryable_path: ft_retryable, other_path: ft_success},+ temp_file_paths=[retryable_path, other_path],+ )++ with (+ patch("utils.file_utils.safe_unlink") as mock_unlink,+ patch("os.path.exists", return_value=False),+ ):+ task_service._cleanup_upload_temp_files(task)++ mock_unlink.assert_called_once_with(other_path)+ assert task.temp_file_paths == [retryable_path]+++def test_cleanup_upload_temp_files_removes_non_retryable_failure(task_service):+ temp_path = "/var/folders/tmp/corrupt.tmp"+ ft = FileTask(file_path=temp_path, filename="bad.pdf")+ ft.status = TaskStatus.FAILED+ ft.error = "The file appears corrupted or invalid"+ task = UploadTask(+ task_id="task-1",+ total_files=1,+ file_tasks={temp_path: ft},+ temp_file_paths=[temp_path],+ )++ with (+ patch("utils.file_utils.safe_unlink") as mock_unlink,+ patch("os.path.exists", return_value=False),+ ):+ task_service._cleanup_upload_temp_files(task)++ mock_unlink.assert_called_once_with(temp_path)+ assert task.temp_file_paths == []+++def test_cleanup_upload_temp_files_retains_path_when_unlink_fails(task_service):+ temp_path = "/var/folders/tmp/still-there.tmp"+ ft = FileTask(file_path=temp_path, filename="ok.pdf")+ ft.status = TaskStatus.COMPLETED+ task = UploadTask(+ task_id="task-1",+ total_files=1,+ file_tasks={temp_path: ft},+ temp_file_paths=[temp_path],+ )++ with (+ patch("utils.file_utils.safe_unlink") as mock_unlink,+ patch("os.path.exists", return_value=True),+ ):+ task_service._cleanup_upload_temp_files(task)++ mock_unlink.assert_called_once_with(temp_path)+ assert task.temp_file_paths == [temp_path]+++def test_cleanup_upload_temp_files_does_not_unlink_retryable_local_failure(task_service):+ retryable_path = "/var/folders/tmp/retryable.tmp"+ ft = _retryable_failed_file(retryable_path)+ task = UploadTask(+ task_id="task-1",+ total_files=1,+ file_tasks={retryable_path: ft},+ temp_file_paths=[retryable_path],+ )++ with patch("utils.file_utils.safe_unlink") as mock_unlink:+ task_service._cleanup_upload_temp_files(task)++ mock_unlink.assert_not_called()+ assert task.temp_file_paths == [retryable_path]+++def test_cleanup_upload_temp_files_retains_unmapped_absolute_temp(task_service):+ temp_path = "/var/folders/tmp/unmapped.tmp"+ task = UploadTask(+ task_id="task-1",+ total_files=0,+ file_tasks={},+ temp_file_paths=[temp_path],+ )++ with patch("utils.file_utils.safe_unlink") as mock_unlink:+ task_service._cleanup_upload_temp_files(task)++ mock_unlink.assert_not_called()+ assert task.temp_file_paths == [temp_path]