OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
(bug): Ingesting folder doesn't seem to work
Mike Pawlowski9f463f12a30b
diff --git a/.env.example b/.env.exampleindex 4c4cd79b6..68e49b7f5 100644--- a/.env.example+++ b/.env.example@@ -10,6 +10,11 @@ DISABLE_INGEST_WITH_LANGFLOW=false# LANGFLOW_TIMEOUT=2400# LANGFLOW_CONNECT_TIMEOUT=30+# Per-file processing timeout for document ingestion tasks (in seconds)+# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete+# Default: 3600 seconds (60 minutes) total timeout+# INGESTION_TIMEOUT=3600+# make one like so https://docs.langflow.org/api-keys-and-authentication#langflow-secret-keyLANGFLOW_SECRET_KEY=diff --git a/.gitignore b/.gitignoreindex 2143d13a6..2c09fc1f9 100644--- a/.gitignore+++ b/.gitignore@@ -26,7 +26,7 @@ wheels/!frontend/*.json.DS_Store-config/+/config/.docling.piddiff --git a/src/config/settings.py b/src/config/settings.pyindex a8e9433ad..c4a5d9ce7 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -65,6 +65,11 @@ DISABLE_INGEST_WITH_LANGFLOW = os.getenv(LANGFLOW_TIMEOUT = float(os.getenv("LANGFLOW_TIMEOUT", "2400")) # 40 minutesLANGFLOW_CONNECT_TIMEOUT = float(os.getenv("LANGFLOW_CONNECT_TIMEOUT", "30")) # 30 seconds+# Per-file processing timeout for document ingestion tasks (in seconds)+# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete+# Default: 3600 seconds (60 minutes)+INGESTION_TIMEOUT = int(os.getenv("INGESTION_TIMEOUT", "3600"))+def is_no_auth_mode():"""Check if we're running in no-auth mode (OAuth credentials missing)"""@@ -90,7 +95,7 @@ OPENAI_EMBEDDING_DIMENSIONS = {WATSONX_EMBEDDING_DIMENSIONS = {# IBM Models-"ibm/granite-embedding-107m-multilingual": 384,+"ibm/granite-embedding-107m-multilingual": 384,"ibm/granite-embedding-278m-multilingual": 1024,"ibm/slate-125m-english-rtrvr": 768,"ibm/slate-125m-english-rtrvr-v2": 768,@@ -410,17 +415,17 @@ class AppClients:# LiteLLM routes based on model name prefixes (openai/, ollama/, watsonx/, etc.)try:config = get_openrag_config()-+# Set OpenAI credentialsif config.providers.openai.api_key:os.environ["OPENAI_API_KEY"] = config.providers.openai.api_keylogger.debug("Loaded OpenAI API key from config")-+# Set Anthropic credentialsif config.providers.anthropic.api_key:os.environ["ANTHROPIC_API_KEY"] = config.providers.anthropic.api_keylogger.debug("Loaded Anthropic API key from config")-+# Set WatsonX credentialsif config.providers.watsonx.api_key:os.environ["WATSONX_API_KEY"] = config.providers.watsonx.api_key@@ -431,13 +436,13 @@ class AppClients:os.environ["WATSONX_PROJECT_ID"] = config.providers.watsonx.project_idif config.providers.watsonx.api_key:logger.debug("Loaded WatsonX credentials from config")-+# Set Ollama endpointif config.providers.ollama.endpoint:os.environ["OLLAMA_BASE_URL"] = config.providers.ollama.endpointos.environ["OLLAMA_ENDPOINT"] = config.providers.ollama.endpointlogger.debug("Loaded Ollama endpoint from config")-+except Exception as e:logger.debug("Could not load provider credentials from config", error=str(e))diff --git a/src/main.py b/src/main.pyindex a0aeb82cd..aa1c907ff 100644--- a/src/main.py+++ b/src/main.py@@ -68,6 +68,7 @@ from config.settings import (API_KEYS_INDEX_BODY,API_KEYS_INDEX_NAME,DISABLE_INGEST_WITH_LANGFLOW,+ INGESTION_TIMEOUT,INDEX_BODY,INDEX_NAME,SESSION_SECRET,@@ -625,7 +626,7 @@ async def initialize_services():# Initialize servicesdocument_service = DocumentService(session_manager=session_manager)search_service = SearchService(session_manager)- task_service = TaskService(document_service, process_pool)+ task_service = TaskService(document_service, process_pool, ingestion_timeout=INGESTION_TIMEOUT)chat_service = ChatService()flows_service = FlowsService()knowledge_filter_service = KnowledgeFilterService(session_manager)@@ -1559,6 +1560,9 @@ async def create_app():app.state.background_tasks.add(t1)t1.add_done_callback(app.state.background_tasks.discard)+ # Start periodic task cleanup scheduler+ services["task_service"].start_cleanup_scheduler()+# Start periodic flow backup task (every 5 minutes)async def periodic_backup():"""Periodic backup task that runs every 15 minutes"""@@ -1603,6 +1607,8 @@ async def create_app():async def shutdown_event():await TelemetryClient.send_event(Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN)await cleanup_subscriptions_proper(services)+ # Cleanup task service (cancels background tasks and process pool)+ await services["task_service"].shutdown()# Cleanup async clientsawait clients.cleanup()# Cleanup telemetry clientdiff --git a/src/services/task_service.py b/src/services/task_service.pyindex 6ebb070d1..9c76379ae 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -2,6 +2,7 @@ import asyncioimport randomimport timeimport uuid+from typing import Any, Coroutine, TypeVarfrom models.tasks import FileTask, TaskStatus, UploadTaskfrom session_manager import AnonymousUser@@ -9,22 +10,68 @@ from utils.gpu_detection import get_worker_countfrom utils.logging_config import get_loggerfrom utils.telemetry import TelemetryClient, Category, MessageId+T = TypeVar("T")logger = get_logger(__name__)+class IngestionTimeoutError(Exception):+ """Raised when file processing exceeds the configured timeout"""+ pass++class TaskService:- def __init__(self, document_service=None, process_pool=None):+ # Cleanup interval in seconds (2 hours)+ CLEANUP_INTERVAL_SECONDS = 2 * 60 * 60++ def __init__(self, document_service=None, process_pool=None, ingestion_timeout=3600):self.document_service = document_serviceself.process_pool = process_poolself.task_store: dict[str, dict[str, UploadTask]] = {} # user_id -> {task_id -> UploadTask}self.background_tasks = set()+ self.ingestion_timeout = ingestion_timeout+ self._cleanup_task: asyncio.Task | None = None+ # Locks for task counter updates, keyed by task_id+ # Kept separate from UploadTask to maintain serialization compatibility+ self._task_locks: dict[str, asyncio.Lock] = {}if self.process_pool is None:raise ValueError("TaskService requires a process_pool parameter")+ def _get_task_lock(self, task_id: str) -> asyncio.Lock:+ """Get or create a lock for a specific task's counter updates"""+ if task_id not in self._task_locks:+ self._task_locks[task_id] = asyncio.Lock()+ return self._task_locks[task_id]++ def start_cleanup_scheduler(self) -> None:+ """Start the periodic cleanup background task.++ Should be called once after the event loop is running (e.g., during app startup).+ """+ if self._cleanup_task is None or self._cleanup_task.done():+ self._cleanup_task = asyncio.create_task(self._periodic_cleanup())+ logger.info(+ "Started periodic task cleanup scheduler",+ interval_seconds=self.CLEANUP_INTERVAL_SECONDS,+ )++ async def _periodic_cleanup(self) -> None:+ """Periodically clean up old completed/failed tasks."""+ while True:+ try:+ await asyncio.sleep(self.CLEANUP_INTERVAL_SECONDS)+ cleaned = await self.cleanup_old_tasks()+ if cleaned > 0:+ logger.debug("Periodic cleanup completed", tasks_cleaned=cleaned)+ except asyncio.CancelledError:+ logger.debug("Periodic cleanup task cancelled")+ raise+ except Exception as e:+ logger.warning("Error during periodic cleanup", error=str(e))+async def exponential_backoff_delay(self, retry_count: int, base_delay: float = 1.0, max_delay: float = 60.0) -> None:@@ -32,6 +79,28 @@ class TaskService:delay = min(base_delay * (2**retry_count) + random.uniform(0, 1), max_delay)await asyncio.sleep(delay)+ async def _process_with_timeout(+ self, coro: Coroutine[Any, Any, T], timeout_seconds: int | None = None+ ) -> T:+ """Wrapper to add timeout protection to file processing++ Args:+ coro: Coroutine to execute with timeout+ timeout_seconds: Timeout in seconds (uses self.ingestion_timeout if None)++ Returns:+ The result of the coroutine++ Raises:+ IngestionTimeoutError: If processing exceeds timeout+ """+ timeout = timeout_seconds or self.ingestion_timeout+ try:+ return await asyncio.wait_for(coro, timeout=timeout)+ except asyncio.TimeoutError:+ logger.error("File processing timed out", timeout_seconds=timeout)+ raise IngestionTimeoutError(f"Processing timed out after {timeout}s") from None+async def create_upload_task(self,user_id: str,@@ -251,7 +320,33 @@ class TaskService:file_task.updated_at = time.time()try:- await processor.process_item(upload_task, item, file_task)+ # Add timeout protection to prevent indefinite hangs+ await self._process_with_timeout(+ processor.process_item(upload_task, item, file_task),+ timeout_seconds=self.ingestion_timeout+ )+ except asyncio.CancelledError:+ # Handle cancellation explicitly+ logger.info("File processing cancelled", item=str(item))+ if file_task.status == TaskStatus.RUNNING:+ file_task.status = TaskStatus.FAILED+ file_task.error = "Task cancelled by user"+ async with self._get_task_lock(task_id):+ upload_task.failed_files += 1+ raise # Re-raise to propagate cancellation+ except IngestionTimeoutError as e:+ # Handle timeout explicitly with specific error message+ logger.warning(+ "File processing timed out",+ item=str(item),+ timeout=self.ingestion_timeout+ )+ if file_task.status == TaskStatus.RUNNING:+ file_task.status = TaskStatus.FAILED+ file_task.error = f"Processing timed out after {self.ingestion_timeout}s"+ async with self._get_task_lock(task_id):+ upload_task.failed_files += 1+ # Don't re-raise - treat as normal failure, not cancellationexcept Exception as e:logger.error("Failed to process item", item=str(item), error=str(e)@@ -268,7 +363,11 @@ class TaskService:file_task.error = str(e)finally:file_task.updated_at = time.time()- upload_task.processed_files += 1+ # Only increment processed_files if the file reached a terminal state+ # This prevents counter inconsistency on cancellation+ if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:+ async with self._get_task_lock(task_id):+ upload_task.processed_files += 1upload_task.updated_at = time.time()tasks = [process_with_semaphore(item, str(item)) for item in items]@@ -447,6 +546,47 @@ class TaskService:tasks.sort(key=lambda x: x["created_at"], reverse=True)return tasks+ async def cleanup_old_tasks(self, max_age_seconds: int = 3600) -> int:+ """Remove completed/failed tasks older than max_age_seconds++ Args:+ max_age_seconds: Maximum age in seconds for completed tasks (default: 1 hour)++ Returns:+ Number of tasks cleaned up+ """+ current_time = time.time()+ cleaned_count = 0++ # Complexity Analysis:+ # O(n) where n = total tasks across all users++ for user_id in list(self.task_store.keys()):+ for task_id in list(self.task_store[user_id].keys()):+ task = self.task_store[user_id][task_id]+ # Only cleanup completed or failed tasks that are old enough+ if (task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED] and+ current_time - task.updated_at > max_age_seconds):+ del self.task_store[user_id][task_id]+ # Clean up the associated lock+ self._task_locks.pop(task_id, None)+ cleaned_count += 1+ logger.debug(+ "Cleaned up old task",+ task_id=task_id,+ user_id=user_id,+ age_seconds=current_time - task.updated_at+ )++ # Remove empty user entries+ if not self.task_store[user_id]:+ del self.task_store[user_id]++ if cleaned_count > 0:+ logger.info("Task cleanup completed", cleaned_count=cleaned_count)++ return cleaned_count+async def cancel_task(self, user_id: str, task_id: str) -> bool:"""Cancel a task if it exists and is not already completed.@@ -493,18 +633,51 @@ class TaskService:# Mark all pending and running file tasks as failedfor file_task in upload_task.file_tasks.values():- if file_task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:- # Increment failed_files counter for both pending and running- # (running files haven't been counted yet in either counter)- upload_task.failed_files += 1-- file_task.status = TaskStatus.FAILED- file_task.error = "Task cancelled by user"- file_task.updated_at = time.time()+ # Lock the entire check-and-modify to prevent race with background tasks+ async with self._get_task_lock(task_id):+ if file_task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:+ # Increment failed_files counter for both pending and running+ # (running files haven't been counted yet in either counter)+ upload_task.failed_files += 1+ file_task.status = TaskStatus.FAILED+ file_task.error = "Task cancelled by user"+ file_task.updated_at = time.time()return True- def shutdown(self):- """Cleanup process pool"""+ async def shutdown(self):+ """Cleanup process pool and cancel all background tasks++ Ensures graceful shutdown by:+ 1. Cancelling the periodic cleanup task+ 2. Cancelling all running background tasks+ 3. Waiting for cancellation to complete+ 4. Shutting down the process pool+ """+ logger.info("Shutting down TaskService", background_tasks_count=len(self.background_tasks))++ # Cancel the periodic cleanup task+ if self._cleanup_task is not None and not self._cleanup_task.done():+ self._cleanup_task.cancel()+ try:+ await self._cleanup_task+ except asyncio.CancelledError:+ pass++ # Cancel all background tasks+ for task in self.background_tasks:+ if not task.done():+ task.cancel()++ # Wait for all tasks to complete cancellation+ if self.background_tasks:+ results = await asyncio.gather(*self.background_tasks, return_exceptions=True)+ # Log any unexpected errors (not CancelledError)+ for i, result in enumerate(results):+ if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):+ logger.warning("Background task raised exception during shutdown", error=str(result))++ # Shutdown the process poolif hasattr(self, "process_pool"):self.process_pool.shutdown(wait=True)+ logger.info("Process pool shutdown complete")diff --git a/tests/unit/__init__.py b/tests/unit/__init__.pynew file mode 100644index 000000000..abb4f7f9a--- /dev/null+++ b/tests/unit/__init__.py@@ -0,0 +1 @@+# Unit tests for OpenRAGdiff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.pynew file mode 100644index 000000000..9fd43cb89--- /dev/null+++ b/tests/unit/test_task_service.py@@ -0,0 +1,246 @@+"""+Basic tests for TaskService improvements+Tests timeout handling, cancellation, and cleanup functionality+"""+import asyncio+import pytest+import time+from unittest.mock import Mock, AsyncMock, patch+from services.task_service import TaskService, IngestionTimeoutError+from models.tasks import TaskStatus, UploadTask, FileTask+++def mock_process_pool():+ """Mock process pool"""+ pool = Mock()+ pool.shutdown = Mock()+ return pool+++def task_service(mock_process_pool):+ """Create TaskService instance with mocked dependencies"""+ mock_doc_service = Mock()+ return TaskService(+ document_service=mock_doc_service,+ process_pool=mock_process_pool,+ ingestion_timeout=2 # Short timeout for testing+ )+++async def test_process_with_timeout_success(task_service):+ """Test that _process_with_timeout completes successfully within timeout"""+ async def quick_task():+ await asyncio.sleep(0.01)+ return "success"++ result = await task_service._process_with_timeout(quick_task(), timeout_seconds=1)+ assert result == "success"+++async def test_process_with_timeout_exceeds(task_service):+ """Test that _process_with_timeout raises IngestionTimeoutError when timeout is exceeded"""+ async def slow_task():+ await asyncio.sleep(10)+ return "should not reach here"++ with pytest.raises(IngestionTimeoutError) as exc_info:+ await task_service._process_with_timeout(slow_task(), timeout_seconds=0.5)++ assert "timed out" in str(exc_info.value).lower()+++async def test_cleanup_old_tasks(task_service):+ """Test that cleanup_old_tasks removes old completed tasks"""+ # Create some test tasks+ old_task = UploadTask(+ task_id="old_task",+ total_files=1,+ file_tasks={"file1": FileTask(file_path="file1")},+ status=TaskStatus.COMPLETED+ )+ old_task.updated_at = time.time() - 7200 # 2 hours ago++ recent_task = UploadTask(+ task_id="recent_task",+ total_files=1,+ file_tasks={"file2": FileTask(file_path="file2")},+ status=TaskStatus.COMPLETED+ )+ recent_task.updated_at = time.time() - 600 # 10 minutes ago++ running_task = UploadTask(+ task_id="running_task",+ total_files=1,+ file_tasks={"file3": FileTask(file_path="file3")},+ status=TaskStatus.RUNNING+ )+ running_task.updated_at = time.time() - 7200 # 2 hours ago but still running++ # Add tasks to store+ task_service.task_store["user1"] = {+ "old_task": old_task,+ "recent_task": recent_task,+ "running_task": running_task+ }+ # Pre-create locks for the tasks+ task_service._task_locks["old_task"] = asyncio.Lock()+ task_service._task_locks["recent_task"] = asyncio.Lock()++ # Cleanup tasks older than 1 hour+ cleaned = await task_service.cleanup_old_tasks(max_age_seconds=3600)++ # Should have cleaned up 1 task (old_task)+ assert cleaned == 1+ assert "old_task" not in task_service.task_store["user1"]+ assert "recent_task" in task_service.task_store["user1"]+ assert "running_task" in task_service.task_store["user1"] # Running tasks not cleaned+ # Verify lock was also cleaned up+ assert "old_task" not in task_service._task_locks+ assert "recent_task" in task_service._task_locks # Recent task lock should remain+++async def test_shutdown_cancels_background_tasks(task_service):+ """Test that shutdown properly cancels background tasks"""+ async def background_work():+ try:+ await asyncio.sleep(10)+ except asyncio.CancelledError:+ raise++ # Add a real background task+ bg_task = asyncio.create_task(background_work())+ task_service.background_tasks.add(bg_task)++ # Give the task a moment to start+ await asyncio.sleep(0.01)++ # Shutdown should cancel it+ await task_service.shutdown()++ # Verify task was actually cancelled (not just done)+ assert bg_task.done()+ assert bg_task.cancelled()+ # Verify process pool was shut down+ task_service.process_pool.shutdown.assert_called_once_with(wait=True)+++def test_counter_consistency_logic(task_service):+ """Test that processed_files counter logic only counts terminal states"""+ # This tests the logic pattern used in the finally block++ task = UploadTask(+ task_id="test_task",+ total_files=3,+ file_tasks={+ "file1": FileTask(file_path="file1", status=TaskStatus.COMPLETED),+ "file2": FileTask(file_path="file2", status=TaskStatus.FAILED),+ "file3": FileTask(file_path="file3", status=TaskStatus.RUNNING),+ }+ )++ # Simulate the counter logic from the finally block+ processed_count = 0+ for file_task in task.file_tasks.values():+ if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:+ processed_count += 1++ # Should only count completed and failed, not running+ assert processed_count == 2++++async def test_concurrent_counter_updates(task_service):+ """Test that concurrent counter updates are thread-safe with locks"""+ # Create a task with multiple files+ task = UploadTask(+ task_id="concurrent_task",+ total_files=10,+ file_tasks={+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)+ for i in range(10)+ },+ status=TaskStatus.RUNNING+ )++ task_service.task_store["user1"] = {"concurrent_task": task}++ # Simulate concurrent updates to counters+ async def update_counter(file_id: str):+ """Simulate processing a file and updating counters"""+ file_task = task.file_tasks[file_id]++ # Simulate some processing time+ await asyncio.sleep(0.01)++ # Update file status+ file_task.status = TaskStatus.COMPLETED++ # Update counters with lock (as done in the actual code)+ async with task_service._get_task_lock("concurrent_task"):+ task.processed_files += 1+ task.successful_files += 1++ # Run all updates concurrently+ await asyncio.gather(*[update_counter(f"file{i}") for i in range(10)])++ # Verify all counters are correct (no race conditions)+ assert task.processed_files == 10+ assert task.successful_files == 10+ assert task.failed_files == 0++ # Verify all file tasks are completed+ for file_task in task.file_tasks.values():+ assert file_task.status == TaskStatus.COMPLETED+++async def test_concurrent_mixed_counter_updates(task_service):+ """Test concurrent updates with mixed success/failure outcomes"""+ task = UploadTask(+ task_id="mixed_task",+ total_files=20,+ file_tasks={+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)+ for i in range(20)+ },+ status=TaskStatus.RUNNING+ )++ task_service.task_store["user1"] = {"mixed_task": task}++ async def update_counter(file_id: str, should_fail: bool):+ """Simulate processing with success or failure"""+ file_task = task.file_tasks[file_id]+ await asyncio.sleep(0.01)++ # Update file status and counters atomically+ async with task_service._get_task_lock("mixed_task"):+ if should_fail:+ file_task.status = TaskStatus.FAILED+ task.failed_files += 1+ else:+ file_task.status = TaskStatus.COMPLETED+ task.successful_files += 1+ task.processed_files += 1++ # Create mix of successful and failed tasks+ tasks = [+ update_counter(f"file{i}", should_fail=(i % 3 == 0))+ for i in range(20)+ ]++ await asyncio.gather(*tasks)++ # Verify counters: 7 failures (0,3,6,9,12,15,18), 13 successes+ assert task.processed_files == 20+ assert task.failed_files == 7+ assert task.successful_files == 13+ assert task.processed_files == task.successful_files + task.failed_files+