Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

(bug): Ingesting folder doesn't seem to work

Mike Pawlowski9f463f12a30b
diff --git a/.env.example b/.env.example
index 4c4cd79b6..68e49b7f5 100644
--- a/.env.example
+++ b/.env.example
@@ -10,6 +10,11 @@ DISABLE_INGEST_WITH_LANGFLOW=false
# LANGFLOW_TIMEOUT=2400
# LANGFLOW_CONNECT_TIMEOUT=30
+# Per-file processing timeout for document ingestion tasks (in seconds)
+# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete
+# Default: 3600 seconds (60 minutes) total timeout
+# INGESTION_TIMEOUT=3600
+
# make one like so https://docs.langflow.org/api-keys-and-authentication#langflow-secret-key
LANGFLOW_SECRET_KEY=
diff --git a/.gitignore b/.gitignore
index 2143d13a6..2c09fc1f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,7 +26,7 @@ wheels/
!frontend/*.json
.DS_Store
-config/
+/config/
.docling.pid
diff --git a/src/config/settings.py b/src/config/settings.py
index a8e9433ad..c4a5d9ce7 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -65,6 +65,11 @@ DISABLE_INGEST_WITH_LANGFLOW = os.getenv(
LANGFLOW_TIMEOUT = float(os.getenv("LANGFLOW_TIMEOUT", "2400")) # 40 minutes
LANGFLOW_CONNECT_TIMEOUT = float(os.getenv("LANGFLOW_CONNECT_TIMEOUT", "30")) # 30 seconds
+# Per-file processing timeout for document ingestion tasks (in seconds)
+# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete
+# Default: 3600 seconds (60 minutes)
+INGESTION_TIMEOUT = int(os.getenv("INGESTION_TIMEOUT", "3600"))
+
def is_no_auth_mode():
"""Check if we're running in no-auth mode (OAuth credentials missing)"""
@@ -90,7 +95,7 @@ OPENAI_EMBEDDING_DIMENSIONS = {
WATSONX_EMBEDDING_DIMENSIONS = {
# IBM Models
-"ibm/granite-embedding-107m-multilingual": 384,
+"ibm/granite-embedding-107m-multilingual": 384,
"ibm/granite-embedding-278m-multilingual": 1024,
"ibm/slate-125m-english-rtrvr": 768,
"ibm/slate-125m-english-rtrvr-v2": 768,
@@ -410,17 +415,17 @@ class AppClients:
# LiteLLM routes based on model name prefixes (openai/, ollama/, watsonx/, etc.)
try:
config = get_openrag_config()
-
+
# Set OpenAI credentials
if config.providers.openai.api_key:
os.environ["OPENAI_API_KEY"] = config.providers.openai.api_key
logger.debug("Loaded OpenAI API key from config")
-
+
# Set Anthropic credentials
if config.providers.anthropic.api_key:
os.environ["ANTHROPIC_API_KEY"] = config.providers.anthropic.api_key
logger.debug("Loaded Anthropic API key from config")
-
+
# Set WatsonX credentials
if config.providers.watsonx.api_key:
os.environ["WATSONX_API_KEY"] = config.providers.watsonx.api_key
@@ -431,13 +436,13 @@ class AppClients:
os.environ["WATSONX_PROJECT_ID"] = config.providers.watsonx.project_id
if config.providers.watsonx.api_key:
logger.debug("Loaded WatsonX credentials from config")
-
+
# Set Ollama endpoint
if config.providers.ollama.endpoint:
os.environ["OLLAMA_BASE_URL"] = config.providers.ollama.endpoint
os.environ["OLLAMA_ENDPOINT"] = config.providers.ollama.endpoint
logger.debug("Loaded Ollama endpoint from config")
-
+
except Exception as e:
logger.debug("Could not load provider credentials from config", error=str(e))
diff --git a/src/main.py b/src/main.py
index a0aeb82cd..aa1c907ff 100644
--- a/src/main.py
+++ b/src/main.py
@@ -68,6 +68,7 @@ from config.settings import (
API_KEYS_INDEX_BODY,
API_KEYS_INDEX_NAME,
DISABLE_INGEST_WITH_LANGFLOW,
+ INGESTION_TIMEOUT,
INDEX_BODY,
INDEX_NAME,
SESSION_SECRET,
@@ -625,7 +626,7 @@ async def initialize_services():
# Initialize services
document_service = DocumentService(session_manager=session_manager)
search_service = SearchService(session_manager)
- task_service = TaskService(document_service, process_pool)
+ task_service = TaskService(document_service, process_pool, ingestion_timeout=INGESTION_TIMEOUT)
chat_service = ChatService()
flows_service = FlowsService()
knowledge_filter_service = KnowledgeFilterService(session_manager)
@@ -1559,6 +1560,9 @@ async def create_app():
app.state.background_tasks.add(t1)
t1.add_done_callback(app.state.background_tasks.discard)
+ # Start periodic task cleanup scheduler
+ services["task_service"].start_cleanup_scheduler()
+
# Start periodic flow backup task (every 5 minutes)
async def periodic_backup():
"""Periodic backup task that runs every 15 minutes"""
@@ -1603,6 +1607,8 @@ async def create_app():
async def shutdown_event():
await TelemetryClient.send_event(Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN)
await cleanup_subscriptions_proper(services)
+ # Cleanup task service (cancels background tasks and process pool)
+ await services["task_service"].shutdown()
# Cleanup async clients
await clients.cleanup()
# Cleanup telemetry client
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 6ebb070d1..9c76379ae 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -2,6 +2,7 @@ import asyncio
import random
import time
import uuid
+from typing import Any, Coroutine, TypeVar
from models.tasks import FileTask, TaskStatus, UploadTask
from session_manager import AnonymousUser
@@ -9,22 +10,68 @@ from utils.gpu_detection import get_worker_count
from utils.logging_config import get_logger
from utils.telemetry import TelemetryClient, Category, MessageId
+T = TypeVar("T")
logger = get_logger(__name__)
+class IngestionTimeoutError(Exception):
+ """Raised when file processing exceeds the configured timeout"""
+ pass
+
+
class TaskService:
- def __init__(self, document_service=None, process_pool=None):
+ # Cleanup interval in seconds (2 hours)
+ CLEANUP_INTERVAL_SECONDS = 2 * 60 * 60
+
+ def __init__(self, document_service=None, process_pool=None, ingestion_timeout=3600):
self.document_service = document_service
self.process_pool = process_pool
self.task_store: dict[
str, dict[str, UploadTask]
] = {} # user_id -> {task_id -> UploadTask}
self.background_tasks = set()
+ self.ingestion_timeout = ingestion_timeout
+ self._cleanup_task: asyncio.Task | None = None
+ # Locks for task counter updates, keyed by task_id
+ # Kept separate from UploadTask to maintain serialization compatibility
+ self._task_locks: dict[str, asyncio.Lock] = {}
if self.process_pool is None:
raise ValueError("TaskService requires a process_pool parameter")
+ def _get_task_lock(self, task_id: str) -> asyncio.Lock:
+ """Get or create a lock for a specific task's counter updates"""
+ if task_id not in self._task_locks:
+ self._task_locks[task_id] = asyncio.Lock()
+ return self._task_locks[task_id]
+
+ def start_cleanup_scheduler(self) -> None:
+ """Start the periodic cleanup background task.
+
+ Should be called once after the event loop is running (e.g., during app startup).
+ """
+ if self._cleanup_task is None or self._cleanup_task.done():
+ self._cleanup_task = asyncio.create_task(self._periodic_cleanup())
+ logger.info(
+ "Started periodic task cleanup scheduler",
+ interval_seconds=self.CLEANUP_INTERVAL_SECONDS,
+ )
+
+ async def _periodic_cleanup(self) -> None:
+ """Periodically clean up old completed/failed tasks."""
+ while True:
+ try:
+ await asyncio.sleep(self.CLEANUP_INTERVAL_SECONDS)
+ cleaned = await self.cleanup_old_tasks()
+ if cleaned > 0:
+ logger.debug("Periodic cleanup completed", tasks_cleaned=cleaned)
+ except asyncio.CancelledError:
+ logger.debug("Periodic cleanup task cancelled")
+ raise
+ except Exception as e:
+ logger.warning("Error during periodic cleanup", error=str(e))
+
async def exponential_backoff_delay(
self, retry_count: int, base_delay: float = 1.0, max_delay: float = 60.0
) -> None:
@@ -32,6 +79,28 @@ class TaskService:
delay = min(base_delay * (2**retry_count) + random.uniform(0, 1), max_delay)
await asyncio.sleep(delay)
+ async def _process_with_timeout(
+ self, coro: Coroutine[Any, Any, T], timeout_seconds: int | None = None
+ ) -> T:
+ """Wrapper to add timeout protection to file processing
+
+ Args:
+ coro: Coroutine to execute with timeout
+ timeout_seconds: Timeout in seconds (uses self.ingestion_timeout if None)
+
+ Returns:
+ The result of the coroutine
+
+ Raises:
+ IngestionTimeoutError: If processing exceeds timeout
+ """
+ timeout = timeout_seconds or self.ingestion_timeout
+ try:
+ return await asyncio.wait_for(coro, timeout=timeout)
+ except asyncio.TimeoutError:
+ logger.error("File processing timed out", timeout_seconds=timeout)
+ raise IngestionTimeoutError(f"Processing timed out after {timeout}s") from None
+
async def create_upload_task(
self,
user_id: str,
@@ -251,7 +320,33 @@ class TaskService:
file_task.updated_at = time.time()
try:
- await processor.process_item(upload_task, item, file_task)
+ # Add timeout protection to prevent indefinite hangs
+ await self._process_with_timeout(
+ processor.process_item(upload_task, item, file_task),
+ timeout_seconds=self.ingestion_timeout
+ )
+ except asyncio.CancelledError:
+ # Handle cancellation explicitly
+ logger.info("File processing cancelled", item=str(item))
+ if file_task.status == TaskStatus.RUNNING:
+ file_task.status = TaskStatus.FAILED
+ file_task.error = "Task cancelled by user"
+ async with self._get_task_lock(task_id):
+ upload_task.failed_files += 1
+ raise # Re-raise to propagate cancellation
+ except IngestionTimeoutError as e:
+ # Handle timeout explicitly with specific error message
+ logger.warning(
+ "File processing timed out",
+ item=str(item),
+ timeout=self.ingestion_timeout
+ )
+ if file_task.status == TaskStatus.RUNNING:
+ file_task.status = TaskStatus.FAILED
+ file_task.error = f"Processing timed out after {self.ingestion_timeout}s"
+ async with self._get_task_lock(task_id):
+ upload_task.failed_files += 1
+ # Don't re-raise - treat as normal failure, not cancellation
except Exception as e:
logger.error(
"Failed to process item", item=str(item), error=str(e)
@@ -268,7 +363,11 @@ class TaskService:
file_task.error = str(e)
finally:
file_task.updated_at = time.time()
- upload_task.processed_files += 1
+ # Only increment processed_files if the file reached a terminal state
+ # This prevents counter inconsistency on cancellation
+ if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+ async with self._get_task_lock(task_id):
+ upload_task.processed_files += 1
upload_task.updated_at = time.time()
tasks = [process_with_semaphore(item, str(item)) for item in items]
@@ -447,6 +546,47 @@ class TaskService:
tasks.sort(key=lambda x: x["created_at"], reverse=True)
return tasks
+ async def cleanup_old_tasks(self, max_age_seconds: int = 3600) -> int:
+ """Remove completed/failed tasks older than max_age_seconds
+
+ Args:
+ max_age_seconds: Maximum age in seconds for completed tasks (default: 1 hour)
+
+ Returns:
+ Number of tasks cleaned up
+ """
+ current_time = time.time()
+ cleaned_count = 0
+
+ # Complexity Analysis:
+ # O(n) where n = total tasks across all users
+
+ for user_id in list(self.task_store.keys()):
+ for task_id in list(self.task_store[user_id].keys()):
+ task = self.task_store[user_id][task_id]
+ # Only cleanup completed or failed tasks that are old enough
+ if (task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED] and
+ current_time - task.updated_at > max_age_seconds):
+ del self.task_store[user_id][task_id]
+ # Clean up the associated lock
+ self._task_locks.pop(task_id, None)
+ cleaned_count += 1
+ logger.debug(
+ "Cleaned up old task",
+ task_id=task_id,
+ user_id=user_id,
+ age_seconds=current_time - task.updated_at
+ )
+
+ # Remove empty user entries
+ if not self.task_store[user_id]:
+ del self.task_store[user_id]
+
+ if cleaned_count > 0:
+ logger.info("Task cleanup completed", cleaned_count=cleaned_count)
+
+ return cleaned_count
+
async def cancel_task(self, user_id: str, task_id: str) -> bool:
"""Cancel a task if it exists and is not already completed.
@@ -493,18 +633,51 @@ class TaskService:
# Mark all pending and running file tasks as failed
for file_task in upload_task.file_tasks.values():
- if file_task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:
- # Increment failed_files counter for both pending and running
- # (running files haven't been counted yet in either counter)
- upload_task.failed_files += 1
-
- file_task.status = TaskStatus.FAILED
- file_task.error = "Task cancelled by user"
- file_task.updated_at = time.time()
+ # Lock the entire check-and-modify to prevent race with background tasks
+ async with self._get_task_lock(task_id):
+ if file_task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:
+ # Increment failed_files counter for both pending and running
+ # (running files haven't been counted yet in either counter)
+ upload_task.failed_files += 1
+ file_task.status = TaskStatus.FAILED
+ file_task.error = "Task cancelled by user"
+ file_task.updated_at = time.time()
return True
- def shutdown(self):
- """Cleanup process pool"""
+ async def shutdown(self):
+ """Cleanup process pool and cancel all background tasks
+
+ Ensures graceful shutdown by:
+ 1. Cancelling the periodic cleanup task
+ 2. Cancelling all running background tasks
+ 3. Waiting for cancellation to complete
+ 4. Shutting down the process pool
+ """
+ logger.info("Shutting down TaskService", background_tasks_count=len(self.background_tasks))
+
+ # Cancel the periodic cleanup task
+ if self._cleanup_task is not None and not self._cleanup_task.done():
+ self._cleanup_task.cancel()
+ try:
+ await self._cleanup_task
+ except asyncio.CancelledError:
+ pass
+
+ # Cancel all background tasks
+ for task in self.background_tasks:
+ if not task.done():
+ task.cancel()
+
+ # Wait for all tasks to complete cancellation
+ if self.background_tasks:
+ results = await asyncio.gather(*self.background_tasks, return_exceptions=True)
+ # Log any unexpected errors (not CancelledError)
+ for i, result in enumerate(results):
+ if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
+ logger.warning("Background task raised exception during shutdown", error=str(result))
+
+ # Shutdown the process pool
if hasattr(self, "process_pool"):
self.process_pool.shutdown(wait=True)
+ logger.info("Process pool shutdown complete")
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 000000000..abb4f7f9a
--- /dev/null
+++ b/tests/unit/__init__.py
@@ -0,0 +1 @@
+# Unit tests for OpenRAG
diff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.py
new file mode 100644
index 000000000..9fd43cb89
--- /dev/null
+++ b/tests/unit/test_task_service.py
@@ -0,0 +1,246 @@
+"""
+Basic tests for TaskService improvements
+Tests timeout handling, cancellation, and cleanup functionality
+"""
+import asyncio
+import pytest
+import time
+from unittest.mock import Mock, AsyncMock, patch
+from services.task_service import TaskService, IngestionTimeoutError
+from models.tasks import TaskStatus, UploadTask, FileTask
+
+
+def mock_process_pool():
+ """Mock process pool"""
+ pool = Mock()
+ pool.shutdown = Mock()
+ return pool
+
+
+def task_service(mock_process_pool):
+ """Create TaskService instance with mocked dependencies"""
+ mock_doc_service = Mock()
+ return TaskService(
+ document_service=mock_doc_service,
+ process_pool=mock_process_pool,
+ ingestion_timeout=2 # Short timeout for testing
+ )
+
+
+async def test_process_with_timeout_success(task_service):
+ """Test that _process_with_timeout completes successfully within timeout"""
+ async def quick_task():
+ await asyncio.sleep(0.01)
+ return "success"
+
+ result = await task_service._process_with_timeout(quick_task(), timeout_seconds=1)
+ assert result == "success"
+
+
+async def test_process_with_timeout_exceeds(task_service):
+ """Test that _process_with_timeout raises IngestionTimeoutError when timeout is exceeded"""
+ async def slow_task():
+ await asyncio.sleep(10)
+ return "should not reach here"
+
+ with pytest.raises(IngestionTimeoutError) as exc_info:
+ await task_service._process_with_timeout(slow_task(), timeout_seconds=0.5)
+
+ assert "timed out" in str(exc_info.value).lower()
+
+
+async def test_cleanup_old_tasks(task_service):
+ """Test that cleanup_old_tasks removes old completed tasks"""
+ # Create some test tasks
+ old_task = UploadTask(
+ task_id="old_task",
+ total_files=1,
+ file_tasks={"file1": FileTask(file_path="file1")},
+ status=TaskStatus.COMPLETED
+ )
+ old_task.updated_at = time.time() - 7200 # 2 hours ago
+
+ recent_task = UploadTask(
+ task_id="recent_task",
+ total_files=1,
+ file_tasks={"file2": FileTask(file_path="file2")},
+ status=TaskStatus.COMPLETED
+ )
+ recent_task.updated_at = time.time() - 600 # 10 minutes ago
+
+ running_task = UploadTask(
+ task_id="running_task",
+ total_files=1,
+ file_tasks={"file3": FileTask(file_path="file3")},
+ status=TaskStatus.RUNNING
+ )
+ running_task.updated_at = time.time() - 7200 # 2 hours ago but still running
+
+ # Add tasks to store
+ task_service.task_store["user1"] = {
+ "old_task": old_task,
+ "recent_task": recent_task,
+ "running_task": running_task
+ }
+ # Pre-create locks for the tasks
+ task_service._task_locks["old_task"] = asyncio.Lock()
+ task_service._task_locks["recent_task"] = asyncio.Lock()
+
+ # Cleanup tasks older than 1 hour
+ cleaned = await task_service.cleanup_old_tasks(max_age_seconds=3600)
+
+ # Should have cleaned up 1 task (old_task)
+ assert cleaned == 1
+ assert "old_task" not in task_service.task_store["user1"]
+ assert "recent_task" in task_service.task_store["user1"]
+ assert "running_task" in task_service.task_store["user1"] # Running tasks not cleaned
+ # Verify lock was also cleaned up
+ assert "old_task" not in task_service._task_locks
+ assert "recent_task" in task_service._task_locks # Recent task lock should remain
+
+
+async def test_shutdown_cancels_background_tasks(task_service):
+ """Test that shutdown properly cancels background tasks"""
+ async def background_work():
+ try:
+ await asyncio.sleep(10)
+ except asyncio.CancelledError:
+ raise
+
+ # Add a real background task
+ bg_task = asyncio.create_task(background_work())
+ task_service.background_tasks.add(bg_task)
+
+ # Give the task a moment to start
+ await asyncio.sleep(0.01)
+
+ # Shutdown should cancel it
+ await task_service.shutdown()
+
+ # Verify task was actually cancelled (not just done)
+ assert bg_task.done()
+ assert bg_task.cancelled()
+ # Verify process pool was shut down
+ task_service.process_pool.shutdown.assert_called_once_with(wait=True)
+
+
+def test_counter_consistency_logic(task_service):
+ """Test that processed_files counter logic only counts terminal states"""
+ # This tests the logic pattern used in the finally block
+
+ task = UploadTask(
+ task_id="test_task",
+ total_files=3,
+ file_tasks={
+ "file1": FileTask(file_path="file1", status=TaskStatus.COMPLETED),
+ "file2": FileTask(file_path="file2", status=TaskStatus.FAILED),
+ "file3": FileTask(file_path="file3", status=TaskStatus.RUNNING),
+ }
+ )
+
+ # Simulate the counter logic from the finally block
+ processed_count = 0
+ for file_task in task.file_tasks.values():
+ if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+ processed_count += 1
+
+ # Should only count completed and failed, not running
+ assert processed_count == 2
+
+
+
+async def test_concurrent_counter_updates(task_service):
+ """Test that concurrent counter updates are thread-safe with locks"""
+ # Create a task with multiple files
+ task = UploadTask(
+ task_id="concurrent_task",
+ total_files=10,
+ file_tasks={
+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)
+ for i in range(10)
+ },
+ status=TaskStatus.RUNNING
+ )
+
+ task_service.task_store["user1"] = {"concurrent_task": task}
+
+ # Simulate concurrent updates to counters
+ async def update_counter(file_id: str):
+ """Simulate processing a file and updating counters"""
+ file_task = task.file_tasks[file_id]
+
+ # Simulate some processing time
+ await asyncio.sleep(0.01)
+
+ # Update file status
+ file_task.status = TaskStatus.COMPLETED
+
+ # Update counters with lock (as done in the actual code)
+ async with task_service._get_task_lock("concurrent_task"):
+ task.processed_files += 1
+ task.successful_files += 1
+
+ # Run all updates concurrently
+ await asyncio.gather(*[update_counter(f"file{i}") for i in range(10)])
+
+ # Verify all counters are correct (no race conditions)
+ assert task.processed_files == 10
+ assert task.successful_files == 10
+ assert task.failed_files == 0
+
+ # Verify all file tasks are completed
+ for file_task in task.file_tasks.values():
+ assert file_task.status == TaskStatus.COMPLETED
+
+
+async def test_concurrent_mixed_counter_updates(task_service):
+ """Test concurrent updates with mixed success/failure outcomes"""
+ task = UploadTask(
+ task_id="mixed_task",
+ total_files=20,
+ file_tasks={
+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)
+ for i in range(20)
+ },
+ status=TaskStatus.RUNNING
+ )
+
+ task_service.task_store["user1"] = {"mixed_task": task}
+
+ async def update_counter(file_id: str, should_fail: bool):
+ """Simulate processing with success or failure"""
+ file_task = task.file_tasks[file_id]
+ await asyncio.sleep(0.01)
+
+ # Update file status and counters atomically
+ async with task_service._get_task_lock("mixed_task"):
+ if should_fail:
+ file_task.status = TaskStatus.FAILED
+ task.failed_files += 1
+ else:
+ file_task.status = TaskStatus.COMPLETED
+ task.successful_files += 1
+ task.processed_files += 1
+
+ # Create mix of successful and failed tasks
+ tasks = [
+ update_counter(f"file{i}", should_fail=(i % 3 == 0))
+ for i in range(20)
+ ]
+
+ await asyncio.gather(*tasks)
+
+ # Verify counters: 7 failures (0,3,6,9,12,15,18), 13 successes
+ assert task.processed_files == 20
+ assert task.failed_files == 7
+ assert task.successful_files == 13
+ assert task.processed_files == task.successful_files + task.failed_files
+