OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
(bug): Ingesting folder doesn't seem to work
Mike Pawlowski3d5e1893c772
diff --git a/.env.example b/.env.exampleindex 68e49b7f5..97ffa94c2 100644--- a/.env.example+++ b/.env.example@@ -114,3 +114,8 @@ NEXT_ALLOWED_DEV_ORIGINS=# MAX_WORKERS=4 # Backend ingestion workers (default: min(4, CPU_COUNT // 2))# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)# DOCLING_WORKERS=2 # Docling workers (default: 1)++# OPTIONAL: Enable or disable HTTP access logging events+# - e.g. INFO: 127.0.0.1:45132 - "GET /tasks HTTP/1.1" 200 OK+# - Default: true+# ACCESS_LOG=falsediff --git a/src/main.py b/src/main.pyindex aa1c907ff..5168209c7 100644--- a/src/main.py+++ b/src/main.py@@ -1682,6 +1682,9 @@ if __name__ == "__main__":# Create app asynchronouslyapp = asyncio.run(create_app())+ # Enable or disable HTTP access logging events+ access_log = os.getenv("ACCESS_LOG", "true").lower() == "true"+# Run the server (startup tasks now handled by Starlette startup event)uvicorn.run(app,@@ -1689,4 +1692,5 @@ if __name__ == "__main__":host="0.0.0.0",port=8000,reload=False, # Disable reload since we're running from main+ access_log=access_log,)diff --git a/src/models/tasks.py b/src/models/tasks.pyindex 253cabb54..af0466587 100644--- a/src/models/tasks.py+++ b/src/models/tasks.py@@ -1,7 +1,8 @@+import itertoolsimport timefrom dataclasses import dataclass, fieldfrom enum import Enum-from typing import Dict, Optional+from typing import ClassVar, Dict, Optionalclass TaskStatus(Enum):@@ -30,6 +31,8 @@ class FileTask:@dataclassclass UploadTask:+ _id_counter: ClassVar[itertools.count] = itertools.count(1)+task_id: strtotal_files: intprocessed_files: int = 0@@ -39,7 +42,15 @@ class UploadTask:status: TaskStatus = TaskStatus.PENDINGcreated_at: float = field(default_factory=time.time)updated_at: float = field(default_factory=time.time)-+ _sequence_number: int = field(init=False, repr=False)++ def __post_init__(self):+ self._sequence_number = next(UploadTask._id_counter)++ @property+ def sequence_number(self) -> int:+ return self._sequence_number+@propertydef duration_seconds(self) -> float:"""Duration in seconds from creation to last update"""diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 9c76379ae..c7ae2f3b8 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -1,4 +1,5 @@import asyncio+import osimport randomimport timeimport uuid@@ -94,12 +95,11 @@ class TaskService:Raises:IngestionTimeoutError: If processing exceeds timeout"""- timeout = timeout_seconds or self.ingestion_timeout+ timeout: int = timeout_seconds or self.ingestion_timeouttry:return await asyncio.wait_for(coro, timeout=timeout)except asyncio.TimeoutError:- logger.error("File processing timed out", timeout_seconds=timeout)- raise IngestionTimeoutError(f"Processing timed out after {timeout}s") from None+ raise IngestionTimeoutError(f"File processing timed out after {timeout} seconds.") from Noneasync def create_upload_task(self,@@ -215,98 +215,61 @@ class TaskService:return task_id- async def background_upload_processor(self, user_id: str, task_id: str) -> None:- """Background task to process all files in an upload job with concurrency control"""- try:- upload_task = self.task_store[user_id][task_id]- upload_task.status = TaskStatus.RUNNING- upload_task.updated_at = time.time()+ def _get_display_filenames(self, upload_task: UploadTask) -> list[str]:+ filenames: list[str] = [+ task.filename or os.path.basename(task.file_path)+ for task in upload_task.file_tasks.values()+ ]- # Process items with limited concurrency- # - Semaphore matches worker count- # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O- worker_count = get_worker_count()- semaphore = asyncio.Semaphore(worker_count)+ if len(filenames) <= 3:+ # e.g. ['book-1.xlsx']+ # e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx']+ return filenames+ # e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx', '...']+ return filenames[:3] + ["..."]- async def process_with_semaphore(file_path: str):- async with semaphore:- from models.processors import DocumentFileProcessor- file_task = upload_task.file_tasks[file_path]-- # Create processor with user context (all None for background processing)- processor = DocumentFileProcessor(- document_service=self.document_service,- owner_user_id=None,- jwt_token=None,- owner_name=None,- owner_email=None,- )+ def _format_duration(self, duration: float | int) -> str:+ """+ Convert specified duration (seconds) into a human-readable string:+ - < 60 s → "45s"+ - < 60 min → "3m 42s"+ - ≥ 60 min → "2h 14m 35s"+ """+ total_seconds: int = max(0, int(duration))- # Process the file- await processor.process_item(upload_task, file_path, file_task)+ if total_seconds < 60:+ return f"{total_seconds}s"- tasks = [- process_with_semaphore(file_path)- for file_path in upload_task.file_tasks.keys()- ]+ mins, secs = divmod(total_seconds, 60)- await asyncio.gather(*tasks, return_exceptions=True)+ if mins < 60:+ return f"{mins}m {secs}s"- # Check if task is complete- if upload_task.processed_files >= upload_task.total_files:- upload_task.status = TaskStatus.COMPLETED- upload_task.updated_at = time.time()+ hours, mins = divmod(mins, 60)- # Send telemetry for task completion- asyncio.create_task(- TelemetryClient.send_event(- Category.TASK_OPERATIONS,- MessageId.ORB_TASK_COMPLETE,- metadata={- "total_files": upload_task.total_files,- "successful_files": upload_task.successful_files,- "failed_files": upload_task.failed_files,- }- )- )-- except Exception as e:- logger.error(- "Background upload processor failed", task_id=task_id, error=str(e)- )- import traceback-- traceback.print_exc()- if user_id in self.task_store and task_id in self.task_store[user_id]:- failed_task = self.task_store[user_id][task_id]- failed_task.status = TaskStatus.FAILED- failed_task.updated_at = time.time()-- # Send telemetry for task failure- asyncio.create_task(- TelemetryClient.send_event(- Category.TASK_OPERATIONS,- MessageId.ORB_TASK_FAILED,- metadata={- "total_files": failed_task.total_files,- "processed_files": failed_task.processed_files,- "successful_files": failed_task.successful_files,- "failed_files": failed_task.failed_files,- }- )- )+ return f"{hours}h {mins}m {secs}s"async def background_custom_processor(self, user_id: str, task_id: str, items: list) -> None:"""Background task to process items using custom processor"""try:- upload_task = self.task_store[user_id][task_id]+ upload_task: UploadTask = self.task_store[user_id][task_id]upload_task.status = TaskStatus.RUNNINGupload_task.updated_at = time.time()processor = upload_task.processor+ logger.info(+ "Upload / ingestion task started",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ total_files=upload_task.total_files,+ filenames=self._get_display_filenames(upload_task),+ processor_type=processor.__class__.__name__,+ user_id=user_id,+ )+# Process items with limited concurrency# - Semaphore matches worker count# - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O@@ -319,41 +282,65 @@ class TaskService:file_task.status = TaskStatus.RUNNINGfile_task.updated_at = time.time()+ logger.info(+ "File processing task running",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ filename=file_task.filename,+ )+try:# Add timeout protection to prevent indefinite hangsawait self._process_with_timeout(processor.process_item(upload_task, item, file_task),timeout_seconds=self.ingestion_timeout)++ logger.info(+ "File processing task succeeded",+ status="PASSED",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ filename=file_task.filename,+ )+except asyncio.CancelledError:# Handle cancellation explicitly- logger.info("File processing cancelled", item=str(item))+if file_task.status == TaskStatus.RUNNING:file_task.status = TaskStatus.FAILED- file_task.error = "Task cancelled by user"+ file_task.error = "File processing task cancelled."async with self._get_task_lock(task_id):upload_task.failed_files += 1- raise # Re-raise to propagate cancellation- except IngestionTimeoutError as e:- # Handle timeout explicitly with specific error message+logger.warning(- "File processing timed out",- item=str(item),- timeout=self.ingestion_timeout+ "File processing task cancelled",+ status="FAILED",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ filename=file_task.filename,)++ raise # Re-raise to propagate cancellation+ except IngestionTimeoutError as e:+ # Handle timeout explicitlyif file_task.status == TaskStatus.RUNNING:file_task.status = TaskStatus.FAILED- file_task.error = f"Processing timed out after {self.ingestion_timeout}s"+ file_task.error = str(e)async with self._get_task_lock(task_id):upload_task.failed_files += 1# Don't re-raise - treat as normal failure, not cancellation- except Exception as e:+logger.error(- "Failed to process item", item=str(item), error=str(e)+ "File processing task timed out",+ status="FAILED",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ filename=file_task.filename,+ exception=str(e),)- import traceback- traceback.print_exc()+ except Exception as e:# Note: Processors already handle incrementing failed_files and# setting file_task status/error, so we don't duplicate that here.# Only update timestamp if processor didn't already set it@@ -361,6 +348,16 @@ class TaskService:file_task.status = TaskStatus.FAILEDif not file_task.error:file_task.error = str(e)++ logger.error(+ "File processing task exception encountered",+ status="FAILED",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ filename=file_task.filename,+ exception=str(e),+ )+finally:file_task.updated_at = time.time()# Only increment processed_files if the file reached a terminal state@@ -378,6 +375,28 @@ class TaskService:upload_task.status = TaskStatus.COMPLETEDupload_task.updated_at = time.time()+ status: str = "FAILED"++ if upload_task.failed_files == 0:+ status = "PASSED"+ elif upload_task.successful_files > 0:+ status = "FAILED (partial success)"++ logger.info(+ "Upload / ingestion task finished",+ status=status,+ task_number=upload_task.sequence_number,+ task_id=task_id,+ duration=self._format_duration(upload_task.duration_seconds),+ total_files=upload_task.total_files,+ processed_files=upload_task.processed_files,+ successful_files=upload_task.successful_files,+ failed_files=upload_task.failed_files,+ filenames=self._get_display_filenames(upload_task),+ processor_type=processor.__class__.__name__,+ user_id=user_id,+ )+# Send telemetry for task completionasyncio.create_task(TelemetryClient.send_event(@@ -392,22 +411,54 @@ class TaskService:)except asyncio.CancelledError:- logger.info("Background processor cancelled", task_id=task_id)if user_id in self.task_store and task_id in self.task_store[user_id]:# Task status and pending files already handled by cancel_task()- pass+ upload_task = self.task_store[user_id][task_id]++ logger.warning(+ "Upload / ingestion task cancelled",+ status="FAILED",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ duration=self._format_duration(upload_task.duration_seconds),+ total_files=upload_task.total_files,+ processed_files=upload_task.processed_files,+ successful_files=upload_task.successful_files,+ failed_files=upload_task.failed_files,+ filenames=self._get_display_filenames(upload_task),+ processor_type=upload_task.processor.__class__.__name__,+ user_id=user_id,+ )+ else:+ logger.warning(+ "Upload / ingestion task cancelled",+ status="FAILED",+ task_id=task_id,+ user_id=user_id,+ )+raise # Re-raise to properly handle cancellationexcept Exception as e:- logger.error(- "Background custom processor failed", task_id=task_id, error=str(e)- )- import traceback-- traceback.print_exc()if user_id in self.task_store and task_id in self.task_store[user_id]:- failed_task = self.task_store[user_id][task_id]- failed_task.status = TaskStatus.FAILED- failed_task.updated_at = time.time()+ upload_task = self.task_store[user_id][task_id]+ upload_task.status = TaskStatus.FAILED+ upload_task.updated_at = time.time()++ logger.error(+ "Upload / ingestion task exception encountered",+ status="FAILED",+ task_number=upload_task.sequence_number,+ task_id=task_id,+ duration=self._format_duration(upload_task.duration_seconds),+ total_files=upload_task.total_files,+ processed_files=upload_task.processed_files,+ successful_files=upload_task.successful_files,+ failed_files=upload_task.failed_files,+ filenames=self._get_display_filenames(upload_task),+ processor_type=upload_task.processor.__class__.__name__,+ user_id=user_id,+ exception=str(e),+ )# Send telemetry for task failureasyncio.create_task(@@ -415,13 +466,21 @@ class TaskService:Category.TASK_OPERATIONS,MessageId.ORB_TASK_FAILED,metadata={- "total_files": failed_task.total_files,- "processed_files": failed_task.processed_files,- "successful_files": failed_task.successful_files,- "failed_files": failed_task.failed_files,+ "total_files": upload_task.total_files,+ "processed_files": upload_task.processed_files,+ "successful_files": upload_task.successful_files,+ "failed_files": upload_task.failed_files,}))+ else:+ logger.error(+ "Upload / ingestion exception encountered",+ status="FAILED",+ task_id=task_id,+ user_id=user_id,+ exception=str(e),+ )def get_task_status(self, user_id: str, task_id: str) -> dict | None:"""Get the status of a specific upload taskdiff --git a/src/utils/logging_config.py b/src/utils/logging_config.pyindex 76fd3cc7b..b7a66389c 100644--- a/src/utils/logging_config.py+++ b/src/utils/logging_config.py@@ -48,24 +48,31 @@ def configure_logging(console_renderer = structlog.processors.JSONRenderer()else:# Custom clean format: timestamp path/file:loc logentry+ LOC_WIDTH_SHORT = 35+ LOC_WIDTH_LONG = 60+def custom_formatter(logger, log_method, event_dict):timestamp = event_dict.pop("timestamp", "")pathname = event_dict.pop("pathname", "")filename = event_dict.pop("filename", "")lineno = event_dict.pop("lineno", "")- level = event_dict.pop("level", "")+ level = event_dict.pop("level", "").upper()- # Build file location - prefer pathname for full path, fallback to filename- if pathname and lineno:- location = f"{pathname}:{lineno}"- elif filename and lineno:+ if filename and lineno:location = f"{filename}:{lineno}"- elif pathname:- location = pathname+ loc_width = LOC_WIDTH_SHORT+ elif pathname and lineno:+ location = f"{pathname}:{lineno}"+ loc_width = LOC_WIDTH_LONGelif filename:location = filename+ loc_width = LOC_WIDTH_SHORT+ elif pathname:+ location = pathname+ loc_width = LOC_WIDTH_LONGelse:location = "unknown"+ loc_width = LOC_WIDTH_SHORT# Build the main messagemessage_parts = []@@ -73,14 +80,18 @@ def configure_logging(if event:message_parts.append(event)- # Add any remaining context- for key, value in event_dict.items():- if key not in ["service", "func_name"]: # Skip internal fields- message_parts.append(f"{key}={value}")+ header = f"[{timestamp}] [{level:<8}] [{location:<{loc_width}}] "++ # Add any remaining context as indented multi-line fields+ extra = {k: v for k, v in event_dict.items() if k not in ["service", "func_name"]}+ if extra:+ padding = " " * len(header)+ for key, value in extra.items():+ message_parts.append(f"\n{padding}- {key}: {value}")- message = " ".join(message_parts)+ message = "".join(message_parts)- return f"{timestamp} {location} {message}"+ return f"{header}{message}"console_renderer = custom_formatter