Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

(bug): Ingesting folder doesn't seem to work

Mike Pawlowski3d5e1893c772
diff --git a/.env.example b/.env.example
index 68e49b7f5..97ffa94c2 100644
--- a/.env.example
+++ b/.env.example
@@ -114,3 +114,8 @@ NEXT_ALLOWED_DEV_ORIGINS=
# MAX_WORKERS=4 # Backend ingestion workers (default: min(4, CPU_COUNT // 2))
# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)
# DOCLING_WORKERS=2 # Docling workers (default: 1)
+
+# OPTIONAL: Enable or disable HTTP access logging events
+# - e.g. INFO: 127.0.0.1:45132 - "GET /tasks HTTP/1.1" 200 OK
+# - Default: true
+# ACCESS_LOG=false
diff --git a/src/main.py b/src/main.py
index aa1c907ff..5168209c7 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1682,6 +1682,9 @@ if __name__ == "__main__":
# Create app asynchronously
app = asyncio.run(create_app())
+ # Enable or disable HTTP access logging events
+ access_log = os.getenv("ACCESS_LOG", "true").lower() == "true"
+
# Run the server (startup tasks now handled by Starlette startup event)
uvicorn.run(
app,
@@ -1689,4 +1692,5 @@ if __name__ == "__main__":
host="0.0.0.0",
port=8000,
reload=False, # Disable reload since we're running from main
+ access_log=access_log,
)
diff --git a/src/models/tasks.py b/src/models/tasks.py
index 253cabb54..af0466587 100644
--- a/src/models/tasks.py
+++ b/src/models/tasks.py
@@ -1,7 +1,8 @@
+import itertools
import time
from dataclasses import dataclass, field
from enum import Enum
-from typing import Dict, Optional
+from typing import ClassVar, Dict, Optional
class TaskStatus(Enum):
@@ -30,6 +31,8 @@ class FileTask:
@dataclass
class UploadTask:
+ _id_counter: ClassVar[itertools.count] = itertools.count(1)
+
task_id: str
total_files: int
processed_files: int = 0
@@ -39,7 +42,15 @@ class UploadTask:
status: TaskStatus = TaskStatus.PENDING
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
-
+ _sequence_number: int = field(init=False, repr=False)
+
+ def __post_init__(self):
+ self._sequence_number = next(UploadTask._id_counter)
+
+ @property
+ def sequence_number(self) -> int:
+ return self._sequence_number
+
@property
def duration_seconds(self) -> float:
"""Duration in seconds from creation to last update"""
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 9c76379ae..c7ae2f3b8 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -1,4 +1,5 @@
import asyncio
+import os
import random
import time
import uuid
@@ -94,12 +95,11 @@ class TaskService:
Raises:
IngestionTimeoutError: If processing exceeds timeout
"""
- timeout = timeout_seconds or self.ingestion_timeout
+ timeout: int = timeout_seconds or self.ingestion_timeout
try:
return await asyncio.wait_for(coro, timeout=timeout)
except asyncio.TimeoutError:
- logger.error("File processing timed out", timeout_seconds=timeout)
- raise IngestionTimeoutError(f"Processing timed out after {timeout}s") from None
+ raise IngestionTimeoutError(f"File processing timed out after {timeout} seconds.") from None
async def create_upload_task(
self,
@@ -215,98 +215,61 @@ class TaskService:
return task_id
- async def background_upload_processor(self, user_id: str, task_id: str) -> None:
- """Background task to process all files in an upload job with concurrency control"""
- try:
- upload_task = self.task_store[user_id][task_id]
- upload_task.status = TaskStatus.RUNNING
- upload_task.updated_at = time.time()
+ def _get_display_filenames(self, upload_task: UploadTask) -> list[str]:
+ filenames: list[str] = [
+ task.filename or os.path.basename(task.file_path)
+ for task in upload_task.file_tasks.values()
+ ]
- # Process items with limited concurrency
- # - Semaphore matches worker count
- # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
- worker_count = get_worker_count()
- semaphore = asyncio.Semaphore(worker_count)
+ if len(filenames) <= 3:
+ # e.g. ['book-1.xlsx']
+ # e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx']
+ return filenames
+ # e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx', '...']
+ return filenames[:3] + ["..."]
- async def process_with_semaphore(file_path: str):
- async with semaphore:
- from models.processors import DocumentFileProcessor
- file_task = upload_task.file_tasks[file_path]
-
- # Create processor with user context (all None for background processing)
- processor = DocumentFileProcessor(
- document_service=self.document_service,
- owner_user_id=None,
- jwt_token=None,
- owner_name=None,
- owner_email=None,
- )
+ def _format_duration(self, duration: float | int) -> str:
+ """
+ Convert specified duration (seconds) into a human-readable string:
+ - < 60 s → "45s"
+ - < 60 min → "3m 42s"
+ - ≥ 60 min → "2h 14m 35s"
+ """
+ total_seconds: int = max(0, int(duration))
- # Process the file
- await processor.process_item(upload_task, file_path, file_task)
+ if total_seconds < 60:
+ return f"{total_seconds}s"
- tasks = [
- process_with_semaphore(file_path)
- for file_path in upload_task.file_tasks.keys()
- ]
+ mins, secs = divmod(total_seconds, 60)
- await asyncio.gather(*tasks, return_exceptions=True)
+ if mins < 60:
+ return f"{mins}m {secs}s"
- # Check if task is complete
- if upload_task.processed_files >= upload_task.total_files:
- upload_task.status = TaskStatus.COMPLETED
- upload_task.updated_at = time.time()
+ hours, mins = divmod(mins, 60)
- # Send telemetry for task completion
- asyncio.create_task(
- TelemetryClient.send_event(
- Category.TASK_OPERATIONS,
- MessageId.ORB_TASK_COMPLETE,
- metadata={
- "total_files": upload_task.total_files,
- "successful_files": upload_task.successful_files,
- "failed_files": upload_task.failed_files,
- }
- )
- )
-
- except Exception as e:
- logger.error(
- "Background upload processor failed", task_id=task_id, error=str(e)
- )
- import traceback
-
- traceback.print_exc()
- if user_id in self.task_store and task_id in self.task_store[user_id]:
- failed_task = self.task_store[user_id][task_id]
- failed_task.status = TaskStatus.FAILED
- failed_task.updated_at = time.time()
-
- # Send telemetry for task failure
- asyncio.create_task(
- TelemetryClient.send_event(
- Category.TASK_OPERATIONS,
- MessageId.ORB_TASK_FAILED,
- metadata={
- "total_files": failed_task.total_files,
- "processed_files": failed_task.processed_files,
- "successful_files": failed_task.successful_files,
- "failed_files": failed_task.failed_files,
- }
- )
- )
+ return f"{hours}h {mins}m {secs}s"
async def background_custom_processor(
self, user_id: str, task_id: str, items: list
) -> None:
"""Background task to process items using custom processor"""
try:
- upload_task = self.task_store[user_id][task_id]
+ upload_task: UploadTask = self.task_store[user_id][task_id]
upload_task.status = TaskStatus.RUNNING
upload_task.updated_at = time.time()
processor = upload_task.processor
+ logger.info(
+ "Upload / ingestion task started",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ total_files=upload_task.total_files,
+ filenames=self._get_display_filenames(upload_task),
+ processor_type=processor.__class__.__name__,
+ user_id=user_id,
+ )
+
# Process items with limited concurrency
# - Semaphore matches worker count
# - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
@@ -319,41 +282,65 @@ class TaskService:
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
+ logger.info(
+ "File processing task running",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ filename=file_task.filename,
+ )
+
try:
# Add timeout protection to prevent indefinite hangs
await self._process_with_timeout(
processor.process_item(upload_task, item, file_task),
timeout_seconds=self.ingestion_timeout
)
+
+ logger.info(
+ "File processing task succeeded",
+ status="PASSED",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ filename=file_task.filename,
+ )
+
except asyncio.CancelledError:
# Handle cancellation explicitly
- logger.info("File processing cancelled", item=str(item))
+
if file_task.status == TaskStatus.RUNNING:
file_task.status = TaskStatus.FAILED
- file_task.error = "Task cancelled by user"
+ file_task.error = "File processing task cancelled."
async with self._get_task_lock(task_id):
upload_task.failed_files += 1
- raise # Re-raise to propagate cancellation
- except IngestionTimeoutError as e:
- # Handle timeout explicitly with specific error message
+
logger.warning(
- "File processing timed out",
- item=str(item),
- timeout=self.ingestion_timeout
+ "File processing task cancelled",
+ status="FAILED",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ filename=file_task.filename,
)
+
+ raise # Re-raise to propagate cancellation
+ except IngestionTimeoutError as e:
+ # Handle timeout explicitly
if file_task.status == TaskStatus.RUNNING:
file_task.status = TaskStatus.FAILED
- file_task.error = f"Processing timed out after {self.ingestion_timeout}s"
+ file_task.error = str(e)
async with self._get_task_lock(task_id):
upload_task.failed_files += 1
# Don't re-raise - treat as normal failure, not cancellation
- except Exception as e:
+
logger.error(
- "Failed to process item", item=str(item), error=str(e)
+ "File processing task timed out",
+ status="FAILED",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ filename=file_task.filename,
+ exception=str(e),
)
- import traceback
- traceback.print_exc()
+ except Exception as e:
# Note: Processors already handle incrementing failed_files and
# setting file_task status/error, so we don't duplicate that here.
# Only update timestamp if processor didn't already set it
@@ -361,6 +348,16 @@ class TaskService:
file_task.status = TaskStatus.FAILED
if not file_task.error:
file_task.error = str(e)
+
+ logger.error(
+ "File processing task exception encountered",
+ status="FAILED",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ filename=file_task.filename,
+ exception=str(e),
+ )
+
finally:
file_task.updated_at = time.time()
# Only increment processed_files if the file reached a terminal state
@@ -378,6 +375,28 @@ class TaskService:
upload_task.status = TaskStatus.COMPLETED
upload_task.updated_at = time.time()
+ status: str = "FAILED"
+
+ if upload_task.failed_files == 0:
+ status = "PASSED"
+ elif upload_task.successful_files > 0:
+ status = "FAILED (partial success)"
+
+ logger.info(
+ "Upload / ingestion task finished",
+ status=status,
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ duration=self._format_duration(upload_task.duration_seconds),
+ total_files=upload_task.total_files,
+ processed_files=upload_task.processed_files,
+ successful_files=upload_task.successful_files,
+ failed_files=upload_task.failed_files,
+ filenames=self._get_display_filenames(upload_task),
+ processor_type=processor.__class__.__name__,
+ user_id=user_id,
+ )
+
# Send telemetry for task completion
asyncio.create_task(
TelemetryClient.send_event(
@@ -392,22 +411,54 @@ class TaskService:
)
except asyncio.CancelledError:
- logger.info("Background processor cancelled", task_id=task_id)
if user_id in self.task_store and task_id in self.task_store[user_id]:
# Task status and pending files already handled by cancel_task()
- pass
+ upload_task = self.task_store[user_id][task_id]
+
+ logger.warning(
+ "Upload / ingestion task cancelled",
+ status="FAILED",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ duration=self._format_duration(upload_task.duration_seconds),
+ total_files=upload_task.total_files,
+ processed_files=upload_task.processed_files,
+ successful_files=upload_task.successful_files,
+ failed_files=upload_task.failed_files,
+ filenames=self._get_display_filenames(upload_task),
+ processor_type=upload_task.processor.__class__.__name__,
+ user_id=user_id,
+ )
+ else:
+ logger.warning(
+ "Upload / ingestion task cancelled",
+ status="FAILED",
+ task_id=task_id,
+ user_id=user_id,
+ )
+
raise # Re-raise to properly handle cancellation
except Exception as e:
- logger.error(
- "Background custom processor failed", task_id=task_id, error=str(e)
- )
- import traceback
-
- traceback.print_exc()
if user_id in self.task_store and task_id in self.task_store[user_id]:
- failed_task = self.task_store[user_id][task_id]
- failed_task.status = TaskStatus.FAILED
- failed_task.updated_at = time.time()
+ upload_task = self.task_store[user_id][task_id]
+ upload_task.status = TaskStatus.FAILED
+ upload_task.updated_at = time.time()
+
+ logger.error(
+ "Upload / ingestion task exception encountered",
+ status="FAILED",
+ task_number=upload_task.sequence_number,
+ task_id=task_id,
+ duration=self._format_duration(upload_task.duration_seconds),
+ total_files=upload_task.total_files,
+ processed_files=upload_task.processed_files,
+ successful_files=upload_task.successful_files,
+ failed_files=upload_task.failed_files,
+ filenames=self._get_display_filenames(upload_task),
+ processor_type=upload_task.processor.__class__.__name__,
+ user_id=user_id,
+ exception=str(e),
+ )
# Send telemetry for task failure
asyncio.create_task(
@@ -415,13 +466,21 @@ class TaskService:
Category.TASK_OPERATIONS,
MessageId.ORB_TASK_FAILED,
metadata={
- "total_files": failed_task.total_files,
- "processed_files": failed_task.processed_files,
- "successful_files": failed_task.successful_files,
- "failed_files": failed_task.failed_files,
+ "total_files": upload_task.total_files,
+ "processed_files": upload_task.processed_files,
+ "successful_files": upload_task.successful_files,
+ "failed_files": upload_task.failed_files,
}
)
)
+ else:
+ logger.error(
+ "Upload / ingestion exception encountered",
+ status="FAILED",
+ task_id=task_id,
+ user_id=user_id,
+ exception=str(e),
+ )
def get_task_status(self, user_id: str, task_id: str) -> dict | None:
"""Get the status of a specific upload task
diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py
index 76fd3cc7b..b7a66389c 100644
--- a/src/utils/logging_config.py
+++ b/src/utils/logging_config.py
@@ -48,24 +48,31 @@ def configure_logging(
console_renderer = structlog.processors.JSONRenderer()
else:
# Custom clean format: timestamp path/file:loc logentry
+ LOC_WIDTH_SHORT = 35
+ LOC_WIDTH_LONG = 60
+
def custom_formatter(logger, log_method, event_dict):
timestamp = event_dict.pop("timestamp", "")
pathname = event_dict.pop("pathname", "")
filename = event_dict.pop("filename", "")
lineno = event_dict.pop("lineno", "")
- level = event_dict.pop("level", "")
+ level = event_dict.pop("level", "").upper()
- # Build file location - prefer pathname for full path, fallback to filename
- if pathname and lineno:
- location = f"{pathname}:{lineno}"
- elif filename and lineno:
+ if filename and lineno:
location = f"{filename}:{lineno}"
- elif pathname:
- location = pathname
+ loc_width = LOC_WIDTH_SHORT
+ elif pathname and lineno:
+ location = f"{pathname}:{lineno}"
+ loc_width = LOC_WIDTH_LONG
elif filename:
location = filename
+ loc_width = LOC_WIDTH_SHORT
+ elif pathname:
+ location = pathname
+ loc_width = LOC_WIDTH_LONG
else:
location = "unknown"
+ loc_width = LOC_WIDTH_SHORT
# Build the main message
message_parts = []
@@ -73,14 +80,18 @@ def configure_logging(
if event:
message_parts.append(event)
- # Add any remaining context
- for key, value in event_dict.items():
- if key not in ["service", "func_name"]: # Skip internal fields
- message_parts.append(f"{key}={value}")
+ header = f"[{timestamp}] [{level:<8}] [{location:<{loc_width}}] "
+
+ # Add any remaining context as indented multi-line fields
+ extra = {k: v for k, v in event_dict.items() if k not in ["service", "func_name"]}
+ if extra:
+ padding = " " * len(header)
+ for key, value in extra.items():
+ message_parts.append(f"\n{padding}- {key}: {value}")
- message = " ".join(message_parts)
+ message = "".join(message_parts)
- return f"{timestamp} {location} {message}"
+ return f"{header}{message}"
console_renderer = custom_formatter