Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Ingesting folder doesn't seem to work

Mike Pawlowski58fc15ceb6c8
diff --git a/src/services/task_service.py b/src/services/task_service.py
index c7ae2f3b8..1081f6470 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -37,6 +37,10 @@ class TaskService:
# Locks for task counter updates, keyed by task_id
# Kept separate from UploadTask to maintain serialization compatibility
self._task_locks: dict[str, asyncio.Lock] = {}
+ # Global semaphore to limit concurrent file processing across all tasks.
+ # TaskService is a singleton, so this limits concurrency system-wide.
+ self._worker_count = get_worker_count()
+ self._processing_semaphore = asyncio.Semaphore(self._worker_count)
if self.process_pool is None:
raise ValueError("TaskService requires a process_pool parameter")
@@ -268,16 +272,14 @@ class TaskService:
filenames=self._get_display_filenames(upload_task),
processor_type=processor.__class__.__name__,
user_id=user_id,
+ worker_count=self._worker_count,
)
- # Process items with limited concurrency
- # - Semaphore matches worker count
+ # Process items with limited concurrency using the global semaphore
+ # - Limits concurrency across all tasks, not just within this one
# - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
- worker_count = get_worker_count()
- semaphore = asyncio.Semaphore(worker_count)
-
async def process_with_semaphore(item, item_key: str):
- async with semaphore:
+ async with self._processing_semaphore:
file_task = upload_task.file_tasks[item_key]
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -286,7 +288,7 @@ class TaskService:
"File processing task running",
task_number=upload_task.sequence_number,
task_id=task_id,
- filename=file_task.filename,
+ file_path=file_task.file_path,
)
try:
@@ -301,7 +303,7 @@ class TaskService:
status="PASSED",
task_number=upload_task.sequence_number,
task_id=task_id,
- filename=file_task.filename,
+ file_path=file_task.file_path,
)
except asyncio.CancelledError:
@@ -318,7 +320,7 @@ class TaskService:
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
- filename=file_task.filename,
+ file_path=file_task.file_path,
)
raise # Re-raise to propagate cancellation
@@ -336,7 +338,7 @@ class TaskService:
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
- filename=file_task.filename,
+ file_path=file_task.file_path,
exception=str(e),
)
@@ -354,7 +356,7 @@ class TaskService:
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
- filename=file_task.filename,
+ file_path=file_task.file_path,
exception=str(e),
)
@@ -395,6 +397,7 @@ class TaskService:
filenames=self._get_display_filenames(upload_task),
processor_type=processor.__class__.__name__,
user_id=user_id,
+ worker_count=self._worker_count,
)
# Send telemetry for task completion
@@ -428,6 +431,7 @@ class TaskService:
filenames=self._get_display_filenames(upload_task),
processor_type=upload_task.processor.__class__.__name__,
user_id=user_id,
+ worker_count=self._worker_count,
)
else:
logger.warning(
@@ -435,6 +439,7 @@ class TaskService:
status="FAILED",
task_id=task_id,
user_id=user_id,
+ worker_count=self._worker_count,
)
raise # Re-raise to properly handle cancellation
@@ -457,6 +462,7 @@ class TaskService:
filenames=self._get_display_filenames(upload_task),
processor_type=upload_task.processor.__class__.__name__,
user_id=user_id,
+ worker_count=self._worker_count,
exception=str(e),
)
@@ -479,6 +485,7 @@ class TaskService:
status="FAILED",
task_id=task_id,
user_id=user_id,
+ worker_count=self._worker_count,
exception=str(e),
)
diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py
index b7a66389c..d516909b4 100644
--- a/src/utils/logging_config.py
+++ b/src/utils/logging_config.py
@@ -48,7 +48,7 @@ def configure_logging(
console_renderer = structlog.processors.JSONRenderer()
else:
# Custom clean format: timestamp path/file:loc logentry
- LOC_WIDTH_SHORT = 35
+ LOC_WIDTH_SHORT = 30
LOC_WIDTH_LONG = 60
def custom_formatter(logger, log_method, event_dict):
@@ -80,7 +80,7 @@ def configure_logging(
if event:
message_parts.append(event)
- header = f"[{timestamp}] [{level:<8}] [{location:<{loc_width}}] "
+ header = f"[{timestamp}] [{level:<7}] [{location:<{loc_width}}] "
# Add any remaining context as indented multi-line fields
extra = {k: v for k, v in event_dict.items() if k not in ["service", "func_name"]}