Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Implement the ability to scale up Langflow and Docling workers...

Mike Pawlowski14cbdc809bef
diff --git a/.env.example b/.env.example
index b97e01f18..4c4cd79b6 100644
--- a/.env.example
+++ b/.env.example
@@ -103,3 +103,9 @@ LANGFUSE_HOST=
# - A list of comma-separate hostnames may be specified
# - e.g. 9.46.110.49,10.21.103.227,openrag.example.com
NEXT_ALLOWED_DEV_ORIGINS=
+
+# OPTIONAL: Number of worker processes for concurrent request handling
+# - Increase for higher throughput; but, factor in CPU / memory constraints
+# MAX_WORKERS=4 # Backend ingestion workers (default: min(4, CPU_COUNT // 2))
+# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)
+# DOCLING_WORKERS=2 # Docling workers (default: 1)
diff --git a/Makefile b/Makefile
index 21b2d67e8..19f151f32 100644
--- a/Makefile
+++ b/Makefile
@@ -71,7 +71,7 @@ endef
dev dev-cpu dev-local dev-local-cpu stop clean build logs \
shell-backend shell-frontend install \
test test-integration test-ci test-ci-local test-sdk test-os-jwt lint \
- backend frontend docling docling-stop install-be install-fe build-be build-fe logs-be logs-fe logs-lf logs-os \
+ backend frontend docling docling-stop install-be install-fe build-be build-fe build-os build-lf logs-be logs-fe logs-lf logs-os \
shell-be shell-lf shell-os restart status health db-reset clear-os-data flow-upload setup factory-reset \
dev-branch build-langflow-dev stop-dev clean-dev logs-dev logs-lf-dev shell-lf-dev restart-dev status-dev
@@ -197,8 +197,10 @@ help_docker: ## Show Docker and container commands
@echo ''
@echo "$(PURPLE)Build Images:$(NC)"
@echo " $(PURPLE)make build$(NC) - Build all Docker images locally"
+ @echo " $(PURPLE)make build-os$(NC) - Build OpenSearch Docker image only"
@echo " $(PURPLE)make build-be$(NC) - Build backend Docker image only"
@echo " $(PURPLE)make build-fe$(NC) - Build frontend Docker image only"
+ @echo " $(PURPLE)make build-lf$(NC) - Build Langflow Docker image only"
@echo ''
@echo "$(PURPLE)Container Management:$(NC)"
@echo " $(PURPLE)make stop$(NC) - Stop and remove all OpenRAG containers"
@@ -534,18 +536,14 @@ install-fe: ## Install frontend dependencies
# DOCKER BUILD
######################
-build: ## Build all Docker images locally
- @echo "$(YELLOW)Building all Docker images locally...$(NC)"
- @echo "$(CYAN)Building OpenSearch image...$(NC)"
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:latest -f Dockerfile .
- @echo "$(CYAN)Building Backend image...$(NC)"
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:latest -f Dockerfile.backend .
- @echo "$(CYAN)Building Frontend image...$(NC)"
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:latest -f Dockerfile.frontend .
- @echo "$(CYAN)Building Langflow image...$(NC)"
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .
+build: build-os build-be build-fe build-lf ## Build all Docker images locally
@echo "$(PURPLE)All images built successfully!$(NC)"
+build-os: ## Build OpenSearch Docker image
+ @echo "$(YELLOW)Building OpenSearch image...$(NC)"
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:latest -f Dockerfile .
+ @echo "$(PURPLE)OpenSearch image built.$(NC)"
+
build-be: ## Build backend Docker image
@echo "$(YELLOW)Building backend image...$(NC)"
$(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:latest -f Dockerfile.backend .
@@ -556,6 +554,11 @@ build-fe: ## Build frontend Docker image
$(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:latest -f Dockerfile.frontend .
@echo "$(PURPLE)Frontend image built.$(NC)"
+build-lf: ## Build Langflow Docker image
+ @echo "$(YELLOW)Building Langflow image...$(NC)"
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .
+ @echo "$(PURPLE)Langflow image built.$(NC)"
+
######################
# LOGGING
######################
diff --git a/docker-compose.yml b/docker-compose.yml
index 113688edb..633dbd21d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,7 +15,7 @@ services:
bash -c "
# Ensure data directory has correct permissions
sudo chown -R opensearch:opensearch /usr/share/opensearch/data || true
-
+
# Start OpenSearch in background
/usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch &
@@ -123,7 +123,7 @@ services:
- OLLAMA_BASE_URL=${OLLAMA_ENDPOINT:-None}
- LANGFLOW_LOAD_FLOWS_PATH=/app/flows
- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}
- - JWT=None
+ - JWT=None
- OWNER=None
- OWNER_NAME=None
- OWNER_EMAIL=None
@@ -146,6 +146,7 @@ services:
- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-}
- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME
- LANGFLOW_LOG_LEVEL=DEBUG
+ - LANGFLOW_WORKERS=${LANGFLOW_WORKERS:-1}
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index dd6fc4925..a71af6616 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -154,10 +154,12 @@ Configure general system components, session management, and logging.
| `FRONTEND_PORT` | `3000` | Host port for the OpenRAG frontend web interface. Change this if port 3000 is already in use on your system. |
| `OPENRAG_VERSION` | `latest` | The version of the OpenRAG Docker images to run. For more information, see [Upgrade OpenRAG](/upgrade) |
| `NEXT_ALLOWED_DEV_ORIGINS` | `http://localhost:3000` | Only used when [running OpenRAG in development mode](/support/contribute). Accepts a comma-separated list of hostnames to allow additional origins to make requests to the Next.js development server. |
+| `MAX_WORKERS` | `min(4, CPU_COUNT // 2)` | Number of Backend worker processes for concurrent request handling. |
+| `LANGFLOW_WORKERS` | `1` | Number of Langflow worker processes for concurrent request handling. |
+| `DOCLING_WORKERS` | `1` | Number of Docling worker processes for concurrent request handling. |
<!-- | `LOG_FORMAT` | Not set | Set to `json` to enabled JSON-formatted log output. If this variable isn't set, then the default logging format is used. | -->
<!-- | `LOG_LEVEL` | `INFO` | Logging level. Can be one of `DEBUG`, `INFO`, `WARNING`, or `ERROR`. `DEBUG` provides the most detailed logs but can impact performance. | -->
-<!-- | `MAX_WORKERS` | `1` | Maximum number of workers for document processing. | -->
<!-- | `SERVICE_NAME` | `openrag` | Service name for logging. | -->
<!-- | `SESSION_SECRET` | Automatically generated | Session management. | -->
<!-- TBD: OPENRAG_VERSION -->
diff --git a/scripts/docling_ctl.py b/scripts/docling_ctl.py
index 8dc5c879a..dee3a05eb 100644
--- a/scripts/docling_ctl.py
+++ b/scripts/docling_ctl.py
@@ -12,7 +12,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
from tui.managers.docling_manager import DoclingManager
-async def start_docling(port: int = 5001, host: str = None, enable_ui: bool = False):
+async def start_docling(port: int = 5001, host: str = None, enable_ui: bool = False, workers: int | None = None):
"""Start docling-serve."""
manager = DoclingManager()
@@ -23,8 +23,9 @@ async def start_docling(port: int = 5001, host: str = None, enable_ui: bool = Fa
return 0
host_msg = f"{host}:{port}" if host else f"auto-detected host:{port}"
- print(f"Starting docling-serve on {host_msg}...")
- success, message = await manager.start(port=port, host=host, enable_ui=enable_ui)
+ workers_msg = f" with {workers} workers" if workers else ""
+ print(f"Starting docling-serve on {host_msg}{workers_msg}...")
+ success, message = await manager.start(port=port, host=host, enable_ui=enable_ui, workers=workers)
if success:
print(f"{message}")
@@ -75,12 +76,13 @@ async def main():
parser.add_argument("command", choices=["start", "stop", "status"], help="Command to run")
parser.add_argument("--port", type=int, default=5001, help="Port to run on (default: 5001)")
parser.add_argument("--host", default=None, help="Host to bind to (default: auto-detect for containers)")
+ parser.add_argument("--workers", type=int, default=None, help="Number of worker processes (default: DOCLING_WORKERS env var value or 1)")
parser.add_argument("--enable-ui", action="store_true", help="Enable UI")
args = parser.parse_args()
if args.command == "start":
- return await start_docling(port=args.port, host=args.host if args.host else None, enable_ui=args.enable_ui)
+ return await start_docling(port=args.port, host=args.host if args.host else None, enable_ui=args.enable_ui, workers=args.workers)
elif args.command == "stop":
return await stop_docling()
elif args.command == "status":
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 596483b4c..6ebb070d1 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -153,11 +153,11 @@ class TaskService:
upload_task.status = TaskStatus.RUNNING
upload_task.updated_at = time.time()
- # Process files with limited concurrency to avoid overwhelming the system
- max_workers = get_worker_count()
- semaphore = asyncio.Semaphore(
- max_workers * 2
- ) # Allow 2x process pool size for async I/O
+ # Process items with limited concurrency
+ # - Semaphore matches worker count
+ # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
+ worker_count = get_worker_count()
+ semaphore = asyncio.Semaphore(worker_count)
async def process_with_semaphore(file_path: str):
async with semaphore:
@@ -187,7 +187,7 @@ class TaskService:
if upload_task.processed_files >= upload_task.total_files:
upload_task.status = TaskStatus.COMPLETED
upload_task.updated_at = time.time()
-
+
# Send telemetry for task completion
asyncio.create_task(
TelemetryClient.send_event(
@@ -212,7 +212,7 @@ class TaskService:
failed_task = self.task_store[user_id][task_id]
failed_task.status = TaskStatus.FAILED
failed_task.updated_at = time.time()
-
+
# Send telemetry for task failure
asyncio.create_task(
TelemetryClient.send_event(
@@ -239,8 +239,10 @@ class TaskService:
processor = upload_task.processor
# Process items with limited concurrency
- max_workers = get_worker_count()
- semaphore = asyncio.Semaphore(max_workers * 2)
+ # - Semaphore matches worker count
+ # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
+ worker_count = get_worker_count()
+ semaphore = asyncio.Semaphore(worker_count)
async def process_with_semaphore(item, item_key: str):
async with semaphore:
@@ -276,7 +278,7 @@ class TaskService:
# Mark task as completed
upload_task.status = TaskStatus.COMPLETED
upload_task.updated_at = time.time()
-
+
# Send telemetry for task completion
asyncio.create_task(
TelemetryClient.send_event(
@@ -307,7 +309,7 @@ class TaskService:
failed_task = self.task_store[user_id][task_id]
failed_task.status = TaskStatus.FAILED
failed_task.updated_at = time.time()
-
+
# Send telemetry for task failure
asyncio.create_task(
TelemetryClient.send_event(
diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.py
index 0b5b6e492..67d55bc20 100644
--- a/src/tui/managers/docling_manager.py
+++ b/src/tui/managers/docling_manager.py
@@ -33,6 +33,8 @@ class DoclingManager:
self._port = 5001
# Bind to all interfaces by default (can be overridden with DOCLING_BIND_HOST env var)
self._host = os.getenv('DOCLING_BIND_HOST', '0.0.0.0')
+ # Default workers for concurrent document processing (can be overridden with DOCLING_WORKERS env var)
+ self._workers = int(os.getenv('DOCLING_WORKERS', '1'))
self._running = False
self._starting = False
self._external_process = False
@@ -57,7 +59,7 @@ class DoclingManager:
# Just clean up our references
self._add_log_entry("TUI exiting - docling-serve will continue running")
# Note: We keep the PID file so we can reconnect in future sessions
-
+
def _save_pid(self, pid: int) -> None:
"""Save the process PID to a file for persistence across sessions."""
try:
@@ -120,7 +122,7 @@ class DoclingManager:
# Keep buffer size limited
if len(self._log_buffer) > self._max_log_lines:
self._log_buffer = self._log_buffer[-self._max_log_lines:]
-
+
def is_running(self) -> bool:
"""Check if docling serve is running (by PID only)."""
# Check if we have a direct process handle
@@ -142,7 +144,7 @@ class DoclingManager:
self._running = False
self._external_process = False
return False
-
+
def check_port_available(self) -> tuple[bool, Optional[str]]:
"""Check if the native service port is available.
@@ -175,6 +177,7 @@ class DoclingManager:
"status": "starting",
"port": self._port,
"host": self._host,
+ "workers": self._workers,
"endpoint": None,
"docs_url": None,
"ui_url": None,
@@ -196,6 +199,7 @@ class DoclingManager:
"status": "running",
"port": self._port,
"host": self._host,
+ "workers": self._workers,
"endpoint": f"http://{display_host}:{self._port}",
"docs_url": f"http://{display_host}:{self._port}/docs",
"ui_url": f"http://{display_host}:{self._port}/ui",
@@ -207,14 +211,22 @@ class DoclingManager:
"status": "stopped",
"port": self._port,
"host": self._host,
+ "workers": self._workers,
"endpoint": None,
"docs_url": None,
"ui_url": None,
"pid": None
}
-
- async def start(self, port: int = 5001, host: Optional[str] = None, enable_ui: bool = False) -> Tuple[bool, str]:
- """Start docling serve as external process."""
+
+ async def start(self, port: int = 5001, host: str | None = None, enable_ui: bool = False, workers: int | None = None) -> Tuple[bool, str]:
+ """Start docling serve as external process.
+
+ Args:
+ port: Port to listen on (default: 5001)
+ host: Host to bind to (default: from env or 0.0.0.0)
+ enable_ui: Enable Gradio UI (default: False)
+ workers: Number of worker processes (default: from env or 1)
+ """
if self.is_running():
return False, "Docling serve is already running"
@@ -222,6 +234,9 @@ class DoclingManager:
# Use provided host or keep default from __init__
if host is not None:
self._host = host
+ # Use provided workers or keep default from __init__
+ if workers is not None:
+ self._workers = workers
# Check if port is already in use before trying to start
import socket
@@ -251,12 +266,14 @@ class DoclingManager:
"uv", "run", "python", "-m", "docling_serve", "run",
"--host", self._host,
"--port", str(self._port),
+ "--workers", str(self._workers),
]
else:
cmd = [
sys.executable, "-m", "docling_serve", "run",
"--host", self._host,
"--port", str(self._port),
+ "--workers", str(self._workers),
]
if enable_ui:
@@ -467,7 +484,7 @@ class DoclingManager:
except Exception as e:
self._add_log_entry(f"Error stopping docling serve: {e}")
return False, f"Error stopping docling serve: {str(e)}"
-
+
async def restart(self, port: Optional[int] = None, host: Optional[str] = None, enable_ui: bool = False) -> Tuple[bool, str]:
"""Restart docling serve."""
# Use current settings if not specified
@@ -475,23 +492,23 @@ class DoclingManager:
port = self._port
if host is None:
host = self._host
-
+
# Stop if running
if self.is_running():
success, msg = await self.stop()
if not success:
return False, f"Failed to stop: {msg}"
-
+
# Wait a moment for cleanup
await asyncio.sleep(1)
-
+
# Start with new settings
return await self.start(port, host, enable_ui)
-
+
def add_manual_log_entry(self, message: str) -> None:
"""Add a manual log entry - useful for debugging."""
self._add_log_entry(f"MANUAL: {message}")
-
+
def get_logs(self, lines: int = 50) -> Tuple[bool, str]:
"""Get logs from the docling-serve process."""
if self.is_running():
@@ -506,7 +523,7 @@ class DoclingManager:
return True, logs
else:
return True, "Docling serve is not running."
-
+
async def follow_logs(self) -> AsyncIterator[str]:
"""Follow logs from the docling-serve process in real-time."""
# First yield status message and any existing logs
diff --git a/src/utils/gpu_detection.py b/src/utils/gpu_detection.py
index 6bf5576dc..7b4664873 100644
--- a/src/utils/gpu_detection.py
+++ b/src/utils/gpu_detection.py
@@ -28,20 +28,27 @@ def detect_gpu_devices():
def get_worker_count():
- """Get optimal worker count based on GPU availability"""
+ """Get optimal worker count based on downstream service capacity.
+
+ The worker count controls concurrent ingestion requests to Langflow/Docling.
+ The bottleneck is not the backend CPU (which is I/O-bound waiting on HTTP),
+ but rather Langflow and Docling's processing capacity (default: 1 worker each).
+
+ Uses min(4, cpu_count // 2) for both GPU and CPU modes to maintain a
+ reasonable ratio with downstream services (4 backend : 1 Langflow worker).
+ """
has_gpu_devices, gpu_count = detect_gpu_devices()
- if has_gpu_devices:
- default_workers = min(4, multiprocessing.cpu_count() // 2)
- logger.info(
- "GPU mode enabled with limited concurrency",
- gpu_count=gpu_count,
- worker_count=default_workers,
- )
- else:
- default_workers = multiprocessing.cpu_count()
- logger.info(
- "CPU-only mode enabled with full concurrency", worker_count=default_workers
- )
-
- return int(os.getenv("MAX_WORKERS", default_workers))
+ # Same formula for both modes: cap at 4, use half of available CPUs
+ default_worker_count = max(1, min(4, multiprocessing.cpu_count() // 2))
+ worker_count = max(1, int(os.getenv("MAX_WORKERS", default_worker_count)))
+ mode = "GPU" if has_gpu_devices else "CPU-only"
+
+ logger.info(
+ f"{mode} mode enabled",
+ gpu_count=gpu_count,
+ worker_count=worker_count,
+ default_worker_count=default_worker_count,
+ )
+
+ return worker_count