OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Implement the ability to scale up Langflow and Docling workers...
Mike Pawlowski14cbdc809bef
diff --git a/.env.example b/.env.exampleindex b97e01f18..4c4cd79b6 100644--- a/.env.example+++ b/.env.example@@ -103,3 +103,9 @@ LANGFUSE_HOST=# - A list of comma-separate hostnames may be specified# - e.g. 9.46.110.49,10.21.103.227,openrag.example.comNEXT_ALLOWED_DEV_ORIGINS=++# OPTIONAL: Number of worker processes for concurrent request handling+# - Increase for higher throughput; but, factor in CPU / memory constraints+# MAX_WORKERS=4 # Backend ingestion workers (default: min(4, CPU_COUNT // 2))+# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)+# DOCLING_WORKERS=2 # Docling workers (default: 1)diff --git a/Makefile b/Makefileindex 21b2d67e8..19f151f32 100644--- a/Makefile+++ b/Makefile@@ -71,7 +71,7 @@ endefdev dev-cpu dev-local dev-local-cpu stop clean build logs \shell-backend shell-frontend install \test test-integration test-ci test-ci-local test-sdk test-os-jwt lint \- backend frontend docling docling-stop install-be install-fe build-be build-fe logs-be logs-fe logs-lf logs-os \+ backend frontend docling docling-stop install-be install-fe build-be build-fe build-os build-lf logs-be logs-fe logs-lf logs-os \shell-be shell-lf shell-os restart status health db-reset clear-os-data flow-upload setup factory-reset \dev-branch build-langflow-dev stop-dev clean-dev logs-dev logs-lf-dev shell-lf-dev restart-dev status-dev@@ -197,8 +197,10 @@ help_docker: ## Show Docker and container commands@echo ''@echo "$(PURPLE)Build Images:$(NC)"@echo " $(PURPLE)make build$(NC) - Build all Docker images locally"+ @echo " $(PURPLE)make build-os$(NC) - Build OpenSearch Docker image only"@echo " $(PURPLE)make build-be$(NC) - Build backend Docker image only"@echo " $(PURPLE)make build-fe$(NC) - Build frontend Docker image only"+ @echo " $(PURPLE)make build-lf$(NC) - Build Langflow Docker image only"@echo ''@echo "$(PURPLE)Container Management:$(NC)"@echo " $(PURPLE)make stop$(NC) - Stop and remove all OpenRAG containers"@@ -534,18 +536,14 @@ install-fe: ## Install frontend dependencies# DOCKER BUILD######################-build: ## Build all Docker images locally- @echo "$(YELLOW)Building all Docker images locally...$(NC)"- @echo "$(CYAN)Building OpenSearch image...$(NC)"- $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:latest -f Dockerfile .- @echo "$(CYAN)Building Backend image...$(NC)"- $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:latest -f Dockerfile.backend .- @echo "$(CYAN)Building Frontend image...$(NC)"- $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:latest -f Dockerfile.frontend .- @echo "$(CYAN)Building Langflow image...$(NC)"- $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .+build: build-os build-be build-fe build-lf ## Build all Docker images locally@echo "$(PURPLE)All images built successfully!$(NC)"+build-os: ## Build OpenSearch Docker image+ @echo "$(YELLOW)Building OpenSearch image...$(NC)"+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:latest -f Dockerfile .+ @echo "$(PURPLE)OpenSearch image built.$(NC)"+build-be: ## Build backend Docker image@echo "$(YELLOW)Building backend image...$(NC)"$(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:latest -f Dockerfile.backend .@@ -556,6 +554,11 @@ build-fe: ## Build frontend Docker image$(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:latest -f Dockerfile.frontend .@echo "$(PURPLE)Frontend image built.$(NC)"+build-lf: ## Build Langflow Docker image+ @echo "$(YELLOW)Building Langflow image...$(NC)"+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .+ @echo "$(PURPLE)Langflow image built.$(NC)"+####################### LOGGING######################diff --git a/docker-compose.yml b/docker-compose.ymlindex 113688edb..633dbd21d 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -15,7 +15,7 @@ services:bash -c "# Ensure data directory has correct permissionssudo chown -R opensearch:opensearch /usr/share/opensearch/data || true-+# Start OpenSearch in background/usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch &@@ -123,7 +123,7 @@ services:- OLLAMA_BASE_URL=${OLLAMA_ENDPOINT:-None}- LANGFLOW_LOAD_FLOWS_PATH=/app/flows- LANGFLOW_SECRET_KEY=${LANGFLOW_SECRET_KEY}- - JWT=None+ - JWT=None- OWNER=None- OWNER_NAME=None- OWNER_EMAIL=None@@ -146,6 +146,7 @@ services:- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-}- LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME- LANGFLOW_LOG_LEVEL=DEBUG+ - LANGFLOW_WORKERS=${LANGFLOW_WORKERS:-1}- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}- LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD}diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdxindex dd6fc4925..a71af6616 100644--- a/docs/docs/reference/configuration.mdx+++ b/docs/docs/reference/configuration.mdx@@ -154,10 +154,12 @@ Configure general system components, session management, and logging.| `FRONTEND_PORT` | `3000` | Host port for the OpenRAG frontend web interface. Change this if port 3000 is already in use on your system. || `OPENRAG_VERSION` | `latest` | The version of the OpenRAG Docker images to run. For more information, see [Upgrade OpenRAG](/upgrade) || `NEXT_ALLOWED_DEV_ORIGINS` | `http://localhost:3000` | Only used when [running OpenRAG in development mode](/support/contribute). Accepts a comma-separated list of hostnames to allow additional origins to make requests to the Next.js development server. |+| `MAX_WORKERS` | `min(4, CPU_COUNT // 2)` | Number of Backend worker processes for concurrent request handling. |+| `LANGFLOW_WORKERS` | `1` | Number of Langflow worker processes for concurrent request handling. |+| `DOCLING_WORKERS` | `1` | Number of Docling worker processes for concurrent request handling. |<!-- | `LOG_FORMAT` | Not set | Set to `json` to enabled JSON-formatted log output. If this variable isn't set, then the default logging format is used. | --><!-- | `LOG_LEVEL` | `INFO` | Logging level. Can be one of `DEBUG`, `INFO`, `WARNING`, or `ERROR`. `DEBUG` provides the most detailed logs but can impact performance. | -->-<!-- | `MAX_WORKERS` | `1` | Maximum number of workers for document processing. | --><!-- | `SERVICE_NAME` | `openrag` | Service name for logging. | --><!-- | `SESSION_SECRET` | Automatically generated | Session management. | --><!-- TBD: OPENRAG_VERSION -->diff --git a/scripts/docling_ctl.py b/scripts/docling_ctl.pyindex 8dc5c879a..dee3a05eb 100644--- a/scripts/docling_ctl.py+++ b/scripts/docling_ctl.py@@ -12,7 +12,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "src"))from tui.managers.docling_manager import DoclingManager-async def start_docling(port: int = 5001, host: str = None, enable_ui: bool = False):+async def start_docling(port: int = 5001, host: str = None, enable_ui: bool = False, workers: int | None = None):"""Start docling-serve."""manager = DoclingManager()@@ -23,8 +23,9 @@ async def start_docling(port: int = 5001, host: str = None, enable_ui: bool = Fareturn 0host_msg = f"{host}:{port}" if host else f"auto-detected host:{port}"- print(f"Starting docling-serve on {host_msg}...")- success, message = await manager.start(port=port, host=host, enable_ui=enable_ui)+ workers_msg = f" with {workers} workers" if workers else ""+ print(f"Starting docling-serve on {host_msg}{workers_msg}...")+ success, message = await manager.start(port=port, host=host, enable_ui=enable_ui, workers=workers)if success:print(f"{message}")@@ -75,12 +76,13 @@ async def main():parser.add_argument("command", choices=["start", "stop", "status"], help="Command to run")parser.add_argument("--port", type=int, default=5001, help="Port to run on (default: 5001)")parser.add_argument("--host", default=None, help="Host to bind to (default: auto-detect for containers)")+ parser.add_argument("--workers", type=int, default=None, help="Number of worker processes (default: DOCLING_WORKERS env var value or 1)")parser.add_argument("--enable-ui", action="store_true", help="Enable UI")args = parser.parse_args()if args.command == "start":- return await start_docling(port=args.port, host=args.host if args.host else None, enable_ui=args.enable_ui)+ return await start_docling(port=args.port, host=args.host if args.host else None, enable_ui=args.enable_ui, workers=args.workers)elif args.command == "stop":return await stop_docling()elif args.command == "status":diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 596483b4c..6ebb070d1 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -153,11 +153,11 @@ class TaskService:upload_task.status = TaskStatus.RUNNINGupload_task.updated_at = time.time()- # Process files with limited concurrency to avoid overwhelming the system- max_workers = get_worker_count()- semaphore = asyncio.Semaphore(- max_workers * 2- ) # Allow 2x process pool size for async I/O+ # Process items with limited concurrency+ # - Semaphore matches worker count+ # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O+ worker_count = get_worker_count()+ semaphore = asyncio.Semaphore(worker_count)async def process_with_semaphore(file_path: str):async with semaphore:@@ -187,7 +187,7 @@ class TaskService:if upload_task.processed_files >= upload_task.total_files:upload_task.status = TaskStatus.COMPLETEDupload_task.updated_at = time.time()-+# Send telemetry for task completionasyncio.create_task(TelemetryClient.send_event(@@ -212,7 +212,7 @@ class TaskService:failed_task = self.task_store[user_id][task_id]failed_task.status = TaskStatus.FAILEDfailed_task.updated_at = time.time()-+# Send telemetry for task failureasyncio.create_task(TelemetryClient.send_event(@@ -239,8 +239,10 @@ class TaskService:processor = upload_task.processor# Process items with limited concurrency- max_workers = get_worker_count()- semaphore = asyncio.Semaphore(max_workers * 2)+ # - Semaphore matches worker count+ # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O+ worker_count = get_worker_count()+ semaphore = asyncio.Semaphore(worker_count)async def process_with_semaphore(item, item_key: str):async with semaphore:@@ -276,7 +278,7 @@ class TaskService:# Mark task as completedupload_task.status = TaskStatus.COMPLETEDupload_task.updated_at = time.time()-+# Send telemetry for task completionasyncio.create_task(TelemetryClient.send_event(@@ -307,7 +309,7 @@ class TaskService:failed_task = self.task_store[user_id][task_id]failed_task.status = TaskStatus.FAILEDfailed_task.updated_at = time.time()-+# Send telemetry for task failureasyncio.create_task(TelemetryClient.send_event(diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.pyindex 0b5b6e492..67d55bc20 100644--- a/src/tui/managers/docling_manager.py+++ b/src/tui/managers/docling_manager.py@@ -33,6 +33,8 @@ class DoclingManager:self._port = 5001# Bind to all interfaces by default (can be overridden with DOCLING_BIND_HOST env var)self._host = os.getenv('DOCLING_BIND_HOST', '0.0.0.0')+ # Default workers for concurrent document processing (can be overridden with DOCLING_WORKERS env var)+ self._workers = int(os.getenv('DOCLING_WORKERS', '1'))self._running = Falseself._starting = Falseself._external_process = False@@ -57,7 +59,7 @@ class DoclingManager:# Just clean up our referencesself._add_log_entry("TUI exiting - docling-serve will continue running")# Note: We keep the PID file so we can reconnect in future sessions-+def _save_pid(self, pid: int) -> None:"""Save the process PID to a file for persistence across sessions."""try:@@ -120,7 +122,7 @@ class DoclingManager:# Keep buffer size limitedif len(self._log_buffer) > self._max_log_lines:self._log_buffer = self._log_buffer[-self._max_log_lines:]-+def is_running(self) -> bool:"""Check if docling serve is running (by PID only)."""# Check if we have a direct process handle@@ -142,7 +144,7 @@ class DoclingManager:self._running = Falseself._external_process = Falsereturn False-+def check_port_available(self) -> tuple[bool, Optional[str]]:"""Check if the native service port is available.@@ -175,6 +177,7 @@ class DoclingManager:"status": "starting","port": self._port,"host": self._host,+ "workers": self._workers,"endpoint": None,"docs_url": None,"ui_url": None,@@ -196,6 +199,7 @@ class DoclingManager:"status": "running","port": self._port,"host": self._host,+ "workers": self._workers,"endpoint": f"http://{display_host}:{self._port}","docs_url": f"http://{display_host}:{self._port}/docs","ui_url": f"http://{display_host}:{self._port}/ui",@@ -207,14 +211,22 @@ class DoclingManager:"status": "stopped","port": self._port,"host": self._host,+ "workers": self._workers,"endpoint": None,"docs_url": None,"ui_url": None,"pid": None}-- async def start(self, port: int = 5001, host: Optional[str] = None, enable_ui: bool = False) -> Tuple[bool, str]:- """Start docling serve as external process."""++ async def start(self, port: int = 5001, host: str | None = None, enable_ui: bool = False, workers: int | None = None) -> Tuple[bool, str]:+ """Start docling serve as external process.++ Args:+ port: Port to listen on (default: 5001)+ host: Host to bind to (default: from env or 0.0.0.0)+ enable_ui: Enable Gradio UI (default: False)+ workers: Number of worker processes (default: from env or 1)+ """if self.is_running():return False, "Docling serve is already running"@@ -222,6 +234,9 @@ class DoclingManager:# Use provided host or keep default from __init__if host is not None:self._host = host+ # Use provided workers or keep default from __init__+ if workers is not None:+ self._workers = workers# Check if port is already in use before trying to startimport socket@@ -251,12 +266,14 @@ class DoclingManager:"uv", "run", "python", "-m", "docling_serve", "run","--host", self._host,"--port", str(self._port),+ "--workers", str(self._workers),]else:cmd = [sys.executable, "-m", "docling_serve", "run","--host", self._host,"--port", str(self._port),+ "--workers", str(self._workers),]if enable_ui:@@ -467,7 +484,7 @@ class DoclingManager:except Exception as e:self._add_log_entry(f"Error stopping docling serve: {e}")return False, f"Error stopping docling serve: {str(e)}"-+async def restart(self, port: Optional[int] = None, host: Optional[str] = None, enable_ui: bool = False) -> Tuple[bool, str]:"""Restart docling serve."""# Use current settings if not specified@@ -475,23 +492,23 @@ class DoclingManager:port = self._portif host is None:host = self._host-+# Stop if runningif self.is_running():success, msg = await self.stop()if not success:return False, f"Failed to stop: {msg}"-+# Wait a moment for cleanupawait asyncio.sleep(1)-+# Start with new settingsreturn await self.start(port, host, enable_ui)-+def add_manual_log_entry(self, message: str) -> None:"""Add a manual log entry - useful for debugging."""self._add_log_entry(f"MANUAL: {message}")-+def get_logs(self, lines: int = 50) -> Tuple[bool, str]:"""Get logs from the docling-serve process."""if self.is_running():@@ -506,7 +523,7 @@ class DoclingManager:return True, logselse:return True, "Docling serve is not running."-+async def follow_logs(self) -> AsyncIterator[str]:"""Follow logs from the docling-serve process in real-time."""# First yield status message and any existing logsdiff --git a/src/utils/gpu_detection.py b/src/utils/gpu_detection.pyindex 6bf5576dc..7b4664873 100644--- a/src/utils/gpu_detection.py+++ b/src/utils/gpu_detection.py@@ -28,20 +28,27 @@ def detect_gpu_devices():def get_worker_count():- """Get optimal worker count based on GPU availability"""+ """Get optimal worker count based on downstream service capacity.++ The worker count controls concurrent ingestion requests to Langflow/Docling.+ The bottleneck is not the backend CPU (which is I/O-bound waiting on HTTP),+ but rather Langflow and Docling's processing capacity (default: 1 worker each).++ Uses min(4, cpu_count // 2) for both GPU and CPU modes to maintain a+ reasonable ratio with downstream services (4 backend : 1 Langflow worker).+ """has_gpu_devices, gpu_count = detect_gpu_devices()- if has_gpu_devices:- default_workers = min(4, multiprocessing.cpu_count() // 2)- logger.info(- "GPU mode enabled with limited concurrency",- gpu_count=gpu_count,- worker_count=default_workers,- )- else:- default_workers = multiprocessing.cpu_count()- logger.info(- "CPU-only mode enabled with full concurrency", worker_count=default_workers- )-- return int(os.getenv("MAX_WORKERS", default_workers))+ # Same formula for both modes: cap at 4, use half of available CPUs+ default_worker_count = max(1, min(4, multiprocessing.cpu_count() // 2))+ worker_count = max(1, int(os.getenv("MAX_WORKERS", default_worker_count)))+ mode = "GPU" if has_gpu_devices else "CPU-only"++ logger.info(+ f"{mode} mode enabled",+ gpu_count=gpu_count,+ worker_count=worker_count,+ default_worker_count=default_worker_count,+ )++ return worker_count