OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
test-fix:Improve docling startup and Markdown handling
Edwin Jose61804a96982f
diff --git a/.gitignore b/.gitignoreindex 2c09fc1f9..9de3e2574 100644--- a/.gitignore+++ b/.gitignore@@ -42,3 +42,7 @@ node_modules/GEMINI.md/.bob/AGENTS.md+documents/warmup_ocr.pdf+documents/openrag-documentation.pdf+documents/ibm_anthropic.pdf+documents/docling.pdfdiff --git a/Makefile b/Makefileindex 19f151f32..29b62df39 100644--- a/Makefile+++ b/Makefile@@ -633,9 +633,18 @@ test-ci: ## Start infra, run integration + SDK tests, tear down (uses DockerHub$(CONTAINER_RUNTIME) build --no-cache -t langflowai/openrag-opensearch:latest -f Dockerfile .; \echo "$(YELLOW)Starting infra (OpenSearch + Dashboards + Langflow + Backend + Frontend) with CPU containers$(NC)"; \$(COMPOSE_CMD) up -d opensearch dashboards langflow openrag-backend openrag-frontend; \+ echo "$(CYAN)Architecture: $$(uname -m), Platform: $$(uname -s)$(NC)"; \echo "$(YELLOW)Starting docling-serve...$(NC)"; \- DOCLING_ENDPOINT=$$(uv run python scripts/docling_ctl.py start --port 5001 | grep "Endpoint:" | awk '{print $$2}'); \+ DOCLING_START_OUTPUT=$$(uv run python scripts/docling_ctl.py start --port 5001 2>&1); \+ echo "$$DOCLING_START_OUTPUT"; \+ DOCLING_ENDPOINT=$$(echo "$$DOCLING_START_OUTPUT" | grep "Endpoint:" | awk '{print $$2}'); \+ if [ -z "$$DOCLING_ENDPOINT" ]; then \+ echo "$(RED)WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001$(NC)"; \+ DOCLING_ENDPOINT="http://localhost:5001"; \+ fi; \echo "$(PURPLE)Docling-serve started at $$DOCLING_ENDPOINT$(NC)"; \+ echo "$(YELLOW)Docling-serve status check:$(NC)"; \+ uv run python scripts/docling_ctl.py status 2>&1 || true; \echo "$(YELLOW)Waiting for backend OIDC endpoint...$(NC)"; \for i in $$(seq 1 60); do \$(CONTAINER_RUNTIME) exec openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1 && break || sleep 2; \@@ -667,6 +676,12 @@ test-ci: ## Start infra, run integration + SDK tests, tear down (uses DockerHubfor i in $$(seq 1 60); do \curl -s $${DOCLING_ENDPOINT}/health >/dev/null 2>&1 && break || sleep 2; \done; \+ if ! curl -s $${DOCLING_ENDPOINT}/health >/dev/null 2>&1; then \+ echo "$(RED)ERROR: docling-serve is not healthy at $$DOCLING_ENDPOINT after 120s$(NC)"; \+ echo "$(YELLOW)Docling status:$(NC)"; \+ uv run python scripts/docling_ctl.py status 2>&1 || true; \+ echo "$(RED)Integration tests that depend on docling will fail.$(NC)"; \+ fi; \echo "$(PURPLE)Running integration tests$(NC)"; \LOG_LEVEL=$${LOG_LEVEL:-DEBUG} \GOOGLE_OAUTH_CLIENT_ID="" \@@ -720,9 +735,18 @@ test-ci-local: ## Same as test-ci but builds all images locally$(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:latest -f Dockerfile.langflow .; \echo "$(YELLOW)Starting infra (OpenSearch + Dashboards + Langflow + Backend + Frontend) with CPU containers$(NC)"; \$(COMPOSE_CMD) up -d opensearch dashboards langflow openrag-backend openrag-frontend; \+ echo "$(CYAN)Architecture: $$(uname -m), Platform: $$(uname -s)$(NC)"; \echo "$(YELLOW)Starting docling-serve...$(NC)"; \- DOCLING_ENDPOINT=$$(uv run python scripts/docling_ctl.py start --port 5001 | grep "Endpoint:" | awk '{print $$2}'); \+ DOCLING_START_OUTPUT=$$(uv run python scripts/docling_ctl.py start --port 5001 2>&1); \+ echo "$$DOCLING_START_OUTPUT"; \+ DOCLING_ENDPOINT=$$(echo "$$DOCLING_START_OUTPUT" | grep "Endpoint:" | awk '{print $$2}'); \+ if [ -z "$$DOCLING_ENDPOINT" ]; then \+ echo "$(RED)WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001$(NC)"; \+ DOCLING_ENDPOINT="http://localhost:5001"; \+ fi; \echo "$(PURPLE)Docling-serve started at $$DOCLING_ENDPOINT$(NC)"; \+ echo "$(YELLOW)Docling-serve status check:$(NC)"; \+ uv run python scripts/docling_ctl.py status 2>&1 || true; \echo "$(YELLOW)Waiting for backend OIDC endpoint...$(NC)"; \for i in $$(seq 1 60); do \$(CONTAINER_RUNTIME) exec openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1 && break || sleep 2; \@@ -754,6 +778,12 @@ test-ci-local: ## Same as test-ci but builds all images locallyfor i in $$(seq 1 60); do \curl -s $${DOCLING_ENDPOINT}/health >/dev/null 2>&1 && break || sleep 2; \done; \+ if ! curl -s $${DOCLING_ENDPOINT}/health >/dev/null 2>&1; then \+ echo "$(RED)ERROR: docling-serve is not healthy at $$DOCLING_ENDPOINT after 120s$(NC)"; \+ echo "$(YELLOW)Docling status:$(NC)"; \+ uv run python scripts/docling_ctl.py status 2>&1 || true; \+ echo "$(RED)Integration tests that depend on docling will fail.$(NC)"; \+ fi; \echo "$(PURPLE)Running integration tests$(NC)"; \LOG_LEVEL=$${LOG_LEVEL:-DEBUG} \GOOGLE_OAUTH_CLIENT_ID="" \diff --git a/src/api/docling.py b/src/api/docling.pyindex 2f9adcbdd..7d172c2a7 100644--- a/src/api/docling.py+++ b/src/api/docling.py@@ -39,38 +39,41 @@ def _get_gateway_ip_from_route() -> str | None:def determine_docling_host() -> str:- """Determine the host address used for docling health checks."""+ """Determine the host address used for docling health checks.++ Works for containers, Docker Desktop hosts, and plain Linux hosts / CI runners.+ """container_type = detect_container_environment()++ # Container-specific env var (e.g. HOST_DOCKER_INTERNAL set explicitly)if container_type:- # Try HOST_DOCKER_INTERNAL env var firstcontainer_host = get_container_host()if container_host:logger.info("Using container-aware host '%s'", container_host)return container_host- # Try special hostnames (Docker Desktop and rootless podman)- import socket- for hostname in ["host.docker.internal", "host.containers.internal"]:- try:- socket.getaddrinfo(hostname, None)- logger.info("Using %s for container-to-host communication", hostname)- return hostname- except socket.gaierror:- logger.debug("%s not available", hostname)-- # Try gateway IP detection (Docker on Linux)+ # Try well-known hostnames (resolves on Docker Desktop even outside a+ # container, and inside Docker/Podman containers with host networking)+ for hostname in ["host.docker.internal", "host.containers.internal"]:+ try:+ socket.getaddrinfo(hostname, None)+ logger.info("Using %s for docling-serve", hostname)+ return hostname+ except socket.gaierror:+ logger.debug("%s not available", hostname)++ # Container-only fallbacks: gateway IP or bridge IP+ if container_type:gateway_ip = _get_gateway_ip_from_route()if gateway_ip:logger.info("Detected host gateway IP: %s", gateway_ip)return gateway_ip- # Fallback to bridge IPfallback_ip = guess_host_ip_for_containers(logger=logger)logger.info("Falling back to container bridge host %s", fallback_ip)return fallback_ip- # Running outside a container- logger.info("Running outside a container; using localhost")+ logger.info("Using localhost for docling-serve")return "localhost"diff --git a/src/models/processors.py b/src/models/processors.pyindex 051cf9713..ec6eaaa18 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -200,11 +200,11 @@ class TaskProcessor:file_hash=file_hash,)- # Check if this is a .txt file - use simple processing instead of docling+ # Check if this is a .txt or .md file - use simple processing instead of doclingimport osfile_ext = os.path.splitext(file_path)[1].lower()- if file_ext == '.txt':+ if file_ext in ('.txt', '.md'):# Simple text file processing without doclingfrom utils.document_processing import process_text_filelogger.info(diff --git a/src/services/document_service.py b/src/services/document_service.pyindex 7bb33e9eb..e46bdea7f 100644--- a/src/services/document_service.py+++ b/src/services/document_service.py@@ -170,10 +170,10 @@ class DocumentService:content.write(chunk)content.seek(0) # Reset to beginning for reading- # Check if this is a .txt file - use simple processing+ # Check if this is a .txt or .md file - use simple processingfile_ext = os.path.splitext(filename)[1].lower()- if file_ext == '.txt':+ if file_ext in ('.txt', '.md'):# Simple text file processing for chat contexttext_content = content.read().decode('utf-8', errors='replace')