OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: upgraded version of docling (#1995)
Lucas Oliveirabdd873f3538a
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex 9e0ee40f1..c5e04465e 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -72,6 +72,7 @@ jobs:- name: Cleanup Docker cacherun: |docker compose -f docker-compose.yml down -v --remove-orphans || true+ docker system prune -f || true- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)run: |@@ -92,25 +93,99 @@ jobs:echo "Workspace: ${GITHUB_WORKSPACE}"ls -la+ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@v3++ - name: Generate Image Cache Keys+ id: cache-keys+ run: |+ OPENSEARCH_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)+ BACKEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)+ FRONTEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)+ LANGFLOW_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)++ echo "opensearch=$OPENSEARCH_HASH" >> $GITHUB_OUTPUT+ echo "backend=$BACKEND_HASH" >> $GITHUB_OUTPUT+ echo "frontend=$FRONTEND_HASH" >> $GITHUB_OUTPUT+ echo "langflow=$LANGFLOW_HASH" >> $GITHUB_OUTPUT++ - name: Cache OpenSearch Image+ uses: actions/cache@v4+ with:+ path: /tmp/image-cache/opensearch.tar+ key: ${{ runner.os }}-image-opensearch-${{ steps.cache-keys.outputs.opensearch }}++ - name: Cache Backend Image+ uses: actions/cache@v4+ with:+ path: /tmp/image-cache/backend.tar+ key: ${{ runner.os }}-image-backend-${{ steps.cache-keys.outputs.backend }}++ - name: Cache Frontend Image+ uses: actions/cache@v4+ with:+ path: /tmp/image-cache/frontend.tar+ key: ${{ runner.os }}-image-frontend-${{ steps.cache-keys.outputs.frontend }}++ - name: Cache Langflow Image+ uses: actions/cache@v4+ with:+ path: /tmp/image-cache/langflow.tar+ key: ${{ runner.os }}-image-langflow-${{ steps.cache-keys.outputs.langflow }}+- name: Build OpenRAG imagesrun: |USE_LOCAL="${{ inputs.use_local_images }}"++ mkdir -p /tmp/image-cache++ build_or_load_image() {+ local name=$1+ local dockerfile=$2+ local tar_path="/tmp/image-cache/${name}.tar"++ if [ -f "$tar_path" ]; then+ echo "Cache hit for ${name}. Loading image from ${tar_path}..."+ docker load -i "$tar_path"+ echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..."+ docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}"+ else+ echo "Cache miss for ${name}. Building..."+ if docker buildx build \+ --load \+ -t "langflowai/openrag-${name}:cached" \+ -t "langflowai/openrag-${name}:${CI_IMAGE_TAG}" \+ -f "$dockerfile" .; then+ echo "Saving built image to ${tar_path}..."+ docker save "langflowai/openrag-${name}:cached" -o "$tar_path"+ else+ echo "Build failed for ${name}."+ return 1+ fi+ fi+ }+for i in 1 2 3; doecho "Build attempt $i..."if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; thenecho "Building PR images locally..."- if OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-build-images; then++ if build_or_load_image "opensearch" "Dockerfile" && \+ build_or_load_image "backend" "Dockerfile.backend" && \+ build_or_load_image "frontend" "Dockerfile.frontend" && \+ build_or_load_image "langflow" "Dockerfile.langflow"; thenbreakfielseecho "Preparing DockerHub images..."+if docker pull langflowai/openrag-backend:latest && \docker pull langflowai/openrag-frontend:latest && \docker pull langflowai/openrag-langflow:latest && \docker tag langflowai/openrag-backend:latest "langflowai/openrag-backend:${CI_IMAGE_TAG}" && \docker tag langflowai/openrag-frontend:latest "langflowai/openrag-frontend:${CI_IMAGE_TAG}" && \docker tag langflowai/openrag-langflow:latest "langflowai/openrag-langflow:${CI_IMAGE_TAG}" && \- docker build --no-cache -t "langflowai/openrag-opensearch:${CI_IMAGE_TAG}" -f Dockerfile .; then+ build_or_load_image "opensearch" "Dockerfile"; thenbreakfifi@@ -149,6 +224,7 @@ jobs:- name: Cleanup Docker cacherun: |docker compose -f docker-compose.yml down -v --remove-orphans || true+ docker system prune -f || true- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)run: |@@ -227,6 +303,7 @@ jobs:- name: Cleanup Docker cacherun: |docker compose -f docker-compose.yml down -v --remove-orphans || true+ docker system prune -f || true- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)run: |@@ -302,10 +379,11 @@ jobs:if: failure()run: |mkdir -p service-logs- docker logs os > service-logs/opensearch.log 2>&1 || true+ docker logs openrag-opensearch > service-logs/opensearch.log 2>&1 || truedocker logs openrag-backend > service-logs/backend.log 2>&1 || true- docker logs langflow > service-logs/langflow.log 2>&1 || true- docker logs openrag-backend-proxy > service-logs/backend-proxy.log 2>&1 || true+ docker logs openrag-frontend > service-logs/frontend.log 2>&1 || true+ docker logs openrag-langflow > service-logs/langflow.log 2>&1 || true+ cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>&1 || true- name: Upload service logsuses: actions/upload-artifact@v4diff --git a/Makefile b/Makefileindex 6aaf8f94b..6cc45fc86 100644--- a/Makefile+++ b/Makefile@@ -877,14 +877,10 @@ ci-build-images: ## Build all OpenRAG images for CI artifact sharing@set -e; \IMAGE_TAG=$${OPENRAG_VERSION:-latest}; \echo "$(YELLOW)Building all OpenRAG images with tag '$$IMAGE_TAG'...$(NC)"; \- $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:$$IMAGE_TAG -f Dockerfile . & PID1=$$!; \- $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:$$IMAGE_TAG -f Dockerfile.backend . & PID2=$$!; \- $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:$$IMAGE_TAG -f Dockerfile.frontend . & PID3=$$!; \- $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:$$IMAGE_TAG -f Dockerfile.langflow . & PID4=$$!; \- wait $$PID1 || exit 1; \- wait $$PID2 || exit 1; \- wait $$PID3 || exit 1; \- wait $$PID4 || exit 1; \+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:$$IMAGE_TAG -f Dockerfile .; \+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:$$IMAGE_TAG -f Dockerfile.backend .; \+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:$$IMAGE_TAG -f Dockerfile.frontend .; \+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:$$IMAGE_TAG -f Dockerfile.langflow .; \echo "$(GREEN)All images built successfully!$(NC)"ci-save-images: ## Save CI-built OpenRAG images to .ci-artifacts/openrag-ci-images.tar@@ -1153,10 +1149,11 @@ test-ci-local: ensure-langflow-data ensure-backend-volumes ## Same as test-ci buif [ $$TEST_RESULT -ne 0 ]; then \echo "$(RED)=== Tests failed, saving container logs to service-logs/ ===$(NC)"; \mkdir -p service-logs; \- $(CONTAINER_RUNTIME) logs langflow > service-logs/langflow.log 2>&1 || echo "$(RED)Could not get Langflow logs$(NC)"; \- $(CONTAINER_RUNTIME) logs openrag-backend > service-logs/backend.log 2>&1 || echo "$(RED)Could not get backend logs$(NC)"; \- $(CONTAINER_RUNTIME) logs openrag-frontend > service-logs/frontend.log 2>&1 || echo "$(RED)Could not get frontend logs$(NC)"; \- $(CONTAINER_RUNTIME) logs os > service-logs/opensearch.log 2>&1 || echo "$(RED)Could not get OpenSearch logs$(NC)"; \+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-langflow > service-logs/langflow.log 2>&1 || echo "$(RED)Could not get Langflow logs$(NC)"; \+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-backend > service-logs/backend.log 2>&1 || echo "$(RED)Could not get backend logs$(NC)"; \+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-frontend > service-logs/frontend.log 2>&1 || echo "$(RED)Could not get frontend logs$(NC)"; \+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-opensearch > service-logs/opensearch.log 2>&1 || echo "$(RED)Could not get OpenSearch logs$(NC)"; \+ if [ -f ~/.openrag/tui/docling-serve.log ]; then cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>/dev/null || echo "$(RED)Could not get Docling logs$(NC)"; fi; \fi; \echo "::group::Test Failure Report"; \uv run python scripts/ci/generate_test_report.py service-logs || true; \diff --git a/scripts/ci/get_image_hash.py b/scripts/ci/get_image_hash.pynew file mode 100644index 000000000..1ddc2c6f3--- /dev/null+++ b/scripts/ci/get_image_hash.py@@ -0,0 +1,56 @@+#!/usr/bin/env python3+import glob+import hashlib+import os+import sys+++def get_hash(paths):+ hasher = hashlib.sha256()++ # Track files to hash to ensure deterministic sorting+ files_to_hash = []++ for path in paths:+ if os.path.isdir(path):+ for root, _, files in os.walk(path):+ for f in files:+ full_path = os.path.join(root, f)+ files_to_hash.append(full_path)+ elif "*" in path or "?" in path:+ # Handle glob patterns+ for matched in glob.glob(path, recursive=True):+ if os.path.isdir(matched):+ for root, _, files in os.walk(matched):+ for f in files:+ files_to_hash.append(os.path.join(root, f))+ else:+ files_to_hash.append(matched)+ elif os.path.exists(path):+ files_to_hash.append(path)++ # Sort files deterministically and hash them+ for fp in sorted(set(files_to_hash)):+ # We only hash standard files+ if os.path.isfile(fp):+ # Ignore files that might be unreadable (e.g. root-owned leftovers)+ try:+ # Include filename in hash to distinguish different file structures+ hasher.update(os.path.relpath(fp).encode("utf-8"))+ hasher.update(b"\0")+ with open(fp, "rb") as f:+ while chunk := f.read(65536):+ hasher.update(chunk)+ hasher.update(b"\0")+ except (PermissionError, FileNotFoundError):+ continue++ return hasher.hexdigest()+++if __name__ == "__main__":+ if len(sys.argv) < 2:+ print("Usage: get_image_hash.py <file_or_dir1> [file_or_dir2 ...]")+ sys.exit(1)++ print(get_hash(sys.argv[1:]))diff --git a/scripts/ci/run_integration_suite.sh b/scripts/ci/run_integration_suite.shindex 710d8cdb9..ac703503e 100755--- a/scripts/ci/run_integration_suite.sh+++ b/scripts/ci/run_integration_suite.sh@@ -96,10 +96,13 @@ dump_logs() {-e "s/${pw}/**REDACTED**/g"}- "$container_runtime" logs --tail 10000 langflow 2>&1 | redact > service-logs/langflow.log || echo "${red}Could not get Langflow logs${nc}"- "$container_runtime" logs --tail 10000 openrag-backend 2>&1 | redact > service-logs/backend.log || echo "${red}Could not get backend logs${nc}"- "$container_runtime" logs --tail 10000 openrag-frontend 2>&1 | redact > service-logs/frontend.log || echo "${red}Could not get frontend logs${nc}"- "$container_runtime" logs --tail 10000 os 2>&1 | redact > service-logs/opensearch.log || echo "${red}Could not get OpenSearch logs${nc}"+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-langflow" 2>&1 | redact > service-logs/langflow.log || echo "${red}Could not get Langflow logs${nc}"+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 | redact > service-logs/backend.log || echo "${red}Could not get backend logs${nc}"+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-frontend" 2>&1 | redact > service-logs/frontend.log || echo "${red}Could not get frontend logs${nc}"+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-opensearch" 2>&1 | redact > service-logs/opensearch.log || echo "${red}Could not get OpenSearch logs${nc}"+ if [[ -f ~/.openrag/tui/docling-serve.log ]]; then+ redact < ~/.openrag/tui/docling-serve.log > service-logs/docling.log || echo "${red}Could not get Docling logs${nc}"+ fi}generate_report() {diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.pyindex b8aa47892..441c803a2 100644--- a/src/tui/managers/docling_manager.py+++ b/src/tui/managers/docling_manager.py@@ -298,17 +298,15 @@ class DoclingManager:"--python","3.13","--from",- "docling-serve[ui]==1.20.0",+ "docling-serve[ui]==1.26.0","--with","onnxruntime","--with","easyocr","--with",- f"docling[{docling_extras}]==2.95.0",+ f"docling[{docling_extras}]==2.108.0","--with",- "docling-core==2.77.1",- "--with",- "docling-jobkit==1.20.0",+ "docling-core==2.85.0","--with","transformers>=5.8.1,<5.9.0",]@@ -337,11 +335,20 @@ class DoclingManager:# async task state (leading to 404 on /v1/result/{id}).self._log_file_path.parent.mkdir(parents=True, exist_ok=True)log_file = open(self._log_file_path, "w")++ env = os.environ.copy()++ # Add docling serve environment variable to change Tenant Id header name+ # so that it's not used to enforce ownership locally.+ if "DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER" not in env:+ env["DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER"] = "X-Docling-Tenant-Id"+self._process = subprocess.Popen(cmd,stdout=log_file,stderr=subprocess.STDOUT, # Merge stderr into stdout log filestart_new_session=True, # Detach from parent process group+ env=env,)# Close parent's copy of the fd; the child has its ownlog_file.close()