Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: upgraded version of docling (#1995)

Lucas Oliveirabdd873f3538a
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index 9e0ee40f1..c5e04465e 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -72,6 +72,7 @@ jobs:
- name: Cleanup Docker cache
run: |
docker compose -f docker-compose.yml down -v --remove-orphans || true
+ docker system prune -f || true
- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
run: |
@@ -92,25 +93,99 @@ jobs:
echo "Workspace: ${GITHUB_WORKSPACE}"
ls -la
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Generate Image Cache Keys
+ id: cache-keys
+ run: |
+ OPENSEARCH_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)
+ BACKEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)
+ FRONTEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)
+ LANGFLOW_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)
+
+ echo "opensearch=$OPENSEARCH_HASH" >> $GITHUB_OUTPUT
+ echo "backend=$BACKEND_HASH" >> $GITHUB_OUTPUT
+ echo "frontend=$FRONTEND_HASH" >> $GITHUB_OUTPUT
+ echo "langflow=$LANGFLOW_HASH" >> $GITHUB_OUTPUT
+
+ - name: Cache OpenSearch Image
+ uses: actions/cache@v4
+ with:
+ path: /tmp/image-cache/opensearch.tar
+ key: ${{ runner.os }}-image-opensearch-${{ steps.cache-keys.outputs.opensearch }}
+
+ - name: Cache Backend Image
+ uses: actions/cache@v4
+ with:
+ path: /tmp/image-cache/backend.tar
+ key: ${{ runner.os }}-image-backend-${{ steps.cache-keys.outputs.backend }}
+
+ - name: Cache Frontend Image
+ uses: actions/cache@v4
+ with:
+ path: /tmp/image-cache/frontend.tar
+ key: ${{ runner.os }}-image-frontend-${{ steps.cache-keys.outputs.frontend }}
+
+ - name: Cache Langflow Image
+ uses: actions/cache@v4
+ with:
+ path: /tmp/image-cache/langflow.tar
+ key: ${{ runner.os }}-image-langflow-${{ steps.cache-keys.outputs.langflow }}
+
- name: Build OpenRAG images
run: |
USE_LOCAL="${{ inputs.use_local_images }}"
+
+ mkdir -p /tmp/image-cache
+
+ build_or_load_image() {
+ local name=$1
+ local dockerfile=$2
+ local tar_path="/tmp/image-cache/${name}.tar"
+
+ if [ -f "$tar_path" ]; then
+ echo "Cache hit for ${name}. Loading image from ${tar_path}..."
+ docker load -i "$tar_path"
+ echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..."
+ docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}"
+ else
+ echo "Cache miss for ${name}. Building..."
+ if docker buildx build \
+ --load \
+ -t "langflowai/openrag-${name}:cached" \
+ -t "langflowai/openrag-${name}:${CI_IMAGE_TAG}" \
+ -f "$dockerfile" .; then
+ echo "Saving built image to ${tar_path}..."
+ docker save "langflowai/openrag-${name}:cached" -o "$tar_path"
+ else
+ echo "Build failed for ${name}."
+ return 1
+ fi
+ fi
+ }
+
for i in 1 2 3; do
echo "Build attempt $i..."
if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; then
echo "Building PR images locally..."
- if OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-build-images; then
+
+ if build_or_load_image "opensearch" "Dockerfile" && \
+ build_or_load_image "backend" "Dockerfile.backend" && \
+ build_or_load_image "frontend" "Dockerfile.frontend" && \
+ build_or_load_image "langflow" "Dockerfile.langflow"; then
break
fi
else
echo "Preparing DockerHub images..."
+
if docker pull langflowai/openrag-backend:latest && \
docker pull langflowai/openrag-frontend:latest && \
docker pull langflowai/openrag-langflow:latest && \
docker tag langflowai/openrag-backend:latest "langflowai/openrag-backend:${CI_IMAGE_TAG}" && \
docker tag langflowai/openrag-frontend:latest "langflowai/openrag-frontend:${CI_IMAGE_TAG}" && \
docker tag langflowai/openrag-langflow:latest "langflowai/openrag-langflow:${CI_IMAGE_TAG}" && \
- docker build --no-cache -t "langflowai/openrag-opensearch:${CI_IMAGE_TAG}" -f Dockerfile .; then
+ build_or_load_image "opensearch" "Dockerfile"; then
break
fi
fi
@@ -149,6 +224,7 @@ jobs:
- name: Cleanup Docker cache
run: |
docker compose -f docker-compose.yml down -v --remove-orphans || true
+ docker system prune -f || true
- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
run: |
@@ -227,6 +303,7 @@ jobs:
- name: Cleanup Docker cache
run: |
docker compose -f docker-compose.yml down -v --remove-orphans || true
+ docker system prune -f || true
- name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
run: |
@@ -302,10 +379,11 @@ jobs:
if: failure()
run: |
mkdir -p service-logs
- docker logs os > service-logs/opensearch.log 2>&1 || true
+ docker logs openrag-opensearch > service-logs/opensearch.log 2>&1 || true
docker logs openrag-backend > service-logs/backend.log 2>&1 || true
- docker logs langflow > service-logs/langflow.log 2>&1 || true
- docker logs openrag-backend-proxy > service-logs/backend-proxy.log 2>&1 || true
+ docker logs openrag-frontend > service-logs/frontend.log 2>&1 || true
+ docker logs openrag-langflow > service-logs/langflow.log 2>&1 || true
+ cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>&1 || true
- name: Upload service logs
uses: actions/upload-artifact@v4
diff --git a/Makefile b/Makefile
index 6aaf8f94b..6cc45fc86 100644
--- a/Makefile
+++ b/Makefile
@@ -877,14 +877,10 @@ ci-build-images: ## Build all OpenRAG images for CI artifact sharing
@set -e; \
IMAGE_TAG=$${OPENRAG_VERSION:-latest}; \
echo "$(YELLOW)Building all OpenRAG images with tag '$$IMAGE_TAG'...$(NC)"; \
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:$$IMAGE_TAG -f Dockerfile . & PID1=$$!; \
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:$$IMAGE_TAG -f Dockerfile.backend . & PID2=$$!; \
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:$$IMAGE_TAG -f Dockerfile.frontend . & PID3=$$!; \
- $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:$$IMAGE_TAG -f Dockerfile.langflow . & PID4=$$!; \
- wait $$PID1 || exit 1; \
- wait $$PID2 || exit 1; \
- wait $$PID3 || exit 1; \
- wait $$PID4 || exit 1; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:$$IMAGE_TAG -f Dockerfile .; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:$$IMAGE_TAG -f Dockerfile.backend .; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:$$IMAGE_TAG -f Dockerfile.frontend .; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:$$IMAGE_TAG -f Dockerfile.langflow .; \
echo "$(GREEN)All images built successfully!$(NC)"
ci-save-images: ## Save CI-built OpenRAG images to .ci-artifacts/openrag-ci-images.tar
@@ -1153,10 +1149,11 @@ test-ci-local: ensure-langflow-data ensure-backend-volumes ## Same as test-ci bu
if [ $$TEST_RESULT -ne 0 ]; then \
echo "$(RED)=== Tests failed, saving container logs to service-logs/ ===$(NC)"; \
mkdir -p service-logs; \
- $(CONTAINER_RUNTIME) logs langflow > service-logs/langflow.log 2>&1 || echo "$(RED)Could not get Langflow logs$(NC)"; \
- $(CONTAINER_RUNTIME) logs openrag-backend > service-logs/backend.log 2>&1 || echo "$(RED)Could not get backend logs$(NC)"; \
- $(CONTAINER_RUNTIME) logs openrag-frontend > service-logs/frontend.log 2>&1 || echo "$(RED)Could not get frontend logs$(NC)"; \
- $(CONTAINER_RUNTIME) logs os > service-logs/opensearch.log 2>&1 || echo "$(RED)Could not get OpenSearch logs$(NC)"; \
+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-langflow > service-logs/langflow.log 2>&1 || echo "$(RED)Could not get Langflow logs$(NC)"; \
+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-backend > service-logs/backend.log 2>&1 || echo "$(RED)Could not get backend logs$(NC)"; \
+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-frontend > service-logs/frontend.log 2>&1 || echo "$(RED)Could not get frontend logs$(NC)"; \
+ $(CONTAINER_RUNTIME) logs $(COMPOSE_PROJECT_NAME)-opensearch > service-logs/opensearch.log 2>&1 || echo "$(RED)Could not get OpenSearch logs$(NC)"; \
+ if [ -f ~/.openrag/tui/docling-serve.log ]; then cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>/dev/null || echo "$(RED)Could not get Docling logs$(NC)"; fi; \
fi; \
echo "::group::Test Failure Report"; \
uv run python scripts/ci/generate_test_report.py service-logs || true; \
diff --git a/scripts/ci/get_image_hash.py b/scripts/ci/get_image_hash.py
new file mode 100644
index 000000000..1ddc2c6f3
--- /dev/null
+++ b/scripts/ci/get_image_hash.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+import glob
+import hashlib
+import os
+import sys
+
+
+def get_hash(paths):
+ hasher = hashlib.sha256()
+
+ # Track files to hash to ensure deterministic sorting
+ files_to_hash = []
+
+ for path in paths:
+ if os.path.isdir(path):
+ for root, _, files in os.walk(path):
+ for f in files:
+ full_path = os.path.join(root, f)
+ files_to_hash.append(full_path)
+ elif "*" in path or "?" in path:
+ # Handle glob patterns
+ for matched in glob.glob(path, recursive=True):
+ if os.path.isdir(matched):
+ for root, _, files in os.walk(matched):
+ for f in files:
+ files_to_hash.append(os.path.join(root, f))
+ else:
+ files_to_hash.append(matched)
+ elif os.path.exists(path):
+ files_to_hash.append(path)
+
+ # Sort files deterministically and hash them
+ for fp in sorted(set(files_to_hash)):
+ # We only hash standard files
+ if os.path.isfile(fp):
+ # Ignore files that might be unreadable (e.g. root-owned leftovers)
+ try:
+ # Include filename in hash to distinguish different file structures
+ hasher.update(os.path.relpath(fp).encode("utf-8"))
+ hasher.update(b"\0")
+ with open(fp, "rb") as f:
+ while chunk := f.read(65536):
+ hasher.update(chunk)
+ hasher.update(b"\0")
+ except (PermissionError, FileNotFoundError):
+ continue
+
+ return hasher.hexdigest()
+
+
+if __name__ == "__main__":
+ if len(sys.argv) < 2:
+ print("Usage: get_image_hash.py <file_or_dir1> [file_or_dir2 ...]")
+ sys.exit(1)
+
+ print(get_hash(sys.argv[1:]))
diff --git a/scripts/ci/run_integration_suite.sh b/scripts/ci/run_integration_suite.sh
index 710d8cdb9..ac703503e 100755
--- a/scripts/ci/run_integration_suite.sh
+++ b/scripts/ci/run_integration_suite.sh
@@ -96,10 +96,13 @@ dump_logs() {
-e "s/${pw}/**REDACTED**/g"
}
- "$container_runtime" logs --tail 10000 langflow 2>&1 | redact > service-logs/langflow.log || echo "${red}Could not get Langflow logs${nc}"
- "$container_runtime" logs --tail 10000 openrag-backend 2>&1 | redact > service-logs/backend.log || echo "${red}Could not get backend logs${nc}"
- "$container_runtime" logs --tail 10000 openrag-frontend 2>&1 | redact > service-logs/frontend.log || echo "${red}Could not get frontend logs${nc}"
- "$container_runtime" logs --tail 10000 os 2>&1 | redact > service-logs/opensearch.log || echo "${red}Could not get OpenSearch logs${nc}"
+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-langflow" 2>&1 | redact > service-logs/langflow.log || echo "${red}Could not get Langflow logs${nc}"
+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 | redact > service-logs/backend.log || echo "${red}Could not get backend logs${nc}"
+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-frontend" 2>&1 | redact > service-logs/frontend.log || echo "${red}Could not get frontend logs${nc}"
+ "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-opensearch" 2>&1 | redact > service-logs/opensearch.log || echo "${red}Could not get OpenSearch logs${nc}"
+ if [[ -f ~/.openrag/tui/docling-serve.log ]]; then
+ redact < ~/.openrag/tui/docling-serve.log > service-logs/docling.log || echo "${red}Could not get Docling logs${nc}"
+ fi
}
generate_report() {
diff --git a/src/tui/managers/docling_manager.py b/src/tui/managers/docling_manager.py
index b8aa47892..441c803a2 100644
--- a/src/tui/managers/docling_manager.py
+++ b/src/tui/managers/docling_manager.py
@@ -298,17 +298,15 @@ class DoclingManager:
"--python",
"3.13",
"--from",
- "docling-serve[ui]==1.20.0",
+ "docling-serve[ui]==1.26.0",
"--with",
"onnxruntime",
"--with",
"easyocr",
"--with",
- f"docling[{docling_extras}]==2.95.0",
+ f"docling[{docling_extras}]==2.108.0",
"--with",
- "docling-core==2.77.1",
- "--with",
- "docling-jobkit==1.20.0",
+ "docling-core==2.85.0",
"--with",
"transformers>=5.8.1,<5.9.0",
]
@@ -337,11 +335,20 @@ class DoclingManager:
# async task state (leading to 404 on /v1/result/{id}).
self._log_file_path.parent.mkdir(parents=True, exist_ok=True)
log_file = open(self._log_file_path, "w")
+
+ env = os.environ.copy()
+
+ # Add docling serve environment variable to change Tenant Id header name
+ # so that it's not used to enforce ownership locally.
+ if "DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER" not in env:
+ env["DOCLING_SERVE_ENG_RAY_TENANT_ID_HEADER"] = "X-Docling-Tenant-Id"
+
self._process = subprocess.Popen(
cmd,
stdout=log_file,
stderr=subprocess.STDOUT, # Merge stderr into stdout log file
start_new_session=True, # Detach from parent process group
+ env=env,
)
# Close parent's copy of the fd; the child has its own
log_file.close()