Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: add build images in matrix and push to ghcr (#2102)

Lucas Oliveira32f36e9bcec1
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index 45ac1cfb8..c3af9ce59 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -10,6 +10,10 @@ on:
type: boolean
default: true
+concurrency:
+ group: test-ci-${{ github.ref }}
+ cancel-in-progress: true
+
env:
NODE_VERSION: "22"
PYTHON_VERSION: "3.13"
@@ -17,6 +21,7 @@ env:
PLAYWRIGHT_VERSION: "1.57.0"
# Total number of parallel E2E test shards
TOTAL_SHARDS: 8
+ IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}
OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LANGFLOW_AUTO_LOGIN: "True"
@@ -42,6 +47,8 @@ env:
jobs:
changes:
runs-on: ubuntu-latest
+ permissions:
+ contents: read
outputs:
e2e: ${{ steps.filter.outputs.e2e }}
integration: ${{ steps.filter.outputs.integration }}
@@ -62,99 +69,71 @@ jobs:
- '**'
- '!docs/**'
- '!**/*.md'
+ - '!frontend/**'
build-images:
needs: changes
if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
runs-on:
labels: "ubuntu-latest"
+ permissions:
+ contents: read
+ packages: write
+ strategy:
+ fail-fast: false
+ matrix:
+ image: [opensearch, backend, frontend, langflow]
+ name: build-image (${{ matrix.image }})
steps:
- - name: Maximize build space
- uses: easimon/maximize-build-space@v10
- continue-on-error: true
- with:
- root-reserve-mb: 40960
- temp-reserve-mb: 30720
- remove-dotnet: 'true'
- remove-android: 'true'
- remove-haskell: 'true'
- remove-codeql: 'true'
- remove-docker-images: 'true'
- build-mount-path: '/var/lib/docker'
-
- - name: Restart Docker
- continue-on-error: true
- run: sudo service docker restart || sudo systemctl restart docker || true
-
- - run: df -h
-
- - name: Cleanup Docker cache
- run: |
- docker compose -f docker-compose.yml down -v --remove-orphans || true
- docker system prune -f || true
-
- - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
- run: |
- for i in 1 2 3; do
- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
- echo "Attempt $i failed, retrying in 5s..."
- sleep 5
- done || true
-
- - run: df -h
-
- name: Checkout
uses: actions/checkout@v4
- - name: Verify workspace
- run: |
- echo "Current directory: $(pwd)"
- echo "Workspace: ${GITHUB_WORKSPACE}"
- ls -la
-
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- - name: Generate Image Cache Keys
- id: cache-keys
- run: |
- OPENSEARCH_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)
- BACKEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)
- FRONTEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)
- LANGFLOW_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)
-
- echo "opensearch=$OPENSEARCH_HASH" >> $GITHUB_OUTPUT
- echo "backend=$BACKEND_HASH" >> $GITHUB_OUTPUT
- echo "frontend=$FRONTEND_HASH" >> $GITHUB_OUTPUT
- echo "langflow=$LANGFLOW_HASH" >> $GITHUB_OUTPUT
-
- - name: Cache OpenSearch Image
- uses: actions/cache@v4
+ - name: Log in to GHCR
+ if: env.IS_FORK != 'true'
+ uses: docker/login-action@v3
with:
- path: /tmp/image-cache/opensearch.tar
- key: ${{ runner.os }}-${{ runner.arch }}-opensearch-${{ steps.cache-keys.outputs.opensearch }}
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
- - name: Cache Backend Image
- uses: actions/cache@v4
- with:
- path: /tmp/image-cache/backend.tar
- key: ${{ runner.os }}-${{ runner.arch }}-backend-${{ steps.cache-keys.outputs.backend }}
+ - name: Set lower case repository name
+ id: repo
+ run: |
+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
- - name: Cache Frontend Image
- uses: actions/cache@v4
- with:
- path: /tmp/image-cache/frontend.tar
- key: ${{ runner.os }}-${{ runner.arch }}-frontend-${{ steps.cache-keys.outputs.frontend }}
+ - name: Generate Image Cache Key
+ id: cache-key
+ run: |
+ if [ "${{ matrix.image }}" = "opensearch" ]; then
+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)
+ elif [ "${{ matrix.image }}" = "backend" ]; then
+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)
+ elif [ "${{ matrix.image }}" = "frontend" ]; then
+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)
+ elif [ "${{ matrix.image }}" = "langflow" ]; then
+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)
+ fi
+ echo "key=$HASH" >> $GITHUB_OUTPUT
- - name: Cache Langflow Image
+ - name: Cache Image
uses: actions/cache@v4
with:
- path: /tmp/image-cache/langflow.tar
- key: ${{ runner.os }}-${{ runner.arch }}-langflow-${{ steps.cache-keys.outputs.langflow }}
+ path: /tmp/image-cache/${{ matrix.image }}.tar
+ key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.image }}-${{ steps.cache-key.outputs.key }}
- - name: Build OpenRAG images
+ - name: Build and Push OpenRAG image
run: |
USE_LOCAL="${{ inputs.use_local_images }}"
+ IS_FORK="${{ env.IS_FORK }}"
+
+ if [ "$IS_FORK" = "true" ]; then
+ REGISTRY_IMAGE="langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}"
+ else
+ REGISTRY_IMAGE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}"
+ fi
mkdir -p /tmp/image-cache
@@ -185,8 +164,8 @@ jobs:
if [ "$loaded_arch" = "$host_arch_norm" ]; then
echo "Cache hit for ${name}. Loaded image matches host architecture ($host_arch_norm)."
use_cache=true
- echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..."
- docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}"
+ echo "Tagging loaded image as ${REGISTRY_IMAGE}..."
+ docker tag "langflowai/openrag-${name}:cached" "${REGISTRY_IMAGE}"
else
echo "Cache image architecture ($loaded_arch) does not match host architecture ($host_arch_norm). Discarding cache."
docker rmi "langflowai/openrag-${name}:cached" 2>/dev/null || true
@@ -199,7 +178,7 @@ jobs:
if docker buildx build \
--load \
-t "langflowai/openrag-${name}:cached" \
- -t "langflowai/openrag-${name}:${CI_IMAGE_TAG}" \
+ -t "${REGISTRY_IMAGE}" \
-f "$dockerfile" .; then
echo "Saving built image to ${tar_path}..."
docker save "langflowai/openrag-${name}:cached" -o "$tar_path"
@@ -210,46 +189,60 @@ jobs:
fi
}
+ pull_image() {
+ local name=$1
+ echo "Pulling latest image for ${name} from DockerHub..."
+ if docker pull "langflowai/openrag-${name}:latest"; then
+ docker tag "langflowai/openrag-${name}:latest" "${REGISTRY_IMAGE}"
+ else
+ echo "Pull failed for ${name}."
+ return 1
+ fi
+ }
+
+ if [ "${{ matrix.image }}" = "opensearch" ]; then
+ DOCKERFILE="Dockerfile"
+ else
+ DOCKERFILE="Dockerfile.${{ matrix.image }}"
+ fi
+
for i in 1 2 3; do
- echo "Build attempt $i..."
- if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; then
- echo "Building PR images locally..."
-
- if build_or_load_image "opensearch" "Dockerfile" && \
- build_or_load_image "backend" "Dockerfile.backend" && \
- build_or_load_image "frontend" "Dockerfile.frontend" && \
- build_or_load_image "langflow" "Dockerfile.langflow"; then
+ echo "Build/pull attempt $i..."
+ if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then
+ if build_or_load_image "${{ matrix.image }}" "$DOCKERFILE"; then
+ if [ "$IS_FORK" != "true" ]; then
+ docker push "${REGISTRY_IMAGE}"
+ fi
break
fi
else
- echo "Preparing DockerHub images..."
-
- if docker pull langflowai/openrag-backend:latest && \
- docker pull langflowai/openrag-frontend:latest && \
- docker pull langflowai/openrag-langflow:latest && \
- docker tag langflowai/openrag-backend:latest "langflowai/openrag-backend:${CI_IMAGE_TAG}" && \
- docker tag langflowai/openrag-frontend:latest "langflowai/openrag-frontend:${CI_IMAGE_TAG}" && \
- docker tag langflowai/openrag-langflow:latest "langflowai/openrag-langflow:${CI_IMAGE_TAG}" && \
- build_or_load_image "opensearch" "Dockerfile"; then
+ if pull_image "${{ matrix.image }}"; then
+ if [ "$IS_FORK" != "true" ]; then
+ docker push "${REGISTRY_IMAGE}"
+ fi
break
fi
fi
- echo "Build attempt $i failed. Retrying in 10 seconds..."
+ echo "Attempt $i failed. Retrying in 10 seconds..."
sleep 10
if [ $i -eq 3 ]; then
- echo "Build failed after 3 attempts."
+ echo "Build/pull failed after 3 attempts."
exit 1
fi
done
- - name: Save OpenRAG images
- run: OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-save-images
+ - name: Save OpenRAG image (Fork fallback)
+ if: env.IS_FORK == 'true'
+ run: |
+ mkdir -p .ci-artifacts
+ docker save -o .ci-artifacts/${{ matrix.image }}.tar langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}
- - name: Upload OpenRAG image artifact
+ - name: Upload OpenRAG image artifact (Fork fallback)
+ if: env.IS_FORK == 'true'
uses: actions/upload-artifact@v4
with:
- name: openrag-ci-images-${{ github.sha }}
- path: .ci-artifacts/openrag-ci-images.tar
+ name: openrag-ci-image-${{ matrix.image }}-${{ github.sha }}
+ path: .ci-artifacts/${{ matrix.image }}.tar
retention-days: 1
compression-level: 0
@@ -270,59 +263,23 @@ jobs:
if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
runs-on:
labels: "ubuntu-latest"
+ permissions:
+ contents: read
+ packages: read
strategy:
fail-fast: false
matrix:
suite: [core, sdk-python, sdk-typescript]
name: tests (${{ matrix.suite }})
steps:
- - name: Maximize build space
- uses: easimon/maximize-build-space@v10
- continue-on-error: true
- with:
- root-reserve-mb: 40960
- temp-reserve-mb: 30720
- remove-dotnet: 'true'
- remove-android: 'true'
- remove-haskell: 'true'
- remove-codeql: 'true'
- remove-docker-images: 'true'
- build-mount-path: '/var/lib/docker'
-
- - name: Restart Docker
- continue-on-error: true
- run: sudo service docker restart || sudo systemctl restart docker || true
-
- - run: df -h
-
- - name: Cleanup Docker cache
- run: |
- docker compose -f docker-compose.yml down -v --remove-orphans || true
- docker system prune -f || true
-
- - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
- run: |
- for i in 1 2 3; do
- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
- echo "Attempt $i failed, retrying in 5s..."
- sleep 5
- done || true
-
- - run: df -h
-
- name: Checkout
uses: actions/checkout@v4
- - name: Verify workspace
- run: |
- echo "Current directory: $(pwd)"
- echo "Workspace: ${GITHUB_WORKSPACE}"
- ls -la
-
- name: Set up UV
uses: astral-sh/setup-uv@v3
with:
version: latest
+ enable-cache: true
- name: Set up Node.js
if: matrix.suite == 'sdk-typescript'
@@ -335,17 +292,45 @@ jobs:
- name: Python version
run: uv python install 3.13
- - name: Download OpenRAG image artifact
+ - name: Download OpenRAG image artifacts (Fork fallback)
+ if: env.IS_FORK == 'true'
uses: actions/download-artifact@v4
with:
- name: openrag-ci-images-${{ github.sha }}
+ pattern: openrag-ci-image-*-${{ github.sha }}
path: .ci-artifacts
+ merge-multiple: true
- - name: Load OpenRAG images
+ - name: Load OpenRAG images (Fork fallback)
+ if: env.IS_FORK == 'true'
run: |
- docker load -i .ci-artifacts/openrag-ci-images.tar
+ for f in .ci-artifacts/*.tar; do
+ docker load -i "$f"
+ done
rm -rf .ci-artifacts
+ - name: Log in to GHCR
+ if: env.IS_FORK != 'true'
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set lower case repository name
+ id: repo
+ if: env.IS_FORK != 'true'
+ run: |
+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+
+ - name: Pull and tag OpenRAG images
+ if: env.IS_FORK != 'true'
+ run: |
+ REGISTRY_PREFIX="ghcr.io/${{ steps.repo.outputs.name }}"
+ for image in opensearch backend frontend langflow; do
+ docker pull "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}"
+ docker tag "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}" "langflowai/openrag-${image}:${CI_IMAGE_TAG}"
+ done
+
- name: Run ${{ matrix.suite }} integration suite
env:
OPENSEARCH_HOST: localhost
@@ -388,6 +373,9 @@ jobs:
if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}
runs-on:
labels: "ubuntu-latest"
+ permissions:
+ contents: read
+ packages: read
strategy:
fail-fast: false
matrix:
@@ -395,36 +383,6 @@ jobs:
total_shards: [8]
name: e2e (shard ${{ matrix.shard }}/${{ matrix.total_shards }})
steps:
- - name: Maximize build space
- uses: easimon/maximize-build-space@v10
- continue-on-error: true
- with:
- root-reserve-mb: 40960
- temp-reserve-mb: 30720
- remove-dotnet: 'true'
- remove-android: 'true'
- remove-haskell: 'true'
- remove-codeql: 'true'
- remove-docker-images: 'true'
- build-mount-path: '/var/lib/docker'
-
- - name: Restart Docker
- continue-on-error: true
- run: sudo service docker restart || sudo systemctl restart docker || true
-
- - name: Cleanup Docker cache
- run: |
- docker compose -f docker-compose.yml down -v --remove-orphans || true
- docker system prune -f || true
-
- - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
- run: |
- for i in 1 2 3; do
- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
- echo "Attempt $i failed, retrying in 5s..."
- sleep 5
- done || true
-
- name: Checkout
uses: actions/checkout@v4
@@ -455,6 +413,13 @@ jobs:
sleep 5
done
+ - name: Cache Playwright Browsers
+ id: playwright-cache
+ uses: actions/cache@v4
+ with:
+ path: frontend/${{ env.PLAYWRIGHT_BROWSERS_PATH }}
+ key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
+
- name: Install Playwright Browsers
working-directory: frontend
run: |
@@ -471,17 +436,45 @@ jobs:
fi
done
- - name: Download OpenRAG image artifact
+ - name: Download OpenRAG image artifacts (Fork fallback)
+ if: env.IS_FORK == 'true'
uses: actions/download-artifact@v4
with:
- name: openrag-ci-images-${{ github.sha }}
+ pattern: openrag-ci-image-*-${{ github.sha }}
path: .ci-artifacts
+ merge-multiple: true
- - name: Load OpenRAG images
+ - name: Load OpenRAG images (Fork fallback)
+ if: env.IS_FORK == 'true'
run: |
- docker load -i .ci-artifacts/openrag-ci-images.tar
+ for f in .ci-artifacts/*.tar; do
+ docker load -i "$f"
+ done
rm -rf .ci-artifacts
+ - name: Log in to GHCR
+ if: env.IS_FORK != 'true'
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set lower case repository name
+ id: repo
+ if: env.IS_FORK != 'true'
+ run: |
+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+
+ - name: Pull and tag OpenRAG images
+ if: env.IS_FORK != 'true'
+ run: |
+ REGISTRY_PREFIX="ghcr.io/${{ steps.repo.outputs.name }}"
+ for image in opensearch backend frontend langflow; do
+ docker pull "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}"
+ docker tag "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}" "langflowai/openrag-${image}:${CI_IMAGE_TAG}"
+ done
+
- name: Setup E2E infrastructure
run: |
chmod +x scripts/setup-e2e.sh
@@ -591,6 +584,8 @@ jobs:
if: always()
runs-on:
labels: "ubuntu-latest"
+ permissions:
+ contents: read
steps:
- name: Check split integration result
run: |
@@ -611,6 +606,8 @@ jobs:
if: always()
runs-on:
labels: "ubuntu-latest"
+ permissions:
+ contents: read
steps:
- name: Check E2E result
run: |
@@ -625,3 +622,50 @@ jobs:
fi
fi
echo "E2E tests passed or skipped"
+
+ cleanup-packages:
+ needs: [tests, e2e]
+ if: always() && env.IS_FORK != 'true'
+ runs-on:
+ labels: "ubuntu-latest"
+ permissions:
+ packages: write
+ steps:
+ - name: Log in to GHCR
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set lower case repository name
+ id: repo
+ run: |
+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+
+ - name: Delete package versions
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ for image in opensearch backend frontend langflow; do
+ PACKAGE_NAME="openrag-${image}"
+ echo "Searching for version of package ${PACKAGE_NAME} with tag ${{ github.sha }}..."
+ OWNER="${{ github.repository_owner }}"
+
+ IS_ORG=$(gh api "/users/${OWNER}" --jq '.type == "Organization"' || echo "false")
+ if [ "$IS_ORG" = "true" ]; then
+ ENDPOINT="orgs/${OWNER}"
+ else
+ ENDPOINT="users/${OWNER}"
+ fi
+
+ VERSIONS_JSON=$(gh api "/${ENDPOINT}/packages/container/${PACKAGE_NAME}/versions" 2>/dev/null || echo "[]")
+ VERSION_ID=$(echo "$VERSIONS_JSON" | jq -r --arg tag "${{ github.sha }}" '.[] | select(.metadata.container.tags[] == $tag) | .id' 2>/dev/null || true)
+
+ if [ -n "$VERSION_ID" ]; then
+ echo "Deleting package version ID ${VERSION_ID} for ${PACKAGE_NAME}..."
+ gh api -X DELETE "/${ENDPOINT}/packages/container/${PACKAGE_NAME}/versions/${VERSION_ID}" || true
+ else
+ echo "No package version found with tag ${{ github.sha }} for ${PACKAGE_NAME}"
+ fi
+ done
diff --git a/scripts/ci/run_integration_suite.sh b/scripts/ci/run_integration_suite.sh
index 79f386c08..43a8cf00e 100755
--- a/scripts/ci/run_integration_suite.sh
+++ b/scripts/ci/run_integration_suite.sh
@@ -43,7 +43,7 @@ wait_for_url() {
echo "${yellow}Waiting for ${label}...${nc}"
for _ in $(seq 1 "$attempts"); do
- if curl -s "$url" >/dev/null 2>&1; then
+ if curl -sf "$url" >/dev/null 2>&1; then
return 0
fi
sleep 2
@@ -112,177 +112,248 @@ generate_report() {
fi
}
-teardown() {
- local status=$?
- if [[ "$status" -ne 0 && "$test_result" -eq 0 ]]; then
- test_result="$status"
+clean_infra() {
+ echo "${yellow}Tearing down infra and cleaning up files...${nc}"
+ uv run python scripts/docling_ctl.py stop || true
+
+ local exit_status=0
+ if ! "${compose_cmd[@]}" down -v 2>/dev/null; then
+ echo "${red}ERROR: docker compose down failed${nc}"
+ exit_status=1
fi
-
- generate_report || true
-
- if [[ "$test_result" -ne 0 ]]; then
- dump_logs || true
+
+ if command -v sudo >/dev/null 2>&1; then
+ if ! sudo rm -rf langflow-data config data keys opensearch-data openrag-documents flows; then
+ echo "${red}ERROR: sudo rm -rf failed${nc}"
+ exit_status=1
+ fi
+ else
+ if ! rm -rf langflow-data config data keys opensearch-data openrag-documents flows 2>/dev/null; then
+ local docker_rm_success=false
+ for i in 1 2 3; do
+ if docker run --rm -v "$(pwd):/work" alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents"; then
+ docker_rm_success=true
+ break
+ fi
+ echo "Attempt $i failed, retrying in 5s..."
+ sleep 5
+ done
+ if [ "$docker_rm_success" = "false" ]; then
+ echo "${red}ERROR: alpine rm -rf failed${nc}"
+ exit_status=1
+ fi
+ fi
fi
-
- echo "${yellow}Tearing down infra${nc}"
- uv run python scripts/docling_ctl.py stop || true
- "${compose_cmd[@]}" down -v 2>/dev/null || true
-
- exit "$test_result"
+
+ return "$exit_status"
}
-trap teardown EXIT
-if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]; then
- echo "${red}OPENSEARCH_PASSWORD is required${nc}"
+trap_cleanup() {
+ echo "${red}Interrupted! Cleaning up...${nc}"
+ clean_infra || true
exit 1
-fi
+}
+trap trap_cleanup SIGINT SIGTERM
-echo "${yellow}Installing test dependencies...${nc}"
-uv sync --quiet --group dev
+run_attempt() {
+ local exit_code=0
-echo "::group::Start Infrastructure"
-echo "${yellow}Cleaning up old containers and volumes...${nc}"
-"${compose_cmd[@]}" down -v 2>/dev/null || true
+ echo "::group::Start Infrastructure"
+ echo "${yellow}Starting infra for suite '${suite}' with OpenRAG version '${OPENRAG_VERSION:-latest}'${nc}"
+ if ! OPENSEARCH_HOST=opensearch "${compose_cmd[@]}" up -d opensearch langflow openrag-backend openrag-frontend; then
+ echo "${red}ERROR: docker compose up failed${nc}"
+ return 1
+ fi
-echo "${yellow}Starting infra for suite '${suite}' with OpenRAG version '${OPENRAG_VERSION:-latest}'${nc}"
-OPENSEARCH_HOST=opensearch "${compose_cmd[@]}" up -d opensearch langflow openrag-backend openrag-frontend
+ echo "${cyan}Architecture: $(uname -m), Platform: $(uname -s)${nc}"
+
+ # Normalize host architecture
+ host_arch="$(uname -m)"
+ host_arch_norm=""
+ if [[ "$host_arch" == "x86_64" ]]; then
+ host_arch_norm="amd64"
+ elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then
+ host_arch_norm="arm64"
+ else
+ host_arch_norm="$host_arch"
+ fi
-echo "${cyan}Architecture: $(uname -m), Platform: $(uname -s)${nc}"
+ # Get image architecture
+ image_name="langflowai/openrag-backend:${OPENRAG_VERSION:-latest}"
+ image_arch=$("$container_runtime" inspect --format '{{.Architecture}}' "$image_name" 2>/dev/null || echo "")
-# Normalize host architecture
-host_arch="$(uname -m)"
-host_arch_norm=""
-if [[ "$host_arch" == "x86_64" ]]; then
- host_arch_norm="amd64"
-elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then
- host_arch_norm="arm64"
-else
- host_arch_norm="$host_arch"
-fi
+ if [[ -n "$image_arch" && "$image_arch" != "$host_arch_norm" ]]; then
+ echo "${red}WARNING: Architecture mismatch detected!${nc}"
+ echo "${red}Host architecture is '${host_arch}' (${host_arch_norm}), but container image architecture is '${image_arch}'.${nc}"
+ echo "${red}The backend will run under QEMU emulation, which is extremely slow and may cause timeouts.${nc}"
+ fi
-# Get image architecture
-image_name="langflowai/openrag-backend:${OPENRAG_VERSION:-latest}"
-image_arch=$("$container_runtime" inspect --format '{{.Architecture}}' "$image_name" 2>/dev/null || echo "")
+ echo "${yellow}Starting docling-serve...${nc}"
+ docling_start_failed=0
+ docling_start_output="$(uv run python scripts/docling_ctl.py start --port 5001 --timeout 180 2>&1)" || docling_start_failed=1
+ echo "$docling_start_output"
+ if [[ "$docling_start_failed" = "1" ]]; then
+ echo "${red}ERROR: docling_ctl.py start failed. Output above.${nc}"
+ uv run python scripts/docling_ctl.py status 2>&1 || true
+ return 1
+ fi
-if [[ -n "$image_arch" && "$image_arch" != "$host_arch_norm" ]]; then
- echo "${red}WARNING: Architecture mismatch detected!${nc}"
- echo "${red}Host architecture is '${host_arch}' (${host_arch_norm}), but container image architecture is '${image_arch}'.${nc}"
- echo "${red}The backend will run under QEMU emulation, which is extremely slow and may cause timeouts.${nc}"
-fi
+ docling_endpoint="$(echo "$docling_start_output" | grep "Endpoint:" | awk '{print $2}')"
+ if [[ -z "$docling_endpoint" ]]; then
+ echo "${red}WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001${nc}"
+ docling_endpoint="http://localhost:5001"
+ fi
-echo "${yellow}Starting docling-serve...${nc}"
-docling_start_failed=0
-docling_start_output="$(uv run python scripts/docling_ctl.py start --port 5001 --timeout 180 2>&1)" || docling_start_failed=1
-echo "$docling_start_output"
-if [[ "$docling_start_failed" = "1" ]]; then
- echo "${red}ERROR: docling_ctl.py start failed. Output above.${nc}"
+ echo "${purple}Docling-serve started at ${docling_endpoint}${nc}"
+ echo "${yellow}Docling-serve status check:${nc}"
uv run python scripts/docling_ctl.py status 2>&1 || true
- exit 1
-fi
-docling_endpoint="$(echo "$docling_start_output" | grep "Endpoint:" | awk '{print $2}')"
-if [[ -z "$docling_endpoint" ]]; then
- echo "${red}WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001${nc}"
- docling_endpoint="http://localhost:5001"
-fi
+ echo "${yellow}Waiting for backend OIDC endpoint...${nc}"
+ for i in $(seq 1 60); do
+ if "${compose_cmd[@]}" exec -T openrag-backend curl -sf http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1; then
+ break
+ fi
+ if [[ "$i" -eq 60 ]]; then
+ echo "${red}Backend OIDC endpoint was not reachable in time${nc}"
+ echo "${yellow}--- Last 50 lines of backend logs ---${nc}"
+ "$container_runtime" logs --tail 50 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 || true
+ echo "${yellow}------------------------------------${nc}"
+ return 1
+ fi
+ sleep 2
+ done
-echo "${purple}Docling-serve started at ${docling_endpoint}${nc}"
-echo "${yellow}Docling-serve status check:${nc}"
-uv run python scripts/docling_ctl.py status 2>&1 || true
+ echo "${yellow}Fixing JWT key ownership for test runner (host UID $(id -u))...${nc}"
+ "$container_runtime" run --rm -v "$(pwd)/keys:/keys" alpine sh -c "chown $(id -u):$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true
-echo "${yellow}Waiting for backend OIDC endpoint...${nc}"
-for i in $(seq 1 60); do
- if "${compose_cmd[@]}" exec -T openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1; then
- break
- fi
- if [[ "$i" -eq 60 ]]; then
- echo "${red}Backend OIDC endpoint was not reachable in time${nc}"
- echo "${yellow}--- Last 50 lines of backend logs ---${nc}"
- "$container_runtime" logs --tail 50 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 || true
- echo "${yellow}------------------------------------${nc}"
- exit 1
- fi
- sleep 2
-done
+ echo "${yellow}Waiting for OpenSearch security config to be fully applied...${nc}"
+ for i in $(seq 1 60); do
+ if "${compose_cmd[@]}" logs opensearch 2>&1 | grep -q "Security configuration applied successfully"; then
+ echo "${purple}Security configuration applied${nc}"
+ break
+ fi
+ if [[ "$i" -eq 60 ]]; then
+ echo "${red}OpenSearch security config was not applied in time${nc}"
+ return 1
+ fi
+ sleep 2
+ done
-echo "${yellow}Fixing JWT key ownership for test runner (host UID $(id -u))...${nc}"
-"$container_runtime" run --rm -v "$(pwd)/keys:/keys" alpine sh -c "chown $(id -u):$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true
+ echo "${yellow}Verifying OIDC authenticator is active in OpenSearch...${nc}"
+ for i in $(seq 1 30); do
+ authc_config="$(curl -k -s -u "admin:${OPENSEARCH_PASSWORD}" https://localhost:${OPENSEARCH_PORT}/_opendistro/_security/api/securityconfig 2>/dev/null || true)"
+ if echo "$authc_config" | grep -q "openid_auth_domain"; then
+ echo "${purple}OIDC authenticator configured${nc}"
+ echo "$authc_config" | grep -A 5 "openid_auth_domain" || true
+ break
+ fi
+ if [[ "$i" -eq 30 ]]; then
+ echo "${red}OIDC authenticator NOT found or unreachable in time!${nc}"
+ echo "Security config output: $authc_config"
+ return 1
+ fi
+ sleep 2
+ done
-echo "${yellow}Waiting for OpenSearch security config to be fully applied...${nc}"
-for i in $(seq 1 60); do
- if "${compose_cmd[@]}" logs opensearch 2>&1 | grep -q "Security configuration applied successfully"; then
- echo "${purple}Security configuration applied${nc}"
- break
+ if ! wait_for_url "Langflow" "http://localhost:${LANGFLOW_PORT}/" 60; then
+ return 1
fi
- if [[ "$i" -eq 60 ]]; then
- echo "${red}OpenSearch security config was not applied in time${nc}"
- exit 1
+ if ! wait_for_url "docling-serve at ${docling_endpoint}" "${docling_endpoint}/health" 60; then
+ return 1
fi
- sleep 2
-done
+ echo "::endgroup::"
+
+ # Clear and recreate service logs directory to isolate diagnostic artifacts per attempt
+ rm -rf service-logs
+ mkdir -p service-logs
-echo "${yellow}Verifying OIDC authenticator is active in OpenSearch...${nc}"
-for i in $(seq 1 30); do
- authc_config="$(curl -k -s -u "admin:${OPENSEARCH_PASSWORD}" https://localhost:${OPENSEARCH_PORT}/_opendistro/_security/api/securityconfig 2>/dev/null || true)"
- if echo "$authc_config" | grep -q "openid_auth_domain"; then
- echo "${purple}OIDC authenticator configured${nc}"
- echo "$authc_config" | grep -A 5 "openid_auth_domain" || true
+ case "$suite" in
+ core)
+ echo "::group::Core Integration Tests"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ echo "${purple} Core Integration Tests${nc}"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ LOG_LEVEL="${LOG_LEVEL:-DEBUG}" \
+ GOOGLE_OAUTH_CLIENT_ID="" \
+ GOOGLE_OAUTH_CLIENT_SECRET="" \
+ OPENSEARCH_HOST=localhost OPENSEARCH_PORT=${OPENSEARCH_PORT} \
+ OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" \
+ DISABLE_STARTUP_INGEST="${DISABLE_STARTUP_INGEST:-true}" \
+ uv run pytest tests/integration/core -vv -s --log-file=service-logs/pytest-core.log --log-file-level=DEBUG --junitxml=service-logs/junit-core.xml || exit_code=1
+ echo "::endgroup::"
+ test_jwt_opensearch || exit_code=1
+ ;;
+ sdk-python)
+ if ! wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60; then
+ return 1
+ fi
+ echo "::group::SDK Integration Tests (Python)"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ echo "${purple} SDK Integration Tests (Python)${nc}"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ if ! uv pip install --quiet -e sdks/python; then
+ echo "${red}ERROR: uv pip install failed${nc}"
+ return 1
+ fi
+ SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s --log-file=service-logs/pytest-sdk.log --log-file-level=DEBUG --junitxml=service-logs/junit-sdk-python.xml || exit_code=1
+ echo "::endgroup::"
+ ;;
+ sdk-typescript)
+ if ! wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60; then
+ return 1
+ fi
+ echo "::group::SDK Integration Tests (TypeScript)"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ echo "${purple} SDK Integration Tests (TypeScript)${nc}"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ cd sdks/typescript
+ npm install && npm run build && OPENRAG_URL=http://localhost:3000 npm test -- --reporter=junit --outputFile=../../service-logs/junit-sdk-typescript.xml || exit_code=1
+ cd ../..
+ echo "::endgroup::"
+ ;;
+ *)
+ echo "${red}Unknown integration suite: ${suite}${nc}"
+ exit_code=1
+ ;;
+ esac
+
+ return "$exit_code"
+}
+
+if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]; then
+ echo "${red}OPENSEARCH_PASSWORD is required${nc}"
+ exit 1
+fi
+
+echo "${yellow}Installing test dependencies...${nc}"
+uv sync --quiet --group dev
+
+max_attempts=2
+test_result=0
+
+for attempt in $(seq 1 "$max_attempts"); do
+ echo "${yellow}=== Starting Integration Suite '${suite}' - Attempt $attempt of $max_attempts ===${nc}"
+
+ if run_attempt; then
+ echo "${green}Attempt $attempt succeeded!${nc}"
+ test_result=0
break
+ else
+ echo "${red}Attempt $attempt failed!${nc}"
+ test_result=1
+ dump_logs || true
+
+ if [ "$attempt" -lt "$max_attempts" ]; then
+ if ! clean_infra; then
+ echo "${red}ERROR: Cleanup failed. Aborting further attempts.${nc}"
+ test_result=1
+ break
+ fi
+ echo "${yellow}Retrying in 5 seconds...${nc}"
+ sleep 5
+ fi
fi
- if [[ "$i" -eq 30 ]]; then
- echo "${red}OIDC authenticator NOT found or unreachable in time!${nc}"
- echo "Security config output: $authc_config"
- exit 1
- fi
- sleep 2
done
-wait_for_url "Langflow" "http://localhost:${LANGFLOW_PORT}/" 60
-wait_for_url "docling-serve at ${docling_endpoint}" "${docling_endpoint}/health" 60
-echo "::endgroup::"
-
-mkdir -p service-logs
-
-case "$suite" in
- core)
- echo "::group::Core Integration Tests"
- echo "${cyan}════════════════════════════════════════${nc}"
- echo "${purple} Core Integration Tests${nc}"
- echo "${cyan}════════════════════════════════════════${nc}"
- mkdir -p service-logs
- LOG_LEVEL="${LOG_LEVEL:-DEBUG}" \
- GOOGLE_OAUTH_CLIENT_ID="" \
- GOOGLE_OAUTH_CLIENT_SECRET="" \
- OPENSEARCH_HOST=localhost OPENSEARCH_PORT=${OPENSEARCH_PORT} \
- OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" \
- DISABLE_STARTUP_INGEST="${DISABLE_STARTUP_INGEST:-true}" \
- uv run pytest tests/integration/core -vv -s --log-file=service-logs/pytest-core.log --log-file-level=DEBUG --junitxml=service-logs/junit-core.xml || test_result=1
- echo "::endgroup::"
- test_jwt_opensearch || test_result=1
- ;;
- sdk-python)
- wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60
- echo "::group::SDK Integration Tests (Python)"
- echo "${cyan}════════════════════════════════════════${nc}"
- echo "${purple} SDK Integration Tests (Python)${nc}"
- echo "${cyan}════════════════════════════════════════${nc}"
- uv pip install --quiet -e sdks/python
- SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s --log-file=service-logs/pytest-sdk.log --log-file-level=DEBUG --junitxml=service-logs/junit-sdk-python.xml || test_result=1
- echo "::endgroup::"
- ;;
- sdk-typescript)
- wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60
- echo "::group::SDK Integration Tests (TypeScript)"
- echo "${cyan}════════════════════════════════════════${nc}"
- echo "${purple} SDK Integration Tests (TypeScript)${nc}"
- echo "${cyan}════════════════════════════════════════${nc}"
- cd sdks/typescript
- npm install && npm run build && OPENRAG_URL=http://localhost:3000 npm test -- --reporter=junit --outputFile=../../service-logs/junit-sdk-typescript.xml || test_result=1
- cd ../..
- echo "::endgroup::"
- ;;
- *)
- echo "${red}Unknown integration suite: ${suite}${nc}"
- test_result=1
- ;;
-esac
+generate_report || true
+exit "$test_result"