OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: add build images in matrix and push to ghcr (#2102)
Lucas Oliveira32f36e9bcec1
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex 45ac1cfb8..c3af9ce59 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -10,6 +10,10 @@ on:type: booleandefault: true+concurrency:+ group: test-ci-${{ github.ref }}+ cancel-in-progress: true+env:NODE_VERSION: "22"PYTHON_VERSION: "3.13"@@ -17,6 +21,7 @@ env:PLAYWRIGHT_VERSION: "1.57.0"# Total number of parallel E2E test shardsTOTAL_SHARDS: 8+ IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }}OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }}OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}LANGFLOW_AUTO_LOGIN: "True"@@ -42,6 +47,8 @@ env:jobs:changes:runs-on: ubuntu-latest+ permissions:+ contents: readoutputs:e2e: ${{ steps.filter.outputs.e2e }}integration: ${{ steps.filter.outputs.integration }}@@ -62,99 +69,71 @@ jobs:- '**'- '!docs/**'- '!**/*.md'+ - '!frontend/**'build-images:needs: changesif: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}runs-on:labels: "ubuntu-latest"+ permissions:+ contents: read+ packages: write+ strategy:+ fail-fast: false+ matrix:+ image: [opensearch, backend, frontend, langflow]+ name: build-image (${{ matrix.image }})steps:- - name: Maximize build space- uses: easimon/maximize-build-space@v10- continue-on-error: true- with:- root-reserve-mb: 40960- temp-reserve-mb: 30720- remove-dotnet: 'true'- remove-android: 'true'- remove-haskell: 'true'- remove-codeql: 'true'- remove-docker-images: 'true'- build-mount-path: '/var/lib/docker'-- - name: Restart Docker- continue-on-error: true- run: sudo service docker restart || sudo systemctl restart docker || true-- - run: df -h-- - name: Cleanup Docker cache- run: |- docker compose -f docker-compose.yml down -v --remove-orphans || true- docker system prune -f || true-- - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)- run: |- for i in 1 2 3; do- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break- echo "Attempt $i failed, retrying in 5s..."- sleep 5- done || true-- - run: df -h-- name: Checkoutuses: actions/checkout@v4- - name: Verify workspace- run: |- echo "Current directory: $(pwd)"- echo "Workspace: ${GITHUB_WORKSPACE}"- ls -la-- name: Set up Docker Buildxuses: docker/setup-buildx-action@v3- - name: Generate Image Cache Keys- id: cache-keys- run: |- OPENSEARCH_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)- BACKEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)- FRONTEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)- LANGFLOW_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)-- echo "opensearch=$OPENSEARCH_HASH" >> $GITHUB_OUTPUT- echo "backend=$BACKEND_HASH" >> $GITHUB_OUTPUT- echo "frontend=$FRONTEND_HASH" >> $GITHUB_OUTPUT- echo "langflow=$LANGFLOW_HASH" >> $GITHUB_OUTPUT-- - name: Cache OpenSearch Image- uses: actions/cache@v4+ - name: Log in to GHCR+ if: env.IS_FORK != 'true'+ uses: docker/login-action@v3with:- path: /tmp/image-cache/opensearch.tar- key: ${{ runner.os }}-${{ runner.arch }}-opensearch-${{ steps.cache-keys.outputs.opensearch }}+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}- - name: Cache Backend Image- uses: actions/cache@v4- with:- path: /tmp/image-cache/backend.tar- key: ${{ runner.os }}-${{ runner.arch }}-backend-${{ steps.cache-keys.outputs.backend }}+ - name: Set lower case repository name+ id: repo+ run: |+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT- - name: Cache Frontend Image- uses: actions/cache@v4- with:- path: /tmp/image-cache/frontend.tar- key: ${{ runner.os }}-${{ runner.arch }}-frontend-${{ steps.cache-keys.outputs.frontend }}+ - name: Generate Image Cache Key+ id: cache-key+ run: |+ if [ "${{ matrix.image }}" = "opensearch" ]; then+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)+ elif [ "${{ matrix.image }}" = "backend" ]; then+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)+ elif [ "${{ matrix.image }}" = "frontend" ]; then+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)+ elif [ "${{ matrix.image }}" = "langflow" ]; then+ HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)+ fi+ echo "key=$HASH" >> $GITHUB_OUTPUT- - name: Cache Langflow Image+ - name: Cache Imageuses: actions/cache@v4with:- path: /tmp/image-cache/langflow.tar- key: ${{ runner.os }}-${{ runner.arch }}-langflow-${{ steps.cache-keys.outputs.langflow }}+ path: /tmp/image-cache/${{ matrix.image }}.tar+ key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.image }}-${{ steps.cache-key.outputs.key }}- - name: Build OpenRAG images+ - name: Build and Push OpenRAG imagerun: |USE_LOCAL="${{ inputs.use_local_images }}"+ IS_FORK="${{ env.IS_FORK }}"++ if [ "$IS_FORK" = "true" ]; then+ REGISTRY_IMAGE="langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}"+ else+ REGISTRY_IMAGE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}"+ fimkdir -p /tmp/image-cache@@ -185,8 +164,8 @@ jobs:if [ "$loaded_arch" = "$host_arch_norm" ]; thenecho "Cache hit for ${name}. Loaded image matches host architecture ($host_arch_norm)."use_cache=true- echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..."- docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}"+ echo "Tagging loaded image as ${REGISTRY_IMAGE}..."+ docker tag "langflowai/openrag-${name}:cached" "${REGISTRY_IMAGE}"elseecho "Cache image architecture ($loaded_arch) does not match host architecture ($host_arch_norm). Discarding cache."docker rmi "langflowai/openrag-${name}:cached" 2>/dev/null || true@@ -199,7 +178,7 @@ jobs:if docker buildx build \--load \-t "langflowai/openrag-${name}:cached" \- -t "langflowai/openrag-${name}:${CI_IMAGE_TAG}" \+ -t "${REGISTRY_IMAGE}" \-f "$dockerfile" .; thenecho "Saving built image to ${tar_path}..."docker save "langflowai/openrag-${name}:cached" -o "$tar_path"@@ -210,46 +189,60 @@ jobs:fi}+ pull_image() {+ local name=$1+ echo "Pulling latest image for ${name} from DockerHub..."+ if docker pull "langflowai/openrag-${name}:latest"; then+ docker tag "langflowai/openrag-${name}:latest" "${REGISTRY_IMAGE}"+ else+ echo "Pull failed for ${name}."+ return 1+ fi+ }++ if [ "${{ matrix.image }}" = "opensearch" ]; then+ DOCKERFILE="Dockerfile"+ else+ DOCKERFILE="Dockerfile.${{ matrix.image }}"+ fi+for i in 1 2 3; do- echo "Build attempt $i..."- if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; then- echo "Building PR images locally..."-- if build_or_load_image "opensearch" "Dockerfile" && \- build_or_load_image "backend" "Dockerfile.backend" && \- build_or_load_image "frontend" "Dockerfile.frontend" && \- build_or_load_image "langflow" "Dockerfile.langflow"; then+ echo "Build/pull attempt $i..."+ if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then+ if build_or_load_image "${{ matrix.image }}" "$DOCKERFILE"; then+ if [ "$IS_FORK" != "true" ]; then+ docker push "${REGISTRY_IMAGE}"+ fibreakfielse- echo "Preparing DockerHub images..."-- if docker pull langflowai/openrag-backend:latest && \- docker pull langflowai/openrag-frontend:latest && \- docker pull langflowai/openrag-langflow:latest && \- docker tag langflowai/openrag-backend:latest "langflowai/openrag-backend:${CI_IMAGE_TAG}" && \- docker tag langflowai/openrag-frontend:latest "langflowai/openrag-frontend:${CI_IMAGE_TAG}" && \- docker tag langflowai/openrag-langflow:latest "langflowai/openrag-langflow:${CI_IMAGE_TAG}" && \- build_or_load_image "opensearch" "Dockerfile"; then+ if pull_image "${{ matrix.image }}"; then+ if [ "$IS_FORK" != "true" ]; then+ docker push "${REGISTRY_IMAGE}"+ fibreakfifi- echo "Build attempt $i failed. Retrying in 10 seconds..."+ echo "Attempt $i failed. Retrying in 10 seconds..."sleep 10if [ $i -eq 3 ]; then- echo "Build failed after 3 attempts."+ echo "Build/pull failed after 3 attempts."exit 1fidone- - name: Save OpenRAG images- run: OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-save-images+ - name: Save OpenRAG image (Fork fallback)+ if: env.IS_FORK == 'true'+ run: |+ mkdir -p .ci-artifacts+ docker save -o .ci-artifacts/${{ matrix.image }}.tar langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}- - name: Upload OpenRAG image artifact+ - name: Upload OpenRAG image artifact (Fork fallback)+ if: env.IS_FORK == 'true'uses: actions/upload-artifact@v4with:- name: openrag-ci-images-${{ github.sha }}- path: .ci-artifacts/openrag-ci-images.tar+ name: openrag-ci-image-${{ matrix.image }}-${{ github.sha }}+ path: .ci-artifacts/${{ matrix.image }}.tarretention-days: 1compression-level: 0@@ -270,59 +263,23 @@ jobs:if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}runs-on:labels: "ubuntu-latest"+ permissions:+ contents: read+ packages: readstrategy:fail-fast: falsematrix:suite: [core, sdk-python, sdk-typescript]name: tests (${{ matrix.suite }})steps:- - name: Maximize build space- uses: easimon/maximize-build-space@v10- continue-on-error: true- with:- root-reserve-mb: 40960- temp-reserve-mb: 30720- remove-dotnet: 'true'- remove-android: 'true'- remove-haskell: 'true'- remove-codeql: 'true'- remove-docker-images: 'true'- build-mount-path: '/var/lib/docker'-- - name: Restart Docker- continue-on-error: true- run: sudo service docker restart || sudo systemctl restart docker || true-- - run: df -h-- - name: Cleanup Docker cache- run: |- docker compose -f docker-compose.yml down -v --remove-orphans || true- docker system prune -f || true-- - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)- run: |- for i in 1 2 3; do- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break- echo "Attempt $i failed, retrying in 5s..."- sleep 5- done || true-- - run: df -h-- name: Checkoutuses: actions/checkout@v4- - name: Verify workspace- run: |- echo "Current directory: $(pwd)"- echo "Workspace: ${GITHUB_WORKSPACE}"- ls -la-- name: Set up UVuses: astral-sh/setup-uv@v3with:version: latest+ enable-cache: true- name: Set up Node.jsif: matrix.suite == 'sdk-typescript'@@ -335,17 +292,45 @@ jobs:- name: Python versionrun: uv python install 3.13- - name: Download OpenRAG image artifact+ - name: Download OpenRAG image artifacts (Fork fallback)+ if: env.IS_FORK == 'true'uses: actions/download-artifact@v4with:- name: openrag-ci-images-${{ github.sha }}+ pattern: openrag-ci-image-*-${{ github.sha }}path: .ci-artifacts+ merge-multiple: true- - name: Load OpenRAG images+ - name: Load OpenRAG images (Fork fallback)+ if: env.IS_FORK == 'true'run: |- docker load -i .ci-artifacts/openrag-ci-images.tar+ for f in .ci-artifacts/*.tar; do+ docker load -i "$f"+ donerm -rf .ci-artifacts+ - name: Log in to GHCR+ if: env.IS_FORK != 'true'+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Set lower case repository name+ id: repo+ if: env.IS_FORK != 'true'+ run: |+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT++ - name: Pull and tag OpenRAG images+ if: env.IS_FORK != 'true'+ run: |+ REGISTRY_PREFIX="ghcr.io/${{ steps.repo.outputs.name }}"+ for image in opensearch backend frontend langflow; do+ docker pull "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}"+ docker tag "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}" "langflowai/openrag-${image}:${CI_IMAGE_TAG}"+ done+- name: Run ${{ matrix.suite }} integration suiteenv:OPENSEARCH_HOST: localhost@@ -388,6 +373,9 @@ jobs:if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}runs-on:labels: "ubuntu-latest"+ permissions:+ contents: read+ packages: readstrategy:fail-fast: falsematrix:@@ -395,36 +383,6 @@ jobs:total_shards: [8]name: e2e (shard ${{ matrix.shard }}/${{ matrix.total_shards }})steps:- - name: Maximize build space- uses: easimon/maximize-build-space@v10- continue-on-error: true- with:- root-reserve-mb: 40960- temp-reserve-mb: 30720- remove-dotnet: 'true'- remove-android: 'true'- remove-haskell: 'true'- remove-codeql: 'true'- remove-docker-images: 'true'- build-mount-path: '/var/lib/docker'-- - name: Restart Docker- continue-on-error: true- run: sudo service docker restart || sudo systemctl restart docker || true-- - name: Cleanup Docker cache- run: |- docker compose -f docker-compose.yml down -v --remove-orphans || true- docker system prune -f || true-- - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)- run: |- for i in 1 2 3; do- docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break- echo "Attempt $i failed, retrying in 5s..."- sleep 5- done || true-- name: Checkoutuses: actions/checkout@v4@@ -455,6 +413,13 @@ jobs:sleep 5done+ - name: Cache Playwright Browsers+ id: playwright-cache+ uses: actions/cache@v4+ with:+ path: frontend/${{ env.PLAYWRIGHT_BROWSERS_PATH }}+ key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}+- name: Install Playwright Browsersworking-directory: frontendrun: |@@ -471,17 +436,45 @@ jobs:fidone- - name: Download OpenRAG image artifact+ - name: Download OpenRAG image artifacts (Fork fallback)+ if: env.IS_FORK == 'true'uses: actions/download-artifact@v4with:- name: openrag-ci-images-${{ github.sha }}+ pattern: openrag-ci-image-*-${{ github.sha }}path: .ci-artifacts+ merge-multiple: true- - name: Load OpenRAG images+ - name: Load OpenRAG images (Fork fallback)+ if: env.IS_FORK == 'true'run: |- docker load -i .ci-artifacts/openrag-ci-images.tar+ for f in .ci-artifacts/*.tar; do+ docker load -i "$f"+ donerm -rf .ci-artifacts+ - name: Log in to GHCR+ if: env.IS_FORK != 'true'+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Set lower case repository name+ id: repo+ if: env.IS_FORK != 'true'+ run: |+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT++ - name: Pull and tag OpenRAG images+ if: env.IS_FORK != 'true'+ run: |+ REGISTRY_PREFIX="ghcr.io/${{ steps.repo.outputs.name }}"+ for image in opensearch backend frontend langflow; do+ docker pull "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}"+ docker tag "${REGISTRY_PREFIX}/openrag-${image}:${{ github.sha }}" "langflowai/openrag-${image}:${CI_IMAGE_TAG}"+ done+- name: Setup E2E infrastructurerun: |chmod +x scripts/setup-e2e.sh@@ -591,6 +584,8 @@ jobs:if: always()runs-on:labels: "ubuntu-latest"+ permissions:+ contents: readsteps:- name: Check split integration resultrun: |@@ -611,6 +606,8 @@ jobs:if: always()runs-on:labels: "ubuntu-latest"+ permissions:+ contents: readsteps:- name: Check E2E resultrun: |@@ -625,3 +622,50 @@ jobs:fifiecho "E2E tests passed or skipped"++ cleanup-packages:+ needs: [tests, e2e]+ if: always() && env.IS_FORK != 'true'+ runs-on:+ labels: "ubuntu-latest"+ permissions:+ packages: write+ steps:+ - name: Log in to GHCR+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Set lower case repository name+ id: repo+ run: |+ echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT++ - name: Delete package versions+ env:+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ for image in opensearch backend frontend langflow; do+ PACKAGE_NAME="openrag-${image}"+ echo "Searching for version of package ${PACKAGE_NAME} with tag ${{ github.sha }}..."+ OWNER="${{ github.repository_owner }}"++ IS_ORG=$(gh api "/users/${OWNER}" --jq '.type == "Organization"' || echo "false")+ if [ "$IS_ORG" = "true" ]; then+ ENDPOINT="orgs/${OWNER}"+ else+ ENDPOINT="users/${OWNER}"+ fi++ VERSIONS_JSON=$(gh api "/${ENDPOINT}/packages/container/${PACKAGE_NAME}/versions" 2>/dev/null || echo "[]")+ VERSION_ID=$(echo "$VERSIONS_JSON" | jq -r --arg tag "${{ github.sha }}" '.[] | select(.metadata.container.tags[] == $tag) | .id' 2>/dev/null || true)++ if [ -n "$VERSION_ID" ]; then+ echo "Deleting package version ID ${VERSION_ID} for ${PACKAGE_NAME}..."+ gh api -X DELETE "/${ENDPOINT}/packages/container/${PACKAGE_NAME}/versions/${VERSION_ID}" || true+ else+ echo "No package version found with tag ${{ github.sha }} for ${PACKAGE_NAME}"+ fi+ donediff --git a/scripts/ci/run_integration_suite.sh b/scripts/ci/run_integration_suite.shindex 79f386c08..43a8cf00e 100755--- a/scripts/ci/run_integration_suite.sh+++ b/scripts/ci/run_integration_suite.sh@@ -43,7 +43,7 @@ wait_for_url() {echo "${yellow}Waiting for ${label}...${nc}"for _ in $(seq 1 "$attempts"); do- if curl -s "$url" >/dev/null 2>&1; then+ if curl -sf "$url" >/dev/null 2>&1; thenreturn 0fisleep 2@@ -112,177 +112,248 @@ generate_report() {fi}-teardown() {- local status=$?- if [[ "$status" -ne 0 && "$test_result" -eq 0 ]]; then- test_result="$status"+clean_infra() {+ echo "${yellow}Tearing down infra and cleaning up files...${nc}"+ uv run python scripts/docling_ctl.py stop || true++ local exit_status=0+ if ! "${compose_cmd[@]}" down -v 2>/dev/null; then+ echo "${red}ERROR: docker compose down failed${nc}"+ exit_status=1fi-- generate_report || true-- if [[ "$test_result" -ne 0 ]]; then- dump_logs || true++ if command -v sudo >/dev/null 2>&1; then+ if ! sudo rm -rf langflow-data config data keys opensearch-data openrag-documents flows; then+ echo "${red}ERROR: sudo rm -rf failed${nc}"+ exit_status=1+ fi+ else+ if ! rm -rf langflow-data config data keys opensearch-data openrag-documents flows 2>/dev/null; then+ local docker_rm_success=false+ for i in 1 2 3; do+ if docker run --rm -v "$(pwd):/work" alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents"; then+ docker_rm_success=true+ break+ fi+ echo "Attempt $i failed, retrying in 5s..."+ sleep 5+ done+ if [ "$docker_rm_success" = "false" ]; then+ echo "${red}ERROR: alpine rm -rf failed${nc}"+ exit_status=1+ fi+ fifi-- echo "${yellow}Tearing down infra${nc}"- uv run python scripts/docling_ctl.py stop || true- "${compose_cmd[@]}" down -v 2>/dev/null || true-- exit "$test_result"++ return "$exit_status"}-trap teardown EXIT-if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]; then- echo "${red}OPENSEARCH_PASSWORD is required${nc}"+trap_cleanup() {+ echo "${red}Interrupted! Cleaning up...${nc}"+ clean_infra || trueexit 1-fi+}+trap trap_cleanup SIGINT SIGTERM-echo "${yellow}Installing test dependencies...${nc}"-uv sync --quiet --group dev+run_attempt() {+ local exit_code=0-echo "::group::Start Infrastructure"-echo "${yellow}Cleaning up old containers and volumes...${nc}"-"${compose_cmd[@]}" down -v 2>/dev/null || true+ echo "::group::Start Infrastructure"+ echo "${yellow}Starting infra for suite '${suite}' with OpenRAG version '${OPENRAG_VERSION:-latest}'${nc}"+ if ! OPENSEARCH_HOST=opensearch "${compose_cmd[@]}" up -d opensearch langflow openrag-backend openrag-frontend; then+ echo "${red}ERROR: docker compose up failed${nc}"+ return 1+ fi-echo "${yellow}Starting infra for suite '${suite}' with OpenRAG version '${OPENRAG_VERSION:-latest}'${nc}"-OPENSEARCH_HOST=opensearch "${compose_cmd[@]}" up -d opensearch langflow openrag-backend openrag-frontend+ echo "${cyan}Architecture: $(uname -m), Platform: $(uname -s)${nc}"++ # Normalize host architecture+ host_arch="$(uname -m)"+ host_arch_norm=""+ if [[ "$host_arch" == "x86_64" ]]; then+ host_arch_norm="amd64"+ elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then+ host_arch_norm="arm64"+ else+ host_arch_norm="$host_arch"+ fi-echo "${cyan}Architecture: $(uname -m), Platform: $(uname -s)${nc}"+ # Get image architecture+ image_name="langflowai/openrag-backend:${OPENRAG_VERSION:-latest}"+ image_arch=$("$container_runtime" inspect --format '{{.Architecture}}' "$image_name" 2>/dev/null || echo "")-# Normalize host architecture-host_arch="$(uname -m)"-host_arch_norm=""-if [[ "$host_arch" == "x86_64" ]]; then- host_arch_norm="amd64"-elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then- host_arch_norm="arm64"-else- host_arch_norm="$host_arch"-fi+ if [[ -n "$image_arch" && "$image_arch" != "$host_arch_norm" ]]; then+ echo "${red}WARNING: Architecture mismatch detected!${nc}"+ echo "${red}Host architecture is '${host_arch}' (${host_arch_norm}), but container image architecture is '${image_arch}'.${nc}"+ echo "${red}The backend will run under QEMU emulation, which is extremely slow and may cause timeouts.${nc}"+ fi-# Get image architecture-image_name="langflowai/openrag-backend:${OPENRAG_VERSION:-latest}"-image_arch=$("$container_runtime" inspect --format '{{.Architecture}}' "$image_name" 2>/dev/null || echo "")+ echo "${yellow}Starting docling-serve...${nc}"+ docling_start_failed=0+ docling_start_output="$(uv run python scripts/docling_ctl.py start --port 5001 --timeout 180 2>&1)" || docling_start_failed=1+ echo "$docling_start_output"+ if [[ "$docling_start_failed" = "1" ]]; then+ echo "${red}ERROR: docling_ctl.py start failed. Output above.${nc}"+ uv run python scripts/docling_ctl.py status 2>&1 || true+ return 1+ fi-if [[ -n "$image_arch" && "$image_arch" != "$host_arch_norm" ]]; then- echo "${red}WARNING: Architecture mismatch detected!${nc}"- echo "${red}Host architecture is '${host_arch}' (${host_arch_norm}), but container image architecture is '${image_arch}'.${nc}"- echo "${red}The backend will run under QEMU emulation, which is extremely slow and may cause timeouts.${nc}"-fi+ docling_endpoint="$(echo "$docling_start_output" | grep "Endpoint:" | awk '{print $2}')"+ if [[ -z "$docling_endpoint" ]]; then+ echo "${red}WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001${nc}"+ docling_endpoint="http://localhost:5001"+ fi-echo "${yellow}Starting docling-serve...${nc}"-docling_start_failed=0-docling_start_output="$(uv run python scripts/docling_ctl.py start --port 5001 --timeout 180 2>&1)" || docling_start_failed=1-echo "$docling_start_output"-if [[ "$docling_start_failed" = "1" ]]; then- echo "${red}ERROR: docling_ctl.py start failed. Output above.${nc}"+ echo "${purple}Docling-serve started at ${docling_endpoint}${nc}"+ echo "${yellow}Docling-serve status check:${nc}"uv run python scripts/docling_ctl.py status 2>&1 || true- exit 1-fi-docling_endpoint="$(echo "$docling_start_output" | grep "Endpoint:" | awk '{print $2}')"-if [[ -z "$docling_endpoint" ]]; then- echo "${red}WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001${nc}"- docling_endpoint="http://localhost:5001"-fi+ echo "${yellow}Waiting for backend OIDC endpoint...${nc}"+ for i in $(seq 1 60); do+ if "${compose_cmd[@]}" exec -T openrag-backend curl -sf http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1; then+ break+ fi+ if [[ "$i" -eq 60 ]]; then+ echo "${red}Backend OIDC endpoint was not reachable in time${nc}"+ echo "${yellow}--- Last 50 lines of backend logs ---${nc}"+ "$container_runtime" logs --tail 50 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 || true+ echo "${yellow}------------------------------------${nc}"+ return 1+ fi+ sleep 2+ done-echo "${purple}Docling-serve started at ${docling_endpoint}${nc}"-echo "${yellow}Docling-serve status check:${nc}"-uv run python scripts/docling_ctl.py status 2>&1 || true+ echo "${yellow}Fixing JWT key ownership for test runner (host UID $(id -u))...${nc}"+ "$container_runtime" run --rm -v "$(pwd)/keys:/keys" alpine sh -c "chown $(id -u):$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true-echo "${yellow}Waiting for backend OIDC endpoint...${nc}"-for i in $(seq 1 60); do- if "${compose_cmd[@]}" exec -T openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1; then- break- fi- if [[ "$i" -eq 60 ]]; then- echo "${red}Backend OIDC endpoint was not reachable in time${nc}"- echo "${yellow}--- Last 50 lines of backend logs ---${nc}"- "$container_runtime" logs --tail 50 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 || true- echo "${yellow}------------------------------------${nc}"- exit 1- fi- sleep 2-done+ echo "${yellow}Waiting for OpenSearch security config to be fully applied...${nc}"+ for i in $(seq 1 60); do+ if "${compose_cmd[@]}" logs opensearch 2>&1 | grep -q "Security configuration applied successfully"; then+ echo "${purple}Security configuration applied${nc}"+ break+ fi+ if [[ "$i" -eq 60 ]]; then+ echo "${red}OpenSearch security config was not applied in time${nc}"+ return 1+ fi+ sleep 2+ done-echo "${yellow}Fixing JWT key ownership for test runner (host UID $(id -u))...${nc}"-"$container_runtime" run --rm -v "$(pwd)/keys:/keys" alpine sh -c "chown $(id -u):$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true+ echo "${yellow}Verifying OIDC authenticator is active in OpenSearch...${nc}"+ for i in $(seq 1 30); do+ authc_config="$(curl -k -s -u "admin:${OPENSEARCH_PASSWORD}" https://localhost:${OPENSEARCH_PORT}/_opendistro/_security/api/securityconfig 2>/dev/null || true)"+ if echo "$authc_config" | grep -q "openid_auth_domain"; then+ echo "${purple}OIDC authenticator configured${nc}"+ echo "$authc_config" | grep -A 5 "openid_auth_domain" || true+ break+ fi+ if [[ "$i" -eq 30 ]]; then+ echo "${red}OIDC authenticator NOT found or unreachable in time!${nc}"+ echo "Security config output: $authc_config"+ return 1+ fi+ sleep 2+ done-echo "${yellow}Waiting for OpenSearch security config to be fully applied...${nc}"-for i in $(seq 1 60); do- if "${compose_cmd[@]}" logs opensearch 2>&1 | grep -q "Security configuration applied successfully"; then- echo "${purple}Security configuration applied${nc}"- break+ if ! wait_for_url "Langflow" "http://localhost:${LANGFLOW_PORT}/" 60; then+ return 1fi- if [[ "$i" -eq 60 ]]; then- echo "${red}OpenSearch security config was not applied in time${nc}"- exit 1+ if ! wait_for_url "docling-serve at ${docling_endpoint}" "${docling_endpoint}/health" 60; then+ return 1fi- sleep 2-done+ echo "::endgroup::"++ # Clear and recreate service logs directory to isolate diagnostic artifacts per attempt+ rm -rf service-logs+ mkdir -p service-logs-echo "${yellow}Verifying OIDC authenticator is active in OpenSearch...${nc}"-for i in $(seq 1 30); do- authc_config="$(curl -k -s -u "admin:${OPENSEARCH_PASSWORD}" https://localhost:${OPENSEARCH_PORT}/_opendistro/_security/api/securityconfig 2>/dev/null || true)"- if echo "$authc_config" | grep -q "openid_auth_domain"; then- echo "${purple}OIDC authenticator configured${nc}"- echo "$authc_config" | grep -A 5 "openid_auth_domain" || true+ case "$suite" in+ core)+ echo "::group::Core Integration Tests"+ echo "${cyan}════════════════════════════════════════${nc}"+ echo "${purple} Core Integration Tests${nc}"+ echo "${cyan}════════════════════════════════════════${nc}"+ LOG_LEVEL="${LOG_LEVEL:-DEBUG}" \+ GOOGLE_OAUTH_CLIENT_ID="" \+ GOOGLE_OAUTH_CLIENT_SECRET="" \+ OPENSEARCH_HOST=localhost OPENSEARCH_PORT=${OPENSEARCH_PORT} \+ OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" \+ DISABLE_STARTUP_INGEST="${DISABLE_STARTUP_INGEST:-true}" \+ uv run pytest tests/integration/core -vv -s --log-file=service-logs/pytest-core.log --log-file-level=DEBUG --junitxml=service-logs/junit-core.xml || exit_code=1+ echo "::endgroup::"+ test_jwt_opensearch || exit_code=1+ ;;+ sdk-python)+ if ! wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60; then+ return 1+ fi+ echo "::group::SDK Integration Tests (Python)"+ echo "${cyan}════════════════════════════════════════${nc}"+ echo "${purple} SDK Integration Tests (Python)${nc}"+ echo "${cyan}════════════════════════════════════════${nc}"+ if ! uv pip install --quiet -e sdks/python; then+ echo "${red}ERROR: uv pip install failed${nc}"+ return 1+ fi+ SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s --log-file=service-logs/pytest-sdk.log --log-file-level=DEBUG --junitxml=service-logs/junit-sdk-python.xml || exit_code=1+ echo "::endgroup::"+ ;;+ sdk-typescript)+ if ! wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60; then+ return 1+ fi+ echo "::group::SDK Integration Tests (TypeScript)"+ echo "${cyan}════════════════════════════════════════${nc}"+ echo "${purple} SDK Integration Tests (TypeScript)${nc}"+ echo "${cyan}════════════════════════════════════════${nc}"+ cd sdks/typescript+ npm install && npm run build && OPENRAG_URL=http://localhost:3000 npm test -- --reporter=junit --outputFile=../../service-logs/junit-sdk-typescript.xml || exit_code=1+ cd ../..+ echo "::endgroup::"+ ;;+ *)+ echo "${red}Unknown integration suite: ${suite}${nc}"+ exit_code=1+ ;;+ esac++ return "$exit_code"+}++if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]; then+ echo "${red}OPENSEARCH_PASSWORD is required${nc}"+ exit 1+fi++echo "${yellow}Installing test dependencies...${nc}"+uv sync --quiet --group dev++max_attempts=2+test_result=0++for attempt in $(seq 1 "$max_attempts"); do+ echo "${yellow}=== Starting Integration Suite '${suite}' - Attempt $attempt of $max_attempts ===${nc}"++ if run_attempt; then+ echo "${green}Attempt $attempt succeeded!${nc}"+ test_result=0break+ else+ echo "${red}Attempt $attempt failed!${nc}"+ test_result=1+ dump_logs || true++ if [ "$attempt" -lt "$max_attempts" ]; then+ if ! clean_infra; then+ echo "${red}ERROR: Cleanup failed. Aborting further attempts.${nc}"+ test_result=1+ break+ fi+ echo "${yellow}Retrying in 5 seconds...${nc}"+ sleep 5+ fifi- if [[ "$i" -eq 30 ]]; then- echo "${red}OIDC authenticator NOT found or unreachable in time!${nc}"- echo "Security config output: $authc_config"- exit 1- fi- sleep 2done-wait_for_url "Langflow" "http://localhost:${LANGFLOW_PORT}/" 60-wait_for_url "docling-serve at ${docling_endpoint}" "${docling_endpoint}/health" 60-echo "::endgroup::"--mkdir -p service-logs--case "$suite" in- core)- echo "::group::Core Integration Tests"- echo "${cyan}════════════════════════════════════════${nc}"- echo "${purple} Core Integration Tests${nc}"- echo "${cyan}════════════════════════════════════════${nc}"- mkdir -p service-logs- LOG_LEVEL="${LOG_LEVEL:-DEBUG}" \- GOOGLE_OAUTH_CLIENT_ID="" \- GOOGLE_OAUTH_CLIENT_SECRET="" \- OPENSEARCH_HOST=localhost OPENSEARCH_PORT=${OPENSEARCH_PORT} \- OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" \- DISABLE_STARTUP_INGEST="${DISABLE_STARTUP_INGEST:-true}" \- uv run pytest tests/integration/core -vv -s --log-file=service-logs/pytest-core.log --log-file-level=DEBUG --junitxml=service-logs/junit-core.xml || test_result=1- echo "::endgroup::"- test_jwt_opensearch || test_result=1- ;;- sdk-python)- wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60- echo "::group::SDK Integration Tests (Python)"- echo "${cyan}════════════════════════════════════════${nc}"- echo "${purple} SDK Integration Tests (Python)${nc}"- echo "${cyan}════════════════════════════════════════${nc}"- uv pip install --quiet -e sdks/python- SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s --log-file=service-logs/pytest-sdk.log --log-file-level=DEBUG --junitxml=service-logs/junit-sdk-python.xml || test_result=1- echo "::endgroup::"- ;;- sdk-typescript)- wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60- echo "::group::SDK Integration Tests (TypeScript)"- echo "${cyan}════════════════════════════════════════${nc}"- echo "${purple} SDK Integration Tests (TypeScript)${nc}"- echo "${cyan}════════════════════════════════════════${nc}"- cd sdks/typescript- npm install && npm run build && OPENRAG_URL=http://localhost:3000 npm test -- --reporter=junit --outputFile=../../service-logs/junit-sdk-typescript.xml || test_result=1- cd ../..- echo "::endgroup::"- ;;- *)- echo "${red}Unknown integration suite: ${suite}${nc}"- test_result=1- ;;-esac+generate_report || true+exit "$test_result"