OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: speed up CI image builds with BuildKit layer caching and a shared cross-PR cache (#2106)
Wallgau57b1fe76d4fc
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex d01250457..86ad9fa05 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -2,6 +2,10 @@ name: CI Testson:pull_request:+ # Push to main only warms the shared layer cache (build-images); the test/e2e+ # jobs are gated off `push` below so a merge doesn't run the full suite.+ push:+ branches: [main]workflow_dispatch:inputs:use_local_images:@@ -73,7 +77,7 @@ jobs:build-images:needs: changes- if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}+ if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' }}runs-on:labels: "ubuntu-latest"permissions:@@ -104,132 +108,93 @@ jobs:run: |echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT- - name: Generate Image Cache Key- id: cache-key+ - name: Determine build parameters+ id: paramsrun: |if [ "${{ matrix.image }}" = "opensearch" ]; then- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)- elif [ "${{ matrix.image }}" = "backend" ]; then- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)- elif [ "${{ matrix.image }}" = "frontend" ]; then- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)- elif [ "${{ matrix.image }}" = "langflow" ]; then- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)+ echo "dockerfile=Dockerfile" >> "$GITHUB_OUTPUT"+ else+ echo "dockerfile=Dockerfile.${{ matrix.image }}" >> "$GITHUB_OUTPUT"fi- echo "key=$HASH" >> $GITHUB_OUTPUT-- - name: Cache Image- uses: actions/cache@v4- with:- path: /tmp/image-cache/${{ matrix.image }}.tar- key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.image }}-${{ steps.cache-key.outputs.key }}-- - name: Build and Push OpenRAG image- run: |- USE_LOCAL="${{ inputs.use_local_images }}"- IS_FORK="${{ env.IS_FORK }}"-- if [ "$IS_FORK" = "true" ]; then- REGISTRY_IMAGE="langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}"+ if [ "${{ env.IS_FORK }}" = "true" ]; then+ echo "image=langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}" >> "$GITHUB_OUTPUT"else- REGISTRY_IMAGE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}"+ echo "image=ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}" >> "$GITHUB_OUTPUT"fi-- mkdir -p /tmp/image-cache-- build_or_load_image() {- local name=$1- local dockerfile=$2- local tar_path="/tmp/image-cache/${name}.tar"- local use_cache=false-- if [ -f "$tar_path" ]; then- echo "Found cache file for ${name}. Inspecting architecture..."- docker load -i "$tar_path"-- # Get host normalized architecture- local host_arch="$(uname -m)"- local host_arch_norm=""- if [[ "$host_arch" == "x86_64" ]]; then- host_arch_norm="amd64"- elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then- host_arch_norm="arm64"- else- host_arch_norm="$host_arch"- fi-- local loaded_arch=""- loaded_arch=$(docker inspect --format '{{.Architecture}}' "langflowai/openrag-${name}:cached" 2>/dev/null || echo "unknown")-- if [ "$loaded_arch" = "$host_arch_norm" ]; then- echo "Cache hit for ${name}. Loaded image matches host architecture ($host_arch_norm)."- use_cache=true- echo "Tagging loaded image as ${REGISTRY_IMAGE}..."- docker tag "langflowai/openrag-${name}:cached" "${REGISTRY_IMAGE}"- else- echo "Cache image architecture ($loaded_arch) does not match host architecture ($host_arch_norm). Discarding cache."- docker rmi "langflowai/openrag-${name}:cached" 2>/dev/null || true- rm -f "$tar_path"- fi- fi-- if [ "$use_cache" = "false" ]; then- echo "Cache miss/incompatible for ${name}. Building..."- if docker buildx build \- --load \- -t "langflowai/openrag-${name}:cached" \- -t "${REGISTRY_IMAGE}" \- -f "$dockerfile" .; then- echo "Saving built image to ${tar_path}..."- docker save "langflowai/openrag-${name}:cached" -o "$tar_path"- else- echo "Build failed for ${name}."- return 1- fi- fi- }-- pull_image() {- local name=$1- echo "Pulling latest image for ${name} from DockerHub..."- if docker pull "langflowai/openrag-${name}:latest"; then- docker tag "langflowai/openrag-${name}:latest" "${REGISTRY_IMAGE}"- else- echo "Pull failed for ${name}."- return 1- fi- }-- if [ "${{ matrix.image }}" = "opensearch" ]; then- DOCKERFILE="Dockerfile"+ # Build locally for PRs, local dispatch, and the OpenSearch image+ # (which has no DockerHub publish); otherwise reuse the prebuilt+ # DockerHub image.+ if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "push" ] || [ "${{ inputs.use_local_images }}" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then+ echo "mode=build" >> "$GITHUB_OUTPUT"else- DOCKERFILE="Dockerfile.${{ matrix.image }}"+ echo "mode=pull" >> "$GITHUB_OUTPUT"+ fi+ # Shared, non-branch-scoped registry cache. `main` (push) writes it;+ # every non-fork PR reads it, so a PR that doesn't touch this image+ # gets its layers for free on the first run. PRs also keep a per-PR+ # gha cache so a second commit in the same PR reuses its own layers.+ REG_CACHE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:buildcache"+ {+ echo "cache_from<<EOF"+ echo "type=registry,ref=${REG_CACHE}"+ echo "type=gha,scope=${{ matrix.image }}"+ echo "EOF"+ } >> "$GITHUB_OUTPUT"+ if [ "${{ github.event_name }}" = "push" ]; then+ echo "cache_to=type=registry,ref=${REG_CACHE},mode=max" >> "$GITHUB_OUTPUT"+ else+ echo "cache_to=type=gha,mode=max,scope=${{ matrix.image }}" >> "$GITHUB_OUTPUT"fi+ # Build with BuildKit layer caching (cache_from/cache_to computed above)+ # so unchanged layers -- e.g. Langflow's torch/pip-install layer or the+ # backend uv-sync layer -- are reused instead of rebuilt. Reads the shared+ # registry cache seeded by main plus this PR's own gha cache; a flows-only+ # change no longer forces a full re-install because that layer's cache key+ # doesn't depend on flows/. Non-fork builds push straight to GHCR (no local+ # load / docker save / tar shuttle).+ - name: Build and push image (GHCR)+ if: steps.params.outputs.mode == 'build' && env.IS_FORK != 'true'+ uses: docker/build-push-action@v6+ with:+ context: .+ file: ${{ steps.params.outputs.dockerfile }}+ push: true+ tags: ${{ steps.params.outputs.image }}+ cache-from: ${{ steps.params.outputs.cache_from }}+ cache-to: ${{ steps.params.outputs.cache_to }}+ provenance: false++ # Fork PRs cannot push to GHCR or write the Actions cache, so build and+ # load locally (still reading any available layer cache). The fork+ # fallback steps below save and upload the image as an artifact.+ - name: Build image (fork fallback)+ if: steps.params.outputs.mode == 'build' && env.IS_FORK == 'true'+ uses: docker/build-push-action@v6+ with:+ context: .+ file: ${{ steps.params.outputs.dockerfile }}+ load: true+ tags: ${{ steps.params.outputs.image }}+ cache-from: type=gha,scope=${{ matrix.image }}+ provenance: false++ - name: Pull prebuilt image from DockerHub+ if: steps.params.outputs.mode == 'pull'+ run: |for i in 1 2 3; do- echo "Build/pull attempt $i..."- if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then- if build_or_load_image "${{ matrix.image }}" "$DOCKERFILE"; then- if [ "$IS_FORK" != "true" ]; then- docker push "${REGISTRY_IMAGE}"- fi- break- fi- else- if pull_image "${{ matrix.image }}"; then- if [ "$IS_FORK" != "true" ]; then- docker push "${REGISTRY_IMAGE}"- fi- break+ if docker pull "langflowai/openrag-${{ matrix.image }}:latest"; then+ docker tag "langflowai/openrag-${{ matrix.image }}:latest" "${{ steps.params.outputs.image }}"+ if [ "${{ env.IS_FORK }}" != "true" ]; then+ docker push "${{ steps.params.outputs.image }}"fi+ exit 0fi- echo "Attempt $i failed. Retrying in 10 seconds..."+ echo "Pull attempt $i failed. Retrying in 10s..."sleep 10- if [ $i -eq 3 ]; then- echo "Build/pull failed after 3 attempts."- exit 1- fidone+ echo "Pull failed after 3 attempts."+ exit 1- name: Save OpenRAG image (Fork fallback)if: env.IS_FORK == 'true'@@ -260,7 +225,7 @@ jobs:test-suite:needs: [changes, build-images]- if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}+ if: ${{ github.event_name != 'push' && (needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch') }}runs-on:labels: "ubuntu-latest"permissions:@@ -370,7 +335,7 @@ jobs:e2e-run:needs: [changes, build-images]- if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}+ if: ${{ github.event_name != 'push' && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') }}runs-on:labels: "ubuntu-latest"permissions:@@ -540,7 +505,7 @@ jobs:done || truee2e-merge-reports:- if: ${{ !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}+ if: ${{ github.event_name != 'push' && !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}needs: [changes, e2e-run]runs-on:labels: "ubuntu-latest"@@ -581,7 +546,7 @@ jobs:tests:needs: [changes, build-images, test-suite]- if: always()+ if: always() && github.event_name != 'push'runs-on:labels: "ubuntu-latest"permissions:@@ -603,7 +568,7 @@ jobs:e2e:needs: [changes, build-images, e2e-run]- if: always()+ if: always() && github.event_name != 'push'runs-on:labels: "ubuntu-latest"permissions:diff --git a/Makefile b/Makefileindex 992dd7567..8cd6a7547 100644--- a/Makefile+++ b/Makefile@@ -3,6 +3,7 @@ ENV_FILE ?= .env# Load variables from $(ENV_FILE) if present so `make` commands pick them up# Strip quotes from values to avoid issues with tools that don't handle them like python-dotenv does+ifneq (,$(wildcard $(ENV_FILE)))include $(ENV_FILE)export $(shell sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' $(ENV_FILE))diff --git a/frontend/components/chat-renderer.tsx b/frontend/components/chat-renderer.tsxindex 885516d2d..1e641943a 100644--- a/frontend/components/chat-renderer.tsx+++ b/frontend/components/chat-renderer.tsx@@ -53,6 +53,8 @@ export function ChatRenderer({setOnboardingComplete,} = useChat();+ //TODO: comment to test CI+// Initialize onboarding state from backend settingsconst [currentStep, setCurrentStep] = useState<number>(settings?.onboarding?.current_step ?? 0,