Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: speed up CI image builds with BuildKit layer caching and a shared cross-PR cache (#2106)

Wallgau57b1fe76d4fc
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index d01250457..86ad9fa05 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -2,6 +2,10 @@ name: CI Tests
on:
pull_request:
+ # Push to main only warms the shared layer cache (build-images); the test/e2e
+ # jobs are gated off `push` below so a merge doesn't run the full suite.
+ push:
+ branches: [main]
workflow_dispatch:
inputs:
use_local_images:
@@ -73,7 +77,7 @@ jobs:
build-images:
needs: changes
- if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
+ if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' }}
runs-on:
labels: "ubuntu-latest"
permissions:
@@ -104,132 +108,93 @@ jobs:
run: |
echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
- - name: Generate Image Cache Key
- id: cache-key
+ - name: Determine build parameters
+ id: params
run: |
if [ "${{ matrix.image }}" = "opensearch" ]; then
- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)
- elif [ "${{ matrix.image }}" = "backend" ]; then
- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)
- elif [ "${{ matrix.image }}" = "frontend" ]; then
- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)
- elif [ "${{ matrix.image }}" = "langflow" ]; then
- HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)
+ echo "dockerfile=Dockerfile" >> "$GITHUB_OUTPUT"
+ else
+ echo "dockerfile=Dockerfile.${{ matrix.image }}" >> "$GITHUB_OUTPUT"
fi
- echo "key=$HASH" >> $GITHUB_OUTPUT
-
- - name: Cache Image
- uses: actions/cache@v4
- with:
- path: /tmp/image-cache/${{ matrix.image }}.tar
- key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.image }}-${{ steps.cache-key.outputs.key }}
-
- - name: Build and Push OpenRAG image
- run: |
- USE_LOCAL="${{ inputs.use_local_images }}"
- IS_FORK="${{ env.IS_FORK }}"
-
- if [ "$IS_FORK" = "true" ]; then
- REGISTRY_IMAGE="langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}"
+ if [ "${{ env.IS_FORK }}" = "true" ]; then
+ echo "image=langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
else
- REGISTRY_IMAGE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}"
+ echo "image=ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
-
- mkdir -p /tmp/image-cache
-
- build_or_load_image() {
- local name=$1
- local dockerfile=$2
- local tar_path="/tmp/image-cache/${name}.tar"
- local use_cache=false
-
- if [ -f "$tar_path" ]; then
- echo "Found cache file for ${name}. Inspecting architecture..."
- docker load -i "$tar_path"
-
- # Get host normalized architecture
- local host_arch="$(uname -m)"
- local host_arch_norm=""
- if [[ "$host_arch" == "x86_64" ]]; then
- host_arch_norm="amd64"
- elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then
- host_arch_norm="arm64"
- else
- host_arch_norm="$host_arch"
- fi
-
- local loaded_arch=""
- loaded_arch=$(docker inspect --format '{{.Architecture}}' "langflowai/openrag-${name}:cached" 2>/dev/null || echo "unknown")
-
- if [ "$loaded_arch" = "$host_arch_norm" ]; then
- echo "Cache hit for ${name}. Loaded image matches host architecture ($host_arch_norm)."
- use_cache=true
- echo "Tagging loaded image as ${REGISTRY_IMAGE}..."
- docker tag "langflowai/openrag-${name}:cached" "${REGISTRY_IMAGE}"
- else
- echo "Cache image architecture ($loaded_arch) does not match host architecture ($host_arch_norm). Discarding cache."
- docker rmi "langflowai/openrag-${name}:cached" 2>/dev/null || true
- rm -f "$tar_path"
- fi
- fi
-
- if [ "$use_cache" = "false" ]; then
- echo "Cache miss/incompatible for ${name}. Building..."
- if docker buildx build \
- --load \
- -t "langflowai/openrag-${name}:cached" \
- -t "${REGISTRY_IMAGE}" \
- -f "$dockerfile" .; then
- echo "Saving built image to ${tar_path}..."
- docker save "langflowai/openrag-${name}:cached" -o "$tar_path"
- else
- echo "Build failed for ${name}."
- return 1
- fi
- fi
- }
-
- pull_image() {
- local name=$1
- echo "Pulling latest image for ${name} from DockerHub..."
- if docker pull "langflowai/openrag-${name}:latest"; then
- docker tag "langflowai/openrag-${name}:latest" "${REGISTRY_IMAGE}"
- else
- echo "Pull failed for ${name}."
- return 1
- fi
- }
-
- if [ "${{ matrix.image }}" = "opensearch" ]; then
- DOCKERFILE="Dockerfile"
+ # Build locally for PRs, local dispatch, and the OpenSearch image
+ # (which has no DockerHub publish); otherwise reuse the prebuilt
+ # DockerHub image.
+ if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "push" ] || [ "${{ inputs.use_local_images }}" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then
+ echo "mode=build" >> "$GITHUB_OUTPUT"
else
- DOCKERFILE="Dockerfile.${{ matrix.image }}"
+ echo "mode=pull" >> "$GITHUB_OUTPUT"
+ fi
+ # Shared, non-branch-scoped registry cache. `main` (push) writes it;
+ # every non-fork PR reads it, so a PR that doesn't touch this image
+ # gets its layers for free on the first run. PRs also keep a per-PR
+ # gha cache so a second commit in the same PR reuses its own layers.
+ REG_CACHE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:buildcache"
+ {
+ echo "cache_from<<EOF"
+ echo "type=registry,ref=${REG_CACHE}"
+ echo "type=gha,scope=${{ matrix.image }}"
+ echo "EOF"
+ } >> "$GITHUB_OUTPUT"
+ if [ "${{ github.event_name }}" = "push" ]; then
+ echo "cache_to=type=registry,ref=${REG_CACHE},mode=max" >> "$GITHUB_OUTPUT"
+ else
+ echo "cache_to=type=gha,mode=max,scope=${{ matrix.image }}" >> "$GITHUB_OUTPUT"
fi
+ # Build with BuildKit layer caching (cache_from/cache_to computed above)
+ # so unchanged layers -- e.g. Langflow's torch/pip-install layer or the
+ # backend uv-sync layer -- are reused instead of rebuilt. Reads the shared
+ # registry cache seeded by main plus this PR's own gha cache; a flows-only
+ # change no longer forces a full re-install because that layer's cache key
+ # doesn't depend on flows/. Non-fork builds push straight to GHCR (no local
+ # load / docker save / tar shuttle).
+ - name: Build and push image (GHCR)
+ if: steps.params.outputs.mode == 'build' && env.IS_FORK != 'true'
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: ${{ steps.params.outputs.dockerfile }}
+ push: true
+ tags: ${{ steps.params.outputs.image }}
+ cache-from: ${{ steps.params.outputs.cache_from }}
+ cache-to: ${{ steps.params.outputs.cache_to }}
+ provenance: false
+
+ # Fork PRs cannot push to GHCR or write the Actions cache, so build and
+ # load locally (still reading any available layer cache). The fork
+ # fallback steps below save and upload the image as an artifact.
+ - name: Build image (fork fallback)
+ if: steps.params.outputs.mode == 'build' && env.IS_FORK == 'true'
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: ${{ steps.params.outputs.dockerfile }}
+ load: true
+ tags: ${{ steps.params.outputs.image }}
+ cache-from: type=gha,scope=${{ matrix.image }}
+ provenance: false
+
+ - name: Pull prebuilt image from DockerHub
+ if: steps.params.outputs.mode == 'pull'
+ run: |
for i in 1 2 3; do
- echo "Build/pull attempt $i..."
- if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then
- if build_or_load_image "${{ matrix.image }}" "$DOCKERFILE"; then
- if [ "$IS_FORK" != "true" ]; then
- docker push "${REGISTRY_IMAGE}"
- fi
- break
- fi
- else
- if pull_image "${{ matrix.image }}"; then
- if [ "$IS_FORK" != "true" ]; then
- docker push "${REGISTRY_IMAGE}"
- fi
- break
+ if docker pull "langflowai/openrag-${{ matrix.image }}:latest"; then
+ docker tag "langflowai/openrag-${{ matrix.image }}:latest" "${{ steps.params.outputs.image }}"
+ if [ "${{ env.IS_FORK }}" != "true" ]; then
+ docker push "${{ steps.params.outputs.image }}"
fi
+ exit 0
fi
- echo "Attempt $i failed. Retrying in 10 seconds..."
+ echo "Pull attempt $i failed. Retrying in 10s..."
sleep 10
- if [ $i -eq 3 ]; then
- echo "Build/pull failed after 3 attempts."
- exit 1
- fi
done
+ echo "Pull failed after 3 attempts."
+ exit 1
- name: Save OpenRAG image (Fork fallback)
if: env.IS_FORK == 'true'
@@ -260,7 +225,7 @@ jobs:
test-suite:
needs: [changes, build-images]
- if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
+ if: ${{ github.event_name != 'push' && (needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch') }}
runs-on:
labels: "ubuntu-latest"
permissions:
@@ -370,7 +335,7 @@ jobs:
e2e-run:
needs: [changes, build-images]
- if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}
+ if: ${{ github.event_name != 'push' && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') }}
runs-on:
labels: "ubuntu-latest"
permissions:
@@ -540,7 +505,7 @@ jobs:
done || true
e2e-merge-reports:
- if: ${{ !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}
+ if: ${{ github.event_name != 'push' && !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}
needs: [changes, e2e-run]
runs-on:
labels: "ubuntu-latest"
@@ -581,7 +546,7 @@ jobs:
tests:
needs: [changes, build-images, test-suite]
- if: always()
+ if: always() && github.event_name != 'push'
runs-on:
labels: "ubuntu-latest"
permissions:
@@ -603,7 +568,7 @@ jobs:
e2e:
needs: [changes, build-images, e2e-run]
- if: always()
+ if: always() && github.event_name != 'push'
runs-on:
labels: "ubuntu-latest"
permissions:
diff --git a/Makefile b/Makefile
index 992dd7567..8cd6a7547 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,7 @@ ENV_FILE ?= .env
# Load variables from $(ENV_FILE) if present so `make` commands pick them up
# Strip quotes from values to avoid issues with tools that don't handle them like python-dotenv does
+
ifneq (,$(wildcard $(ENV_FILE)))
include $(ENV_FILE)
export $(shell sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' $(ENV_FILE))
diff --git a/frontend/components/chat-renderer.tsx b/frontend/components/chat-renderer.tsx
index 885516d2d..1e641943a 100644
--- a/frontend/components/chat-renderer.tsx
+++ b/frontend/components/chat-renderer.tsx
@@ -53,6 +53,8 @@ export function ChatRenderer({
setOnboardingComplete,
} = useChat();
+ //TODO: comment to test CI
+
// Initialize onboarding state from backend settings
const [currentStep, setCurrentStep] = useState<number>(
settings?.onboarding?.current_step ?? 0,