Skip to content

High performance self-hosted photo and video management solution.

ci: machine learning separate native docker image builds (#16102)

Zack Pollard425c87bce4f3
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b60e02e806..c31a9a5257 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -95,44 +95,56 @@ jobs:
name: Build and Push ML
needs: pre-job
if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }}
- runs-on: ubuntu-latest
+ runs-on: ${{ matrix.runner }}
env:
image: immich-machine-learning
context: machine-learning
file: machine-learning/Dockerfile
+ GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning
+ DOCKER_REPO: altran1502/immich-machine-learning
strategy:
# Prevent a failure in one image from stopping the other builds
fail-fast: false
matrix:
include:
- - platforms: linux/amd64,linux/arm64
+ - platform: linux/amd64
+ runner: ubuntu-latest
+ device: cpu
+
+ - platform: linux/arm64
+ runner: ubuntu-24.04-arm
device: cpu
- - platforms: linux/amd64
+ - platform: linux/amd64
+ runner: ubuntu-latest
device: cuda
suffix: -cuda
- - platforms: linux/amd64
+ - platform: linux/amd64
+ runner: ubuntu-latest
device: openvino
suffix: -openvino
- - platforms: linux/arm64
+ - platform: linux/arm64
+ runner: ubuntu-24.04-arm
device: armnn
suffix: -armnn
steps:
+ - name: Prepare
+ run: |
+ platform=${{ matrix.platform }}
+ echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
- name: Checkout
uses: actions/checkout@v4
- - name: Set up QEMU
- uses: docker/[email protected]
-
- name: Set up Docker Buildx
uses: docker/[email protected]
- name: Login to Docker Hub
# Only push to Docker Hub when making a release
- if: ${{ github.event_name == 'release' }}
+ if: ${{ !github.event.pull_request.head.repo.fork }}
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -147,16 +159,92 @@ jobs:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Build and push image
+ id: build
+ uses: docker/[email protected]
+ with:
+ context: ${{ env.context }}
+ file: ${{ env.file }}
+ platforms: ${{ matrix.platforms }}
+ # Skip pushing when PR from a fork
+ push: ${{ !github.event.pull_request.head.repo.fork }}
+ labels: ${{ steps.metadata.outputs.labels }}
+ outputs: type=image,"name=${{ env.GHCR_REPO }},${{ env.DOCKER_REPO }}",push-by-digest=true,name-canonical=true,push=true
+ build-args: |
+ DEVICE=${{ matrix.device }}
+ BUILD_ID=${{ github.run_id }}
+ BUILD_IMAGE=${{ github.event_name == 'release' && github.ref_name || steps.metadata.outputs.tags }}
+ BUILD_SOURCE_REF=${{ github.ref_name }}
+ BUILD_SOURCE_COMMIT=${{ github.sha }}
+
+ - name: Export digest
+ run: |
+ mkdir -p ${{ runner.temp }}/digests
+ digest="${{ steps.build.outputs.digest }}"
+ touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+ - name: Upload digest
+ uses: actions/upload-artifact@v4
+ with:
+ name: ml-digests-${{ matrix.device }}-${{ env.PLATFORM_PAIR }}
+ path: ${{ runner.temp }}/digests/*
+ if-no-files-found: error
+ retention-days: 1
+
+ merge_ml:
+ name: Merge & Push ML
+ runs-on: ubuntu-latest
+ if: ${{ needs.pre-job.outputs.should_run_ml == 'true' && !github.event.pull_request.head.repo.fork }}
+ env:
+ GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning
+ DOCKER_REPO: altran1502/immich-machine-learning
+ strategy:
+ matrix:
+ include:
+ - device: cpu
+ - device: cuda
+ suffix: -cuda
+ - device: openvino
+ suffix: -openvino
+ - device: armnn
+ suffix: -armnn
+ needs:
+ - build_and_push_ml
+ steps:
+ - name: Download digests
+ uses: actions/download-artifact@v4
+ with:
+ path: ${{ runner.temp }}/digests
+ pattern: ml-digests-${{ matrix.device }}-*
+ merge-multiple: true
+
+ - name: Login to Docker Hub
+ if: ${{ github.event_name == 'release' }}
+ uses: docker/login-action@v3
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to GHCR
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{ github.repository_owner }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
- name: Generate docker image tags
- id: metadata
+ id: meta
uses: docker/metadata-action@v5
with:
flavor: |
# Disable latest tag
latest=false
images: |
- name=ghcr.io/${{ github.repository_owner }}/${{env.image}}
- name=altran1502/${{env.image}},enable=${{ github.event_name == 'release' }}
+ name=${{ env.GHCR_REPO }}
+ name=${{ env.DOCKER_REPO }},enable=${{ github.event_name == 'release' }}
tags: |
# Tag with branch name
type=ref,event=branch,suffix=${{ matrix.suffix }}
@@ -168,35 +256,12 @@ jobs:
type=ref,event=tag,suffix=${{ matrix.suffix }}
type=raw,value=release,enable=${{ github.event_name == 'release' }},suffix=${{ matrix.suffix }}
- - name: Determine build cache output
- id: cache-target
+ - name: Create manifest list and push
+ working-directory: ${{ runner.temp }}/digests
run: |
- if [[ "${{ github.event_name }}" == "pull_request" ]]; then
- # Essentially just ignore the cache output (PR can't write to registry cache)
- echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT
- else
- echo "cache-to=type=registry,mode=max,ref=ghcr.io/${{ github.repository_owner }}/immich-build-cache:${{ env.image }}" >> $GITHUB_OUTPUT
- fi
-
- - name: Build and push image
- uses: docker/[email protected]
- with:
- context: ${{ env.context }}
- file: ${{ env.file }}
- platforms: ${{ matrix.platforms }}
- # Skip pushing when PR from a fork
- push: ${{ !github.event.pull_request.head.repo.fork }}
- cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/immich-build-cache:${{env.image}}
- cache-to: ${{ steps.cache-target.outputs.cache-to }}
- tags: ${{ steps.metadata.outputs.tags }}
- labels: ${{ steps.metadata.outputs.labels }}
- build-args: |
- DEVICE=${{ matrix.device }}
- BUILD_ID=${{ github.run_id }}
- BUILD_IMAGE=${{ github.event_name == 'release' && github.ref_name || steps.metadata.outputs.tags }}
- BUILD_SOURCE_REF=${{ github.ref_name }}
- BUILD_SOURCE_COMMIT=${{ github.sha }}
-
+ docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+ $(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) \
+ $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
build_and_push_server:
name: Build and Push Server
@@ -272,7 +337,7 @@ jobs:
- name: Upload digest
uses: actions/upload-artifact@v4
with:
- name: digests-${{ env.PLATFORM_PAIR }}
+ name: server-digests-${{ env.PLATFORM_PAIR }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
@@ -291,10 +356,11 @@ jobs:
uses: actions/download-artifact@v4
with:
path: ${{ runner.temp }}/digests
- pattern: digests-*
+ pattern: server-digests-*
merge-multiple: true
- name: Login to Docker Hub
+ if: ${{ github.event_name == 'release' }}
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -353,7 +419,7 @@ jobs:
success-check-ml:
name: Docker Build & Push ML Success
- needs: [build_and_push_ml, retag_ml]
+ needs: [merge_ml, retag_ml]
runs-on: ubuntu-latest
if: always()
steps: