Skip to content

High performance self-hosted photo and video management solution.

fix: isolate docker host/container filesystem for node_modules and build output (#21167)

Min Idzelis44e1c83c84e3
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 79126fd658..9563be6125 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -5,7 +5,8 @@
"immich-server",
"redis",
"database",
- "immich-machine-learning"
+ "immich-machine-learning",
+ "init"
],
"dockerComposeFile": [
"../docker/docker-compose.dev.yml",
diff --git a/.devcontainer/server/container-compose-overrides.yml b/.devcontainer/server/container-compose-overrides.yml
index 869b6c2624..59975baad3 100644
--- a/.devcontainer/server/container-compose-overrides.yml
+++ b/.devcontainer/server/container-compose-overrides.yml
@@ -11,6 +11,18 @@ services:
- ${UPLOAD_LOCATION:-upload1-devcontainer-volume}${UPLOAD_LOCATION:+/photos}:/data
- ${UPLOAD_LOCATION:-upload2-devcontainer-volume}${UPLOAD_LOCATION:+/photos/upload}:/data/upload
- /etc/localtime:/etc/localtime:ro
+ - pnpm-store:/usr/src/app/.pnpm-store
+ - server-node-modules:/usr/src/app/server/node_modules
+ - server-dist:/usr/src/app/server/dist
+ - web-node_modules:/usr/src/app/web/node_modules
+ - github-node_modules:/usr/src/app/.github/node_modules
+ - cli-node_modules:/usr/src/app/cli/node_modules
+ - docs-node_modules:/usr/src/app/docs/node_modules
+ - e2e-node_modules:/usr/src/app/e2e/node_modules
+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
+ - app-node_modules:/usr/src/app/node_modules
+ - sveltekit:/usr/src/app/web/.svelte-kit
+ - coverage:/usr/src/app/web/coverage
immich-web:
env_file: !reset []
immich-machine-learning:
diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs
index feda217821..0e76dabe66 100644
--- a/.pnpmfile.cjs
+++ b/.pnpmfile.cjs
@@ -4,34 +4,13 @@ module.exports = {
if (!pkg.name) {
return pkg;
}
- switch (pkg.name) {
- case "exiftool-vendored":
- if (pkg.optionalDependencies["exiftool-vendored.pl"]) {
- // make exiftool-vendored.pl a regular dependency
- pkg.dependencies["exiftool-vendored.pl"] =
- pkg.optionalDependencies["exiftool-vendored.pl"];
- delete pkg.optionalDependencies["exiftool-vendored.pl"];
- }
- break;
- case "sharp":
- const optionalDeps = Object.keys(pkg.optionalDependencies).filter(
- (dep) => dep.startsWith("@img")
- );
- for (const dep of optionalDeps) {
- // remove all optionalDependencies from sharp (they will be compiled from source), except:
- // include the precompiled musl version of sharp, for web
- // include precompiled linux-x64 version of sharp, for server (stage: web-prod)
- // include precompiled linux-arm64 version of sharp, for server (stage: web-prod)
- if (
- dep.includes("musl") ||
- dep.includes("linux-x64") ||
- dep.includes("linux-arm64")
- ) {
- continue;
- }
- delete pkg.optionalDependencies[dep];
- }
- break;
+ if (pkg.name === "exiftool-vendored") {
+ if (pkg.optionalDependencies["exiftool-vendored.pl"]) {
+ // make exiftool-vendored.pl a regular dependency
+ pkg.dependencies["exiftool-vendored.pl"] =
+ pkg.optionalDependencies["exiftool-vendored.pl"];
+ delete pkg.optionalDependencies["exiftool-vendored.pl"];
+ }
}
return pkg;
},
diff --git a/Makefile b/Makefile
index a9faceadf4..31a00ee6be 100644
--- a/Makefile
+++ b/Makefile
@@ -1,29 +1,29 @@
-dev:
+dev: prepare-volumes
@trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --remove-orphans
dev-down:
docker compose -f ./docker/docker-compose.dev.yml down --remove-orphans
-dev-update:
+dev-update: prepare-volumes
@trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --build -V --remove-orphans
-dev-scale:
+dev-scale: prepare-volumes
@trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --build -V --scale immich-server=3 --remove-orphans
-dev-docs:
+dev-docs: prepare-volumes
npm --prefix docs run start
.PHONY: e2e
-e2e:
- @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --build -V --remove-orphans
+e2e: prepare-volumes
+ @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --remove-orphans
-e2e-update:
+e2e-update: prepare-volumes
@trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --build -V --remove-orphans
e2e-down:
docker compose -f ./e2e/docker-compose.yml down --remove-orphans
-prod:
+prod:
@trap 'make prod-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.prod.yml up --build -V --remove-orphans
prod-down:
@@ -33,16 +33,16 @@ prod-scale:
@trap 'make prod-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.prod.yml up --build -V --scale immich-server=3 --scale immich-microservices=3 --remove-orphans
.PHONY: open-api
-open-api:
+open-api: prepare-volumes
cd ./open-api && bash ./bin/generate-open-api.sh
-open-api-dart:
+open-api-dart: prepare-volumes
cd ./open-api && bash ./bin/generate-open-api.sh dart
-open-api-typescript:
+open-api-typescript: prepare-volumes
cd ./open-api && bash ./bin/generate-open-api.sh typescript
-sql:
+sql: prepare-volumes
pnpm --filter immich run sync:sql
attach-server:
@@ -51,6 +51,30 @@ attach-server:
renovate:
LOG_LEVEL=debug npx renovate --platform=local --repository-cache=reset
+# Directories that need to be created for volumes or build output
+VOLUME_DIRS = \
+ ./.pnpm-store \
+ ./web/.svelte-kit \
+ ./web/node_modules \
+ ./web/coverage \
+ ./e2e/node_modules \
+ ./docs/node_modules \
+ ./server/node_modules \
+ ./server/dist \
+ ./open-api/typescript-sdk/node_modules \
+ ./.github/node_modules \
+ ./node_modules \
+ ./cli/node_modules
+
+# create empty directories and chown to current user
+prepare-volumes:
+ @for dir in $(VOLUME_DIRS); do \
+ mkdir -p $$dir; \
+ done
+ @if [ -n "$(VOLUME_DIRS)" ]; then \
+ chown -R $$(id -u):$$(id -g) $(VOLUME_DIRS); \
+ fi
+
MODULES = e2e server web cli sdk docs .github
# directory to package name mapping function
diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
index c6393fd132..fe621c768f 100644
--- a/docker/docker-compose.dev.yml
+++ b/docker/docker-compose.dev.yml
@@ -32,6 +32,18 @@ services:
- ${UPLOAD_LOCATION}/photos:/data
- ${UPLOAD_LOCATION}/photos/upload:/data/upload
- /etc/localtime:/etc/localtime:ro
+ - pnpm-store:/usr/src/app/.pnpm-store
+ - server-node-modules:/usr/src/app/server/node_modules
+ - server-dist:/usr/src/app/server/dist
+ - web-node_modules:/usr/src/app/web/node_modules
+ - github-node_modules:/usr/src/app/.github/node_modules
+ - cli-node_modules:/usr/src/app/cli/node_modules
+ - docs-node_modules:/usr/src/app/docs/node_modules
+ - e2e-node_modules:/usr/src/app/e2e/node_modules
+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
+ - app-node_modules:/usr/src/app/node_modules
+ - sveltekit:/usr/src/app/web/.svelte-kit
+ - coverage:/usr/src/app/web/coverage
env_file:
- .env
environment:
@@ -84,6 +96,18 @@ services:
- 24678:24678
volumes:
- ..:/usr/src/app
+ - pnpm-store:/usr/src/app/.pnpm-store
+ - server-node-modules:/usr/src/app/server/node_modules
+ - server-dist:/usr/src/app/server/dist
+ - web-node_modules:/usr/src/app/web/node_modules
+ - github-node_modules:/usr/src/app/.github/node_modules
+ - cli-node_modules:/usr/src/app/cli/node_modules
+ - docs-node_modules:/usr/src/app/docs/node_modules
+ - e2e-node_modules:/usr/src/app/e2e/node_modules
+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
+ - app-node_modules:/usr/src/app/node_modules
+ - sveltekit:/usr/src/app/web/.svelte-kit
+ - coverage:/usr/src/app/web/coverage
ulimits:
nofile:
soft: 1048576
@@ -167,9 +191,33 @@ services:
env_file:
- .env
user: 0:0
- command: sh -c 'for path in /usr/src/app/.pnpm-store /usr/src/app/server/node_modules /usr/src/app/.github/node_modules /usr/src/app/cli/node_modules /usr/src/app/docs/node_modules /usr/src/app/e2e/node_modules /usr/src/app/open-api/typescript-sdk/node_modules /usr/src/app/web/.svelte-kit /usr/src/app/web/coverage /usr/src/app/node_modules /usr/src/app/web/node_modules; do [ -e "$$path" ] && chown -R ${UID:-1000}:${GID:-1000} "$$path" || true; done'
-
+ command: sh -c 'for path in /usr/src/app/.pnpm-store /usr/src/app/server/node_modules /usr/src/app/server/dist /usr/src/app/.github/node_modules /usr/src/app/cli/node_modules /usr/src/app/docs/node_modules /usr/src/app/e2e/node_modules /usr/src/app/open-api/typescript-sdk/node_modules /usr/src/app/web/.svelte-kit /usr/src/app/web/coverage /usr/src/app/node_modules /usr/src/app/web/node_modules; do [ -e "$$path" ] && chown -R ${UID:-1000}:${GID:-1000} "$$path" || true; done'
+ volumes:
+ - pnpm-store:/usr/src/app/.pnpm-store
+ - server-node-modules:/usr/src/app/server/node_modules
+ - server-dist:/usr/src/app/server/dist
+ - web-node_modules:/usr/src/app/web/node_modules
+ - github-node_modules:/usr/src/app/.github/node_modules
+ - cli-node_modules:/usr/src/app/cli/node_modules
+ - docs-node_modules:/usr/src/app/docs/node_modules
+ - e2e-node_modules:/usr/src/app/e2e/node_modules
+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules
+ - app-node_modules:/usr/src/app/node_modules
+ - sveltekit:/usr/src/app/web/.svelte-kit
+ - coverage:/usr/src/app/web/coverage
volumes:
model-cache:
prometheus-data:
grafana-data:
+ pnpm-store:
+ server-node-modules:
+ server-dist:
+ web-node_modules:
+ github-node_modules:
+ cli-node_modules:
+ docs-node_modules:
+ e2e-node_modules:
+ sdk-node_modules:
+ app-node_modules:
+ sveltekit:
+ coverage:
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 103cacfdc0..fa24e5b31f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -11,7 +11,7 @@ overrides:
packageExtensionsChecksum: sha256-DAYr0FTkvKYnvBH4muAER9UE1FVGKhqfRU4/QwA2xPQ=
-pnpmfileChecksum: sha256-7GOLcTtuczNumtarIG1mbRinBOSpiOOVzgbeV3Xp4X4=
+pnpmfileChecksum: sha256-AG/qwrPNpmy9q60PZwCpecoYVptglTHgH+N6RKQHOM0=
importers:
@@ -2124,6 +2124,9 @@ packages:
resolution: {integrity: sha512-P1ml0nvOmEFdmu0smSXOqTS1sxU5tqvnc0dA4MTKV39kye+bhQnjkIKEE18fNOvxjyB86k8esoCIFM3x4RykOQ==}
engines: {node: '>=18.0'}
+ '@emnapi/[email protected]':
+ resolution: {integrity: sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==}
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==}
engines: {node: '>=12'}
@@ -2553,11 +2556,48 @@ packages:
resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==}
engines: {node: '>=18.18'}
+ resolution: {integrity: sha512-OfXHZPppddivUJnqyKoi5YVeHRkkNE2zUFT2gbpKxp/JZCFYEYubnMg+gOp6lWfasPrTS+KPosKqdI+ELYVDtg==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [arm64]
+ os: [darwin]
+
+ resolution: {integrity: sha512-dYvWqmjU9VxqXmjEtjmvHnGqF8GrVjM2Epj9rJ6BUIXvk8slvNDJbhGFvIoXzkDhrJC2jUxNLz/GUjjvSzfw+g==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [x64]
+ os: [darwin]
+
+ resolution: {integrity: sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==}
+ cpu: [arm64]
+ os: [darwin]
+
+ resolution: {integrity: sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==}
+ cpu: [x64]
+ os: [darwin]
+
resolution: {integrity: sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==}
cpu: [arm64]
os: [linux]
+ resolution: {integrity: sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==}
+ cpu: [arm]
+ os: [linux]
+
+ resolution: {integrity: sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==}
+ cpu: [ppc64]
+ os: [linux]
+
+ resolution: {integrity: sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==}
+ cpu: [s390x]
+ os: [linux]
+
resolution: {integrity: sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==}
cpu: [x64]
@@ -2579,6 +2619,18 @@ packages:
cpu: [arm64]
os: [linux]
+ resolution: {integrity: sha512-0DZzkvuEOqQUP9mo2kjjKNok5AmnOr1jB2XYjkaoNRwpAYMDzRmAqUIa1nRi58S2WswqSfPOWLNOr0FDT3H5RQ==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [arm]
+ os: [linux]
+
+ resolution: {integrity: sha512-EGZ1xwhBI7dNISwxjChqBGELCWMGDvmxZXKjQRuqMrakhO8QoMgqCrdjnAqJq/CScxfRn+Bb7suXBElKQpPDiw==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [s390x]
+ os: [linux]
+
resolution: {integrity: sha512-sD7J+h5nFLMMmOXYH4DD9UtSNBD05tWSSdWAcEyzqW8Cn5UxXvsHAxmxSesYUsTOBmUnjtxghKDl15EvfqLFbQ==}
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
@@ -2597,6 +2649,29 @@ packages:
cpu: [x64]
os: [linux]
+ resolution: {integrity: sha512-/VI4mdlJ9zkaq53MbIG6rZY+QRN3MLbR6usYlgITEzi4Rpx5S6LFKsycOQjkOGmqTNmkIdLjEvooFKwww6OpdQ==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [wasm32]
+
+ resolution: {integrity: sha512-cfP/r9FdS63VA5k0xiqaNaEoGxBg9k7uE+RQGzuK9fHt7jib4zAVVseR9LsE4gJcNWgT6APKMNnCcnyOtmSEUQ==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [arm64]
+ os: [win32]
+
+ resolution: {integrity: sha512-QLjGGvAbj0X/FXl8n1WbtQ6iVBpWU7JO94u/P2M4a8CFYsvQi4GW2mRy/JqkRx0qpBzaOdKJKw8uc930EX2AHw==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [ia32]
+ os: [win32]
+
+ resolution: {integrity: sha512-aUdT6zEYtDKCaxkofmmJDJYGCf0+pJg3eU9/oBuqvEeoB9dKI6ZLc/1iLJCTuJQDO4ptntAlkUmHgGjyuobZbw==}
+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
+ cpu: [x64]
+ os: [win32]
+
'@immich/[email protected]':
resolution: {integrity: sha512-phJ9BHV0+OnKsxXD+5+Te5Amnb1N4ExYpRGSJPYFqutd5WXeN7kZGKZXd3CfcQ1e31SXRy4DsHSGdM1pY7AUgA==}
peerDependencies:
@@ -13888,6 +13963,11 @@ snapshots:
- uglify-js
- webpack-cli
+ '@emnapi/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+ optional: true
+
'@esbuild/[email protected]':
optional: true
@@ -14184,9 +14264,34 @@ snapshots:
'@humanwhocodes/[email protected]': {}
+ optionalDependencies:
+ '@img/sharp-libvips-darwin-arm64': 1.1.0
+ optional: true
+
+ optionalDependencies:
+ '@img/sharp-libvips-darwin-x64': 1.1.0
+ optional: true
+
+ optional: true
+
+ optional: true
+
optional: true
+ optional: true
+
+ optional: true
+
+ optional: true
+
optional: true
@@ -14201,6 +14306,16 @@ snapshots:
'@img/sharp-libvips-linux-arm64': 1.1.0
optional: true
+ optionalDependencies:
+ '@img/sharp-libvips-linux-arm': 1.1.0
+ optional: true
+
+ optionalDependencies:
+ '@img/sharp-libvips-linux-s390x': 1.1.0
+ optional: true
+
optionalDependencies:
'@img/sharp-libvips-linux-x64': 1.1.0
@@ -14216,6 +14331,20 @@ snapshots:
'@img/sharp-libvips-linuxmusl-x64': 1.1.0
optional: true
+ dependencies:
+ '@emnapi/runtime': 1.4.5
+ optional: true
+
+ optional: true
+
+ optional: true
+
+ optional: true
+
dependencies:
'@mdi/js': 7.4.47
@@ -23758,14 +23887,27 @@ snapshots:
node-gyp: 11.2.0
semver: 7.7.2
optionalDependencies:
+ '@img/sharp-darwin-arm64': 0.34.2
+ '@img/sharp-darwin-x64': 0.34.2
+ '@img/sharp-libvips-darwin-arm64': 1.1.0
+ '@img/sharp-libvips-darwin-x64': 1.1.0
+ '@img/sharp-libvips-linux-arm': 1.1.0
'@img/sharp-libvips-linux-arm64': 1.1.0
+ '@img/sharp-libvips-linux-ppc64': 1.1.0
+ '@img/sharp-libvips-linux-s390x': 1.1.0
'@img/sharp-libvips-linux-x64': 1.1.0
'@img/sharp-libvips-linuxmusl-arm64': 1.1.0
'@img/sharp-libvips-linuxmusl-x64': 1.1.0
+ '@img/sharp-linux-arm': 0.34.2
'@img/sharp-linux-arm64': 0.34.2
+ '@img/sharp-linux-s390x': 0.34.2
'@img/sharp-linux-x64': 0.34.2
'@img/sharp-linuxmusl-arm64': 0.34.2
'@img/sharp-linuxmusl-x64': 0.34.2
+ '@img/sharp-wasm32': 0.34.2
+ '@img/sharp-win32-arm64': 0.34.2
+ '@img/sharp-win32-ia32': 0.34.2
+ '@img/sharp-win32-x64': 0.34.2
transitivePeerDependencies:
- supports-color
diff --git a/server/Dockerfile b/server/Dockerfile
index ae3198c5d3..bd5c55402e 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -91,9 +91,8 @@ FROM prod-builder-base AS server-prod
WORKDIR /usr/src/app
COPY ./package* ./pnpm* .pnpmfile.cjs ./
COPY ./server ./server/
-# SHARP_IGNORE_GLOBAL_LIBVIPS because 'deploy' will always build sharp bindings from source
RUN SHARP_IGNORE_GLOBAL_LIBVIPS=true pnpm --filter immich --frozen-lockfile build && \
- pnpm --filter immich --frozen-lockfile --prod --no-optional deploy /output/server-pruned
+ SHARP_FORCE_GLOBAL_LIBVIPS=true pnpm --filter immich --frozen-lockfile --prod --no-optional deploy /output/server-pruned
# web production build
FROM prod-builder-base AS web-prod
diff --git a/server/nest-cli.json b/server/nest-cli.json
index 1eaf1888d5..16a8b8a09b 100644
--- a/server/nest-cli.json
+++ b/server/nest-cli.json
@@ -3,7 +3,7 @@
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
- "deleteOutDir": true,
+ "deleteOutDir": false,
"webpack": false,
"plugins": [
{