High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix: isolate docker host/container filesystem for node_modules and build output (#21167)
Min Idzelis44e1c83c84e3
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.jsonindex 79126fd658..9563be6125 100644--- a/.devcontainer/devcontainer.json+++ b/.devcontainer/devcontainer.json@@ -5,7 +5,8 @@"immich-server","redis","database",- "immich-machine-learning"+ "immich-machine-learning",+ "init"],"dockerComposeFile": ["../docker/docker-compose.dev.yml",diff --git a/.devcontainer/server/container-compose-overrides.yml b/.devcontainer/server/container-compose-overrides.ymlindex 869b6c2624..59975baad3 100644--- a/.devcontainer/server/container-compose-overrides.yml+++ b/.devcontainer/server/container-compose-overrides.yml@@ -11,6 +11,18 @@ services:- ${UPLOAD_LOCATION:-upload1-devcontainer-volume}${UPLOAD_LOCATION:+/photos}:/data- ${UPLOAD_LOCATION:-upload2-devcontainer-volume}${UPLOAD_LOCATION:+/photos/upload}:/data/upload- /etc/localtime:/etc/localtime:ro+ - pnpm-store:/usr/src/app/.pnpm-store+ - server-node-modules:/usr/src/app/server/node_modules+ - server-dist:/usr/src/app/server/dist+ - web-node_modules:/usr/src/app/web/node_modules+ - github-node_modules:/usr/src/app/.github/node_modules+ - cli-node_modules:/usr/src/app/cli/node_modules+ - docs-node_modules:/usr/src/app/docs/node_modules+ - e2e-node_modules:/usr/src/app/e2e/node_modules+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules+ - app-node_modules:/usr/src/app/node_modules+ - sveltekit:/usr/src/app/web/.svelte-kit+ - coverage:/usr/src/app/web/coverageimmich-web:env_file: !reset []immich-machine-learning:diff --git a/.pnpmfile.cjs b/.pnpmfile.cjsindex feda217821..0e76dabe66 100644--- a/.pnpmfile.cjs+++ b/.pnpmfile.cjs@@ -4,34 +4,13 @@ module.exports = {if (!pkg.name) {return pkg;}- switch (pkg.name) {- case "exiftool-vendored":- if (pkg.optionalDependencies["exiftool-vendored.pl"]) {- // make exiftool-vendored.pl a regular dependency- pkg.dependencies["exiftool-vendored.pl"] =- pkg.optionalDependencies["exiftool-vendored.pl"];- delete pkg.optionalDependencies["exiftool-vendored.pl"];- }- break;- case "sharp":- const optionalDeps = Object.keys(pkg.optionalDependencies).filter(- (dep) => dep.startsWith("@img")- );- for (const dep of optionalDeps) {- // remove all optionalDependencies from sharp (they will be compiled from source), except:- // include the precompiled musl version of sharp, for web- // include precompiled linux-x64 version of sharp, for server (stage: web-prod)- // include precompiled linux-arm64 version of sharp, for server (stage: web-prod)- if (- dep.includes("musl") ||- dep.includes("linux-x64") ||- dep.includes("linux-arm64")- ) {- continue;- }- delete pkg.optionalDependencies[dep];- }- break;+ if (pkg.name === "exiftool-vendored") {+ if (pkg.optionalDependencies["exiftool-vendored.pl"]) {+ // make exiftool-vendored.pl a regular dependency+ pkg.dependencies["exiftool-vendored.pl"] =+ pkg.optionalDependencies["exiftool-vendored.pl"];+ delete pkg.optionalDependencies["exiftool-vendored.pl"];+ }}return pkg;},diff --git a/Makefile b/Makefileindex a9faceadf4..31a00ee6be 100644--- a/Makefile+++ b/Makefile@@ -1,29 +1,29 @@-dev:+dev: prepare-volumes@trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --remove-orphansdev-down:docker compose -f ./docker/docker-compose.dev.yml down --remove-orphans-dev-update:+dev-update: prepare-volumes@trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --build -V --remove-orphans-dev-scale:+dev-scale: prepare-volumes@trap 'make dev-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.dev.yml up --build -V --scale immich-server=3 --remove-orphans-dev-docs:+dev-docs: prepare-volumesnpm --prefix docs run start.PHONY: e2e-e2e:- @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --build -V --remove-orphans+e2e: prepare-volumes+ @trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --remove-orphans-e2e-update:+e2e-update: prepare-volumes@trap 'make e2e-down' EXIT; COMPOSE_BAKE=true docker compose -f ./e2e/docker-compose.yml up --build -V --remove-orphanse2e-down:docker compose -f ./e2e/docker-compose.yml down --remove-orphans-prod:+prod:@trap 'make prod-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.prod.yml up --build -V --remove-orphansprod-down:@@ -33,16 +33,16 @@ prod-scale:@trap 'make prod-down' EXIT; COMPOSE_BAKE=true docker compose -f ./docker/docker-compose.prod.yml up --build -V --scale immich-server=3 --scale immich-microservices=3 --remove-orphans.PHONY: open-api-open-api:+open-api: prepare-volumescd ./open-api && bash ./bin/generate-open-api.sh-open-api-dart:+open-api-dart: prepare-volumescd ./open-api && bash ./bin/generate-open-api.sh dart-open-api-typescript:+open-api-typescript: prepare-volumescd ./open-api && bash ./bin/generate-open-api.sh typescript-sql:+sql: prepare-volumespnpm --filter immich run sync:sqlattach-server:@@ -51,6 +51,30 @@ attach-server:renovate:LOG_LEVEL=debug npx renovate --platform=local --repository-cache=reset+# Directories that need to be created for volumes or build output+VOLUME_DIRS = \+ ./.pnpm-store \+ ./web/.svelte-kit \+ ./web/node_modules \+ ./web/coverage \+ ./e2e/node_modules \+ ./docs/node_modules \+ ./server/node_modules \+ ./server/dist \+ ./open-api/typescript-sdk/node_modules \+ ./.github/node_modules \+ ./node_modules \+ ./cli/node_modules++# create empty directories and chown to current user+prepare-volumes:+ @for dir in $(VOLUME_DIRS); do \+ mkdir -p $$dir; \+ done+ @if [ -n "$(VOLUME_DIRS)" ]; then \+ chown -R $$(id -u):$$(id -g) $(VOLUME_DIRS); \+ fi+MODULES = e2e server web cli sdk docs .github# directory to package name mapping functiondiff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.ymlindex c6393fd132..fe621c768f 100644--- a/docker/docker-compose.dev.yml+++ b/docker/docker-compose.dev.yml@@ -32,6 +32,18 @@ services:- ${UPLOAD_LOCATION}/photos:/data- ${UPLOAD_LOCATION}/photos/upload:/data/upload- /etc/localtime:/etc/localtime:ro+ - pnpm-store:/usr/src/app/.pnpm-store+ - server-node-modules:/usr/src/app/server/node_modules+ - server-dist:/usr/src/app/server/dist+ - web-node_modules:/usr/src/app/web/node_modules+ - github-node_modules:/usr/src/app/.github/node_modules+ - cli-node_modules:/usr/src/app/cli/node_modules+ - docs-node_modules:/usr/src/app/docs/node_modules+ - e2e-node_modules:/usr/src/app/e2e/node_modules+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules+ - app-node_modules:/usr/src/app/node_modules+ - sveltekit:/usr/src/app/web/.svelte-kit+ - coverage:/usr/src/app/web/coverageenv_file:- .envenvironment:@@ -84,6 +96,18 @@ services:- 24678:24678volumes:- ..:/usr/src/app+ - pnpm-store:/usr/src/app/.pnpm-store+ - server-node-modules:/usr/src/app/server/node_modules+ - server-dist:/usr/src/app/server/dist+ - web-node_modules:/usr/src/app/web/node_modules+ - github-node_modules:/usr/src/app/.github/node_modules+ - cli-node_modules:/usr/src/app/cli/node_modules+ - docs-node_modules:/usr/src/app/docs/node_modules+ - e2e-node_modules:/usr/src/app/e2e/node_modules+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules+ - app-node_modules:/usr/src/app/node_modules+ - sveltekit:/usr/src/app/web/.svelte-kit+ - coverage:/usr/src/app/web/coverageulimits:nofile:soft: 1048576@@ -167,9 +191,33 @@ services:env_file:- .envuser: 0:0- command: sh -c 'for path in /usr/src/app/.pnpm-store /usr/src/app/server/node_modules /usr/src/app/.github/node_modules /usr/src/app/cli/node_modules /usr/src/app/docs/node_modules /usr/src/app/e2e/node_modules /usr/src/app/open-api/typescript-sdk/node_modules /usr/src/app/web/.svelte-kit /usr/src/app/web/coverage /usr/src/app/node_modules /usr/src/app/web/node_modules; do [ -e "$$path" ] && chown -R ${UID:-1000}:${GID:-1000} "$$path" || true; done'-+ command: sh -c 'for path in /usr/src/app/.pnpm-store /usr/src/app/server/node_modules /usr/src/app/server/dist /usr/src/app/.github/node_modules /usr/src/app/cli/node_modules /usr/src/app/docs/node_modules /usr/src/app/e2e/node_modules /usr/src/app/open-api/typescript-sdk/node_modules /usr/src/app/web/.svelte-kit /usr/src/app/web/coverage /usr/src/app/node_modules /usr/src/app/web/node_modules; do [ -e "$$path" ] && chown -R ${UID:-1000}:${GID:-1000} "$$path" || true; done'+ volumes:+ - pnpm-store:/usr/src/app/.pnpm-store+ - server-node-modules:/usr/src/app/server/node_modules+ - server-dist:/usr/src/app/server/dist+ - web-node_modules:/usr/src/app/web/node_modules+ - github-node_modules:/usr/src/app/.github/node_modules+ - cli-node_modules:/usr/src/app/cli/node_modules+ - docs-node_modules:/usr/src/app/docs/node_modules+ - e2e-node_modules:/usr/src/app/e2e/node_modules+ - sdk-node_modules:/usr/src/app/open-api/typescript-sdk/node_modules+ - app-node_modules:/usr/src/app/node_modules+ - sveltekit:/usr/src/app/web/.svelte-kit+ - coverage:/usr/src/app/web/coveragevolumes:model-cache:prometheus-data:grafana-data:+ pnpm-store:+ server-node-modules:+ server-dist:+ web-node_modules:+ github-node_modules:+ cli-node_modules:+ docs-node_modules:+ e2e-node_modules:+ sdk-node_modules:+ app-node_modules:+ sveltekit:+ coverage:diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 103cacfdc0..fa24e5b31f 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -11,7 +11,7 @@ overrides:packageExtensionsChecksum: sha256-DAYr0FTkvKYnvBH4muAER9UE1FVGKhqfRU4/QwA2xPQ=-pnpmfileChecksum: sha256-7GOLcTtuczNumtarIG1mbRinBOSpiOOVzgbeV3Xp4X4=+pnpmfileChecksum: sha256-AG/qwrPNpmy9q60PZwCpecoYVptglTHgH+N6RKQHOM0=importers:@@ -2124,6 +2124,9 @@ packages:resolution: {integrity: sha512-P1ml0nvOmEFdmu0smSXOqTS1sxU5tqvnc0dA4MTKV39kye+bhQnjkIKEE18fNOvxjyB86k8esoCIFM3x4RykOQ==}engines: {node: '>=18.0'}+ '@emnapi/[email protected]':+ resolution: {integrity: sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==}+'@esbuild/[email protected]':resolution: {integrity: sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==}engines: {node: '>=12'}@@ -2553,11 +2556,48 @@ packages:resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==}engines: {node: '>=18.18'}+ '@img/[email protected]':+ resolution: {integrity: sha512-OfXHZPppddivUJnqyKoi5YVeHRkkNE2zUFT2gbpKxp/JZCFYEYubnMg+gOp6lWfasPrTS+KPosKqdI+ELYVDtg==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [arm64]+ os: [darwin]++ '@img/[email protected]':+ resolution: {integrity: sha512-dYvWqmjU9VxqXmjEtjmvHnGqF8GrVjM2Epj9rJ6BUIXvk8slvNDJbhGFvIoXzkDhrJC2jUxNLz/GUjjvSzfw+g==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [x64]+ os: [darwin]++ '@img/[email protected]':+ resolution: {integrity: sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==}+ cpu: [arm64]+ os: [darwin]++ '@img/[email protected]':+ resolution: {integrity: sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==}+ cpu: [x64]+ os: [darwin]+'@img/[email protected]':resolution: {integrity: sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==}cpu: [arm64]os: [linux]+ '@img/[email protected]':+ resolution: {integrity: sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==}+ cpu: [arm]+ os: [linux]++ '@img/[email protected]':+ resolution: {integrity: sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==}+ cpu: [ppc64]+ os: [linux]++ '@img/[email protected]':+ resolution: {integrity: sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==}+ cpu: [s390x]+ os: [linux]+'@img/[email protected]':resolution: {integrity: sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==}cpu: [x64]@@ -2579,6 +2619,18 @@ packages:cpu: [arm64]os: [linux]+ '@img/[email protected]':+ resolution: {integrity: sha512-0DZzkvuEOqQUP9mo2kjjKNok5AmnOr1jB2XYjkaoNRwpAYMDzRmAqUIa1nRi58S2WswqSfPOWLNOr0FDT3H5RQ==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [arm]+ os: [linux]++ '@img/[email protected]':+ resolution: {integrity: sha512-EGZ1xwhBI7dNISwxjChqBGELCWMGDvmxZXKjQRuqMrakhO8QoMgqCrdjnAqJq/CScxfRn+Bb7suXBElKQpPDiw==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [s390x]+ os: [linux]+'@img/[email protected]':resolution: {integrity: sha512-sD7J+h5nFLMMmOXYH4DD9UtSNBD05tWSSdWAcEyzqW8Cn5UxXvsHAxmxSesYUsTOBmUnjtxghKDl15EvfqLFbQ==}engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}@@ -2597,6 +2649,29 @@ packages:cpu: [x64]os: [linux]+ '@img/[email protected]':+ resolution: {integrity: sha512-/VI4mdlJ9zkaq53MbIG6rZY+QRN3MLbR6usYlgITEzi4Rpx5S6LFKsycOQjkOGmqTNmkIdLjEvooFKwww6OpdQ==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [wasm32]++ '@img/[email protected]':+ resolution: {integrity: sha512-cfP/r9FdS63VA5k0xiqaNaEoGxBg9k7uE+RQGzuK9fHt7jib4zAVVseR9LsE4gJcNWgT6APKMNnCcnyOtmSEUQ==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [arm64]+ os: [win32]++ '@img/[email protected]':+ resolution: {integrity: sha512-QLjGGvAbj0X/FXl8n1WbtQ6iVBpWU7JO94u/P2M4a8CFYsvQi4GW2mRy/JqkRx0qpBzaOdKJKw8uc930EX2AHw==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [ia32]+ os: [win32]++ '@img/[email protected]':+ resolution: {integrity: sha512-aUdT6zEYtDKCaxkofmmJDJYGCf0+pJg3eU9/oBuqvEeoB9dKI6ZLc/1iLJCTuJQDO4ptntAlkUmHgGjyuobZbw==}+ engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}+ cpu: [x64]+ os: [win32]+'@immich/[email protected]':resolution: {integrity: sha512-phJ9BHV0+OnKsxXD+5+Te5Amnb1N4ExYpRGSJPYFqutd5WXeN7kZGKZXd3CfcQ1e31SXRy4DsHSGdM1pY7AUgA==}peerDependencies:@@ -13888,6 +13963,11 @@ snapshots:- uglify-js- webpack-cli+ '@emnapi/[email protected]':+ dependencies:+ tslib: 2.8.1+ optional: true+'@esbuild/[email protected]':optional: true@@ -14184,9 +14264,34 @@ snapshots:'@humanwhocodes/[email protected]': {}+ '@img/[email protected]':+ optionalDependencies:+ '@img/sharp-libvips-darwin-arm64': 1.1.0+ optional: true++ '@img/[email protected]':+ optionalDependencies:+ '@img/sharp-libvips-darwin-x64': 1.1.0+ optional: true++ '@img/[email protected]':+ optional: true++ '@img/[email protected]':+ optional: true+'@img/[email protected]':optional: true+ '@img/[email protected]':+ optional: true++ '@img/[email protected]':+ optional: true++ '@img/[email protected]':+ optional: true+'@img/[email protected]':optional: true@@ -14201,6 +14306,16 @@ snapshots:'@img/sharp-libvips-linux-arm64': 1.1.0optional: true+ '@img/[email protected]':+ optionalDependencies:+ '@img/sharp-libvips-linux-arm': 1.1.0+ optional: true++ '@img/[email protected]':+ optionalDependencies:+ '@img/sharp-libvips-linux-s390x': 1.1.0+ optional: true+'@img/[email protected]':optionalDependencies:'@img/sharp-libvips-linux-x64': 1.1.0@@ -14216,6 +14331,20 @@ snapshots:'@img/sharp-libvips-linuxmusl-x64': 1.1.0optional: true+ '@img/[email protected]':+ dependencies:+ '@emnapi/runtime': 1.4.5+ optional: true++ '@img/[email protected]':+ optional: true++ '@img/[email protected]':+ optional: true++ '@img/[email protected]':+ optional: true+dependencies:'@mdi/js': 7.4.47@@ -23758,14 +23887,27 @@ snapshots:node-gyp: 11.2.0semver: 7.7.2optionalDependencies:+ '@img/sharp-darwin-arm64': 0.34.2+ '@img/sharp-darwin-x64': 0.34.2+ '@img/sharp-libvips-darwin-arm64': 1.1.0+ '@img/sharp-libvips-darwin-x64': 1.1.0+ '@img/sharp-libvips-linux-arm': 1.1.0'@img/sharp-libvips-linux-arm64': 1.1.0+ '@img/sharp-libvips-linux-ppc64': 1.1.0+ '@img/sharp-libvips-linux-s390x': 1.1.0'@img/sharp-libvips-linux-x64': 1.1.0'@img/sharp-libvips-linuxmusl-arm64': 1.1.0'@img/sharp-libvips-linuxmusl-x64': 1.1.0+ '@img/sharp-linux-arm': 0.34.2'@img/sharp-linux-arm64': 0.34.2+ '@img/sharp-linux-s390x': 0.34.2'@img/sharp-linux-x64': 0.34.2'@img/sharp-linuxmusl-arm64': 0.34.2'@img/sharp-linuxmusl-x64': 0.34.2+ '@img/sharp-wasm32': 0.34.2+ '@img/sharp-win32-arm64': 0.34.2+ '@img/sharp-win32-ia32': 0.34.2+ '@img/sharp-win32-x64': 0.34.2transitivePeerDependencies:- supports-colordiff --git a/server/Dockerfile b/server/Dockerfileindex ae3198c5d3..bd5c55402e 100644--- a/server/Dockerfile+++ b/server/Dockerfile@@ -91,9 +91,8 @@ FROM prod-builder-base AS server-prodWORKDIR /usr/src/appCOPY ./package* ./pnpm* .pnpmfile.cjs ./COPY ./server ./server/-# SHARP_IGNORE_GLOBAL_LIBVIPS because 'deploy' will always build sharp bindings from sourceRUN SHARP_IGNORE_GLOBAL_LIBVIPS=true pnpm --filter immich --frozen-lockfile build && \- pnpm --filter immich --frozen-lockfile --prod --no-optional deploy /output/server-pruned+ SHARP_FORCE_GLOBAL_LIBVIPS=true pnpm --filter immich --frozen-lockfile --prod --no-optional deploy /output/server-pruned# web production buildFROM prod-builder-base AS web-proddiff --git a/server/nest-cli.json b/server/nest-cli.jsonindex 1eaf1888d5..16a8b8a09b 100644--- a/server/nest-cli.json+++ b/server/nest-cli.json@@ -3,7 +3,7 @@"collection": "@nestjs/schematics","sourceRoot": "src","compilerOptions": {- "deleteOutDir": true,+ "deleteOutDir": false,"webpack": false,"plugins": [{