Skip to content

High performance self-hosted photo and video management solution.

fix(web): compute hashes for uploads in chunks (#27878)

Freddie Floydd36ebcaf00cfa
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index dcfd991ba1..d7a5e57a21 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -67,7 +67,7 @@ importers:
version: 24.12.2
'@vitest/coverage-v8':
specifier: ^4.0.0
byte-size:
specifier: ^9.0.0
version: 9.0.1
@@ -112,10 +112,10 @@ importers:
vitest:
specifier: ^4.0.0
vitest-fetch-mock:
specifier: ^0.4.0
yaml:
specifier: ^2.3.1
version: 2.8.3
@@ -670,7 +670,7 @@ importers:
version: 13.15.10
'@vitest/coverage-v8':
specifier: ^3.0.0
eslint:
specifier: ^10.0.0
version: 10.1.0([email protected])
@@ -727,7 +727,7 @@ importers:
vitest:
specifier: ^3.0.0
web:
dependencies:
@@ -749,6 +749,9 @@ importers:
'@mdi/js':
specifier: ^7.4.47
version: 7.4.47
+ '@noble/hashes':
+ specifier: ^2.2.0
+ version: 2.2.0
'@photo-sphere-viewer/core':
specifier: ^5.14.0
version: 5.14.1
@@ -781,7 +784,7 @@ importers:
version: 2.6.0
fabric:
specifier: ^7.0.0
- version: 7.2.0
+ version: 7.2.0([email protected])
geo-coordinates-parser:
specifier: ^1.7.4
version: 1.7.4
@@ -887,7 +890,7 @@ importers:
version: 6.9.1
'@testing-library/svelte':
specifier: ^5.2.8
'@testing-library/user-event':
specifier: ^14.5.2
version: 14.6.1(@testing-library/[email protected])
@@ -911,7 +914,7 @@ importers:
version: 1.5.6
'@vitest/coverage-v8':
specifier: ^4.0.0
dotenv:
specifier: ^17.0.0
version: 17.3.1
@@ -974,7 +977,7 @@ importers:
vitest:
specifier: ^4.0.0
packages:
@@ -3585,6 +3588,10 @@ packages:
resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}
engines: {node: ^14.21.3 || >=16}
+ '@noble/[email protected]':
+ resolution: {integrity: sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg==}
+ engines: {node: '>= 20.19.0'}
+
'@nodelib/[email protected]':
resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
engines: {node: '>= 8'}
@@ -15549,22 +15556,6 @@ snapshots:
'@mapbox/[email protected]': {}
- '@mapbox/[email protected]':
- dependencies:
- detect-libc: 2.1.2
- https-proxy-agent: 5.0.1
- make-dir: 3.1.0
- node-fetch: 2.7.0
- nopt: 5.0.0
- npmlog: 5.0.1
- rimraf: 3.0.2
- semver: 7.7.4
- tar: 6.2.1
- transitivePeerDependencies:
- - encoding
- - supports-color
- optional: true
-
dependencies:
detect-libc: 2.1.2
@@ -15871,6 +15862,8 @@ snapshots:
'@noble/[email protected]': {}
+ '@noble/[email protected]': {}
+
'@nodelib/[email protected]':
dependencies:
'@nodelib/fs.stat': 2.0.5
@@ -16950,14 +16943,14 @@ snapshots:
dependencies:
svelte: 5.55.1
dependencies:
'@testing-library/dom': 10.4.1
'@testing-library/svelte-core': 1.0.0([email protected])
svelte: 5.55.1
optionalDependencies:
'@testing-library/[email protected](@testing-library/[email protected])':
dependencies:
@@ -17652,7 +17645,7 @@ snapshots:
'@vercel/[email protected]': {}
dependencies:
'@ampproject/remapping': 2.3.0
'@bcoe/v8-coverage': 1.0.2
@@ -17667,11 +17660,11 @@ snapshots:
std-env: 3.10.0
test-exclude: 7.0.2
tinyrainbow: 2.0.0
transitivePeerDependencies:
- supports-color
dependencies:
'@bcoe/v8-coverage': 1.0.2
'@vitest/utils': 4.1.2
@@ -17683,9 +17676,9 @@ snapshots:
obug: 2.1.1
std-env: 4.0.0
tinyrainbow: 3.1.0
dependencies:
'@bcoe/v8-coverage': 1.0.2
'@vitest/utils': 4.1.2
@@ -17697,7 +17690,7 @@ snapshots:
obug: 2.1.1
std-env: 4.0.0
tinyrainbow: 3.1.0
'@vitest/[email protected]':
dependencies:
@@ -18473,16 +18466,6 @@ snapshots:
- dependencies:
- '@mapbox/node-pre-gyp': 1.0.11
- nan: 2.26.2
- simple-get: 3.1.1
- transitivePeerDependencies:
- - encoding
- - supports-color
- optional: true
-
dependencies:
'@mapbox/node-pre-gyp': 1.0.11([email protected])
@@ -20114,10 +20097,10 @@ snapshots:
optionalDependencies:
- canvas: 2.11.2
- jsdom: 26.1.0([email protected])
+ canvas: 2.11.2([email protected])
transitivePeerDependencies:
- bufferutil
- encoding
@@ -21284,36 +21267,6 @@ snapshots:
- utf-8-validate
optional: true
- dependencies:
- cssstyle: 4.6.0
- data-urls: 5.0.0
- decimal.js: 10.6.0
- html-encoding-sniffer: 4.0.0
- http-proxy-agent: 7.0.2
- https-proxy-agent: 7.0.6
- is-potential-custom-element-name: 1.0.1
- nwsapi: 2.2.23
- parse5: 7.3.0
- rrweb-cssom: 0.8.0
- saxes: 6.0.0
- symbol-tree: 3.2.4
- tough-cookie: 5.1.2
- w3c-xmlserializer: 5.0.0
- webidl-conversions: 7.0.0
- whatwg-encoding: 3.1.1
- whatwg-mimetype: 4.0.0
- whatwg-url: 14.2.0
- ws: 8.20.0
- xml-name-validator: 5.0.0
- optionalDependencies:
- canvas: 2.11.2
- transitivePeerDependencies:
- - bufferutil
- - supports-color
- - utf-8-validate
- optional: true
-
@@ -22555,11 +22508,6 @@ snapshots:
emojilib: 2.4.0
skin-tone: 2.0.0
- dependencies:
- whatwg-url: 5.0.0
- optional: true
-
dependencies:
whatwg-url: 5.0.0
@@ -25700,11 +25648,11 @@ snapshots:
optionalDependencies:
dependencies:
dependencies:
'@types/chai': 5.2.3
'@vitest/expect': 3.2.4
@@ -25733,7 +25681,7 @@ snapshots:
'@types/debug': 4.1.12
'@types/node': 24.12.2
happy-dom: 20.8.9
- jsdom: 26.1.0([email protected])
transitivePeerDependencies:
- jiti
- less
@@ -25778,37 +25726,7 @@ snapshots:
transitivePeerDependencies:
- msw
- dependencies:
- '@vitest/expect': 4.1.2
- '@vitest/pretty-format': 4.1.2
- '@vitest/runner': 4.1.2
- '@vitest/snapshot': 4.1.2
- '@vitest/spy': 4.1.2
- '@vitest/utils': 4.1.2
- es-module-lexer: 2.0.0
- expect-type: 1.3.0
- magic-string: 0.30.21
- obug: 2.1.1
- pathe: 2.0.3
- picomatch: 4.0.4
- std-env: 4.0.0
- tinybench: 2.9.0
- tinyexec: 1.0.4
- tinyglobby: 0.2.15
- tinyrainbow: 3.1.0
- why-is-node-running: 2.3.0
- optionalDependencies:
- '@opentelemetry/api': 1.9.0
- '@types/node': 24.12.2
- happy-dom: 20.8.9
- jsdom: 26.1.0([email protected])
- transitivePeerDependencies:
- - msw
-
dependencies:
'@vitest/expect': 4.1.2
@@ -25834,7 +25752,7 @@ snapshots:
'@opentelemetry/api': 1.9.0
'@types/node': 25.5.0
happy-dom: 20.8.9
- jsdom: 26.1.0([email protected])
transitivePeerDependencies:
- msw
diff --git a/web/package.json b/web/package.json
index 71509541f0..6061458b08 100644
--- a/web/package.json
+++ b/web/package.json
@@ -30,6 +30,7 @@
"@immich/ui": "^0.76.0",
"@mapbox/mapbox-gl-rtl-text": "0.3.0",
"@mdi/js": "^7.4.47",
+ "@noble/hashes": "^2.2.0",
"@photo-sphere-viewer/core": "^5.14.0",
"@photo-sphere-viewer/equirectangular-video-adapter": "^5.14.0",
"@photo-sphere-viewer/markers-plugin": "^5.14.0",
diff --git a/web/src/lib/utils/file-uploader.spec.ts b/web/src/lib/utils/file-uploader.spec.ts
index 2218059412..e8ee482b60 100644
--- a/web/src/lib/utils/file-uploader.spec.ts
+++ b/web/src/lib/utils/file-uploader.spec.ts
@@ -20,9 +20,6 @@ describe('fileUploader error handling', () => {
vi.spyOn(uploadManager, 'getExtensions').mockReturnValue(['.jpg']);
uploadAssetsStore.reset();
authManager.reset();
-
- // Stub out crypto to avoid that branch
- vi.stubGlobal('crypto', undefined);
});
for (const [name, mockUser] of [
diff --git a/web/src/lib/utils/file-uploader.ts b/web/src/lib/utils/file-uploader.ts
index b50a468b00..245cb43d76 100644
--- a/web/src/lib/utils/file-uploader.ts
+++ b/web/src/lib/utils/file-uploader.ts
@@ -127,6 +127,30 @@ function getDeviceAssetId(asset: File) {
return 'web' + '-' + asset.name + '-' + asset.lastModified;
}
+function hashFile(file: File): Promise<string> {
+ return new Promise<string>((resolve, reject) => {
+ const worker = new Worker(new URL('$lib/workers/hash-file.ts', import.meta.url), { type: 'module' });
+
+ worker.addEventListener('message', ({ data }: MessageEvent<{ result?: string; error?: string }>) => {
+ worker.terminate();
+
+ if (data.error) {
+ reject(new Error(data.error));
+ } else {
+ resolve(data.result!);
+ }
+ });
+
+ worker.addEventListener('error', (event) => {
+ worker.terminate();
+
+ reject(new Error(event.message));
+ });
+
+ worker.postMessage(file);
+ });
+}
+
type FileUploaderParams = {
assetFile: File;
albumId?: string;
@@ -165,15 +189,11 @@ async function fileUploader({
}
let responseData: { id: string; status: AssetMediaStatus; isTrashed?: boolean } | undefined;
- if (crypto?.subtle?.digest && !authManager.isSharedLink) {
+ if (!authManager.isSharedLink) {
uploadAssetsStore.updateItem(deviceAssetId, { message: $t('asset_hashing') });
await tick();
try {
- const bytes = await assetFile.arrayBuffer();
- const hash = await crypto.subtle.digest('SHA-1', bytes);
- const checksum = Array.from(new Uint8Array(hash))
- .map((b) => b.toString(16).padStart(2, '0'))
- .join('');
+ const checksum = await hashFile(assetFile);
const {
results: [checkUploadResult],
diff --git a/web/src/lib/workers/hash-file.ts b/web/src/lib/workers/hash-file.ts
new file mode 100644
index 0000000000..6927eb7db7
--- /dev/null
+++ b/web/src/lib/workers/hash-file.ts
@@ -0,0 +1,24 @@
+import { sha1 } from '@noble/hashes/legacy.js';
+import { bytesToHex } from '@noble/hashes/utils.js';
+
+const HASH_CHUNK_SIZE = 5 * 1024 * 1024;
+
+async function hashFile(file: File): Promise<string> {
+ const hasher = sha1.create();
+
+ for (let offset = 0; offset < file.size; offset += HASH_CHUNK_SIZE) {
+ const slice = file.slice(offset, Math.min(offset + HASH_CHUNK_SIZE, file.size));
+
+ const buffer = await slice.arrayBuffer();
+
+ hasher.update(new Uint8Array(buffer));
+ }
+
+ return bytesToHex(hasher.digest());
+}
+
+addEventListener('message', (event: MessageEvent<File>) => {
+ void hashFile(event.data)
+ .then((result) => postMessage({ result }))
+ .catch((error: unknown) => postMessage({ error: error instanceof Error ? error.message : String(error) }));
+});