Skip to content

High performance self-hosted photo and video management solution.

fix(server): sync files to disk (#26881)

Thomas16749ff8ba6f
diff --git a/server/src/middleware/file-upload.interceptor.ts b/server/src/middleware/file-upload.interceptor.ts
index 6dfd11ee4b..63acb13789 100644
--- a/server/src/middleware/file-upload.interceptor.ts
+++ b/server/src/middleware/file-upload.interceptor.ts
@@ -3,13 +3,16 @@ import { PATH_METADATA } from '@nestjs/common/constants';
import { Reflector } from '@nestjs/core';
import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
import { NextFunction, RequestHandler } from 'express';
-import multer, { StorageEngine, diskStorage } from 'multer';
+import multer from 'multer';
import { createHash, randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { pipeline } from 'node:stream';
import { Observable } from 'rxjs';
import { UploadFieldName } from 'src/dtos/asset-media.dto';
import { RouteKey } from 'src/enum';
import { AuthRequest } from 'src/middleware/auth.guard';
import { LoggingRepository } from 'src/repositories/logging.repository';
+import { StorageRepository } from 'src/repositories/storage.repository';
import { AssetMediaService } from 'src/services/asset-media.service';
import { ImmichFile, UploadFile, UploadFiles } from 'src/types';
import { asUploadRequest, mapToUploadFile } from 'src/utils/asset.util';
@@ -26,8 +29,6 @@ export function getFiles(files: UploadFiles) {
};
}
-type DiskStorageCallback = (error: Error | null, result: string) => void;
-
type ImmichMulterFile = Express.Multer.File & { uuid: string };
interface Callback<T> {
@@ -35,34 +36,21 @@ interface Callback<T> {
(error: null, result: T): void;
}
-const callbackify = <T>(target: (...arguments_: any[]) => T, callback: Callback<T>) => {
- try {
- return callback(null, target());
- } catch (error: Error | any) {
- return callback(error);
- }
-};
-
@Injectable()
export class FileUploadInterceptor implements NestInterceptor {
private handlers: {
userProfile: RequestHandler;
assetUpload: RequestHandler;
};
- private defaultStorage: StorageEngine;
constructor(
private reflect: Reflector,
private assetService: AssetMediaService,
+ private storageRepository: StorageRepository,
private logger: LoggingRepository,
) {
this.logger.setContext(FileUploadInterceptor.name);
- this.defaultStorage = diskStorage({
- filename: this.filename.bind(this),
- destination: this.destination.bind(this),
- });
-
const instance = multer({
fileFilter: this.fileFilter.bind(this),
storage: {
@@ -99,60 +87,60 @@ export class FileUploadInterceptor implements NestInterceptor {
}
private fileFilter(request: AuthRequest, file: Express.Multer.File, callback: multer.FileFilterCallback) {
- return callbackify(() => this.assetService.canUploadFile(asUploadRequest(request, file)), callback);
- }
-
- private filename(request: AuthRequest, file: Express.Multer.File, callback: DiskStorageCallback) {
- return callbackify(
- () => this.assetService.getUploadFilename(asUploadRequest(request, file)),
- callback as Callback<string>,
- );
- }
-
- private destination(request: AuthRequest, file: Express.Multer.File, callback: DiskStorageCallback) {
- return callbackify(
- () => this.assetService.getUploadFolder(asUploadRequest(request, file)),
- callback as Callback<string>,
- );
+ try {
+ callback(null, this.assetService.canUploadFile(asUploadRequest(request, file)));
+ } catch (error: Error | any) {
+ callback(error);
+ }
}
private handleFile(request: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) {
- (file as ImmichMulterFile).uuid = randomUUID();
-
request.on('error', (error) => {
this.logger.warn('Request error while uploading file, cleaning up', error);
this.assetService.onUploadError(request, file).catch(this.logger.error);
});
- if (!this.isAssetUploadFile(file)) {
- this.defaultStorage._handleFile(request, file, callback);
- return;
- }
+ try {
+ (file as ImmichMulterFile).uuid = randomUUID();
- const hash = createHash('sha1');
- file.stream.on('data', (chunk) => hash.update(chunk));
- this.defaultStorage._handleFile(request, file, (error, info) => {
- if (error) {
- hash.destroy();
- callback(error);
- } else {
- callback(null, { ...info, checksum: hash.digest() });
- }
- });
- }
+ const uploadRequest = asUploadRequest(request, file);
- private removeFile(request: AuthRequest, file: Express.Multer.File, callback: (error: Error | null) => void) {
- this.defaultStorage._removeFile(request, file, callback);
- }
+ const path = join(
+ this.assetService.getUploadFolder(uploadRequest),
+ this.assetService.getUploadFilename(uploadRequest),
+ );
- private isAssetUploadFile(file: Express.Multer.File) {
- switch (file.fieldname as UploadFieldName) {
- case UploadFieldName.ASSET_DATA: {
- return true;
- }
+ const writeStream = this.storageRepository.createWriteStream(path);
+ const hash = file.fieldname === UploadFieldName.ASSET_DATA ? createHash('sha1') : null;
+
+ let size = 0;
+
+ file.stream.on('data', (chunk) => {
+ hash?.update(chunk);
+ size += chunk.length;
+ });
+
+ pipeline(file.stream, writeStream, (error) => {
+ if (error) {
+ hash?.destroy();
+ return callback(error);
+ }
+ callback(null, {
+ path,
+ size,
+ checksum: hash?.digest(),
+ });
+ });
+ } catch (error: Error | any) {
+ callback(error);
}
+ }
- return false;
+ private removeFile(_request: AuthRequest, file: Express.Multer.File, callback: (error: Error | null) => void) {
+ this.storageRepository
+ .unlink(file.path)
+ .then(() => callback(null))
+ .catch(callback);
}
private getHandler(route: RouteKey) {
diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.ts
index 5a1a936e77..c7ba4ab6cc 100644
--- a/server/src/repositories/storage.repository.ts
+++ b/server/src/repositories/storage.repository.ts
@@ -63,7 +63,7 @@ export class StorageRepository {
}
createWriteStream(filepath: string): Writable {
- return createWriteStream(filepath, { flags: 'w' });
+ return createWriteStream(filepath, { flags: 'w', flush: true });
}
createOrOverwriteFile(filepath: string, buffer: Buffer) {