Skip to content

High performance self-hosted photo and video management solution.

feat(server): start up folder checks (#12401)

Jason Rasmussen1e3052bd0bda
diff --git a/server/src/entities/system-metadata.entity.ts b/server/src/entities/system-metadata.entity.ts
index ae01c47b84..0a238e1da5 100644
--- a/server/src/entities/system-metadata.entity.ts
+++ b/server/src/entities/system-metadata.entity.ts
@@ -12,12 +12,14 @@ export class SystemMetadataEntity<T extends keyof SystemMetadata = SystemMetadat
}
export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string };
+export type SystemFlags = { mountFiles: boolean };
export interface SystemMetadata extends Record<SystemMetadataKey, Record<string, any>> {
- [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
- [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
[SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean };
+ [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
+ [SystemMetadataKey.LICENSE]: { licenseKey: string; activationKey: string; activatedAt: Date };
+ [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
[SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial<SystemConfig>;
+ [SystemMetadataKey.SYSTEM_FLAGS]: SystemFlags;
[SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;
- [SystemMetadataKey.LICENSE]: { licenseKey: string; activationKey: string; activatedAt: Date };
}
diff --git a/server/src/enum.ts b/server/src/enum.ts
index 28973e0205..32254854e4 100644
--- a/server/src/enum.ts
+++ b/server/src/enum.ts
@@ -153,6 +153,7 @@ export enum SystemMetadataKey {
FACIAL_RECOGNITION_STATE = 'facial-recognition-state',
ADMIN_ONBOARDING = 'admin-onboarding',
SYSTEM_CONFIG = 'system-config',
+ SYSTEM_FLAGS = 'system-flags',
VERSION_CHECK_STATE = 'version-check-state',
LICENSE = 'license',
}
diff --git a/server/src/interfaces/database.interface.ts b/server/src/interfaces/database.interface.ts
index 373f109142..51b39b95a8 100644
--- a/server/src/interfaces/database.interface.ts
+++ b/server/src/interfaces/database.interface.ts
@@ -15,6 +15,7 @@ export enum VectorIndex {
export enum DatabaseLock {
GeodataImport = 100,
Migrations = 200,
+ SystemFileMounts = 300,
StorageTemplateMigration = 420,
CLIPDimSize = 512,
LibraryWatch = 1337,
diff --git a/server/src/main.ts b/server/src/main.ts
index 7839bafd2f..ee4de1a259 100644
--- a/server/src/main.ts
+++ b/server/src/main.ts
@@ -17,7 +17,13 @@ async function bootstrapImmichAdmin() {
function bootstrapWorker(name: string) {
console.log(`Starting ${name} worker`);
+
const worker = name === 'api' ? fork(`./dist/workers/${name}.js`) : new Worker(`./dist/workers/${name}.js`);
+
+ worker.on('error', (error) => {
+ console.error(`${name} worker error: ${error}`);
+ });
+
worker.on('exit', (exitCode) => {
if (exitCode !== 0) {
console.error(`${name} worker exited with code ${exitCode}`);
diff --git a/server/src/services/storage.service.spec.ts b/server/src/services/storage.service.spec.ts
index d9b4c8eefb..b0f38554cb 100644
--- a/server/src/services/storage.service.spec.ts
+++ b/server/src/services/storage.service.spec.ts
@@ -1,19 +1,29 @@
+import { SystemMetadataKey } from 'src/enum';
+import { IDatabaseRepository } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IStorageRepository } from 'src/interfaces/storage.interface';
+import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { StorageService } from 'src/services/storage.service';
+import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock';
+import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock';
import { Mocked } from 'vitest';
describe(StorageService.name, () => {
let sut: StorageService;
+ let databaseMock: Mocked<IDatabaseRepository>;
let storageMock: Mocked<IStorageRepository>;
let loggerMock: Mocked<ILoggerRepository>;
+ let systemMock: Mocked<ISystemMetadataRepository>;
beforeEach(() => {
+ databaseMock = newDatabaseRepositoryMock();
storageMock = newStorageRepositoryMock();
loggerMock = newLoggerRepositoryMock();
- sut = new StorageService(storageMock, loggerMock);
+ systemMock = newSystemMetadataRepositoryMock();
+
+ sut = new StorageService(databaseMock, storageMock, loggerMock, systemMock);
});
it('should work', () => {
@@ -21,9 +31,35 @@ describe(StorageService.name, () => {
});
describe('onBootstrap', () => {
- it('should create the library folder on initialization', () => {
- sut.onBootstrap();
+ it('should enable mount folder checking', async () => {
+ systemMock.get.mockResolvedValue(null);
+
+ await expect(sut.onBootstrap()).resolves.toBeUndefined();
+
+ expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, { mountFiles: true });
+ expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/encoded-video');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/library');
+ expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/profile');
+ expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs');
+ });
+
+ it('should throw an error if .immich is missing', async () => {
+ systemMock.get.mockResolvedValue({ mountFiles: true });
+ storageMock.readFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
+
+ await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount');
+
+ expect(storageMock.writeFile).not.toHaveBeenCalled();
+ expect(systemMock.set).not.toHaveBeenCalled();
+ });
+
+ it('should throw an error if .immich is present but read-only', async () => {
+ systemMock.get.mockResolvedValue({ mountFiles: true });
+ storageMock.writeFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
+
+ await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount');
+
+ expect(systemMock.set).not.toHaveBeenCalled();
});
});
diff --git a/server/src/services/storage.service.ts b/server/src/services/storage.service.ts
index c3f2c06438..a8f6a76e74 100644
--- a/server/src/services/storage.service.ts
+++ b/server/src/services/storage.service.ts
@@ -1,23 +1,52 @@
import { Inject, Injectable } from '@nestjs/common';
+import { join } from 'node:path';
import { StorageCore, StorageFolder } from 'src/cores/storage.core';
import { OnEmit } from 'src/decorators';
+import { SystemMetadataKey } from 'src/enum';
+import { DatabaseLock, IDatabaseRepository } from 'src/interfaces/database.interface';
import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IStorageRepository } from 'src/interfaces/storage.interface';
+import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
+import { ImmichStartupError } from 'src/utils/events';
@Injectable()
export class StorageService {
constructor(
+ @Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
+ @Inject(ISystemMetadataRepository) private systemMetadata: ISystemMetadataRepository,
) {
this.logger.setContext(StorageService.name);
}
@OnEmit({ event: 'app.bootstrap' })
- onBootstrap() {
- const libraryBase = StorageCore.getBaseFolder(StorageFolder.LIBRARY);
- this.storageRepository.mkdirSync(libraryBase);
+ async onBootstrap() {
+ await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => {
+ const flags = (await this.systemMetadata.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false };
+
+ this.logger.log('Verifying system mount folder checks');
+
+ // check each folder exists and is writable
+ for (const folder of Object.values(StorageFolder)) {
+ if (!flags.mountFiles) {
+ this.logger.log(`Writing initial mount file for the ${folder} folder`);
+ await this.verifyWriteAccess(folder);
+ }
+
+ await this.verifyReadAccess(folder);
+ await this.verifyWriteAccess(folder);
+ }
+
+ if (!flags.mountFiles) {
+ flags.mountFiles = true;
+ await this.systemMetadata.set(SystemMetadataKey.SYSTEM_FLAGS, flags);
+ this.logger.log('Successfully enabled system mount folders checks');
+ }
+
+ this.logger.log('Successfully verified system mount folder checks');
+ });
}
async handleDeleteFiles(job: IDeleteFilesJob) {
@@ -38,4 +67,38 @@ export class StorageService {
return JobStatus.SUCCESS;
}
+
+ private async verifyReadAccess(folder: StorageFolder) {
+ const { filePath } = this.getMountFilePaths(folder);
+ try {
+ await this.storageRepository.readFile(filePath);
+ } catch (error) {
+ this.logger.error(`Failed to read ${filePath}: ${error}`);
+ this.logger.error(
+ `The "${folder}" folder appears to be offline/missing, please make sure the volume is mounted with the correct permissions`,
+ );
+ throw new ImmichStartupError(`Failed to validate folder mount (read from "<MEDIA_LOCATION>/${folder}")`);
+ }
+ }
+
+ private async verifyWriteAccess(folder: StorageFolder) {
+ const { folderPath, filePath } = this.getMountFilePaths(folder);
+ try {
+ this.storageRepository.mkdirSync(folderPath);
+ await this.storageRepository.writeFile(filePath, Buffer.from(`${Date.now()}`));
+ } catch (error) {
+ this.logger.error(`Failed to write ${filePath}: ${error}`);
+ this.logger.error(
+ `The "${folder}" folder cannot be written to, please make sure the volume is mounted with the correct permissions`,
+ );
+ throw new ImmichStartupError(`Failed to validate folder mount (write to "<MEDIA_LOCATION>/${folder}")`);
+ }
+ }
+
+ private getMountFilePaths(folder: StorageFolder) {
+ const folderPath = StorageCore.getBaseFolder(folder);
+ const filePath = join(folderPath, '.immich');
+
+ return { folderPath, filePath };
+ }
}
diff --git a/server/src/utils/events.ts b/server/src/utils/events.ts
index 2dd7e7fd5d..064c9f7507 100644
--- a/server/src/utils/events.ts
+++ b/server/src/utils/events.ts
@@ -12,6 +12,9 @@ type Item<T extends EmitEvent> = {
label: string;
};
+export class ImmichStartupError extends Error {}
+export const isStartUpError = (error: unknown): error is ImmichStartupError => error instanceof ImmichStartupError;
+
export const setupEventHandlers = (moduleRef: ModuleRef) => {
const reflector = moduleRef.get(Reflector, { strict: false });
const repository = moduleRef.get<IEventRepository>(IEventRepository);
diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts
index 5857f587a0..629c50c653 100644
--- a/server/src/workers/api.ts
+++ b/server/src/workers/api.ts
@@ -9,6 +9,7 @@ import { envName, excludePaths, isDev, resourcePaths, serverVersion } from 'src/
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { ApiService } from 'src/services/api.service';
+import { isStartUpError } from 'src/utils/events';
import { otelStart } from 'src/utils/instrumentation';
import { useSwagger } from 'src/utils/misc';
@@ -73,6 +74,9 @@ async function bootstrap() {
}
bootstrap().catch((error) => {
- console.error(error);
- throw error;
+ if (!isStartUpError(error)) {
+ console.error(error);
+ }
+ // eslint-disable-next-line unicorn/no-process-exit
+ process.exit(1);
});
diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts
index f920e8c947..789b6f5287 100644
--- a/server/src/workers/microservices.ts
+++ b/server/src/workers/microservices.ts
@@ -4,6 +4,7 @@ import { MicroservicesModule } from 'src/app.module';
import { envName, serverVersion } from 'src/constants';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
+import { isStartUpError } from 'src/utils/events';
import { otelStart } from 'src/utils/instrumentation';
export async function bootstrap() {
@@ -25,7 +26,9 @@ export async function bootstrap() {
if (!isMainThread) {
bootstrap().catch((error) => {
- console.error(error);
- process.exit(1);
+ if (!isStartUpError(error)) {
+ console.error(error);
+ }
+ throw error;
});
}