High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(maintenance): prevent enable/disable maintenance CLI hanging on occasion (#24713)
Paul Maklesa17f188e976a
diff --git a/server/src/maintenance/maintenance-websocket.repository.ts b/server/src/maintenance/maintenance-websocket.repository.tsindex 5d8368cf69..cf04c0ad12 100644--- a/server/src/maintenance/maintenance-websocket.repository.ts+++ b/server/src/maintenance/maintenance-websocket.repository.ts@@ -37,7 +37,13 @@ export class MaintenanceWebsocketRepository implements OnGatewayConnection, OnGaafterInit(websocketServer: Server) {this.logger.log('Initialized websocket server');- websocketServer.on('AppRestart', () => this.appRepository.exitApp());++ websocketServer.on('AppRestart', (event: ArgsOf<'AppRestart'>, ack?: (ok: 'ok') => void) => {+ this.logger.log(`Restarting due to event... ${JSON.stringify(event)}`);++ ack?.('ok');+ this.appRepository.exitApp();+ });}clientBroadcast<T extends keyof ClientEventMap>(event: T, ...data: ClientEventMap[T]) {diff --git a/server/src/repositories/app.repository.ts b/server/src/repositories/app.repository.tsindex e6181ef7f3..96e413232f 100644--- a/server/src/repositories/app.repository.ts+++ b/server/src/repositories/app.repository.ts@@ -1,5 +1,10 @@import { Injectable } from '@nestjs/common';+import { createAdapter } from '@socket.io/redis-adapter';+import Redis from 'ioredis';+import { Server as SocketIO } from 'socket.io';import { ExitCode } from 'src/enum';+import { ConfigRepository } from 'src/repositories/config.repository';+import { AppRestartEvent } from 'src/repositories/event.repository';@Injectable()export class AppRepository {@@ -17,4 +22,26 @@ export class AppRepository {setCloseFn(fn: () => Promise<void>) {this.closeFn = fn;}++ async sendOneShotAppRestart(state: AppRestartEvent): Promise<void> {+ const server = new SocketIO();+ const { redis } = new ConfigRepository().getEnv();+ const pubClient = new Redis({ ...redis, lazyConnect: true });+ const subClient = pubClient.duplicate();++ await Promise.all([pubClient.connect(), subClient.connect()]);++ server.adapter(createAdapter(pubClient, subClient));++ // => corresponds to notification.service.ts#onAppRestart+ server.emit('AppRestartV1', state, async () => {+ const responses = await server.serverSideEmitWithAck('AppRestart', state);+ if (responses.some((response) => response !== 'ok')) {+ throw new Error("One or more node(s) returned a non-'ok' response to our restart request!");+ }++ pubClient.disconnect();+ subClient.disconnect();+ });+ }}diff --git a/server/src/services/cli.service.spec.ts b/server/src/services/cli.service.spec.tsindex 49fa5cf5b8..f4f14c3e68 100644--- a/server/src/services/cli.service.spec.ts+++ b/server/src/services/cli.service.spec.ts@@ -89,6 +89,7 @@ describe(CliService.name, () => {alreadyDisabled: true,});+ expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalledTimes(0);expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);expect(mocks.event.emit).toHaveBeenCalledTimes(0);});@@ -99,6 +100,7 @@ describe(CliService.name, () => {alreadyDisabled: false,});+ expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalled();expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {isMaintenanceMode: false,});@@ -114,6 +116,7 @@ describe(CliService.name, () => {}),);+ expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalledTimes(0);expect(mocks.systemMetadata.set).toHaveBeenCalledTimes(0);expect(mocks.event.emit).toHaveBeenCalledTimes(0);});@@ -126,6 +129,7 @@ describe(CliService.name, () => {}),);+ expect(mocks.app.sendOneShotAppRestart).toHaveBeenCalled();expect(mocks.systemMetadata.set).toHaveBeenCalledWith(SystemMetadataKey.MaintenanceMode, {isMaintenanceMode: true,secret: expect.stringMatching(/^\w{128}$/),diff --git a/server/src/services/cli.service.ts b/server/src/services/cli.service.tsindex 3d248edc7a..8d2f1b0e99 100644--- a/server/src/services/cli.service.ts+++ b/server/src/services/cli.service.ts@@ -5,7 +5,7 @@ import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto';import { SystemMetadataKey } from 'src/enum';import { BaseService } from 'src/services/base.service';-import { createMaintenanceLoginUrl, generateMaintenanceSecret, sendOneShotAppRestart } from 'src/utils/maintenance';+import { createMaintenanceLoginUrl, generateMaintenanceSecret } from 'src/utils/maintenance';import { getExternalDomain } from 'src/utils/misc';@Injectable()@@ -55,8 +55,7 @@ export class CliService extends BaseService {const state = { isMaintenanceMode: false as const };await this.systemMetadataRepository.set(SystemMetadataKey.MaintenanceMode, state);-- sendOneShotAppRestart(state);+ await this.appRepository.sendOneShotAppRestart(state);return {alreadyDisabled: false,@@ -89,7 +88,7 @@ export class CliService extends BaseService {secret,});- sendOneShotAppRestart({+ await this.appRepository.sendOneShotAppRestart({isMaintenanceMode: true,});diff --git a/server/src/services/maintenance.service.ts b/server/src/services/maintenance.service.tsindex e6808300bc..0f5fa06957 100644--- a/server/src/services/maintenance.service.ts+++ b/server/src/services/maintenance.service.ts@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';import { OnEvent } from 'src/decorators';import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';import { SystemMetadataKey } from 'src/enum';+import { ArgOf } from 'src/repositories/event.repository';import { BaseService } from 'src/services/base.service';import { MaintenanceModeState } from 'src/types';import { createMaintenanceLoginUrl, generateMaintenanceSecret, signMaintenanceJwt } from 'src/utils/maintenance';@@ -31,7 +32,10 @@ export class MaintenanceService extends BaseService {}@OnEvent({ name: 'AppRestart', server: true })- onRestart(): void {+ onRestart(event: ArgOf<'AppRestart'>, ack?: (ok: 'ok') => void): void {+ this.logger.log(`Restarting due to event... ${JSON.stringify(event)}`);++ ack?.('ok');this.appRepository.exitApp();}diff --git a/server/src/utils/maintenance.ts b/server/src/utils/maintenance.tsindex 22de2e4083..faa92395d6 100644--- a/server/src/utils/maintenance.ts+++ b/server/src/utils/maintenance.ts@@ -1,55 +1,6 @@-import { createAdapter } from '@socket.io/redis-adapter';-import Redis from 'ioredis';import { SignJWT } from 'jose';import { randomBytes } from 'node:crypto';-import { Server as SocketIO } from 'socket.io';import { MaintenanceAuthDto } from 'src/dtos/maintenance.dto';-import { ConfigRepository } from 'src/repositories/config.repository';-import { AppRestartEvent } from 'src/repositories/event.repository';--export function sendOneShotAppRestart(state: AppRestartEvent): void {- const server = new SocketIO();- const { redis } = new ConfigRepository().getEnv();- const pubClient = new Redis(redis);- const subClient = pubClient.duplicate();- server.adapter(createAdapter(pubClient, subClient));-- /**- * Keep trying until we manage to stop Immich- *- * Sometimes there appear to be communication- * issues between to the other servers.- *- * This issue only occurs with this method.- */- async function tryTerminate() {- while (true) {- try {- const responses = await server.serverSideEmitWithAck('AppRestart', state);- if (responses.length > 0) {- return;- }- } catch (error) {- console.error(error);- console.error('Encountered an error while telling Immich to stop.');- }-- console.info(- "\nIt doesn't appear that Immich stopped, trying again in a moment.\nIf Immich is already not running, you can ignore this error.",- );-- await new Promise((r) => setTimeout(r, 1e3));- }- }-- // => corresponds to notification.service.ts#onAppRestart- server.emit('AppRestartV1', state, () => {- void tryTerminate().finally(() => {- pubClient.disconnect();- subClient.disconnect();- });- });-}export async function createMaintenanceLoginUrl(baseUrl: string,