Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:availability): monitor dependencies and handle service outages

johavenf54f182f884f
diff --git a/backend/src/app.bootstrap.ts b/backend/src/app.bootstrap.ts
index 39c52c2a..0592bec1 100644
--- a/backend/src/app.bootstrap.ts
+++ b/backend/src/app.bootstrap.ts
@@ -36,6 +36,8 @@ export async function appBootstrap(): Promise<NestFastifyApplication> {
/* LOGGER */
app.useLogger(IS_TEST_ENV ? ['fatal'] : app.get(Logger))
+ // Flush bootstrap logs through Pino before lifecycle hooks can block app.listen().
+ app.flushLogs()
/* WEBDAV BOOTSTRAP RULES */
bootstrapWebDAV(app, fastifyInstance)
diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts
index 0dc2c4f2..c7d8e68c 100755
--- a/backend/src/app.module.ts
+++ b/backend/src/app.module.ts
@@ -8,6 +8,7 @@ import { ApplicationsModule } from './applications/applications.module'
import { AuthModule } from './authentication/auth.module'
import { configuration, exportConfiguration } from './configuration/config.environment'
import { configLogger } from './configuration/config.logger'
+import { AvailabilityModule } from './infrastructure/availability/availability.module'
import { CacheModule } from './infrastructure/cache/cache.module'
import { ContextModule } from './infrastructure/context/context.module'
import { DatabaseModule } from './infrastructure/database/database.module'
@@ -22,6 +23,7 @@ import { SchedulerModule } from './infrastructure/scheduler/scheduler.module'
pinoHttp: configLogger(configuration.logger)
})
}),
+ AvailabilityModule,
AuthModule,
DatabaseModule,
CacheModule,
diff --git a/backend/src/applications/files/services/files-favorites-queries.e2e-spec.ts b/backend/src/applications/files/services/files-favorites-queries.e2e-spec.ts
index cc526c08..fdca710e 100644
--- a/backend/src/applications/files/services/files-favorites-queries.e2e-spec.ts
+++ b/backend/src/applications/files/services/files-favorites-queries.e2e-spec.ts
@@ -1,6 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing'
import { eq, inArray } from 'drizzle-orm'
import { randomUUID } from 'node:crypto'
+import { AvailabilityModule } from '../../../infrastructure/availability/availability.module'
import type { Cache } from '../../../infrastructure/cache/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DatabaseModule } from '../../../infrastructure/database/database.module'
@@ -54,7 +55,7 @@ describe('Files favorites queries (e2e)', () => {
} as unknown as Cache
beforeAll(async () => {
- module = await Test.createTestingModule({ imports: [DatabaseModule] }).compile()
+ module = await Test.createTestingModule({ imports: [AvailabilityModule, DatabaseModule] }).compile()
module.useLogger(['fatal'])
db = await module.resolve<DBSchema>(DB_TOKEN_PROVIDER)
const filesQueries = new FilesQueries(db)
diff --git a/backend/src/applications/shares/shares.e2e-spec.ts b/backend/src/applications/shares/shares.e2e-spec.ts
index abb563af..732c3290 100644
--- a/backend/src/applications/shares/shares.e2e-spec.ts
+++ b/backend/src/applications/shares/shares.e2e-spec.ts
@@ -1,6 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing'
import { inArray } from 'drizzle-orm'
import { randomUUID } from 'node:crypto'
+import { AvailabilityModule } from '../../infrastructure/availability/availability.module'
import type { Cache } from '../../infrastructure/cache/cache.service'
import { DB_TOKEN_PROVIDER } from '../../infrastructure/database/constants'
import { DatabaseModule } from '../../infrastructure/database/database.module'
@@ -49,7 +50,7 @@ describe('Shares (e2e)', () => {
} as unknown as Cache
beforeAll(async () => {
- module = await Test.createTestingModule({ imports: [DatabaseModule] }).compile()
+ module = await Test.createTestingModule({ imports: [AvailabilityModule, DatabaseModule] }).compile()
module.useLogger(['fatal'])
db = await module.resolve<DBSchema>(DB_TOKEN_PROVIDER)
sharesQueries = new SharesQueries(db, noCache)
diff --git a/backend/src/authentication/auth.controller.ts b/backend/src/authentication/auth.controller.ts
index 8519be72..844e8865 100755
--- a/backend/src/authentication/auth.controller.ts
+++ b/backend/src/authentication/auth.controller.ts
@@ -5,6 +5,7 @@ import { UserHaveRole } from '../applications/users/decorators/roles.decorator'
import { GetUser } from '../applications/users/decorators/user.decorator'
import { UserRolesGuard } from '../applications/users/guards/roles.guard'
import { UserModel } from '../applications/users/models/user.model'
+import { AvailabilitySkip } from '../infrastructure/availability/availability.decorator'
import { AuthManager } from './auth.service'
import { ACCESS_KEY, TOKEN_PATHS } from './constants/auth'
import { AUTH_ROUTE } from './constants/routes'
@@ -39,6 +40,7 @@ export class AuthController {
@Post(AUTH_ROUTE.LOGOUT)
@AuthTokenSkip()
+ @AvailabilitySkip()
logout(@Res({ passthrough: true }) res: FastifyReply) {
return this.authManager.clearCookies(res)
}
diff --git a/backend/src/infrastructure/availability/availability.constants.ts b/backend/src/infrastructure/availability/availability.constants.ts
new file mode 100644
index 00000000..74bc0e1d
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.constants.ts
@@ -0,0 +1,8 @@
+import type { AvailabilityDependency } from './availability.interfaces'
+
+export const INFRASTRUCTURE_CONNECTION_RETRY_DELAY = 6000
+export const INFRASTRUCTURE_DEPENDENCY = {
+ DATABASE: 'database',
+ CACHE: 'cache',
+ WEBSOCKET: 'websocket'
+} as const satisfies Record<string, AvailabilityDependency>
diff --git a/backend/src/infrastructure/availability/availability.decorator.ts b/backend/src/infrastructure/availability/availability.decorator.ts
new file mode 100644
index 00000000..a0971686
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.decorator.ts
@@ -0,0 +1,4 @@
+import { SetMetadata } from '@nestjs/common'
+
+export const AVAILABILITY_SKIP = 'availabilitySkip'
+export const AvailabilitySkip = () => SetMetadata(AVAILABILITY_SKIP, true)
diff --git a/backend/src/infrastructure/availability/availability.guard.spec.ts b/backend/src/infrastructure/availability/availability.guard.spec.ts
new file mode 100644
index 00000000..a56adbe8
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.guard.spec.ts
@@ -0,0 +1,38 @@
+import { createMock, type DeepMocked } from '@golevelup/ts-vitest'
+import { ExecutionContext, ServiceUnavailableException } from '@nestjs/common'
+import { Reflector } from '@nestjs/core'
+import { INFRASTRUCTURE_DEPENDENCY } from './availability.constants'
+import { AvailabilitySkip } from './availability.decorator'
+import { AvailabilityGuard } from './availability.guard'
+import { Availability } from './availability.service'
+
+describe(AvailabilityGuard.name, () => {
+ let availability: Availability
+ let context: DeepMocked<ExecutionContext>
+ let guard: AvailabilityGuard
+
+ beforeEach(() => {
+ availability = new Availability()
+ context = createMock<ExecutionContext>()
+ guard = new AvailabilityGuard(availability, new Reflector())
+ })
+
+ it('allows requests when every registered dependency is available', () => {
+ availability.register(INFRASTRUCTURE_DEPENDENCY.DATABASE, true)
+
+ expect(guard.canActivate(context)).toBe(true)
+ })
+
+ it('returns a service unavailable error when a dependency is unavailable', () => {
+ availability.register(INFRASTRUCTURE_DEPENDENCY.DATABASE)
+
+ expect(() => guard.canActivate(context)).toThrow(new ServiceUnavailableException('Service unavailable'))
+ })
+
+ it('allows an unavailable dependency on an exempted route', () => {
+ availability.register(INFRASTRUCTURE_DEPENDENCY.DATABASE)
+ AvailabilitySkip()(context.getHandler())
+
+ expect(guard.canActivate(context)).toBe(true)
+ })
+})
diff --git a/backend/src/infrastructure/availability/availability.guard.ts b/backend/src/infrastructure/availability/availability.guard.ts
new file mode 100644
index 00000000..b71bd04b
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.guard.ts
@@ -0,0 +1,22 @@
+import { CanActivate, ExecutionContext, Injectable, ServiceUnavailableException } from '@nestjs/common'
+import { Reflector } from '@nestjs/core'
+import { AVAILABILITY_SKIP } from './availability.decorator'
+import { Availability } from './availability.service'
+
+@Injectable()
+export class AvailabilityGuard implements CanActivate {
+ constructor(
+ private readonly availability: Availability,
+ private readonly reflector: Reflector
+ ) {}
+
+ canActivate(context: ExecutionContext): boolean {
+ const availabilitySkip = this.reflector.getAllAndOverride<boolean>(AVAILABILITY_SKIP, [context.getHandler(), context.getClass()])
+ if (availabilitySkip) return true
+
+ if (!this.availability.allAvailable()) {
+ throw new ServiceUnavailableException('Service unavailable')
+ }
+ return true
+ }
+}
diff --git a/backend/src/infrastructure/availability/availability.interfaces.ts b/backend/src/infrastructure/availability/availability.interfaces.ts
new file mode 100644
index 00000000..8e4f193b
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.interfaces.ts
@@ -0,0 +1 @@
+export type AvailabilityDependency = 'database' | 'cache' | 'websocket'
diff --git a/backend/src/infrastructure/availability/availability.md b/backend/src/infrastructure/availability/availability.md
new file mode 100644
index 00000000..fa0de929
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.md
@@ -0,0 +1,21 @@
+# Availability
+
+`AvailabilityModule` tracks the runtime state of infrastructure dependencies such as the database, cache, and WebSocket transport.
+It is global and must be imported once by `AppModule`. Isolated Nest test modules must import it explicitly when they rely on this behavior.
+
+Infrastructure adapters own their connection and retry logic. They register their dependency with `Availability` and update its state when the
+connection is established, lost, or restored:
+
+```typescript
+this.availability.register(INFRASTRUCTURE_DEPENDENCY.DATABASE)
+this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, true)
+```
+
+`AvailabilityGuard` is registered as an application guard. It returns HTTP `503 Service unavailable` while at least one registered dependency is
+unavailable.
+
+Routes that must remain reachable during an outage can use `@AvailabilitySkip()`. This exemption should be limited to endpoints that do not access the
+monitored dependencies, such as `/api/auth/logout`.
+
+`INFRASTRUCTURE_CONNECTION_RETRY_DELAY` provides the shared delay used by infrastructure reconnection loops. The availability module only stores and
+exposes state; it does not reconnect dependencies itself.
diff --git a/backend/src/infrastructure/availability/availability.module.ts b/backend/src/infrastructure/availability/availability.module.ts
new file mode 100644
index 00000000..d78119b5
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.module.ts
@@ -0,0 +1,17 @@
+import { Global, Module } from '@nestjs/common'
+import { APP_GUARD } from '@nestjs/core'
+import { AvailabilityGuard } from './availability.guard'
+import { Availability } from './availability.service'
+
+@Global()
+@Module({
+ providers: [
+ Availability,
+ {
+ provide: APP_GUARD,
+ useClass: AvailabilityGuard
+ }
+ ],
+ exports: [Availability]
+})
+export class AvailabilityModule {}
diff --git a/backend/src/infrastructure/availability/availability.service.ts b/backend/src/infrastructure/availability/availability.service.ts
new file mode 100644
index 00000000..858af800
--- /dev/null
+++ b/backend/src/infrastructure/availability/availability.service.ts
@@ -0,0 +1,28 @@
+import { Injectable } from '@nestjs/common'
+import type { AvailabilityDependency } from './availability.interfaces'
+
+@Injectable()
+export class Availability {
+ private readonly dependencies = new Map<AvailabilityDependency, boolean>()
+
+ register(dependency: AvailabilityDependency, isAvailable = false): void {
+ if (!this.dependencies.has(dependency)) {
+ this.dependencies.set(dependency, isAvailable)
+ }
+ }
+
+ setAvailable(dependency: AvailabilityDependency, isAvailable: boolean): void {
+ this.dependencies.set(dependency, isAvailable)
+ }
+
+ isAvailable(dependency: AvailabilityDependency): boolean {
+ return this.dependencies.get(dependency) === true
+ }
+
+ allAvailable(): boolean {
+ for (const isAvailable of this.dependencies.values()) {
+ if (!isAvailable) return false
+ }
+ return true
+ }
+}
diff --git a/backend/src/infrastructure/cache/adapters/mysql-cache.adapter.ts b/backend/src/infrastructure/cache/adapters/mysql-cache.adapter.ts
index 3c6e73b0..09149d08 100644
--- a/backend/src/infrastructure/cache/adapters/mysql-cache.adapter.ts
+++ b/backend/src/infrastructure/cache/adapters/mysql-cache.adapter.ts
@@ -5,6 +5,8 @@ import { and, between, eq, exists, inArray, notBetween, SQL, sql } from 'drizzle
import cluster from 'node:cluster'
import { createCacheKeySlug, currentTimeStamp } from '../../../common/shared'
import { configuration } from '../../../configuration/config.environment'
+import { INFRASTRUCTURE_DEPENDENCY } from '../../availability/availability.constants'
+import { Availability } from '../../availability/availability.service'
import { DB_TOKEN_PROVIDER } from '../../database/constants'
import { DBSchema } from '../../database/interfaces/database.interface'
import { dbCheckAffectedRows, dbParseJson } from '../../database/utils'
@@ -29,8 +31,11 @@ export class MysqlCacheAdapter implements Cache {
constructor(
@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,
- private readonly scheduler: SchedulerRegistry
- ) {}
+ private readonly scheduler: SchedulerRegistry,
+ availability: Availability
+ ) {
+ availability.register(INFRASTRUCTURE_DEPENDENCY.CACHE, true)
+ }
async onModuleInit(): Promise<void> {
if (cluster.isWorker && process.env[SCHEDULER_ENV] === SCHEDULER_STATE.ENABLED) {
diff --git a/backend/src/infrastructure/cache/adapters/redis-cache.adapter.ts b/backend/src/infrastructure/cache/adapters/redis-cache.adapter.ts
index 00ea691a..065533cb 100644
--- a/backend/src/infrastructure/cache/adapters/redis-cache.adapter.ts
+++ b/backend/src/infrastructure/cache/adapters/redis-cache.adapter.ts
@@ -3,7 +3,9 @@ import { RedisClientOptions } from '@redis/client'
import { createClient, RedisClientType } from 'redis'
import { createCacheKeySlug } from '../../../common/shared'
import { configuration } from '../../../configuration/config.environment'
-import { redactRedisUrl } from '../../utils'
+import { INFRASTRUCTURE_CONNECTION_RETRY_DELAY, INFRASTRUCTURE_DEPENDENCY } from '../../availability/availability.constants'
+import { Availability } from '../../availability/availability.service'
+import { connectionErrorMessage, isRetryableConnectionError, redactRedisUrl } from '../../utils'
import { Cache } from '../cache.service'
import type { CacheRateLimitResult } from '../interfaces/cache-rate-limit.interface'
@@ -14,22 +16,30 @@ export class RedisCacheAdapter implements Cache {
private readonly logger = new Logger(Cache.name.toUpperCase())
private readonly client: RedisClientType
private readonly redactedRedisUrl = redactRedisUrl(configuration.cache.redis)
- private readonly reconnectOptions = { maxAttempts: 3, minConnectDelay: 1000, maxConnectDelay: 2000 }
- constructor() {
+ constructor(private readonly availability: Availability) {
+ this.availability.register(INFRASTRUCTURE_DEPENDENCY.CACHE)
this.client = createClient({
url: configuration.cache.redis,
socket: { noDelay: true, reconnectStrategy: this.reconnectStrategy }
} satisfies RedisClientOptions)
}
- async onModuleInit() {
- this.client.on('error', (e: Error) => this.logger.error(e.message || e))
- this.client.on('ready', () => this.logger.log(`Connected to Redis Server at ${this.redactedRedisUrl}`))
- this.client.connect().catch((e: Error) => this.logger.error(e))
+ async onModuleInit(): Promise<void> {
+ this.client.on('error', (e: Error) => {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.CACHE, this.client.isReady)
+ this.logger.error(e.message || e)
+ })
+ this.client.on('reconnecting', () => this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.CACHE, false))
+ this.client.on('ready', () => {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.CACHE, true)
+ this.logger.log(`Connected to Redis Server at ${this.redactedRedisUrl}`)
+ })
+ await this.client.connect()
}
async onModuleDestroy() {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.CACHE, false)
if (this.client?.isOpen) {
await this.client.close()
}
@@ -160,15 +170,13 @@ export class RedisCacheAdapter implements Cache {
return createCacheKeySlug(args)
}
- private readonly reconnectStrategy = (attempts: number): number => {
- if (attempts > this.reconnectOptions.maxAttempts) {
- this.logger.error('Too many retries on Redis server. Exiting')
- process.exit()
- } else {
- const wait: number = Math.min(this.reconnectOptions.minConnectDelay * Math.pow(2, attempts), this.reconnectOptions.maxConnectDelay)
- this.logger.warn(`Retrying connection to Redis server in ${wait / 1000}s`)
- return wait
+ private readonly reconnectStrategy = (_attempts: number, cause: Error): number | Error => {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.CACHE, false)
+ if (!isRetryableConnectionError(cause)) {
+ return new Error(`Unable to connect to Redis server: ${connectionErrorMessage(cause)}`)
}
+ this.logger.warn(`Retrying connection to Redis server in ${INFRASTRUCTURE_CONNECTION_RETRY_DELAY / 1000}s`)
+ return INFRASTRUCTURE_CONNECTION_RETRY_DELAY
}
private getTTL(ttl: number): number {
diff --git a/backend/src/infrastructure/cache/cache.e2e-spec.ts b/backend/src/infrastructure/cache/cache.e2e-spec.ts
index aa2e267b..641c842e 100644
--- a/backend/src/infrastructure/cache/cache.e2e-spec.ts
+++ b/backend/src/infrastructure/cache/cache.e2e-spec.ts
@@ -1,6 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing'
import { LoggerModule } from 'nestjs-pino'
import { setTimeout } from 'node:timers/promises'
+import { AvailabilityModule } from '../availability/availability.module'
import { DatabaseModule } from '../database/database.module'
import { CacheModule } from './cache.module'
import { Cache } from './cache.service'
@@ -11,7 +12,7 @@ describe(Cache.name, () => {
beforeAll(async () => {
module = await Test.createTestingModule({
- imports: [CacheModule, LoggerModule.forRoot(), DatabaseModule]
+ imports: [AvailabilityModule, CacheModule, LoggerModule.forRoot(), DatabaseModule]
}).compile()
module.useLogger(['fatal'])
diff --git a/backend/src/infrastructure/database/database.module.ts b/backend/src/infrastructure/database/database.module.ts
index 38a5db7d..e9130e6f 100644
--- a/backend/src/infrastructure/database/database.module.ts
+++ b/backend/src/infrastructure/database/database.module.ts
@@ -1,8 +1,11 @@
-import { DrizzleMySqlConfig, DrizzleMySqlModule } from '@knaadh/nestjs-drizzle-mysql2'
-import { BeforeApplicationShutdown, Global, Inject, Module, OnModuleInit } from '@nestjs/common'
-import { MySql2Client } from 'drizzle-orm/mysql2'
-import { Connection, Pool } from 'mysql2'
+import { type DrizzleMySqlConfig, DrizzleMySqlModule } from '@knaadh/nestjs-drizzle-mysql2'
+import { BeforeApplicationShutdown, Global, Inject, Logger, Module, OnModuleInit } from '@nestjs/common'
+import type { Connection, Pool } from 'mysql2'
+import { setTimeout } from 'node:timers/promises'
import { configuration } from '../../configuration/config.environment'
+import { INFRASTRUCTURE_CONNECTION_RETRY_DELAY, INFRASTRUCTURE_DEPENDENCY } from '../availability/availability.constants'
+import { Availability } from '../availability/availability.service'
+import { connectionErrorMessage, isRetryableConnectionError } from '../utils'
import { DB_SESSION_INIT_QUERIES, DB_TOKEN_PROVIDER } from './constants'
import { DatabaseLogger } from './database.logger'
import type { DBSchema } from './interfaces/database.interface'
@@ -13,7 +16,7 @@ import * as schema from './schema'
imports: [
DrizzleMySqlModule.registerAsync({
tag: DB_TOKEN_PROVIDER,
- useFactory: async (): Promise<DrizzleMySqlConfig> => ({
+ useFactory: (): DrizzleMySqlConfig => ({
mysql: {
connection: 'pool',
config: configuration.mysql.url
@@ -28,26 +31,97 @@ import * as schema from './schema'
]
})
export class DatabaseModule implements OnModuleInit, BeforeApplicationShutdown {
- constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema & { session: { client: MySql2Client } }) {}
+ private readonly logger = new Logger(DatabaseModule.name)
+ private readonly pool: Pool
+ private readonly shutdownController = new AbortController()
+ private monitorPromise?: Promise<void>
+ private poolClosed = false
- async onModuleInit() {
- const pool: Pool = (this.db as any).$client
- pool.on('connection', (conn: Connection) => {
+ constructor(
+ @Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema,
+ private readonly availability: Availability
+ ) {
+ this.pool = this.db.$client
+ this.availability.register(INFRASTRUCTURE_DEPENDENCY.DATABASE)
+ }
+
+ async onModuleInit(): Promise<void> {
+ this.pool.on('connection', (conn: Connection) => {
+ conn.on('error', () => this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, false))
for (const query of DB_SESSION_INIT_QUERIES) {
conn.query(query)
}
})
- // Ensure MySQL connection is healthy
+ await this.waitUntilAvailable()
+ if (!this.shutdownController.signal.aborted) {
+ this.monitorPromise = this.monitorAvailability()
+ }
+ }
+
+ async beforeApplicationShutdown(): Promise<void> {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, false)
+ this.shutdownController.abort()
+ await this.monitorPromise
+ await this.closePool()
+ }
+
+ private async waitUntilAvailable(): Promise<void> {
+ while (!this.shutdownController.signal.aborted) {
+ try {
+ await this.pool.promise().query('SELECT 1')
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, true)
+ this.logger.log('Connected to MySQL server')
+ return
+ } catch (error) {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, false)
+ const message = `Unable to connect to MySQL server: ${connectionErrorMessage(error)}`
+ if (!isRetryableConnectionError(error)) {
+ await this.closePool()
+ throw new Error(message)
+ }
+ this.logger.error(message)
+ this.logger.warn(`Retrying connection to MySQL server in ${INFRASTRUCTURE_CONNECTION_RETRY_DELAY / 1000}s`)
+ if (!(await this.waitRetryDelay())) return
+ }
+ }
+ }
+
+ private async monitorAvailability(): Promise<void> {
+ while (await this.waitRetryDelay()) {
+ try {
+ await this.pool.promise().query('SELECT 1')
+ if (!this.availability.isAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE)) {
+ this.logger.log('Connection to MySQL server restored')
+ }
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, true)
+ } catch (error) {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.DATABASE, false)
+ const message = `Connection to MySQL server lost: ${connectionErrorMessage(error)}`
+ this.logger.error(message)
+ if (!isRetryableConnectionError(error)) {
+ this.logger.error('MySQL connection cannot be retried without changing the configuration')
+ return
+ }
+ this.logger.warn(`Retrying connection to MySQL server in ${INFRASTRUCTURE_CONNECTION_RETRY_DELAY / 1000}s`)
+ }
+ }
+ }
+
+ private async waitRetryDelay(): Promise<boolean> {
try {
- await pool.promise().query('SELECT 1')
+ await setTimeout(INFRASTRUCTURE_CONNECTION_RETRY_DELAY, undefined, { signal: this.shutdownController.signal })
+ return true
} catch (error) {
- await this.db.session.client.end()
+ if ((error as Error).name === 'AbortError') return false
throw error
}
}
- async beforeApplicationShutdown() {
- await this.db.session.client.end()
+ private async closePool(): Promise<void> {
+ if (!this.poolClosed) {
+ this.poolClosed = true
+ await this.pool.promise().end()
+ }
}
}
diff --git a/backend/src/infrastructure/database/interfaces/database.interface.ts b/backend/src/infrastructure/database/interfaces/database.interface.ts
index c82619ed..8e1ec442 100644
--- a/backend/src/infrastructure/database/interfaces/database.interface.ts
+++ b/backend/src/infrastructure/database/interfaces/database.interface.ts
@@ -1,4 +1,5 @@
-import { MySql2Database } from 'drizzle-orm/mysql2'
+import type { MySql2Database } from 'drizzle-orm/mysql2'
+import type { Pool } from 'mysql2'
import * as schema from '../schema'
-export type DBSchema = MySql2Database<typeof schema>
+export type DBSchema = MySql2Database<typeof schema> & { $client: Pool }
diff --git a/backend/src/infrastructure/utils.spec.ts b/backend/src/infrastructure/utils.spec.ts
index 7d155d5e..dfc8e4b7 100644
--- a/backend/src/infrastructure/utils.spec.ts
+++ b/backend/src/infrastructure/utils.spec.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
-import { redactRedisUrl } from './utils'
+import { connectionErrorMessage, isRetryableConnectionError, redactRedisUrl } from './utils'
describe(redactRedisUrl.name, () => {
it.each([
@@ -11,3 +11,31 @@ describe(redactRedisUrl.name, () => {
expect(redactRedisUrl(url)).toBe(expected)
})
})
+
+describe(isRetryableConnectionError.name, () => {
+ it('recognizes retryable direct and aggregate connection errors', () => {
+ const refused = connectionError('ECONNREFUSED')
+ const timedOut = connectionError('ETIMEDOUT')
+
+ expect(isRetryableConnectionError(refused)).toBe(true)
+ expect(isRetryableConnectionError(new AggregateError([refused, timedOut]))).toBe(true)
+ })
+
+ it('rejects unknown and mixed aggregate errors', () => {
+ const refused = connectionError('ECONNREFUSED')
+ const invalidConfiguration = connectionError('ERR_INVALID_URL')
+
+ expect(isRetryableConnectionError(invalidConfiguration)).toBe(false)
+ expect(isRetryableConnectionError(new AggregateError([refused, invalidConfiguration]))).toBe(false)
+ })
+
+ it('formats aggregate connection errors for logs', () => {
+ const error = new AggregateError([new Error('first'), new Error('second')])
+
+ expect(connectionErrorMessage(error)).toBe('Error: first, Error: second')
+ })
+})
+
+function connectionError(code: string): Error & { code: string } {
+ return Object.assign(new Error(code), { code })
+}
diff --git a/backend/src/infrastructure/utils.ts b/backend/src/infrastructure/utils.ts
index c53ad5ba..c6f67554 100644
--- a/backend/src/infrastructure/utils.ts
+++ b/backend/src/infrastructure/utils.ts
@@ -5,3 +5,38 @@ export function redactRedisUrl(url: string): string {
parsedUrl.password = '********'
return parsedUrl.toString()
}
+
+const RETRYABLE_CONNECTION_ERROR_CODES = new Set([
+ 'EAI_AGAIN',
+ 'ECONNABORTED',
+ 'ECONNREFUSED',
+ 'ECONNRESET',
+ 'EHOSTUNREACH',
+ 'ENETUNREACH',
+ 'ENOTFOUND',
+ 'EPIPE',
+ 'ER_CON_COUNT_ERROR',
+ 'ER_SERVER_SHUTDOWN',
+ 'ETIMEDOUT',
+ 'PROTOCOL_CONNECTION_LOST'
+])
+
+export function isRetryableConnectionError(error: unknown): boolean {
+ const errors = connectionErrors(error)
+ return errors.length > 0 && errors.every((currentError) => isRetryableError(currentError))
+}
+
+export function connectionErrorMessage(error: unknown): string {
+ return connectionErrors(error).map(String).join(', ')
+}
+
+function connectionErrors(error: unknown): unknown[] {
+ const errors = (error as { errors?: unknown[] })?.errors
+ return Array.isArray(errors) ? errors : [error]
+}
+
+function isRetryableError(error: unknown): boolean {
+ const code = (error as { code?: string })?.code
+ const message = error instanceof Error ? error.message : String(error)
+ return (!!code && RETRYABLE_CONNECTION_ERROR_CODES.has(code)) || message.includes('Socket closed unexpectedly')
+}
diff --git a/backend/src/infrastructure/websocket/adapters/redis.adapter.ts b/backend/src/infrastructure/websocket/adapters/redis.adapter.ts
index e0985737..ce670a11 100644
--- a/backend/src/infrastructure/websocket/adapters/redis.adapter.ts
+++ b/backend/src/infrastructure/websocket/adapters/redis.adapter.ts
@@ -1,43 +1,82 @@
import { Logger } from '@nestjs/common'
+import { NestFastifyApplication } from '@nestjs/platform-fastify'
import { IoAdapter } from '@nestjs/platform-socket.io'
+import type { createAdapter as createRedisAdapter } from '@socket.io/redis-adapter'
+import type { RedisClientType } from 'redis'
import { ServerOptions } from 'socket.io'
import { loadOptionalModule } from '../../../common/functions'
-import { redactRedisUrl } from '../../utils'
+import { INFRASTRUCTURE_CONNECTION_RETRY_DELAY, INFRASTRUCTURE_DEPENDENCY } from '../../availability/availability.constants'
+import { Availability } from '../../availability/availability.service'
+import { connectionErrorMessage, isRetryableConnectionError, redactRedisUrl } from '../../utils'
import type { WebSocketConfig } from '../web-socket.config'
export class RedisAdapter extends IoAdapter {
private readonly logger = new Logger('WebSocketAdapter')
- private adapterConstructor: any
- private readonly reconnectOptions = { maxAttempts: 3, minConnectDelay: 6000, maxConnectDelay: 30000 }
- private readonly reconnectStrategy = (attempts: number): number => {
- if (attempts > this.reconnectOptions.maxAttempts) {
- this.logger.error('Too many retries on Redis server. Exiting')
- process.exit()
- } else {
- const wait: number = Math.min(this.reconnectOptions.minConnectDelay * Math.pow(2, attempts), this.reconnectOptions.maxConnectDelay)
- this.logger.warn(`Retrying connection to Redis server in ${wait / 1000}s`)
- return wait
+ private readonly availability: Availability
+ private adapterConstructor: ReturnType<typeof createRedisAdapter>
+ private pubClient?: RedisClientType
+ private subClient?: RedisClientType
+
+ constructor(app: NestFastifyApplication) {
+ super(app)
+ this.availability = app.get(Availability)
+ this.availability.register(INFRASTRUCTURE_DEPENDENCY.WEBSOCKET)
+ }
+
+ private readonly reconnectStrategy = (_attempts: number, cause: Error): number | Error => {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.WEBSOCKET, false)
+ if (!isRetryableConnectionError(cause)) {
+ return new Error(`Unable to connect to Redis WebSocket server: ${connectionErrorMessage(cause)}`)
}
+ this.logger.warn(`Retrying connection to Redis WebSocket server in ${INFRASTRUCTURE_CONNECTION_RETRY_DELAY / 1000}s`)
+ return INFRASTRUCTURE_CONNECTION_RETRY_DELAY
}
- async connectToRedis(redisUrl: WebSocketConfig['redis']): Promise<void> {
+ async connect(redisUrl: WebSocketConfig['redis']): Promise<void> {
const redactedRedisUrl = redactRedisUrl(redisUrl)
const { createAdapter } = await loadOptionalModule('@socket.io/redis-adapter')
const { createClient } = await loadOptionalModule('redis')
const pubClient = createClient({ url: redisUrl, socket: { noDelay: true, reconnectStrategy: this.reconnectStrategy } })
const subClient = pubClient.duplicate()
- pubClient.on('error', (e: Error) => this.logger.error(`PubClient: ${e.message || e}`))
- pubClient.on('ready', () => this.logger.log(`PubClient: Connected to Redis Server at ${redactedRedisUrl}`))
- subClient.on('error', (e: Error) => this.logger.error(`SubClient: ${e.message || e}`))
- subClient.on('ready', () => this.logger.log(`SubClient: Connected to Redis Server at ${redactedRedisUrl}`))
- pubClient.connect()
- subClient.connect()
+ this.pubClient = pubClient
+ this.subClient = subClient
+ this.registerClientEvents(pubClient, 'PubClient', redactedRedisUrl)
+ this.registerClientEvents(subClient, 'SubClient', redactedRedisUrl)
+ try {
+ await Promise.all([pubClient.connect(), subClient.connect()])
+ } catch (error) {
+ await this.disconnect()
+ throw error
+ }
+ this.updateAvailability()
this.adapterConstructor = createAdapter(pubClient, subClient)
}
+ async disconnect(): Promise<void> {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.WEBSOCKET, false)
+ const clients = [this.pubClient, this.subClient].filter((client): client is RedisClientType => !!client?.isOpen)
+ await Promise.all(clients.map((client) => client.close()))
+ }
+
createIOServer(port: number, options?: ServerOptions): any {
const server = super.createIOServer(port, options)
server.adapter(this.adapterConstructor)
return server
}
+
+ private registerClientEvents(client: RedisClientType, name: 'PubClient' | 'SubClient', redactedRedisUrl: string): void {
+ client.on('error', (error: Error) => {
+ this.updateAvailability()
+ this.logger.error(`${name}: ${error.message || error}`)
+ })
+ client.on('reconnecting', () => this.updateAvailability())
+ client.on('ready', () => {
+ this.updateAvailability()
+ this.logger.log(`${name}: Connected to Redis Server at ${redactedRedisUrl}`)
+ })
+ }
+
+ private updateAvailability(): void {
+ this.availability.setAvailable(INFRASTRUCTURE_DEPENDENCY.WEBSOCKET, this.pubClient?.isReady === true && this.subClient?.isReady === true)
+ }
}
diff --git a/backend/src/infrastructure/websocket/adapters/web-socket.adapter.ts b/backend/src/infrastructure/websocket/adapters/web-socket.adapter.ts
index 2f23cf95..adaaf0f4 100644
--- a/backend/src/infrastructure/websocket/adapters/web-socket.adapter.ts
+++ b/backend/src/infrastructure/websocket/adapters/web-socket.adapter.ts
@@ -52,15 +52,21 @@ export class WebSocketAdapter extends IoAdapter {
return server
}
+ async dispose(): Promise<void> {
+ if (this.adapter instanceof RedisAdapter) {
+ await this.adapter.disconnect()
+ }
+ }
+
async getAdapter() {
if (configuration.websocket.adapter === 'redis') {
try {
const redisIoAdapter = new RedisAdapter(this.app)
- await redisIoAdapter.connectToRedis(configuration.websocket.redis)
+ await redisIoAdapter.connect(configuration.websocket.redis)
return redisIoAdapter
} catch (e) {
this.logger.error(e.message)
- process.exit(1)
+ throw e
}
} else {
if (cluster.isWorker) {
diff --git a/backend/src/main.ts b/backend/src/main.ts
index 181487f4..5ae6b0bb 100755
--- a/backend/src/main.ts
+++ b/backend/src/main.ts
@@ -7,7 +7,7 @@ import { AppService } from './app.service'
import { configuration } from './configuration/config.environment'
async function bootstrap(): Promise<void> {
- let logger: Logger
+ let logger: Logger | undefined
try {
const app: NestFastifyApplication = await appBootstrap()
logger = app.get<Logger>(Logger)
@@ -31,8 +31,15 @@ async function bootstrap(): Promise<void> {
}
)
} catch (e) {
- logger.error(`Bootstrap failed: ${e?.errors || e}`, 'BOOTSTRAP')
- logger.error(`${configuration.server.restartOnFailure ? 'Retrying' : 'Exiting'} ...`, 'BOOTSTRAP')
+ const failureMessage = `Bootstrap failed: ${e?.errors || e}`
+ const nextActionMessage = `${configuration.server.restartOnFailure ? 'Retrying' : 'Exiting'} ...`
+ if (logger) {
+ logger.error(failureMessage, 'BOOTSTRAP')
+ logger.error(nextActionMessage, 'BOOTSTRAP')
+ } else {
+ console.error(`[BOOTSTRAP] ${failureMessage}`)
+ console.error(`[BOOTSTRAP] ${nextActionMessage}`)
+ }
await setTimeout(6000)
throw e
}
diff --git a/frontend/src/app/app.constants.ts b/frontend/src/app/app.constants.ts
index 07da5351..aeb93d8c 100644
--- a/frontend/src/app/app.constants.ts
+++ b/frontend/src/app/app.constants.ts
@@ -17,4 +17,5 @@ export const APP_MENU: AppMenu = {
submenus: []
} as const
-export const SERVER_CONNECTION_ERROR = 'Server connection error'
+export const SERVICE_INTERRUPTION_ERROR = 'Service interruption'
+export const SERVICE_UNAVAILABLE_ERROR = 'Service unavailable'
diff --git a/frontend/src/app/applications/spaces/components/spaces-browser.component.html b/frontend/src/app/applications/spaces/components/spaces-browser.component.html
index b95fc52b..c1391f89 100644
--- a/frontend/src/app/applications/spaces/components/spaces-browser.component.html
+++ b/frontend/src/app/applications/spaces/components/spaces-browser.component.html
@@ -574,9 +574,9 @@
<div class="fs-sm text-center text-danger text-bold no-select p-4">
<span l10nTranslate>Location not found</span>
</div>
- } @else if (serverConnectionError) {
+ } @else if (serviceError) {
<div class="fs-sm text-center text-danger text-bold no-select p-4">
- <span l10nTranslate>{{ serverConnectionErrorMessage }}</span>
+ <span l10nTranslate>{{ serviceErrorMsg }}</span>
</div>
} @else if (!isTrashRepo && !loading && !stats.elements) {
<div class="fs-base text-center text-gray fw-bold no-select p-4">
diff --git a/frontend/src/app/applications/spaces/components/spaces-browser.component.ts b/frontend/src/app/applications/spaces/components/spaces-browser.component.ts
index 42ed1998..6491ab1e 100644
--- a/frontend/src/app/applications/spaces/components/spaces-browser.component.ts
+++ b/frontend/src/app/applications/spaces/components/spaces-browser.component.ts
@@ -47,7 +47,7 @@ import { BsModalRef } from 'ngx-bootstrap/modal'
import { TooltipModule } from 'ngx-bootstrap/tooltip'
import { Subscription } from 'rxjs'
import { take } from 'rxjs/operators'
-import { SERVER_CONNECTION_ERROR } from '../../../app.constants'
+import { SERVICE_UNAVAILABLE_ERROR } from '../../../app.constants'
import { BadgePermissionsComponent } from '../../../common/components/badge-permissions.component'
import { FilterComponent } from '../../../common/components/filter.component'
import { NavigationViewComponent, ViewMode } from '../../../common/components/navigation-view/navigation-view.component'
@@ -168,8 +168,8 @@ export class SpacesBrowserComponent implements OnInit, AfterViewInit, OnDestroy
protected loading = false
protected locationNotFound = false
protected forbiddenResource = false
- protected serverConnectionError = false
- protected serverConnectionErrorMessage = SERVER_CONNECTION_ERROR
+ protected serviceError = false
+ protected serviceErrorMsg = SERVICE_UNAVAILABLE_ERROR
protected isFilesRepo: boolean
protected isSharesRepo: boolean
protected isTrashRepo: boolean
@@ -313,7 +313,7 @@ export class SpacesBrowserComponent implements OnInit, AfterViewInit, OnDestroy
this.loading = true
this.forbiddenResource = false
this.locationNotFound = false
- this.serverConnectionError = false
+ this.serviceError = false
this.inputFilter.clear()
this.resetFilesSelection()
this.spacesBrowser.loadFiles().subscribe({
@@ -350,8 +350,8 @@ export class SpacesBrowserComponent implements OnInit, AfterViewInit, OnDestroy
if (e.status !== 401) {
this.forbiddenResource = e.status === 403
this.locationNotFound = e.status === 404
- this.serverConnectionError = e.status === 0
- this.layout.sendNotification('error', 'Files', e.error?.message || this.serverConnectionErrorMessage)
+ this.serviceError = e.status === 0
+ this.layout.sendNotification('error', 'Files', e.error?.message || this.serviceErrorMsg)
}
this.loading = false
}
diff --git a/frontend/src/app/auth/auth.interceptor.ts b/frontend/src/app/auth/auth.interceptor.ts
index 39a42088..7027c898 100644
--- a/frontend/src/app/auth/auth.interceptor.ts
+++ b/frontend/src/app/auth/auth.interceptor.ts
@@ -1,23 +1,27 @@
import { HttpErrorResponse, HttpEvent, HttpHandler, HttpInterceptor, HttpRequest } from '@angular/common/http'
import { inject, Injectable, Injector } from '@angular/core'
+import { API_ADMIN_IMPERSONATE_LOGOUT } from '@sync-in-server/backend/src/applications/users/constants/routes'
import { API_AUTH_LOGIN, API_AUTH_LOGOUT, API_AUTH_REFRESH } from '@sync-in-server/backend/src/authentication/constants/routes'
-import { BehaviorSubject, concatMap, delay, Observable, of, retryWhen, throwError } from 'rxjs'
+import { BehaviorSubject, Observable, retry, throwError, timer } from 'rxjs'
import { catchError, filter, finalize, switchMap, take } from 'rxjs/operators'
-import { SERVER_CONNECTION_ERROR } from '../app.constants'
+import { SERVICE_UNAVAILABLE_ERROR } from '../app.constants'
import { hasReservedUrlChars } from '../common/utils/functions'
import { AuthService } from './auth.service'
-import { API_ADMIN_IMPERSONATE_LOGOUT } from '@sync-in-server/backend/src/applications/users/constants/routes'
@Injectable({
providedIn: 'root'
})
export class AuthInterceptor implements HttpInterceptor {
private readonly injector = inject(Injector)
- private auth: AuthService | null = null
+ private _auth?: AuthService
private isRefreshingToken = false
- private waitForRefreshToken: BehaviorSubject<boolean> = new BehaviorSubject<boolean>(false)
- private retryCount = 3
- private retryWaitMilliSeconds = 2000
+ private readonly waitForRefreshToken = new BehaviorSubject<boolean>(false)
+ private readonly retryCount = 3
+ private readonly retryWaitMilliSeconds = 2000
+
+ private get auth(): AuthService {
+ return (this._auth ??= this.injector.get(AuthService))
+ }
intercept(request: HttpRequest<any>, next: HttpHandler): Observable<HttpEvent<any>> {
const encodedUrl = hasReservedUrlChars(request.url)
@@ -26,26 +30,41 @@ export class AuthInterceptor implements HttpInterceptor {
}
return next.handle(request).pipe(
+ retry({
+ count: this.retryCount,
+ delay: (error: HttpErrorResponse) => {
+ if (error.status !== 0 || request.body instanceof FormData || request.url === API_AUTH_LOGOUT) {
+ return throwError(() => error)
+ }
+ return timer(this.retryWaitMilliSeconds)
+ }
+ }),
catchError((e: HttpErrorResponse) => {
if (e.status === 401) {
return this.handleAuthorizationError(request, next, e)
+ } else if (e.status === 503) {
+ return this.handleServiceUnavailable(request, e)
} else if (e.status === 0) {
// Do not retry multipart uploads on a transport error: the connection may have
// been closed after a quota or size rejection, and replaying would resend all files.
- if (request.body instanceof FormData) {
+ if (request.body instanceof FormData || request.url === API_AUTH_LOGOUT) {
return throwError(() => e)
}
- return this.handleRetries(request, next, e)
+ return this.handleServiceUnavailable(request, e, SERVICE_UNAVAILABLE_ERROR)
}
return throwError(() => e)
})
)
}
- private handleAuthorizationError(request: HttpRequest<any>, next: HttpHandler, error: HttpErrorResponse): Observable<any> {
- if (!this.auth) {
- this.auth = this.injector.get(AuthService)
+ private handleServiceUnavailable(request: HttpRequest<any>, error: HttpErrorResponse, message?: string): Observable<never> {
+ if (request.url !== API_AUTH_LOGOUT) {
+ this.auth.logout(true, false, message ?? error.error?.message ?? SERVICE_UNAVAILABLE_ERROR)
}
+ return throwError(() => error)
+ }
+
+ private handleAuthorizationError(request: HttpRequest<any>, next: HttpHandler, error: HttpErrorResponse): Observable<any> {
console.debug('AuthInterceptor:', request.url, error.status)
if ([API_AUTH_REFRESH, API_AUTH_LOGIN, API_AUTH_LOGOUT, API_ADMIN_IMPERSONATE_LOGOUT].indexOf(request.url) === -1) {
if (this.isRefreshingToken) {
@@ -70,23 +89,4 @@ export class AuthInterceptor implements HttpInterceptor {
}
return throwError(() => error)
}
-
- private handleRetries(request: HttpRequest<any>, next: HttpHandler, _error: HttpErrorResponse): Observable<any> {
- return next.handle(request).pipe(
- retryWhen((error) =>
- error.pipe(
- concatMap((error, count) => {
- if (count < this.retryCount) {
- return of(error)
- }
- if (error.status === 0) {
- error.message = SERVER_CONNECTION_ERROR
- }
- return throwError(() => error)
- }),
- delay(this.retryWaitMilliSeconds)
- )
- )
- )
- }
}
diff --git a/frontend/src/app/auth/auth.service.ts b/frontend/src/app/auth/auth.service.ts
index 2972f4a1..8cab7628 100644
--- a/frontend/src/app/auth/auth.service.ts
+++ b/frontend/src/app/auth/auth.service.ts
@@ -24,6 +24,7 @@ import type { TwoFaResponseDto, TwoFaVerifyDto } from '@sync-in-server/backend/s
import { currentTimeStamp } from '@sync-in-server/backend/src/common/shared'
import { catchError, finalize, map, Observable, of, throwError } from 'rxjs'
import { switchMap, tap } from 'rxjs/operators'
+import { SERVICE_INTERRUPTION_ERROR } from '../app.constants'
import { USER_PATH } from '../applications/users/user.constants'
import { UserService } from '../applications/users/user.service'
import { getCookie } from '../common/utils/functions'
@@ -101,8 +102,8 @@ export class AuthService {
)
}
- logout(redirect = true, expired = false) {
- if ((redirect || expired) && this.store.userImpersonate()) {
+ logout(redirect = true, expired = false, errorMsg?: string) {
+ if (!errorMsg && (redirect || expired) && this.store.userImpersonate()) {
this.logoutImpersonateUser()
return
}
@@ -117,12 +118,14 @@ export class AuthService {
if (redirect) {
this.router.navigate([AUTH_PATHS.BASE, AUTH_PATHS.LOGIN]).catch(console.error)
}
- if (expired) {
+ if (errorMsg) {
+ this.layout.sendNotification('error', SERVICE_INTERRUPTION_ERROR, errorMsg)
+ } else if (expired) {
this.layout.sendNotification('warning', 'Session has expired', 'Please sign in')
}
})
)
- .subscribe()
+ .subscribe({ error: (error: HttpErrorResponse) => console.error(error) })
}
initUserFromResponse(r: LoginResponseDto, impersonate = false) {
diff --git a/frontend/src/i18n/de.json b/frontend/src/i18n/de.json
index 62ff11b2..fe5b0841 100644
--- a/frontend/src/i18n/de.json
+++ b/frontend/src/i18n/de.json
@@ -20,6 +20,8 @@
"Close": "Schließen",
"Open": "Öffnen",
"Server connection error": "Fehler bei der Serververbindung",
+ "Service interruption": "Dienstunterbrechung",
+ "Service unavailable": "Dienst nicht verfügbar",
"Filter": "Filtern",
"Filters": "Filter",
"search": "suchen",
diff --git a/frontend/src/i18n/es.json b/frontend/src/i18n/es.json
index 99ad77f3..fe24a4ee 100644
--- a/frontend/src/i18n/es.json
+++ b/frontend/src/i18n/es.json
@@ -20,6 +20,8 @@
"Close": "Cerrar",
"Open": "Abrir",
"Server connection error": "Error de conexión con el servidor",
+ "Service interruption": "Interrupción del servicio",
+ "Service unavailable": "Servicio no disponible",
"Filter": "Filtrar",
"Filters": "Filtros",
"search": "buscar",
diff --git a/frontend/src/i18n/fr.json b/frontend/src/i18n/fr.json
index e8cc1d3c..b5b23b16 100644
--- a/frontend/src/i18n/fr.json
+++ b/frontend/src/i18n/fr.json
@@ -20,6 +20,8 @@
"Close": "Fermer",
"Open": "Ouvrir",
"Server connection error": "Erreur de connexion au serveur",
+ "Service interruption": "Interruption de service",
+ "Service unavailable": "Service indisponible",
"Filter": "Filtrer",
"Filters": "Filtres",
"search": "rechercher",
diff --git a/frontend/src/i18n/hi.json b/frontend/src/i18n/hi.json
index 9d8bc982..362992ce 100644
--- a/frontend/src/i18n/hi.json
+++ b/frontend/src/i18n/hi.json
@@ -20,6 +20,8 @@
"Close": "बंद करें",
"Open": "खोलें",
"Server connection error": "सर्वर कनेक्शन त्रुटि",
+ "Service interruption": "सेवा में व्यवधान",
+ "Service unavailable": "सेवा उपलब्ध नहीं है",
"Filter": "फ़िल्टर",
"Filters": "फ़िल्टर",
"search": "खोजें",
diff --git a/frontend/src/i18n/it.json b/frontend/src/i18n/it.json
index 889fb926..b80fecec 100644
--- a/frontend/src/i18n/it.json
+++ b/frontend/src/i18n/it.json
@@ -20,6 +20,8 @@
"Close": "Chiudi",
"Open": "Apri",
"Server connection error": "Errore di connessione al server",
+ "Service interruption": "Interruzione del servizio",
+ "Service unavailable": "Servizio non disponibile",
"Filter": "Filtra",
"Filters": "Filtri",
"search": "cerca",
diff --git a/frontend/src/i18n/ja.json b/frontend/src/i18n/ja.json
index ec83c1b3..4feaf77f 100644
--- a/frontend/src/i18n/ja.json
+++ b/frontend/src/i18n/ja.json
@@ -20,6 +20,8 @@
"Close": "閉じる",
"Open": "開く",
"Server connection error": "サーバー接続エラー",
+ "Service interruption": "サービスの中断",
+ "Service unavailable": "サービスを利用できません",
"Filter": "フィルター",
"Filters": "フィルター",
"search": "検索",
diff --git a/frontend/src/i18n/ko.json b/frontend/src/i18n/ko.json
index c40c64ed..0f8b4f81 100644
--- a/frontend/src/i18n/ko.json
+++ b/frontend/src/i18n/ko.json
@@ -20,6 +20,8 @@
"Close": "닫기",
"Open": "열기",
"Server connection error": "서버 연결 오류",
+ "Service interruption": "서비스 중단",
+ "Service unavailable": "서비스를 사용할 수 없습니다",
"Filter": "필터",
"Filters": "필터",
"search": "검색",
diff --git a/frontend/src/i18n/nl.json b/frontend/src/i18n/nl.json
index 5612ce42..b2048071 100644
--- a/frontend/src/i18n/nl.json
+++ b/frontend/src/i18n/nl.json
@@ -20,6 +20,8 @@
"Close": "Sluiten",
"Open": "Openen",
"Server connection error": "Fout met serververbinding",
+ "Service interruption": "Dienstonderbreking",
+ "Service unavailable": "Dienst niet beschikbaar",
"Filter": "Filteren",
"Filters": "Filters",
"search": "zoeken",
diff --git a/frontend/src/i18n/pl.json b/frontend/src/i18n/pl.json
index 0c3fbbd9..e77b60b7 100644
--- a/frontend/src/i18n/pl.json
+++ b/frontend/src/i18n/pl.json
@@ -20,6 +20,8 @@
"Close": "Zamknij",
"Open": "Otwórz",
"Server connection error": "Błąd połączenia z serwerem",
+ "Service interruption": "Przerwa w działaniu usługi",
+ "Service unavailable": "Usługa niedostępna",
"Filter": "Filtr",
"Filters": "Filtry",
"search": "szukaj",
diff --git a/frontend/src/i18n/pt-BR.json b/frontend/src/i18n/pt-BR.json
index c0442041..598fa6fc 100644
--- a/frontend/src/i18n/pt-BR.json
+++ b/frontend/src/i18n/pt-BR.json
@@ -20,6 +20,8 @@
"Close": "Fechar",
"Open": "Abrir",
"Server connection error": "Erro de conexão com o servidor",
+ "Service interruption": "Interrupção do serviço",
+ "Service unavailable": "Serviço indisponível",
"Filter": "Filtrar",
"Filters": "Filtros",
"search": "pesquisar",
diff --git a/frontend/src/i18n/pt.json b/frontend/src/i18n/pt.json
index d7d135f9..00a150f3 100644
--- a/frontend/src/i18n/pt.json
+++ b/frontend/src/i18n/pt.json
@@ -20,6 +20,8 @@
"Close": "Fechar",
"Open": "Abrir",
"Server connection error": "Erro de conexão com o servidor",
+ "Service interruption": "Interrupção do serviço",
+ "Service unavailable": "Serviço indisponível",
"Filter": "Filtrar",
"Filters": "Filtros",
"search": "pesquisar",
diff --git a/frontend/src/i18n/ru.json b/frontend/src/i18n/ru.json
index 902c4a5b..a034c805 100644
--- a/frontend/src/i18n/ru.json
+++ b/frontend/src/i18n/ru.json
@@ -20,6 +20,8 @@
"Close": "Закрыть",
"Open": "Открыть",
"Server connection error": "Ошибка подключения к серверу",
+ "Service interruption": "Перерыв в работе сервиса",
+ "Service unavailable": "Сервис недоступен",
"Filter": "Фильтровать",
"Filters": "Фильтры",
"search": "поиск",
diff --git a/frontend/src/i18n/tr.json b/frontend/src/i18n/tr.json
index 005344cd..392bd394 100644
--- a/frontend/src/i18n/tr.json
+++ b/frontend/src/i18n/tr.json
@@ -20,6 +20,8 @@
"Close": "Kapat",
"Open": "Aç",
"Server connection error": "Sunucu bağlantı hatası",
+ "Service interruption": "Hizmet kesintisi",
+ "Service unavailable": "Hizmet kullanılamıyor",
"Filter": "Filtre",
"Filters": "Filtreler",
"search": "ara",
diff --git a/frontend/src/i18n/zh.json b/frontend/src/i18n/zh.json
index e810deb5..9184db88 100644
--- a/frontend/src/i18n/zh.json
+++ b/frontend/src/i18n/zh.json
@@ -20,6 +20,8 @@
"Close": "关闭",
"Open": "打开",
"Server connection error": "服务器连接错误",
+ "Service interruption": "服务中断",
+ "Service unavailable": "服务不可用",
"Filter": "筛选",
"Filters": "筛选器",
"search": "搜索",