High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor(server): db env (#13167)
Jason Rasmussen3ac00b0ffaa5
diff --git a/server/src/database.config.ts b/server/src/database.config.tsindex 9cc317a734..2a46067bc1 100644--- a/server/src/database.config.ts+++ b/server/src/database.config.ts@@ -1,16 +1,17 @@-import { DatabaseExtension } from 'src/interfaces/database.interface';+import { ConfigRepository } from 'src/repositories/config.repository';import { DataSource } from 'typeorm';import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';-const url = process.env.DB_URL;+const { database } = new ConfigRepository().getEnv();+const { url, host, port, username, password, name } = database;const urlOrParts = url? { url }: {- host: process.env.DB_HOSTNAME || 'database',- port: Number.parseInt(process.env.DB_PORT || '5432'),- username: process.env.DB_USERNAME || 'postgres',- password: process.env.DB_PASSWORD || 'postgres',- database: process.env.DB_DATABASE_NAME || 'immich',+ host,+ port,+ username,+ password,+ database: name,};/* eslint unicorn/prefer-module: "off" -- We can fix this when migrating to ESM*/@@ -32,6 +33,3 @@ export const databaseConfig: PostgresConnectionOptions = {* this export is ONLY to be used for TypeORM commands in package.json#scripts*/export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' });--export const getVectorExtension = () =>- process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.tsindex bad4a83222..23a3803284 100644--- a/server/src/interfaces/config.interface.ts+++ b/server/src/interfaces/config.interface.ts@@ -26,6 +26,12 @@ export interface EnvData {};database: {+ url?: string;+ host: string;+ port: number;+ username: string;+ password: string;+ name: string;skipMigrations: boolean;vectorExtension: VectorExtension;};diff --git a/server/src/migrations/1700713871511-UsePgVectors.ts b/server/src/migrations/1700713871511-UsePgVectors.tsindex 033e2ba9ad..e67c7275a7 100644--- a/server/src/migrations/1700713871511-UsePgVectors.ts+++ b/server/src/migrations/1700713871511-UsePgVectors.ts@@ -1,13 +1,15 @@-import { getVectorExtension } from 'src/database.config';+import { ConfigRepository } from 'src/repositories/config.repository';import { getCLIPModelInfo } from 'src/utils/misc';import { MigrationInterface, QueryRunner } from 'typeorm';+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;+export class UsePgVectors1700713871511 implements MigrationInterface {name = 'UsePgVectors1700713871511';public async up(queryRunner: QueryRunner): Promise<void> {await queryRunner.query(`SET search_path TO "$user", public, vectors`);- await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`);+ await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExtension}`);const faceDimQuery = await queryRunner.query(`SELECT CARDINALITY(embedding::real[]) as dimsizeFROM asset_facesdiff --git a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.tsindex e325f270fd..f9ea5a0dc3 100644--- a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts+++ b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts@@ -1,12 +1,14 @@-import { getVectorExtension } from 'src/database.config';import { DatabaseExtension } from 'src/interfaces/database.interface';+import { ConfigRepository } from 'src/repositories/config.repository';import { MigrationInterface, QueryRunner } from 'typeorm';+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;+export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {name = 'AddCLIPEmbeddingIndex1700713994428';public async up(queryRunner: QueryRunner): Promise<void> {- if (getVectorExtension() === DatabaseExtension.VECTORS) {+ if (vectorExtension === DatabaseExtension.VECTORS) {await queryRunner.query(`SET vectors.pgvector_compatibility=on`);}await queryRunner.query(`SET search_path TO "$user", public, vectors`);diff --git a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.tsindex bc6bad6dbd..d11e7b921e 100644--- a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts+++ b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts@@ -1,12 +1,14 @@-import { getVectorExtension } from 'src/database.config';import { DatabaseExtension } from 'src/interfaces/database.interface';+import { ConfigRepository } from 'src/repositories/config.repository';import { MigrationInterface, QueryRunner } from 'typeorm';+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;+export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {name = 'AddFaceEmbeddingIndex1700714033632';public async up(queryRunner: QueryRunner): Promise<void> {- if (getVectorExtension() === DatabaseExtension.VECTORS) {+ if (vectorExtension === DatabaseExtension.VECTORS) {await queryRunner.query(`SET vectors.pgvector_compatibility=on`);}await queryRunner.query(`SET search_path TO "$user", public, vectors`);diff --git a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts b/server/src/migrations/1718486162779-AddFaceSearchRelation.tsindex c8e02ec0c5..ae6d752c65 100644--- a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts+++ b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts@@ -1,10 +1,12 @@-import { getVectorExtension } from 'src/database.config';import { DatabaseExtension } from 'src/interfaces/database.interface';+import { ConfigRepository } from 'src/repositories/config.repository';import { MigrationInterface, QueryRunner } from 'typeorm';+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;+export class AddFaceSearchRelation1718486162779 implements MigrationInterface {public async up(queryRunner: QueryRunner): Promise<void> {- if (getVectorExtension() === DatabaseExtension.VECTORS) {+ if (vectorExtension === DatabaseExtension.VECTORS) {await queryRunner.query(`SET search_path TO "$user", public, vectors`);await queryRunner.query(`SET vectors.pgvector_compatibility=on`);}@@ -13,9 +15,10 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {const columns = await queryRunner.query(`SELECT column_name as nameFROM information_schema.columns- WHERE table_name = '${tableName}'`);+ WHERE table_name = '${tableName}'`,+ );return columns.some((column: { name: string }) => column.name === 'embedding');- }+ };const hasAssetEmbeddings = await hasEmbeddings('smart_search');if (!hasAssetEmbeddings) {@@ -31,7 +34,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);- const hasFaceEmbeddings = await hasEmbeddings('asset_faces')+ const hasFaceEmbeddings = await hasEmbeddings('asset_faces');if (hasFaceEmbeddings) {await queryRunner.query(`INSERT INTO face_search("faceId", embedding)@@ -56,7 +59,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {}public async down(queryRunner: QueryRunner): Promise<void> {- if (getVectorExtension() === DatabaseExtension.VECTORS) {+ if (vectorExtension === DatabaseExtension.VECTORS) {await queryRunner.query(`SET search_path TO "$user", public, vectors`);await queryRunner.query(`SET vectors.pgvector_compatibility=on`);}diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.tsindex 37013d8512..ed9d80a980 100644--- a/server/src/repositories/config.repository.ts+++ b/server/src/repositories/config.repository.ts@@ -1,7 +1,7 @@import { Injectable } from '@nestjs/common';-import { getVectorExtension } from 'src/database.config';import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';+import { DatabaseExtension } from 'src/interfaces/database.interface';import { setDifference } from 'src/utils/set';// TODO replace src/config validation with class-validator, here@@ -65,8 +65,16 @@ export class ConfigRepository implements IConfigRepository {},database: {+ url: process.env.DB_URL,+ host: process.env.DB_HOSTNAME || 'database',+ port: Number(process.env.DB_PORT) || 5432,+ username: process.env.DB_USERNAME || 'postgres',+ password: process.env.DB_PASSWORD || 'postgres',+ name: process.env.DB_DATABASE_NAME || 'immich',+skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true',- vectorExtension: getVectorExtension(),+ vectorExtension:+ process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,},licensePublicKey: isProd ? productionKeys : stagingKeys,diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.tsindex 0453421a39..76998b5239 100644--- a/server/src/repositories/database.repository.ts+++ b/server/src/repositories/database.repository.ts@@ -3,7 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm';import AsyncLock from 'async-lock';import semver from 'semver';import { POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';-import { getVectorExtension } from 'src/database.config';+import { IConfigRepository } from 'src/interfaces/config.interface';import {DatabaseExtension,DatabaseLock,@@ -22,12 +22,15 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm';@Instrumentation()@Injectable()export class DatabaseRepository implements IDatabaseRepository {+ private vectorExtension: VectorExtension;readonly asyncLock = new AsyncLock();constructor(@InjectDataSource() private dataSource: DataSource,@Inject(ILoggerRepository) private logger: ILoggerRepository,+ @Inject(IConfigRepository) configRepository: IConfigRepository,) {+ this.vectorExtension = configRepository.getEnv().database.vectorExtension;this.logger.setContext(DatabaseRepository.name);}@@ -119,7 +122,7 @@ export class DatabaseRepository implements IDatabaseRepository {try {await this.dataSource.query(`REINDEX INDEX ${index}`);} catch (error) {- if (getVectorExtension() !== DatabaseExtension.VECTORS) {+ if (this.vectorExtension !== DatabaseExtension.VECTORS) {throw error;}this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);@@ -141,7 +144,7 @@ export class DatabaseRepository implements IDatabaseRepository {}async shouldReindex(name: VectorIndex): Promise<boolean> {- if (getVectorExtension() !== DatabaseExtension.VECTORS) {+ if (this.vectorExtension !== DatabaseExtension.VECTORS) {return false;}diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.tsindex cb80c8d2f1..1e4b32357c 100644--- a/server/src/repositories/search.repository.ts+++ b/server/src/repositories/search.repository.ts@@ -1,7 +1,6 @@import { Inject, Injectable } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { randomUUID } from 'node:crypto';-import { getVectorExtension } from 'src/database.config';import { DummyValue, GenerateSql } from 'src/decorators';import { AssetFaceEntity } from 'src/entities/asset-face.entity';import { AssetEntity } from 'src/entities/asset.entity';@@ -10,7 +9,8 @@ import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';import { SmartInfoEntity } from 'src/entities/smart-info.entity';import { SmartSearchEntity } from 'src/entities/smart-search.entity';import { AssetType, PaginationMode } from 'src/enum';-import { DatabaseExtension } from 'src/interfaces/database.interface';+import { IConfigRepository } from 'src/interfaces/config.interface';+import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface';import { ILoggerRepository } from 'src/interfaces/logger.interface';import {AssetDuplicateResult,@@ -31,6 +31,7 @@ import { Repository, SelectQueryBuilder } from 'typeorm';@Instrumentation()@Injectable()export class SearchRepository implements ISearchRepository {+ private vectorExtension: VectorExtension;private faceColumns: string[];private assetsByCityQuery: string;@@ -42,7 +43,9 @@ export class SearchRepository implements ISearchRepository {@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,@Inject(ILoggerRepository) private logger: ILoggerRepository,+ @Inject(IConfigRepository) configRepository: IConfigRepository,) {+ this.vectorExtension = configRepository.getEnv().database.vectorExtension;this.logger.setContext(SearchRepository.name);this.faceColumns = this.assetFaceRepository.manager.connection.getMetadata(AssetFaceEntity)@@ -440,7 +443,7 @@ export class SearchRepository implements ISearchRepository {}private getRuntimeConfig(numResults?: number): string {- if (getVectorExtension() === DatabaseExtension.VECTOR) {+ if (this.vectorExtension === DatabaseExtension.VECTOR) {return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall}diff --git a/server/src/services/database.service.spec.ts b/server/src/services/database.service.spec.tsindex e28bf1649a..0bf851f41d 100644--- a/server/src/services/database.service.spec.ts+++ b/server/src/services/database.service.spec.ts@@ -57,7 +57,17 @@ describe(DatabaseService.name, () => {])('should work with $extensionName', ({ extension, extensionName }) => {beforeEach(() => {configMock.getEnv.mockReturnValue(- mockEnvData({ database: { skipMigrations: false, vectorExtension: extension } }),+ mockEnvData({+ database: {+ host: 'database',+ port: 5432,+ username: 'postgres',+ password: 'postgres',+ name: 'immich',+ skipMigrations: false,+ vectorExtension: extension,+ },+ }),);});@@ -245,6 +255,11 @@ describe(DatabaseService.name, () => {configMock.getEnv.mockReturnValue(mockEnvData({database: {+ host: 'database',+ port: 5432,+ username: 'postgres',+ password: 'postgres',+ name: 'immich',skipMigrations: true,vectorExtension: DatabaseExtension.VECTORS,},@@ -260,6 +275,11 @@ describe(DatabaseService.name, () => {configMock.getEnv.mockReturnValue(mockEnvData({database: {+ host: 'database',+ port: 5432,+ username: 'postgres',+ password: 'postgres',+ name: 'immich',skipMigrations: true,vectorExtension: DatabaseExtension.VECTOR,},diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.tsindex 42a113534e..960a7c1e83 100644--- a/server/test/repositories/config.repository.mock.ts+++ b/server/test/repositories/config.repository.mock.ts@@ -10,6 +10,12 @@ const envData: EnvData = {buildMetadata: {},database: {+ host: 'database',+ port: 5432,+ username: 'postgres',+ password: 'postgres',+ name: 'immich',+skipMigrations: false,vectorExtension: DatabaseExtension.VECTORS,},