Skip to content

High performance self-hosted photo and video management solution.

refactor(server): db env (#13167)

Jason Rasmussen3ac00b0ffaa5
diff --git a/server/src/database.config.ts b/server/src/database.config.ts
index 9cc317a734..2a46067bc1 100644
--- a/server/src/database.config.ts
+++ b/server/src/database.config.ts
@@ -1,16 +1,17 @@
-import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
import { DataSource } from 'typeorm';
import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions.js';
-const url = process.env.DB_URL;
+const { database } = new ConfigRepository().getEnv();
+const { url, host, port, username, password, name } = database;
const urlOrParts = url
? { url }
: {
- host: process.env.DB_HOSTNAME || 'database',
- port: Number.parseInt(process.env.DB_PORT || '5432'),
- username: process.env.DB_USERNAME || 'postgres',
- password: process.env.DB_PASSWORD || 'postgres',
- database: process.env.DB_DATABASE_NAME || 'immich',
+ host,
+ port,
+ username,
+ password,
+ database: name,
};
/* eslint unicorn/prefer-module: "off" -- We can fix this when migrating to ESM*/
@@ -32,6 +33,3 @@ export const databaseConfig: PostgresConnectionOptions = {
* this export is ONLY to be used for TypeORM commands in package.json#scripts
*/
export const dataSource = new DataSource({ ...databaseConfig, host: 'localhost' });
-
-export const getVectorExtension = () =>
- process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS;
diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts
index bad4a83222..23a3803284 100644
--- a/server/src/interfaces/config.interface.ts
+++ b/server/src/interfaces/config.interface.ts
@@ -26,6 +26,12 @@ export interface EnvData {
};
database: {
+ url?: string;
+ host: string;
+ port: number;
+ username: string;
+ password: string;
+ name: string;
skipMigrations: boolean;
vectorExtension: VectorExtension;
};
diff --git a/server/src/migrations/1700713871511-UsePgVectors.ts b/server/src/migrations/1700713871511-UsePgVectors.ts
index 033e2ba9ad..e67c7275a7 100644
--- a/server/src/migrations/1700713871511-UsePgVectors.ts
+++ b/server/src/migrations/1700713871511-UsePgVectors.ts
@@ -1,13 +1,15 @@
-import { getVectorExtension } from 'src/database.config';
+import { ConfigRepository } from 'src/repositories/config.repository';
import { getCLIPModelInfo } from 'src/utils/misc';
import { MigrationInterface, QueryRunner } from 'typeorm';
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
export class UsePgVectors1700713871511 implements MigrationInterface {
name = 'UsePgVectors1700713871511';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
- await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${getVectorExtension()}`);
+ await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS ${vectorExtension}`);
const faceDimQuery = await queryRunner.query(`
SELECT CARDINALITY(embedding::real[]) as dimsize
FROM asset_faces
diff --git a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts
index e325f270fd..f9ea5a0dc3 100644
--- a/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts
+++ b/server/src/migrations/1700713994428-AddCLIPEmbeddingIndex.ts
@@ -1,12 +1,14 @@
-import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
import { MigrationInterface, QueryRunner } from 'typeorm';
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
name = 'AddCLIPEmbeddingIndex1700713994428';
public async up(queryRunner: QueryRunner): Promise<void> {
- if (getVectorExtension() === DatabaseExtension.VECTORS) {
+ if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
diff --git a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts
index bc6bad6dbd..d11e7b921e 100644
--- a/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts
+++ b/server/src/migrations/1700714033632-AddFaceEmbeddingIndex.ts
@@ -1,12 +1,14 @@
-import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
import { MigrationInterface, QueryRunner } from 'typeorm';
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
name = 'AddFaceEmbeddingIndex1700714033632';
public async up(queryRunner: QueryRunner): Promise<void> {
- if (getVectorExtension() === DatabaseExtension.VECTORS) {
+ if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
diff --git a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts
index c8e02ec0c5..ae6d752c65 100644
--- a/server/src/migrations/1718486162779-AddFaceSearchRelation.ts
+++ b/server/src/migrations/1718486162779-AddFaceSearchRelation.ts
@@ -1,10 +1,12 @@
-import { getVectorExtension } from 'src/database.config';
import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { ConfigRepository } from 'src/repositories/config.repository';
import { MigrationInterface, QueryRunner } from 'typeorm';
+const vectorExtension = new ConfigRepository().getEnv().database.vectorExtension;
+
export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
- if (getVectorExtension() === DatabaseExtension.VECTORS) {
+ if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
@@ -13,9 +15,10 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
const columns = await queryRunner.query(
`SELECT column_name as name
FROM information_schema.columns
- WHERE table_name = '${tableName}'`);
+ WHERE table_name = '${tableName}'`,
+ );
return columns.some((column: { name: string }) => column.name === 'embedding');
- }
+ };
const hasAssetEmbeddings = await hasEmbeddings('smart_search');
if (!hasAssetEmbeddings) {
@@ -31,7 +34,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
await queryRunner.query(`ALTER TABLE face_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);
await queryRunner.query(`ALTER TABLE smart_search ALTER COLUMN embedding SET STORAGE EXTERNAL`);
- const hasFaceEmbeddings = await hasEmbeddings('asset_faces')
+ const hasFaceEmbeddings = await hasEmbeddings('asset_faces');
if (hasFaceEmbeddings) {
await queryRunner.query(`
INSERT INTO face_search("faceId", embedding)
@@ -56,7 +59,7 @@ export class AddFaceSearchRelation1718486162779 implements MigrationInterface {
}
public async down(queryRunner: QueryRunner): Promise<void> {
- if (getVectorExtension() === DatabaseExtension.VECTORS) {
+ if (vectorExtension === DatabaseExtension.VECTORS) {
await queryRunner.query(`SET search_path TO "$user", public, vectors`);
await queryRunner.query(`SET vectors.pgvector_compatibility=on`);
}
diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts
index 37013d8512..ed9d80a980 100644
--- a/server/src/repositories/config.repository.ts
+++ b/server/src/repositories/config.repository.ts
@@ -1,7 +1,7 @@
import { Injectable } from '@nestjs/common';
-import { getVectorExtension } from 'src/database.config';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
+import { DatabaseExtension } from 'src/interfaces/database.interface';
import { setDifference } from 'src/utils/set';
// TODO replace src/config validation with class-validator, here
@@ -65,8 +65,16 @@ export class ConfigRepository implements IConfigRepository {
},
database: {
+ url: process.env.DB_URL,
+ host: process.env.DB_HOSTNAME || 'database',
+ port: Number(process.env.DB_PORT) || 5432,
+ username: process.env.DB_USERNAME || 'postgres',
+ password: process.env.DB_PASSWORD || 'postgres',
+ name: process.env.DB_DATABASE_NAME || 'immich',
+
skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true',
- vectorExtension: getVectorExtension(),
+ vectorExtension:
+ process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS,
},
licensePublicKey: isProd ? productionKeys : stagingKeys,
diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts
index 0453421a39..76998b5239 100644
--- a/server/src/repositories/database.repository.ts
+++ b/server/src/repositories/database.repository.ts
@@ -3,7 +3,7 @@ import { InjectDataSource } from '@nestjs/typeorm';
import AsyncLock from 'async-lock';
import semver from 'semver';
import { POSTGRES_VERSION_RANGE, VECTOR_VERSION_RANGE, VECTORS_VERSION_RANGE } from 'src/constants';
-import { getVectorExtension } from 'src/database.config';
+import { IConfigRepository } from 'src/interfaces/config.interface';
import {
DatabaseExtension,
DatabaseLock,
@@ -22,12 +22,15 @@ import { DataSource, EntityManager, QueryRunner } from 'typeorm';
@Instrumentation()
@Injectable()
export class DatabaseRepository implements IDatabaseRepository {
+ private vectorExtension: VectorExtension;
readonly asyncLock = new AsyncLock();
constructor(
@InjectDataSource() private dataSource: DataSource,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
+ @Inject(IConfigRepository) configRepository: IConfigRepository,
) {
+ this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(DatabaseRepository.name);
}
@@ -119,7 +122,7 @@ export class DatabaseRepository implements IDatabaseRepository {
try {
await this.dataSource.query(`REINDEX INDEX ${index}`);
} catch (error) {
- if (getVectorExtension() !== DatabaseExtension.VECTORS) {
+ if (this.vectorExtension !== DatabaseExtension.VECTORS) {
throw error;
}
this.logger.warn(`Could not reindex index ${index}. Attempting to auto-fix.`);
@@ -141,7 +144,7 @@ export class DatabaseRepository implements IDatabaseRepository {
}
async shouldReindex(name: VectorIndex): Promise<boolean> {
- if (getVectorExtension() !== DatabaseExtension.VECTORS) {
+ if (this.vectorExtension !== DatabaseExtension.VECTORS) {
return false;
}
diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts
index cb80c8d2f1..1e4b32357c 100644
--- a/server/src/repositories/search.repository.ts
+++ b/server/src/repositories/search.repository.ts
@@ -1,7 +1,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { randomUUID } from 'node:crypto';
-import { getVectorExtension } from 'src/database.config';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
@@ -10,7 +9,8 @@ import { GeodataPlacesEntity } from 'src/entities/geodata-places.entity';
import { SmartInfoEntity } from 'src/entities/smart-info.entity';
import { SmartSearchEntity } from 'src/entities/smart-search.entity';
import { AssetType, PaginationMode } from 'src/enum';
-import { DatabaseExtension } from 'src/interfaces/database.interface';
+import { IConfigRepository } from 'src/interfaces/config.interface';
+import { DatabaseExtension, VectorExtension } from 'src/interfaces/database.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import {
AssetDuplicateResult,
@@ -31,6 +31,7 @@ import { Repository, SelectQueryBuilder } from 'typeorm';
@Instrumentation()
@Injectable()
export class SearchRepository implements ISearchRepository {
+ private vectorExtension: VectorExtension;
private faceColumns: string[];
private assetsByCityQuery: string;
@@ -42,7 +43,9 @@ export class SearchRepository implements ISearchRepository {
@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
@InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
+ @Inject(IConfigRepository) configRepository: IConfigRepository,
) {
+ this.vectorExtension = configRepository.getEnv().database.vectorExtension;
this.logger.setContext(SearchRepository.name);
this.faceColumns = this.assetFaceRepository.manager.connection
.getMetadata(AssetFaceEntity)
@@ -440,7 +443,7 @@ export class SearchRepository implements ISearchRepository {
}
private getRuntimeConfig(numResults?: number): string {
- if (getVectorExtension() === DatabaseExtension.VECTOR) {
+ if (this.vectorExtension === DatabaseExtension.VECTOR) {
return 'SET LOCAL hnsw.ef_search = 1000;'; // mitigate post-filter recall
}
diff --git a/server/src/services/database.service.spec.ts b/server/src/services/database.service.spec.ts
index e28bf1649a..0bf851f41d 100644
--- a/server/src/services/database.service.spec.ts
+++ b/server/src/services/database.service.spec.ts
@@ -57,7 +57,17 @@ describe(DatabaseService.name, () => {
])('should work with $extensionName', ({ extension, extensionName }) => {
beforeEach(() => {
configMock.getEnv.mockReturnValue(
- mockEnvData({ database: { skipMigrations: false, vectorExtension: extension } }),
+ mockEnvData({
+ database: {
+ host: 'database',
+ port: 5432,
+ username: 'postgres',
+ password: 'postgres',
+ name: 'immich',
+ skipMigrations: false,
+ vectorExtension: extension,
+ },
+ }),
);
});
@@ -245,6 +255,11 @@ describe(DatabaseService.name, () => {
configMock.getEnv.mockReturnValue(
mockEnvData({
database: {
+ host: 'database',
+ port: 5432,
+ username: 'postgres',
+ password: 'postgres',
+ name: 'immich',
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTORS,
},
@@ -260,6 +275,11 @@ describe(DatabaseService.name, () => {
configMock.getEnv.mockReturnValue(
mockEnvData({
database: {
+ host: 'database',
+ port: 5432,
+ username: 'postgres',
+ password: 'postgres',
+ name: 'immich',
skipMigrations: true,
vectorExtension: DatabaseExtension.VECTOR,
},
diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts
index 42a113534e..960a7c1e83 100644
--- a/server/test/repositories/config.repository.mock.ts
+++ b/server/test/repositories/config.repository.mock.ts
@@ -10,6 +10,12 @@ const envData: EnvData = {
buildMetadata: {},
database: {
+ host: 'database',
+ port: 5432,
+ username: 'postgres',
+ password: 'postgres',
+ name: 'immich',
+
skipMigrations: false,
vectorExtension: DatabaseExtension.VECTORS,
},