Skip to content

High performance self-hosted photo and video management solution.

refactor: migrate map repository to kysely (#15348)

Zack Pollardc821458e6cef
diff --git a/server/.prettierignore b/server/.prettierignore
index 27a20b3f50..8dbb1067a9 100644
--- a/server/.prettierignore
+++ b/server/.prettierignore
@@ -10,6 +10,7 @@ node_modules
coverage
dist
**/migrations/**
+db.d.ts
# Ignore files for PNPM, NPM and YARN
pnpm-lock.yaml
diff --git a/server/package.json b/server/package.json
index 14b4e2fa32..be6ff2632f 100644
--- a/server/package.json
+++ b/server/package.json
@@ -29,6 +29,7 @@
"typeorm:migrations:revert": "typeorm migration:revert -d ./dist/bin/database.js",
"typeorm:schema:drop": "typeorm query -d ./dist/bin/database.js 'DROP schema public cascade; CREATE schema public;'",
"typeorm:schema:reset": "npm run typeorm:schema:drop && npm run typeorm:migrations:run",
+ "kysely:codegen": "npx kysely-codegen --include-pattern=\"(public|vectors).*\" --dialect postgres --url postgres://postgres:postgres@localhost/immich --log-level debug --out-file=./src/db.d.ts",
"sync:open-api": "node ./dist/bin/sync-open-api.js",
"sync:sql": "node ./dist/bin/sync-sql.js",
"email:dev": "email dev -p 3050 --dir src/emails"
diff --git a/server/src/db.d.ts b/server/src/db.d.ts
index a5cab5dab7..6242914bee 100644
--- a/server/src/db.d.ts
+++ b/server/src/db.d.ts
@@ -3,16 +3,21 @@
* Please do not edit it manually.
*/
-import type { ColumnType } from 'kysely';
+import type { ColumnType } from "kysely";
-export type ArrayType<T> = ArrayTypeImpl<T> extends (infer U)[] ? U[] : ArrayTypeImpl<T>;
+export type ArrayType<T> = ArrayTypeImpl<T> extends (infer U)[]
+ ? U[]
+ : ArrayTypeImpl<T>;
-export type ArrayTypeImpl<T> = T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S[], I[], U[]> : T[];
+export type ArrayTypeImpl<T> = T extends ColumnType<infer S, infer I, infer U>
+ ? ColumnType<S[], I[], U[]>
+ : T[];
-export type AssetsStatusEnum = 'active' | 'deleted' | 'trashed';
+export type AssetsStatusEnum = "active" | "deleted" | "trashed";
-export type Generated<T> =
- T extends ColumnType<infer S, infer I, infer U> ? ColumnType<S, I | undefined, U> : ColumnType<T, T | undefined, T>;
+export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
+ ? ColumnType<S, I | undefined, U>
+ : ColumnType<T, T | undefined, T>;
export type Int8 = ColumnType<string, bigint | number | string, bigint | number | string>;
@@ -28,7 +33,7 @@ export type JsonPrimitive = boolean | number | string | null;
export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
-export type Sourcetype = 'exif' | 'machine-learning';
+export type Sourcetype = "exif" | "machine-learning";
export type Timestamp = ColumnType<Date, Date | string, Date | string>;
@@ -257,7 +262,7 @@ export interface NaturalearthCountries {
admin: string;
admin_a3: string;
coordinates: string;
- id: number;
+ id: Generated<number>;
type: string;
}
@@ -433,6 +438,6 @@ export interface DB {
tags_closure: TagsClosure;
user_metadata: UserMetadata;
users: Users;
- 'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat;
+ "vectors.pg_vector_index_stat": VectorsPgVectorIndexStat;
version_history: VersionHistory;
}
diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts
index 8c3a2549f0..19068ddc5d 100644
--- a/server/src/repositories/config.repository.spec.ts
+++ b/server/src/repositories/config.repository.spec.ts
@@ -83,7 +83,7 @@ describe('getEnv', () => {
config: {
kysely: {
dialect: expect.any(PostgresJSDialect),
- log: ['error'],
+ log: expect.any(Function),
},
typeorm: expect.objectContaining({
type: 'postgres',
diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts
index 15775d0471..67699880bd 100644
--- a/server/src/repositories/config.repository.ts
+++ b/server/src/repositories/config.repository.ts
@@ -5,7 +5,7 @@ import { Request, Response } from 'express';
import { PostgresJSDialect } from 'kysely-postgres-js';
import { CLS_ID } from 'nestjs-cls';
import { join, resolve } from 'node:path';
-import postgres from 'postgres';
+import postgres, { Notice } from 'postgres';
import { citiesFile, excludePaths, IWorker } from 'src/constants';
import { Telemetry } from 'src/decorators';
import { EnvDto } from 'src/dtos/env.dto';
@@ -99,6 +99,11 @@ const getEnv = (): EnvData => {
}
const driverOptions = {
+ onnotice: (notice: Notice) => {
+ if (notice['severity'] !== 'NOTICE') {
+ console.warn('Postgres notice:', notice);
+ }
+ },
max: 10,
types: {
date: {
@@ -194,7 +199,16 @@ const getEnv = (): EnvData => {
dialect: new PostgresJSDialect({
postgres: databaseUrl ? postgres(databaseUrl, driverOptions) : postgres({ ...parts, ...driverOptions }),
}),
- log: ['error'] as const,
+ log(event) {
+ if (event.level === 'error') {
+ console.error('Query failed :', {
+ durationMs: event.queryDurationMillis,
+ error: event.error,
+ sql: event.query.sql,
+ params: event.query.parameters,
+ });
+ }
+ },
},
},
diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts
index b9d17686a1..00870e78eb 100644
--- a/server/src/repositories/map.repository.ts
+++ b/server/src/repositories/map.repository.ts
@@ -1,17 +1,15 @@
import { Inject, Injectable } from '@nestjs/common';
-import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { getName } from 'i18n-iso-countries';
+import { Expression, Kysely, sql, SqlBool } from 'kysely';
+import { InjectKysely } from 'nestjs-kysely';
import { randomUUID } from 'node:crypto';
import { createReadStream, existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import readLine from 'node:readline';
import { citiesFile } from 'src/constants';
-import { AssetEntity } from 'src/entities/asset.entity';
-import { GeodataPlacesEntity, GeodataPlacesTempEntity } from 'src/entities/geodata-places.entity';
-import {
- NaturalEarthCountriesEntity,
- NaturalEarthCountriesTempEntity,
-} from 'src/entities/natural-earth-countries.entity';
+import { DB, GeodataPlaces, NaturalearthCountries } from 'src/db';
+import { AssetEntity, withExif } from 'src/entities/asset.entity';
+import { NaturalEarthCountriesTempEntity } from 'src/entities/natural-earth-countries.entity';
import { LogLevel, SystemMetadataKey } from 'src/enum';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
@@ -23,21 +21,19 @@ import {
ReverseGeocodeResult,
} from 'src/interfaces/map.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
-import { OptionalBetween } from 'src/utils/database';
-import { DataSource, In, IsNull, Not, Repository } from 'typeorm';
-import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
+
+interface MapDB extends DB {
+ geodata_places_tmp: GeodataPlaces;
+ naturalearth_countries_tmp: NaturalearthCountries;
+}
@Injectable()
export class MapRepository implements IMapRepository {
constructor(
- @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
- @InjectRepository(GeodataPlacesEntity) private geodataPlacesRepository: Repository<GeodataPlacesEntity>,
- @InjectRepository(NaturalEarthCountriesEntity)
- private naturalEarthCountriesRepository: Repository<NaturalEarthCountriesEntity>,
- @InjectDataSource() private dataSource: DataSource,
@Inject(IConfigRepository) private configRepository: IConfigRepository,
@Inject(ISystemMetadataRepository) private metadataRepository: ISystemMetadataRepository,
@Inject(ILoggerRepository) private logger: ILoggerRepository,
+ @InjectKysely() private db: Kysely<MapDB>,
) {
this.logger.setContext(MapRepository.name);
}
@@ -70,39 +66,34 @@ export class MapRepository implements IMapRepository {
): Promise<MapMarker[]> {
const { isArchived, isFavorite, fileCreatedAfter, fileCreatedBefore } = options;
- const where = {
- isVisible: true,
- isArchived,
- exifInfo: {
- latitude: Not(IsNull()),
- longitude: Not(IsNull()),
- },
- isFavorite,
- fileCreatedAt: OptionalBetween(fileCreatedAfter, fileCreatedBefore),
- };
-
- const assets = await this.assetRepository.find({
- select: {
- id: true,
- exifInfo: {
- city: true,
- state: true,
- country: true,
- latitude: true,
- longitude: true,
- },
- },
- where: [
- { ...where, ownerId: In([...ownerIds]) },
- { ...where, albums: { id: In([...albumIds]) } },
- ],
- relations: {
- exifInfo: true,
- },
- order: {
- fileCreatedAt: 'DESC',
- },
- });
+ const assets = (await this.db
+ .selectFrom('assets')
+ .$call(withExif)
+ .select('id')
+ .leftJoin('albums_assets_assets', (join) => join.onRef('assets.id', '=', 'albums_assets_assets.assetsId'))
+ .where('isVisible', '=', true)
+ .$if(isArchived !== undefined, (q) => q.where('isArchived', '=', isArchived!))
+ .$if(isFavorite !== undefined, (q) => q.where('isFavorite', '=', isFavorite!))
+ .$if(fileCreatedAfter !== undefined, (q) => q.where('fileCreatedAt', '>=', fileCreatedAfter!))
+ .$if(fileCreatedBefore !== undefined, (q) => q.where('fileCreatedAt', '<=', fileCreatedBefore!))
+ .where('deletedAt', 'is', null)
+ .where('exif.latitude', 'is not', null)
+ .where('exif.longitude', 'is not', null)
+ .where((eb) => {
+ const ors: Expression<SqlBool>[] = [];
+
+ if (ownerIds.length > 0) {
+ ors.push(eb('ownerId', 'in', ownerIds));
+ }
+
+ if (albumIds.length > 0) {
+ ors.push(eb('albums_assets_assets.albumsId', 'in', albumIds));
+ }
+
+ return eb.or(ors);
+ })
+ .orderBy('fileCreatedAt', 'desc')
+ .execute()) as any as AssetEntity[];
return assets.map((asset) => ({
id: asset.id,
@@ -117,15 +108,19 @@ export class MapRepository implements IMapRepository {
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
- const response = await this.geodataPlacesRepository
- .createQueryBuilder('geoplaces')
+ const response = await this.db
+ .selectFrom('geodata_places')
+ .selectAll()
.where(
- 'earth_box(ll_to_earth_public(:latitude, :longitude), 25000) @> ll_to_earth_public(latitude, longitude)',
- point,
+ sql`earth_box(ll_to_earth_public(${point.latitude}, ${point.longitude}), 25000)`,
+ '@>',
+ sql`ll_to_earth_public(latitude, longitude)`,
+ )
+ .orderBy(
+ sql`(earth_distance(ll_to_earth_public(${point.latitude}, ${point.longitude}), ll_to_earth_public(latitude, longitude)))`,
)
- .orderBy('earth_distance(ll_to_earth_public(:latitude, :longitude), ll_to_earth_public(latitude, longitude))')
.limit(1)
- .getOne();
+ .executeTakeFirst();
if (response) {
if (this.logger.isLevelEnabled(LogLevel.VERBOSE)) {
@@ -143,11 +138,12 @@ export class MapRepository implements IMapRepository {
`Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
);
- const ne_response = await this.naturalEarthCountriesRepository
- .createQueryBuilder('naturalearth_countries')
- .where('coordinates @> point (:longitude, :latitude)', point)
+ const ne_response = await this.db
+ .selectFrom('naturalearth_countries')
+ .selectAll()
+ .where('coordinates', '@>', sql<string>`point(${point.longitude}, ${point.latitude})`)
.limit(1)
- .getOne();
+ .executeTakeFirst();
if (!ne_response) {
this.logger.warn(
@@ -176,10 +172,11 @@ export class MapRepository implements IMapRepository {
return;
}
- await this.dataSource.query('DROP TABLE IF EXISTS naturalearth_countries_tmp');
- await this.dataSource.query(
- 'CREATE TABLE naturalearth_countries_tmp (LIKE naturalearth_countries INCLUDING ALL EXCLUDING INDEXES)',
+ await this.db.schema.dropTable('naturalearth_countries_tmp').ifExists().execute();
+ await sql`CREATE TABLE naturalearth_countries_tmp (LIKE naturalearth_countries INCLUDING ALL EXCLUDING INDEXES)`.execute(
+ this.db,
);
+
const entities: Omit<NaturalEarthCountriesTempEntity, 'id'>[] = [];
for (const feature of geoJSONData.features) {
for (const entry of feature.geometry.coordinates) {
@@ -196,14 +193,14 @@ export class MapRepository implements IMapRepository {
}
}
}
- await this.dataSource.manager.insert(NaturalEarthCountriesTempEntity, entities);
+ await this.db.insertInto('naturalearth_countries_tmp').values(entities).execute();
- await this.dataSource.query(`ALTER TABLE naturalearth_countries_tmp ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`);
+ await sql`ALTER TABLE naturalearth_countries_tmp ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`.execute(this.db);
- await this.dataSource.transaction(async (manager) => {
- await manager.query('ALTER TABLE naturalearth_countries RENAME TO naturalearth_countries_old');
- await manager.query('ALTER TABLE naturalearth_countries_tmp RENAME TO naturalearth_countries');
- await manager.query('DROP TABLE naturalearth_countries_old');
+ await this.db.transaction().execute(async (manager) => {
+ await manager.schema.alterTable('naturalearth_countries').renameTo('naturalearth_countries_old').execute();
+ await manager.schema.alterTable('naturalearth_countries_tmp').renameTo('naturalearth_countries').execute();
+ await manager.schema.dropTable('naturalearth_countries_old').execute();
});
}
@@ -214,17 +211,15 @@ export class MapRepository implements IMapRepository {
this.loadAdmin(resourcePaths.geodata.admin2),
]);
- await this.dataSource.query('DROP TABLE IF EXISTS geodata_places_tmp');
- await this.dataSource.query(
- 'CREATE TABLE geodata_places_tmp (LIKE geodata_places INCLUDING ALL EXCLUDING INDEXES)',
- );
+ await this.db.schema.dropTable('geodata_places_tmp').ifExists().execute();
+ await sql`CREATE TABLE geodata_places_tmp (LIKE geodata_places INCLUDING ALL EXCLUDING INDEXES)`.execute(this.db);
await this.loadCities500(admin1, admin2);
await this.createGeodataIndices();
- await this.dataSource.transaction(async (manager) => {
- await manager.query('ALTER TABLE geodata_places RENAME TO geodata_places_old');
- await manager.query('ALTER TABLE geodata_places_tmp RENAME TO geodata_places');
- await manager.query('DROP TABLE geodata_places_old');
+ await this.db.transaction().execute(async (manager) => {
+ await manager.schema.alterTable('geodata_places').renameTo('geodata_places_old').execute();
+ await manager.schema.alterTable('geodata_places_tmp').renameTo('geodata_places').execute();
+ await manager.schema.dropTable('geodata_places_old').execute();
});
}
@@ -236,7 +231,7 @@ export class MapRepository implements IMapRepository {
}
const input = createReadStream(cities500, { highWaterMark: 512 * 1024 * 1024 });
- let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesTempEntity>[] = [];
+ let bufferGeodata = [];
const lineReader = readLine.createInterface({ input });
let count = 0;
@@ -257,19 +252,23 @@ export class MapRepository implements IMapRepository {
admin1Code: lineSplit[10],
admin2Code: lineSplit[11],
modificationDate: lineSplit[18],
- admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`),
- admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`),
+ admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`) ?? null,
+ admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`) ?? null,
};
bufferGeodata.push(geoData);
if (bufferGeodata.length >= 5000) {
const curLength = bufferGeodata.length;
futures.push(
- this.dataSource.manager.insert(GeodataPlacesTempEntity, bufferGeodata).then(() => {
- count += curLength;
- if (count % 10_000 === 0) {
- this.logger.log(`${count} geodata records imported`);
- }
- }),
+ this.db
+ .insertInto('geodata_places_tmp')
+ .values(bufferGeodata)
+ .execute()
+ .then(() => {
+ count += curLength;
+ if (count % 10_000 === 0) {
+ this.logger.log(`${count} geodata records imported`);
+ }
+ }),
);
bufferGeodata = [];
// leave spare connection for other queries
@@ -280,7 +279,7 @@ export class MapRepository implements IMapRepository {
}
}
- await this.dataSource.manager.insert(GeodataPlacesTempEntity, bufferGeodata);
+ await this.db.insertInto('geodata_places_tmp').values(bufferGeodata).execute();
}
private async loadAdmin(filePath: string) {
@@ -303,24 +302,28 @@ export class MapRepository implements IMapRepository {
private createGeodataIndices() {
return Promise.all([
- this.dataSource.query(`ALTER TABLE geodata_places_tmp ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`),
- this.dataSource.query(`
- CREATE INDEX IDX_geodata_gist_earthcoord_${randomUUID().replaceAll('-', '_')}
- ON geodata_places_tmp
- USING gist (ll_to_earth_public(latitude, longitude))
- WITH (fillfactor = 100)`),
- this.dataSource.query(`
- CREATE INDEX idx_geodata_places_name_${randomUUID().replaceAll('-', '_')}
- ON geodata_places_tmp
- USING gin (f_unaccent(name) gin_trgm_ops)`),
- this.dataSource.query(`
- CREATE INDEX idx_geodata_places_admin1_name_${randomUUID().replaceAll('-', '_')}
- ON geodata_places_tmp
- USING gin (f_unaccent("admin1Name") gin_trgm_ops)`),
- this.dataSource.query(`
- CREATE INDEX idx_geodata_places_admin2_name_${randomUUID().replaceAll('-', '_')}
- ON geodata_places_tmp
- USING gin (f_unaccent("admin2Name") gin_trgm_ops)`),
+ sql`ALTER TABLE geodata_places_tmp ADD PRIMARY KEY (id) WITH (FILLFACTOR = 100)`.execute(this.db),
+ sql`
+ CREATE INDEX IDX_geodata_gist_earthcoord_${sql.raw(randomUUID().replaceAll('-', '_'))}
+ ON geodata_places_tmp
+ USING gist (ll_to_earth_public(latitude, longitude))
+ WITH (fillfactor = 100)
+ `.execute(this.db),
+ this.db.schema
+ .createIndex(`idx_geodata_places_country_code_${randomUUID().replaceAll('-', '_')}`)
+ .on('geodata_places_tmp')
+ .using('gin (f_unaccent(name) gin_trgm_ops)')
+ .execute(),
+ this.db.schema
+ .createIndex(`idx_geodata_places_country_code_${randomUUID().replaceAll('-', '_')}`)
+ .on('geodata_places_tmp')
+ .using('gin (f_unaccent("admin1Name") gin_trgm_ops)')
+ .execute(),
+ this.db.schema
+ .createIndex(`idx_geodata_places_admin2_name_${randomUUID().replaceAll('-', '_')}`)
+ .on('geodata_places_tmp')
+ .using('gin (f_unaccent("admin2Name") gin_trgm_ops)')
+ .execute(),
]);
}
}