High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat: create table with constraints (#19828)
Jason Rasmussen51ab7498e90b
diff --git a/server/src/schema/migrations/1752004072340-UpdateIndexOverrides.ts b/server/src/schema/migrations/1752004072340-UpdateIndexOverrides.tsnew file mode 100644index 0000000000..55638ab122--- /dev/null+++ b/server/src/schema/migrations/1752004072340-UpdateIndexOverrides.ts@@ -0,0 +1,31 @@+import { Kysely, sql } from 'kysely';++export async function up(db: Kysely<any>): Promise<void> {+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_originalfilename_trigram","sql":"CREATE INDEX \\"idx_originalfilename_trigram\\" ON \\"assets\\" USING gin (f_unaccent(\\"originalFileName\\") gin_trgm_ops);"}'::jsonb WHERE "name" = 'index_idx_originalfilename_trigram';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_local_date_time_month","sql":"CREATE INDEX \\"idx_local_date_time_month\\" ON \\"assets\\" ((date_trunc(''MONTH''::text, (\\"localDateTime\\" AT TIME ZONE ''UTC''::text)) AT TIME ZONE ''UTC''::text));"}'::jsonb WHERE "name" = 'index_idx_local_date_time_month';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_local_date_time","sql":"CREATE INDEX \\"idx_local_date_time\\" ON \\"assets\\" (((\\"localDateTime\\" at time zone ''UTC'')::date));"}'::jsonb WHERE "name" = 'index_idx_local_date_time';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"UQ_assets_owner_library_checksum","sql":"CREATE UNIQUE INDEX \\"UQ_assets_owner_library_checksum\\" ON \\"assets\\" (\\"ownerId\\", \\"libraryId\\", \\"checksum\\") WHERE (\\"libraryId\\" IS NOT NULL);"}'::jsonb WHERE "name" = 'index_UQ_assets_owner_library_checksum';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"UQ_assets_owner_checksum","sql":"CREATE UNIQUE INDEX \\"UQ_assets_owner_checksum\\" ON \\"assets\\" (\\"ownerId\\", \\"checksum\\") WHERE (\\"libraryId\\" IS NULL);"}'::jsonb WHERE "name" = 'index_UQ_assets_owner_checksum';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"IDX_activity_like","sql":"CREATE UNIQUE INDEX \\"IDX_activity_like\\" ON \\"activity\\" (\\"assetId\\", \\"userId\\", \\"albumId\\") WHERE (\\"isLiked\\" = true);"}'::jsonb WHERE "name" = 'index_IDX_activity_like';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"face_index","sql":"CREATE INDEX \\"face_index\\" ON \\"face_search\\" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16);"}'::jsonb WHERE "name" = 'index_face_index';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"IDX_geodata_gist_earthcoord","sql":"CREATE INDEX \\"IDX_geodata_gist_earthcoord\\" ON \\"geodata_places\\" (ll_to_earth_public(latitude, longitude));"}'::jsonb WHERE "name" = 'index_IDX_geodata_gist_earthcoord';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_geodata_places_name","sql":"CREATE INDEX \\"idx_geodata_places_name\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"name\\") gin_trgm_ops);"}'::jsonb WHERE "name" = 'index_idx_geodata_places_name';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_geodata_places_admin2_name","sql":"CREATE INDEX \\"idx_geodata_places_admin2_name\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"admin2Name\\") gin_trgm_ops);"}'::jsonb WHERE "name" = 'index_idx_geodata_places_admin2_name';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_geodata_places_admin1_name","sql":"CREATE INDEX \\"idx_geodata_places_admin1_name\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"admin1Name\\") gin_trgm_ops);"}'::jsonb WHERE "name" = 'index_idx_geodata_places_admin1_name';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"type":"index","name":"idx_geodata_places_alternate_names","sql":"CREATE INDEX \\"idx_geodata_places_alternate_names\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"alternateNames\\") gin_trgm_ops);"}'::jsonb WHERE "name" = 'index_idx_geodata_places_alternate_names';`.execute(db);+}++export async function down(db: Kysely<any>): Promise<void> {+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_originalfilename_trigram\\" ON \\"assets\\" USING gin (f_unaccent(\\"originalFileName\\") gin_trgm_ops)","name":"idx_originalfilename_trigram","type":"index"}'::jsonb WHERE "name" = 'index_idx_originalfilename_trigram';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_local_date_time_month\\" ON \\"assets\\" ((date_trunc(''MONTH''::text, (\\"localDateTime\\" AT TIME ZONE ''UTC''::text)) AT TIME ZONE ''UTC''::text))","name":"idx_local_date_time_month","type":"index"}'::jsonb WHERE "name" = 'index_idx_local_date_time_month';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_local_date_time\\" ON \\"assets\\" (((\\"localDateTime\\" at time zone ''UTC'')::date))","name":"idx_local_date_time","type":"index"}'::jsonb WHERE "name" = 'index_idx_local_date_time';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE UNIQUE INDEX \\"UQ_assets_owner_library_checksum\\" ON \\"assets\\" (\\"ownerId\\", \\"libraryId\\", \\"checksum\\") WHERE (\\"libraryId\\" IS NOT NULL)","name":"UQ_assets_owner_library_checksum","type":"index"}'::jsonb WHERE "name" = 'index_UQ_assets_owner_library_checksum';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE UNIQUE INDEX \\"UQ_assets_owner_checksum\\" ON \\"assets\\" (\\"ownerId\\", \\"checksum\\") WHERE (\\"libraryId\\" IS NULL)","name":"UQ_assets_owner_checksum","type":"index"}'::jsonb WHERE "name" = 'index_UQ_assets_owner_checksum';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE UNIQUE INDEX \\"IDX_activity_like\\" ON \\"activity\\" (\\"assetId\\", \\"userId\\", \\"albumId\\") WHERE (\\"isLiked\\" = true)","name":"IDX_activity_like","type":"index"}'::jsonb WHERE "name" = 'index_IDX_activity_like';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"face_index\\" ON \\"face_search\\" USING hnsw (embedding vector_cosine_ops) WITH (ef_construction = 300, m = 16)","name":"face_index","type":"index"}'::jsonb WHERE "name" = 'index_face_index';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"IDX_geodata_gist_earthcoord\\" ON \\"geodata_places\\" (ll_to_earth_public(latitude, longitude))","name":"IDX_geodata_gist_earthcoord","type":"index"}'::jsonb WHERE "name" = 'index_IDX_geodata_gist_earthcoord';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_geodata_places_name\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"name\\") gin_trgm_ops)","name":"idx_geodata_places_name","type":"index"}'::jsonb WHERE "name" = 'index_idx_geodata_places_name';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_geodata_places_admin2_name\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"admin2Name\\") gin_trgm_ops)","name":"idx_geodata_places_admin2_name","type":"index"}'::jsonb WHERE "name" = 'index_idx_geodata_places_admin2_name';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_geodata_places_admin1_name\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"admin1Name\\") gin_trgm_ops)","name":"idx_geodata_places_admin1_name","type":"index"}'::jsonb WHERE "name" = 'index_idx_geodata_places_admin1_name';`.execute(db);+ await sql`UPDATE "migration_overrides" SET "value" = '{"sql":"CREATE INDEX \\"idx_geodata_places_alternate_names\\" ON \\"geodata_places\\" USING gin (f_unaccent(\\"alternateNames\\") gin_trgm_ops)","name":"idx_geodata_places_alternate_names","type":"index"}'::jsonb WHERE "name" = 'index_idx_geodata_places_alternate_names';`.execute(db);+}diff --git a/server/src/sql-tools/comparers/table.comparer.ts b/server/src/sql-tools/comparers/table.comparer.tsindex c920a1d07a..0b36b7fce4 100644--- a/server/src/sql-tools/comparers/table.comparer.ts+++ b/server/src/sql-tools/comparers/table.comparer.ts@@ -5,15 +5,6 @@ import { compareTriggers } from 'src/sql-tools/comparers/trigger.comparer';import { compare } from 'src/sql-tools/helpers';import { Comparer, DatabaseTable, Reason, SchemaDiff } from 'src/sql-tools/types';-const newTable = (name: string) => ({- name,- columns: [],- indexes: [],- constraints: [],- triggers: [],- synchronize: true,-});-export const compareTables: Comparer<DatabaseTable> = {onMissing: (source) => [{@@ -21,23 +12,20 @@ export const compareTables: Comparer<DatabaseTable> = {table: source,reason: Reason.MissingInTarget,},- // TODO merge constraints into table create record when possible- ...compareTable(source, newTable(source.name), { columns: false }),],onExtra: (target) => [- ...compareTable(newTable(target.name), target, { columns: false }),{type: 'TableDrop',tableName: target.name,reason: Reason.MissingInSource,},],- onCompare: (source, target) => compareTable(source, target, { columns: true }),+ onCompare: (source, target) => compareTable(source, target),};-const compareTable = (source: DatabaseTable, target: DatabaseTable, options: { columns?: boolean }): SchemaDiff[] => {+const compareTable = (source: DatabaseTable, target: DatabaseTable): SchemaDiff[] => {return [- ...(options.columns ? compare(source.columns, target.columns, {}, compareColumns) : []),+ ...compare(source.columns, target.columns, {}, compareColumns),...compare(source.indexes, target.indexes, {}, compareIndexes),...compare(source.constraints, target.constraints, {}, compareConstraints),...compare(source.triggers, target.triggers, {}, compareTriggers),diff --git a/server/src/sql-tools/transformers/constraint.transformer.ts b/server/src/sql-tools/transformers/constraint.transformer.tsindex 921774b7a9..bb8b30e7ce 100644--- a/server/src/sql-tools/transformers/constraint.transformer.ts+++ b/server/src/sql-tools/transformers/constraint.transformer.ts@@ -20,12 +20,13 @@ export const transformConstraints: SqlTransformer = (ctx, item) => {const withAction = (constraint: { onDelete?: ActionType; onUpdate?: ActionType }) =>` ON UPDATE ${constraint.onUpdate ?? ActionType.NO_ACTION} ON DELETE ${constraint.onDelete ?? ActionType.NO_ACTION}`;-export const asConstraintAdd = (constraint: DatabaseConstraint): string | string[] => {- const base = `ALTER TABLE "${constraint.tableName}" ADD CONSTRAINT "${constraint.name}"`;+export const asConstraintBody = (constraint: DatabaseConstraint): string => {+ const base = `CONSTRAINT "${constraint.name}"`;+switch (constraint.type) {case ConstraintType.PRIMARY_KEY: {const columnNames = asColumnList(constraint.columnNames);- return `${base} PRIMARY KEY (${columnNames});`;+ return `${base} PRIMARY KEY (${columnNames})`;}case ConstraintType.FOREIGN_KEY: {@@ -33,26 +34,29 @@ export const asConstraintAdd = (constraint: DatabaseConstraint): string | stringconst referenceColumnNames = asColumnList(constraint.referenceColumnNames);return (`${base} FOREIGN KEY (${columnNames}) REFERENCES "${constraint.referenceTableName}" (${referenceColumnNames})` +- withAction(constraint) +- ';'+ withAction(constraint));}case ConstraintType.UNIQUE: {const columnNames = asColumnList(constraint.columnNames);- return `${base} UNIQUE (${columnNames});`;+ return `${base} UNIQUE (${columnNames})`;}case ConstraintType.CHECK: {- return `${base} CHECK (${constraint.expression});`;+ return `${base} CHECK (${constraint.expression})`;}default: {- return [];+ throw new Error(`Unknown constraint type: ${(constraint as any).type}`);}}};+export const asConstraintAdd = (constraint: DatabaseConstraint): string | string[] => {+ return `ALTER TABLE "${constraint.tableName}" ADD ${asConstraintBody(constraint)};`;+};+export const asConstraintDrop = (tableName: string, constraintName: string): string => {return `ALTER TABLE "${tableName}" DROP CONSTRAINT "${constraintName}";`;};diff --git a/server/src/sql-tools/transformers/index.transformer.spec.ts b/server/src/sql-tools/transformers/index.transformer.spec.tsindex 45fa9546c5..c9656463bf 100644--- a/server/src/sql-tools/transformers/index.transformer.spec.ts+++ b/server/src/sql-tools/transformers/index.transformer.spec.ts@@ -19,7 +19,7 @@ describe(transformIndexes.name, () => {},reason: 'unknown',}),- ).toEqual('CREATE INDEX "IDX_test" ON "table1" ("column1")');+ ).toEqual('CREATE INDEX "IDX_test" ON "table1" ("column1");');});it('should create an unique index', () => {@@ -35,7 +35,7 @@ describe(transformIndexes.name, () => {},reason: 'unknown',}),- ).toEqual('CREATE UNIQUE INDEX "IDX_test" ON "table1" ("column1")');+ ).toEqual('CREATE UNIQUE INDEX "IDX_test" ON "table1" ("column1");');});it('should create an index with a custom expression', () => {@@ -51,7 +51,7 @@ describe(transformIndexes.name, () => {},reason: 'unknown',}),- ).toEqual('CREATE INDEX "IDX_test" ON "table1" ("id" IS NOT NULL)');+ ).toEqual('CREATE INDEX "IDX_test" ON "table1" ("id" IS NOT NULL);');});it('should create an index with a where clause', () => {@@ -68,7 +68,7 @@ describe(transformIndexes.name, () => {},reason: 'unknown',}),- ).toEqual('CREATE INDEX "IDX_test" ON "table1" ("id") WHERE ("id" IS NOT NULL)');+ ).toEqual('CREATE INDEX "IDX_test" ON "table1" ("id") WHERE ("id" IS NOT NULL);');});it('should create an index with a custom expression', () => {@@ -85,7 +85,7 @@ describe(transformIndexes.name, () => {},reason: 'unknown',}),- ).toEqual('CREATE INDEX "IDX_test" ON "table1" USING gin ("id" IS NOT NULL)');+ ).toEqual('CREATE INDEX "IDX_test" ON "table1" USING gin ("id" IS NOT NULL);');});});diff --git a/server/src/sql-tools/transformers/index.transformer.ts b/server/src/sql-tools/transformers/index.transformer.tsindex a8a1303b8b..bfdbf8e157 100644--- a/server/src/sql-tools/transformers/index.transformer.ts+++ b/server/src/sql-tools/transformers/index.transformer.ts@@ -48,7 +48,7 @@ export const asIndexCreate = (index: DatabaseIndex): string => {sql += ` WHERE ${index.where}`;}- return sql;+ return sql + ';';};export const asIndexDrop = (indexName: string): string => {diff --git a/server/src/sql-tools/transformers/table.transformer.spec.ts b/server/src/sql-tools/transformers/table.transformer.spec.tsindex 756e6aeafa..662e1ba7fd 100644--- a/server/src/sql-tools/transformers/table.transformer.spec.ts+++ b/server/src/sql-tools/transformers/table.transformer.spec.ts@@ -1,9 +1,69 @@import { BaseContext } from 'src/sql-tools/contexts/base-context';import { transformTables } from 'src/sql-tools/transformers/table.transformer';+import { ConstraintType, DatabaseTable } from 'src/sql-tools/types';import { describe, expect, it } from 'vitest';const ctx = new BaseContext({});+const table1: DatabaseTable = {+ name: 'table1',+ columns: [+ {+ name: 'column1',+ tableName: 'table1',+ primary: true,+ type: 'character varying',+ nullable: true,+ isArray: false,+ synchronize: true,+ },+ {+ name: 'column2',+ tableName: 'table1',+ type: 'character varying',+ nullable: true,+ isArray: false,+ synchronize: true,+ },+ ],+ indexes: [+ {+ name: 'index1',+ tableName: 'table1',+ columnNames: ['column2'],+ unique: false,+ synchronize: true,+ },+ ],+ constraints: [+ {+ name: 'constraint1',+ tableName: 'table1',+ columnNames: ['column1'],+ type: ConstraintType.PRIMARY_KEY,+ synchronize: true,+ },+ {+ name: 'constraint2',+ tableName: 'table1',+ columnNames: ['column1'],+ type: ConstraintType.FOREIGN_KEY,+ referenceTableName: 'table2',+ referenceColumnNames: ['parentId'],+ synchronize: true,+ },+ {+ name: 'constraint3',+ tableName: 'table1',+ columnNames: ['column1'],+ type: ConstraintType.UNIQUE,+ synchronize: true,+ },+ ],+ triggers: [],+ synchronize: true,+};+describe(transformTables.name, () => {describe('TableDrop', () => {it('should work', () => {@@ -22,26 +82,19 @@ describe(transformTables.name, () => {expect(transformTables(ctx, {type: 'TableCreate',- table: {- name: 'table1',- columns: [- {- tableName: 'table1',- name: 'column1',- type: 'character varying',- nullable: true,- isArray: false,- synchronize: true,- },- ],- indexes: [],- constraints: [],- triggers: [],- synchronize: true,- },+ table: table1,reason: 'unknown',}),- ).toEqual([`CREATE TABLE "table1" ("column1" character varying);`]);+ ).toEqual([+ `CREATE TABLE "table1" (+ "column1" character varying,+ "column2" character varying,+ CONSTRAINT "constraint1" PRIMARY KEY ("column1"),+ CONSTRAINT "constraint2" FOREIGN KEY ("column1") REFERENCES "table2" ("parentId") ON UPDATE NO ACTION ON DELETE NO ACTION,+ CONSTRAINT "constraint3" UNIQUE ("column1")+);`,+ `CREATE INDEX "index1" ON "table1" ("column2");`,+ ]);});it('should handle a non-nullable column', () => {@@ -67,7 +120,11 @@ describe(transformTables.name, () => {},reason: 'unknown',}),- ).toEqual([`CREATE TABLE "table1" ("column1" character varying NOT NULL);`]);+ ).toEqual([+ `CREATE TABLE "table1" (+ "column1" character varying NOT NULL+);`,+ ]);});it('should handle a default value', () => {@@ -94,7 +151,11 @@ describe(transformTables.name, () => {},reason: 'unknown',}),- ).toEqual([`CREATE TABLE "table1" ("column1" character varying DEFAULT uuid_generate_v4());`]);+ ).toEqual([+ `CREATE TABLE "table1" (+ "column1" character varying DEFAULT uuid_generate_v4()+);`,+ ]);});it('should handle a string with a fixed length', () => {@@ -121,7 +182,11 @@ describe(transformTables.name, () => {},reason: 'unknown',}),- ).toEqual([`CREATE TABLE "table1" ("column1" character varying(2));`]);+ ).toEqual([+ `CREATE TABLE "table1" (+ "column1" character varying(2)+);`,+ ]);});it('should handle an array type', () => {@@ -147,7 +212,11 @@ describe(transformTables.name, () => {},reason: 'unknown',}),- ).toEqual([`CREATE TABLE "table1" ("column1" character varying[]);`]);+ ).toEqual([+ `CREATE TABLE "table1" (+ "column1" character varying[]+);`,+ ]);});});});diff --git a/server/src/sql-tools/transformers/table.transformer.ts b/server/src/sql-tools/transformers/table.transformer.tsindex 14cafe87f2..a81bfc25aa 100644--- a/server/src/sql-tools/transformers/table.transformer.ts+++ b/server/src/sql-tools/transformers/table.transformer.ts@@ -1,5 +1,8 @@import { asColumnComment, getColumnModifiers, getColumnType } from 'src/sql-tools/helpers';import { asColumnAlter } from 'src/sql-tools/transformers/column.transformer';+import { asConstraintBody } from 'src/sql-tools/transformers/constraint.transformer';+import { asIndexCreate } from 'src/sql-tools/transformers/index.transformer';+import { asTriggerCreate } from 'src/sql-tools/transformers/trigger.transformer';import { SqlTransformer } from 'src/sql-tools/transformers/types';import { DatabaseTable } from 'src/sql-tools/types';@@ -19,26 +22,41 @@ export const transformTables: SqlTransformer = (ctx, item) => {}};-const asTableCreate = (table: DatabaseTable): string[] => {+const asTableCreate = (table: DatabaseTable) => {const tableName = table.name;- const columnsTypes = table.columns- .map((column) => `"${column.name}" ${getColumnType(column)}` + getColumnModifiers(column))- .join(', ');- const items = [`CREATE TABLE "${tableName}" (${columnsTypes});`];++ const items: string[] = [];+ for (const column of table.columns) {+ items.push(`"${column.name}" ${getColumnType(column)}${getColumnModifiers(column)}`);+ }++ for (const constraint of table.constraints) {+ items.push(asConstraintBody(constraint));+ }++ const sql = [`CREATE TABLE "${tableName}" (\n ${items.join(',\n ')}\n);`];for (const column of table.columns) {if (column.comment) {- items.push(asColumnComment(tableName, column.name, column.comment));+ sql.push(asColumnComment(tableName, column.name, column.comment));}if (column.storage) {- items.push(...asColumnAlter(tableName, column.name, { storage: column.storage }));+ sql.push(...asColumnAlter(tableName, column.name, { storage: column.storage }));}}- return items;+ for (const index of table.indexes) {+ sql.push(asIndexCreate(index));+ }++ for (const trigger of table.triggers) {+ sql.push(asTriggerCreate(trigger));+ }++ return sql;};-const asTableDrop = (tableName: string): string => {+const asTableDrop = (tableName: string) => {return `DROP TABLE "${tableName}";`;};