Skip to content

High performance self-hosted photo and video management solution.

refactor: sql-tools readers (#19672)

Jason Rasmussenc435bdb5d3b4
diff --git a/server/src/sql-tools/from-database/index.ts b/server/src/sql-tools/from-database/index.ts
index 3c66788670..771a24d7b1 100644
--- a/server/src/sql-tools/from-database/index.ts
+++ b/server/src/sql-tools/from-database/index.ts
@@ -1,576 +1,54 @@
-import { Kysely, QueryResult, sql } from 'kysely';
+import { Kysely } from 'kysely';
import { PostgresJSDialect } from 'kysely-postgres-js';
-import { jsonArrayFrom } from 'kysely/helpers/postgres';
import { Sql } from 'postgres';
-import { parseTriggerType } from 'src/sql-tools/helpers';
-import {
- ColumnType,
- DatabaseActionType,
- DatabaseClient,
- DatabaseColumn,
- DatabaseConstraintType,
- DatabaseEnum,
- DatabaseExtension,
- DatabaseFunction,
- DatabaseParameter,
- DatabaseSchema,
- DatabaseTable,
- LoadSchemaOptions,
- ParameterScope,
- PostgresDB,
-} from 'src/sql-tools/types';
+import { readColumns } from 'src/sql-tools/from-database/readers/column.reader';
+import { readComments } from 'src/sql-tools/from-database/readers/comment.reader';
+import { readConstraints } from 'src/sql-tools/from-database/readers/constraint.reader';
+import { readExtensions } from 'src/sql-tools/from-database/readers/extension.reader';
+import { readFunctions } from 'src/sql-tools/from-database/readers/function.reader';
+import { readIndexes } from 'src/sql-tools/from-database/readers/index.reader';
+import { readName } from 'src/sql-tools/from-database/readers/name.reader';
+import { readParameters } from 'src/sql-tools/from-database/readers/parameter.reader';
+import { readTables } from 'src/sql-tools/from-database/readers/table.reader';
+import { readTriggers } from 'src/sql-tools/from-database/readers/trigger.reader';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+import { DatabaseSchema, LoadSchemaOptions, PostgresDB } from 'src/sql-tools/types';
+
+const readers: DatabaseReader[] = [
+ //
+ readName,
+ readParameters,
+ readExtensions,
+ readFunctions,
+ readTables,
+ readColumns,
+ readIndexes,
+ readConstraints,
+ readTriggers,
+ readComments,
+];
/**
* Load the database schema from the database
*/
export const schemaFromDatabase = async (postgres: Sql, options: LoadSchemaOptions = {}): Promise<DatabaseSchema> => {
- const db = createDatabaseClient(postgres);
-
- const warnings: string[] = [];
- const warn = (message: string) => {
- warnings.push(message);
+ const schema: DatabaseSchema = {
+ name: 'immich',
+ schemaName: options.schemaName || 'public',
+ parameters: [],
+ functions: [],
+ enums: [],
+ extensions: [],
+ tables: [],
+ warnings: [],
};
- const schemaName = options.schemaName || 'public';
- const tablesMap: Record<string, DatabaseTable> = {};
-
- const [
- databaseName,
- tables,
- columns,
- indexes,
- constraints,
- enums,
- routines,
- extensions,
- triggers,
- parameters,
- comments,
- ] = await Promise.all([
- getDatabaseName(db),
- getTables(db, schemaName),
- getTableColumns(db, schemaName),
- getTableIndexes(db, schemaName),
- getTableConstraints(db, schemaName),
- getUserDefinedEnums(db, schemaName),
- getRoutines(db, schemaName),
- getExtensions(db),
- getTriggers(db, schemaName),
- getParameters(db),
- getObjectComments(db),
- ]);
-
- const schemaEnums: DatabaseEnum[] = [];
- const schemaFunctions: DatabaseFunction[] = [];
- const schemaExtensions: DatabaseExtension[] = [];
- const schemaParameters: DatabaseParameter[] = [];
-
- const enumMap = Object.fromEntries(enums.map((e) => [e.name, e.values]));
-
- for (const { name } of extensions) {
- schemaExtensions.push({ name, synchronize: true });
- }
-
- for (const { name, values } of enums) {
- schemaEnums.push({ name, values, synchronize: true });
- }
-
- for (const parameter of parameters) {
- schemaParameters.push({
- name: parameter.name,
- value: parameter.value,
- databaseName,
- scope: parameter.scope as ParameterScope,
- synchronize: true,
- });
- }
-
- for (const { name, expression } of routines) {
- schemaFunctions.push({
- name,
- // TODO read expression from the overrides table
- expression,
- synchronize: true,
- });
- }
-
- // add tables
- for (const table of tables) {
- const tableName = table.table_name;
- if (tablesMap[tableName]) {
- continue;
- }
-
- tablesMap[table.table_name] = {
- name: table.table_name,
- columns: [],
- indexes: [],
- triggers: [],
- constraints: [],
- synchronize: true,
- };
- }
-
- // add columns to tables
- for (const column of columns) {
- const table = tablesMap[column.table_name];
- if (!table) {
- continue;
- }
-
- const columnName = column.column_name;
-
- const item: DatabaseColumn = {
- type: column.data_type as ColumnType,
- name: columnName,
- tableName: column.table_name,
- nullable: column.is_nullable === 'YES',
- isArray: column.array_type !== null,
- numericPrecision: column.numeric_precision ?? undefined,
- numericScale: column.numeric_scale ?? undefined,
- length: column.character_maximum_length ?? undefined,
- default: column.column_default ?? undefined,
- synchronize: true,
- };
-
- const columnLabel = `${table.name}.${columnName}`;
-
- switch (column.data_type) {
- // array types
- case 'ARRAY': {
- if (!column.array_type) {
- warn(`Unable to find type for ${columnLabel} (ARRAY)`);
- continue;
- }
- item.type = column.array_type as ColumnType;
- break;
- }
-
- // enum types
- case 'USER-DEFINED': {
- if (!enumMap[column.udt_name]) {
- warn(`Unable to find type for ${columnLabel} (ENUM)`);
- continue;
- }
-
- item.type = 'enum';
- item.enumName = column.udt_name;
- break;
- }
- }
-
- table.columns.push(item);
- }
-
- // add table indexes
- for (const index of indexes) {
- const table = tablesMap[index.table_name];
- if (!table) {
- continue;
- }
-
- const indexName = index.index_name;
-
- table.indexes.push({
- name: indexName,
- tableName: index.table_name,
- columnNames: index.column_names ?? undefined,
- expression: index.expression ?? undefined,
- using: index.using,
- where: index.where ?? undefined,
- unique: index.unique,
- synchronize: true,
- });
- }
-
- // add table constraints
- for (const constraint of constraints) {
- const table = tablesMap[constraint.table_name];
- if (!table) {
- continue;
- }
-
- const constraintName = constraint.constraint_name;
-
- switch (constraint.constraint_type) {
- // primary key constraint
- case 'p': {
- if (!constraint.column_names) {
- warn(`Skipping CONSTRAINT "${constraintName}", no columns found`);
- continue;
- }
- table.constraints.push({
- type: DatabaseConstraintType.PRIMARY_KEY,
- name: constraintName,
- tableName: constraint.table_name,
- columnNames: constraint.column_names,
- synchronize: true,
- });
- break;
- }
-
- // foreign key constraint
- case 'f': {
- if (!constraint.column_names || !constraint.reference_table_name || !constraint.reference_column_names) {
- warn(
- `Skipping CONSTRAINT "${constraintName}", missing either columns, referenced table, or referenced columns,`,
- );
- continue;
- }
-
- table.constraints.push({
- type: DatabaseConstraintType.FOREIGN_KEY,
- name: constraintName,
- tableName: constraint.table_name,
- columnNames: constraint.column_names,
- referenceTableName: constraint.reference_table_name,
- referenceColumnNames: constraint.reference_column_names,
- onUpdate: asDatabaseAction(constraint.update_action),
- onDelete: asDatabaseAction(constraint.delete_action),
- synchronize: true,
- });
- break;
- }
-
- // unique constraint
- case 'u': {
- table.constraints.push({
- type: DatabaseConstraintType.UNIQUE,
- name: constraintName,
- tableName: constraint.table_name,
- columnNames: constraint.column_names as string[],
- synchronize: true,
- });
- break;
- }
-
- // check constraint
- case 'c': {
- table.constraints.push({
- type: DatabaseConstraintType.CHECK,
- name: constraint.constraint_name,
- tableName: constraint.table_name,
- expression: constraint.expression.replace('CHECK ', ''),
- synchronize: true,
- });
- break;
- }
- }
- }
-
- // add triggers to tables
- for (const trigger of triggers) {
- const table = tablesMap[trigger.table_name];
- if (!table) {
- continue;
- }
-
- table.triggers.push({
- name: trigger.name,
- tableName: trigger.table_name,
- functionName: trigger.function_name,
- referencingNewTableAs: trigger.referencing_new_table_as ?? undefined,
- referencingOldTableAs: trigger.referencing_old_table_as ?? undefined,
- when: trigger.when_expression,
- synchronize: true,
- ...parseTriggerType(trigger.type),
- });
- }
-
- for (const comment of comments) {
- if (comment.object_type === 'r') {
- const table = tablesMap[comment.object_name];
- if (!table) {
- continue;
- }
-
- if (comment.column_name) {
- const column = table.columns.find(({ name }) => name === comment.column_name);
- if (column) {
- column.comment = comment.value;
- }
- }
- }
+ const db = new Kysely<PostgresDB>({ dialect: new PostgresJSDialect({ postgres }) });
+ for (const reader of readers) {
+ await reader(schema, db);
}
await db.destroy();
- return {
- name: databaseName,
- schemaName,
- parameters: schemaParameters,
- functions: schemaFunctions,
- enums: schemaEnums,
- extensions: schemaExtensions,
- tables: Object.values(tablesMap),
- warnings,
- };
-};
-
-const createDatabaseClient = (postgres: Sql): DatabaseClient =>
- new Kysely<PostgresDB>({ dialect: new PostgresJSDialect({ postgres }) });
-
-const asDatabaseAction = (action: string) => {
- switch (action) {
- case 'a': {
- return DatabaseActionType.NO_ACTION;
- }
- case 'c': {
- return DatabaseActionType.CASCADE;
- }
- case 'r': {
- return DatabaseActionType.RESTRICT;
- }
- case 'n': {
- return DatabaseActionType.SET_NULL;
- }
- case 'd': {
- return DatabaseActionType.SET_DEFAULT;
- }
-
- default: {
- return DatabaseActionType.NO_ACTION;
- }
- }
-};
-
-const getDatabaseName = async (db: DatabaseClient) => {
- const result = (await sql`SELECT current_database() as name`.execute(db)) as QueryResult<{ name: string }>;
- return result.rows[0].name;
-};
-
-const getTables = (db: DatabaseClient, schemaName: string) => {
- return db
- .selectFrom('information_schema.tables')
- .where('table_schema', '=', schemaName)
- .where('table_type', '=', sql.lit('BASE TABLE'))
- .selectAll()
- .execute();
-};
-
-const getTableColumns = (db: DatabaseClient, schemaName: string) => {
- return db
- .selectFrom('information_schema.columns as c')
- .leftJoin('information_schema.element_types as o', (join) =>
- join
- .onRef('c.table_catalog', '=', 'o.object_catalog')
- .onRef('c.table_schema', '=', 'o.object_schema')
- .onRef('c.table_name', '=', 'o.object_name')
- .on('o.object_type', '=', sql.lit('TABLE'))
- .onRef('c.dtd_identifier', '=', 'o.collection_type_identifier'),
- )
- .leftJoin('pg_type as t', (join) =>
- join.onRef('t.typname', '=', 'c.udt_name').on('c.data_type', '=', sql.lit('USER-DEFINED')),
- )
- .leftJoin('pg_enum as e', (join) => join.onRef('e.enumtypid', '=', 't.oid'))
- .select([
- 'c.table_name',
- 'c.column_name',
-
- // is ARRAY, USER-DEFINED, or data type
- 'c.data_type',
- 'c.column_default',
- 'c.is_nullable',
- 'c.character_maximum_length',
-
- // number types
- 'c.numeric_precision',
- 'c.numeric_scale',
-
- // date types
- 'c.datetime_precision',
-
- // user defined type
- 'c.udt_catalog',
- 'c.udt_schema',
- 'c.udt_name',
-
- // data type for ARRAYs
- 'o.data_type as array_type',
- ])
- .where('table_schema', '=', schemaName)
- .execute();
-};
-
-const getTableIndexes = (db: DatabaseClient, schemaName: string) => {
- return (
- db
- .selectFrom('pg_index as ix')
- // matching index, which has column information
- .innerJoin('pg_class as i', 'ix.indexrelid', 'i.oid')
- .innerJoin('pg_am as a', 'i.relam', 'a.oid')
- // matching table
- .innerJoin('pg_class as t', 'ix.indrelid', 't.oid')
- // namespace
- .innerJoin('pg_namespace', 'pg_namespace.oid', 'i.relnamespace')
- // PK and UQ constraints automatically have indexes, so we can ignore those
- .leftJoin('pg_constraint', (join) =>
- join
- .onRef('pg_constraint.conindid', '=', 'i.oid')
- .on('pg_constraint.contype', 'in', [sql.lit('p'), sql.lit('u')]),
- )
- .where('pg_constraint.oid', 'is', null)
- .select((eb) => [
- 'i.relname as index_name',
- 't.relname as table_name',
- 'ix.indisunique as unique',
- 'a.amname as using',
- eb.fn<string>('pg_get_expr', ['ix.indexprs', 'ix.indrelid']).as('expression'),
- eb.fn<string>('pg_get_expr', ['ix.indpred', 'ix.indrelid']).as('where'),
- eb
- .selectFrom('pg_attribute as a')
- .where('t.relkind', '=', sql.lit('r'))
- .whereRef('a.attrelid', '=', 't.oid')
- // list of columns numbers in the index
- .whereRef('a.attnum', '=', sql`any("ix"."indkey")`)
- .select((eb) => eb.fn<string[]>('json_agg', ['a.attname']).as('column_name'))
- .as('column_names'),
- ])
- .where('pg_namespace.nspname', '=', schemaName)
- .where('ix.indisprimary', '=', sql.lit(false))
- .execute()
- );
-};
-
-const getTableConstraints = (db: DatabaseClient, schemaName: string) => {
- return db
- .selectFrom('pg_constraint')
- .innerJoin('pg_namespace', 'pg_namespace.oid', 'pg_constraint.connamespace') // namespace
- .innerJoin('pg_class as source_table', (join) =>
- join.onRef('source_table.oid', '=', 'pg_constraint.conrelid').on('source_table.relkind', 'in', [
- // ordinary table
- sql.lit('r'),
- // partitioned table
- sql.lit('p'),
- // foreign table
- sql.lit('f'),
- ]),
- ) // table
- .leftJoin('pg_class as reference_table', 'reference_table.oid', 'pg_constraint.confrelid') // reference table
- .select((eb) => [
- 'pg_constraint.contype as constraint_type',
- 'pg_constraint.conname as constraint_name',
- 'source_table.relname as table_name',
- 'reference_table.relname as reference_table_name',
- 'pg_constraint.confupdtype as update_action',
- 'pg_constraint.confdeltype as delete_action',
- // 'pg_constraint.oid as constraint_id',
- eb
- .selectFrom('pg_attribute')
- // matching table for PK, FK, and UQ
- .whereRef('pg_attribute.attrelid', '=', 'pg_constraint.conrelid')
- .whereRef('pg_attribute.attnum', '=', sql`any("pg_constraint"."conkey")`)
- .select((eb) => eb.fn<string[]>('json_agg', ['pg_attribute.attname']).as('column_name'))
- .as('column_names'),
- eb
- .selectFrom('pg_attribute')
- // matching foreign table for FK
- .whereRef('pg_attribute.attrelid', '=', 'pg_constraint.confrelid')
- .whereRef('pg_attribute.attnum', '=', sql`any("pg_constraint"."confkey")`)
- .select((eb) => eb.fn<string[]>('json_agg', ['pg_attribute.attname']).as('column_name'))
- .as('reference_column_names'),
- eb.fn<string>('pg_get_constraintdef', ['pg_constraint.oid']).as('expression'),
- ])
- .where('pg_namespace.nspname', '=', schemaName)
- .execute();
-};
-
-const getUserDefinedEnums = async (db: DatabaseClient, schemaName: string) => {
- const items = await db
- .selectFrom('pg_type')
- .innerJoin('pg_namespace', (join) =>
- join.onRef('pg_namespace.oid', '=', 'pg_type.typnamespace').on('pg_namespace.nspname', '=', schemaName),
- )
- .where('typtype', '=', sql.lit('e'))
- .select((eb) => [
- 'pg_type.typname as name',
- jsonArrayFrom(
- eb.selectFrom('pg_enum as e').select(['e.enumlabel as value']).whereRef('e.enumtypid', '=', 'pg_type.oid'),
- ).as('values'),
- ])
- .execute();
-
- return items.map((item) => ({
- name: item.name,
- values: item.values.map(({ value }) => value),
- }));
-};
-
-const getRoutines = async (db: DatabaseClient, schemaName: string) => {
- return db
- .selectFrom('pg_proc as p')
- .innerJoin('pg_namespace', 'pg_namespace.oid', 'p.pronamespace')
- .leftJoin('pg_depend as d', (join) => join.onRef('d.objid', '=', 'p.oid').on('d.deptype', '=', sql.lit('e')))
- .where('d.objid', 'is', sql.lit(null))
- .where('p.prokind', '=', sql.lit('f'))
- .where('pg_namespace.nspname', '=', schemaName)
- .select((eb) => [
- 'p.proname as name',
- eb.fn<string>('pg_get_function_identity_arguments', ['p.oid']).as('arguments'),
- eb.fn<string>('pg_get_functiondef', ['p.oid']).as('expression'),
- ])
- .execute();
-};
-
-const getExtensions = async (db: DatabaseClient) => {
- return (
- db
- .selectFrom('pg_catalog.pg_extension')
- // .innerJoin('pg_namespace', 'pg_namespace.oid', 'pg_catalog.pg_extension.extnamespace')
- // .where('pg_namespace.nspname', '=', schemaName)
- .select(['extname as name', 'extversion as version'])
- .execute()
- );
-};
-
-const getTriggers = async (db: Kysely<PostgresDB>, schemaName: string) => {
- return db
- .selectFrom('pg_trigger as t')
- .innerJoin('pg_proc as p', 't.tgfoid', 'p.oid')
- .innerJoin('pg_namespace as n', 'p.pronamespace', 'n.oid')
- .innerJoin('pg_class as c', 't.tgrelid', 'c.oid')
- .select((eb) => [
- 't.tgname as name',
- 't.tgenabled as enabled',
- 't.tgtype as type',
- 't.tgconstraint as _constraint',
- 't.tgdeferrable as is_deferrable',
- 't.tginitdeferred as is_initially_deferred',
- 't.tgargs as arguments',
- 't.tgoldtable as referencing_old_table_as',
- 't.tgnewtable as referencing_new_table_as',
- eb.fn<string>('pg_get_expr', ['t.tgqual', 't.tgrelid']).as('when_expression'),
- 'p.proname as function_name',
- 'c.relname as table_name',
- ])
- .where('t.tgisinternal', '=', false) // Exclude internal system triggers
- .where('n.nspname', '=', schemaName)
- .execute();
-};
-
-const getParameters = async (db: Kysely<PostgresDB>) => {
- return db
- .selectFrom('pg_settings')
- .where('source', 'in', [sql.lit('database'), sql.lit('user')])
- .select(['name', 'setting as value', 'source as scope'])
- .execute();
-};
-
-const getObjectComments = async (db: Kysely<PostgresDB>) => {
- return db
- .selectFrom('pg_description as d')
- .innerJoin('pg_class as c', 'd.objoid', 'c.oid')
- .leftJoin('pg_attribute as a', (join) =>
- join.onRef('a.attrelid', '=', 'c.oid').onRef('a.attnum', '=', 'd.objsubid'),
- )
- .select([
- 'c.relname as object_name',
- 'c.relkind as object_type',
- 'd.description as value',
- 'a.attname as column_name',
- ])
- .where('d.description', 'is not', null)
- .orderBy('object_type')
- .orderBy('object_name')
- .execute();
+ return schema;
};
diff --git a/server/src/sql-tools/from-database/readers/column.reader.ts b/server/src/sql-tools/from-database/readers/column.reader.ts
new file mode 100644
index 0000000000..167124bac9
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/column.reader.ts
@@ -0,0 +1,119 @@
+import { sql } from 'kysely';
+import { jsonArrayFrom } from 'kysely/helpers/postgres';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+import { ColumnType, DatabaseColumn } from 'src/sql-tools/types';
+
+export const readColumns: DatabaseReader = async (schema, db) => {
+ const columns = await db
+ .selectFrom('information_schema.columns as c')
+ .leftJoin('information_schema.element_types as o', (join) =>
+ join
+ .onRef('c.table_catalog', '=', 'o.object_catalog')
+ .onRef('c.table_schema', '=', 'o.object_schema')
+ .onRef('c.table_name', '=', 'o.object_name')
+ .on('o.object_type', '=', sql.lit('TABLE'))
+ .onRef('c.dtd_identifier', '=', 'o.collection_type_identifier'),
+ )
+ .leftJoin('pg_type as t', (join) =>
+ join.onRef('t.typname', '=', 'c.udt_name').on('c.data_type', '=', sql.lit('USER-DEFINED')),
+ )
+ .leftJoin('pg_enum as e', (join) => join.onRef('e.enumtypid', '=', 't.oid'))
+ .select([
+ 'c.table_name',
+ 'c.column_name',
+
+ // is ARRAY, USER-DEFINED, or data type
+ 'c.data_type',
+ 'c.column_default',
+ 'c.is_nullable',
+ 'c.character_maximum_length',
+
+ // number types
+ 'c.numeric_precision',
+ 'c.numeric_scale',
+
+ // date types
+ 'c.datetime_precision',
+
+ // user defined type
+ 'c.udt_catalog',
+ 'c.udt_schema',
+ 'c.udt_name',
+
+ // data type for ARRAYs
+ 'o.data_type as array_type',
+ ])
+ .where('table_schema', '=', schema.schemaName)
+ .execute();
+
+ const enumRaw = await db
+ .selectFrom('pg_type')
+ .innerJoin('pg_namespace', (join) =>
+ join.onRef('pg_namespace.oid', '=', 'pg_type.typnamespace').on('pg_namespace.nspname', '=', schema.schemaName),
+ )
+ .where('typtype', '=', sql.lit('e'))
+ .select((eb) => [
+ 'pg_type.typname as name',
+ jsonArrayFrom(
+ eb.selectFrom('pg_enum as e').select(['e.enumlabel as value']).whereRef('e.enumtypid', '=', 'pg_type.oid'),
+ ).as('values'),
+ ])
+ .execute();
+
+ const enums = enumRaw.map((item) => ({ name: item.name, values: item.values.map(({ value }) => value) }));
+ for (const { name, values } of enums) {
+ schema.enums.push({ name, values, synchronize: true });
+ }
+
+ const enumMap = Object.fromEntries(enums.map((e) => [e.name, e.values]));
+ // add columns to tables
+ for (const column of columns) {
+ const table = schema.tables.find((table) => table.name === column.table_name);
+ if (!table) {
+ continue;
+ }
+
+ const columnName = column.column_name;
+
+ const item: DatabaseColumn = {
+ type: column.data_type as ColumnType,
+ name: columnName,
+ tableName: column.table_name,
+ nullable: column.is_nullable === 'YES',
+ isArray: column.array_type !== null,
+ numericPrecision: column.numeric_precision ?? undefined,
+ numericScale: column.numeric_scale ?? undefined,
+ length: column.character_maximum_length ?? undefined,
+ default: column.column_default ?? undefined,
+ synchronize: true,
+ };
+
+ const columnLabel = `${table.name}.${columnName}`;
+
+ switch (column.data_type) {
+ // array types
+ case 'ARRAY': {
+ if (!column.array_type) {
+ schema.warnings.push(`Unable to find type for ${columnLabel} (ARRAY)`);
+ continue;
+ }
+ item.type = column.array_type as ColumnType;
+ break;
+ }
+
+ // enum types
+ case 'USER-DEFINED': {
+ if (!enumMap[column.udt_name]) {
+ schema.warnings.push(`Unable to find type for ${columnLabel} (ENUM)`);
+ continue;
+ }
+
+ item.type = 'enum';
+ item.enumName = column.udt_name;
+ break;
+ }
+ }
+
+ table.columns.push(item);
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/comment.reader.ts b/server/src/sql-tools/from-database/readers/comment.reader.ts
new file mode 100644
index 0000000000..3dd4f4adc9
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/comment.reader.ts
@@ -0,0 +1,36 @@
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+
+export const readComments: DatabaseReader = async (schema, db) => {
+ const comments = await db
+ .selectFrom('pg_description as d')
+ .innerJoin('pg_class as c', 'd.objoid', 'c.oid')
+ .leftJoin('pg_attribute as a', (join) =>
+ join.onRef('a.attrelid', '=', 'c.oid').onRef('a.attnum', '=', 'd.objsubid'),
+ )
+ .select([
+ 'c.relname as object_name',
+ 'c.relkind as object_type',
+ 'd.description as value',
+ 'a.attname as column_name',
+ ])
+ .where('d.description', 'is not', null)
+ .orderBy('object_type')
+ .orderBy('object_name')
+ .execute();
+
+ for (const comment of comments) {
+ if (comment.object_type === 'r') {
+ const table = schema.tables.find((table) => table.name === comment.object_name);
+ if (!table) {
+ continue;
+ }
+
+ if (comment.column_name) {
+ const column = table.columns.find(({ name }) => name === comment.column_name);
+ if (column) {
+ column.comment = comment.value;
+ }
+ }
+ }
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/constraint.reader.ts b/server/src/sql-tools/from-database/readers/constraint.reader.ts
new file mode 100644
index 0000000000..a633324f25
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/constraint.reader.ts
@@ -0,0 +1,144 @@
+import { sql } from 'kysely';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+import { DatabaseActionType, DatabaseConstraintType } from 'src/sql-tools/types';
+
+export const readConstraints: DatabaseReader = async (schema, db) => {
+ const constraints = await db
+ .selectFrom('pg_constraint')
+ .innerJoin('pg_namespace', 'pg_namespace.oid', 'pg_constraint.connamespace') // namespace
+ .innerJoin('pg_class as source_table', (join) =>
+ join.onRef('source_table.oid', '=', 'pg_constraint.conrelid').on('source_table.relkind', 'in', [
+ // ordinary table
+ sql.lit('r'),
+ // partitioned table
+ sql.lit('p'),
+ // foreign table
+ sql.lit('f'),
+ ]),
+ ) // table
+ .leftJoin('pg_class as reference_table', 'reference_table.oid', 'pg_constraint.confrelid') // reference table
+ .select((eb) => [
+ 'pg_constraint.contype as constraint_type',
+ 'pg_constraint.conname as constraint_name',
+ 'source_table.relname as table_name',
+ 'reference_table.relname as reference_table_name',
+ 'pg_constraint.confupdtype as update_action',
+ 'pg_constraint.confdeltype as delete_action',
+ // 'pg_constraint.oid as constraint_id',
+ eb
+ .selectFrom('pg_attribute')
+ // matching table for PK, FK, and UQ
+ .whereRef('pg_attribute.attrelid', '=', 'pg_constraint.conrelid')
+ .whereRef('pg_attribute.attnum', '=', sql`any("pg_constraint"."conkey")`)
+ .select((eb) => eb.fn<string[]>('json_agg', ['pg_attribute.attname']).as('column_name'))
+ .as('column_names'),
+ eb
+ .selectFrom('pg_attribute')
+ // matching foreign table for FK
+ .whereRef('pg_attribute.attrelid', '=', 'pg_constraint.confrelid')
+ .whereRef('pg_attribute.attnum', '=', sql`any("pg_constraint"."confkey")`)
+ .select((eb) => eb.fn<string[]>('json_agg', ['pg_attribute.attname']).as('column_name'))
+ .as('reference_column_names'),
+ eb.fn<string>('pg_get_constraintdef', ['pg_constraint.oid']).as('expression'),
+ ])
+ .where('pg_namespace.nspname', '=', schema.schemaName)
+ .execute();
+
+ for (const constraint of constraints) {
+ const table = schema.tables.find((table) => table.name === constraint.table_name);
+ if (!table) {
+ continue;
+ }
+
+ const constraintName = constraint.constraint_name;
+
+ switch (constraint.constraint_type) {
+ // primary key constraint
+ case 'p': {
+ if (!constraint.column_names) {
+ schema.warnings.push(`Skipping CONSTRAINT "${constraintName}", no columns found`);
+ continue;
+ }
+ table.constraints.push({
+ type: DatabaseConstraintType.PRIMARY_KEY,
+ name: constraintName,
+ tableName: constraint.table_name,
+ columnNames: constraint.column_names,
+ synchronize: true,
+ });
+ break;
+ }
+
+ // foreign key constraint
+ case 'f': {
+ if (!constraint.column_names || !constraint.reference_table_name || !constraint.reference_column_names) {
+ schema.warnings.push(
+ `Skipping CONSTRAINT "${constraintName}", missing either columns, referenced table, or referenced columns,`,
+ );
+ continue;
+ }
+
+ table.constraints.push({
+ type: DatabaseConstraintType.FOREIGN_KEY,
+ name: constraintName,
+ tableName: constraint.table_name,
+ columnNames: constraint.column_names,
+ referenceTableName: constraint.reference_table_name,
+ referenceColumnNames: constraint.reference_column_names,
+ onUpdate: asDatabaseAction(constraint.update_action),
+ onDelete: asDatabaseAction(constraint.delete_action),
+ synchronize: true,
+ });
+ break;
+ }
+
+ // unique constraint
+ case 'u': {
+ table.constraints.push({
+ type: DatabaseConstraintType.UNIQUE,
+ name: constraintName,
+ tableName: constraint.table_name,
+ columnNames: constraint.column_names as string[],
+ synchronize: true,
+ });
+ break;
+ }
+
+ // check constraint
+ case 'c': {
+ table.constraints.push({
+ type: DatabaseConstraintType.CHECK,
+ name: constraint.constraint_name,
+ tableName: constraint.table_name,
+ expression: constraint.expression.replace('CHECK ', ''),
+ synchronize: true,
+ });
+ break;
+ }
+ }
+ }
+};
+
+const asDatabaseAction = (action: string) => {
+ switch (action) {
+ case 'a': {
+ return DatabaseActionType.NO_ACTION;
+ }
+ case 'c': {
+ return DatabaseActionType.CASCADE;
+ }
+ case 'r': {
+ return DatabaseActionType.RESTRICT;
+ }
+ case 'n': {
+ return DatabaseActionType.SET_NULL;
+ }
+ case 'd': {
+ return DatabaseActionType.SET_DEFAULT;
+ }
+
+ default: {
+ return DatabaseActionType.NO_ACTION;
+ }
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/extension.reader.ts b/server/src/sql-tools/from-database/readers/extension.reader.ts
new file mode 100644
index 0000000000..c0a59f85d8
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/extension.reader.ts
@@ -0,0 +1,14 @@
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+
+export const readExtensions: DatabaseReader = async (schema, db) => {
+ const extensions = await db
+ .selectFrom('pg_catalog.pg_extension')
+ // .innerJoin('pg_namespace', 'pg_namespace.oid', 'pg_catalog.pg_extension.extnamespace')
+ // .where('pg_namespace.nspname', '=', schemaName)
+ .select(['extname as name', 'extversion as version'])
+ .execute();
+
+ for (const { name } of extensions) {
+ schema.extensions.push({ name, synchronize: true });
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/function.reader.ts b/server/src/sql-tools/from-database/readers/function.reader.ts
new file mode 100644
index 0000000000..0c81117d7c
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/function.reader.ts
@@ -0,0 +1,27 @@
+import { sql } from 'kysely';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+
+export const readFunctions: DatabaseReader = async (schema, db) => {
+ const routines = await db
+ .selectFrom('pg_proc as p')
+ .innerJoin('pg_namespace', 'pg_namespace.oid', 'p.pronamespace')
+ .leftJoin('pg_depend as d', (join) => join.onRef('d.objid', '=', 'p.oid').on('d.deptype', '=', sql.lit('e')))
+ .where('d.objid', 'is', sql.lit(null))
+ .where('p.prokind', '=', sql.lit('f'))
+ .where('pg_namespace.nspname', '=', schema.schemaName)
+ .select((eb) => [
+ 'p.proname as name',
+ eb.fn<string>('pg_get_function_identity_arguments', ['p.oid']).as('arguments'),
+ eb.fn<string>('pg_get_functiondef', ['p.oid']).as('expression'),
+ ])
+ .execute();
+
+ for (const { name, expression } of routines) {
+ schema.functions.push({
+ name,
+ // TODO read expression from the overrides table
+ expression,
+ synchronize: true,
+ });
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/index.reader.ts b/server/src/sql-tools/from-database/readers/index.reader.ts
new file mode 100644
index 0000000000..681ba8f9ef
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/index.reader.ts
@@ -0,0 +1,58 @@
+import { sql } from 'kysely';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+
+export const readIndexes: DatabaseReader = async (schema, db) => {
+ const indexes = await db
+ .selectFrom('pg_index as ix')
+ // matching index, which has column information
+ .innerJoin('pg_class as i', 'ix.indexrelid', 'i.oid')
+ .innerJoin('pg_am as a', 'i.relam', 'a.oid')
+ // matching table
+ .innerJoin('pg_class as t', 'ix.indrelid', 't.oid')
+ // namespace
+ .innerJoin('pg_namespace', 'pg_namespace.oid', 'i.relnamespace')
+ // PK and UQ constraints automatically have indexes, so we can ignore those
+ .leftJoin('pg_constraint', (join) =>
+ join
+ .onRef('pg_constraint.conindid', '=', 'i.oid')
+ .on('pg_constraint.contype', 'in', [sql.lit('p'), sql.lit('u')]),
+ )
+ .where('pg_constraint.oid', 'is', null)
+ .select((eb) => [
+ 'i.relname as index_name',
+ 't.relname as table_name',
+ 'ix.indisunique as unique',
+ 'a.amname as using',
+ eb.fn<string>('pg_get_expr', ['ix.indexprs', 'ix.indrelid']).as('expression'),
+ eb.fn<string>('pg_get_expr', ['ix.indpred', 'ix.indrelid']).as('where'),
+ eb
+ .selectFrom('pg_attribute as a')
+ .where('t.relkind', '=', sql.lit('r'))
+ .whereRef('a.attrelid', '=', 't.oid')
+ // list of columns numbers in the index
+ .whereRef('a.attnum', '=', sql`any("ix"."indkey")`)
+ .select((eb) => eb.fn<string[]>('json_agg', ['a.attname']).as('column_name'))
+ .as('column_names'),
+ ])
+ .where('pg_namespace.nspname', '=', schema.schemaName)
+ .where('ix.indisprimary', '=', sql.lit(false))
+ .execute();
+
+ for (const index of indexes) {
+ const table = schema.tables.find((table) => table.name === index.table_name);
+ if (!table) {
+ continue;
+ }
+
+ table.indexes.push({
+ name: index.index_name,
+ tableName: index.table_name,
+ columnNames: index.column_names ?? undefined,
+ expression: index.expression ?? undefined,
+ using: index.using,
+ where: index.where ?? undefined,
+ unique: index.unique,
+ synchronize: true,
+ });
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/name.reader.ts b/server/src/sql-tools/from-database/readers/name.reader.ts
new file mode 100644
index 0000000000..dc3d4ec745
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/name.reader.ts
@@ -0,0 +1,8 @@
+import { QueryResult, sql } from 'kysely';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+
+export const readName: DatabaseReader = async (schema, db) => {
+ const result = (await sql`SELECT current_database() as name`.execute(db)) as QueryResult<{ name: string }>;
+
+ schema.name = result.rows[0].name;
+};
diff --git a/server/src/sql-tools/from-database/readers/parameter.reader.ts b/server/src/sql-tools/from-database/readers/parameter.reader.ts
new file mode 100644
index 0000000000..f58718a6b6
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/parameter.reader.ts
@@ -0,0 +1,21 @@
+import { sql } from 'kysely';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+import { ParameterScope } from 'src/sql-tools/types';
+
+export const readParameters: DatabaseReader = async (schema, db) => {
+ const parameters = await db
+ .selectFrom('pg_settings')
+ .where('source', 'in', [sql.lit('database'), sql.lit('user')])
+ .select(['name', 'setting as value', 'source as scope'])
+ .execute();
+
+ for (const parameter of parameters) {
+ schema.parameters.push({
+ name: parameter.name,
+ value: parameter.value,
+ databaseName: schema.name,
+ scope: parameter.scope as ParameterScope,
+ synchronize: true,
+ });
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/table.reader.ts b/server/src/sql-tools/from-database/readers/table.reader.ts
new file mode 100644
index 0000000000..6dbf9fc8b8
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/table.reader.ts
@@ -0,0 +1,22 @@
+import { sql } from 'kysely';
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+
+export const readTables: DatabaseReader = async (schema, db) => {
+ const tables = await db
+ .selectFrom('information_schema.tables')
+ .where('table_schema', '=', schema.schemaName)
+ .where('table_type', '=', sql.lit('BASE TABLE'))
+ .selectAll()
+ .execute();
+
+ for (const table of tables) {
+ schema.tables.push({
+ name: table.table_name,
+ columns: [],
+ indexes: [],
+ triggers: [],
+ constraints: [],
+ synchronize: true,
+ });
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/trigger.reader.ts b/server/src/sql-tools/from-database/readers/trigger.reader.ts
new file mode 100644
index 0000000000..a174a7502b
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/trigger.reader.ts
@@ -0,0 +1,46 @@
+import { DatabaseReader } from 'src/sql-tools/from-database/readers/type';
+import { parseTriggerType } from 'src/sql-tools/helpers';
+
+export const readTriggers: DatabaseReader = async (schema, db) => {
+ const triggers = await db
+ .selectFrom('pg_trigger as t')
+ .innerJoin('pg_proc as p', 't.tgfoid', 'p.oid')
+ .innerJoin('pg_namespace as n', 'p.pronamespace', 'n.oid')
+ .innerJoin('pg_class as c', 't.tgrelid', 'c.oid')
+ .select((eb) => [
+ 't.tgname as name',
+ 't.tgenabled as enabled',
+ 't.tgtype as type',
+ 't.tgconstraint as _constraint',
+ 't.tgdeferrable as is_deferrable',
+ 't.tginitdeferred as is_initially_deferred',
+ 't.tgargs as arguments',
+ 't.tgoldtable as referencing_old_table_as',
+ 't.tgnewtable as referencing_new_table_as',
+ eb.fn<string>('pg_get_expr', ['t.tgqual', 't.tgrelid']).as('when_expression'),
+ 'p.proname as function_name',
+ 'c.relname as table_name',
+ ])
+ .where('t.tgisinternal', '=', false) // Exclude internal system triggers
+ .where('n.nspname', '=', schema.schemaName)
+ .execute();
+
+ // add triggers to tables
+ for (const trigger of triggers) {
+ const table = schema.tables.find((table) => table.name === trigger.table_name);
+ if (!table) {
+ continue;
+ }
+
+ table.triggers.push({
+ name: trigger.name,
+ tableName: trigger.table_name,
+ functionName: trigger.function_name,
+ referencingNewTableAs: trigger.referencing_new_table_as ?? undefined,
+ referencingOldTableAs: trigger.referencing_old_table_as ?? undefined,
+ when: trigger.when_expression,
+ synchronize: true,
+ ...parseTriggerType(trigger.type),
+ });
+ }
+};
diff --git a/server/src/sql-tools/from-database/readers/type.ts b/server/src/sql-tools/from-database/readers/type.ts
new file mode 100644
index 0000000000..d8a21d486b
--- /dev/null
+++ b/server/src/sql-tools/from-database/readers/type.ts
@@ -0,0 +1,3 @@
+import { DatabaseClient, DatabaseSchema } from 'src/sql-tools/types';
+
+export type DatabaseReader = (schema: DatabaseSchema, db: DatabaseClient) => Promise<void>;