Skip to content

Database diagrams editor that allows you to visualize and design your DB with a single query.

fix: export sql + import metadata lib (#902)

Guy Ben-Aharonffddcdcc987b
diff --git a/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx b/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx
index ab8af742..b08a8390 100644
--- a/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx
+++ b/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx
@@ -3,7 +3,7 @@ import { Dialog, DialogContent } from '@/components/dialog/dialog';
import { DatabaseType } from '@/lib/domain/database-type';
import { useStorage } from '@/hooks/use-storage';
import type { Diagram } from '@/lib/domain/diagram';
-import { loadFromDatabaseMetadata } from '@/lib/domain/diagram';
+import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import';
import { useNavigate } from 'react-router-dom';
import { useConfig } from '@/hooks/use-config';
import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
diff --git a/src/dialogs/export-sql-dialog/export-sql-dialog.tsx b/src/dialogs/export-sql-dialog/export-sql-dialog.tsx
index 949d3a53..df9d29cd 100644
--- a/src/dialogs/export-sql-dialog/export-sql-dialog.tsx
+++ b/src/dialogs/export-sql-dialog/export-sql-dialog.tsx
@@ -17,7 +17,7 @@ import { useDialog } from '@/hooks/use-dialog';
import {
exportBaseSQL,
exportSQL,
-} from '@/lib/data/export-metadata/export-sql-script';
+} from '@/lib/data/sql-export/export-sql-script';
import { databaseTypeToLabelMap } from '@/lib/databases';
import { DatabaseType } from '@/lib/domain/database-type';
import { Annoyed, Sparkles } from 'lucide-react';
diff --git a/src/dialogs/import-database-dialog/import-database-dialog.tsx b/src/dialogs/import-database-dialog/import-database-dialog.tsx
index ce29e75b..e7dd28af 100644
--- a/src/dialogs/import-database-dialog/import-database-dialog.tsx
+++ b/src/dialogs/import-database-dialog/import-database-dialog.tsx
@@ -7,7 +7,7 @@ import type { DatabaseEdition } from '@/lib/domain/database-edition';
import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
import { loadDatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
import type { Diagram } from '@/lib/domain/diagram';
-import { loadFromDatabaseMetadata } from '@/lib/domain/diagram';
+import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import';
import { useChartDB } from '@/hooks/use-chartdb';
import { useRedoUndoStack } from '@/hooks/use-redo-undo-stack';
import { Trans, useTranslation } from 'react-i18next';
diff --git a/src/lib/data/import-metadata/import/custom-types.ts b/src/lib/data/import-metadata/import/custom-types.ts
new file mode 100644
index 00000000..b3269f2a
--- /dev/null
+++ b/src/lib/data/import-metadata/import/custom-types.ts
@@ -0,0 +1,21 @@
+import type { DBCustomType, DBCustomTypeKind } from '@/lib/domain';
+import { schemaNameToDomainSchemaName } from '@/lib/domain';
+import type { DBCustomTypeInfo } from '../metadata-types/custom-type-info';
+import { generateId } from '@/lib/utils';
+
+export const createCustomTypesFromMetadata = ({
+ customTypes,
+}: {
+ customTypes: DBCustomTypeInfo[];
+}): DBCustomType[] => {
+ return customTypes.map((customType) => {
+ return {
+ id: generateId(),
+ schema: schemaNameToDomainSchemaName(customType.schema),
+ name: customType.type,
+ kind: customType.kind as DBCustomTypeKind,
+ values: customType.values,
+ fields: customType.fields,
+ };
+ });
+};
diff --git a/src/lib/data/import-metadata/import/dependencies.ts b/src/lib/data/import-metadata/import/dependencies.ts
new file mode 100644
index 00000000..1937c5ec
--- /dev/null
+++ b/src/lib/data/import-metadata/import/dependencies.ts
@@ -0,0 +1,351 @@
+import { generateId } from '@/lib/utils';
+import type { AST } from 'node-sql-parser';
+import type { DBDependency, DBTable } from '@/lib/domain';
+import { DatabaseType, schemaNameToDomainSchemaName } from '@/lib/domain';
+import type { ViewInfo } from '../metadata-types/view-info';
+import { decodeViewDefinition } from './tables';
+
+const astDatabaseTypes: Record<DatabaseType, string> = {
+ [DatabaseType.POSTGRESQL]: 'postgresql',
+ [DatabaseType.MYSQL]: 'postgresql',
+ [DatabaseType.MARIADB]: 'postgresql',
+ [DatabaseType.GENERIC]: 'postgresql',
+ [DatabaseType.SQLITE]: 'postgresql',
+ [DatabaseType.SQL_SERVER]: 'postgresql',
+ [DatabaseType.CLICKHOUSE]: 'postgresql',
+ [DatabaseType.COCKROACHDB]: 'postgresql',
+ [DatabaseType.ORACLE]: 'postgresql',
+};
+
+export const createDependenciesFromMetadata = async ({
+ views,
+ tables,
+ databaseType,
+}: {
+ views: ViewInfo[];
+ tables: DBTable[];
+ databaseType: DatabaseType;
+}): Promise<DBDependency[]> => {
+ if (!views || views.length === 0) {
+ return [];
+ }
+
+ const { Parser } = await import('node-sql-parser');
+ const parser = new Parser();
+
+ const dependencies = views
+ .flatMap((view) => {
+ const viewSchema = schemaNameToDomainSchemaName(view.schema);
+ const viewTable = tables.find(
+ (table) =>
+ table.name === view.view_name && viewSchema === table.schema
+ );
+
+ if (!viewTable) {
+ console.warn(
+ `Source table for view ${view.view_name} not found (schema: ${viewSchema})`
+ );
+ return []; // Skip this view and proceed to the next
+ }
+
+ if (view.view_definition) {
+ try {
+ const decodedViewDefinition = decodeViewDefinition(
+ databaseType,
+ view.view_definition
+ );
+
+ let modifiedViewDefinition = '';
+ if (
+ databaseType === DatabaseType.MYSQL ||
+ databaseType === DatabaseType.MARIADB
+ ) {
+ modifiedViewDefinition = preprocessViewDefinitionMySQL(
+ decodedViewDefinition
+ );
+ } else if (databaseType === DatabaseType.SQL_SERVER) {
+ modifiedViewDefinition =
+ preprocessViewDefinitionSQLServer(
+ decodedViewDefinition
+ );
+ } else {
+ modifiedViewDefinition = preprocessViewDefinition(
+ decodedViewDefinition
+ );
+ }
+
+ // Parse using the appropriate dialect
+ const ast = parser.astify(modifiedViewDefinition, {
+ database: astDatabaseTypes[databaseType],
+ type: 'select', // Parsing a SELECT statement
+ });
+
+ let relatedTables = extractTablesFromAST(ast);
+
+ // Filter out duplicate tables without schema
+ relatedTables = filterDuplicateTables(relatedTables);
+
+ return relatedTables.map((relTable) => {
+ const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined
+ const relTableName = relTable.tableName;
+
+ const table = tables.find(
+ (table) =>
+ table.name === relTableName &&
+ (table.schema || '') === relSchema
+ );
+
+ if (table) {
+ const dependency: DBDependency = {
+ id: generateId(),
+ schema: view.schema,
+ tableId: table.id, // related table
+ dependentSchema: table.schema,
+ dependentTableId: viewTable.id, // dependent view
+ createdAt: Date.now(),
+ };
+
+ return dependency;
+ } else {
+ console.warn(
+ `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}`
+ );
+ return null;
+ }
+ });
+ } catch (error) {
+ console.error(
+ `Error parsing view ${view.schema}.${view.view_name}:`,
+ error
+ );
+ return [];
+ }
+ } else {
+ console.warn(
+ `View definition missing for ${view.schema}.${view.view_name}`
+ );
+ return [];
+ }
+ })
+ .filter((dependency) => dependency !== null);
+
+ return dependencies;
+};
+
+// Add this new function to filter out duplicate tables
+function filterDuplicateTables(
+ tables: { schema?: string; tableName: string }[]
+): { schema?: string; tableName: string }[] {
+ const tableMap = new Map<string, { schema?: string; tableName: string }>();
+
+ for (const table of tables) {
+ const key = table.tableName;
+ const existingTable = tableMap.get(key);
+
+ if (!existingTable || (table.schema && !existingTable.schema)) {
+ tableMap.set(key, table);
+ }
+ }
+
+ return Array.from(tableMap.values());
+}
+
+// Preprocess the view_definition to remove schema from CREATE VIEW
+function preprocessViewDefinition(viewDefinition: string): string {
+ if (!viewDefinition) {
+ return '';
+ }
+
+ // Remove leading and trailing whitespace
+ viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
+
+ // Replace escaped double quotes with regular ones
+ viewDefinition = viewDefinition.replace(/\\"/g, '"');
+
+ // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW'
+ viewDefinition = viewDefinition.replace(
+ /CREATE\s+MATERIALIZED\s+VIEW/i,
+ 'CREATE VIEW'
+ );
+
+ // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS'
+ // This regex captures the view name and skips any content between the view name and 'AS'
+ const regex =
+ /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i;
+
+ const match = viewDefinition.match(regex);
+ let modifiedDefinition: string;
+
+ if (match) {
+ const viewName = match[1] || match[2] || match[3];
+ // Extract the SQL after the 'AS' keyword
+ const restOfDefinition = viewDefinition.substring(
+ match.index! + match[0].length
+ );
+
+ // Replace double-quoted identifiers with unquoted ones
+ let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1');
+
+ // Replace '::' type casts with 'CAST' expressions
+ modifiedSQL = modifiedSQL.replace(
+ /\(([^()]+)\)::(\w+)/g,
+ 'CAST($1 AS $2)'
+ );
+
+ // Remove ClickHouse-specific syntax that may still be present
+ // For example, remove SETTINGS clauses inside the SELECT statement
+ modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, '');
+
+ modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`;
+ } else {
+ console.warn('Could not preprocess view definition:', viewDefinition);
+ modifiedDefinition = viewDefinition;
+ }
+
+ return modifiedDefinition;
+}
+
+// Preprocess the view_definition for SQL Server
+function preprocessViewDefinitionSQLServer(viewDefinition: string): string {
+ if (!viewDefinition) {
+ return '';
+ }
+
+ // Remove BOM if present
+ viewDefinition = viewDefinition.replace(/^\uFEFF/, '');
+
+ // Normalize whitespace
+ viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
+
+ // Remove square brackets and replace with double quotes
+ viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"');
+
+ // Remove database names from fully qualified identifiers
+ viewDefinition = viewDefinition.replace(
+ /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g,
+ '"$2"."$3"'
+ );
+
+ // Replace SQL Server functions with PostgreSQL equivalents
+ viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()');
+ viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE(');
+
+ // Replace 'TOP N' with 'LIMIT N' at the end of the query
+ const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i);
+ if (topMatch) {
+ const topN = topMatch[1];
+ viewDefinition = viewDefinition.replace(
+ /SELECT\s+TOP\s+\d+/i,
+ 'SELECT'
+ );
+ viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end
+ viewDefinition += ` LIMIT ${topN}`;
+ }
+
+ viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines
+
+ // Adjust CREATE VIEW syntax
+ const regex =
+ /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i;
+ const match = viewDefinition.match(regex);
+ let modifiedDefinition: string;
+
+ if (match) {
+ const viewName = match[2];
+ const modifiedSQL = viewDefinition.substring(
+ match.index! + match[0].length
+ );
+
+ // Remove semicolons at the end
+ const finalSQL = modifiedSQL.replace(/;+\s*$/, '');
+
+ modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`;
+ } else {
+ console.warn('Could not preprocess view definition:', viewDefinition);
+ modifiedDefinition = viewDefinition;
+ }
+
+ return modifiedDefinition;
+}
+
+// Preprocess the view_definition to remove schema from CREATE VIEW
+function preprocessViewDefinitionMySQL(viewDefinition: string): string {
+ if (!viewDefinition) {
+ return '';
+ }
+
+ // Remove any trailing semicolons
+ viewDefinition = viewDefinition.replace(/;\s*$/, '');
+
+ // Remove backticks from identifiers
+ viewDefinition = viewDefinition.replace(/`/g, '');
+
+ // Remove unnecessary parentheses around joins and ON clauses
+ viewDefinition = removeRedundantParentheses(viewDefinition);
+
+ return viewDefinition;
+}
+
+function removeRedundantParentheses(sql: string): string {
+ // Regular expressions to match unnecessary parentheses
+ const patterns = [
+ /\(\s*(JOIN\s+[^()]+?)\s*\)/gi,
+ /\(\s*(ON\s+[^()]+?)\s*\)/gi,
+ // Additional patterns if necessary
+ ];
+
+ let prevSql;
+ do {
+ prevSql = sql;
+ patterns.forEach((pattern) => {
+ sql = sql.replace(pattern, '$1');
+ });
+ } while (sql !== prevSql);
+
+ return sql;
+}
+
+function extractTablesFromAST(
+ ast: AST | AST[]
+): { schema?: string; tableName: string }[] {
+ const tablesMap = new Map<string, { schema: string; tableName: string }>();
+ const visitedNodes = new Set();
+
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ function traverse(node: any) {
+ if (!node || visitedNodes.has(node)) return;
+ visitedNodes.add(node);
+
+ if (Array.isArray(node)) {
+ node.forEach(traverse);
+ } else if (typeof node === 'object') {
+ // Check if node represents a table
+ if (
+ Object.hasOwnProperty.call(node, 'table') &&
+ typeof node.table === 'string'
+ ) {
+ let schema = node.db || node.schema;
+ const tableName = node.table;
+ if (tableName) {
+ // Assign default schema if undefined
+ schema = schemaNameToDomainSchemaName(schema) || '';
+ const key = `${schema}.${tableName}`;
+ if (!tablesMap.has(key)) {
+ tablesMap.set(key, { schema, tableName });
+ }
+ }
+ }
+
+ // Recursively traverse all properties
+ for (const key in node) {
+ if (Object.hasOwnProperty.call(node, key)) {
+ traverse(node[key]);
+ }
+ }
+ }
+ }
+
+ traverse(ast);
+
+ return Array.from(tablesMap.values());
+}
diff --git a/src/lib/data/import-metadata/import/fields.ts b/src/lib/data/import-metadata/import/fields.ts
new file mode 100644
index 00000000..01c901c8
--- /dev/null
+++ b/src/lib/data/import-metadata/import/fields.ts
@@ -0,0 +1,64 @@
+import type { DBField } from '@/lib/domain';
+import type { ColumnInfo } from '../metadata-types/column-info';
+import type { AggregatedIndexInfo } from '../metadata-types/index-info';
+import type { PrimaryKeyInfo } from '../metadata-types/primary-key-info';
+import type { TableInfo } from '../metadata-types/table-info';
+import { generateId } from '@/lib/utils';
+
+export const createFieldsFromMetadata = ({
+ tableColumns,
+ tablePrimaryKeys,
+ aggregatedIndexes,
+}: {
+ tableColumns: ColumnInfo[];
+ tableSchema?: string;
+ tableInfo: TableInfo;
+ tablePrimaryKeys: PrimaryKeyInfo[];
+ aggregatedIndexes: AggregatedIndexInfo[];
+}) => {
+ const uniqueColumns = tableColumns.reduce((acc, col) => {
+ if (!acc.has(col.name)) {
+ acc.set(col.name, col);
+ }
+ return acc;
+ }, new Map<string, ColumnInfo>());
+
+ const sortedColumns = Array.from(uniqueColumns.values()).sort(
+ (a, b) => a.ordinal_position - b.ordinal_position
+ );
+
+ const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) =>
+ pk.column.trim()
+ );
+
+ return sortedColumns.map(
+ (col: ColumnInfo): DBField => ({
+ id: generateId(),
+ name: col.name,
+ type: {
+ id: col.type.split(' ').join('_').toLowerCase(),
+ name: col.type.toLowerCase(),
+ },
+ primaryKey: tablePrimaryKeysColumns.includes(col.name),
+ unique: Object.values(aggregatedIndexes).some(
+ (idx) =>
+ idx.unique &&
+ idx.columns.length === 1 &&
+ idx.columns[0].name === col.name
+ ),
+ nullable: Boolean(col.nullable),
+ ...(col.character_maximum_length &&
+ col.character_maximum_length !== 'null'
+ ? { characterMaximumLength: col.character_maximum_length }
+ : {}),
+ ...(col.precision?.precision
+ ? { precision: col.precision.precision }
+ : {}),
+ ...(col.precision?.scale ? { scale: col.precision.scale } : {}),
+ ...(col.default ? { default: col.default } : {}),
+ ...(col.collation ? { collation: col.collation } : {}),
+ createdAt: Date.now(),
+ comments: col.comment ? col.comment : undefined,
+ })
+ );
+};
diff --git a/src/lib/data/import-metadata/import/index.ts b/src/lib/data/import-metadata/import/index.ts
new file mode 100644
index 00000000..ecc907fb
--- /dev/null
+++ b/src/lib/data/import-metadata/import/index.ts
@@ -0,0 +1,82 @@
+import type { DatabaseEdition, Diagram } from '@/lib/domain';
+import { adjustTablePositions, DatabaseType } from '@/lib/domain';
+import { generateDiagramId } from '@/lib/utils';
+import type { DatabaseMetadata } from '../metadata-types/database-metadata';
+import { createCustomTypesFromMetadata } from './custom-types';
+import { createRelationshipsFromMetadata } from './relationships';
+import { createTablesFromMetadata } from './tables';
+import { createDependenciesFromMetadata } from './dependencies';
+
+export const loadFromDatabaseMetadata = async ({
+ databaseType,
+ databaseMetadata,
+ diagramNumber,
+ databaseEdition,
+}: {
+ databaseType: DatabaseType;
+ databaseMetadata: DatabaseMetadata;
+ diagramNumber?: number;
+ databaseEdition?: DatabaseEdition;
+}): Promise<Diagram> => {
+ const {
+ fk_info: foreignKeys,
+ views: views,
+ custom_types: customTypes,
+ } = databaseMetadata;
+
+ const tables = createTablesFromMetadata({
+ databaseMetadata,
+ databaseType,
+ });
+
+ const relationships = createRelationshipsFromMetadata({
+ foreignKeys,
+ tables,
+ });
+
+ const dependencies = await createDependenciesFromMetadata({
+ views,
+ tables,
+ databaseType,
+ });
+
+ const dbCustomTypes = customTypes
+ ? createCustomTypesFromMetadata({
+ customTypes,
+ })
+ : [];
+
+ const adjustedTables = adjustTablePositions({
+ tables,
+ relationships,
+ mode: 'perSchema',
+ });
+
+ const sortedTables = adjustedTables.sort((a, b) => {
+ if (a.isView === b.isView) {
+ // Both are either tables or views, so sort alphabetically by name
+ return a.name.localeCompare(b.name);
+ }
+ // If one is a view and the other is not, put tables first
+ return a.isView ? 1 : -1;
+ });
+
+ const diagram: Diagram = {
+ id: generateDiagramId(),
+ name: databaseMetadata.database_name
+ ? `${databaseMetadata.database_name}-db`
+ : diagramNumber
+ ? `Diagram ${diagramNumber}`
+ : 'New Diagram',
+ databaseType: databaseType ?? DatabaseType.GENERIC,
+ databaseEdition,
+ tables: sortedTables,
+ relationships,
+ dependencies,
+ customTypes: dbCustomTypes,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ };
+
+ return diagram;
+};
diff --git a/src/lib/data/import-metadata/import/indexes.ts b/src/lib/data/import-metadata/import/indexes.ts
new file mode 100644
index 00000000..c020fd41
--- /dev/null
+++ b/src/lib/data/import-metadata/import/indexes.ts
@@ -0,0 +1,24 @@
+import type { DBField, DBIndex, IndexType } from '@/lib/domain';
+import type { AggregatedIndexInfo } from '../metadata-types/index-info';
+import { generateId } from '@/lib/utils';
+
+export const createIndexesFromMetadata = ({
+ aggregatedIndexes,
+ fields,
+}: {
+ aggregatedIndexes: AggregatedIndexInfo[];
+ fields: DBField[];
+}): DBIndex[] =>
+ aggregatedIndexes.map(
+ (idx): DBIndex => ({
+ id: generateId(),
+ name: idx.name,
+ unique: Boolean(idx.unique),
+ fieldIds: idx.columns
+ .sort((a, b) => a.position - b.position)
+ .map((c) => fields.find((f) => f.name === c.name)?.id)
+ .filter((id): id is string => id !== undefined),
+ createdAt: Date.now(),
+ type: idx.index_type?.toLowerCase() as IndexType,
+ })
+ );
diff --git a/src/lib/data/import-metadata/import/relationships.ts b/src/lib/data/import-metadata/import/relationships.ts
new file mode 100644
index 00000000..0b0fd7aa
--- /dev/null
+++ b/src/lib/data/import-metadata/import/relationships.ts
@@ -0,0 +1,85 @@
+import type {
+ Cardinality,
+ DBField,
+ DBRelationship,
+ DBTable,
+} from '@/lib/domain';
+import { schemaNameToDomainSchemaName } from '@/lib/domain';
+import type { ForeignKeyInfo } from '../metadata-types/foreign-key-info';
+import { generateId } from '@/lib/utils';
+
+const determineCardinality = (
+ field: DBField,
+ isTablePKComplex: boolean
+): Cardinality => {
+ return field.unique || (field.primaryKey && !isTablePKComplex)
+ ? 'one'
+ : 'many';
+};
+
+export const createRelationshipsFromMetadata = ({
+ foreignKeys,
+ tables,
+}: {
+ foreignKeys: ForeignKeyInfo[];
+ tables: DBTable[];
+}): DBRelationship[] => {
+ return foreignKeys
+ .map((fk: ForeignKeyInfo): DBRelationship | null => {
+ const schema = schemaNameToDomainSchemaName(fk.schema);
+ const sourceTable = tables.find(
+ (table) => table.name === fk.table && table.schema === schema
+ );
+
+ const targetSchema = schemaNameToDomainSchemaName(
+ fk.reference_schema
+ );
+
+ const targetTable = tables.find(
+ (table) =>
+ table.name === fk.reference_table &&
+ table.schema === targetSchema
+ );
+ const sourceField = sourceTable?.fields.find(
+ (field) => field.name === fk.column
+ );
+ const targetField = targetTable?.fields.find(
+ (field) => field.name === fk.reference_column
+ );
+
+ const isSourceTablePKComplex =
+ (sourceTable?.fields.filter((field) => field.primaryKey) ?? [])
+ .length > 1;
+ const isTargetTablePKComplex =
+ (targetTable?.fields.filter((field) => field.primaryKey) ?? [])
+ .length > 1;
+
+ if (sourceTable && targetTable && sourceField && targetField) {
+ const sourceCardinality = determineCardinality(
+ sourceField,
+ isSourceTablePKComplex
+ );
+ const targetCardinality = determineCardinality(
+ targetField,
+ isTargetTablePKComplex
+ );
+
+ return {
+ id: generateId(),
+ name: fk.foreign_key_name,
+ sourceSchema: schema,
+ targetSchema: targetSchema,
+ sourceTableId: sourceTable.id,
+ targetTableId: targetTable.id,
+ sourceFieldId: sourceField.id,
+ targetFieldId: targetField.id,
+ sourceCardinality,
+ targetCardinality,
+ createdAt: Date.now(),
+ };
+ }
+
+ return null;
+ })
+ .filter((rel) => rel !== null) as DBRelationship[];
+};
diff --git a/src/lib/data/import-metadata/import/tables.ts b/src/lib/data/import-metadata/import/tables.ts
new file mode 100644
index 00000000..b45689b8
--- /dev/null
+++ b/src/lib/data/import-metadata/import/tables.ts
@@ -0,0 +1,228 @@
+import type { DBIndex, DBTable } from '@/lib/domain';
+import {
+ DatabaseType,
+ generateTableKey,
+ schemaNameToDomainSchemaName,
+} from '@/lib/domain';
+import type { DatabaseMetadata } from '../metadata-types/database-metadata';
+import type { TableInfo } from '../metadata-types/table-info';
+import { createAggregatedIndexes } from '../metadata-types/index-info';
+import {
+ decodeBase64ToUtf16LE,
+ decodeBase64ToUtf8,
+ generateId,
+} from '@/lib/utils';
+import {
+ defaultTableColor,
+ materializedViewColor,
+ viewColor,
+} from '@/lib/colors';
+import { createFieldsFromMetadata } from './fields';
+import { createIndexesFromMetadata } from './indexes';
+
+export const decodeViewDefinition = (
+ databaseType: DatabaseType,
+ viewDefinition?: string
+): string => {
+ if (!viewDefinition) {
+ return '';
+ }
+
+ let decodedViewDefinition: string;
+ if (databaseType === DatabaseType.SQL_SERVER) {
+ decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition);
+ } else {
+ decodedViewDefinition = decodeBase64ToUtf8(viewDefinition);
+ }
+
+ return decodedViewDefinition;
+};
+
+export const createTablesFromMetadata = ({
+ databaseMetadata,
+ databaseType,
+}: {
+ databaseMetadata: DatabaseMetadata;
+ databaseType: DatabaseType;
+}): DBTable[] => {
+ const {
+ tables: tableInfos,
+ pk_info: primaryKeys,
+ columns,
+ indexes,
+ views: views,
+ } = databaseMetadata;
+
+ // Pre-compute view names for faster lookup if there are views
+ const viewNamesSet = new Set<string>();
+ const materializedViewNamesSet = new Set<string>();
+
+ if (views && views.length > 0) {
+ views.forEach((view) => {
+ const key = generateTableKey({
+ schemaName: view.schema,
+ tableName: view.view_name,
+ });
+ viewNamesSet.add(key);
+
+ if (
+ view.view_definition &&
+ decodeViewDefinition(databaseType, view.view_definition)
+ .toLowerCase()
+ .includes('materialized')
+ ) {
+ materializedViewNamesSet.add(key);
+ }
+ });
+ }
+
+ // Pre-compute lookup maps for better performance
+ const columnsByTable = new Map<string, (typeof columns)[0][]>();
+ const indexesByTable = new Map<string, (typeof indexes)[0][]>();
+ const primaryKeysByTable = new Map<string, (typeof primaryKeys)[0][]>();
+
+ // Group columns by table
+ columns.forEach((col) => {
+ const key = generateTableKey({
+ schemaName: col.schema,
+ tableName: col.table,
+ });
+ if (!columnsByTable.has(key)) {
+ columnsByTable.set(key, []);
+ }
+ columnsByTable.get(key)!.push(col);
+ });
+
+ // Group indexes by table
+ indexes.forEach((idx) => {
+ const key = generateTableKey({
+ schemaName: idx.schema,
+ tableName: idx.table,
+ });
+ if (!indexesByTable.has(key)) {
+ indexesByTable.set(key, []);
+ }
+ indexesByTable.get(key)!.push(idx);
+ });
+
+ // Group primary keys by table
+ primaryKeys.forEach((pk) => {
+ const key = generateTableKey({
+ schemaName: pk.schema,
+ tableName: pk.table,
+ });
+ if (!primaryKeysByTable.has(key)) {
+ primaryKeysByTable.set(key, []);
+ }
+ primaryKeysByTable.get(key)!.push(pk);
+ });
+
+ const result = tableInfos.map((tableInfo: TableInfo) => {
+ const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema);
+ const tableKey = generateTableKey({
+ schemaName: tableInfo.schema,
+ tableName: tableInfo.table,
+ });
+
+ // Use pre-computed lookups instead of filtering entire arrays
+ const tableIndexes = indexesByTable.get(tableKey) || [];
+ const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || [];
+ const tableColumns = columnsByTable.get(tableKey) || [];
+
+ // Aggregate indexes with multiple columns
+ const aggregatedIndexes = createAggregatedIndexes({
+ tableInfo,
+ tableSchema,
+ tableIndexes,
+ });
+
+ const fields = createFieldsFromMetadata({
+ aggregatedIndexes,
+ tableColumns,
+ tablePrimaryKeys,
+ tableInfo,
+ tableSchema,
+ });
+
+ // Check for composite primary key and find matching index name
+ const primaryKeyFields = fields.filter((f) => f.primaryKey);
+ let pkMatchingIndexName: string | undefined;
+ let pkIndex: DBIndex | undefined;
+
+ if (primaryKeyFields.length >= 1) {
+ // We have a composite primary key, look for an index that matches all PK columns
+ const pkFieldNames = primaryKeyFields.map((f) => f.name).sort();
+
+ // Find an index that matches the primary key columns exactly
+ const matchingIndex = aggregatedIndexes.find((index) => {
+ const indexColumnNames = index.columns
+ .map((c) => c.name)
+ .sort();
+ return (
+ indexColumnNames.length === pkFieldNames.length &&
+ indexColumnNames.every((col, i) => col === pkFieldNames[i])
+ );
+ });
+
+ if (matchingIndex) {
+ pkMatchingIndexName = matchingIndex.name;
+ // Create a special PK index
+ pkIndex = {
+ id: generateId(),
+ name: matchingIndex.name,
+ unique: true,
+ fieldIds: primaryKeyFields.map((f) => f.id),
+ createdAt: Date.now(),
+ isPrimaryKey: true,
+ };
+ }
+ }
+
+ // Filter out the index that matches the composite PK (to avoid duplication)
+ const filteredAggregatedIndexes = pkMatchingIndexName
+ ? aggregatedIndexes.filter(
+ (idx) => idx.name !== pkMatchingIndexName
+ )
+ : aggregatedIndexes;
+
+ const dbIndexes = createIndexesFromMetadata({
+ aggregatedIndexes: filteredAggregatedIndexes,
+ fields,
+ });
+
+ // Add the PK index if it exists
+ if (pkIndex) {
+ dbIndexes.push(pkIndex);
+ }
+
+ // Determine if the current table is a view by checking against pre-computed sets
+ const viewKey = generateTableKey({
+ schemaName: tableSchema,
+ tableName: tableInfo.table,
+ });
+ const isView = viewNamesSet.has(viewKey);
+ const isMaterializedView = materializedViewNamesSet.has(viewKey);
+
+ // Initial random positions; these will be adjusted later
+ return {
+ id: generateId(),
+ name: tableInfo.table,
+ schema: tableSchema,
+ x: Math.random() * 1000, // Placeholder X
+ y: Math.random() * 800, // Placeholder Y
+ fields,
+ indexes: dbIndexes,
+ color: isMaterializedView
+ ? materializedViewColor
+ : isView
+ ? viewColor
+ : defaultTableColor,
+ isView: isView,
+ isMaterializedView: isMaterializedView,
+ createdAt: Date.now(),
+ comments: tableInfo.comment ? tableInfo.comment : undefined,
+ };
+ });
+
+ return result;
+};
diff --git a/src/lib/data/export-metadata/__tests__/export-sql-dbml.test.ts b/src/lib/data/sql-export/__tests__/export-sql-dbml.test.ts
similarity index 100%
rename from src/lib/data/export-metadata/__tests__/export-sql-dbml.test.ts
rename to src/lib/data/sql-export/__tests__/export-sql-dbml.test.ts
diff --git a/src/lib/data/export-metadata/__tests__/export-sql-quoted-identifiers.test.ts b/src/lib/data/sql-export/__tests__/export-sql-quoted-identifiers.test.ts
similarity index 100%
rename from src/lib/data/export-metadata/__tests__/export-sql-quoted-identifiers.test.ts
rename to src/lib/data/sql-export/__tests__/export-sql-quoted-identifiers.test.ts
diff --git a/src/lib/data/export-metadata/export-per-type/common.ts b/src/lib/data/sql-export/export-per-type/common.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-per-type/common.ts
rename to src/lib/data/sql-export/export-per-type/common.ts
diff --git a/src/lib/data/export-metadata/export-per-type/mssql.ts b/src/lib/data/sql-export/export-per-type/mssql.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-per-type/mssql.ts
rename to src/lib/data/sql-export/export-per-type/mssql.ts
diff --git a/src/lib/data/export-metadata/export-per-type/mysql.ts b/src/lib/data/sql-export/export-per-type/mysql.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-per-type/mysql.ts
rename to src/lib/data/sql-export/export-per-type/mysql.ts
diff --git a/src/lib/data/export-metadata/export-per-type/postgresql.ts b/src/lib/data/sql-export/export-per-type/postgresql.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-per-type/postgresql.ts
rename to src/lib/data/sql-export/export-per-type/postgresql.ts
diff --git a/src/lib/data/export-metadata/export-per-type/sqlite.ts b/src/lib/data/sql-export/export-per-type/sqlite.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-per-type/sqlite.ts
rename to src/lib/data/sql-export/export-per-type/sqlite.ts
diff --git a/src/lib/data/export-metadata/export-sql-cache.ts b/src/lib/data/sql-export/export-sql-cache.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-sql-cache.ts
rename to src/lib/data/sql-export/export-sql-cache.ts
diff --git a/src/lib/data/export-metadata/export-sql-script.ts b/src/lib/data/sql-export/export-sql-script.ts
similarity index 100%
rename from src/lib/data/export-metadata/export-sql-script.ts
rename to src/lib/data/sql-export/export-sql-script.ts
diff --git a/src/lib/dbml/dbml-export/dbml-export.ts b/src/lib/dbml/dbml-export/dbml-export.ts
index 5030085d..b0049583 100644
--- a/src/lib/dbml/dbml-export/dbml-export.ts
+++ b/src/lib/dbml/dbml-export/dbml-export.ts
@@ -1,5 +1,5 @@
import { importer } from '@dbml/core';
-import { exportBaseSQL } from '@/lib/data/export-metadata/export-sql-script';
+import { exportBaseSQL } from '@/lib/data/sql-export/export-sql-script';
import type { Diagram } from '@/lib/domain/diagram';
import { DatabaseType } from '@/lib/domain/database-type';
import type { DBTable } from '@/lib/domain/db-table';
diff --git a/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts b/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts
index a2fe3e5d..6cc40513 100644
--- a/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts
+++ b/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts
@@ -1,8 +1,8 @@
import { describe, it, expect } from 'vitest';
import { importDBMLToDiagram } from '../dbml-import';
-import { exportPostgreSQL } from '@/lib/data/export-metadata/export-per-type/postgresql';
-import { exportMySQL } from '@/lib/data/export-metadata/export-per-type/mysql';
-import { exportMSSQL } from '@/lib/data/export-metadata/export-per-type/mssql';
+import { exportPostgreSQL } from '@/lib/data/sql-export/export-per-type/postgresql';
+import { exportMySQL } from '@/lib/data/sql-export/export-per-type/mysql';
+import { exportMSSQL } from '@/lib/data/sql-export/export-per-type/mssql';
import { DatabaseType } from '@/lib/domain/database-type';
describe('Composite Primary Key with Name', () => {
diff --git a/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts b/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts
index 252ff579..22d1c76a 100644
--- a/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts
+++ b/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { createTablesFromMetadata } from '../db-table';
+import { createTablesFromMetadata } from '@/lib/data/import-metadata/import/tables';
import { DatabaseType } from '../database-type';
import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';
diff --git a/src/lib/domain/db-custom-type.ts b/src/lib/domain/db-custom-type.ts
index 59a00679..2252d031 100644
--- a/src/lib/domain/db-custom-type.ts
+++ b/src/lib/domain/db-custom-type.ts
@@ -1,7 +1,4 @@
import { z } from 'zod';
-import type { DBCustomTypeInfo } from '@/lib/data/import-metadata/metadata-types/custom-type-info';
-import { generateId } from '../utils';
-import { schemaNameToDomainSchemaName } from './db-schema';
export enum DBCustomTypeKind {
enum = 'enum',
@@ -38,23 +35,6 @@ export const dbCustomTypeSchema: z.ZodType<DBCustomType> = z.object({
order: z.number().or(z.null()).optional(),
});
-export const createCustomTypesFromMetadata = ({
- customTypes,
-}: {
- customTypes: DBCustomTypeInfo[];
-}): DBCustomType[] => {
- return customTypes.map((customType) => {
- return {
- id: generateId(),
- schema: schemaNameToDomainSchemaName(customType.schema),
- name: customType.type,
- kind: customType.kind as DBCustomTypeKind,
- values: customType.values,
- fields: customType.fields,
- };
- });
-};
-
export const customTypeKindToLabel: Record<DBCustomTypeKind, string> = {
enum: 'Enum',
composite: 'Composite',
diff --git a/src/lib/domain/db-dependency.ts b/src/lib/domain/db-dependency.ts
index 347194e2..4f9c4bba 100644
--- a/src/lib/domain/db-dependency.ts
+++ b/src/lib/domain/db-dependency.ts
@@ -1,10 +1,4 @@
import { z } from 'zod';
-import type { ViewInfo } from '../data/import-metadata/metadata-types/view-info';
-import { DatabaseType } from './database-type';
-import { schemaNameToDomainSchemaName } from './db-schema';
-import { decodeViewDefinition, type DBTable } from './db-table';
-import { generateId } from '@/lib/utils';
-import type { AST } from 'node-sql-parser';
export interface DBDependency {
id: string;
@@ -23,348 +17,3 @@ export const dbDependencySchema: z.ZodType<DBDependency> = z.object({
dependentTableId: z.string(),
createdAt: z.number(),
});
-
-const astDatabaseTypes: Record<DatabaseType, string> = {
- [DatabaseType.POSTGRESQL]: 'postgresql',
- [DatabaseType.MYSQL]: 'postgresql',
- [DatabaseType.MARIADB]: 'postgresql',
- [DatabaseType.GENERIC]: 'postgresql',
- [DatabaseType.SQLITE]: 'postgresql',
- [DatabaseType.SQL_SERVER]: 'postgresql',
- [DatabaseType.CLICKHOUSE]: 'postgresql',
- [DatabaseType.COCKROACHDB]: 'postgresql',
- [DatabaseType.ORACLE]: 'postgresql',
-};
-
-export const createDependenciesFromMetadata = async ({
- views,
- tables,
- databaseType,
-}: {
- views: ViewInfo[];
- tables: DBTable[];
- databaseType: DatabaseType;
-}): Promise<DBDependency[]> => {
- if (!views || views.length === 0) {
- return [];
- }
-
- const { Parser } = await import('node-sql-parser');
- const parser = new Parser();
-
- const dependencies = views
- .flatMap((view) => {
- const viewSchema = schemaNameToDomainSchemaName(view.schema);
- const viewTable = tables.find(
- (table) =>
- table.name === view.view_name && viewSchema === table.schema
- );
-
- if (!viewTable) {
- console.warn(
- `Source table for view ${view.view_name} not found (schema: ${viewSchema})`
- );
- return []; // Skip this view and proceed to the next
- }
-
- if (view.view_definition) {
- try {
- const decodedViewDefinition = decodeViewDefinition(
- databaseType,
- view.view_definition
- );
-
- let modifiedViewDefinition = '';
- if (
- databaseType === DatabaseType.MYSQL ||
- databaseType === DatabaseType.MARIADB
- ) {
- modifiedViewDefinition = preprocessViewDefinitionMySQL(
- decodedViewDefinition
- );
- } else if (databaseType === DatabaseType.SQL_SERVER) {
- modifiedViewDefinition =
- preprocessViewDefinitionSQLServer(
- decodedViewDefinition
- );
- } else {
- modifiedViewDefinition = preprocessViewDefinition(
- decodedViewDefinition
- );
- }
-
- // Parse using the appropriate dialect
- const ast = parser.astify(modifiedViewDefinition, {
- database: astDatabaseTypes[databaseType],
- type: 'select', // Parsing a SELECT statement
- });
-
- let relatedTables = extractTablesFromAST(ast);
-
- // Filter out duplicate tables without schema
- relatedTables = filterDuplicateTables(relatedTables);
-
- return relatedTables.map((relTable) => {
- const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined
- const relTableName = relTable.tableName;
-
- const table = tables.find(
- (table) =>
- table.name === relTableName &&
- (table.schema || '') === relSchema
- );
-
- if (table) {
- const dependency: DBDependency = {
- id: generateId(),
- schema: view.schema,
- tableId: table.id, // related table
- dependentSchema: table.schema,
- dependentTableId: viewTable.id, // dependent view
- createdAt: Date.now(),
- };
-
- return dependency;
- } else {
- console.warn(
- `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}`
- );
- return null;
- }
- });
- } catch (error) {
- console.error(
- `Error parsing view ${view.schema}.${view.view_name}:`,
- error
- );
- return [];
- }
- } else {
- console.warn(
- `View definition missing for ${view.schema}.${view.view_name}`
- );
- return [];
- }
- })
- .filter((dependency) => dependency !== null);
-
- return dependencies;
-};
-
-// Add this new function to filter out duplicate tables
-function filterDuplicateTables(
- tables: { schema?: string; tableName: string }[]
-): { schema?: string; tableName: string }[] {
- const tableMap = new Map<string, { schema?: string; tableName: string }>();
-
- for (const table of tables) {
- const key = table.tableName;
- const existingTable = tableMap.get(key);
-
- if (!existingTable || (table.schema && !existingTable.schema)) {
- tableMap.set(key, table);
- }
- }
-
- return Array.from(tableMap.values());
-}
-
-// Preprocess the view_definition to remove schema from CREATE VIEW
-function preprocessViewDefinition(viewDefinition: string): string {
- if (!viewDefinition) {
- return '';
- }
-
- // Remove leading and trailing whitespace
- viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
-
- // Replace escaped double quotes with regular ones
- viewDefinition = viewDefinition.replace(/\\"/g, '"');
-
- // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW'
- viewDefinition = viewDefinition.replace(
- /CREATE\s+MATERIALIZED\s+VIEW/i,
- 'CREATE VIEW'
- );
-
- // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS'
- // This regex captures the view name and skips any content between the view name and 'AS'
- const regex =
- /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i;
-
- const match = viewDefinition.match(regex);
- let modifiedDefinition: string;
-
- if (match) {
- const viewName = match[1] || match[2] || match[3];
- // Extract the SQL after the 'AS' keyword
- const restOfDefinition = viewDefinition.substring(
- match.index! + match[0].length
- );
-
- // Replace double-quoted identifiers with unquoted ones
- let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1');
-
- // Replace '::' type casts with 'CAST' expressions
- modifiedSQL = modifiedSQL.replace(
- /\(([^()]+)\)::(\w+)/g,
- 'CAST($1 AS $2)'
- );
-
- // Remove ClickHouse-specific syntax that may still be present
- // For example, remove SETTINGS clauses inside the SELECT statement
- modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, '');
-
- modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`;
- } else {
- console.warn('Could not preprocess view definition:', viewDefinition);
- modifiedDefinition = viewDefinition;
- }
-
- return modifiedDefinition;
-}
-
-// Preprocess the view_definition for SQL Server
-function preprocessViewDefinitionSQLServer(viewDefinition: string): string {
- if (!viewDefinition) {
- return '';
- }
-
- // Remove BOM if present
- viewDefinition = viewDefinition.replace(/^\uFEFF/, '');
-
- // Normalize whitespace
- viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();
-
- // Remove square brackets and replace with double quotes
- viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"');
-
- // Remove database names from fully qualified identifiers
- viewDefinition = viewDefinition.replace(
- /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g,
- '"$2"."$3"'
- );
-
- // Replace SQL Server functions with PostgreSQL equivalents
- viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()');
- viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE(');
-
- // Replace 'TOP N' with 'LIMIT N' at the end of the query
- const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i);
- if (topMatch) {
- const topN = topMatch[1];
- viewDefinition = viewDefinition.replace(
- /SELECT\s+TOP\s+\d+/i,
- 'SELECT'
- );
- viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end
- viewDefinition += ` LIMIT ${topN}`;
- }
-
- viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines
-
- // Adjust CREATE VIEW syntax
- const regex =
- /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i;
- const match = viewDefinition.match(regex);
- let modifiedDefinition: string;
-
- if (match) {
- const viewName = match[2];
- const modifiedSQL = viewDefinition.substring(
- match.index! + match[0].length
- );
-
- // Remove semicolons at the end
- const finalSQL = modifiedSQL.replace(/;+\s*$/, '');
-
- modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`;
- } else {
- console.warn('Could not preprocess view definition:', viewDefinition);
- modifiedDefinition = viewDefinition;
- }
-
- return modifiedDefinition;
-}
-
-// Preprocess the view_definition to remove schema from CREATE VIEW
-function preprocessViewDefinitionMySQL(viewDefinition: string): string {
- if (!viewDefinition) {
- return '';
- }
-
- // Remove any trailing semicolons
- viewDefinition = viewDefinition.replace(/;\s*$/, '');
-
- // Remove backticks from identifiers
- viewDefinition = viewDefinition.replace(/`/g, '');
-
- // Remove unnecessary parentheses around joins and ON clauses
- viewDefinition = removeRedundantParentheses(viewDefinition);
-
- return viewDefinition;
-}
-
-function removeRedundantParentheses(sql: string): string {
- // Regular expressions to match unnecessary parentheses
- const patterns = [
- /\(\s*(JOIN\s+[^()]+?)\s*\)/gi,
- /\(\s*(ON\s+[^()]+?)\s*\)/gi,
- // Additional patterns if necessary
- ];
-
- let prevSql;
- do {
- prevSql = sql;
- patterns.forEach((pattern) => {
- sql = sql.replace(pattern, '$1');
- });
- } while (sql !== prevSql);
-
- return sql;
-}
-
-function extractTablesFromAST(
- ast: AST | AST[]
-): { schema?: string; tableName: string }[] {
- const tablesMap = new Map<string, { schema: string; tableName: string }>();
- const visitedNodes = new Set();
-
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- function traverse(node: any) {
- if (!node || visitedNodes.has(node)) return;
- visitedNodes.add(node);
-
- if (Array.isArray(node)) {
- node.forEach(traverse);
- } else if (typeof node === 'object') {
- // Check if node represents a table
- if (
- Object.hasOwnProperty.call(node, 'table') &&
- typeof node.table === 'string'
- ) {
- let schema = node.db || node.schema;
- const tableName = node.table;
- if (tableName) {
- // Assign default schema if undefined
- schema = schemaNameToDomainSchemaName(schema) || '';
- const key = `${schema}.${tableName}`;
- if (!tablesMap.has(key)) {
- tablesMap.set(key, { schema, tableName });
- }
- }
- }
-
- // Recursively traverse all properties
- for (const key in node) {
- if (Object.hasOwnProperty.call(node, key)) {
- traverse(node[key]);
- }
- }
- }
- }
-
- traverse(ast);
-
- return Array.from(tablesMap.values());
-}
diff --git a/src/lib/domain/db-field.ts b/src/lib/domain/db-field.ts
index dfc1a032..042530e3 100644
--- a/src/lib/domain/db-field.ts
+++ b/src/lib/domain/db-field.ts
@@ -4,11 +4,6 @@ import {
findDataTypeDataById,
type DataType,
} from '../data/data-types/data-types';
-import type { ColumnInfo } from '../data/import-metadata/metadata-types/column-info';
-import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info';
-import type { PrimaryKeyInfo } from '../data/import-metadata/metadata-types/primary-key-info';
-import type { TableInfo } from '../data/import-metadata/metadata-types/table-info';
-import { generateId } from '../utils';
import type { DatabaseType } from './database-type';
export interface DBField {
@@ -45,64 +40,6 @@ export const dbFieldSchema: z.ZodType<DBField> = z.object({
comments: z.string().or(z.null()).optional(),
});
-export const createFieldsFromMetadata = ({
- tableColumns,
- tablePrimaryKeys,
- aggregatedIndexes,
-}: {
- tableColumns: ColumnInfo[];
- tableSchema?: string;
- tableInfo: TableInfo;
- tablePrimaryKeys: PrimaryKeyInfo[];
- aggregatedIndexes: AggregatedIndexInfo[];
-}) => {
- const uniqueColumns = tableColumns.reduce((acc, col) => {
- if (!acc.has(col.name)) {
- acc.set(col.name, col);
- }
- return acc;
- }, new Map<string, ColumnInfo>());
-
- const sortedColumns = Array.from(uniqueColumns.values()).sort(
- (a, b) => a.ordinal_position - b.ordinal_position
- );
-
- const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) =>
- pk.column.trim()
- );
-
- return sortedColumns.map(
- (col: ColumnInfo): DBField => ({
- id: generateId(),
- name: col.name,
- type: {
- id: col.type.split(' ').join('_').toLowerCase(),
- name: col.type.toLowerCase(),
- },
- primaryKey: tablePrimaryKeysColumns.includes(col.name),
- unique: Object.values(aggregatedIndexes).some(
- (idx) =>
- idx.unique &&
- idx.columns.length === 1 &&
- idx.columns[0].name === col.name
- ),
- nullable: Boolean(col.nullable),
- ...(col.character_maximum_length &&
- col.character_maximum_length !== 'null'
- ? { characterMaximumLength: col.character_maximum_length }
- : {}),
- ...(col.precision?.precision
- ? { precision: col.precision.precision }
- : {}),
- ...(col.precision?.scale ? { scale: col.precision.scale } : {}),
- ...(col.default ? { default: col.default } : {}),
- ...(col.collation ? { collation: col.collation } : {}),
- createdAt: Date.now(),
- comments: col.comment ? col.comment : undefined,
- })
- );
-};
-
export const generateDBFieldSuffix = (
field: DBField,
{
diff --git a/src/lib/domain/db-index.ts b/src/lib/domain/db-index.ts
index b9a8c900..b9668daf 100644
--- a/src/lib/domain/db-index.ts
+++ b/src/lib/domain/db-index.ts
@@ -1,7 +1,5 @@
import { z } from 'zod';
-import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info';
import { generateId } from '../utils';
-import type { DBField } from './db-field';
import { DatabaseType } from './database-type';
import type { DBTable } from './db-table';
@@ -43,27 +41,6 @@ export const dbIndexSchema: z.ZodType<DBIndex> = z.object({
isPrimaryKey: z.boolean().or(z.null()).optional(),
});
-export const createIndexesFromMetadata = ({
- aggregatedIndexes,
- fields,
-}: {
- aggregatedIndexes: AggregatedIndexInfo[];
- fields: DBField[];
-}): DBIndex[] =>
- aggregatedIndexes.map(
- (idx): DBIndex => ({
- id: generateId(),
- name: idx.name,
- unique: Boolean(idx.unique),
- fieldIds: idx.columns
- .sort((a, b) => a.position - b.position)
- .map((c) => fields.find((f) => f.name === c.name)?.id)
- .filter((id): id is string => id !== undefined),
- createdAt: Date.now(),
- type: idx.index_type?.toLowerCase() as IndexType,
- })
- );
-
export const databaseIndexTypes: { [key in DatabaseType]?: IndexType[] } = {
[DatabaseType.POSTGRESQL]: ['btree', 'hash'],
};
diff --git a/src/lib/domain/db-relationship.ts b/src/lib/domain/db-relationship.ts
index fd5debd2..c470a002 100644
--- a/src/lib/domain/db-relationship.ts
+++ b/src/lib/domain/db-relationship.ts
@@ -1,9 +1,4 @@
import { z } from 'zod';
-import type { ForeignKeyInfo } from '../data/import-metadata/metadata-types/foreign-key-info';
-import type { DBField } from './db-field';
-import { schemaNameToDomainSchemaName } from './db-schema';
-import type { DBTable } from './db-table';
-import { generateId } from '@/lib/utils';
export interface DBRelationship {
id: string;
@@ -40,82 +35,6 @@ export type RelationshipType =
| 'many_to_many';
export type Cardinality = 'one' | 'many';
-const determineCardinality = (
- field: DBField,
- isTablePKComplex: boolean
-): Cardinality => {
- return field.unique || (field.primaryKey && !isTablePKComplex)
- ? 'one'
- : 'many';
-};
-
-export const createRelationshipsFromMetadata = ({
- foreignKeys,
- tables,
-}: {
- foreignKeys: ForeignKeyInfo[];
- tables: DBTable[];
-}): DBRelationship[] => {
- return foreignKeys
- .map((fk: ForeignKeyInfo): DBRelationship | null => {
- const schema = schemaNameToDomainSchemaName(fk.schema);
- const sourceTable = tables.find(
- (table) => table.name === fk.table && table.schema === schema
- );
-
- const targetSchema = schemaNameToDomainSchemaName(
- fk.reference_schema
- );
-
- const targetTable = tables.find(
- (table) =>
- table.name === fk.reference_table &&
- table.schema === targetSchema
- );
- const sourceField = sourceTable?.fields.find(
- (field) => field.name === fk.column
- );
- const targetField = targetTable?.fields.find(
- (field) => field.name === fk.reference_column
- );
-
- const isSourceTablePKComplex =
- (sourceTable?.fields.filter((field) => field.primaryKey) ?? [])
- .length > 1;
- const isTargetTablePKComplex =
- (targetTable?.fields.filter((field) => field.primaryKey) ?? [])
- .length > 1;
-
- if (sourceTable && targetTable && sourceField && targetField) {
- const sourceCardinality = determineCardinality(
- sourceField,
- isSourceTablePKComplex
- );
- const targetCardinality = determineCardinality(
- targetField,
- isTargetTablePKComplex
- );
-
- return {
- id: generateId(),
- name: fk.foreign_key_name,
- sourceSchema: schema,
- targetSchema: targetSchema,
- sourceTableId: sourceTable.id,
- targetTableId: targetTable.id,
- sourceFieldId: sourceField.id,
- targetFieldId: targetField.id,
- sourceCardinality,
- targetCardinality,
- createdAt: Date.now(),
- };
- }
-
- return null;
- })
- .filter((rel) => rel !== null) as DBRelationship[];
-};
-
export const determineRelationshipType = ({
sourceCardinality,
targetCardinality,
diff --git a/src/lib/domain/db-table.ts b/src/lib/domain/db-table.ts
index 4879f469..87f6e62f 100644
--- a/src/lib/domain/db-table.ts
+++ b/src/lib/domain/db-table.ts
@@ -1,30 +1,8 @@
-import {
- createIndexesFromMetadata,
- dbIndexSchema,
- type DBIndex,
-} from './db-index';
-import {
- createFieldsFromMetadata,
- dbFieldSchema,
- type DBField,
-} from './db-field';
-import type { TableInfo } from '../data/import-metadata/metadata-types/table-info';
-import { createAggregatedIndexes } from '../data/import-metadata/metadata-types/index-info';
-import {
- materializedViewColor,
- viewColor,
- defaultTableColor,
-} from '@/lib/colors';
+import { dbIndexSchema, type DBIndex } from './db-index';
+import { dbFieldSchema, type DBField } from './db-field';
import type { DBRelationship } from './db-relationship';
-import {
- decodeBase64ToUtf16LE,
- decodeBase64ToUtf8,
- deepCopy,
- generateId,
-} from '../utils';
+import { deepCopy } from '../utils';
import { schemaNameToDomainSchemaName } from './db-schema';
-import { DatabaseType } from './database-type';
-import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata';
import { z } from 'zod';
import type { Area } from './area';
@@ -79,213 +57,6 @@ export const generateTableKey = ({
tableName: string;
}) => `${schemaNameToDomainSchemaName(schemaName) ?? ''}.${tableName}`;
-export const decodeViewDefinition = (
- databaseType: DatabaseType,
- viewDefinition?: string
-): string => {
- if (!viewDefinition) {
- return '';
- }
-
- let decodedViewDefinition: string;
- if (databaseType === DatabaseType.SQL_SERVER) {
- decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition);
- } else {
- decodedViewDefinition = decodeBase64ToUtf8(viewDefinition);
- }
-
- return decodedViewDefinition;
-};
-
-export const createTablesFromMetadata = ({
- databaseMetadata,
- databaseType,
-}: {
- databaseMetadata: DatabaseMetadata;
- databaseType: DatabaseType;
-}): DBTable[] => {
- const {
- tables: tableInfos,
- pk_info: primaryKeys,
- columns,
- indexes,
- views: views,
- } = databaseMetadata;
-
- // Pre-compute view names for faster lookup if there are views
- const viewNamesSet = new Set<string>();
- const materializedViewNamesSet = new Set<string>();
-
- if (views && views.length > 0) {
- views.forEach((view) => {
- const key = generateTableKey({
- schemaName: view.schema,
- tableName: view.view_name,
- });
- viewNamesSet.add(key);
-
- if (
- view.view_definition &&
- decodeViewDefinition(databaseType, view.view_definition)
- .toLowerCase()
- .includes('materialized')
- ) {
- materializedViewNamesSet.add(key);
- }
- });
- }
-
- // Pre-compute lookup maps for better performance
- const columnsByTable = new Map<string, (typeof columns)[0][]>();
- const indexesByTable = new Map<string, (typeof indexes)[0][]>();
- const primaryKeysByTable = new Map<string, (typeof primaryKeys)[0][]>();
-
- // Group columns by table
- columns.forEach((col) => {
- const key = generateTableKey({
- schemaName: col.schema,
- tableName: col.table,
- });
- if (!columnsByTable.has(key)) {
- columnsByTable.set(key, []);
- }
- columnsByTable.get(key)!.push(col);
- });
-
- // Group indexes by table
- indexes.forEach((idx) => {
- const key = generateTableKey({
- schemaName: idx.schema,
- tableName: idx.table,
- });
- if (!indexesByTable.has(key)) {
- indexesByTable.set(key, []);
- }
- indexesByTable.get(key)!.push(idx);
- });
-
- // Group primary keys by table
- primaryKeys.forEach((pk) => {
- const key = generateTableKey({
- schemaName: pk.schema,
- tableName: pk.table,
- });
- if (!primaryKeysByTable.has(key)) {
- primaryKeysByTable.set(key, []);
- }
- primaryKeysByTable.get(key)!.push(pk);
- });
-
- const result = tableInfos.map((tableInfo: TableInfo) => {
- const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema);
- const tableKey = generateTableKey({
- schemaName: tableInfo.schema,
- tableName: tableInfo.table,
- });
-
- // Use pre-computed lookups instead of filtering entire arrays
- const tableIndexes = indexesByTable.get(tableKey) || [];
- const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || [];
- const tableColumns = columnsByTable.get(tableKey) || [];
-
- // Aggregate indexes with multiple columns
- const aggregatedIndexes = createAggregatedIndexes({
- tableInfo,
- tableSchema,
- tableIndexes,
- });
-
- const fields = createFieldsFromMetadata({
- aggregatedIndexes,
- tableColumns,
- tablePrimaryKeys,
- tableInfo,
- tableSchema,
- });
-
- // Check for composite primary key and find matching index name
- const primaryKeyFields = fields.filter((f) => f.primaryKey);
- let pkMatchingIndexName: string | undefined;
- let pkIndex: DBIndex | undefined;
-
- if (primaryKeyFields.length >= 1) {
- // We have a composite primary key, look for an index that matches all PK columns
- const pkFieldNames = primaryKeyFields.map((f) => f.name).sort();
-
- // Find an index that matches the primary key columns exactly
- const matchingIndex = aggregatedIndexes.find((index) => {
- const indexColumnNames = index.columns
- .map((c) => c.name)
- .sort();
- return (
- indexColumnNames.length === pkFieldNames.length &&
- indexColumnNames.every((col, i) => col === pkFieldNames[i])
- );
- });
-
- if (matchingIndex) {
- pkMatchingIndexName = matchingIndex.name;
- // Create a special PK index
- pkIndex = {
- id: generateId(),
- name: matchingIndex.name,
- unique: true,
- fieldIds: primaryKeyFields.map((f) => f.id),
- createdAt: Date.now(),
- isPrimaryKey: true,
- };
- }
- }
-
- // Filter out the index that matches the composite PK (to avoid duplication)
- const filteredAggregatedIndexes = pkMatchingIndexName
- ? aggregatedIndexes.filter(
- (idx) => idx.name !== pkMatchingIndexName
- )
- : aggregatedIndexes;
-
- const dbIndexes = createIndexesFromMetadata({
- aggregatedIndexes: filteredAggregatedIndexes,
- fields,
- });
-
- // Add the PK index if it exists
- if (pkIndex) {
- dbIndexes.push(pkIndex);
- }
-
- // Determine if the current table is a view by checking against pre-computed sets
- const viewKey = generateTableKey({
- schemaName: tableSchema,
- tableName: tableInfo.table,
- });
- const isView = viewNamesSet.has(viewKey);
- const isMaterializedView = materializedViewNamesSet.has(viewKey);
-
- // Initial random positions; these will be adjusted later
- return {
- id: generateId(),
- name: tableInfo.table,
- schema: tableSchema,
- x: Math.random() * 1000, // Placeholder X
- y: Math.random() * 800, // Placeholder Y
- fields,
- indexes: dbIndexes,
- color: isMaterializedView
- ? materializedViewColor
- : isView
- ? viewColor
- : defaultTableColor,
- isView: isView,
- isMaterializedView: isMaterializedView,
- createdAt: Date.now(),
- comments: tableInfo.comment ? tableInfo.comment : undefined,
- };
- });
-
- return result;
-};
-
export const adjustTablePositions = ({
relationships: inputRelationships,
tables: inputTables,
diff --git a/src/lib/domain/diagram.ts b/src/lib/domain/diagram.ts
index ed171e72..a0548e41 100644
--- a/src/lib/domain/diagram.ts
+++ b/src/lib/domain/diagram.ts
@@ -1,30 +1,15 @@
import { z } from 'zod';
-import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata';
import { DatabaseEdition } from './database-edition';
import { DatabaseType } from './database-type';
import type { DBDependency } from './db-dependency';
-import {
- createDependenciesFromMetadata,
- dbDependencySchema,
-} from './db-dependency';
+import { dbDependencySchema } from './db-dependency';
import type { DBRelationship } from './db-relationship';
-import {
- createRelationshipsFromMetadata,
- dbRelationshipSchema,
-} from './db-relationship';
+import { dbRelationshipSchema } from './db-relationship';
import type { DBTable } from './db-table';
-import {
- adjustTablePositions,
- createTablesFromMetadata,
- dbTableSchema,
-} from './db-table';
-import { generateDiagramId } from '@/lib/utils';
+import { dbTableSchema } from './db-table';
import { areaSchema, type Area } from './area';
import type { DBCustomType } from './db-custom-type';
-import {
- dbCustomTypeSchema,
- createCustomTypesFromMetadata,
-} from './db-custom-type';
+import { dbCustomTypeSchema } from './db-custom-type';
export interface Diagram {
id: string;
@@ -53,77 +38,3 @@ export const diagramSchema: z.ZodType<Diagram> = z.object({
createdAt: z.date(),
updatedAt: z.date(),
});
-
-export const loadFromDatabaseMetadata = async ({
- databaseType,
- databaseMetadata,
- diagramNumber,
- databaseEdition,
-}: {
- databaseType: DatabaseType;
- databaseMetadata: DatabaseMetadata;
- diagramNumber?: number;
- databaseEdition?: DatabaseEdition;
-}): Promise<Diagram> => {
- const {
- fk_info: foreignKeys,
- views: views,
- custom_types: customTypes,
- } = databaseMetadata;
-
- const tables = createTablesFromMetadata({
- databaseMetadata,
- databaseType,
- });
-
- const relationships = createRelationshipsFromMetadata({
- foreignKeys,
- tables,
- });
-
- const dependencies = await createDependenciesFromMetadata({
- views,
- tables,
- databaseType,
- });
-
- const dbCustomTypes = customTypes
- ? createCustomTypesFromMetadata({
- customTypes,
- })
- : [];
-
- const adjustedTables = adjustTablePositions({
- tables,
- relationships,
- mode: 'perSchema',
- });
-
- const sortedTables = adjustedTables.sort((a, b) => {
- if (a.isView === b.isView) {
- // Both are either tables or views, so sort alphabetically by name
- return a.name.localeCompare(b.name);
- }
- // If one is a view and the other is not, put tables first
- return a.isView ? 1 : -1;
- });
-
- const diagram: Diagram = {
- id: generateDiagramId(),
- name: databaseMetadata.database_name
- ? `${databaseMetadata.database_name}-db`
- : diagramNumber
- ? `Diagram ${diagramNumber}`
- : 'New Diagram',
- databaseType: databaseType ?? DatabaseType.GENERIC,
- databaseEdition,
- tables: sortedTables,
- relationships,
- dependencies,
- customTypes: dbCustomTypes,
- createdAt: new Date(),
- updatedAt: new Date(),
- };
-
- return diagram;
-};