Database diagrams editor that allows you to visualize and design your DB with a single query.
TypeScript databasedbdb-migrationeditorerdmariadbmssqlmysqlpostgresqlreactreact-flowschema-migrationssqlsqlitetypescriptvisualizationxyflow
fix: export sql + import metadata lib (#902)
Guy Ben-Aharonffddcdcc987b
diff --git a/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx b/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsxindex ab8af742..b08a8390 100644--- a/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx+++ b/src/dialogs/create-diagram-dialog/create-diagram-dialog.tsx@@ -3,7 +3,7 @@ import { Dialog, DialogContent } from '@/components/dialog/dialog';import { DatabaseType } from '@/lib/domain/database-type';import { useStorage } from '@/hooks/use-storage';import type { Diagram } from '@/lib/domain/diagram';-import { loadFromDatabaseMetadata } from '@/lib/domain/diagram';+import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import';import { useNavigate } from 'react-router-dom';import { useConfig } from '@/hooks/use-config';import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';diff --git a/src/dialogs/export-sql-dialog/export-sql-dialog.tsx b/src/dialogs/export-sql-dialog/export-sql-dialog.tsxindex 949d3a53..df9d29cd 100644--- a/src/dialogs/export-sql-dialog/export-sql-dialog.tsx+++ b/src/dialogs/export-sql-dialog/export-sql-dialog.tsx@@ -17,7 +17,7 @@ import { useDialog } from '@/hooks/use-dialog';import {exportBaseSQL,exportSQL,-} from '@/lib/data/export-metadata/export-sql-script';+} from '@/lib/data/sql-export/export-sql-script';import { databaseTypeToLabelMap } from '@/lib/databases';import { DatabaseType } from '@/lib/domain/database-type';import { Annoyed, Sparkles } from 'lucide-react';diff --git a/src/dialogs/import-database-dialog/import-database-dialog.tsx b/src/dialogs/import-database-dialog/import-database-dialog.tsxindex ce29e75b..e7dd28af 100644--- a/src/dialogs/import-database-dialog/import-database-dialog.tsx+++ b/src/dialogs/import-database-dialog/import-database-dialog.tsx@@ -7,7 +7,7 @@ import type { DatabaseEdition } from '@/lib/domain/database-edition';import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';import { loadDatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';import type { Diagram } from '@/lib/domain/diagram';-import { loadFromDatabaseMetadata } from '@/lib/domain/diagram';+import { loadFromDatabaseMetadata } from '@/lib/data/import-metadata/import';import { useChartDB } from '@/hooks/use-chartdb';import { useRedoUndoStack } from '@/hooks/use-redo-undo-stack';import { Trans, useTranslation } from 'react-i18next';diff --git a/src/lib/data/import-metadata/import/custom-types.ts b/src/lib/data/import-metadata/import/custom-types.tsnew file mode 100644index 00000000..b3269f2a--- /dev/null+++ b/src/lib/data/import-metadata/import/custom-types.ts@@ -0,0 +1,21 @@+import type { DBCustomType, DBCustomTypeKind } from '@/lib/domain';+import { schemaNameToDomainSchemaName } from '@/lib/domain';+import type { DBCustomTypeInfo } from '../metadata-types/custom-type-info';+import { generateId } from '@/lib/utils';++export const createCustomTypesFromMetadata = ({+ customTypes,+}: {+ customTypes: DBCustomTypeInfo[];+}): DBCustomType[] => {+ return customTypes.map((customType) => {+ return {+ id: generateId(),+ schema: schemaNameToDomainSchemaName(customType.schema),+ name: customType.type,+ kind: customType.kind as DBCustomTypeKind,+ values: customType.values,+ fields: customType.fields,+ };+ });+};diff --git a/src/lib/data/import-metadata/import/dependencies.ts b/src/lib/data/import-metadata/import/dependencies.tsnew file mode 100644index 00000000..1937c5ec--- /dev/null+++ b/src/lib/data/import-metadata/import/dependencies.ts@@ -0,0 +1,351 @@+import { generateId } from '@/lib/utils';+import type { AST } from 'node-sql-parser';+import type { DBDependency, DBTable } from '@/lib/domain';+import { DatabaseType, schemaNameToDomainSchemaName } from '@/lib/domain';+import type { ViewInfo } from '../metadata-types/view-info';+import { decodeViewDefinition } from './tables';++const astDatabaseTypes: Record<DatabaseType, string> = {+ [DatabaseType.POSTGRESQL]: 'postgresql',+ [DatabaseType.MYSQL]: 'postgresql',+ [DatabaseType.MARIADB]: 'postgresql',+ [DatabaseType.GENERIC]: 'postgresql',+ [DatabaseType.SQLITE]: 'postgresql',+ [DatabaseType.SQL_SERVER]: 'postgresql',+ [DatabaseType.CLICKHOUSE]: 'postgresql',+ [DatabaseType.COCKROACHDB]: 'postgresql',+ [DatabaseType.ORACLE]: 'postgresql',+};++export const createDependenciesFromMetadata = async ({+ views,+ tables,+ databaseType,+}: {+ views: ViewInfo[];+ tables: DBTable[];+ databaseType: DatabaseType;+}): Promise<DBDependency[]> => {+ if (!views || views.length === 0) {+ return [];+ }++ const { Parser } = await import('node-sql-parser');+ const parser = new Parser();++ const dependencies = views+ .flatMap((view) => {+ const viewSchema = schemaNameToDomainSchemaName(view.schema);+ const viewTable = tables.find(+ (table) =>+ table.name === view.view_name && viewSchema === table.schema+ );++ if (!viewTable) {+ console.warn(+ `Source table for view ${view.view_name} not found (schema: ${viewSchema})`+ );+ return []; // Skip this view and proceed to the next+ }++ if (view.view_definition) {+ try {+ const decodedViewDefinition = decodeViewDefinition(+ databaseType,+ view.view_definition+ );++ let modifiedViewDefinition = '';+ if (+ databaseType === DatabaseType.MYSQL ||+ databaseType === DatabaseType.MARIADB+ ) {+ modifiedViewDefinition = preprocessViewDefinitionMySQL(+ decodedViewDefinition+ );+ } else if (databaseType === DatabaseType.SQL_SERVER) {+ modifiedViewDefinition =+ preprocessViewDefinitionSQLServer(+ decodedViewDefinition+ );+ } else {+ modifiedViewDefinition = preprocessViewDefinition(+ decodedViewDefinition+ );+ }++ // Parse using the appropriate dialect+ const ast = parser.astify(modifiedViewDefinition, {+ database: astDatabaseTypes[databaseType],+ type: 'select', // Parsing a SELECT statement+ });++ let relatedTables = extractTablesFromAST(ast);++ // Filter out duplicate tables without schema+ relatedTables = filterDuplicateTables(relatedTables);++ return relatedTables.map((relTable) => {+ const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined+ const relTableName = relTable.tableName;++ const table = tables.find(+ (table) =>+ table.name === relTableName &&+ (table.schema || '') === relSchema+ );++ if (table) {+ const dependency: DBDependency = {+ id: generateId(),+ schema: view.schema,+ tableId: table.id, // related table+ dependentSchema: table.schema,+ dependentTableId: viewTable.id, // dependent view+ createdAt: Date.now(),+ };++ return dependency;+ } else {+ console.warn(+ `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}`+ );+ return null;+ }+ });+ } catch (error) {+ console.error(+ `Error parsing view ${view.schema}.${view.view_name}:`,+ error+ );+ return [];+ }+ } else {+ console.warn(+ `View definition missing for ${view.schema}.${view.view_name}`+ );+ return [];+ }+ })+ .filter((dependency) => dependency !== null);++ return dependencies;+};++// Add this new function to filter out duplicate tables+function filterDuplicateTables(+ tables: { schema?: string; tableName: string }[]+): { schema?: string; tableName: string }[] {+ const tableMap = new Map<string, { schema?: string; tableName: string }>();++ for (const table of tables) {+ const key = table.tableName;+ const existingTable = tableMap.get(key);++ if (!existingTable || (table.schema && !existingTable.schema)) {+ tableMap.set(key, table);+ }+ }++ return Array.from(tableMap.values());+}++// Preprocess the view_definition to remove schema from CREATE VIEW+function preprocessViewDefinition(viewDefinition: string): string {+ if (!viewDefinition) {+ return '';+ }++ // Remove leading and trailing whitespace+ viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();++ // Replace escaped double quotes with regular ones+ viewDefinition = viewDefinition.replace(/\\"/g, '"');++ // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW'+ viewDefinition = viewDefinition.replace(+ /CREATE\s+MATERIALIZED\s+VIEW/i,+ 'CREATE VIEW'+ );++ // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS'+ // This regex captures the view name and skips any content between the view name and 'AS'+ const regex =+ /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i;++ const match = viewDefinition.match(regex);+ let modifiedDefinition: string;++ if (match) {+ const viewName = match[1] || match[2] || match[3];+ // Extract the SQL after the 'AS' keyword+ const restOfDefinition = viewDefinition.substring(+ match.index! + match[0].length+ );++ // Replace double-quoted identifiers with unquoted ones+ let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1');++ // Replace '::' type casts with 'CAST' expressions+ modifiedSQL = modifiedSQL.replace(+ /\(([^()]+)\)::(\w+)/g,+ 'CAST($1 AS $2)'+ );++ // Remove ClickHouse-specific syntax that may still be present+ // For example, remove SETTINGS clauses inside the SELECT statement+ modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, '');++ modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`;+ } else {+ console.warn('Could not preprocess view definition:', viewDefinition);+ modifiedDefinition = viewDefinition;+ }++ return modifiedDefinition;+}++// Preprocess the view_definition for SQL Server+function preprocessViewDefinitionSQLServer(viewDefinition: string): string {+ if (!viewDefinition) {+ return '';+ }++ // Remove BOM if present+ viewDefinition = viewDefinition.replace(/^\uFEFF/, '');++ // Normalize whitespace+ viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();++ // Remove square brackets and replace with double quotes+ viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"');++ // Remove database names from fully qualified identifiers+ viewDefinition = viewDefinition.replace(+ /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g,+ '"$2"."$3"'+ );++ // Replace SQL Server functions with PostgreSQL equivalents+ viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()');+ viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE(');++ // Replace 'TOP N' with 'LIMIT N' at the end of the query+ const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i);+ if (topMatch) {+ const topN = topMatch[1];+ viewDefinition = viewDefinition.replace(+ /SELECT\s+TOP\s+\d+/i,+ 'SELECT'+ );+ viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end+ viewDefinition += ` LIMIT ${topN}`;+ }++ viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines++ // Adjust CREATE VIEW syntax+ const regex =+ /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i;+ const match = viewDefinition.match(regex);+ let modifiedDefinition: string;++ if (match) {+ const viewName = match[2];+ const modifiedSQL = viewDefinition.substring(+ match.index! + match[0].length+ );++ // Remove semicolons at the end+ const finalSQL = modifiedSQL.replace(/;+\s*$/, '');++ modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`;+ } else {+ console.warn('Could not preprocess view definition:', viewDefinition);+ modifiedDefinition = viewDefinition;+ }++ return modifiedDefinition;+}++// Preprocess the view_definition to remove schema from CREATE VIEW+function preprocessViewDefinitionMySQL(viewDefinition: string): string {+ if (!viewDefinition) {+ return '';+ }++ // Remove any trailing semicolons+ viewDefinition = viewDefinition.replace(/;\s*$/, '');++ // Remove backticks from identifiers+ viewDefinition = viewDefinition.replace(/`/g, '');++ // Remove unnecessary parentheses around joins and ON clauses+ viewDefinition = removeRedundantParentheses(viewDefinition);++ return viewDefinition;+}++function removeRedundantParentheses(sql: string): string {+ // Regular expressions to match unnecessary parentheses+ const patterns = [+ /\(\s*(JOIN\s+[^()]+?)\s*\)/gi,+ /\(\s*(ON\s+[^()]+?)\s*\)/gi,+ // Additional patterns if necessary+ ];++ let prevSql;+ do {+ prevSql = sql;+ patterns.forEach((pattern) => {+ sql = sql.replace(pattern, '$1');+ });+ } while (sql !== prevSql);++ return sql;+}++function extractTablesFromAST(+ ast: AST | AST[]+): { schema?: string; tableName: string }[] {+ const tablesMap = new Map<string, { schema: string; tableName: string }>();+ const visitedNodes = new Set();++ // eslint-disable-next-line @typescript-eslint/no-explicit-any+ function traverse(node: any) {+ if (!node || visitedNodes.has(node)) return;+ visitedNodes.add(node);++ if (Array.isArray(node)) {+ node.forEach(traverse);+ } else if (typeof node === 'object') {+ // Check if node represents a table+ if (+ Object.hasOwnProperty.call(node, 'table') &&+ typeof node.table === 'string'+ ) {+ let schema = node.db || node.schema;+ const tableName = node.table;+ if (tableName) {+ // Assign default schema if undefined+ schema = schemaNameToDomainSchemaName(schema) || '';+ const key = `${schema}.${tableName}`;+ if (!tablesMap.has(key)) {+ tablesMap.set(key, { schema, tableName });+ }+ }+ }++ // Recursively traverse all properties+ for (const key in node) {+ if (Object.hasOwnProperty.call(node, key)) {+ traverse(node[key]);+ }+ }+ }+ }++ traverse(ast);++ return Array.from(tablesMap.values());+}diff --git a/src/lib/data/import-metadata/import/fields.ts b/src/lib/data/import-metadata/import/fields.tsnew file mode 100644index 00000000..01c901c8--- /dev/null+++ b/src/lib/data/import-metadata/import/fields.ts@@ -0,0 +1,64 @@+import type { DBField } from '@/lib/domain';+import type { ColumnInfo } from '../metadata-types/column-info';+import type { AggregatedIndexInfo } from '../metadata-types/index-info';+import type { PrimaryKeyInfo } from '../metadata-types/primary-key-info';+import type { TableInfo } from '../metadata-types/table-info';+import { generateId } from '@/lib/utils';++export const createFieldsFromMetadata = ({+ tableColumns,+ tablePrimaryKeys,+ aggregatedIndexes,+}: {+ tableColumns: ColumnInfo[];+ tableSchema?: string;+ tableInfo: TableInfo;+ tablePrimaryKeys: PrimaryKeyInfo[];+ aggregatedIndexes: AggregatedIndexInfo[];+}) => {+ const uniqueColumns = tableColumns.reduce((acc, col) => {+ if (!acc.has(col.name)) {+ acc.set(col.name, col);+ }+ return acc;+ }, new Map<string, ColumnInfo>());++ const sortedColumns = Array.from(uniqueColumns.values()).sort(+ (a, b) => a.ordinal_position - b.ordinal_position+ );++ const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) =>+ pk.column.trim()+ );++ return sortedColumns.map(+ (col: ColumnInfo): DBField => ({+ id: generateId(),+ name: col.name,+ type: {+ id: col.type.split(' ').join('_').toLowerCase(),+ name: col.type.toLowerCase(),+ },+ primaryKey: tablePrimaryKeysColumns.includes(col.name),+ unique: Object.values(aggregatedIndexes).some(+ (idx) =>+ idx.unique &&+ idx.columns.length === 1 &&+ idx.columns[0].name === col.name+ ),+ nullable: Boolean(col.nullable),+ ...(col.character_maximum_length &&+ col.character_maximum_length !== 'null'+ ? { characterMaximumLength: col.character_maximum_length }+ : {}),+ ...(col.precision?.precision+ ? { precision: col.precision.precision }+ : {}),+ ...(col.precision?.scale ? { scale: col.precision.scale } : {}),+ ...(col.default ? { default: col.default } : {}),+ ...(col.collation ? { collation: col.collation } : {}),+ createdAt: Date.now(),+ comments: col.comment ? col.comment : undefined,+ })+ );+};diff --git a/src/lib/data/import-metadata/import/index.ts b/src/lib/data/import-metadata/import/index.tsnew file mode 100644index 00000000..ecc907fb--- /dev/null+++ b/src/lib/data/import-metadata/import/index.ts@@ -0,0 +1,82 @@+import type { DatabaseEdition, Diagram } from '@/lib/domain';+import { adjustTablePositions, DatabaseType } from '@/lib/domain';+import { generateDiagramId } from '@/lib/utils';+import type { DatabaseMetadata } from '../metadata-types/database-metadata';+import { createCustomTypesFromMetadata } from './custom-types';+import { createRelationshipsFromMetadata } from './relationships';+import { createTablesFromMetadata } from './tables';+import { createDependenciesFromMetadata } from './dependencies';++export const loadFromDatabaseMetadata = async ({+ databaseType,+ databaseMetadata,+ diagramNumber,+ databaseEdition,+}: {+ databaseType: DatabaseType;+ databaseMetadata: DatabaseMetadata;+ diagramNumber?: number;+ databaseEdition?: DatabaseEdition;+}): Promise<Diagram> => {+ const {+ fk_info: foreignKeys,+ views: views,+ custom_types: customTypes,+ } = databaseMetadata;++ const tables = createTablesFromMetadata({+ databaseMetadata,+ databaseType,+ });++ const relationships = createRelationshipsFromMetadata({+ foreignKeys,+ tables,+ });++ const dependencies = await createDependenciesFromMetadata({+ views,+ tables,+ databaseType,+ });++ const dbCustomTypes = customTypes+ ? createCustomTypesFromMetadata({+ customTypes,+ })+ : [];++ const adjustedTables = adjustTablePositions({+ tables,+ relationships,+ mode: 'perSchema',+ });++ const sortedTables = adjustedTables.sort((a, b) => {+ if (a.isView === b.isView) {+ // Both are either tables or views, so sort alphabetically by name+ return a.name.localeCompare(b.name);+ }+ // If one is a view and the other is not, put tables first+ return a.isView ? 1 : -1;+ });++ const diagram: Diagram = {+ id: generateDiagramId(),+ name: databaseMetadata.database_name+ ? `${databaseMetadata.database_name}-db`+ : diagramNumber+ ? `Diagram ${diagramNumber}`+ : 'New Diagram',+ databaseType: databaseType ?? DatabaseType.GENERIC,+ databaseEdition,+ tables: sortedTables,+ relationships,+ dependencies,+ customTypes: dbCustomTypes,+ createdAt: new Date(),+ updatedAt: new Date(),+ };++ return diagram;+};diff --git a/src/lib/data/import-metadata/import/indexes.ts b/src/lib/data/import-metadata/import/indexes.tsnew file mode 100644index 00000000..c020fd41--- /dev/null+++ b/src/lib/data/import-metadata/import/indexes.ts@@ -0,0 +1,24 @@+import type { DBField, DBIndex, IndexType } from '@/lib/domain';+import type { AggregatedIndexInfo } from '../metadata-types/index-info';+import { generateId } from '@/lib/utils';++export const createIndexesFromMetadata = ({+ aggregatedIndexes,+ fields,+}: {+ aggregatedIndexes: AggregatedIndexInfo[];+ fields: DBField[];+}): DBIndex[] =>+ aggregatedIndexes.map(+ (idx): DBIndex => ({+ id: generateId(),+ name: idx.name,+ unique: Boolean(idx.unique),+ fieldIds: idx.columns+ .sort((a, b) => a.position - b.position)+ .map((c) => fields.find((f) => f.name === c.name)?.id)+ .filter((id): id is string => id !== undefined),+ createdAt: Date.now(),+ type: idx.index_type?.toLowerCase() as IndexType,+ })+ );diff --git a/src/lib/data/import-metadata/import/relationships.ts b/src/lib/data/import-metadata/import/relationships.tsnew file mode 100644index 00000000..0b0fd7aa--- /dev/null+++ b/src/lib/data/import-metadata/import/relationships.ts@@ -0,0 +1,85 @@+import type {+ Cardinality,+ DBField,+ DBRelationship,+ DBTable,+} from '@/lib/domain';+import { schemaNameToDomainSchemaName } from '@/lib/domain';+import type { ForeignKeyInfo } from '../metadata-types/foreign-key-info';+import { generateId } from '@/lib/utils';++const determineCardinality = (+ field: DBField,+ isTablePKComplex: boolean+): Cardinality => {+ return field.unique || (field.primaryKey && !isTablePKComplex)+ ? 'one'+ : 'many';+};++export const createRelationshipsFromMetadata = ({+ foreignKeys,+ tables,+}: {+ foreignKeys: ForeignKeyInfo[];+ tables: DBTable[];+}): DBRelationship[] => {+ return foreignKeys+ .map((fk: ForeignKeyInfo): DBRelationship | null => {+ const schema = schemaNameToDomainSchemaName(fk.schema);+ const sourceTable = tables.find(+ (table) => table.name === fk.table && table.schema === schema+ );++ const targetSchema = schemaNameToDomainSchemaName(+ fk.reference_schema+ );++ const targetTable = tables.find(+ (table) =>+ table.name === fk.reference_table &&+ table.schema === targetSchema+ );+ const sourceField = sourceTable?.fields.find(+ (field) => field.name === fk.column+ );+ const targetField = targetTable?.fields.find(+ (field) => field.name === fk.reference_column+ );++ const isSourceTablePKComplex =+ (sourceTable?.fields.filter((field) => field.primaryKey) ?? [])+ .length > 1;+ const isTargetTablePKComplex =+ (targetTable?.fields.filter((field) => field.primaryKey) ?? [])+ .length > 1;++ if (sourceTable && targetTable && sourceField && targetField) {+ const sourceCardinality = determineCardinality(+ sourceField,+ isSourceTablePKComplex+ );+ const targetCardinality = determineCardinality(+ targetField,+ isTargetTablePKComplex+ );++ return {+ id: generateId(),+ name: fk.foreign_key_name,+ sourceSchema: schema,+ targetSchema: targetSchema,+ sourceTableId: sourceTable.id,+ targetTableId: targetTable.id,+ sourceFieldId: sourceField.id,+ targetFieldId: targetField.id,+ sourceCardinality,+ targetCardinality,+ createdAt: Date.now(),+ };+ }++ return null;+ })+ .filter((rel) => rel !== null) as DBRelationship[];+};diff --git a/src/lib/data/import-metadata/import/tables.ts b/src/lib/data/import-metadata/import/tables.tsnew file mode 100644index 00000000..b45689b8--- /dev/null+++ b/src/lib/data/import-metadata/import/tables.ts@@ -0,0 +1,228 @@+import type { DBIndex, DBTable } from '@/lib/domain';+import {+ DatabaseType,+ generateTableKey,+ schemaNameToDomainSchemaName,+} from '@/lib/domain';+import type { DatabaseMetadata } from '../metadata-types/database-metadata';+import type { TableInfo } from '../metadata-types/table-info';+import { createAggregatedIndexes } from '../metadata-types/index-info';+import {+ decodeBase64ToUtf16LE,+ decodeBase64ToUtf8,+ generateId,+} from '@/lib/utils';+import {+ defaultTableColor,+ materializedViewColor,+ viewColor,+} from '@/lib/colors';+import { createFieldsFromMetadata } from './fields';+import { createIndexesFromMetadata } from './indexes';++export const decodeViewDefinition = (+ databaseType: DatabaseType,+ viewDefinition?: string+): string => {+ if (!viewDefinition) {+ return '';+ }++ let decodedViewDefinition: string;+ if (databaseType === DatabaseType.SQL_SERVER) {+ decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition);+ } else {+ decodedViewDefinition = decodeBase64ToUtf8(viewDefinition);+ }++ return decodedViewDefinition;+};++export const createTablesFromMetadata = ({+ databaseMetadata,+ databaseType,+}: {+ databaseMetadata: DatabaseMetadata;+ databaseType: DatabaseType;+}): DBTable[] => {+ const {+ tables: tableInfos,+ pk_info: primaryKeys,+ columns,+ indexes,+ views: views,+ } = databaseMetadata;++ // Pre-compute view names for faster lookup if there are views+ const viewNamesSet = new Set<string>();+ const materializedViewNamesSet = new Set<string>();++ if (views && views.length > 0) {+ views.forEach((view) => {+ const key = generateTableKey({+ schemaName: view.schema,+ tableName: view.view_name,+ });+ viewNamesSet.add(key);++ if (+ view.view_definition &&+ decodeViewDefinition(databaseType, view.view_definition)+ .toLowerCase()+ .includes('materialized')+ ) {+ materializedViewNamesSet.add(key);+ }+ });+ }++ // Pre-compute lookup maps for better performance+ const columnsByTable = new Map<string, (typeof columns)[0][]>();+ const indexesByTable = new Map<string, (typeof indexes)[0][]>();+ const primaryKeysByTable = new Map<string, (typeof primaryKeys)[0][]>();++ // Group columns by table+ columns.forEach((col) => {+ const key = generateTableKey({+ schemaName: col.schema,+ tableName: col.table,+ });+ if (!columnsByTable.has(key)) {+ columnsByTable.set(key, []);+ }+ columnsByTable.get(key)!.push(col);+ });++ // Group indexes by table+ indexes.forEach((idx) => {+ const key = generateTableKey({+ schemaName: idx.schema,+ tableName: idx.table,+ });+ if (!indexesByTable.has(key)) {+ indexesByTable.set(key, []);+ }+ indexesByTable.get(key)!.push(idx);+ });++ // Group primary keys by table+ primaryKeys.forEach((pk) => {+ const key = generateTableKey({+ schemaName: pk.schema,+ tableName: pk.table,+ });+ if (!primaryKeysByTable.has(key)) {+ primaryKeysByTable.set(key, []);+ }+ primaryKeysByTable.get(key)!.push(pk);+ });++ const result = tableInfos.map((tableInfo: TableInfo) => {+ const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema);+ const tableKey = generateTableKey({+ schemaName: tableInfo.schema,+ tableName: tableInfo.table,+ });++ // Use pre-computed lookups instead of filtering entire arrays+ const tableIndexes = indexesByTable.get(tableKey) || [];+ const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || [];+ const tableColumns = columnsByTable.get(tableKey) || [];++ // Aggregate indexes with multiple columns+ const aggregatedIndexes = createAggregatedIndexes({+ tableInfo,+ tableSchema,+ tableIndexes,+ });++ const fields = createFieldsFromMetadata({+ aggregatedIndexes,+ tableColumns,+ tablePrimaryKeys,+ tableInfo,+ tableSchema,+ });++ // Check for composite primary key and find matching index name+ const primaryKeyFields = fields.filter((f) => f.primaryKey);+ let pkMatchingIndexName: string | undefined;+ let pkIndex: DBIndex | undefined;++ if (primaryKeyFields.length >= 1) {+ // We have a composite primary key, look for an index that matches all PK columns+ const pkFieldNames = primaryKeyFields.map((f) => f.name).sort();++ // Find an index that matches the primary key columns exactly+ const matchingIndex = aggregatedIndexes.find((index) => {+ const indexColumnNames = index.columns+ .map((c) => c.name)+ .sort();+ return (+ indexColumnNames.length === pkFieldNames.length &&+ indexColumnNames.every((col, i) => col === pkFieldNames[i])+ );+ });++ if (matchingIndex) {+ pkMatchingIndexName = matchingIndex.name;+ // Create a special PK index+ pkIndex = {+ id: generateId(),+ name: matchingIndex.name,+ unique: true,+ fieldIds: primaryKeyFields.map((f) => f.id),+ createdAt: Date.now(),+ isPrimaryKey: true,+ };+ }+ }++ // Filter out the index that matches the composite PK (to avoid duplication)+ const filteredAggregatedIndexes = pkMatchingIndexName+ ? aggregatedIndexes.filter(+ (idx) => idx.name !== pkMatchingIndexName+ )+ : aggregatedIndexes;++ const dbIndexes = createIndexesFromMetadata({+ aggregatedIndexes: filteredAggregatedIndexes,+ fields,+ });++ // Add the PK index if it exists+ if (pkIndex) {+ dbIndexes.push(pkIndex);+ }++ // Determine if the current table is a view by checking against pre-computed sets+ const viewKey = generateTableKey({+ schemaName: tableSchema,+ tableName: tableInfo.table,+ });+ const isView = viewNamesSet.has(viewKey);+ const isMaterializedView = materializedViewNamesSet.has(viewKey);++ // Initial random positions; these will be adjusted later+ return {+ id: generateId(),+ name: tableInfo.table,+ schema: tableSchema,+ x: Math.random() * 1000, // Placeholder X+ y: Math.random() * 800, // Placeholder Y+ fields,+ indexes: dbIndexes,+ color: isMaterializedView+ ? materializedViewColor+ : isView+ ? viewColor+ : defaultTableColor,+ isView: isView,+ isMaterializedView: isMaterializedView,+ createdAt: Date.now(),+ comments: tableInfo.comment ? tableInfo.comment : undefined,+ };+ });++ return result;+};diff --git a/src/lib/data/export-metadata/__tests__/export-sql-dbml.test.ts b/src/lib/data/sql-export/__tests__/export-sql-dbml.test.tssimilarity index 100%rename from src/lib/data/export-metadata/__tests__/export-sql-dbml.test.tsrename to src/lib/data/sql-export/__tests__/export-sql-dbml.test.tsdiff --git a/src/lib/data/export-metadata/__tests__/export-sql-quoted-identifiers.test.ts b/src/lib/data/sql-export/__tests__/export-sql-quoted-identifiers.test.tssimilarity index 100%rename from src/lib/data/export-metadata/__tests__/export-sql-quoted-identifiers.test.tsrename to src/lib/data/sql-export/__tests__/export-sql-quoted-identifiers.test.tsdiff --git a/src/lib/data/export-metadata/export-per-type/common.ts b/src/lib/data/sql-export/export-per-type/common.tssimilarity index 100%rename from src/lib/data/export-metadata/export-per-type/common.tsrename to src/lib/data/sql-export/export-per-type/common.tsdiff --git a/src/lib/data/export-metadata/export-per-type/mssql.ts b/src/lib/data/sql-export/export-per-type/mssql.tssimilarity index 100%rename from src/lib/data/export-metadata/export-per-type/mssql.tsrename to src/lib/data/sql-export/export-per-type/mssql.tsdiff --git a/src/lib/data/export-metadata/export-per-type/mysql.ts b/src/lib/data/sql-export/export-per-type/mysql.tssimilarity index 100%rename from src/lib/data/export-metadata/export-per-type/mysql.tsrename to src/lib/data/sql-export/export-per-type/mysql.tsdiff --git a/src/lib/data/export-metadata/export-per-type/postgresql.ts b/src/lib/data/sql-export/export-per-type/postgresql.tssimilarity index 100%rename from src/lib/data/export-metadata/export-per-type/postgresql.tsrename to src/lib/data/sql-export/export-per-type/postgresql.tsdiff --git a/src/lib/data/export-metadata/export-per-type/sqlite.ts b/src/lib/data/sql-export/export-per-type/sqlite.tssimilarity index 100%rename from src/lib/data/export-metadata/export-per-type/sqlite.tsrename to src/lib/data/sql-export/export-per-type/sqlite.tsdiff --git a/src/lib/data/export-metadata/export-sql-cache.ts b/src/lib/data/sql-export/export-sql-cache.tssimilarity index 100%rename from src/lib/data/export-metadata/export-sql-cache.tsrename to src/lib/data/sql-export/export-sql-cache.tsdiff --git a/src/lib/data/export-metadata/export-sql-script.ts b/src/lib/data/sql-export/export-sql-script.tssimilarity index 100%rename from src/lib/data/export-metadata/export-sql-script.tsrename to src/lib/data/sql-export/export-sql-script.tsdiff --git a/src/lib/dbml/dbml-export/dbml-export.ts b/src/lib/dbml/dbml-export/dbml-export.tsindex 5030085d..b0049583 100644--- a/src/lib/dbml/dbml-export/dbml-export.ts+++ b/src/lib/dbml/dbml-export/dbml-export.ts@@ -1,5 +1,5 @@import { importer } from '@dbml/core';-import { exportBaseSQL } from '@/lib/data/export-metadata/export-sql-script';+import { exportBaseSQL } from '@/lib/data/sql-export/export-sql-script';import type { Diagram } from '@/lib/domain/diagram';import { DatabaseType } from '@/lib/domain/database-type';import type { DBTable } from '@/lib/domain/db-table';diff --git a/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts b/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.tsindex a2fe3e5d..6cc40513 100644--- a/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts+++ b/src/lib/dbml/dbml-import/__tests__/composite-pk-name.test.ts@@ -1,8 +1,8 @@import { describe, it, expect } from 'vitest';import { importDBMLToDiagram } from '../dbml-import';-import { exportPostgreSQL } from '@/lib/data/export-metadata/export-per-type/postgresql';-import { exportMySQL } from '@/lib/data/export-metadata/export-per-type/mysql';-import { exportMSSQL } from '@/lib/data/export-metadata/export-per-type/mssql';+import { exportPostgreSQL } from '@/lib/data/sql-export/export-per-type/postgresql';+import { exportMySQL } from '@/lib/data/sql-export/export-per-type/mysql';+import { exportMSSQL } from '@/lib/data/sql-export/export-per-type/mssql';import { DatabaseType } from '@/lib/domain/database-type';describe('Composite Primary Key with Name', () => {diff --git a/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts b/src/lib/domain/__tests__/composite-pk-metadata-import.test.tsindex 252ff579..22d1c76a 100644--- a/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts+++ b/src/lib/domain/__tests__/composite-pk-metadata-import.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { createTablesFromMetadata } from '../db-table';+import { createTablesFromMetadata } from '@/lib/data/import-metadata/import/tables';import { DatabaseType } from '../database-type';import type { DatabaseMetadata } from '@/lib/data/import-metadata/metadata-types/database-metadata';diff --git a/src/lib/domain/db-custom-type.ts b/src/lib/domain/db-custom-type.tsindex 59a00679..2252d031 100644--- a/src/lib/domain/db-custom-type.ts+++ b/src/lib/domain/db-custom-type.ts@@ -1,7 +1,4 @@import { z } from 'zod';-import type { DBCustomTypeInfo } from '@/lib/data/import-metadata/metadata-types/custom-type-info';-import { generateId } from '../utils';-import { schemaNameToDomainSchemaName } from './db-schema';export enum DBCustomTypeKind {enum = 'enum',@@ -38,23 +35,6 @@ export const dbCustomTypeSchema: z.ZodType<DBCustomType> = z.object({order: z.number().or(z.null()).optional(),});-export const createCustomTypesFromMetadata = ({- customTypes,-}: {- customTypes: DBCustomTypeInfo[];-}): DBCustomType[] => {- return customTypes.map((customType) => {- return {- id: generateId(),- schema: schemaNameToDomainSchemaName(customType.schema),- name: customType.type,- kind: customType.kind as DBCustomTypeKind,- values: customType.values,- fields: customType.fields,- };- });-};-export const customTypeKindToLabel: Record<DBCustomTypeKind, string> = {enum: 'Enum',composite: 'Composite',diff --git a/src/lib/domain/db-dependency.ts b/src/lib/domain/db-dependency.tsindex 347194e2..4f9c4bba 100644--- a/src/lib/domain/db-dependency.ts+++ b/src/lib/domain/db-dependency.ts@@ -1,10 +1,4 @@import { z } from 'zod';-import type { ViewInfo } from '../data/import-metadata/metadata-types/view-info';-import { DatabaseType } from './database-type';-import { schemaNameToDomainSchemaName } from './db-schema';-import { decodeViewDefinition, type DBTable } from './db-table';-import { generateId } from '@/lib/utils';-import type { AST } from 'node-sql-parser';export interface DBDependency {id: string;@@ -23,348 +17,3 @@ export const dbDependencySchema: z.ZodType<DBDependency> = z.object({dependentTableId: z.string(),createdAt: z.number(),});--const astDatabaseTypes: Record<DatabaseType, string> = {- [DatabaseType.POSTGRESQL]: 'postgresql',- [DatabaseType.MYSQL]: 'postgresql',- [DatabaseType.MARIADB]: 'postgresql',- [DatabaseType.GENERIC]: 'postgresql',- [DatabaseType.SQLITE]: 'postgresql',- [DatabaseType.SQL_SERVER]: 'postgresql',- [DatabaseType.CLICKHOUSE]: 'postgresql',- [DatabaseType.COCKROACHDB]: 'postgresql',- [DatabaseType.ORACLE]: 'postgresql',-};--export const createDependenciesFromMetadata = async ({- views,- tables,- databaseType,-}: {- views: ViewInfo[];- tables: DBTable[];- databaseType: DatabaseType;-}): Promise<DBDependency[]> => {- if (!views || views.length === 0) {- return [];- }-- const { Parser } = await import('node-sql-parser');- const parser = new Parser();-- const dependencies = views- .flatMap((view) => {- const viewSchema = schemaNameToDomainSchemaName(view.schema);- const viewTable = tables.find(- (table) =>- table.name === view.view_name && viewSchema === table.schema- );-- if (!viewTable) {- console.warn(- `Source table for view ${view.view_name} not found (schema: ${viewSchema})`- );- return []; // Skip this view and proceed to the next- }-- if (view.view_definition) {- try {- const decodedViewDefinition = decodeViewDefinition(- databaseType,- view.view_definition- );-- let modifiedViewDefinition = '';- if (- databaseType === DatabaseType.MYSQL ||- databaseType === DatabaseType.MARIADB- ) {- modifiedViewDefinition = preprocessViewDefinitionMySQL(- decodedViewDefinition- );- } else if (databaseType === DatabaseType.SQL_SERVER) {- modifiedViewDefinition =- preprocessViewDefinitionSQLServer(- decodedViewDefinition- );- } else {- modifiedViewDefinition = preprocessViewDefinition(- decodedViewDefinition- );- }-- // Parse using the appropriate dialect- const ast = parser.astify(modifiedViewDefinition, {- database: astDatabaseTypes[databaseType],- type: 'select', // Parsing a SELECT statement- });-- let relatedTables = extractTablesFromAST(ast);-- // Filter out duplicate tables without schema- relatedTables = filterDuplicateTables(relatedTables);-- return relatedTables.map((relTable) => {- const relSchema = relTable.schema || view.schema; // Use view's schema if relSchema is undefined- const relTableName = relTable.tableName;-- const table = tables.find(- (table) =>- table.name === relTableName &&- (table.schema || '') === relSchema- );-- if (table) {- const dependency: DBDependency = {- id: generateId(),- schema: view.schema,- tableId: table.id, // related table- dependentSchema: table.schema,- dependentTableId: viewTable.id, // dependent view- createdAt: Date.now(),- };-- return dependency;- } else {- console.warn(- `Dependent table ${relSchema}.${relTableName} not found for view ${view.schema}.${view.view_name}`- );- return null;- }- });- } catch (error) {- console.error(- `Error parsing view ${view.schema}.${view.view_name}:`,- error- );- return [];- }- } else {- console.warn(- `View definition missing for ${view.schema}.${view.view_name}`- );- return [];- }- })- .filter((dependency) => dependency !== null);-- return dependencies;-};--// Add this new function to filter out duplicate tables-function filterDuplicateTables(- tables: { schema?: string; tableName: string }[]-): { schema?: string; tableName: string }[] {- const tableMap = new Map<string, { schema?: string; tableName: string }>();-- for (const table of tables) {- const key = table.tableName;- const existingTable = tableMap.get(key);-- if (!existingTable || (table.schema && !existingTable.schema)) {- tableMap.set(key, table);- }- }-- return Array.from(tableMap.values());-}--// Preprocess the view_definition to remove schema from CREATE VIEW-function preprocessViewDefinition(viewDefinition: string): string {- if (!viewDefinition) {- return '';- }-- // Remove leading and trailing whitespace- viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();-- // Replace escaped double quotes with regular ones- viewDefinition = viewDefinition.replace(/\\"/g, '"');-- // Replace 'CREATE MATERIALIZED VIEW' with 'CREATE VIEW'- viewDefinition = viewDefinition.replace(- /CREATE\s+MATERIALIZED\s+VIEW/i,- 'CREATE VIEW'- );-- // Regular expression to match 'CREATE VIEW [schema.]view_name [ (column definitions) ] AS'- // This regex captures the view name and skips any content between the view name and 'AS'- const regex =- /CREATE\s+VIEW\s+(?:(?:`[^`]+`|"[^"]+"|\w+)\.)?(?:`([^`]+)`|"([^"]+)"|(\w+))[\s\S]*?\bAS\b\s+/i;-- const match = viewDefinition.match(regex);- let modifiedDefinition: string;-- if (match) {- const viewName = match[1] || match[2] || match[3];- // Extract the SQL after the 'AS' keyword- const restOfDefinition = viewDefinition.substring(- match.index! + match[0].length- );-- // Replace double-quoted identifiers with unquoted ones- let modifiedSQL = restOfDefinition.replace(/"(\w+)"/g, '$1');-- // Replace '::' type casts with 'CAST' expressions- modifiedSQL = modifiedSQL.replace(- /\(([^()]+)\)::(\w+)/g,- 'CAST($1 AS $2)'- );-- // Remove ClickHouse-specific syntax that may still be present- // For example, remove SETTINGS clauses inside the SELECT statement- modifiedSQL = modifiedSQL.replace(/\bSETTINGS\b[\s\S]*$/i, '');-- modifiedDefinition = `CREATE VIEW ${viewName} AS ${modifiedSQL}`;- } else {- console.warn('Could not preprocess view definition:', viewDefinition);- modifiedDefinition = viewDefinition;- }-- return modifiedDefinition;-}--// Preprocess the view_definition for SQL Server-function preprocessViewDefinitionSQLServer(viewDefinition: string): string {- if (!viewDefinition) {- return '';- }-- // Remove BOM if present- viewDefinition = viewDefinition.replace(/^\uFEFF/, '');-- // Normalize whitespace- viewDefinition = viewDefinition.replace(/\s+/g, ' ').trim();-- // Remove square brackets and replace with double quotes- viewDefinition = viewDefinition.replace(/\[([^\]]+)\]/g, '"$1"');-- // Remove database names from fully qualified identifiers- viewDefinition = viewDefinition.replace(- /"([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"\."([a-zA-Z0-9_]+)"/g,- '"$2"."$3"'- );-- // Replace SQL Server functions with PostgreSQL equivalents- viewDefinition = viewDefinition.replace(/\bGETDATE\(\)/gi, 'NOW()');- viewDefinition = viewDefinition.replace(/\bISNULL\(/gi, 'COALESCE(');-- // Replace 'TOP N' with 'LIMIT N' at the end of the query- const topMatch = viewDefinition.match(/SELECT\s+TOP\s+(\d+)/i);- if (topMatch) {- const topN = topMatch[1];- viewDefinition = viewDefinition.replace(- /SELECT\s+TOP\s+\d+/i,- 'SELECT'- );- viewDefinition = viewDefinition.replace(/;+\s*$/, ''); // Remove semicolons at the end- viewDefinition += ` LIMIT ${topN}`;- }-- viewDefinition = viewDefinition.replace(/\n/g, ''); // Remove newlines-- // Adjust CREATE VIEW syntax- const regex =- /CREATE\s+VIEW\s+(?:"?([^".\s]+)"?\.)?"?([^".\s]+)"?\s+AS\s+/i;- const match = viewDefinition.match(regex);- let modifiedDefinition: string;-- if (match) {- const viewName = match[2];- const modifiedSQL = viewDefinition.substring(- match.index! + match[0].length- );-- // Remove semicolons at the end- const finalSQL = modifiedSQL.replace(/;+\s*$/, '');-- modifiedDefinition = `CREATE VIEW "${viewName}" AS ${finalSQL}`;- } else {- console.warn('Could not preprocess view definition:', viewDefinition);- modifiedDefinition = viewDefinition;- }-- return modifiedDefinition;-}--// Preprocess the view_definition to remove schema from CREATE VIEW-function preprocessViewDefinitionMySQL(viewDefinition: string): string {- if (!viewDefinition) {- return '';- }-- // Remove any trailing semicolons- viewDefinition = viewDefinition.replace(/;\s*$/, '');-- // Remove backticks from identifiers- viewDefinition = viewDefinition.replace(/`/g, '');-- // Remove unnecessary parentheses around joins and ON clauses- viewDefinition = removeRedundantParentheses(viewDefinition);-- return viewDefinition;-}--function removeRedundantParentheses(sql: string): string {- // Regular expressions to match unnecessary parentheses- const patterns = [- /\(\s*(JOIN\s+[^()]+?)\s*\)/gi,- /\(\s*(ON\s+[^()]+?)\s*\)/gi,- // Additional patterns if necessary- ];-- let prevSql;- do {- prevSql = sql;- patterns.forEach((pattern) => {- sql = sql.replace(pattern, '$1');- });- } while (sql !== prevSql);-- return sql;-}--function extractTablesFromAST(- ast: AST | AST[]-): { schema?: string; tableName: string }[] {- const tablesMap = new Map<string, { schema: string; tableName: string }>();- const visitedNodes = new Set();-- // eslint-disable-next-line @typescript-eslint/no-explicit-any- function traverse(node: any) {- if (!node || visitedNodes.has(node)) return;- visitedNodes.add(node);-- if (Array.isArray(node)) {- node.forEach(traverse);- } else if (typeof node === 'object') {- // Check if node represents a table- if (- Object.hasOwnProperty.call(node, 'table') &&- typeof node.table === 'string'- ) {- let schema = node.db || node.schema;- const tableName = node.table;- if (tableName) {- // Assign default schema if undefined- schema = schemaNameToDomainSchemaName(schema) || '';- const key = `${schema}.${tableName}`;- if (!tablesMap.has(key)) {- tablesMap.set(key, { schema, tableName });- }- }- }-- // Recursively traverse all properties- for (const key in node) {- if (Object.hasOwnProperty.call(node, key)) {- traverse(node[key]);- }- }- }- }-- traverse(ast);-- return Array.from(tablesMap.values());-}diff --git a/src/lib/domain/db-field.ts b/src/lib/domain/db-field.tsindex dfc1a032..042530e3 100644--- a/src/lib/domain/db-field.ts+++ b/src/lib/domain/db-field.ts@@ -4,11 +4,6 @@ import {findDataTypeDataById,type DataType,} from '../data/data-types/data-types';-import type { ColumnInfo } from '../data/import-metadata/metadata-types/column-info';-import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info';-import type { PrimaryKeyInfo } from '../data/import-metadata/metadata-types/primary-key-info';-import type { TableInfo } from '../data/import-metadata/metadata-types/table-info';-import { generateId } from '../utils';import type { DatabaseType } from './database-type';export interface DBField {@@ -45,64 +40,6 @@ export const dbFieldSchema: z.ZodType<DBField> = z.object({comments: z.string().or(z.null()).optional(),});-export const createFieldsFromMetadata = ({- tableColumns,- tablePrimaryKeys,- aggregatedIndexes,-}: {- tableColumns: ColumnInfo[];- tableSchema?: string;- tableInfo: TableInfo;- tablePrimaryKeys: PrimaryKeyInfo[];- aggregatedIndexes: AggregatedIndexInfo[];-}) => {- const uniqueColumns = tableColumns.reduce((acc, col) => {- if (!acc.has(col.name)) {- acc.set(col.name, col);- }- return acc;- }, new Map<string, ColumnInfo>());-- const sortedColumns = Array.from(uniqueColumns.values()).sort(- (a, b) => a.ordinal_position - b.ordinal_position- );-- const tablePrimaryKeysColumns = tablePrimaryKeys.map((pk) =>- pk.column.trim()- );-- return sortedColumns.map(- (col: ColumnInfo): DBField => ({- id: generateId(),- name: col.name,- type: {- id: col.type.split(' ').join('_').toLowerCase(),- name: col.type.toLowerCase(),- },- primaryKey: tablePrimaryKeysColumns.includes(col.name),- unique: Object.values(aggregatedIndexes).some(- (idx) =>- idx.unique &&- idx.columns.length === 1 &&- idx.columns[0].name === col.name- ),- nullable: Boolean(col.nullable),- ...(col.character_maximum_length &&- col.character_maximum_length !== 'null'- ? { characterMaximumLength: col.character_maximum_length }- : {}),- ...(col.precision?.precision- ? { precision: col.precision.precision }- : {}),- ...(col.precision?.scale ? { scale: col.precision.scale } : {}),- ...(col.default ? { default: col.default } : {}),- ...(col.collation ? { collation: col.collation } : {}),- createdAt: Date.now(),- comments: col.comment ? col.comment : undefined,- })- );-};-export const generateDBFieldSuffix = (field: DBField,{diff --git a/src/lib/domain/db-index.ts b/src/lib/domain/db-index.tsindex b9a8c900..b9668daf 100644--- a/src/lib/domain/db-index.ts+++ b/src/lib/domain/db-index.ts@@ -1,7 +1,5 @@import { z } from 'zod';-import type { AggregatedIndexInfo } from '../data/import-metadata/metadata-types/index-info';import { generateId } from '../utils';-import type { DBField } from './db-field';import { DatabaseType } from './database-type';import type { DBTable } from './db-table';@@ -43,27 +41,6 @@ export const dbIndexSchema: z.ZodType<DBIndex> = z.object({isPrimaryKey: z.boolean().or(z.null()).optional(),});-export const createIndexesFromMetadata = ({- aggregatedIndexes,- fields,-}: {- aggregatedIndexes: AggregatedIndexInfo[];- fields: DBField[];-}): DBIndex[] =>- aggregatedIndexes.map(- (idx): DBIndex => ({- id: generateId(),- name: idx.name,- unique: Boolean(idx.unique),- fieldIds: idx.columns- .sort((a, b) => a.position - b.position)- .map((c) => fields.find((f) => f.name === c.name)?.id)- .filter((id): id is string => id !== undefined),- createdAt: Date.now(),- type: idx.index_type?.toLowerCase() as IndexType,- })- );-export const databaseIndexTypes: { [key in DatabaseType]?: IndexType[] } = {[DatabaseType.POSTGRESQL]: ['btree', 'hash'],};diff --git a/src/lib/domain/db-relationship.ts b/src/lib/domain/db-relationship.tsindex fd5debd2..c470a002 100644--- a/src/lib/domain/db-relationship.ts+++ b/src/lib/domain/db-relationship.ts@@ -1,9 +1,4 @@import { z } from 'zod';-import type { ForeignKeyInfo } from '../data/import-metadata/metadata-types/foreign-key-info';-import type { DBField } from './db-field';-import { schemaNameToDomainSchemaName } from './db-schema';-import type { DBTable } from './db-table';-import { generateId } from '@/lib/utils';export interface DBRelationship {id: string;@@ -40,82 +35,6 @@ export type RelationshipType =| 'many_to_many';export type Cardinality = 'one' | 'many';-const determineCardinality = (- field: DBField,- isTablePKComplex: boolean-): Cardinality => {- return field.unique || (field.primaryKey && !isTablePKComplex)- ? 'one'- : 'many';-};--export const createRelationshipsFromMetadata = ({- foreignKeys,- tables,-}: {- foreignKeys: ForeignKeyInfo[];- tables: DBTable[];-}): DBRelationship[] => {- return foreignKeys- .map((fk: ForeignKeyInfo): DBRelationship | null => {- const schema = schemaNameToDomainSchemaName(fk.schema);- const sourceTable = tables.find(- (table) => table.name === fk.table && table.schema === schema- );-- const targetSchema = schemaNameToDomainSchemaName(- fk.reference_schema- );-- const targetTable = tables.find(- (table) =>- table.name === fk.reference_table &&- table.schema === targetSchema- );- const sourceField = sourceTable?.fields.find(- (field) => field.name === fk.column- );- const targetField = targetTable?.fields.find(- (field) => field.name === fk.reference_column- );-- const isSourceTablePKComplex =- (sourceTable?.fields.filter((field) => field.primaryKey) ?? [])- .length > 1;- const isTargetTablePKComplex =- (targetTable?.fields.filter((field) => field.primaryKey) ?? [])- .length > 1;-- if (sourceTable && targetTable && sourceField && targetField) {- const sourceCardinality = determineCardinality(- sourceField,- isSourceTablePKComplex- );- const targetCardinality = determineCardinality(- targetField,- isTargetTablePKComplex- );-- return {- id: generateId(),- name: fk.foreign_key_name,- sourceSchema: schema,- targetSchema: targetSchema,- sourceTableId: sourceTable.id,- targetTableId: targetTable.id,- sourceFieldId: sourceField.id,- targetFieldId: targetField.id,- sourceCardinality,- targetCardinality,- createdAt: Date.now(),- };- }-- return null;- })- .filter((rel) => rel !== null) as DBRelationship[];-};-export const determineRelationshipType = ({sourceCardinality,targetCardinality,diff --git a/src/lib/domain/db-table.ts b/src/lib/domain/db-table.tsindex 4879f469..87f6e62f 100644--- a/src/lib/domain/db-table.ts+++ b/src/lib/domain/db-table.ts@@ -1,30 +1,8 @@-import {- createIndexesFromMetadata,- dbIndexSchema,- type DBIndex,-} from './db-index';-import {- createFieldsFromMetadata,- dbFieldSchema,- type DBField,-} from './db-field';-import type { TableInfo } from '../data/import-metadata/metadata-types/table-info';-import { createAggregatedIndexes } from '../data/import-metadata/metadata-types/index-info';-import {- materializedViewColor,- viewColor,- defaultTableColor,-} from '@/lib/colors';+import { dbIndexSchema, type DBIndex } from './db-index';+import { dbFieldSchema, type DBField } from './db-field';import type { DBRelationship } from './db-relationship';-import {- decodeBase64ToUtf16LE,- decodeBase64ToUtf8,- deepCopy,- generateId,-} from '../utils';+import { deepCopy } from '../utils';import { schemaNameToDomainSchemaName } from './db-schema';-import { DatabaseType } from './database-type';-import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata';import { z } from 'zod';import type { Area } from './area';@@ -79,213 +57,6 @@ export const generateTableKey = ({tableName: string;}) => `${schemaNameToDomainSchemaName(schemaName) ?? ''}.${tableName}`;-export const decodeViewDefinition = (- databaseType: DatabaseType,- viewDefinition?: string-): string => {- if (!viewDefinition) {- return '';- }-- let decodedViewDefinition: string;- if (databaseType === DatabaseType.SQL_SERVER) {- decodedViewDefinition = decodeBase64ToUtf16LE(viewDefinition);- } else {- decodedViewDefinition = decodeBase64ToUtf8(viewDefinition);- }-- return decodedViewDefinition;-};--export const createTablesFromMetadata = ({- databaseMetadata,- databaseType,-}: {- databaseMetadata: DatabaseMetadata;- databaseType: DatabaseType;-}): DBTable[] => {- const {- tables: tableInfos,- pk_info: primaryKeys,- columns,- indexes,- views: views,- } = databaseMetadata;-- // Pre-compute view names for faster lookup if there are views- const viewNamesSet = new Set<string>();- const materializedViewNamesSet = new Set<string>();-- if (views && views.length > 0) {- views.forEach((view) => {- const key = generateTableKey({- schemaName: view.schema,- tableName: view.view_name,- });- viewNamesSet.add(key);-- if (- view.view_definition &&- decodeViewDefinition(databaseType, view.view_definition)- .toLowerCase()- .includes('materialized')- ) {- materializedViewNamesSet.add(key);- }- });- }-- // Pre-compute lookup maps for better performance- const columnsByTable = new Map<string, (typeof columns)[0][]>();- const indexesByTable = new Map<string, (typeof indexes)[0][]>();- const primaryKeysByTable = new Map<string, (typeof primaryKeys)[0][]>();-- // Group columns by table- columns.forEach((col) => {- const key = generateTableKey({- schemaName: col.schema,- tableName: col.table,- });- if (!columnsByTable.has(key)) {- columnsByTable.set(key, []);- }- columnsByTable.get(key)!.push(col);- });-- // Group indexes by table- indexes.forEach((idx) => {- const key = generateTableKey({- schemaName: idx.schema,- tableName: idx.table,- });- if (!indexesByTable.has(key)) {- indexesByTable.set(key, []);- }- indexesByTable.get(key)!.push(idx);- });-- // Group primary keys by table- primaryKeys.forEach((pk) => {- const key = generateTableKey({- schemaName: pk.schema,- tableName: pk.table,- });- if (!primaryKeysByTable.has(key)) {- primaryKeysByTable.set(key, []);- }- primaryKeysByTable.get(key)!.push(pk);- });-- const result = tableInfos.map((tableInfo: TableInfo) => {- const tableSchema = schemaNameToDomainSchemaName(tableInfo.schema);- const tableKey = generateTableKey({- schemaName: tableInfo.schema,- tableName: tableInfo.table,- });-- // Use pre-computed lookups instead of filtering entire arrays- const tableIndexes = indexesByTable.get(tableKey) || [];- const tablePrimaryKeys = primaryKeysByTable.get(tableKey) || [];- const tableColumns = columnsByTable.get(tableKey) || [];-- // Aggregate indexes with multiple columns- const aggregatedIndexes = createAggregatedIndexes({- tableInfo,- tableSchema,- tableIndexes,- });-- const fields = createFieldsFromMetadata({- aggregatedIndexes,- tableColumns,- tablePrimaryKeys,- tableInfo,- tableSchema,- });-- // Check for composite primary key and find matching index name- const primaryKeyFields = fields.filter((f) => f.primaryKey);- let pkMatchingIndexName: string | undefined;- let pkIndex: DBIndex | undefined;-- if (primaryKeyFields.length >= 1) {- // We have a composite primary key, look for an index that matches all PK columns- const pkFieldNames = primaryKeyFields.map((f) => f.name).sort();-- // Find an index that matches the primary key columns exactly- const matchingIndex = aggregatedIndexes.find((index) => {- const indexColumnNames = index.columns- .map((c) => c.name)- .sort();- return (- indexColumnNames.length === pkFieldNames.length &&- indexColumnNames.every((col, i) => col === pkFieldNames[i])- );- });-- if (matchingIndex) {- pkMatchingIndexName = matchingIndex.name;- // Create a special PK index- pkIndex = {- id: generateId(),- name: matchingIndex.name,- unique: true,- fieldIds: primaryKeyFields.map((f) => f.id),- createdAt: Date.now(),- isPrimaryKey: true,- };- }- }-- // Filter out the index that matches the composite PK (to avoid duplication)- const filteredAggregatedIndexes = pkMatchingIndexName- ? aggregatedIndexes.filter(- (idx) => idx.name !== pkMatchingIndexName- )- : aggregatedIndexes;-- const dbIndexes = createIndexesFromMetadata({- aggregatedIndexes: filteredAggregatedIndexes,- fields,- });-- // Add the PK index if it exists- if (pkIndex) {- dbIndexes.push(pkIndex);- }-- // Determine if the current table is a view by checking against pre-computed sets- const viewKey = generateTableKey({- schemaName: tableSchema,- tableName: tableInfo.table,- });- const isView = viewNamesSet.has(viewKey);- const isMaterializedView = materializedViewNamesSet.has(viewKey);-- // Initial random positions; these will be adjusted later- return {- id: generateId(),- name: tableInfo.table,- schema: tableSchema,- x: Math.random() * 1000, // Placeholder X- y: Math.random() * 800, // Placeholder Y- fields,- indexes: dbIndexes,- color: isMaterializedView- ? materializedViewColor- : isView- ? viewColor- : defaultTableColor,- isView: isView,- isMaterializedView: isMaterializedView,- createdAt: Date.now(),- comments: tableInfo.comment ? tableInfo.comment : undefined,- };- });-- return result;-};-export const adjustTablePositions = ({relationships: inputRelationships,tables: inputTables,diff --git a/src/lib/domain/diagram.ts b/src/lib/domain/diagram.tsindex ed171e72..a0548e41 100644--- a/src/lib/domain/diagram.ts+++ b/src/lib/domain/diagram.ts@@ -1,30 +1,15 @@import { z } from 'zod';-import type { DatabaseMetadata } from '../data/import-metadata/metadata-types/database-metadata';import { DatabaseEdition } from './database-edition';import { DatabaseType } from './database-type';import type { DBDependency } from './db-dependency';-import {- createDependenciesFromMetadata,- dbDependencySchema,-} from './db-dependency';+import { dbDependencySchema } from './db-dependency';import type { DBRelationship } from './db-relationship';-import {- createRelationshipsFromMetadata,- dbRelationshipSchema,-} from './db-relationship';+import { dbRelationshipSchema } from './db-relationship';import type { DBTable } from './db-table';-import {- adjustTablePositions,- createTablesFromMetadata,- dbTableSchema,-} from './db-table';-import { generateDiagramId } from '@/lib/utils';+import { dbTableSchema } from './db-table';import { areaSchema, type Area } from './area';import type { DBCustomType } from './db-custom-type';-import {- dbCustomTypeSchema,- createCustomTypesFromMetadata,-} from './db-custom-type';+import { dbCustomTypeSchema } from './db-custom-type';export interface Diagram {id: string;@@ -53,77 +38,3 @@ export const diagramSchema: z.ZodType<Diagram> = z.object({createdAt: z.date(),updatedAt: z.date(),});--export const loadFromDatabaseMetadata = async ({- databaseType,- databaseMetadata,- diagramNumber,- databaseEdition,-}: {- databaseType: DatabaseType;- databaseMetadata: DatabaseMetadata;- diagramNumber?: number;- databaseEdition?: DatabaseEdition;-}): Promise<Diagram> => {- const {- fk_info: foreignKeys,- views: views,- custom_types: customTypes,- } = databaseMetadata;-- const tables = createTablesFromMetadata({- databaseMetadata,- databaseType,- });-- const relationships = createRelationshipsFromMetadata({- foreignKeys,- tables,- });-- const dependencies = await createDependenciesFromMetadata({- views,- tables,- databaseType,- });-- const dbCustomTypes = customTypes- ? createCustomTypesFromMetadata({- customTypes,- })- : [];-- const adjustedTables = adjustTablePositions({- tables,- relationships,- mode: 'perSchema',- });-- const sortedTables = adjustedTables.sort((a, b) => {- if (a.isView === b.isView) {- // Both are either tables or views, so sort alphabetically by name- return a.name.localeCompare(b.name);- }- // If one is a view and the other is not, put tables first- return a.isView ? 1 : -1;- });-- const diagram: Diagram = {- id: generateDiagramId(),- name: databaseMetadata.database_name- ? `${databaseMetadata.database_name}-db`- : diagramNumber- ? `Diagram ${diagramNumber}`- : 'New Diagram',- databaseType: databaseType ?? DatabaseType.GENERIC,- databaseEdition,- tables: sortedTables,- relationships,- dependencies,- customTypes: dbCustomTypes,- createdAt: new Date(),- updatedAt: new Date(),- };-- return diagram;-};