Database diagrams editor that allows you to visualize and design your DB with a single query.
TypeScript databasedbdb-migrationeditorerdmariadbmssqlmysqlpostgresqlreactreact-flowschema-migrationssqlsqlitetypescriptvisualizationxyflow
fix(ddl-import): fix datatypes when importing via ddl (#696)
Jonathan Fishnera1144bbf761a
diff --git a/src/lib/data/data-types/sqlite-data-types.ts b/src/lib/data/data-types/sqlite-data-types.tsindex afb14a46..94ff55df 100644--- a/src/lib/data/data-types/sqlite-data-types.ts+++ b/src/lib/data/data-types/sqlite-data-types.ts@@ -1,22 +1,34 @@import type { DataTypeData } from './data-types';export const sqliteDataTypes: readonly DataTypeData[] = [- // Level 1 - Most commonly used types+ // Level 1 - Most commonly used types - SQLite's 5 storage classes{ name: 'integer', id: 'integer', usageLevel: 1 },- { name: 'int', id: 'int', usageLevel: 1 },{ name: 'text', id: 'text', usageLevel: 1 },+ { name: 'real', id: 'real', usageLevel: 1 },+ { name: 'blob', id: 'blob', usageLevel: 1 },+ { name: 'null', id: 'null', usageLevel: 1 },++ // SQLite type aliases and common types+ { name: 'int', id: 'int', usageLevel: 1 },{ name: 'varchar', id: 'varchar', hasCharMaxLength: true, usageLevel: 1 },+ { name: 'timestamp', id: 'timestamp', usageLevel: 1 },{ name: 'date', id: 'date', usageLevel: 1 },{ name: 'datetime', id: 'datetime', usageLevel: 1 },{ name: 'boolean', id: 'boolean', usageLevel: 1 },// Level 2 - Second most common types+ { name: 'numeric', id: 'numeric', usageLevel: 2 },{ name: 'decimal', id: 'decimal', usageLevel: 2 },+ { name: 'float', id: 'float', usageLevel: 2 },+ { name: 'double', id: 'double', usageLevel: 2 },{ name: 'json', id: 'json', usageLevel: 2 },- // Less common types- { name: 'real', id: 'real' },- { name: 'numeric', id: 'numeric' },- { name: 'float', id: 'float' },- { name: 'blob', id: 'blob' },+ // Less common types (all map to SQLite storage classes)+ { name: 'char', id: 'char', hasCharMaxLength: true },+ { name: 'binary', id: 'binary' },+ { name: 'varbinary', id: 'varbinary' },+ { name: 'smallint', id: 'smallint' },+ { name: 'bigint', id: 'bigint' },+ { name: 'bool', id: 'bool' },+ { name: 'time', id: 'time' },] as const;diff --git a/src/lib/data/sql-import/common.ts b/src/lib/data/sql-import/common.tsindex 168a561b..38f7e882 100644--- a/src/lib/data/sql-import/common.ts+++ b/src/lib/data/sql-import/common.ts@@ -209,11 +209,48 @@ export function determineCardinality(}// Map SQL data type to generic data type in our system-export function mapSQLTypeToGenericType(sqlType: string): DataType {- const normalizedType = sqlType.toLowerCase().replace(/\(.*\)/, '');+export function mapSQLTypeToGenericType(+ sqlType: string,+ databaseType?: DatabaseType+): DataType {+ if (!sqlType) {+ return genericDataTypes.find((t) => t.id === 'text')!;+ }++ // Normalize the SQL type to lowercase for consistency+ const normalizedSqlType = sqlType.toLowerCase();++ // Add special case handling for SQLite INTEGER type+ if (+ databaseType === DatabaseType.SQLITE &&+ (normalizedSqlType === 'integer' || normalizedSqlType === 'int')+ ) {+ return genericDataTypes.find((t) => t.id === 'integer')!;+ }++ // Get dialect-specific type mappings+ const dialectAffinity =+ (databaseType && typeAffinity[databaseType]) ||+ typeAffinity[DatabaseType.GENERIC];++ // Handle specific database dialect mappings+ if (databaseType) {+ // Try to find a mapping for the normalized type+ const typeMapping = dialectAffinity[normalizedSqlType];+ if (typeMapping) {+ const foundType = genericDataTypes.find(+ (t) => t.id === typeMapping+ );+ if (foundType) return foundType;+ }+ }++ // Try direct mapping by normalizing the input type+ const normalizedType = normalizedSqlType.replace(/\(.*\)/, '');const matchedType = genericDataTypes.find((t) => t.id === normalizedType);if (matchedType) return matchedType;+ // Generic type mappings as a fallbackconst typeMap: Record<string, string> = {int: 'integer',integer: 'integer',@@ -236,6 +273,8 @@ export function mapSQLTypeToGenericType(sqlType: string): DataType {time: 'time',json: 'json',jsonb: 'json',+ real: 'real',+ blob: 'blob',};const mappedType = typeMap[normalizedType];@@ -244,80 +283,164 @@ export function mapSQLTypeToGenericType(sqlType: string): DataType {if (foundType) return foundType;}- return genericDataTypes.find((t) => t.id === 'varchar')!;+ // Default to text as last resort+ return genericDataTypes.find((t) => t.id === 'text')!;}// Type affinity definitions for different database dialectsexport const typeAffinity: Record<string, Record<string, string>> = {[DatabaseType.POSTGRESQL]: {- INT: 'INTEGER',- INTEGER: 'INTEGER',- MEDIUMINT: 'INTEGER',- BIT: 'BOOLEAN',+ // PostgreSQL data types (all lowercase for consistency)+ int: 'integer',+ integer: 'integer',+ int4: 'integer',+ smallint: 'smallint',+ int2: 'smallint',+ bigint: 'bigint',+ int8: 'bigint',+ decimal: 'decimal',+ numeric: 'numeric',+ real: 'real',+ 'double precision': 'double',+ float: 'float',+ float4: 'float',+ float8: 'double',+ boolean: 'boolean',+ bool: 'boolean',+ varchar: 'varchar',+ 'character varying': 'varchar',+ char: 'char',+ character: 'char',+ text: 'text',+ date: 'date',+ timestamp: 'timestamp',+ time: 'time',+ json: 'json',+ jsonb: 'jsonb',},[DatabaseType.MYSQL]: {- INT: 'INTEGER',- INTEGER: 'INTEGER',- BOOL: 'BOOLEAN',- BOOLEAN: 'TINYINT',+ // MySQL data types (all lowercase for consistency)+ int: 'integer',+ integer: 'integer',+ smallint: 'smallint',+ tinyint: 'tinyint',+ bigint: 'bigint',+ decimal: 'decimal',+ numeric: 'numeric',+ float: 'float',+ double: 'double',+ boolean: 'tinyint',+ bool: 'tinyint',+ varchar: 'varchar',+ char: 'char',+ text: 'text',+ date: 'date',+ datetime: 'datetime',+ timestamp: 'timestamp',+ time: 'time',+ json: 'json',},[DatabaseType.MARIADB]: {- INT: 'INTEGER',- INTEGER: 'INTEGER',- BOOL: 'BOOLEAN',- BOOLEAN: 'TINYINT',+ // MariaDB data types (all lowercase for consistency)+ int: 'integer',+ integer: 'integer',+ smallint: 'smallint',+ tinyint: 'tinyint',+ bigint: 'bigint',+ decimal: 'decimal',+ numeric: 'numeric',+ float: 'float',+ double: 'double',+ boolean: 'tinyint',+ bool: 'tinyint',+ varchar: 'varchar',+ char: 'char',+ text: 'text',+ date: 'date',+ datetime: 'datetime',+ timestamp: 'timestamp',+ time: 'time',+ json: 'json',},[DatabaseType.SQL_SERVER]: {- INT: 'INTEGER',- INTEGER: 'INT',- BOOL: 'BIT',- BOOLEAN: 'BIT',+ // SQL Server data types (all lowercase for consistency)+ int: 'integer',+ integer: 'integer',+ smallint: 'smallint',+ bigint: 'bigint',+ decimal: 'decimal',+ numeric: 'numeric',+ float: 'float',+ real: 'real',+ bit: 'bit',+ boolean: 'bit',+ bool: 'bit',+ varchar: 'varchar',+ nvarchar: 'nvarchar',+ char: 'char',+ nchar: 'nchar',+ text: 'text',+ ntext: 'ntext',+ date: 'date',+ datetime: 'datetime',+ datetime2: 'datetime2',+ time: 'time',+ uniqueidentifier: 'uniqueidentifier',},[DatabaseType.SQLITE]: {- INT: 'INTEGER',- BOOL: 'INTEGER',- BOOLEAN: 'INTEGER',+ // SQLite storage classes (all lowercase for consistency)+ integer: 'integer',+ int: 'integer',+ bigint: 'bigint',+ smallint: 'smallint',+ tinyint: 'tinyint',+ real: 'real',+ float: 'real',+ double: 'real',+ numeric: 'real',+ decimal: 'real',+ text: 'text',+ varchar: 'text',+ char: 'text',+ blob: 'blob',+ binary: 'blob',+ varbinary: 'blob',+ timestamp: 'timestamp',+ datetime: 'timestamp',+ date: 'date',+ boolean: 'integer',+ bool: 'integer',+ time: 'text',+ json: 'text',},[DatabaseType.GENERIC]: {- INTEGER: 'integer',- INT: 'integer',- MEDIUMINT: 'integer',- BIT: 'boolean',- VARCHAR: 'varchar',- 'CHARACTER VARYING': 'varchar',- CHAR: 'char',- CHARACTER: 'char',- TEXT: 'text',- BOOLEAN: 'boolean',- BOOL: 'boolean',- TIMESTAMP: 'timestamp',- DATETIME: 'timestamp',- DATE: 'date',- TIME: 'time',- JSON: 'json',- JSONB: 'json',- DECIMAL: 'decimal',- NUMERIC: 'numeric',- FLOAT: 'float',- DOUBLE: 'double',- BIGINT: 'bigint',- SMALLINT: 'smallint',+ // Generic fallback types (all lowercase for consistency)+ integer: 'integer',+ int: 'integer',+ smallint: 'smallint',+ bigint: 'bigint',+ decimal: 'decimal',+ numeric: 'numeric',+ float: 'float',+ double: 'double',+ real: 'real',+ boolean: 'boolean',+ bool: 'boolean',+ varchar: 'varchar',+ 'character varying': 'varchar',+ char: 'char',+ character: 'char',+ text: 'text',+ date: 'date',+ timestamp: 'timestamp',+ datetime: 'timestamp',+ time: 'time',+ json: 'json',+ jsonb: 'json',+ blob: 'blob',},};-// For safe type conversions-export function getTypeAffinity(- databaseType: DatabaseType,- sqlType: string-): string {- if (!sqlType) return 'varchar';-- const normalizedType = sqlType.toUpperCase();- const dialectAffinity =- typeAffinity[databaseType] || typeAffinity[DatabaseType.GENERIC];- return dialectAffinity[normalizedType] || sqlType.toLowerCase();-}-// Convert SQLParserResult to ChartDB Diagram structureexport function convertToChartDBDiagram(parserResult: SQLParserResult,@@ -337,10 +460,89 @@ export function convertToChartDBDiagram(// Create fields from columnsconst fields: DBField[] = table.columns.map((column) => {+ // Use special case handling for specific database types to ensure correct mapping+ let mappedType: DataType;++ // SQLite-specific handling for numeric types+ if (sourceDatabaseType === DatabaseType.SQLITE) {+ const normalizedType = column.type.toLowerCase();++ if (normalizedType === 'integer' || normalizedType === 'int') {+ // Ensure integer types are preserved+ mappedType = { id: 'integer', name: 'integer' };+ } else if (+ normalizedType === 'real' ||+ normalizedType === 'float' ||+ normalizedType === 'double' ||+ normalizedType === 'numeric' ||+ normalizedType === 'decimal'+ ) {+ // Ensure real types are preserved+ mappedType = { id: 'real', name: 'real' };+ } else if (normalizedType === 'blob') {+ // Ensure blob types are preserved+ mappedType = { id: 'blob', name: 'blob' };+ } else {+ // Use the standard mapping for other types+ mappedType = mapSQLTypeToGenericType(+ column.type,+ sourceDatabaseType+ );+ }+ }+ // Handle MySQL/MariaDB integer types specifically+ else if (+ sourceDatabaseType === DatabaseType.MYSQL ||+ sourceDatabaseType === DatabaseType.MARIADB+ ) {+ const normalizedType = column.type+ .toLowerCase()+ .replace(/\(\d+\)/, '')+ .trim();++ // Handle various integer types+ if (normalizedType === 'tinyint') {+ mappedType = { id: 'tinyint', name: 'tinyint' };+ } else if (+ normalizedType === 'int' ||+ normalizedType === 'integer'+ ) {+ mappedType = { id: 'int', name: 'int' };+ } else if (normalizedType === 'smallint') {+ mappedType = { id: 'smallint', name: 'smallint' };+ } else if (normalizedType === 'mediumint') {+ mappedType = { id: 'mediumint', name: 'mediumint' };+ } else if (normalizedType === 'bigint') {+ mappedType = { id: 'bigint', name: 'bigint' };+ } else {+ // Use the standard mapping for other types+ mappedType = mapSQLTypeToGenericType(+ column.type,+ sourceDatabaseType+ );+ }+ }+ // Handle PostgreSQL integer type specifically+ else if (+ sourceDatabaseType === DatabaseType.POSTGRESQL &&+ (column.type.toLowerCase() === 'integer' ||+ column.type.toLowerCase() === 'int' ||+ column.type.toLowerCase() === 'int4')+ ) {+ // Ensure integer types are preserved+ mappedType = { id: 'integer', name: 'integer' };+ } else {+ // Use the standard mapping for other types+ mappedType = mapSQLTypeToGenericType(+ column.type,+ sourceDatabaseType+ );+ }+const field: DBField = {id: generateId(),name: column.name,- type: mapSQLTypeToGenericType(column.type),+ type: mappedType,nullable: column.nullable,primaryKey: column.primaryKey,unique: column.unique,diff --git a/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts b/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.tsindex a9be199c..bd40500c 100644--- a/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts+++ b/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts@@ -32,12 +32,43 @@ export async function fromSQLite(sqlContent: string): Promise<SQLParserResult> {const tableMap: Record<string, string> = {}; // Maps table name to its IDtry {+ // SPECIAL HANDLING: Direct line-by-line parser for SQLite DDL+ // This ensures we preserve the exact data types from the original DDL+ const directlyParsedTables = parseCreateTableStatements(sqlContent);++ // Check if we successfully parsed tables directly+ if (directlyParsedTables.length > 0) {+ // Map the direct parsing results to the expected SQLParserResult format+ directlyParsedTables.forEach((table) => {+ const tableId = getTableIdWithSchemaSupport(table.name);+ tableMap[table.name] = tableId;++ // Add the table with its columns+ tables.push({+ id: tableId,+ name: table.name,+ columns: table.columns,+ indexes: [],+ order: tables.length,+ });+ });++ // Process foreign keys using the regex approach+ findForeignKeysUsingRegex(sqlContent, tableMap, relationships);++ // Return the result+ return { tables, relationships };+ }++ // Preprocess SQL to handle SQLite quoted identifiers+ const preprocessedSQL = preprocessSQLiteDDL(sqlContent);+// Parse the SQL DDL statementsconst { Parser } = await import('node-sql-parser');const parser = new Parser();const ast = parser.astify(- sqlContent,+ preprocessedSQL,parserOpts) as unknown as SQLASTNode[];@@ -87,6 +118,141 @@ export async function fromSQLite(sqlContent: string): Promise<SQLParserResult> {}}+/**+ * Parse SQLite CREATE TABLE statements directly to preserve exact type information+ */+function parseCreateTableStatements(sqlContent: string): {+ name: string;+ columns: SQLColumn[];+}[] {+ const tables: {+ name: string;+ columns: SQLColumn[];+ }[] = [];++ // Split SQL content into lines+ const lines = sqlContent.split('\n');++ let currentTable: { name: string; columns: SQLColumn[] } | null = null;+ let inCreateTable = false;++ // Process each line+ for (let i = 0; i < lines.length; i++) {+ const line = lines[i].trim();++ // Skip empty lines and comments+ if (!line || line.startsWith('--')) {+ continue;+ }++ // Check for CREATE TABLE statement+ if (line.toUpperCase().startsWith('CREATE TABLE')) {+ // Extract table name+ const tableNameMatch =+ /CREATE\s+TABLE\s+(?:if\s+not\s+exists\s+)?["'`]?(\w+)["'`]?/i.exec(+ line+ );+ if (tableNameMatch && tableNameMatch[1]) {+ inCreateTable = true;+ currentTable = {+ name: tableNameMatch[1],+ columns: [],+ };+ }+ }+ // Check for end of CREATE TABLE statement+ else if (inCreateTable && line.includes(');')) {+ if (currentTable) {+ tables.push(currentTable);+ }+ inCreateTable = false;+ currentTable = null;+ }+ // Process column definitions inside CREATE TABLE+ else if (inCreateTable && currentTable && line.includes('"')) {+ // Column line pattern optimized for user's DDL format+ const columnPattern = /\s*["'`](\w+)["'`]\s+([A-Za-z0-9_]+)(.+)?/i;+ const match = columnPattern.exec(line);++ if (match) {+ const columnName = match[1];+ const rawType = match[2].toUpperCase();+ const restOfLine = match[3] || '';++ // Determine column properties+ const isPrimaryKey = restOfLine+ .toUpperCase()+ .includes('PRIMARY KEY');+ const isNotNull = restOfLine.toUpperCase().includes('NOT NULL');+ const isUnique = restOfLine.toUpperCase().includes('UNIQUE');++ // Extract default value+ let defaultValue = '';+ const defaultMatch = /DEFAULT\s+([^,\s)]+)/i.exec(restOfLine);+ if (defaultMatch) {+ defaultValue = defaultMatch[1];+ }++ // Map to appropriate SQLite storage class+ let columnType = rawType;+ if (rawType === 'INTEGER' || rawType === 'INT') {+ columnType = 'INTEGER';+ } else if (+ ['REAL', 'FLOAT', 'DOUBLE', 'NUMERIC', 'DECIMAL'].includes(+ rawType+ )+ ) {+ columnType = 'REAL';+ } else if (rawType === 'BLOB' || rawType === 'BINARY') {+ columnType = 'BLOB';+ } else if (+ ['TIMESTAMP', 'DATETIME', 'DATE'].includes(rawType)+ ) {+ columnType = 'TIMESTAMP';+ } else {+ columnType = 'TEXT';+ }++ // Add column to the table+ currentTable.columns.push({+ name: columnName,+ type: columnType,+ nullable: !isNotNull,+ primaryKey: isPrimaryKey,+ unique: isUnique || isPrimaryKey,+ default: defaultValue,+ increment: isPrimaryKey && columnType === 'INTEGER',+ });+ }+ }+ }++ return tables;+}++/**+ * Preprocess SQLite DDL to handle specific syntax issues that might cause parsing problems+ */+function preprocessSQLiteDDL(sqlContent: string): string {+ // Replace quoted identifiers with their unquoted equivalents+ let processedSQL = sqlContent;++ // Handle column type declarations with quotes around them+ // For example: "id" "TEXT" PRIMARY KEY -> "id" TEXT PRIMARY KEY+ processedSQL = processedSQL.replace(+ /(['"`])(\w+)(['"`])\s+(['"`])(\w+)(['"`])/g,+ (_match, q1, col, q2, ...rest) => {+ // Extract the type from rest parameters+ // match, q1, col, q2, q3, type, q4+ const type = rest[1];+ // Preserve the quotes around column name, but remove quotes around type+ return `${q1}${col}${q2} ${type}`;+ }+ );++ return processedSQL;+}+/*** Process a CREATE TABLE statement to extract table and column information*/@@ -139,6 +305,7 @@ function processCreateTableStatement(createTableStmt.create_definitions &&Array.isArray(createTableStmt.create_definitions)) {+ // First pass - collect column information from the SQLcreateTableStmt.create_definitions.forEach((def) => {if ('column' in def) {// Process column definition@@ -158,7 +325,34 @@ function processCreateTableStatement(};if (columnDef.dataType) {- typeName = columnDef.dataType.dataType || 'text';+ // Get the raw data type string and clean it up+ typeName =+ columnDef.dataType.dataType?.toUpperCase() || 'TEXT';++ // Set the exact type according to SQLite's type system+ // SQLite has 5 storage classes: NULL, INTEGER, REAL, TEXT, and BLOB+ if (typeName === 'INTEGER' || typeName === 'INT') {+ typeName = 'INTEGER';+ } else if (+ typeName === 'REAL' ||+ typeName === 'FLOAT' ||+ typeName === 'DOUBLE' ||+ typeName === 'NUMERIC' ||+ typeName === 'DECIMAL'+ ) {+ typeName = 'REAL';+ } else if (typeName === 'BLOB') {+ typeName = 'BLOB';+ } else if (+ typeName === 'TIMESTAMP' ||+ typeName === 'DATETIME' ||+ typeName === 'DATE'+ ) {+ typeName = 'TIMESTAMP'; // Preserve TIMESTAMP as a special type+ } else {+ typeName = 'TEXT'; // Default SQLite type+ }+const args = getTypeArgs(columnDef.dataType);typeArgs.length = args.size > 0 ? args.size : undefined;typeArgs.precision = args.precision;