Skip to content

Database diagrams editor that allows you to visualize and design your DB with a single query.

fix(ddl-import): fix datatypes when importing via ddl (#696)

Jonathan Fishnera1144bbf761a
diff --git a/src/lib/data/data-types/sqlite-data-types.ts b/src/lib/data/data-types/sqlite-data-types.ts
index afb14a46..94ff55df 100644
--- a/src/lib/data/data-types/sqlite-data-types.ts
+++ b/src/lib/data/data-types/sqlite-data-types.ts
@@ -1,22 +1,34 @@
import type { DataTypeData } from './data-types';
export const sqliteDataTypes: readonly DataTypeData[] = [
- // Level 1 - Most commonly used types
+ // Level 1 - Most commonly used types - SQLite's 5 storage classes
{ name: 'integer', id: 'integer', usageLevel: 1 },
- { name: 'int', id: 'int', usageLevel: 1 },
{ name: 'text', id: 'text', usageLevel: 1 },
+ { name: 'real', id: 'real', usageLevel: 1 },
+ { name: 'blob', id: 'blob', usageLevel: 1 },
+ { name: 'null', id: 'null', usageLevel: 1 },
+
+ // SQLite type aliases and common types
+ { name: 'int', id: 'int', usageLevel: 1 },
{ name: 'varchar', id: 'varchar', hasCharMaxLength: true, usageLevel: 1 },
+ { name: 'timestamp', id: 'timestamp', usageLevel: 1 },
{ name: 'date', id: 'date', usageLevel: 1 },
{ name: 'datetime', id: 'datetime', usageLevel: 1 },
{ name: 'boolean', id: 'boolean', usageLevel: 1 },
// Level 2 - Second most common types
+ { name: 'numeric', id: 'numeric', usageLevel: 2 },
{ name: 'decimal', id: 'decimal', usageLevel: 2 },
+ { name: 'float', id: 'float', usageLevel: 2 },
+ { name: 'double', id: 'double', usageLevel: 2 },
{ name: 'json', id: 'json', usageLevel: 2 },
- // Less common types
- { name: 'real', id: 'real' },
- { name: 'numeric', id: 'numeric' },
- { name: 'float', id: 'float' },
- { name: 'blob', id: 'blob' },
+ // Less common types (all map to SQLite storage classes)
+ { name: 'char', id: 'char', hasCharMaxLength: true },
+ { name: 'binary', id: 'binary' },
+ { name: 'varbinary', id: 'varbinary' },
+ { name: 'smallint', id: 'smallint' },
+ { name: 'bigint', id: 'bigint' },
+ { name: 'bool', id: 'bool' },
+ { name: 'time', id: 'time' },
] as const;
diff --git a/src/lib/data/sql-import/common.ts b/src/lib/data/sql-import/common.ts
index 168a561b..38f7e882 100644
--- a/src/lib/data/sql-import/common.ts
+++ b/src/lib/data/sql-import/common.ts
@@ -209,11 +209,48 @@ export function determineCardinality(
}
// Map SQL data type to generic data type in our system
-export function mapSQLTypeToGenericType(sqlType: string): DataType {
- const normalizedType = sqlType.toLowerCase().replace(/\(.*\)/, '');
+export function mapSQLTypeToGenericType(
+ sqlType: string,
+ databaseType?: DatabaseType
+): DataType {
+ if (!sqlType) {
+ return genericDataTypes.find((t) => t.id === 'text')!;
+ }
+
+ // Normalize the SQL type to lowercase for consistency
+ const normalizedSqlType = sqlType.toLowerCase();
+
+ // Add special case handling for SQLite INTEGER type
+ if (
+ databaseType === DatabaseType.SQLITE &&
+ (normalizedSqlType === 'integer' || normalizedSqlType === 'int')
+ ) {
+ return genericDataTypes.find((t) => t.id === 'integer')!;
+ }
+
+ // Get dialect-specific type mappings
+ const dialectAffinity =
+ (databaseType && typeAffinity[databaseType]) ||
+ typeAffinity[DatabaseType.GENERIC];
+
+ // Handle specific database dialect mappings
+ if (databaseType) {
+ // Try to find a mapping for the normalized type
+ const typeMapping = dialectAffinity[normalizedSqlType];
+ if (typeMapping) {
+ const foundType = genericDataTypes.find(
+ (t) => t.id === typeMapping
+ );
+ if (foundType) return foundType;
+ }
+ }
+
+ // Try direct mapping by normalizing the input type
+ const normalizedType = normalizedSqlType.replace(/\(.*\)/, '');
const matchedType = genericDataTypes.find((t) => t.id === normalizedType);
if (matchedType) return matchedType;
+ // Generic type mappings as a fallback
const typeMap: Record<string, string> = {
int: 'integer',
integer: 'integer',
@@ -236,6 +273,8 @@ export function mapSQLTypeToGenericType(sqlType: string): DataType {
time: 'time',
json: 'json',
jsonb: 'json',
+ real: 'real',
+ blob: 'blob',
};
const mappedType = typeMap[normalizedType];
@@ -244,80 +283,164 @@ export function mapSQLTypeToGenericType(sqlType: string): DataType {
if (foundType) return foundType;
}
- return genericDataTypes.find((t) => t.id === 'varchar')!;
+ // Default to text as last resort
+ return genericDataTypes.find((t) => t.id === 'text')!;
}
// Type affinity definitions for different database dialects
export const typeAffinity: Record<string, Record<string, string>> = {
[DatabaseType.POSTGRESQL]: {
- INT: 'INTEGER',
- INTEGER: 'INTEGER',
- MEDIUMINT: 'INTEGER',
- BIT: 'BOOLEAN',
+ // PostgreSQL data types (all lowercase for consistency)
+ int: 'integer',
+ integer: 'integer',
+ int4: 'integer',
+ smallint: 'smallint',
+ int2: 'smallint',
+ bigint: 'bigint',
+ int8: 'bigint',
+ decimal: 'decimal',
+ numeric: 'numeric',
+ real: 'real',
+ 'double precision': 'double',
+ float: 'float',
+ float4: 'float',
+ float8: 'double',
+ boolean: 'boolean',
+ bool: 'boolean',
+ varchar: 'varchar',
+ 'character varying': 'varchar',
+ char: 'char',
+ character: 'char',
+ text: 'text',
+ date: 'date',
+ timestamp: 'timestamp',
+ time: 'time',
+ json: 'json',
+ jsonb: 'jsonb',
},
[DatabaseType.MYSQL]: {
- INT: 'INTEGER',
- INTEGER: 'INTEGER',
- BOOL: 'BOOLEAN',
- BOOLEAN: 'TINYINT',
+ // MySQL data types (all lowercase for consistency)
+ int: 'integer',
+ integer: 'integer',
+ smallint: 'smallint',
+ tinyint: 'tinyint',
+ bigint: 'bigint',
+ decimal: 'decimal',
+ numeric: 'numeric',
+ float: 'float',
+ double: 'double',
+ boolean: 'tinyint',
+ bool: 'tinyint',
+ varchar: 'varchar',
+ char: 'char',
+ text: 'text',
+ date: 'date',
+ datetime: 'datetime',
+ timestamp: 'timestamp',
+ time: 'time',
+ json: 'json',
},
[DatabaseType.MARIADB]: {
- INT: 'INTEGER',
- INTEGER: 'INTEGER',
- BOOL: 'BOOLEAN',
- BOOLEAN: 'TINYINT',
+ // MariaDB data types (all lowercase for consistency)
+ int: 'integer',
+ integer: 'integer',
+ smallint: 'smallint',
+ tinyint: 'tinyint',
+ bigint: 'bigint',
+ decimal: 'decimal',
+ numeric: 'numeric',
+ float: 'float',
+ double: 'double',
+ boolean: 'tinyint',
+ bool: 'tinyint',
+ varchar: 'varchar',
+ char: 'char',
+ text: 'text',
+ date: 'date',
+ datetime: 'datetime',
+ timestamp: 'timestamp',
+ time: 'time',
+ json: 'json',
},
[DatabaseType.SQL_SERVER]: {
- INT: 'INTEGER',
- INTEGER: 'INT',
- BOOL: 'BIT',
- BOOLEAN: 'BIT',
+ // SQL Server data types (all lowercase for consistency)
+ int: 'integer',
+ integer: 'integer',
+ smallint: 'smallint',
+ bigint: 'bigint',
+ decimal: 'decimal',
+ numeric: 'numeric',
+ float: 'float',
+ real: 'real',
+ bit: 'bit',
+ boolean: 'bit',
+ bool: 'bit',
+ varchar: 'varchar',
+ nvarchar: 'nvarchar',
+ char: 'char',
+ nchar: 'nchar',
+ text: 'text',
+ ntext: 'ntext',
+ date: 'date',
+ datetime: 'datetime',
+ datetime2: 'datetime2',
+ time: 'time',
+ uniqueidentifier: 'uniqueidentifier',
},
[DatabaseType.SQLITE]: {
- INT: 'INTEGER',
- BOOL: 'INTEGER',
- BOOLEAN: 'INTEGER',
+ // SQLite storage classes (all lowercase for consistency)
+ integer: 'integer',
+ int: 'integer',
+ bigint: 'bigint',
+ smallint: 'smallint',
+ tinyint: 'tinyint',
+ real: 'real',
+ float: 'real',
+ double: 'real',
+ numeric: 'real',
+ decimal: 'real',
+ text: 'text',
+ varchar: 'text',
+ char: 'text',
+ blob: 'blob',
+ binary: 'blob',
+ varbinary: 'blob',
+ timestamp: 'timestamp',
+ datetime: 'timestamp',
+ date: 'date',
+ boolean: 'integer',
+ bool: 'integer',
+ time: 'text',
+ json: 'text',
},
[DatabaseType.GENERIC]: {
- INTEGER: 'integer',
- INT: 'integer',
- MEDIUMINT: 'integer',
- BIT: 'boolean',
- VARCHAR: 'varchar',
- 'CHARACTER VARYING': 'varchar',
- CHAR: 'char',
- CHARACTER: 'char',
- TEXT: 'text',
- BOOLEAN: 'boolean',
- BOOL: 'boolean',
- TIMESTAMP: 'timestamp',
- DATETIME: 'timestamp',
- DATE: 'date',
- TIME: 'time',
- JSON: 'json',
- JSONB: 'json',
- DECIMAL: 'decimal',
- NUMERIC: 'numeric',
- FLOAT: 'float',
- DOUBLE: 'double',
- BIGINT: 'bigint',
- SMALLINT: 'smallint',
+ // Generic fallback types (all lowercase for consistency)
+ integer: 'integer',
+ int: 'integer',
+ smallint: 'smallint',
+ bigint: 'bigint',
+ decimal: 'decimal',
+ numeric: 'numeric',
+ float: 'float',
+ double: 'double',
+ real: 'real',
+ boolean: 'boolean',
+ bool: 'boolean',
+ varchar: 'varchar',
+ 'character varying': 'varchar',
+ char: 'char',
+ character: 'char',
+ text: 'text',
+ date: 'date',
+ timestamp: 'timestamp',
+ datetime: 'timestamp',
+ time: 'time',
+ json: 'json',
+ jsonb: 'json',
+ blob: 'blob',
},
};
-// For safe type conversions
-export function getTypeAffinity(
- databaseType: DatabaseType,
- sqlType: string
-): string {
- if (!sqlType) return 'varchar';
-
- const normalizedType = sqlType.toUpperCase();
- const dialectAffinity =
- typeAffinity[databaseType] || typeAffinity[DatabaseType.GENERIC];
- return dialectAffinity[normalizedType] || sqlType.toLowerCase();
-}
-
// Convert SQLParserResult to ChartDB Diagram structure
export function convertToChartDBDiagram(
parserResult: SQLParserResult,
@@ -337,10 +460,89 @@ export function convertToChartDBDiagram(
// Create fields from columns
const fields: DBField[] = table.columns.map((column) => {
+ // Use special case handling for specific database types to ensure correct mapping
+ let mappedType: DataType;
+
+ // SQLite-specific handling for numeric types
+ if (sourceDatabaseType === DatabaseType.SQLITE) {
+ const normalizedType = column.type.toLowerCase();
+
+ if (normalizedType === 'integer' || normalizedType === 'int') {
+ // Ensure integer types are preserved
+ mappedType = { id: 'integer', name: 'integer' };
+ } else if (
+ normalizedType === 'real' ||
+ normalizedType === 'float' ||
+ normalizedType === 'double' ||
+ normalizedType === 'numeric' ||
+ normalizedType === 'decimal'
+ ) {
+ // Ensure real types are preserved
+ mappedType = { id: 'real', name: 'real' };
+ } else if (normalizedType === 'blob') {
+ // Ensure blob types are preserved
+ mappedType = { id: 'blob', name: 'blob' };
+ } else {
+ // Use the standard mapping for other types
+ mappedType = mapSQLTypeToGenericType(
+ column.type,
+ sourceDatabaseType
+ );
+ }
+ }
+ // Handle MySQL/MariaDB integer types specifically
+ else if (
+ sourceDatabaseType === DatabaseType.MYSQL ||
+ sourceDatabaseType === DatabaseType.MARIADB
+ ) {
+ const normalizedType = column.type
+ .toLowerCase()
+ .replace(/\(\d+\)/, '')
+ .trim();
+
+ // Handle various integer types
+ if (normalizedType === 'tinyint') {
+ mappedType = { id: 'tinyint', name: 'tinyint' };
+ } else if (
+ normalizedType === 'int' ||
+ normalizedType === 'integer'
+ ) {
+ mappedType = { id: 'int', name: 'int' };
+ } else if (normalizedType === 'smallint') {
+ mappedType = { id: 'smallint', name: 'smallint' };
+ } else if (normalizedType === 'mediumint') {
+ mappedType = { id: 'mediumint', name: 'mediumint' };
+ } else if (normalizedType === 'bigint') {
+ mappedType = { id: 'bigint', name: 'bigint' };
+ } else {
+ // Use the standard mapping for other types
+ mappedType = mapSQLTypeToGenericType(
+ column.type,
+ sourceDatabaseType
+ );
+ }
+ }
+ // Handle PostgreSQL integer type specifically
+ else if (
+ sourceDatabaseType === DatabaseType.POSTGRESQL &&
+ (column.type.toLowerCase() === 'integer' ||
+ column.type.toLowerCase() === 'int' ||
+ column.type.toLowerCase() === 'int4')
+ ) {
+ // Ensure integer types are preserved
+ mappedType = { id: 'integer', name: 'integer' };
+ } else {
+ // Use the standard mapping for other types
+ mappedType = mapSQLTypeToGenericType(
+ column.type,
+ sourceDatabaseType
+ );
+ }
+
const field: DBField = {
id: generateId(),
name: column.name,
- type: mapSQLTypeToGenericType(column.type),
+ type: mappedType,
nullable: column.nullable,
primaryKey: column.primaryKey,
unique: column.unique,
diff --git a/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts b/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts
index a9be199c..bd40500c 100644
--- a/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts
+++ b/src/lib/data/sql-import/dialect-importers/sqlite/sqlite.ts
@@ -32,12 +32,43 @@ export async function fromSQLite(sqlContent: string): Promise<SQLParserResult> {
const tableMap: Record<string, string> = {}; // Maps table name to its ID
try {
+ // SPECIAL HANDLING: Direct line-by-line parser for SQLite DDL
+ // This ensures we preserve the exact data types from the original DDL
+ const directlyParsedTables = parseCreateTableStatements(sqlContent);
+
+ // Check if we successfully parsed tables directly
+ if (directlyParsedTables.length > 0) {
+ // Map the direct parsing results to the expected SQLParserResult format
+ directlyParsedTables.forEach((table) => {
+ const tableId = getTableIdWithSchemaSupport(table.name);
+ tableMap[table.name] = tableId;
+
+ // Add the table with its columns
+ tables.push({
+ id: tableId,
+ name: table.name,
+ columns: table.columns,
+ indexes: [],
+ order: tables.length,
+ });
+ });
+
+ // Process foreign keys using the regex approach
+ findForeignKeysUsingRegex(sqlContent, tableMap, relationships);
+
+ // Return the result
+ return { tables, relationships };
+ }
+
+ // Preprocess SQL to handle SQLite quoted identifiers
+ const preprocessedSQL = preprocessSQLiteDDL(sqlContent);
+
// Parse the SQL DDL statements
const { Parser } = await import('node-sql-parser');
const parser = new Parser();
const ast = parser.astify(
- sqlContent,
+ preprocessedSQL,
parserOpts
) as unknown as SQLASTNode[];
@@ -87,6 +118,141 @@ export async function fromSQLite(sqlContent: string): Promise<SQLParserResult> {
}
}
+/**
+ * Parse SQLite CREATE TABLE statements directly to preserve exact type information
+ */
+function parseCreateTableStatements(sqlContent: string): {
+ name: string;
+ columns: SQLColumn[];
+}[] {
+ const tables: {
+ name: string;
+ columns: SQLColumn[];
+ }[] = [];
+
+ // Split SQL content into lines
+ const lines = sqlContent.split('\n');
+
+ let currentTable: { name: string; columns: SQLColumn[] } | null = null;
+ let inCreateTable = false;
+
+ // Process each line
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i].trim();
+
+ // Skip empty lines and comments
+ if (!line || line.startsWith('--')) {
+ continue;
+ }
+
+ // Check for CREATE TABLE statement
+ if (line.toUpperCase().startsWith('CREATE TABLE')) {
+ // Extract table name
+ const tableNameMatch =
+ /CREATE\s+TABLE\s+(?:if\s+not\s+exists\s+)?["'`]?(\w+)["'`]?/i.exec(
+ line
+ );
+ if (tableNameMatch && tableNameMatch[1]) {
+ inCreateTable = true;
+ currentTable = {
+ name: tableNameMatch[1],
+ columns: [],
+ };
+ }
+ }
+ // Check for end of CREATE TABLE statement
+ else if (inCreateTable && line.includes(');')) {
+ if (currentTable) {
+ tables.push(currentTable);
+ }
+ inCreateTable = false;
+ currentTable = null;
+ }
+ // Process column definitions inside CREATE TABLE
+ else if (inCreateTable && currentTable && line.includes('"')) {
+ // Column line pattern optimized for user's DDL format
+ const columnPattern = /\s*["'`](\w+)["'`]\s+([A-Za-z0-9_]+)(.+)?/i;
+ const match = columnPattern.exec(line);
+
+ if (match) {
+ const columnName = match[1];
+ const rawType = match[2].toUpperCase();
+ const restOfLine = match[3] || '';
+
+ // Determine column properties
+ const isPrimaryKey = restOfLine
+ .toUpperCase()
+ .includes('PRIMARY KEY');
+ const isNotNull = restOfLine.toUpperCase().includes('NOT NULL');
+ const isUnique = restOfLine.toUpperCase().includes('UNIQUE');
+
+ // Extract default value
+ let defaultValue = '';
+ const defaultMatch = /DEFAULT\s+([^,\s)]+)/i.exec(restOfLine);
+ if (defaultMatch) {
+ defaultValue = defaultMatch[1];
+ }
+
+ // Map to appropriate SQLite storage class
+ let columnType = rawType;
+ if (rawType === 'INTEGER' || rawType === 'INT') {
+ columnType = 'INTEGER';
+ } else if (
+ ['REAL', 'FLOAT', 'DOUBLE', 'NUMERIC', 'DECIMAL'].includes(
+ rawType
+ )
+ ) {
+ columnType = 'REAL';
+ } else if (rawType === 'BLOB' || rawType === 'BINARY') {
+ columnType = 'BLOB';
+ } else if (
+ ['TIMESTAMP', 'DATETIME', 'DATE'].includes(rawType)
+ ) {
+ columnType = 'TIMESTAMP';
+ } else {
+ columnType = 'TEXT';
+ }
+
+ // Add column to the table
+ currentTable.columns.push({
+ name: columnName,
+ type: columnType,
+ nullable: !isNotNull,
+ primaryKey: isPrimaryKey,
+ unique: isUnique || isPrimaryKey,
+ default: defaultValue,
+ increment: isPrimaryKey && columnType === 'INTEGER',
+ });
+ }
+ }
+ }
+
+ return tables;
+}
+
+/**
+ * Preprocess SQLite DDL to handle specific syntax issues that might cause parsing problems
+ */
+function preprocessSQLiteDDL(sqlContent: string): string {
+ // Replace quoted identifiers with their unquoted equivalents
+ let processedSQL = sqlContent;
+
+ // Handle column type declarations with quotes around them
+ // For example: "id" "TEXT" PRIMARY KEY -> "id" TEXT PRIMARY KEY
+ processedSQL = processedSQL.replace(
+ /(['"`])(\w+)(['"`])\s+(['"`])(\w+)(['"`])/g,
+ (_match, q1, col, q2, ...rest) => {
+ // Extract the type from rest parameters
+ // match, q1, col, q2, q3, type, q4
+ const type = rest[1];
+ // Preserve the quotes around column name, but remove quotes around type
+ return `${q1}${col}${q2} ${type}`;
+ }
+ );
+
+ return processedSQL;
+}
+
/**
* Process a CREATE TABLE statement to extract table and column information
*/
@@ -139,6 +305,7 @@ function processCreateTableStatement(
createTableStmt.create_definitions &&
Array.isArray(createTableStmt.create_definitions)
) {
+ // First pass - collect column information from the SQL
createTableStmt.create_definitions.forEach((def) => {
if ('column' in def) {
// Process column definition
@@ -158,7 +325,34 @@ function processCreateTableStatement(
};
if (columnDef.dataType) {
- typeName = columnDef.dataType.dataType || 'text';
+ // Get the raw data type string and clean it up
+ typeName =
+ columnDef.dataType.dataType?.toUpperCase() || 'TEXT';
+
+ // Set the exact type according to SQLite's type system
+ // SQLite has 5 storage classes: NULL, INTEGER, REAL, TEXT, and BLOB
+ if (typeName === 'INTEGER' || typeName === 'INT') {
+ typeName = 'INTEGER';
+ } else if (
+ typeName === 'REAL' ||
+ typeName === 'FLOAT' ||
+ typeName === 'DOUBLE' ||
+ typeName === 'NUMERIC' ||
+ typeName === 'DECIMAL'
+ ) {
+ typeName = 'REAL';
+ } else if (typeName === 'BLOB') {
+ typeName = 'BLOB';
+ } else if (
+ typeName === 'TIMESTAMP' ||
+ typeName === 'DATETIME' ||
+ typeName === 'DATE'
+ ) {
+ typeName = 'TIMESTAMP'; // Preserve TIMESTAMP as a special type
+ } else {
+ typeName = 'TEXT'; // Default SQLite type
+ }
+
const args = getTypeArgs(columnDef.dataType);
typeArgs.length = args.size > 0 ? args.size : undefined;
typeArgs.precision = args.precision;