Skip to content

Database diagrams editor that allows you to visualize and design your DB with a single query.

fix(import): strict parse of database metadata (#635)

Guy Ben-Aharon0940d72d5d37
diff --git a/src/lib/data/import-metadata/metadata-types/column-info.ts b/src/lib/data/import-metadata/metadata-types/column-info.ts
index 03d3ac14..4e22a20f 100644
--- a/src/lib/data/import-metadata/metadata-types/column-info.ts
+++ b/src/lib/data/import-metadata/metadata-types/column-info.ts
@@ -1,10 +1,12 @@
+import { z } from 'zod';
+
export interface ColumnInfo {
schema: string;
table: string;
name: string;
type: string;
ordinal_position: number;
- nullable: boolean;
+ nullable: boolean | number;
character_maximum_length?: string | null; // The maximum length of the column (if applicable), nullable
precision?: {
precision: number | null; // The precision for numeric types
@@ -14,3 +16,23 @@ export interface ColumnInfo {
collation?: string | null;
comment?: string | null;
}
+
+export const ColumnInfoSchema: z.ZodType<ColumnInfo> = z.object({
+ schema: z.string(),
+ table: z.string(),
+ name: z.string(),
+ type: z.string(),
+ ordinal_position: z.number(),
+ nullable: z.union([z.boolean(), z.number()]),
+ character_maximum_length: z.string().nullable().optional(),
+ precision: z
+ .object({
+ precision: z.number().nullable(),
+ scale: z.number().nullable(),
+ })
+ .nullable()
+ .optional(),
+ default: z.string().nullable().optional(),
+ collation: z.string().nullable().optional(),
+ comment: z.string().nullable().optional(),
+});
diff --git a/src/lib/data/import-metadata/metadata-types/database-metadata.ts b/src/lib/data/import-metadata/metadata-types/database-metadata.ts
index 99a7c962..66b01d93 100644
--- a/src/lib/data/import-metadata/metadata-types/database-metadata.ts
+++ b/src/lib/data/import-metadata/metadata-types/database-metadata.ts
@@ -1,9 +1,11 @@
-import type { ForeignKeyInfo } from './foreign-key-info';
-import type { PrimaryKeyInfo } from './primary-key-info';
-import type { ColumnInfo } from './column-info';
-import type { IndexInfo } from './index-info';
-import type { TableInfo } from './table-info';
-import type { ViewInfo } from './view-info';
+import { z } from 'zod';
+import { ForeignKeyInfoSchema, type ForeignKeyInfo } from './foreign-key-info';
+import { PrimaryKeyInfoSchema, type PrimaryKeyInfo } from './primary-key-info';
+import { ColumnInfoSchema, type ColumnInfo } from './column-info';
+import { IndexInfoSchema, type IndexInfo } from './index-info';
+import { TableInfoSchema, type TableInfo } from './table-info';
+import { ViewInfoSchema, type ViewInfo } from './view-info';
+
export interface DatabaseMetadata {
fk_info: ForeignKeyInfo[];
pk_info: PrimaryKeyInfo[];
@@ -15,16 +17,26 @@ export interface DatabaseMetadata {
version: string;
}
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const isDatabaseMetadata = (obj: any): boolean => {
- return (
- Array.isArray(obj.fk_info) &&
- Array.isArray(obj.pk_info) &&
- Array.isArray(obj.columns) &&
- Array.isArray(obj.indexes) &&
- Array.isArray(obj.tables) &&
- Array.isArray(obj.views)
- );
+export const DatabaseMetadataSchema: z.ZodType<DatabaseMetadata> = z.object({
+ fk_info: z.array(ForeignKeyInfoSchema),
+ pk_info: z.array(PrimaryKeyInfoSchema),
+ columns: z.array(ColumnInfoSchema),
+ indexes: z.array(IndexInfoSchema),
+ tables: z.array(TableInfoSchema),
+ views: z.array(ViewInfoSchema),
+ database_name: z.string(),
+ version: z.string(),
+});
+
+export const isDatabaseMetadata = (obj: unknown): boolean => {
+ const parsedObject = DatabaseMetadataSchema.safeParse(obj);
+
+ if (!parsedObject.success) {
+ console.error(parsedObject.error);
+ return false;
+ }
+
+ return true;
};
export const loadDatabaseMetadata = (jsonString: string): DatabaseMetadata => {
diff --git a/src/lib/data/import-metadata/metadata-types/foreign-key-info.ts b/src/lib/data/import-metadata/metadata-types/foreign-key-info.ts
index 931f0c88..c8c342b3 100644
--- a/src/lib/data/import-metadata/metadata-types/foreign-key-info.ts
+++ b/src/lib/data/import-metadata/metadata-types/foreign-key-info.ts
@@ -1,3 +1,5 @@
+import { z } from 'zod';
+
export interface ForeignKeyInfo {
schema: string;
table: string;
@@ -8,3 +10,14 @@ export interface ForeignKeyInfo {
reference_column: string;
fk_def: string;
}
+
+export const ForeignKeyInfoSchema: z.ZodType<ForeignKeyInfo> = z.object({
+ schema: z.string(),
+ table: z.string(),
+ column: z.string(),
+ foreign_key_name: z.string(),
+ reference_schema: z.string().optional(),
+ reference_table: z.string(),
+ reference_column: z.string(),
+ fk_def: z.string(),
+});
diff --git a/src/lib/data/import-metadata/metadata-types/index-info.ts b/src/lib/data/import-metadata/metadata-types/index-info.ts
index c95df818..7d48cbc6 100644
--- a/src/lib/data/import-metadata/metadata-types/index-info.ts
+++ b/src/lib/data/import-metadata/metadata-types/index-info.ts
@@ -1,5 +1,6 @@
import { schemaNameToDomainSchemaName } from '@/lib/domain/db-schema';
import type { TableInfo } from './table-info';
+import { z } from 'zod';
export interface IndexInfo {
schema: string;
@@ -7,14 +8,26 @@ export interface IndexInfo {
name: string;
column: string;
index_type: string;
- cardinality: number;
- size: number;
- unique: boolean;
- is_partial_index: boolean;
+ cardinality?: number | null;
+ size?: number | null;
+ unique: boolean | number;
direction: string;
column_position: number;
}
+export const IndexInfoSchema: z.ZodType<IndexInfo> = z.object({
+ schema: z.string(),
+ table: z.string(),
+ name: z.string(),
+ column: z.string(),
+ index_type: z.string(),
+ cardinality: z.number().nullable().optional(),
+ size: z.number().nullable().optional(),
+ unique: z.union([z.boolean(), z.number()]),
+ direction: z.string(),
+ column_position: z.number(),
+});
+
export type AggregatedIndexInfo = Omit<IndexInfo, 'column'> & {
columns: { name: string; position: number }[];
};
diff --git a/src/lib/data/import-metadata/metadata-types/primary-key-info.ts b/src/lib/data/import-metadata/metadata-types/primary-key-info.ts
index 2e378dd2..b15910f6 100644
--- a/src/lib/data/import-metadata/metadata-types/primary-key-info.ts
+++ b/src/lib/data/import-metadata/metadata-types/primary-key-info.ts
@@ -1,6 +1,15 @@
+import { z } from 'zod';
+
export interface PrimaryKeyInfo {
schema: string;
table: string;
column: string;
pk_def: string;
}
+
+export const PrimaryKeyInfoSchema: z.ZodType<PrimaryKeyInfo> = z.object({
+ schema: z.string(),
+ table: z.string(),
+ column: z.string(),
+ pk_def: z.string(),
+});
diff --git a/src/lib/data/import-metadata/metadata-types/table-info.ts b/src/lib/data/import-metadata/metadata-types/table-info.ts
index 088f57eb..ad8b0e8d 100644
--- a/src/lib/data/import-metadata/metadata-types/table-info.ts
+++ b/src/lib/data/import-metadata/metadata-types/table-info.ts
@@ -1,9 +1,21 @@
+import { z } from 'zod';
+
export interface TableInfo {
schema: string;
table: string;
- rows: number;
- type: string;
- engine: string;
- collation: string;
+ rows?: number;
+ type?: string;
+ engine?: string;
+ collation?: string;
comment?: string;
}
+
+export const TableInfoSchema: z.ZodType<TableInfo> = z.object({
+ schema: z.string(),
+ table: z.string(),
+ rows: z.number().optional(),
+ type: z.string().optional(),
+ engine: z.string().optional(),
+ collation: z.string().optional(),
+ comment: z.string().optional(),
+});
diff --git a/src/lib/data/import-metadata/metadata-types/view-info.ts b/src/lib/data/import-metadata/metadata-types/view-info.ts
index 0ffee179..a71f84da 100644
--- a/src/lib/data/import-metadata/metadata-types/view-info.ts
+++ b/src/lib/data/import-metadata/metadata-types/view-info.ts
@@ -1,5 +1,13 @@
+import { z } from 'zod';
+
export interface ViewInfo {
schema: string;
view_name: string;
view_definition?: string;
}
+
+export const ViewInfoSchema: z.ZodType<ViewInfo> = z.object({
+ schema: z.string(),
+ view_name: z.string(),
+ view_definition: z.string().optional(),
+});
diff --git a/src/lib/data/import-metadata/scripts/clickhouse-script.ts b/src/lib/data/import-metadata/scripts/clickhouse-script.ts
index 5435017a..26478edd 100644
--- a/src/lib/data/import-metadata/scripts/clickhouse-script.ts
+++ b/src/lib/data/import-metadata/scripts/clickhouse-script.ts
@@ -4,7 +4,7 @@ cols AS (
concat('{"schema":"', col_tuple.1, '"',
',"table":"', col_tuple.2, '"',
',"name":"', col_tuple.3, '"',
- ',"ordinal_position":"', toString(col_tuple.4), '"',
+ ',"ordinal_position":', toString(col_tuple.4),
',"type":"', col_tuple.5, '"',
',"nullable":"', if(col_tuple.6 = 'NULLABLE', 'true', 'false'), '"',
',"default":"', if(col_tuple.7 = '', 'null', col_tuple.7), '"',
diff --git a/src/lib/data/import-metadata/scripts/cockroachdb-script.ts b/src/lib/data/import-metadata/scripts/cockroachdb-script.ts
index a49287c9..9c5cd97f 100644
--- a/src/lib/data/import-metadata/scripts/cockroachdb-script.ts
+++ b/src/lib/data/import-metadata/scripts/cockroachdb-script.ts
@@ -96,8 +96,7 @@ indexes_cols AS (
(CASE WHEN i.indisunique = TRUE THEN 'true' ELSE 'false' END) AS is_unique,
irel.reltuples AS cardinality,
1 + Array_position(i.indkey, a.attnum) AS column_position,
- CASE o.OPTION & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction,
- CASE WHEN indpred IS NOT NULL THEN 'true' ELSE 'false' END AS is_partial_index
+ CASE o.OPTION & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction
FROM pg_index AS i
JOIN pg_class AS trel ON trel.oid = i.indrelid
JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid
@@ -114,8 +113,8 @@ cols AS (
SELECT array_to_string(array_agg(CONCAT('{"schema":"', cols.table_schema::TEXT,
'","table":"', cols.table_name::TEXT,
'","name":"', cols.column_name::TEXT,
- '","ordinal_position":"', cols.ordinal_position::TEXT,
- '","type":"', LOWER(replace(cols.data_type::TEXT, '"', '')),
+ '","ordinal_position":', cols.ordinal_position::TEXT,
+ ',"type":"', LOWER(replace(cols.data_type::TEXT, '"', '')),
'","character_maximum_length":"', COALESCE(cols.character_maximum_length::TEXT, 'null'),
'","precision":',
CASE
@@ -124,7 +123,7 @@ cols AS (
',"scale":', COALESCE(cols.numeric_scale::TEXT, 'null'), '}')
ELSE 'null'
END,
- ',"nullable":', CASE WHEN (cols.IS_NULLABLE = 'YES') THEN 'true' ELSE 'false' END::TEXT,
+ ',"nullable":', CASE WHEN (cols.IS_NULLABLE = 'YES') THEN true ELSE false END::TEXT,
',"default":"', COALESCE(replace(replace(cols.column_default::TEXT, '"', '\\"'), '\\x', '\\\\x'), ''),
'","collation":"', COALESCE(cols.COLLATION_NAME::TEXT, ''),
'","comment":"', COALESCE(replace(replace(dsc.description::TEXT, '"', '\\"'), '\\x', '\\\\x'), ''),
@@ -146,7 +145,6 @@ cols AS (
'","cardinality":', COALESCE(cardinality::TEXT, '0'),
',"size":', COALESCE(index_size::TEXT, 'null'),
',"unique":', is_unique::TEXT,
- ',"is_partial_index":', is_partial_index::TEXT,
',"column_position":', column_position::TEXT,
',"direction":"', LOWER(direction::TEXT),
'"}')), ',') AS indexes_metadata
diff --git a/src/lib/data/import-metadata/scripts/maria-script.ts b/src/lib/data/import-metadata/scripts/maria-script.ts
index 3f4c0044..29ea7023 100644
--- a/src/lib/data/import-metadata/scripts/maria-script.ts
+++ b/src/lib/data/import-metadata/scripts/maria-script.ts
@@ -74,8 +74,8 @@ export const mariaDBQuery = `WITH fk_info as (
',"scale":', IFNULL(cols.numeric_scale, 'null'), '}')
ELSE 'null'
END,
- ',"ordinal_position":"', cols.ordinal_position,
- '","nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'),
+ ',"ordinal_position":', cols.ordinal_position,
+ ',"nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'),
',"default":"', IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', '\\"'), ''),
'","collation":"', IFNULL(cols.collation_name, ''), '"}'
)))))
@@ -88,7 +88,7 @@ export const mariaDBQuery = `WITH fk_info as (
AND (0x00) IN (@indexes:=CONCAT_WS(',', @indexes, CONCAT('{"schema":"',indexes.table_schema,
'","table":"',indexes.table_name,
'","name":"', indexes.index_name,
- '","size":"',
+ '","size":',
(SELECT IFNULL(SUM(stat_value * @@innodb_page_size), -1) AS size_in_bytes
FROM mysql.innodb_index_stats
WHERE stat_name = 'size'
@@ -96,11 +96,12 @@ export const mariaDBQuery = `WITH fk_info as (
AND index_name = indexes.index_name
AND TABLE_NAME = indexes.table_name
AND database_name = indexes.table_schema),
- '","column":"', indexes.column_name,
+ ',"column":"', indexes.column_name,
'","index_type":"', LOWER(indexes.index_type),
'","cardinality":', indexes.cardinality,
',"direction":"', (CASE WHEN indexes.collation = 'D' THEN 'desc' ELSE 'asc' END),
- '","unique":', IF(indexes.non_unique = 1, 'false', 'true'), '}')))))
+ '","column_position":', indexes.seq_in_index,
+ ',"unique":', IF(indexes.non_unique = 1, 'false', 'true'), '}')))))
), tbls as
(
(SELECT (@tbls:=NULL),
diff --git a/src/lib/data/import-metadata/scripts/mysql-script.ts b/src/lib/data/import-metadata/scripts/mysql-script.ts
index 45204a85..034f3c7d 100644
--- a/src/lib/data/import-metadata/scripts/mysql-script.ts
+++ b/src/lib/data/import-metadata/scripts/mysql-script.ts
@@ -84,8 +84,8 @@ export const getMySQLQuery = (
',"scale":', IFNULL(cols.numeric_scale, 'null'), '}')
ELSE 'null'
END,
- ',"ordinal_position":"', cols.ordinal_position,
- '","nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'),
+ ',"ordinal_position":', cols.ordinal_position,
+ ',"nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'),
',"default":"', IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', 'ֿֿֿ\\"'), ''),
'","collation":"', IFNULL(cols.collation_name, ''), '"}'
)))))
@@ -98,7 +98,7 @@ export const getMySQLQuery = (
AND (0x00) IN (@indexes:=CONCAT_WS(',', @indexes, CONCAT('{"schema":"',indexes.table_schema,
'","table":"',indexes.table_name,
'","name":"', indexes.index_name,
- '","size":"',
+ '","size":',
(SELECT IFNULL(SUM(stat_value * @@innodb_page_size), -1) AS size_in_bytes
FROM mysql.innodb_index_stats
WHERE stat_name = 'size'
@@ -106,7 +106,7 @@ export const getMySQLQuery = (
AND index_name = indexes.index_name
AND TABLE_NAME = indexes.table_name
AND database_name = indexes.table_schema),
- '","column":"', indexes.column_name,
+ ',"column":"', indexes.column_name,
'","index_type":"', LOWER(indexes.index_type),
'","cardinality":', indexes.cardinality,
',"direction":"', (CASE WHEN indexes.collation = 'D' THEN 'desc' ELSE 'asc' END),
@@ -209,8 +209,8 @@ export const getMySQLQuery = (
IF(cols.data_type IN ('decimal', 'numeric'),
CONCAT('{"precision":', IFNULL(cols.numeric_precision, 'null'),
',"scale":', IFNULL(cols.numeric_scale, 'null'), '}'), 'null'),
- ',"ordinal_position":"', cols.ordinal_position,
- '","nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'),
+ ',"ordinal_position":', cols.ordinal_position,
+ ',"nullable":', IF(cols.is_nullable = 'YES', 'true', 'false'),
',"default":"', IFNULL(REPLACE(REPLACE(cols.column_default, '\\\\', ''), '"', '\\"'), ''),
'","collation":"', IFNULL(cols.collation_name, ''), '"}')
) FROM (
@@ -233,7 +233,7 @@ export const getMySQLQuery = (
CONCAT('{"schema":"', cast(idx.table_schema as CHAR),
'","table":"', idx.table_name,
'","name":"', idx.index_name,
- '","size":"', IFNULL(
+ '","size":', IFNULL(
(SELECT SUM(stat_value * @@innodb_page_size)
FROM mysql.innodb_index_stats
WHERE stat_name = 'size'
@@ -241,7 +241,7 @@ export const getMySQLQuery = (
AND index_name = idx.index_name
AND TABLE_NAME = idx.table_name
AND database_name = idx.table_schema), -1),
- '","column":"', idx.column_name,
+ ',"column":"', idx.column_name,
'","index_type":"', LOWER(idx.index_type),
'","cardinality":', idx.cardinality,
',"direction":"', (CASE WHEN idx.collation = 'D' THEN 'desc' ELSE 'asc' END),
diff --git a/src/lib/data/import-metadata/scripts/postgres-script.ts b/src/lib/data/import-metadata/scripts/postgres-script.ts
index cd8a01dc..de1665f5 100644
--- a/src/lib/data/import-metadata/scripts/postgres-script.ts
+++ b/src/lib/data/import-metadata/scripts/postgres-script.ts
@@ -147,8 +147,7 @@ indexes_cols AS (
(CASE WHEN i.indisunique = TRUE THEN 'true' ELSE 'false' END) AS is_unique,
irel.reltuples AS cardinality,
1 + Array_position(i.indkey, a.attnum) AS column_position,
- CASE o.OPTION & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction,
- CASE WHEN indpred IS NOT NULL THEN 'true' ELSE 'false' END AS is_partial_index
+ CASE o.OPTION & 1 WHEN 1 THEN 'DESC' ELSE 'ASC' END AS direction
FROM pg_index AS i
JOIN pg_class AS trel ON trel.oid = i.indrelid
JOIN pg_namespace AS tnsp ON trel.relnamespace = tnsp.oid
@@ -165,8 +164,8 @@ cols AS (
SELECT array_to_string(array_agg(CONCAT('{"schema":"', cols.table_schema,
'","table":"', cols.table_name,
'","name":"', cols.column_name,
- '","ordinal_position":"', cols.ordinal_position,
- '","type":"', LOWER(replace(cols.data_type, '"', '')),
+ '","ordinal_position":', cols.ordinal_position,
+ ',"type":"', LOWER(replace(cols.data_type, '"', '')),
'","character_maximum_length":"', COALESCE(cols.character_maximum_length::text, 'null'),
'","precision":',
CASE
@@ -203,7 +202,6 @@ cols AS (
'","cardinality":', cardinality,
',"size":', index_size,
',"unique":', is_unique,
- ',"is_partial_index":', is_partial_index,
',"column_position":', column_position,
',"direction":"', LOWER(direction),
'"}')), ',') AS indexes_metadata
diff --git a/src/lib/data/import-metadata/scripts/sqlite-script.ts b/src/lib/data/import-metadata/scripts/sqlite-script.ts
index dcba3d55..3936e7b0 100644
--- a/src/lib/data/import-metadata/scripts/sqlite-script.ts
+++ b/src/lib/data/import-metadata/scripts/sqlite-script.ts
@@ -60,9 +60,9 @@ WITH fk_info AS (
'name', idx.name,
'column', ic.name,
'index_type', 'B-TREE', -- SQLite uses B-Trees for indexing
- 'cardinality', '', -- SQLite does not provide cardinality
- 'size', '', -- SQLite does not provide index size
- 'unique', (CASE WHEN idx."unique" = 1 THEN 'true' ELSE 'false' END),
+ 'cardinality', null, -- SQLite does not provide cardinality
+ 'size', null, -- SQLite does not provide index size
+ 'unique', (CASE WHEN idx."unique" = 1 THEN true ELSE false END),
'direction', '', -- SQLite does not provide direction info
'column_position', ic.seqno + 1 -- Adding 1 to convert from zero-based to one-based index
)
@@ -107,12 +107,12 @@ WITH fk_info AS (
CASE
WHEN instr(p.type, '(') > 0 THEN
json_object(
- 'precision', substr(p.type, instr(p.type, '(') + 1, instr(p.type, ',') - instr(p.type, '(') - 1),
- 'scale', substr(p.type, instr(p.type, ',') + 1, instr(p.type, ')') - instr(p.type, ',') - 1)
+ 'precision', CAST(substr(p.type, instr(p.type, '(') + 1, instr(p.type, ',') - instr(p.type, '(') - 1) AS INTEGER),
+ 'scale', CAST(substr(p.type, instr(p.type, ',') + 1, instr(p.type, ')') - instr(p.type, ',') - 1) AS INTEGER)
)
- ELSE 'null'
+ ELSE null
END
- ELSE 'null'
+ ELSE null
END,
'default', COALESCE(REPLACE(p.dflt_value, '"', '\\"'), '')
)
@@ -231,9 +231,9 @@ WITH fk_info AS (
'name', idx.name,
'column', ic.name,
'index_type', 'B-TREE',
- 'cardinality', '',
- 'size', '',
- 'unique', (CASE WHEN idx.[unique] = 1 THEN 'true' ELSE 'false' END),
+ 'cardinality', null,
+ 'size', null,
+ 'unique', (CASE WHEN idx.[unique] = 1 THEN true ELSE false END),
'direction', '',
'column_position', ic.seqno + 1
)
@@ -280,12 +280,12 @@ WITH fk_info AS (
CASE
WHEN instr(p.type, '(') > 0 THEN
json_object(
- 'precision', substr(p.type, instr(p.type, '(') + 1, instr(p.type, ',') - instr(p.type, '(') - 1),
- 'scale', substr(p.type, instr(p.type, ',') + 1, instr(p.type, ')') - instr(p.type, ',') - 1)
+ 'precision', CAST(substr(p.type, instr(p.type, '(') + 1, instr(p.type, ',') - instr(p.type, '(') - 1) AS INTEGER),
+ 'scale', CAST(substr(p.type, instr(p.type, ',') + 1, instr(p.type, ')') - instr(p.type, ',') - 1) AS INTEGER)
)
- ELSE 'null'
+ ELSE null
END
- ELSE 'null'
+ ELSE null
END,
'default', COALESCE(REPLACE(p.dflt_value, '"', '\\"'), '')
)
@@ -361,7 +361,7 @@ const generateWranglerCommand = (): string => {
# 5. Replace YOUR_DB_ID with your actual D1 database ID
# Step 1: Write the query to a file
-wrangler d1 execute YOUR_DB_NAME --command $'WITH fk_info AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'column\\', fk.[from], \\'foreign_key_name\\', \\'fk_\\' || m.name || \\'_\\' || fk.[from] || \\'_\\' || fk.[table] || \\'_\\' || fk.[to], \\'reference_schema\\', \\'\\', \\'reference_table\\', fk.[table], \\'reference_column\\', fk.[to], \\'fk_def\\', \\'FOREIGN KEY (\\' || fk.[from] || \\') REFERENCES \\' || fk.[table] || \\'(\\' || fk.[to] || \\')\\' || \\' ON UPDATE \\' || fk.on_update || \\' ON DELETE \\' || fk.on_delete ) ) AS fk_metadata FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) fk ON m.type = \\'table\\' WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), pk_info AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', pk.table_name, \\'field_count\\', pk.field_count, \\'column\\', pk.pk_column, \\'pk_def\\', \\'PRIMARY KEY (\\' || pk.pk_column || \\')\\' ) ) AS pk_metadata FROM ( SELECT m.name AS table_name, COUNT(p.name) AS field_count, GROUP_CONCAT(p.name) AS pk_column FROM sqlite_master m JOIN pragma_table_info(m.name) p ON m.type = \\'table\\' AND p.pk > 0 WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' GROUP BY m.name ) pk ), indexes_metadata AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'name\\', idx.name, \\'column\\', ic.name, \\'index_type\\', \\'B-TREE\\', \\'cardinality\\', \\'\\', \\'size\\', \\'\\', \\'unique\\', CASE WHEN idx.[unique] = 1 THEN \\'true\\' ELSE \\'false\\' END, \\'direction\\', \\'\\', \\'column_position\\', ic.seqno + 1 ) ) AS indexes_metadata FROM sqlite_master m JOIN pragma_index_list(m.name) idx ON m.type = \\'table\\' JOIN pragma_index_info(idx.name) ic WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), cols AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'name\\', p.name, \\'type\\', CASE WHEN INSTR(LOWER(p.type), \\'(\\') > 0 THEN SUBSTR(LOWER(p.type), 1, INSTR(LOWER(p.type), \\'(\\') - 1) ELSE LOWER(p.type) END, \\'ordinal_position\\', p.cid, \\'nullable\\', CASE WHEN p.[notnull] = 0 THEN true ELSE false END, \\'collation\\', \\'\\', \\'character_maximum_length\\', CASE WHEN LOWER(p.type) LIKE \\'char%\\' OR LOWER(p.type) LIKE \\'varchar%\\' THEN CASE WHEN INSTR(p.type, \\'(\\') > 0 THEN REPLACE( SUBSTR(p.type, INSTR(p.type, \\'(\\') + 1, LENGTH(p.type) - INSTR(p.type, \\'(\\') - 1), \\')\\', \\'\\' ) ELSE \\'null\\' END ELSE \\'null\\' END, \\'precision\\', CASE WHEN LOWER(p.type) LIKE \\'decimal%\\' OR LOWER(p.type) LIKE \\'numeric%\\' THEN CASE WHEN instr(p.type, \\'(\\') > 0 THEN json_object( \\'precision\\', substr(p.type, instr(p.type, \\'(\\') + 1, instr(p.type, \\',\\') - instr(p.type, \\'(\\') - 1), \\'scale\\', substr(p.type, instr(p.type, \\',\\') + 1, instr(p.type, \\')\\') - instr(p.type, \\',\\') - 1) ) ELSE \\'null\\' END ELSE \\'null\\' END, \\'default\\', COALESCE(REPLACE(p.dflt_value, \\'"\\', \\'\\\\\\"\\'), \\'\\') ) ) AS cols_metadata FROM sqlite_master m JOIN pragma_table_info(m.name) p ON m.type in (\\'table\\', \\'view\\') WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), tbls AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'rows\\', -1, \\'type\\', \\'table\\', \\'engine\\', \\'\\', \\'collation\\', \\'\\' ) ) AS tbls_metadata FROM sqlite_master m WHERE m.type in (\\'table\\', \\'view\\') AND m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), views AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'view_name\\', m.name ) ) AS views_metadata FROM sqlite_master m WHERE m.type = \\'view\\' AND m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ) SELECT json_object( \\'fk_info\\', json((SELECT fk_metadata FROM fk_info)), \\'pk_info\\', json((SELECT pk_metadata FROM pk_info)), \\'columns\\', json((SELECT cols_metadata FROM cols)), \\'indexes\\', json((SELECT indexes_metadata FROM indexes_metadata)), \\'tables\\', json((SELECT tbls_metadata FROM tbls)), \\'views\\', json((SELECT views_metadata FROM views)), \\'database_name\\', \\'sqlite\\', \\'version\\', \\'\\' ) AS metadata_json_to_import;' --remote
+wrangler d1 execute YOUR_DB_NAME --command $'WITH fk_info AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'column\\', fk.[from], \\'foreign_key_name\\', \\'fk_\\' || m.name || \\'_\\' || fk.[from] || \\'_\\' || fk.[table] || \\'_\\' || fk.[to], \\'reference_schema\\', \\'\\', \\'reference_table\\', fk.[table], \\'reference_column\\', fk.[to], \\'fk_def\\', \\'FOREIGN KEY (\\' || fk.[from] || \\') REFERENCES \\' || fk.[table] || \\'(\\' || fk.[to] || \\')\\' || \\' ON UPDATE \\' || fk.on_update || \\' ON DELETE \\' || fk.on_delete ) ) AS fk_metadata FROM sqlite_master m JOIN pragma_foreign_key_list(m.name) fk ON m.type = \\'table\\' WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), pk_info AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', pk.table_name, \\'field_count\\', pk.field_count, \\'column\\', pk.pk_column, \\'pk_def\\', \\'PRIMARY KEY (\\' || pk.pk_column || \\')\\' ) ) AS pk_metadata FROM ( SELECT m.name AS table_name, COUNT(p.name) AS field_count, GROUP_CONCAT(p.name) AS pk_column FROM sqlite_master m JOIN pragma_table_info(m.name) p ON m.type = \\'table\\' AND p.pk > 0 WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' GROUP BY m.name ) pk ), indexes_metadata AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'name\\', idx.name, \\'column\\', ic.name, \\'index_type\\', \\'B-TREE\\', \\'cardinality\\', \\'\\', \\'size\\', null, \\'unique\\', CASE WHEN idx.[unique] = 1 THEN true ELSE false END, \\'direction\\', \\'\\', \\'column_position\\', ic.seqno + 1 ) ) AS indexes_metadata FROM sqlite_master m JOIN pragma_index_list(m.name) idx ON m.type = \\'table\\' JOIN pragma_index_info(idx.name) ic WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), cols AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'name\\', p.name, \\'type\\', CASE WHEN INSTR(LOWER(p.type), \\'(\\') > 0 THEN SUBSTR(LOWER(p.type), 1, INSTR(LOWER(p.type), \\'(\\') - 1) ELSE LOWER(p.type) END, \\'ordinal_position\\', p.cid, \\'nullable\\', CASE WHEN p.[notnull] = 0 THEN true ELSE false END, \\'collation\\', \\'\\', \\'character_maximum_length\\', CASE WHEN LOWER(p.type) LIKE \\'char%\\' OR LOWER(p.type) LIKE \\'varchar%\\' THEN CASE WHEN INSTR(p.type, \\'(\\') > 0 THEN REPLACE( SUBSTR(p.type, INSTR(p.type, \\'(\\') + 1, LENGTH(p.type) - INSTR(p.type, \\'(\\') - 1), \\')\\', \\'\\' ) ELSE \\'null\\' END ELSE \\'null\\' END, \\'precision\\', CASE WHEN LOWER(p.type) LIKE \\'decimal%\\' OR LOWER(p.type) LIKE \\'numeric%\\' THEN CASE WHEN instr(p.type, \\'(\\') > 0 THEN json_object( \\'precision\\', CAST(substr(p.type, instr(p.type, \\'(\\') + 1, instr(p.type, \\',\\') - instr(p.type, \\'(\\') - 1) as INTIGER), \\'scale\\', CAST(substr(p.type, instr(p.type, \\',\\') + 1, instr(p.type, \\')\\') - instr(p.type, \\',\\') - 1) AS INTIGER) ) ELSE null END ELSE null END, \\'default\\', COALESCE(REPLACE(p.dflt_value, \\'"\\', \\'\\\\\\"\\'), \\'\\') ) ) AS cols_metadata FROM sqlite_master m JOIN pragma_table_info(m.name) p ON m.type in (\\'table\\', \\'view\\') WHERE m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), tbls AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'table\\', m.name, \\'rows\\', -1, \\'type\\', \\'table\\', \\'engine\\', \\'\\', \\'collation\\', \\'\\' ) ) AS tbls_metadata FROM sqlite_master m WHERE m.type in (\\'table\\', \\'view\\') AND m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ), views AS ( SELECT json_group_array( json_object( \\'schema\\', \\'\\', \\'view_name\\', m.name ) ) AS views_metadata FROM sqlite_master m WHERE m.type = \\'view\\' AND m.name NOT LIKE \\'\\\\_cf\\\\_%\\' ESCAPE \\'\\\\\\' ) SELECT json_object( \\'fk_info\\', json((SELECT fk_metadata FROM fk_info)), \\'pk_info\\', json((SELECT pk_metadata FROM pk_info)), \\'columns\\', json((SELECT cols_metadata FROM cols)), \\'indexes\\', json((SELECT indexes_metadata FROM indexes_metadata)), \\'tables\\', json((SELECT tbls_metadata FROM tbls)), \\'views\\', json((SELECT views_metadata FROM views)), \\'database_name\\', \\'sqlite\\', \\'version\\', \\'\\' ) AS metadata_json_to_import;' --remote
# Step 2: Copy the output of the command above and paste it into app.chartdb.io
`;
diff --git a/src/lib/data/import-metadata/scripts/sqlserver-script.ts b/src/lib/data/import-metadata/scripts/sqlserver-script.ts
index efdcf2b3..d00c24fa 100644
--- a/src/lib/data/import-metadata/scripts/sqlserver-script.ts
+++ b/src/lib/data/import-metadata/scripts/sqlserver-script.ts
@@ -68,12 +68,12 @@ cols AS (
'", "name": "' + STRING_ESCAPE(COALESCE(REPLACE(cols.COLUMN_NAME, '"', ''), ''), 'json') +
'", "ordinal_position": ' + CAST(cols.ORDINAL_POSITION AS NVARCHAR(MAX)) +
', "type": "' + STRING_ESCAPE(LOWER(cols.DATA_TYPE), 'json') +
- '", "character_maximum_length": ' +
+ '", "character_maximum_length": "' +
CASE
WHEN cols.CHARACTER_MAXIMUM_LENGTH IS NULL THEN 'null'
ELSE CAST(cols.CHARACTER_MAXIMUM_LENGTH AS NVARCHAR(MAX))
END +
- ', "precision": ' +
+ '", "precision": ' +
CASE
WHEN cols.DATA_TYPE IN ('numeric', 'decimal')
THEN '{"precision":' + COALESCE(CAST(cols.NUMERIC_PRECISION AS NVARCHAR(MAX)), 'null') +
@@ -270,12 +270,12 @@ cols AS (
'", "name": "' + STRING_ESCAPE(COALESCE(REPLACE(cols.COLUMN_NAME, '"', ''), ''), 'json') +
'", "ordinal_position": ' + CAST(cols.ORDINAL_POSITION AS NVARCHAR(MAX)) +
', "type": "' + STRING_ESCAPE(LOWER(cols.DATA_TYPE), 'json') +
- '", "character_maximum_length": ' +
+ '", "character_maximum_length": "' +
CASE
WHEN cols.CHARACTER_MAXIMUM_LENGTH IS NULL THEN 'null'
ELSE CAST(cols.CHARACTER_MAXIMUM_LENGTH AS NVARCHAR(MAX))
END +
- ', "precision": ' +
+ '", "precision": ' +
CASE
WHEN cols.DATA_TYPE IN ('numeric', 'decimal')
THEN '{"precision":' + COALESCE(CAST(cols.NUMERIC_PRECISION AS NVARCHAR(MAX)), 'null') +
diff --git a/src/lib/domain/db-field.ts b/src/lib/domain/db-field.ts
index 89a0f3b7..620baf6e 100644
--- a/src/lib/domain/db-field.ts
+++ b/src/lib/domain/db-field.ts
@@ -92,7 +92,7 @@ export const createFieldsFromMetadata = ({
idx.columns.length === 1 &&
idx.columns[0].name === col.name
),
- nullable: col.nullable,
+ nullable: Boolean(col.nullable),
...(col.character_maximum_length &&
col.character_maximum_length !== 'null'
? { characterMaximumLength: col.character_maximum_length }
diff --git a/src/lib/domain/db-index.ts b/src/lib/domain/db-index.ts
index d909fb4c..ca2b98d2 100644
--- a/src/lib/domain/db-index.ts
+++ b/src/lib/domain/db-index.ts
@@ -30,7 +30,7 @@ export const createIndexesFromMetadata = ({
(idx): DBIndex => ({
id: generateId(),
name: idx.name,
- unique: idx.unique,
+ unique: Boolean(idx.unique),
fieldIds: idx.columns
.sort((a, b) => a.position - b.position)
.map((c) => fields.find((f) => f.name === c.name)?.id)