Database diagrams editor that allows you to visualize and design your DB with a single query.
TypeScript databasedbdb-migrationeditorerdmariadbmssqlmysqlpostgresqlreactreact-flowschema-migrationssqlsqlitetypescriptvisualizationxyflow
fix: handle embedded quotes and CSV format in metadata import (#1116)
Jonathan Fishner344293610cf4
diff --git a/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts b/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.tsindex b8315839..271b65aa 100644--- a/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts+++ b/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts@@ -165,14 +165,21 @@ describe('fixMetadataJson', () => {});describe('escaped backslashes', () => {- it('should convert \\\\\\\\ to \\\\ (double-escaped to single-escaped)', () => {- // Input: {"path": "C:\\\\Users"} (double-escaped backslash in JSON string literal)- // Output: {"path": "C:\\Users"} (single-escaped backslash, valid JSON for path C:\Users)+ it('should preserve valid JSON with double backslashes as-is', () => {+ // Input: {"path": "C:\\\\Users"} is valid JSON (value is C:\\Users)+ // Valid JSON should be returned unchangedconst input = '{"path": "C:\\\\\\\\Users"}';const result = fixMetadataJson(input);- // In JSON, \\ represents a single backslash, so this is correct+ expect(result).toBe('{"path": "C:\\\\\\\\Users"}');+ const parsed = JSON.parse(result);+ expect(parsed.path).toBe('C:\\\\Users');+ });++ it('should convert double-escaped backslashes in stringified JSON', () => {+ // Stringified JSON with double-escaped backslashes+ const input = '{\\"path\\": \\"C:\\\\\\\\Users\\"}';+ const result = fixMetadataJson(input);expect(result).toBe('{"path": "C:\\\\Users"}');- // Verify it parses correctlyconst parsed = JSON.parse(result);expect(parsed.path).toBe('C:\\Users');});@@ -231,6 +238,97 @@ describe('fixMetadataJson', () => {});});+ describe('embedded quotes in values', () => {+ it('should preserve escaped quotes in Snowflake clustering key values', () => {+ const input = `{+ "indexes": [+ {+ "schema": "PUBLIC",+ "table": "snap_lot",+ "name": "snap_lot_cluster",+ "column": "LINEAR(DIV0(\\"snaplot_id\\", 500000))",+ "index_type": "CLUSTERING",+ "cardinality": 0,+ "size": 0,+ "unique": false,+ "column_position": 1,+ "direction": "asc"+ }+ ],+ "fk_info": [],+ "pk_info": [],+ "columns": [],+ "tables": [],+ "views": [],+ "database_name": "test",+ "version": "1.0"+}`;++ const result = fixMetadataJson(input);+ const parsed = JSON.parse(result);+ expect(parsed.indexes[0].column).toBe(+ 'LINEAR(DIV0("snaplot_id", 500000))'+ );+ });++ it('should still fix stringified JSON with embedded quotes', () => {+ const input =+ '{\\"column\\": \\"LINEAR(DIV0(\\\\\\"snaplot_id\\\\\\", 500000))\\"}';+ const result = fixMetadataJson(input);+ const parsed = JSON.parse(result);+ expect(parsed.column).toBe('LINEAR(DIV0("snaplot_id", 500000))');+ });+ });++ describe('control characters and trailing junk', () => {+ it('should strip carriage returns inside string values', () => {+ const input =+ '{"fk_info": [], "pk_info": [{"schema": "dbo",\r"table": "Users",\r"column": "id",\r"pk_def": "PRIMARY KEY"}], "columns": [], "indexes": [], "tables": [], "views": [], "database_name": "db", "version": "1.0"}';+ const result = fixMetadataJson(input);+ expect(isStringMetadataJson(result)).toBe(true);+ });++ it('should remove trailing commas in arrays and objects', () => {+ const input = '{"a": [1, 2, 3,], "b": "x",}';+ const result = fixMetadataJson(input);+ expect(JSON.parse(result)).toEqual({ a: [1, 2, 3], b: 'x' });+ });++ it('should remove leading commas in arrays', () => {+ const input = '{"a": [, 1, 2]}';+ const result = fixMetadataJson(input);+ expect(JSON.parse(result)).toEqual({ a: [1, 2] });+ });++ it('should strip terminal box-drawing characters from copy-paste', () => {+ const input = '{ ↴│ "a": 1, ↳"b": 2}';+ const result = fixMetadataJson(input);+ expect(JSON.parse(result)).toEqual({ a: 1, b: 2 });+ });+ });++ describe('metadata_json_to_import wrapper', () => {+ it('should extract inner JSON from object wrapper', () => {+ // This shape has unescaped quotes inside the wrapper value, making+ // the outer object invalid. We extract the inner object directly.+ const input =+ '{"metadata_json_to_import": "{"fk_info": [], "pk_info": [], "columns": [{"schema": "public", "table": "users", "name": "id", "type": "int", "ordinal_position": 1, "nullable": false}], "indexes": [], "tables": [], "views": [], "database_name": "db", "version": "1.0"}"}';+ const result = fixMetadataJson(input);+ expect(isStringMetadataJson(result)).toBe(true);+ const parsed = JSON.parse(result);+ expect(parsed.columns[0].name).toBe('id');+ });++ it('should extract inner JSON from array wrapper', () => {+ const input =+ '[{"metadata_json_to_import": "{"fk_info": [], "pk_info": [], "columns": [], "indexes": [], "tables": [], "views": [], "database_name": "test_db", "version": "8.0"}"}]';+ const result = fixMetadataJson(input);+ expect(isStringMetadataJson(result)).toBe(true);+ const parsed = JSON.parse(result);+ expect(parsed.database_name).toBe('test_db');+ });+ });+describe('edge cases', () => {it('should handle empty objects', () => {const input = '{}';@@ -311,6 +409,30 @@ describe('isStringMetadataJson', () => {});});+describe('CSV-exported Snowflake output', () => {+ it('should handle Snowflake Snowsight CSV export with "" quoting', () => {+ // Simulates CSV export: outer quotes, all " escaped as ""+ const csvInput = `"{ ""fk_info"": [], ""pk_info"": [{ ""schema"": ""PUBLIC"", ""table"": ""users"", ""column"": ""id"", ""pk_def"": ""pk_users"" }], ""columns"": [{ ""schema"": ""PUBLIC"", ""table"": ""users"", ""name"": ""id"", ""type"": ""NUMBER"", ""ordinal_position"": 1, ""nullable"": false, ""character_maximum_length"": null, ""precision"": { ""precision"": 38, ""scale"": 0 }, ""default"": null, ""collation"": null, ""comment"": null, ""is_identity"": true }], ""indexes"": [], ""tables"": [{ ""schema"": ""PUBLIC"", ""table"": ""users"", ""rows"": 100, ""type"": ""BASE TABLE"", ""engine"": """", ""collation"": """", ""comment"": """" }], ""views"": [], ""database_name"": ""MY_DB"", ""version"": ""10.11.1"" }"`;++ const fixed = fixMetadataJson(csvInput);+ expect(isStringMetadataJson(fixed)).toBe(true);+ const parsed = JSON.parse(fixed);+ expect(parsed.database_name).toBe('MY_DB');+ expect(parsed.pk_info[0].schema).toBe('PUBLIC');+ expect(parsed.columns[0].type).toBe('NUMBER');+ });++ it('should handle CSV export with embedded quotes in clustering keys', () => {+ const csvInput = `"{ ""fk_info"": [], ""pk_info"": [], ""columns"": [], ""indexes"": [{ ""schema"": ""PUBLIC"", ""table"": ""snap_lot"", ""name"": ""snap_lot_cluster"", ""column"": ""LINEAR(DIV0(\\""snaplot_id\\"", 500000))"", ""index_type"": ""CLUSTERING"", ""cardinality"": 0, ""size"": 0, ""unique"": false, ""column_position"": 1, ""direction"": ""asc"" }], ""tables"": [], ""views"": [], ""database_name"": ""TEST_DB"", ""version"": ""10.11.1"" }"`;++ const fixed = fixMetadataJson(csvInput);+ const parsed = JSON.parse(fixed);+ expect(parsed.indexes[0].column).toBe(+ 'LINEAR(DIV0("snaplot_id", 500000))'+ );+ });+});+describe('fixMetadataJson + isStringMetadataJson integration', () => {it('should fix and validate MySQL smart query output', () => {const brokenInput = `{\\n \\"fk_info\\": [],\\n \\"pk_info\\": [],\\n \\"columns\\": [],\\n \\"indexes\\": [],\\n \\"tables\\": [],\\n \\"views\\": [],\\n \\"database_name\\": \\"testdb\\",\\n \\"version\\": \\"8.0.39\\"\\n}`;diff --git a/src/lib/data/import-metadata/utils.ts b/src/lib/data/import-metadata/utils.tsindex 6efed71f..e29f2209 100644--- a/src/lib/data/import-metadata/utils.ts+++ b/src/lib/data/import-metadata/utils.ts@@ -1,5 +1,56 @@import { isDatabaseMetadata } from './metadata-types/database-metadata';+const applyCommonFixups = (json: string): string => {+ return (+ json+ .replace(/\r/g, '') // Strip carriage returns (invalid control char in JSON strings)+ .replace(/\n/g, '')+ // Strip terminal-output decoration characters (e.g. ClickHouse client's+ // ↴│↳ arrows and box-drawing chars that get included in copy-paste).+ .replace(/[\u2190-\u21FF\u2500-\u257F]/g, '')+ .replace(/,(\s*[\]}])/g, '$1') // Remove trailing commas (common JS-to-JSON mistake)+ .replace(/(\[)\s*,/g, '$1') // Remove leading commas in arrays (from NULL IFNULL templates)+ .replace(/"precision": "null"/g, '"precision": null')+ .replace(/"nullable": "false"/g, '"nullable": false')+ .replace(/"nullable": "true"/g, '"nullable": true')+ );+};++// Extract inner JSON from { "metadata_json_to_import": "..." } wrapper.+// The wrapper format has the inner JSON's quotes unescaped, making the outer+// object invalid. We find the inner { ... } using brace counting.+const extractMetadataWrapper = (payload: string): string | null => {+ const wrapperKey = 'metadata_json_to_import';+ if (!payload.includes(wrapperKey)) return null;++ const keyIdx = payload.indexOf(wrapperKey);+ const startIdx = payload.indexOf('{', keyIdx + wrapperKey.length);+ if (startIdx === -1) return null;++ let depth = 0;+ let inString = false;+ let escape = false;+ for (let i = startIdx; i < payload.length; i++) {+ const ch = payload[i];+ if (escape) {+ escape = false;+ continue;+ }+ if (ch === '\\') {+ escape = true;+ continue;+ }+ if (ch === '"') inString = !inString;+ if (inString) continue;+ if (ch === '{') depth++;+ else if (ch === '}') {+ depth--;+ if (depth === 0) return payload.slice(startIdx, i + 1);+ }+ }+ return null;+};+export const fixMetadataJson = (metadataJson: string): string => {// Replace problematic array default values with nullmetadataJson = metadataJson.replace(@@ -13,8 +64,57 @@ export const fixMetadataJson = (metadataJson: string): string => {'"default": "$1"$2');- // TODO: remove this temporary eslint disable- return (+ // Try minimal cleanup first — this preserves valid JSON escaping (e.g. \" in+ // string values like Snowflake clustering keys with quoted identifiers).+ // The destructive \" → " replacement below would break these.+ const minimalCleaned = applyCommonFixups(+ metadataJson+ .trim()+ .replace(/^[^{]*/, '')+ .replace(/}[^}]*$/, '}')+ );++ try {+ JSON.parse(minimalCleaned);+ return minimalCleaned;+ } catch {+ // Not valid JSON as-is, try other formats below+ }++ // Try CSV unwrap — Snowflake Snowsight CSV exports wrap the JSON in quotes+ // and escape all internal " as "". Unwrapping "" → " produces valid JSON+ // while preserving legitimate \" escaping (e.g. clustering keys).+ const csvUnwrapped = applyCommonFixups(+ metadataJson+ .trim()+ .replace(/^[^{]*/, '')+ .replace(/}[^}]*$/, '}')+ .replace(/""/g, '"')+ );++ try {+ JSON.parse(csvUnwrapped);+ return csvUnwrapped;+ } catch {+ // Not CSV format, try wrapper extraction+ }++ // Try metadata_json_to_import wrapper extraction — some users paste the+ // result wrapped in `[{"metadata_json_to_import": "{...inner json...}"}]`+ // where the inner JSON's quotes are not escaped, making the outer object+ // invalid. Extract the inner object directly via brace counting.+ const innerJson = extractMetadataWrapper(metadataJson);+ if (innerJson) {+ const innerCleaned = applyCommonFixups(innerJson);+ try {+ JSON.parse(innerCleaned);+ return innerCleaned;+ } catch {+ // Inner JSON also broken, fall through+ }+ }++ return applyCommonFixups(metadataJson.trim()// First unescape the JSON string@@ -48,20 +148,12 @@ export const fixMetadataJson = (metadataJson: string): string => {/* eslint-disable-next-line no-useless-escape */.replace(/'([^']+)'::\"([^\"]+)\"/g, '\'$1\'::\\\"$2\\\"')- // Convert string "null" to actual null for precision field- .replace(/"precision": "null"/g, '"precision": null')-- // Convert string "true"/"false" to actual boolean for nullable field- .replace(/"nullable": "false"/g, '"nullable": false')- .replace(/"nullable": "true"/g, '"nullable": true')-/* eslint-disable-next-line no-useless-escape */.replace(/\"/g, '___ESCAPED_QUOTE___') // Temporarily replace empty strings.replace(/(:\s*)""(?=\s*[,}])/g, '$1___EMPTY___') // Temporarily replace empty strings (Safari-compatible).replace(/""/g, '"') // Replace remaining double quotes.replace(/___ESCAPED_QUOTE___/g, '"') // Restore empty strings.replace(/___EMPTY___/g, '""') // Restore empty strings- .replace(/\n/g, ''));};