Skip to content

Database diagrams editor that allows you to visualize and design your DB with a single query.

fix: handle embedded quotes and CSV format in metadata import (#1116)

Jonathan Fishner344293610cf4
diff --git a/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts b/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts
index b8315839..271b65aa 100644
--- a/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts
+++ b/src/lib/data/import-metadata/__tests__/fix-metadata-json.test.ts
@@ -165,14 +165,21 @@ describe('fixMetadataJson', () => {
});
describe('escaped backslashes', () => {
- it('should convert \\\\\\\\ to \\\\ (double-escaped to single-escaped)', () => {
- // Input: {"path": "C:\\\\Users"} (double-escaped backslash in JSON string literal)
- // Output: {"path": "C:\\Users"} (single-escaped backslash, valid JSON for path C:\Users)
+ it('should preserve valid JSON with double backslashes as-is', () => {
+ // Input: {"path": "C:\\\\Users"} is valid JSON (value is C:\\Users)
+ // Valid JSON should be returned unchanged
const input = '{"path": "C:\\\\\\\\Users"}';
const result = fixMetadataJson(input);
- // In JSON, \\ represents a single backslash, so this is correct
+ expect(result).toBe('{"path": "C:\\\\\\\\Users"}');
+ const parsed = JSON.parse(result);
+ expect(parsed.path).toBe('C:\\\\Users');
+ });
+
+ it('should convert double-escaped backslashes in stringified JSON', () => {
+ // Stringified JSON with double-escaped backslashes
+ const input = '{\\"path\\": \\"C:\\\\\\\\Users\\"}';
+ const result = fixMetadataJson(input);
expect(result).toBe('{"path": "C:\\\\Users"}');
- // Verify it parses correctly
const parsed = JSON.parse(result);
expect(parsed.path).toBe('C:\\Users');
});
@@ -231,6 +238,97 @@ describe('fixMetadataJson', () => {
});
});
+ describe('embedded quotes in values', () => {
+ it('should preserve escaped quotes in Snowflake clustering key values', () => {
+ const input = `{
+ "indexes": [
+ {
+ "schema": "PUBLIC",
+ "table": "snap_lot",
+ "name": "snap_lot_cluster",
+ "column": "LINEAR(DIV0(\\"snaplot_id\\", 500000))",
+ "index_type": "CLUSTERING",
+ "cardinality": 0,
+ "size": 0,
+ "unique": false,
+ "column_position": 1,
+ "direction": "asc"
+ }
+ ],
+ "fk_info": [],
+ "pk_info": [],
+ "columns": [],
+ "tables": [],
+ "views": [],
+ "database_name": "test",
+ "version": "1.0"
+}`;
+
+ const result = fixMetadataJson(input);
+ const parsed = JSON.parse(result);
+ expect(parsed.indexes[0].column).toBe(
+ 'LINEAR(DIV0("snaplot_id", 500000))'
+ );
+ });
+
+ it('should still fix stringified JSON with embedded quotes', () => {
+ const input =
+ '{\\"column\\": \\"LINEAR(DIV0(\\\\\\"snaplot_id\\\\\\", 500000))\\"}';
+ const result = fixMetadataJson(input);
+ const parsed = JSON.parse(result);
+ expect(parsed.column).toBe('LINEAR(DIV0("snaplot_id", 500000))');
+ });
+ });
+
+ describe('control characters and trailing junk', () => {
+ it('should strip carriage returns inside string values', () => {
+ const input =
+ '{"fk_info": [], "pk_info": [{"schema": "dbo",\r"table": "Users",\r"column": "id",\r"pk_def": "PRIMARY KEY"}], "columns": [], "indexes": [], "tables": [], "views": [], "database_name": "db", "version": "1.0"}';
+ const result = fixMetadataJson(input);
+ expect(isStringMetadataJson(result)).toBe(true);
+ });
+
+ it('should remove trailing commas in arrays and objects', () => {
+ const input = '{"a": [1, 2, 3,], "b": "x",}';
+ const result = fixMetadataJson(input);
+ expect(JSON.parse(result)).toEqual({ a: [1, 2, 3], b: 'x' });
+ });
+
+ it('should remove leading commas in arrays', () => {
+ const input = '{"a": [, 1, 2]}';
+ const result = fixMetadataJson(input);
+ expect(JSON.parse(result)).toEqual({ a: [1, 2] });
+ });
+
+ it('should strip terminal box-drawing characters from copy-paste', () => {
+ const input = '{ ↴│ "a": 1, ↳"b": 2}';
+ const result = fixMetadataJson(input);
+ expect(JSON.parse(result)).toEqual({ a: 1, b: 2 });
+ });
+ });
+
+ describe('metadata_json_to_import wrapper', () => {
+ it('should extract inner JSON from object wrapper', () => {
+ // This shape has unescaped quotes inside the wrapper value, making
+ // the outer object invalid. We extract the inner object directly.
+ const input =
+ '{"metadata_json_to_import": "{"fk_info": [], "pk_info": [], "columns": [{"schema": "public", "table": "users", "name": "id", "type": "int", "ordinal_position": 1, "nullable": false}], "indexes": [], "tables": [], "views": [], "database_name": "db", "version": "1.0"}"}';
+ const result = fixMetadataJson(input);
+ expect(isStringMetadataJson(result)).toBe(true);
+ const parsed = JSON.parse(result);
+ expect(parsed.columns[0].name).toBe('id');
+ });
+
+ it('should extract inner JSON from array wrapper', () => {
+ const input =
+ '[{"metadata_json_to_import": "{"fk_info": [], "pk_info": [], "columns": [], "indexes": [], "tables": [], "views": [], "database_name": "test_db", "version": "8.0"}"}]';
+ const result = fixMetadataJson(input);
+ expect(isStringMetadataJson(result)).toBe(true);
+ const parsed = JSON.parse(result);
+ expect(parsed.database_name).toBe('test_db');
+ });
+ });
+
describe('edge cases', () => {
it('should handle empty objects', () => {
const input = '{}';
@@ -311,6 +409,30 @@ describe('isStringMetadataJson', () => {
});
});
+describe('CSV-exported Snowflake output', () => {
+ it('should handle Snowflake Snowsight CSV export with "" quoting', () => {
+ // Simulates CSV export: outer quotes, all " escaped as ""
+ const csvInput = `"{ ""fk_info"": [], ""pk_info"": [{ ""schema"": ""PUBLIC"", ""table"": ""users"", ""column"": ""id"", ""pk_def"": ""pk_users"" }], ""columns"": [{ ""schema"": ""PUBLIC"", ""table"": ""users"", ""name"": ""id"", ""type"": ""NUMBER"", ""ordinal_position"": 1, ""nullable"": false, ""character_maximum_length"": null, ""precision"": { ""precision"": 38, ""scale"": 0 }, ""default"": null, ""collation"": null, ""comment"": null, ""is_identity"": true }], ""indexes"": [], ""tables"": [{ ""schema"": ""PUBLIC"", ""table"": ""users"", ""rows"": 100, ""type"": ""BASE TABLE"", ""engine"": """", ""collation"": """", ""comment"": """" }], ""views"": [], ""database_name"": ""MY_DB"", ""version"": ""10.11.1"" }"`;
+
+ const fixed = fixMetadataJson(csvInput);
+ expect(isStringMetadataJson(fixed)).toBe(true);
+ const parsed = JSON.parse(fixed);
+ expect(parsed.database_name).toBe('MY_DB');
+ expect(parsed.pk_info[0].schema).toBe('PUBLIC');
+ expect(parsed.columns[0].type).toBe('NUMBER');
+ });
+
+ it('should handle CSV export with embedded quotes in clustering keys', () => {
+ const csvInput = `"{ ""fk_info"": [], ""pk_info"": [], ""columns"": [], ""indexes"": [{ ""schema"": ""PUBLIC"", ""table"": ""snap_lot"", ""name"": ""snap_lot_cluster"", ""column"": ""LINEAR(DIV0(\\""snaplot_id\\"", 500000))"", ""index_type"": ""CLUSTERING"", ""cardinality"": 0, ""size"": 0, ""unique"": false, ""column_position"": 1, ""direction"": ""asc"" }], ""tables"": [], ""views"": [], ""database_name"": ""TEST_DB"", ""version"": ""10.11.1"" }"`;
+
+ const fixed = fixMetadataJson(csvInput);
+ const parsed = JSON.parse(fixed);
+ expect(parsed.indexes[0].column).toBe(
+ 'LINEAR(DIV0("snaplot_id", 500000))'
+ );
+ });
+});
+
describe('fixMetadataJson + isStringMetadataJson integration', () => {
it('should fix and validate MySQL smart query output', () => {
const brokenInput = `{\\n \\"fk_info\\": [],\\n \\"pk_info\\": [],\\n \\"columns\\": [],\\n \\"indexes\\": [],\\n \\"tables\\": [],\\n \\"views\\": [],\\n \\"database_name\\": \\"testdb\\",\\n \\"version\\": \\"8.0.39\\"\\n}`;
diff --git a/src/lib/data/import-metadata/utils.ts b/src/lib/data/import-metadata/utils.ts
index 6efed71f..e29f2209 100644
--- a/src/lib/data/import-metadata/utils.ts
+++ b/src/lib/data/import-metadata/utils.ts
@@ -1,5 +1,56 @@
import { isDatabaseMetadata } from './metadata-types/database-metadata';
+const applyCommonFixups = (json: string): string => {
+ return (
+ json
+ .replace(/\r/g, '') // Strip carriage returns (invalid control char in JSON strings)
+ .replace(/\n/g, '')
+ // Strip terminal-output decoration characters (e.g. ClickHouse client's
+ // ↴│↳ arrows and box-drawing chars that get included in copy-paste).
+ .replace(/[\u2190-\u21FF\u2500-\u257F]/g, '')
+ .replace(/,(\s*[\]}])/g, '$1') // Remove trailing commas (common JS-to-JSON mistake)
+ .replace(/(\[)\s*,/g, '$1') // Remove leading commas in arrays (from NULL IFNULL templates)
+ .replace(/"precision": "null"/g, '"precision": null')
+ .replace(/"nullable": "false"/g, '"nullable": false')
+ .replace(/"nullable": "true"/g, '"nullable": true')
+ );
+};
+
+// Extract inner JSON from { "metadata_json_to_import": "..." } wrapper.
+// The wrapper format has the inner JSON's quotes unescaped, making the outer
+// object invalid. We find the inner { ... } using brace counting.
+const extractMetadataWrapper = (payload: string): string | null => {
+ const wrapperKey = 'metadata_json_to_import';
+ if (!payload.includes(wrapperKey)) return null;
+
+ const keyIdx = payload.indexOf(wrapperKey);
+ const startIdx = payload.indexOf('{', keyIdx + wrapperKey.length);
+ if (startIdx === -1) return null;
+
+ let depth = 0;
+ let inString = false;
+ let escape = false;
+ for (let i = startIdx; i < payload.length; i++) {
+ const ch = payload[i];
+ if (escape) {
+ escape = false;
+ continue;
+ }
+ if (ch === '\\') {
+ escape = true;
+ continue;
+ }
+ if (ch === '"') inString = !inString;
+ if (inString) continue;
+ if (ch === '{') depth++;
+ else if (ch === '}') {
+ depth--;
+ if (depth === 0) return payload.slice(startIdx, i + 1);
+ }
+ }
+ return null;
+};
+
export const fixMetadataJson = (metadataJson: string): string => {
// Replace problematic array default values with null
metadataJson = metadataJson.replace(
@@ -13,8 +64,57 @@ export const fixMetadataJson = (metadataJson: string): string => {
'"default": "$1"$2'
);
- // TODO: remove this temporary eslint disable
- return (
+ // Try minimal cleanup first — this preserves valid JSON escaping (e.g. \" in
+ // string values like Snowflake clustering keys with quoted identifiers).
+ // The destructive \" → " replacement below would break these.
+ const minimalCleaned = applyCommonFixups(
+ metadataJson
+ .trim()
+ .replace(/^[^{]*/, '')
+ .replace(/}[^}]*$/, '}')
+ );
+
+ try {
+ JSON.parse(minimalCleaned);
+ return minimalCleaned;
+ } catch {
+ // Not valid JSON as-is, try other formats below
+ }
+
+ // Try CSV unwrap — Snowflake Snowsight CSV exports wrap the JSON in quotes
+ // and escape all internal " as "". Unwrapping "" → " produces valid JSON
+ // while preserving legitimate \" escaping (e.g. clustering keys).
+ const csvUnwrapped = applyCommonFixups(
+ metadataJson
+ .trim()
+ .replace(/^[^{]*/, '')
+ .replace(/}[^}]*$/, '}')
+ .replace(/""/g, '"')
+ );
+
+ try {
+ JSON.parse(csvUnwrapped);
+ return csvUnwrapped;
+ } catch {
+ // Not CSV format, try wrapper extraction
+ }
+
+ // Try metadata_json_to_import wrapper extraction — some users paste the
+ // result wrapped in `[{"metadata_json_to_import": "{...inner json...}"}]`
+ // where the inner JSON's quotes are not escaped, making the outer object
+ // invalid. Extract the inner object directly via brace counting.
+ const innerJson = extractMetadataWrapper(metadataJson);
+ if (innerJson) {
+ const innerCleaned = applyCommonFixups(innerJson);
+ try {
+ JSON.parse(innerCleaned);
+ return innerCleaned;
+ } catch {
+ // Inner JSON also broken, fall through
+ }
+ }
+
+ return applyCommonFixups(
metadataJson
.trim()
// First unescape the JSON string
@@ -48,20 +148,12 @@ export const fixMetadataJson = (metadataJson: string): string => {
/* eslint-disable-next-line no-useless-escape */
.replace(/'([^']+)'::\"([^\"]+)\"/g, '\'$1\'::\\\"$2\\\"')
- // Convert string "null" to actual null for precision field
- .replace(/"precision": "null"/g, '"precision": null')
-
- // Convert string "true"/"false" to actual boolean for nullable field
- .replace(/"nullable": "false"/g, '"nullable": false')
- .replace(/"nullable": "true"/g, '"nullable": true')
-
/* eslint-disable-next-line no-useless-escape */
.replace(/\"/g, '___ESCAPED_QUOTE___') // Temporarily replace empty strings
.replace(/(:\s*)""(?=\s*[,}])/g, '$1___EMPTY___') // Temporarily replace empty strings (Safari-compatible)
.replace(/""/g, '"') // Replace remaining double quotes
.replace(/___ESCAPED_QUOTE___/g, '"') // Restore empty strings
.replace(/___EMPTY___/g, '""') // Restore empty strings
- .replace(/\n/g, '')
);
};