Skip to content

Privacy-first password manager with built-in email aliasing. Fully encrypted and self-hostable.

Update CsvParser robustness checks (#2413)

Leendert de Borst879b8ebf2fd7
diff --git a/core/client/src/transfer/import/readers/CsvParser.ts b/core/client/src/transfer/import/readers/CsvParser.ts
index d841eff72..026d68d1a 100644
--- a/core/client/src/transfer/import/readers/CsvParser.ts
+++ b/core/client/src/transfer/import/readers/CsvParser.ts
@@ -3,27 +3,79 @@
*/
/**
- * Split CSV text into rows of fields. Blank lines are skipped, a byte order mark is ignored.
+ * The delimiters a file may use: a comma, or a semicolon as spreadsheet apps write for locales with a decimal comma.
+ */
+const CANDIDATE_DELIMITERS = [',', ';'] as const;
+
+/**
+ * How many rows the delimiter detection looks at.
+ */
+const DETECTION_ROW_COUNT = 10;
+
+/**
+ * Split CSV text into rows of fields. The delimiter (comma or semicolon) is detected, blank lines are skipped and a
+ * byte order mark is ignored.
* @param content - The CSV text
* @returns The rows, each an array of field values
+ * @throws {Error} When a quoted field is never closed, which would otherwise swallow every row after it.
*/
export function parseCsvRows(content: string): string[][] {
const text = content.charCodeAt(0) === 0xFEFF ? content.substring(1) : content;
+ const { rows, unterminatedQuote } = splitRows(text, detectDelimiter(text));
+ if (unterminatedQuote) {
+ throw new Error(`The CSV file is malformed: a quoted field that starts on row ${rows.length + 1} is never closed.`);
+ }
+ return rows;
+}
+
+/**
+ * Pick the delimiter the first rows split on most consistently.
+ * @param text - The CSV text
+ * @returns The delimiter
+ */
+function detectDelimiter(text: string): string {
+ let best: { delimiter: string; delta: number } | null = null;
+ for (const delimiter of CANDIDATE_DELIMITERS) {
+ const { rows } = splitRows(text, delimiter, DETECTION_ROW_COUNT);
+ if (rows.length === 0) {
+ continue;
+ }
+
+ const averageFieldCount = rows.reduce((sum, row) => sum + row.length, 0) / rows.length;
+ const delta = rows.slice(1).reduce((sum, row, index) => sum + Math.abs(row.length - rows[index].length), 0);
+ if (averageFieldCount >= 2 && (!best || delta < best.delta)) {
+ best = { delimiter, delta };
+ }
+ }
+ return best?.delimiter ?? ',';
+}
+
+/**
+ * Split CSV text into rows of fields on the given delimiter.
+ * @param text - The CSV text, without a byte order mark
+ * @param delimiter - The field delimiter
+ * @param maxRows - Stop after this many rows
+ * @returns The rows, and whether the text ended inside a quoted field
+ */
+function splitRows(text: string, delimiter: string, maxRows = Infinity): { rows: string[][]; unterminatedQuote: boolean } {
const rows: string[][] = [];
let row: string[] = [];
let field = '';
let i = 0;
- while (i < text.length) {
+ while (i < text.length && rows.length < maxRows) {
const c = text[i];
if (c === '"' && field === '') {
i++;
- while (i < text.length && !(text[i] === '"' && isFieldEnd(text, i + 1))) {
+ while (i < text.length && !(text[i] === '"' && isFieldEnd(text, i + 1, delimiter))) {
field += text[i];
i += text[i] === '"' && text[i + 1] === '"' ? 2 : 1;
}
+ if (i >= text.length) {
+ return { rows, unterminatedQuote: true };
+ }
i++;
- } else if (c === ',') {
+ } else if (c === delimiter) {
row.push(field);
field = '';
i++;
@@ -40,19 +92,20 @@ export function parseCsvRows(content: string): string[][] {
i++;
}
}
- if (row.length > 0 || field !== '') {
+ if (rows.length < maxRows && (row.length > 0 || field !== '')) {
row.push(field);
rows.push(row);
}
- return rows;
+ return { rows, unterminatedQuote: false };
}
/**
* Whether the field ends at `i`: a delimiter, a line break or the end of the text.
* @param text - The CSV text
* @param i - The index to check
+ * @param delimiter - The field delimiter
* @returns True when a field ends there
*/
-function isFieldEnd(text: string, i: number): boolean {
- return i >= text.length || text[i] === ',' || text[i] === '\r' || text[i] === '\n';
+function isFieldEnd(text: string, i: number, delimiter: string): boolean {
+ return i >= text.length || text[i] === delimiter || text[i] === '\r' || text[i] === '\n';
}
diff --git a/core/client/src/transfer/import/readers/__tests__/CsvParser.test.ts b/core/client/src/transfer/import/readers/__tests__/CsvParser.test.ts
new file mode 100644
index 000000000..1716988d6
--- /dev/null
+++ b/core/client/src/transfer/import/readers/__tests__/CsvParser.test.ts
@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+
+import { parseCsvRows } from '../CsvParser';
+
+describe('parseCsvRows', () => {
+ it('parses quoted fields with escaped quotes, delimiters and line breaks', () => {
+ const rows = parseCsvRows('name,notes\r\nA,"say ""hi"", then\nbye"\n\nB,plain\n');
+ expect(rows).toEqual([['name', 'notes'], ['A', 'say "hi", then\nbye'], ['B', 'plain']]);
+ });
+
+ it('throws when a quoted field is never closed', () => {
+ expect(() => parseCsvRows('name,password\nA,"secret\nB,pw2\nC,pw3\n')).toThrow(/row 2 is never closed/);
+ });
+
+ it('detects a semicolon delimiter', () => {
+ const rows = parseCsvRows('name;url;notes\nA;https://a.example;one, two\nB;https://b.example;"x;y"\n');
+ expect(rows).toEqual([['name', 'url', 'notes'], ['A', 'https://a.example', 'one, two'], ['B', 'https://b.example', 'x;y']]);
+ });
+
+ it('keeps the comma when semicolons only appear inside fields', () => {
+ const rows = parseCsvRows('name,url,notes\nA,https://a.example,one; two\nB,https://b.example,three; four; five\n');
+ expect(rows).toEqual([['name', 'url', 'notes'], ['A', 'https://a.example', 'one; two'], ['B', 'https://b.example', 'three; four; five']]);
+ });
+
+ it('prefers the comma when both delimiters split consistently', () => {
+ expect(parseCsvRows('a;b,c;d\ne;f,g;h\n')).toEqual([['a;b', 'c;d'], ['e;f', 'g;h']]);
+ });
+
+ it('falls back to the comma for a single column', () => {
+ expect(parseCsvRows('name\nA\n')).toEqual([['name'], ['A']]);
+ });
+});