Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): read the separator a CSV was written with (#19740)

Lukas3c8205d89fbb
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py
index f5cd7265b6..8cf2365152 100644
--- a/deepdoc/parser/excel_parser.py
+++ b/deepdoc/parser/excel_parser.py
@@ -11,6 +11,7 @@
# limitations under the License.
#
+import csv
import logging
import re
import sys
@@ -25,6 +26,66 @@ from rag.utils.lazy_image import LazyImage
# copied from `/openpyxl/cell/cell.py`
ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]")
+# The separators a spreadsheet actually writes into a file named ".csv". Excel
+# writes the list separator of the machine's locale, which is a semicolon across
+# most of Europe, and a tab separated export is routinely saved as .csv.
+CSV_DELIMITERS = (",", ";", "\t", "|")
+
+# How much of the file the detection looks at. A separator that holds for the
+# first rows holds for the file.
+CSV_SAMPLE_CHARS = 64 * 1024
+CSV_SAMPLE_ROWS = 20
+
+
+def _consistent_column_count(sample, delimiter, truncated):
+ """Columns per row under `delimiter`, or 0 when the rows disagree.
+
+ A separator the file was not written with either does not occur at all (one
+ column) or occurs by accident, and then the rows do not line up. Requiring
+ the same count on every row is what keeps a comma inside a sentence, or a
+ semicolon inside a quoted field, from being read as a separator.
+
+ When `truncated` is set, the sample is a prefix of a longer file, so the row
+ it ends in stops wherever the read did, between two fields or inside a
+ quoted one. That row is left out rather than counted as having fewer columns,
+ also when it is the last of the CSV_SAMPLE_ROWS rows sampled.
+ """
+ rows = []
+ reader = csv.reader(StringIO(sample, newline=""), delimiter=delimiter)
+ try:
+ for row in reader:
+ if any(cell.strip() for cell in row): # a blank or whitespace-only line says nothing
+ rows.append(row)
+ if len(rows) == CSV_SAMPLE_ROWS:
+ break
+ except csv.Error:
+ return 0
+ if truncated and len(rows) > 1 and reader.line_num == len(StringIO(sample, newline="").readlines()):
+ rows.pop()
+ count = 0
+ for row in rows:
+ if count and len(row) != count:
+ return 0
+ count = len(row)
+ return count if count > 1 else 0
+
+
+def detect_csv_delimiter(text):
+ """The separator `text` was written with, defaulting to a comma.
+
+ `pandas.read_csv` and `csv.reader` both default to a comma, and reading a
+ semicolon separated export with one does not fail: it returns a single
+ column holding the whole row, separators included.
+ """
+ truncated = len(text) > CSV_SAMPLE_CHARS
+ sample = text[:CSV_SAMPLE_CHARS]
+ best_delimiter, best_columns = ",", 0
+ for delimiter in CSV_DELIMITERS:
+ columns = _consistent_column_count(sample, delimiter, truncated)
+ if columns > best_columns:
+ best_delimiter, best_columns = delimiter, columns
+ return best_delimiter
+
class RAGFlowExcelParser:
@staticmethod
@@ -36,7 +97,7 @@ class RAGFlowExcelParser:
file_like_object.seek(0)
binary = file_like_object.read()
text, _ = decode_text(binary, document_type="CSV document")
- return pd.read_csv(StringIO(text), on_bad_lines="skip")
+ return pd.read_csv(StringIO(text), sep=detect_csv_delimiter(text), on_bad_lines="skip")
@staticmethod
def _load_excel_to_workbook(file_like_object):
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 2a53e9ef80..156e0ae9d5 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -28,10 +28,24 @@ import (
"encoding/csv"
"fmt"
"strings"
+ "unicode/utf8"
)
const csvSheetName = "Data"
+// csvDelimiters are the separators a spreadsheet actually writes into a file
+// named ".csv". Excel writes the list separator of the machine's locale, which
+// is a semicolon across most of Europe, and a tab separated export is routinely
+// saved as .csv. Mirrors CSV_DELIMITERS in deepdoc/parser/excel_parser.py.
+var csvDelimiters = []rune{',', ';', '\t', '|'}
+
+// How much of the file the separator detection looks at. A separator that
+// holds for the first rows holds for the file.
+const (
+ csvSampleBytes = 64 * 1024
+ csvSampleRows = 20
+)
+
// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.
type CSVParser struct {
ParseMethod string
@@ -125,12 +139,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
}
}
- reader := csv.NewReader(strings.NewReader(text))
- reader.LazyQuotes = true
- reader.TrimLeadingSpace = true
- reader.FieldsPerRecord = -1 // Allow variable column counts, matching Python csv.reader behaviour.
-
- records, err := reader.ReadAll()
+ records, err := newCSVReader(text, detectCSVDelimiter(text)).ReadAll()
if err != nil {
return ParseResult{Err: fmt.Errorf("csv parse: %w", err)}
}
@@ -162,3 +171,88 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
JSON: items,
}
}
+
+// newCSVReader reads text with the given separator, leniently, the same way for
+// the separator detection and for the rows themselves.
+func newCSVReader(text string, comma rune) *csv.Reader {
+ reader := csv.NewReader(strings.NewReader(text))
+ reader.Comma = comma
+ reader.LazyQuotes = true
+ // TrimLeadingSpace also trims a tab when the tab is the separator, which
+ // would swallow the empty field in "Widget\t\t12".
+ reader.TrimLeadingSpace = comma != '\t'
+ reader.FieldsPerRecord = -1 // Allow variable column counts, matching Python csv.reader behaviour.
+ return reader
+}
+
+// detectCSVDelimiter returns the separator text was written with, defaulting
+// to a comma. encoding/csv reads with a comma by default, and reading a
+// semicolon separated export that way does not fail: every row becomes a
+// single column holding the whole line, separators included.
+func detectCSVDelimiter(text string) rune {
+ sample, truncated := text, false
+ if len(text) > csvSampleBytes {
+ cut := csvSampleBytes
+ for cut > 0 && !utf8.RuneStart(text[cut]) {
+ cut--
+ }
+ sample, truncated = text[:cut], true
+ }
+ best, bestColumns := ',', 0
+ for _, delimiter := range csvDelimiters {
+ if columns := consistentColumnCount(sample, delimiter, truncated); columns > bestColumns {
+ best, bestColumns = delimiter, columns
+ }
+ }
+ return best
+}
+
+// consistentColumnCount returns the columns per row under delimiter, or 0 when
+// the rows disagree. A separator the file was not written with either does not
+// occur at all (one column) or occurs by accident, and then the rows do not
+// line up. Requiring the same count on every row is what keeps a comma inside a
+// sentence, or a semicolon inside a quoted field, from being read as a
+// separator.
+//
+// When truncated is set, the sample is a prefix of a longer file, so the row it
+// ends in stops wherever the read did, between two fields or inside a quoted
+// one. That row is left out rather than counted as having fewer columns, also
+// when it is the last of the csvSampleRows rows sampled.
+func consistentColumnCount(sample string, delimiter rune, truncated bool) int {
+ reader := newCSVReader(sample, delimiter)
+ var rows [][]string
+ for len(rows) < csvSampleRows {
+ row, err := reader.Read()
+ if err != nil {
+ break
+ }
+ if !isBlankCSVRow(row) {
+ rows = append(rows, row)
+ }
+ }
+ if truncated && len(rows) > 1 && reader.InputOffset() == int64(len(sample)) {
+ rows = rows[:len(rows)-1]
+ }
+ count := 0
+ for _, row := range rows {
+ if count != 0 && len(row) != count {
+ return 0
+ }
+ count = len(row)
+ }
+ if count > 1 {
+ return count
+ }
+ return 0
+}
+
+// isBlankCSVRow reports whether every cell of row is empty or whitespace; such
+// a row says nothing about the separator.
+func isBlankCSVRow(row []string) bool {
+ for _, cell := range row {
+ if strings.TrimSpace(cell) != "" {
+ return false
+ }
+ }
+ return true
+}
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index ac2f4da99d..8d1a0fc311 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -2,6 +2,8 @@ package parser
import (
"context"
+ "fmt"
+ "strings"
"testing"
)
@@ -136,3 +138,83 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {
t.Fatalf("second data row width = %d, want 3", got)
}
}
+
+func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {
+ // A semicolon, tab or pipe separated .csv used to come out as one column
+ // holding the whole line.
+ for _, sep := range []string{",", ";", "\t", "|"} {
+ data := []byte("Name" + sep + "Region" + sep + "Units\nWidget" + sep + "EU" + sep + "12\n")
+ res := NewCSVParser().ParseWithResult(context.Background(), "export.csv", data)
+ if res.Err != nil {
+ t.Fatalf("separator %q: ParseWithResult failed: %v", sep, res.Err)
+ }
+ if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Region:EU; Units:12 ——Data" {
+ t.Errorf("separator %q: row text = %v", sep, got)
+ }
+ }
+}
+
+func TestDetectCSVDelimiter(t *testing.T) {
+ cases := []struct {
+ name string
+ text string
+ want rune
+ }{
+ {"comma inside a sentence", "Name;Note\nWidget;Red, blue and green\nCable;Short\n", ';'},
+ {"semicolon inside a quoted field", "Name,Note\nWidget,\"a;b\"\nCable,\"c;d;e\"\n", ','},
+ {"single column", "Name\nWidget\nCable\n", ','},
+ {"whitespace-only row", "Name;Region\n \nWidget;EU\n", ';'},
+ {"whitespace-only lines before the header", strings.Repeat(" \n", csvSampleRows) + "Name;Region\nWidget;EU\n", ';'},
+ {"rows disagree under every separator", "Name,Note\nWidget\n", ','},
+ }
+ for _, tc := range cases {
+ if got := detectCSVDelimiter(tc.text); got != tc.want {
+ t.Errorf("%s: detectCSVDelimiter = %q, want %q", tc.name, got, tc.want)
+ }
+ }
+}
+
+func TestDetectCSVDelimiter_LeavesOutTheRowTheSampleCutsShort(t *testing.T) {
+ // The sample ends right after "Cable;E", so its last row has two fields
+ // where the complete rows have three.
+ header := "Name;Region;Units\n"
+ filler := "Widget;EU;" + strings.Repeat("9", csvSampleBytes-len(header)-len("Widget;EU;\nCable;E")) + "\n"
+ text := header + filler + "Cable;EU;12\n"
+ if !strings.HasSuffix(text[:csvSampleBytes], "\nCable;E") {
+ t.Fatalf("sample does not end inside the third row: %q", text[csvSampleBytes-12:csvSampleBytes])
+ }
+ if got := detectCSVDelimiter(text); got != ';' {
+ t.Errorf("detectCSVDelimiter = %q, want ';'", got)
+ }
+}
+
+func TestDetectCSVDelimiter_LeavesOutTheLastSampledRowWhenTheCutEndsIt(t *testing.T) {
+ // csvSampleRows rows fit in the sample only because the last of them is
+ // cut short: "18;xxx" has two fields where the others have three.
+ cell := strings.Repeat("x", 3600)
+ var b strings.Builder
+ b.WriteString("a;b;c\n")
+ for i := 0; i < 40; i++ {
+ fmt.Fprintf(&b, "%d;%s;end\n", i, cell)
+ }
+ text := b.String()
+ if n := strings.Count(text[:csvSampleBytes], "\n"); n != csvSampleRows-1 {
+ t.Fatalf("sample holds %d complete rows, want %d", n, csvSampleRows-1)
+ }
+ if got := detectCSVDelimiter(text); got != ';' {
+ t.Errorf("detectCSVDelimiter = %q, want ';'", got)
+ }
+}
+
+func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {
+ // Trimming leading space would also trim the tab after "Widget", leaving
+ // the row a field short of its header, and the tab would lose to the comma.
+ data := []byte("Name\tRegion\tUnits\nWidget\t\t12\n")
+ res := NewCSVParser().ParseWithResult(context.Background(), "export.csv", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+ if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Units:12 ——Data" {
+ t.Errorf("row text = %v", got)
+ }
+}
diff --git a/rag/app/table.py b/rag/app/table.py
index 5b72eff59c..802e399b45 100644
--- a/rag/app/table.py
+++ b/rag/app/table.py
@@ -35,6 +35,7 @@ from common.constants import MAXIMUM_TASK_PAGE_NUMBER
from deepdoc.parser.utils import get_text
from rag.nlp import rag_tokenizer, tokenize, tokenize_table
from deepdoc.parser import ExcelParser
+from deepdoc.parser.excel_parser import detect_csv_delimiter
from common import settings
logger = logging.getLogger(__name__)
@@ -443,7 +444,8 @@ def column_data_type(arr):
def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER, lang="Chinese", callback=None, **kwargs):
"""
Excel and csv(txt) format files are supported.
- For csv or txt file, the delimiter between columns is TAB.
+ For a txt file the delimiter between columns is TAB; for a csv file it is
+ detected from the file itself, and can be forced with the `delimiter` kwarg.
The first line must be column headers.
Column headers must be meaningful terms inorder to make our NLP model understanding.
It's good to enumerate some synonyms using slash '/' to separate, and even better to
@@ -497,7 +499,10 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
elif re.search(r"\.csv$", filename, re.IGNORECASE):
callback(0.1, "Start to parse.")
txt = get_text(filename, binary)
- delimiter = kwargs.get("delimiter", ",")
+ # A .csv is not always comma separated, and reading one with the wrong
+ # separator does not fail -- every row becomes a single column holding
+ # the whole line.
+ delimiter = kwargs.get("delimiter") or detect_csv_delimiter(txt)
reader = csv.reader(io.StringIO(txt), delimiter=delimiter)
all_rows = list(reader)
diff --git a/test/unit_test/deepdoc/parser/test_csv_delimiter.py b/test/unit_test/deepdoc/parser/test_csv_delimiter.py
new file mode 100644
index 0000000000..bee09c6d55
--- /dev/null
+++ b/test/unit_test/deepdoc/parser/test_csv_delimiter.py
@@ -0,0 +1,152 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A .csv file is not always comma separated.
+
+Excel writes the list separator of the machine's locale -- a semicolon across
+most of Europe -- and a tab separated export is routinely saved as .csv. Both
+`pandas.read_csv` and `csv.reader` default to a comma, and reading such a file
+with one does not fail: every row becomes a single column holding the whole
+line, separators included.
+"""
+
+import importlib.util
+import sys
+from io import BytesIO
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+_REPO_ROOT = Path(__file__).resolve().parents[4]
+
+_MOCK_MODULES = [
+ "xgboost",
+ "pdfplumber",
+ "huggingface_hub",
+ "pypdf",
+ "sklearn",
+ "sklearn.cluster",
+ "sklearn.metrics",
+ "deepdoc.vision",
+]
+for _name in _MOCK_MODULES:
+ sys.modules.setdefault(_name, mock.MagicMock())
+
+
+def _load_excel_parser():
+ """Load excel_parser.py by path, so deepdoc/parser/__init__.py stays out of it."""
+ spec = importlib.util.spec_from_file_location(
+ "deepdoc.parser.excel_parser_under_test",
+ _REPO_ROOT / "deepdoc" / "parser" / "excel_parser.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+excel_parser = _load_excel_parser()
+detect_csv_delimiter = excel_parser.detect_csv_delimiter
+RAGFlowExcelParser = excel_parser.RAGFlowExcelParser
+
+ROWS = ["Name;Region;Units", "Widget;EU;12", "Gadget;US;7"]
+
+
+def _text(delimiter):
+ return "\n".join(row.replace(";", delimiter) for row in ROWS) + "\n"
+
+
[email protected]("delimiter", [",", ";", "\t", "|"])
+def test_detect_csv_delimiter(delimiter):
+ assert detect_csv_delimiter(_text(delimiter)) == delimiter
+
+
+ "text",
+ [
+ # A separator inside a quoted field is not a separator.
+ 'Name,Note\nWidget,"a; b"\nGadget,"c; d"\n',
+ # Rows that do not line up leave the default in place.
+ "Note\na; b\nc; d\n",
+ "name,value\nAlice,1\nBob,2,extra\n",
+ "",
+ ],
+)
+def test_detect_csv_delimiter_falls_back_to_a_comma(text):
+ assert detect_csv_delimiter(text) == ","
+
+
+def test_a_whitespace_only_line_is_not_a_row():
+ assert detect_csv_delimiter("Name;Region\nWidget;EU\n \nGadget;US\n") == ";"
+
+
+def test_the_row_the_sample_cuts_is_left_out():
+ """Fewer than CSV_SAMPLE_ROWS rows fit in the sample, so it ends inside a row."""
+ cell = "x" * 4000
+ text = "a;b;c\n" + "".join(f"{i};{cell};{cell}\n" for i in range(60))
+
+ assert detect_csv_delimiter(text) == ";"
+
+
+def test_the_last_sampled_row_is_left_out_when_the_sample_cuts_it():
+ """CSV_SAMPLE_ROWS rows fit only because the last of them is cut short."""
+ cell = "x" * 3600
+ text = "a;b;c\n" + "".join(f"{i};{cell};end\n" for i in range(40))
+ assert text[: excel_parser.CSV_SAMPLE_CHARS].count("\n") == excel_parser.CSV_SAMPLE_ROWS - 1
+
+ assert detect_csv_delimiter(text) == ";"
+
+
+def test_leading_blank_lines_do_not_use_up_the_sample():
+ text = "\n" * (excel_parser.CSV_SAMPLE_ROWS + 5) + _text(";")
+
+ assert detect_csv_delimiter(text) == ";"
+ workbook = RAGFlowExcelParser._read_csv(BytesIO(text.encode("utf-8")))
+ assert list(workbook.columns) == ["Name", "Region", "Units"]
+
+
+def test_the_sample_can_end_inside_a_quoted_line_break():
+ cell = '"' + ("y" * 70 + "\n") * 60 + '"'
+ text = "a;b;c\n" + "".join(f"{i};{cell};end\n" for i in range(40))
+
+ assert detect_csv_delimiter(text) == ";"
+
+
[email protected]("delimiter", [",", ";", "\t", "|"])
+def test_a_csv_keeps_its_columns_whatever_the_separator(delimiter):
+ """`_read_csv` is the path a .csv upload takes through the Excel parser."""
+ workbook = RAGFlowExcelParser._read_csv(BytesIO(_text(delimiter).encode("utf-8")))
+
+ assert list(workbook.columns) == ["Name", "Region", "Units"]
+ assert workbook.iloc[0].tolist() == ["Widget", "EU", 12]
+
+
+def test_a_single_column_csv_stays_a_single_column():
+ """Guard: a separator that does not line up across the rows is not one."""
+ workbook = RAGFlowExcelParser._read_csv(BytesIO(b"Note\na; b\nc; d\n"))
+
+ assert list(workbook.columns) == ["Note"]
+ assert workbook["Note"].tolist() == ["a; b", "c; d"]
diff --git a/test/unit_test/rag/app/test_table_chunk_column_roles.py b/test/unit_test/rag/app/test_table_chunk_column_roles.py
index ed37a8df78..22dfb442ec 100644
--- a/test/unit_test/rag/app/test_table_chunk_column_roles.py
+++ b/test/unit_test/rag/app/test_table_chunk_column_roles.py
@@ -409,3 +409,32 @@ def test_chunk_keeps_unconvertible_cell_in_content(table_module, mock_update_kb:
assert chunks[2]["amount_long"] == 300
assert chunks[1].get("amount_long") != "N/A"
assert "- amount: N/A" in chunks[1]["content_with_weight"]
+
+
[email protected]("delimiter", [";", "\t", "|"])
+def test_chunk_reads_the_separator_a_csv_was_written_with(table_module, mock_update_kb: MagicMock, delimiter):
+ table_module.chunk(
+ FILENAME,
+ binary=TEST_CSV.replace(b",", delimiter.encode()),
+ callback=_noop_callback,
+ kb_id=KB_ID,
+ parser_config={},
+ lang="Chinese",
+ )
+ args, _ = mock_update_kb.call_args
+ assert args[1]["table_column_names"] == ["row_id", "title", "content", "country", "category"]
+
+
+def test_chunk_uses_an_explicit_delimiter_over_detection(table_module, mock_update_kb: MagicMock):
+ # Detection picks ";" here, which lines up on every row too.
+ table_module.chunk(
+ FILENAME,
+ binary=b"a|b;c\n1|2;3\n4|5;6\n",
+ callback=_noop_callback,
+ kb_id=KB_ID,
+ parser_config={},
+ lang="Chinese",
+ delimiter="|",
+ )
+ args, _ = mock_update_kb.call_args
+ assert args[1]["table_column_names"] == ["a", "b;c"]