RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): read the separator a CSV was written with (#19740)
Lukas3c8205d89fbb
diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.pyindex f5cd7265b6..8cf2365152 100644--- a/deepdoc/parser/excel_parser.py+++ b/deepdoc/parser/excel_parser.py@@ -11,6 +11,7 @@# limitations under the License.#+import csvimport loggingimport reimport sys@@ -25,6 +26,66 @@ from rag.utils.lazy_image import LazyImage# copied from `/openpyxl/cell/cell.py`ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]")+# The separators a spreadsheet actually writes into a file named ".csv". Excel+# writes the list separator of the machine's locale, which is a semicolon across+# most of Europe, and a tab separated export is routinely saved as .csv.+CSV_DELIMITERS = (",", ";", "\t", "|")++# How much of the file the detection looks at. A separator that holds for the+# first rows holds for the file.+CSV_SAMPLE_CHARS = 64 * 1024+CSV_SAMPLE_ROWS = 20+++def _consistent_column_count(sample, delimiter, truncated):+ """Columns per row under `delimiter`, or 0 when the rows disagree.++ A separator the file was not written with either does not occur at all (one+ column) or occurs by accident, and then the rows do not line up. Requiring+ the same count on every row is what keeps a comma inside a sentence, or a+ semicolon inside a quoted field, from being read as a separator.++ When `truncated` is set, the sample is a prefix of a longer file, so the row+ it ends in stops wherever the read did, between two fields or inside a+ quoted one. That row is left out rather than counted as having fewer columns,+ also when it is the last of the CSV_SAMPLE_ROWS rows sampled.+ """+ rows = []+ reader = csv.reader(StringIO(sample, newline=""), delimiter=delimiter)+ try:+ for row in reader:+ if any(cell.strip() for cell in row): # a blank or whitespace-only line says nothing+ rows.append(row)+ if len(rows) == CSV_SAMPLE_ROWS:+ break+ except csv.Error:+ return 0+ if truncated and len(rows) > 1 and reader.line_num == len(StringIO(sample, newline="").readlines()):+ rows.pop()+ count = 0+ for row in rows:+ if count and len(row) != count:+ return 0+ count = len(row)+ return count if count > 1 else 0+++def detect_csv_delimiter(text):+ """The separator `text` was written with, defaulting to a comma.++ `pandas.read_csv` and `csv.reader` both default to a comma, and reading a+ semicolon separated export with one does not fail: it returns a single+ column holding the whole row, separators included.+ """+ truncated = len(text) > CSV_SAMPLE_CHARS+ sample = text[:CSV_SAMPLE_CHARS]+ best_delimiter, best_columns = ",", 0+ for delimiter in CSV_DELIMITERS:+ columns = _consistent_column_count(sample, delimiter, truncated)+ if columns > best_columns:+ best_delimiter, best_columns = delimiter, columns+ return best_delimiter+class RAGFlowExcelParser:@staticmethod@@ -36,7 +97,7 @@ class RAGFlowExcelParser:file_like_object.seek(0)binary = file_like_object.read()text, _ = decode_text(binary, document_type="CSV document")- return pd.read_csv(StringIO(text), on_bad_lines="skip")+ return pd.read_csv(StringIO(text), sep=detect_csv_delimiter(text), on_bad_lines="skip")@staticmethoddef _load_excel_to_workbook(file_like_object):diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 2a53e9ef80..156e0ae9d5 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -28,10 +28,24 @@ import ("encoding/csv""fmt""strings"+ "unicode/utf8")const csvSheetName = "Data"+// csvDelimiters are the separators a spreadsheet actually writes into a file+// named ".csv". Excel writes the list separator of the machine's locale, which+// is a semicolon across most of Europe, and a tab separated export is routinely+// saved as .csv. Mirrors CSV_DELIMITERS in deepdoc/parser/excel_parser.py.+var csvDelimiters = []rune{',', ';', '\t', '|'}++// How much of the file the separator detection looks at. A separator that+// holds for the first rows holds for the file.+const (+ csvSampleBytes = 64 * 1024+ csvSampleRows = 20+)+// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.type CSVParser struct {ParseMethod string@@ -125,12 +139,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [}}- reader := csv.NewReader(strings.NewReader(text))- reader.LazyQuotes = true- reader.TrimLeadingSpace = true- reader.FieldsPerRecord = -1 // Allow variable column counts, matching Python csv.reader behaviour.-- records, err := reader.ReadAll()+ records, err := newCSVReader(text, detectCSVDelimiter(text)).ReadAll()if err != nil {return ParseResult{Err: fmt.Errorf("csv parse: %w", err)}}@@ -162,3 +171,88 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [JSON: items,}}++// newCSVReader reads text with the given separator, leniently, the same way for+// the separator detection and for the rows themselves.+func newCSVReader(text string, comma rune) *csv.Reader {+ reader := csv.NewReader(strings.NewReader(text))+ reader.Comma = comma+ reader.LazyQuotes = true+ // TrimLeadingSpace also trims a tab when the tab is the separator, which+ // would swallow the empty field in "Widget\t\t12".+ reader.TrimLeadingSpace = comma != '\t'+ reader.FieldsPerRecord = -1 // Allow variable column counts, matching Python csv.reader behaviour.+ return reader+}++// detectCSVDelimiter returns the separator text was written with, defaulting+// to a comma. encoding/csv reads with a comma by default, and reading a+// semicolon separated export that way does not fail: every row becomes a+// single column holding the whole line, separators included.+func detectCSVDelimiter(text string) rune {+ sample, truncated := text, false+ if len(text) > csvSampleBytes {+ cut := csvSampleBytes+ for cut > 0 && !utf8.RuneStart(text[cut]) {+ cut--+ }+ sample, truncated = text[:cut], true+ }+ best, bestColumns := ',', 0+ for _, delimiter := range csvDelimiters {+ if columns := consistentColumnCount(sample, delimiter, truncated); columns > bestColumns {+ best, bestColumns = delimiter, columns+ }+ }+ return best+}++// consistentColumnCount returns the columns per row under delimiter, or 0 when+// the rows disagree. A separator the file was not written with either does not+// occur at all (one column) or occurs by accident, and then the rows do not+// line up. Requiring the same count on every row is what keeps a comma inside a+// sentence, or a semicolon inside a quoted field, from being read as a+// separator.+//+// When truncated is set, the sample is a prefix of a longer file, so the row it+// ends in stops wherever the read did, between two fields or inside a quoted+// one. That row is left out rather than counted as having fewer columns, also+// when it is the last of the csvSampleRows rows sampled.+func consistentColumnCount(sample string, delimiter rune, truncated bool) int {+ reader := newCSVReader(sample, delimiter)+ var rows [][]string+ for len(rows) < csvSampleRows {+ row, err := reader.Read()+ if err != nil {+ break+ }+ if !isBlankCSVRow(row) {+ rows = append(rows, row)+ }+ }+ if truncated && len(rows) > 1 && reader.InputOffset() == int64(len(sample)) {+ rows = rows[:len(rows)-1]+ }+ count := 0+ for _, row := range rows {+ if count != 0 && len(row) != count {+ return 0+ }+ count = len(row)+ }+ if count > 1 {+ return count+ }+ return 0+}++// isBlankCSVRow reports whether every cell of row is empty or whitespace; such+// a row says nothing about the separator.+func isBlankCSVRow(row []string) bool {+ for _, cell := range row {+ if strings.TrimSpace(cell) != "" {+ return false+ }+ }+ return true+}diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex ac2f4da99d..8d1a0fc311 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -2,6 +2,8 @@ package parserimport ("context"+ "fmt"+ "strings""testing")@@ -136,3 +138,83 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {t.Fatalf("second data row width = %d, want 3", got)}}++func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {+ // A semicolon, tab or pipe separated .csv used to come out as one column+ // holding the whole line.+ for _, sep := range []string{",", ";", "\t", "|"} {+ data := []byte("Name" + sep + "Region" + sep + "Units\nWidget" + sep + "EU" + sep + "12\n")+ res := NewCSVParser().ParseWithResult(context.Background(), "export.csv", data)+ if res.Err != nil {+ t.Fatalf("separator %q: ParseWithResult failed: %v", sep, res.Err)+ }+ if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Region:EU; Units:12 ——Data" {+ t.Errorf("separator %q: row text = %v", sep, got)+ }+ }+}++func TestDetectCSVDelimiter(t *testing.T) {+ cases := []struct {+ name string+ text string+ want rune+ }{+ {"comma inside a sentence", "Name;Note\nWidget;Red, blue and green\nCable;Short\n", ';'},+ {"semicolon inside a quoted field", "Name,Note\nWidget,\"a;b\"\nCable,\"c;d;e\"\n", ','},+ {"single column", "Name\nWidget\nCable\n", ','},+ {"whitespace-only row", "Name;Region\n \nWidget;EU\n", ';'},+ {"whitespace-only lines before the header", strings.Repeat(" \n", csvSampleRows) + "Name;Region\nWidget;EU\n", ';'},+ {"rows disagree under every separator", "Name,Note\nWidget\n", ','},+ }+ for _, tc := range cases {+ if got := detectCSVDelimiter(tc.text); got != tc.want {+ t.Errorf("%s: detectCSVDelimiter = %q, want %q", tc.name, got, tc.want)+ }+ }+}++func TestDetectCSVDelimiter_LeavesOutTheRowTheSampleCutsShort(t *testing.T) {+ // The sample ends right after "Cable;E", so its last row has two fields+ // where the complete rows have three.+ header := "Name;Region;Units\n"+ filler := "Widget;EU;" + strings.Repeat("9", csvSampleBytes-len(header)-len("Widget;EU;\nCable;E")) + "\n"+ text := header + filler + "Cable;EU;12\n"+ if !strings.HasSuffix(text[:csvSampleBytes], "\nCable;E") {+ t.Fatalf("sample does not end inside the third row: %q", text[csvSampleBytes-12:csvSampleBytes])+ }+ if got := detectCSVDelimiter(text); got != ';' {+ t.Errorf("detectCSVDelimiter = %q, want ';'", got)+ }+}++func TestDetectCSVDelimiter_LeavesOutTheLastSampledRowWhenTheCutEndsIt(t *testing.T) {+ // csvSampleRows rows fit in the sample only because the last of them is+ // cut short: "18;xxx" has two fields where the others have three.+ cell := strings.Repeat("x", 3600)+ var b strings.Builder+ b.WriteString("a;b;c\n")+ for i := 0; i < 40; i++ {+ fmt.Fprintf(&b, "%d;%s;end\n", i, cell)+ }+ text := b.String()+ if n := strings.Count(text[:csvSampleBytes], "\n"); n != csvSampleRows-1 {+ t.Fatalf("sample holds %d complete rows, want %d", n, csvSampleRows-1)+ }+ if got := detectCSVDelimiter(text); got != ';' {+ t.Errorf("detectCSVDelimiter = %q, want ';'", got)+ }+}++func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {+ // Trimming leading space would also trim the tab after "Widget", leaving+ // the row a field short of its header, and the tab would lose to the comma.+ data := []byte("Name\tRegion\tUnits\nWidget\t\t12\n")+ res := NewCSVParser().ParseWithResult(context.Background(), "export.csv", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }+ if got := res.JSON[len(res.JSON)-1]["text"]; got != "Name:Widget; Units:12 ——Data" {+ t.Errorf("row text = %v", got)+ }+}diff --git a/rag/app/table.py b/rag/app/table.pyindex 5b72eff59c..802e399b45 100644--- a/rag/app/table.py+++ b/rag/app/table.py@@ -35,6 +35,7 @@ from common.constants import MAXIMUM_TASK_PAGE_NUMBERfrom deepdoc.parser.utils import get_textfrom rag.nlp import rag_tokenizer, tokenize, tokenize_tablefrom deepdoc.parser import ExcelParser+from deepdoc.parser.excel_parser import detect_csv_delimiterfrom common import settingslogger = logging.getLogger(__name__)@@ -443,7 +444,8 @@ def column_data_type(arr):def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER, lang="Chinese", callback=None, **kwargs):"""Excel and csv(txt) format files are supported.- For csv or txt file, the delimiter between columns is TAB.+ For a txt file the delimiter between columns is TAB; for a csv file it is+ detected from the file itself, and can be forced with the `delimiter` kwarg.The first line must be column headers.Column headers must be meaningful terms inorder to make our NLP model understanding.It's good to enumerate some synonyms using slash '/' to separate, and even better to@@ -497,7 +499,10 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,elif re.search(r"\.csv$", filename, re.IGNORECASE):callback(0.1, "Start to parse.")txt = get_text(filename, binary)- delimiter = kwargs.get("delimiter", ",")+ # A .csv is not always comma separated, and reading one with the wrong+ # separator does not fail -- every row becomes a single column holding+ # the whole line.+ delimiter = kwargs.get("delimiter") or detect_csv_delimiter(txt)reader = csv.reader(io.StringIO(txt), delimiter=delimiter)all_rows = list(reader)diff --git a/test/unit_test/deepdoc/parser/test_csv_delimiter.py b/test/unit_test/deepdoc/parser/test_csv_delimiter.pynew file mode 100644index 0000000000..bee09c6d55--- /dev/null+++ b/test/unit_test/deepdoc/parser/test_csv_delimiter.py@@ -0,0 +1,152 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""A .csv file is not always comma separated.++Excel writes the list separator of the machine's locale -- a semicolon across+most of Europe -- and a tab separated export is routinely saved as .csv. Both+`pandas.read_csv` and `csv.reader` default to a comma, and reading such a file+with one does not fail: every row becomes a single column holding the whole+line, separators included.+"""++import importlib.util+import sys+from io import BytesIO+from pathlib import Path+from unittest import mock++import pytest++_REPO_ROOT = Path(__file__).resolve().parents[4]++_MOCK_MODULES = [+ "xgboost",+ "pdfplumber",+ "huggingface_hub",+ "pypdf",+ "sklearn",+ "sklearn.cluster",+ "sklearn.metrics",+ "deepdoc.vision",+]+for _name in _MOCK_MODULES:+ sys.modules.setdefault(_name, mock.MagicMock())+++def _load_excel_parser():+ """Load excel_parser.py by path, so deepdoc/parser/__init__.py stays out of it."""+ spec = importlib.util.spec_from_file_location(+ "deepdoc.parser.excel_parser_under_test",+ _REPO_ROOT / "deepdoc" / "parser" / "excel_parser.py",+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++excel_parser = _load_excel_parser()+detect_csv_delimiter = excel_parser.detect_csv_delimiter+RAGFlowExcelParser = excel_parser.RAGFlowExcelParser++ROWS = ["Name;Region;Units", "Widget;EU;12", "Gadget;US;7"]+++def _text(delimiter):+ return "\n".join(row.replace(";", delimiter) for row in ROWS) + "\n"++[email protected]("delimiter", [",", ";", "\t", "|"])+def test_detect_csv_delimiter(delimiter):+ assert detect_csv_delimiter(_text(delimiter)) == delimiter+++ "text",+ [+ # A separator inside a quoted field is not a separator.+ 'Name,Note\nWidget,"a; b"\nGadget,"c; d"\n',+ # Rows that do not line up leave the default in place.+ "Note\na; b\nc; d\n",+ "name,value\nAlice,1\nBob,2,extra\n",+ "",+ ],+)+def test_detect_csv_delimiter_falls_back_to_a_comma(text):+ assert detect_csv_delimiter(text) == ","+++def test_a_whitespace_only_line_is_not_a_row():+ assert detect_csv_delimiter("Name;Region\nWidget;EU\n \nGadget;US\n") == ";"+++def test_the_row_the_sample_cuts_is_left_out():+ """Fewer than CSV_SAMPLE_ROWS rows fit in the sample, so it ends inside a row."""+ cell = "x" * 4000+ text = "a;b;c\n" + "".join(f"{i};{cell};{cell}\n" for i in range(60))++ assert detect_csv_delimiter(text) == ";"+++def test_the_last_sampled_row_is_left_out_when_the_sample_cuts_it():+ """CSV_SAMPLE_ROWS rows fit only because the last of them is cut short."""+ cell = "x" * 3600+ text = "a;b;c\n" + "".join(f"{i};{cell};end\n" for i in range(40))+ assert text[: excel_parser.CSV_SAMPLE_CHARS].count("\n") == excel_parser.CSV_SAMPLE_ROWS - 1++ assert detect_csv_delimiter(text) == ";"+++def test_leading_blank_lines_do_not_use_up_the_sample():+ text = "\n" * (excel_parser.CSV_SAMPLE_ROWS + 5) + _text(";")++ assert detect_csv_delimiter(text) == ";"+ workbook = RAGFlowExcelParser._read_csv(BytesIO(text.encode("utf-8")))+ assert list(workbook.columns) == ["Name", "Region", "Units"]+++def test_the_sample_can_end_inside_a_quoted_line_break():+ cell = '"' + ("y" * 70 + "\n") * 60 + '"'+ text = "a;b;c\n" + "".join(f"{i};{cell};end\n" for i in range(40))++ assert detect_csv_delimiter(text) == ";"++[email protected]("delimiter", [",", ";", "\t", "|"])+def test_a_csv_keeps_its_columns_whatever_the_separator(delimiter):+ """`_read_csv` is the path a .csv upload takes through the Excel parser."""+ workbook = RAGFlowExcelParser._read_csv(BytesIO(_text(delimiter).encode("utf-8")))++ assert list(workbook.columns) == ["Name", "Region", "Units"]+ assert workbook.iloc[0].tolist() == ["Widget", "EU", 12]+++def test_a_single_column_csv_stays_a_single_column():+ """Guard: a separator that does not line up across the rows is not one."""+ workbook = RAGFlowExcelParser._read_csv(BytesIO(b"Note\na; b\nc; d\n"))++ assert list(workbook.columns) == ["Note"]+ assert workbook["Note"].tolist() == ["a; b", "c; d"]diff --git a/test/unit_test/rag/app/test_table_chunk_column_roles.py b/test/unit_test/rag/app/test_table_chunk_column_roles.pyindex ed37a8df78..22dfb442ec 100644--- a/test/unit_test/rag/app/test_table_chunk_column_roles.py+++ b/test/unit_test/rag/app/test_table_chunk_column_roles.py@@ -409,3 +409,32 @@ def test_chunk_keeps_unconvertible_cell_in_content(table_module, mock_update_kb:assert chunks[2]["amount_long"] == 300assert chunks[1].get("amount_long") != "N/A"assert "- amount: N/A" in chunks[1]["content_with_weight"]++[email protected]("delimiter", [";", "\t", "|"])+def test_chunk_reads_the_separator_a_csv_was_written_with(table_module, mock_update_kb: MagicMock, delimiter):+ table_module.chunk(+ FILENAME,+ binary=TEST_CSV.replace(b",", delimiter.encode()),+ callback=_noop_callback,+ kb_id=KB_ID,+ parser_config={},+ lang="Chinese",+ )+ args, _ = mock_update_kb.call_args+ assert args[1]["table_column_names"] == ["row_id", "title", "content", "country", "category"]+++def test_chunk_uses_an_explicit_delimiter_over_detection(table_module, mock_update_kb: MagicMock):+ # Detection picks ";" here, which lines up on every row too.+ table_module.chunk(+ FILENAME,+ binary=b"a|b;c\n1|2;3\n4|5;6\n",+ callback=_noop_callback,+ kb_id=KB_ID,+ parser_config={},+ lang="Chinese",+ delimiter="|",+ )+ args, _ = mock_update_kb.call_args+ assert args[1]["table_column_names"] == ["a", "b;c"]