RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): inherit merged data cells, strip invisible chars, trim dataless columns (#20996)
jay7772189a7734408fa
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 28b8bbf02e..153df9012e 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -137,7 +137,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [}// Clean illegal control characters from all cells.- records = cleanIllegalControlChars(records)+ records = trimTrailingEmptyColumns(cleanIllegalControlChars(records))dataRows := make([]int, len(records)-1)for i := range dataRows {diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex 283ac0b245..5f9487875d 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -277,3 +277,26 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {t.Errorf("empty tab-separated field not kept: %q", text)}}++// TestCSVParser_PreservesInternalJoiners asserts format characters that are+// meaningful inside a word survive CSV parsing end to end: U+200D in an+// emoji ZWJ sequence and U+200C as the Persian half-space.+func TestCSVParser_PreservesInternalJoiners(t *testing.T) {+ emoji := "\U0001F469\u200d\U0001F4BB" // woman technologist+ persian := "\u0645\u06cc\u200c\u0631\u0648\u0645"+ data := []byte("Name,Note\nA," + emoji + "\nB," + persian + "\n")+ res := NewCSVParser().ParseWithResult(context.Background(), "t.csv", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one HTML table item", len(res.JSON))+ }+ text := tableItemHTML(t, res.JSON[0])+ if !strings.Contains(text, "<td>"+emoji+"</td>") {+ t.Errorf("emoji ZWJ sequence broken: %q", text)+ }+ if !strings.Contains(text, "<td>"+persian+"</td>") {+ t.Errorf("Persian half-space lost: %q", text)+ }+}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 229e9e2aa7..848ce14f7d 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -36,6 +36,15 @@ import (// except TAB (0x09), LF (0x0A) and CR (0x0D).var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`)+// invisibleFormatCutset lists invisible Unicode format characters stripped+// from the EDGES of cell text only (see cleanIllegalControlChars): ZERO+// WIDTH SPACE / NON-JOINER / JOINER, WORD JOINER and the BOM. The same code+// points are meaningful INSIDE a word — U+200D joins emoji ZWJ sequences+// (e.g. woman technologist), U+200C is the Persian half-space — so they must+// survive mid-cell and only accidental wrappers come off. The escapes below+// are Go string literals naming each code point explicitly.+const invisibleFormatCutset = "\u200b\u200c\u200d\u2060\ufeff"+// numericCellRe recognises number-like cell text (integers, decimals, comma// thousands separators, optional leading sign/$ and trailing %). It is a// deliberately loose heuristic used only for header-vs-data contrast, not for@@ -168,9 +177,21 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow intmatrix := make([][]float64, 0, len(body)+1)matrix = append(matrix, headerTuple)rowsOut := make([][]string, 0, len(body)+1)- rowsOut = append(rowsOut, trimCells(header))+ headerCells := trimCells(header)+ rowsOut = append(rowsOut, headerCells)for _, r := range body {- rowsOut = append(rowsOut, trimCells(r.cells))+ cells := trimCells(r.cells)+ // Pad short rows to the header width so every row renders the+ // full column set with dataless cells left empty. excelize+ // truncates each row at its last valued cell, so without this a+ // trailing dataless column (kept as schema by+ // trimTrailingEmptyColumns) would vanish from data rows while+ // its <th> stays. Padding is empty-only, so the row-aligned+ // position tuple computed above is unaffected.+ for len(cells) < len(headerCells) {+ cells = append(cells, "")+ }+ rowsOut = append(rowsOut, cells)matrix = append(matrix, r.tuple)}grid := make([][]entity.TableCell, 0, len(rowsOut))@@ -394,18 +415,51 @@ func xlsxImageMIMEType(extension string) (string, bool) {const maxMergeExtentCols = 1024// cleanIllegalControlChars replaces illegal control characters in all cells-// with a single space.+// with a single space and trims invisible Unicode format characters from+// cell edges (see invisibleFormatCutset), which otherwise survive TrimSpace+// and pollute search text.func cleanIllegalControlChars(records [][]string) [][]string {out := make([][]string, len(records))for i, row := range records {out[i] = make([]string, len(row))for j, cell := range row {+ cell = strings.Trim(cell, invisibleFormatCutset)out[i][j] = tableIllegalCharsRe.ReplaceAllString(cell, " ")}}return out}+// trimTrailingEmptyColumns drops trailing columns that are blank in every+// row, header included. Only truncation happens here: remaining column+// indices are untouched, so row-aligned position tuples built downstream stay+// valid. A trailing column that carries a header label but no data (e.g. an+// unfilled score column) is intentionally kept — it is real schema, and its+// cells render empty. A header-only sheet is therefore left unchanged, and+// so is any sheet whose trailing columns all have labels.+func trimTrailingEmptyColumns(records [][]string) [][]string {+ if len(records) == 0 {+ return records+ }+ width := 0+ for _, row := range records {+ for c, cell := range row {+ if strings.TrimSpace(cell) != "" && c+1 > width {+ width = c + 1+ }+ }+ }+ if width == 0 {+ return records+ }+ for i, row := range records {+ if len(row) > width {+ records[i] = row[:width]+ }+ }+ return records+}+// ──────────────────────────────────────────────────────────── axis helpers// axisToRC parses an "A1"-style cell reference (optionally with a leading "$")@@ -538,6 +592,53 @@ func inheritMergedHeader(records [][]string, headerRowIdx int, mm map[[2]int][2]}}+// inheritMergedDataCells fills empty cells of non-header rows with the value+// of their merge master, so a vertically merged group label (e.g. A2:A4) is+// visible on every data row instead of only the first. Unlike the header-row+// path above, data rows are never widened: only cells already present in the+// row slice are filled, so a far merge cannot allocate a sea of empty <td>+// cells (see padRowToWidth for the blow-up this avoids). Call with absolute+// sheet row numbers, before records are reordered around the header row.+func inheritMergedDataCells(rows [][]string, headerRow int, ranges []mergeRange) {+ for _, r := range ranges {+ // Vertical merges only: a horizontally merged data cell is already+ // visible via its master column, while copying its text into every+ // slave column would render the same value twice.+ if r.sc != r.ec {+ continue+ }+ if r.sr == headerRow {+ continue+ }+ // A merge mastered on the header row carries a column label, not+ // data: copying it down would stamp the header text into every+ // data row it spans.+ if r.sr < 1 || r.sr > len(rows) || r.sc < 1 {+ continue+ }+ masterRow := rows[r.sr-1]+ if r.sc > len(masterRow) {+ continue+ }+ master := masterRow[r.sc-1]+ if strings.TrimSpace(master) == "" {+ continue+ }+ // Start past the master row: its own cell already holds the value.+ for row := r.sr + 1; row <= r.er && row <= len(rows); row++ {+ if row == headerRow {+ continue+ }+ if r.sc > len(rows[row-1]) {+ continue+ }+ if strings.TrimSpace(rows[row-1][r.sc-1]) == "" {+ rows[row-1][r.sc-1] = master+ }+ }+ }+}+// mergeExtentCol returns the furthest merged column across all ranges, capped// at maxMergeExtentCols so a pathological far merge cannot exhaust parser// memory when the header row is padded to inherit merged text.@@ -785,6 +886,9 @@ func readSpreadsheetRecords(f *excelize.File, sheet string) ([][]string, []int,padRowToWidth(&rows[headerRow-1], mergeExtentCol(ranges))}inheritMergedHeader(rows, headerRow, mm)+ // Data rows inherit their own vertical merge masters in place (no+ // widening); see inheritMergedDataCells.+ inheritMergedDataCells(rows, headerRow, ranges)// Reorder so the detected header row becomes records[0]; every other row is// data. For the common case (header on row 1) this is a no-op.@@ -798,6 +902,7 @@ func readSpreadsheetRecords(f *excelize.File, sheet string) ([][]string, []int,records = append(records, r)absDataRows = append(absDataRows, i+1)}+ records = trimTrailingEmptyColumns(records)return records, absDataRows, headerRow, warnings, nil}diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex a8fd74c909..03727deaaf 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -513,6 +513,174 @@ func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {}}+// TestXLSXParser_MergedDataCellInheritance asserts a vertically merged data+// cell's slave rows inherit the master text, so every row chunk keeps its+// group label instead of rendering an empty first cell.+func TestXLSXParser_MergedDataCellInheritance(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Group")+ mustSetCell(t, f, "Sheet1", "B1", "Item")+ mustSetCell(t, f, "Sheet1", "A2", "G1")+ mustMergeCell(t, f, "Sheet1", "A2", "A3")+ mustSetCell(t, f, "Sheet1", "B2", "x")+ mustSetCell(t, f, "Sheet1", "B3", "y")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ text := spreadsheetText(res)+ if !strings.Contains(text, "<tr><td>G1</td><td>y</td></tr>") {+ t.Fatalf("slave row did not inherit merged master text, got:\n%s", text)+ }+ positions := res.JSON[0]["positions"].([][]float64)+ if len(positions) != 3 {+ t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))+ }+ if got := positions[2]; !reflect.DeepEqual(got, []float64{1, 3, 3, 1, 2}) {+ t.Fatalf("row 3 tuple = %v, want [1 3 3 1 2]", got)+ }+}++// TestXLSXParser_MergedDataCellHorizontalUntouched asserts a horizontally+// merged data cell is NOT duplicated: the slave column stays empty because+// the master column already shows the value.+func TestXLSXParser_MergedDataCellHorizontalUntouched(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Group")+ mustSetCell(t, f, "Sheet1", "B1", "Item")+ mustSetCell(t, f, "Sheet1", "C1", "Note")+ mustSetCell(t, f, "Sheet1", "A2", "G1")+ mustSetCell(t, f, "Sheet1", "B2", "wide")+ mustMergeCell(t, f, "Sheet1", "B2", "C2")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ text := spreadsheetText(res)+ if strings.Contains(text, "<td>wide</td><td>wide</td>") {+ t.Fatalf("horizontal merge master duplicated into slave cell, got:\n%s", text)+ }+}++// TestXLSXParser_MergedHeaderDownIntoDataUntouched asserts a vertical merge+// mastered on the header row does not leak the header label into data rows.+func TestXLSXParser_MergedHeaderDownIntoDataUntouched(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "H")+ mustMergeCell(t, f, "Sheet1", "A1", "A2")+ mustSetCell(t, f, "Sheet1", "B1", "Name")+ mustSetCell(t, f, "Sheet1", "B2", "Alice")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ text := spreadsheetText(res)+ if !strings.Contains(text, "<tr><th>H</th><th>Name</th></tr>") {+ t.Fatalf("want header row untouched, got:\n%s", text)+ }+ if strings.Contains(text, "<td>H</td>") {+ t.Fatalf("header merge master leaked into data row, got:\n%s", text)+ }+}++// TestXLSXParser_StripsInvisibleFormatChars asserts zero-width format+// characters around cell text are removed, so labels wrapped in U+200C match+// their clean form in search text.+func TestXLSXParser_StripsInvisibleFormatChars(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Group")+ mustSetCell(t, f, "Sheet1", "B1", "Item")+ mustSetCell(t, f, "Sheet1", "A2", "\u200cG1\u200c")+ mustSetCell(t, f, "Sheet1", "B2", "x")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ text := spreadsheetText(res)+ if strings.Contains(text, "\u200c") {+ t.Fatalf("invisible format char survived cleaning, got:\n%q", text)+ }+ if !strings.Contains(text, "<tr><td>G1</td><td>x</td></tr>") {+ t.Fatalf("want cleaned cell text, got:\n%s", text)+ }+}++// TestCleanIllegalControlChars_StripsInvisibleFormatChars pins the shared+// cleaner directly, covering the xls/csv paths that reuse it.+// TestCleanIllegalControlChars_PreservesInternalJoiners asserts format+// characters that are meaningful inside a word survive cleaning: U+200D in+// an emoji ZWJ sequence and U+200C as the Persian half-space.+func TestCleanIllegalControlChars_PreservesInternalJoiners(t *testing.T) {+ emoji := "\U0001F469\u200d\U0001F4BB" // woman technologist+ persian := "\u0645\u06cc\u200c\u0631\u0648\u0645"+ in := [][]string{{emoji, persian}}+ got := cleanIllegalControlChars(in)+ if !reflect.DeepEqual(got, in) {+ t.Fatalf("cleanIllegalControlChars = %q, want internal joiners kept %q", got, in)+ }+}++func TestCleanIllegalControlChars_StripsInvisibleFormatChars(t *testing.T) {+ in := [][]string{{"\u200cA\u200c", "B\u200bC D", "D\ufeff"}}+ want := [][]string{{"A", "B\u200bC D", "D"}}+ got := cleanIllegalControlChars(in)+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("cleanIllegalControlChars = %q, want %q", got, want)+ }+}++// TestXLSXParser_KeepsTrailingDatalessColumn asserts a trailing column with+// a header label but no data in any row is kept as real schema: the header+// renders and its cells stay empty.+func TestXLSXParser_KeepsTrailingDatalessColumn(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Group")+ mustSetCell(t, f, "Sheet1", "B1", "Score")+ mustSetCell(t, f, "Sheet1", "A2", "G1")+ mustSetCell(t, f, "Sheet1", "A3", "G2")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ text := spreadsheetText(res)+ if !strings.Contains(text, "<tr><th>Group</th><th>Score</th></tr>") {+ t.Fatalf("trailing labeled column was dropped, got:\n%s", text)+ }+ if !strings.Contains(text, "<tr><td>G1</td><td></td></tr>") {+ t.Fatalf("dataless cell must render empty, got:\n%s", text)+ }+}++// TestTrimTrailingEmptyColumns_DropsFullyBlankColumn pins the helper+// directly: a trailing column blank in every row, header included (e.g. a+// CSV with trailing commas), is truncated.+func TestTrimTrailingEmptyColumns_DropsFullyBlankColumn(t *testing.T) {+ in := [][]string{+ {"Group", "Score", ""},+ {"G1", "", ""},+ {"G2", "", ""},+ }+ got := trimTrailingEmptyColumns(in)+ want := [][]string{+ {"Group", "Score"},+ {"G1", ""},+ {"G2", ""},+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("trimTrailingEmptyColumns = %q, want %q", got, want)+ }+}+// TestDetectHeaderRow_ListObject asserts a ListObject whose top row is > 1 is// detected as the header row.func TestDetectHeaderRow_ListObject(t *testing.T) {diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 0162aa0038..dfaf702cac 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -176,7 +176,8 @@ func readBIFF8Records(ctx context.Context, reader excel.Reader) ([][]string, []irecords = append(records, row.cells)dataRows = append(dataRows, row.number)}- return cleanIllegalControlChars(records), dataRows, rows[0].number+ records = trimTrailingEmptyColumns(cleanIllegalControlChars(records))+ return records, dataRows, rows[0].number}func spreadsheetCellString(value any) string {