Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): inherit merged data cells, strip invisible chars, trim dataless columns (#20996)

jay7772189a7734408fa
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 28b8bbf02e..153df9012e 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -137,7 +137,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
}
// Clean illegal control characters from all cells.
- records = cleanIllegalControlChars(records)
+ records = trimTrailingEmptyColumns(cleanIllegalControlChars(records))
dataRows := make([]int, len(records)-1)
for i := range dataRows {
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 283ac0b245..5f9487875d 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -277,3 +277,26 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {
t.Errorf("empty tab-separated field not kept: %q", text)
}
}
+
+// TestCSVParser_PreservesInternalJoiners asserts format characters that are
+// meaningful inside a word survive CSV parsing end to end: U+200D in an
+// emoji ZWJ sequence and U+200C as the Persian half-space.
+func TestCSVParser_PreservesInternalJoiners(t *testing.T) {
+ emoji := "\U0001F469\u200d\U0001F4BB" // woman technologist
+ persian := "\u0645\u06cc\u200c\u0631\u0648\u0645"
+ data := []byte("Name,Note\nA," + emoji + "\nB," + persian + "\n")
+ res := NewCSVParser().ParseWithResult(context.Background(), "t.csv", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
+ }
+ text := tableItemHTML(t, res.JSON[0])
+ if !strings.Contains(text, "<td>"+emoji+"</td>") {
+ t.Errorf("emoji ZWJ sequence broken: %q", text)
+ }
+ if !strings.Contains(text, "<td>"+persian+"</td>") {
+ t.Errorf("Persian half-space lost: %q", text)
+ }
+}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 229e9e2aa7..848ce14f7d 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -36,6 +36,15 @@ import (
// except TAB (0x09), LF (0x0A) and CR (0x0D).
var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`)
+// invisibleFormatCutset lists invisible Unicode format characters stripped
+// from the EDGES of cell text only (see cleanIllegalControlChars): ZERO
+// WIDTH SPACE / NON-JOINER / JOINER, WORD JOINER and the BOM. The same code
+// points are meaningful INSIDE a word — U+200D joins emoji ZWJ sequences
+// (e.g. woman technologist), U+200C is the Persian half-space — so they must
+// survive mid-cell and only accidental wrappers come off. The escapes below
+// are Go string literals naming each code point explicitly.
+const invisibleFormatCutset = "\u200b\u200c\u200d\u2060\ufeff"
+
// numericCellRe recognises number-like cell text (integers, decimals, comma
// thousands separators, optional leading sign/$ and trailing %). It is a
// deliberately loose heuristic used only for header-vs-data contrast, not for
@@ -168,9 +177,21 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int
matrix := make([][]float64, 0, len(body)+1)
matrix = append(matrix, headerTuple)
rowsOut := make([][]string, 0, len(body)+1)
- rowsOut = append(rowsOut, trimCells(header))
+ headerCells := trimCells(header)
+ rowsOut = append(rowsOut, headerCells)
for _, r := range body {
- rowsOut = append(rowsOut, trimCells(r.cells))
+ cells := trimCells(r.cells)
+ // Pad short rows to the header width so every row renders the
+ // full column set with dataless cells left empty. excelize
+ // truncates each row at its last valued cell, so without this a
+ // trailing dataless column (kept as schema by
+ // trimTrailingEmptyColumns) would vanish from data rows while
+ // its <th> stays. Padding is empty-only, so the row-aligned
+ // position tuple computed above is unaffected.
+ for len(cells) < len(headerCells) {
+ cells = append(cells, "")
+ }
+ rowsOut = append(rowsOut, cells)
matrix = append(matrix, r.tuple)
}
grid := make([][]entity.TableCell, 0, len(rowsOut))
@@ -394,18 +415,51 @@ func xlsxImageMIMEType(extension string) (string, bool) {
const maxMergeExtentCols = 1024
// cleanIllegalControlChars replaces illegal control characters in all cells
-// with a single space.
+// with a single space and trims invisible Unicode format characters from
+// cell edges (see invisibleFormatCutset), which otherwise survive TrimSpace
+// and pollute search text.
func cleanIllegalControlChars(records [][]string) [][]string {
out := make([][]string, len(records))
for i, row := range records {
out[i] = make([]string, len(row))
for j, cell := range row {
+ cell = strings.Trim(cell, invisibleFormatCutset)
out[i][j] = tableIllegalCharsRe.ReplaceAllString(cell, " ")
}
}
return out
}
+// trimTrailingEmptyColumns drops trailing columns that are blank in every
+// row, header included. Only truncation happens here: remaining column
+// indices are untouched, so row-aligned position tuples built downstream stay
+// valid. A trailing column that carries a header label but no data (e.g. an
+// unfilled score column) is intentionally kept — it is real schema, and its
+// cells render empty. A header-only sheet is therefore left unchanged, and
+// so is any sheet whose trailing columns all have labels.
+func trimTrailingEmptyColumns(records [][]string) [][]string {
+ if len(records) == 0 {
+ return records
+ }
+ width := 0
+ for _, row := range records {
+ for c, cell := range row {
+ if strings.TrimSpace(cell) != "" && c+1 > width {
+ width = c + 1
+ }
+ }
+ }
+ if width == 0 {
+ return records
+ }
+ for i, row := range records {
+ if len(row) > width {
+ records[i] = row[:width]
+ }
+ }
+ return records
+}
+
// ──────────────────────────────────────────────────────────── axis helpers
// axisToRC parses an "A1"-style cell reference (optionally with a leading "$")
@@ -538,6 +592,53 @@ func inheritMergedHeader(records [][]string, headerRowIdx int, mm map[[2]int][2]
}
}
+// inheritMergedDataCells fills empty cells of non-header rows with the value
+// of their merge master, so a vertically merged group label (e.g. A2:A4) is
+// visible on every data row instead of only the first. Unlike the header-row
+// path above, data rows are never widened: only cells already present in the
+// row slice are filled, so a far merge cannot allocate a sea of empty <td>
+// cells (see padRowToWidth for the blow-up this avoids). Call with absolute
+// sheet row numbers, before records are reordered around the header row.
+func inheritMergedDataCells(rows [][]string, headerRow int, ranges []mergeRange) {
+ for _, r := range ranges {
+ // Vertical merges only: a horizontally merged data cell is already
+ // visible via its master column, while copying its text into every
+ // slave column would render the same value twice.
+ if r.sc != r.ec {
+ continue
+ }
+ if r.sr == headerRow {
+ continue
+ }
+ // A merge mastered on the header row carries a column label, not
+ // data: copying it down would stamp the header text into every
+ // data row it spans.
+ if r.sr < 1 || r.sr > len(rows) || r.sc < 1 {
+ continue
+ }
+ masterRow := rows[r.sr-1]
+ if r.sc > len(masterRow) {
+ continue
+ }
+ master := masterRow[r.sc-1]
+ if strings.TrimSpace(master) == "" {
+ continue
+ }
+ // Start past the master row: its own cell already holds the value.
+ for row := r.sr + 1; row <= r.er && row <= len(rows); row++ {
+ if row == headerRow {
+ continue
+ }
+ if r.sc > len(rows[row-1]) {
+ continue
+ }
+ if strings.TrimSpace(rows[row-1][r.sc-1]) == "" {
+ rows[row-1][r.sc-1] = master
+ }
+ }
+ }
+}
+
// mergeExtentCol returns the furthest merged column across all ranges, capped
// at maxMergeExtentCols so a pathological far merge cannot exhaust parser
// memory when the header row is padded to inherit merged text.
@@ -785,6 +886,9 @@ func readSpreadsheetRecords(f *excelize.File, sheet string) ([][]string, []int,
padRowToWidth(&rows[headerRow-1], mergeExtentCol(ranges))
}
inheritMergedHeader(rows, headerRow, mm)
+ // Data rows inherit their own vertical merge masters in place (no
+ // widening); see inheritMergedDataCells.
+ inheritMergedDataCells(rows, headerRow, ranges)
// Reorder so the detected header row becomes records[0]; every other row is
// data. For the common case (header on row 1) this is a no-op.
@@ -798,6 +902,7 @@ func readSpreadsheetRecords(f *excelize.File, sheet string) ([][]string, []int,
records = append(records, r)
absDataRows = append(absDataRows, i+1)
}
+ records = trimTrailingEmptyColumns(records)
return records, absDataRows, headerRow, warnings, nil
}
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index a8fd74c909..03727deaaf 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -513,6 +513,174 @@ func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {
}
}
+// TestXLSXParser_MergedDataCellInheritance asserts a vertically merged data
+// cell's slave rows inherit the master text, so every row chunk keeps its
+// group label instead of rendering an empty first cell.
+func TestXLSXParser_MergedDataCellInheritance(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Group")
+ mustSetCell(t, f, "Sheet1", "B1", "Item")
+ mustSetCell(t, f, "Sheet1", "A2", "G1")
+ mustMergeCell(t, f, "Sheet1", "A2", "A3")
+ mustSetCell(t, f, "Sheet1", "B2", "x")
+ mustSetCell(t, f, "Sheet1", "B3", "y")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ text := spreadsheetText(res)
+ if !strings.Contains(text, "<tr><td>G1</td><td>y</td></tr>") {
+ t.Fatalf("slave row did not inherit merged master text, got:\n%s", text)
+ }
+ positions := res.JSON[0]["positions"].([][]float64)
+ if len(positions) != 3 {
+ t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))
+ }
+ if got := positions[2]; !reflect.DeepEqual(got, []float64{1, 3, 3, 1, 2}) {
+ t.Fatalf("row 3 tuple = %v, want [1 3 3 1 2]", got)
+ }
+}
+
+// TestXLSXParser_MergedDataCellHorizontalUntouched asserts a horizontally
+// merged data cell is NOT duplicated: the slave column stays empty because
+// the master column already shows the value.
+func TestXLSXParser_MergedDataCellHorizontalUntouched(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Group")
+ mustSetCell(t, f, "Sheet1", "B1", "Item")
+ mustSetCell(t, f, "Sheet1", "C1", "Note")
+ mustSetCell(t, f, "Sheet1", "A2", "G1")
+ mustSetCell(t, f, "Sheet1", "B2", "wide")
+ mustMergeCell(t, f, "Sheet1", "B2", "C2")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ text := spreadsheetText(res)
+ if strings.Contains(text, "<td>wide</td><td>wide</td>") {
+ t.Fatalf("horizontal merge master duplicated into slave cell, got:\n%s", text)
+ }
+}
+
+// TestXLSXParser_MergedHeaderDownIntoDataUntouched asserts a vertical merge
+// mastered on the header row does not leak the header label into data rows.
+func TestXLSXParser_MergedHeaderDownIntoDataUntouched(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "H")
+ mustMergeCell(t, f, "Sheet1", "A1", "A2")
+ mustSetCell(t, f, "Sheet1", "B1", "Name")
+ mustSetCell(t, f, "Sheet1", "B2", "Alice")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ text := spreadsheetText(res)
+ if !strings.Contains(text, "<tr><th>H</th><th>Name</th></tr>") {
+ t.Fatalf("want header row untouched, got:\n%s", text)
+ }
+ if strings.Contains(text, "<td>H</td>") {
+ t.Fatalf("header merge master leaked into data row, got:\n%s", text)
+ }
+}
+
+// TestXLSXParser_StripsInvisibleFormatChars asserts zero-width format
+// characters around cell text are removed, so labels wrapped in U+200C match
+// their clean form in search text.
+func TestXLSXParser_StripsInvisibleFormatChars(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Group")
+ mustSetCell(t, f, "Sheet1", "B1", "Item")
+ mustSetCell(t, f, "Sheet1", "A2", "\u200cG1\u200c")
+ mustSetCell(t, f, "Sheet1", "B2", "x")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ text := spreadsheetText(res)
+ if strings.Contains(text, "\u200c") {
+ t.Fatalf("invisible format char survived cleaning, got:\n%q", text)
+ }
+ if !strings.Contains(text, "<tr><td>G1</td><td>x</td></tr>") {
+ t.Fatalf("want cleaned cell text, got:\n%s", text)
+ }
+}
+
+// TestCleanIllegalControlChars_StripsInvisibleFormatChars pins the shared
+// cleaner directly, covering the xls/csv paths that reuse it.
+// TestCleanIllegalControlChars_PreservesInternalJoiners asserts format
+// characters that are meaningful inside a word survive cleaning: U+200D in
+// an emoji ZWJ sequence and U+200C as the Persian half-space.
+func TestCleanIllegalControlChars_PreservesInternalJoiners(t *testing.T) {
+ emoji := "\U0001F469\u200d\U0001F4BB" // woman technologist
+ persian := "\u0645\u06cc\u200c\u0631\u0648\u0645"
+ in := [][]string{{emoji, persian}}
+ got := cleanIllegalControlChars(in)
+ if !reflect.DeepEqual(got, in) {
+ t.Fatalf("cleanIllegalControlChars = %q, want internal joiners kept %q", got, in)
+ }
+}
+
+func TestCleanIllegalControlChars_StripsInvisibleFormatChars(t *testing.T) {
+ in := [][]string{{"\u200cA\u200c", "B\u200bC D", "D\ufeff"}}
+ want := [][]string{{"A", "B\u200bC D", "D"}}
+ got := cleanIllegalControlChars(in)
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("cleanIllegalControlChars = %q, want %q", got, want)
+ }
+}
+
+// TestXLSXParser_KeepsTrailingDatalessColumn asserts a trailing column with
+// a header label but no data in any row is kept as real schema: the header
+// renders and its cells stay empty.
+func TestXLSXParser_KeepsTrailingDatalessColumn(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Group")
+ mustSetCell(t, f, "Sheet1", "B1", "Score")
+ mustSetCell(t, f, "Sheet1", "A2", "G1")
+ mustSetCell(t, f, "Sheet1", "A3", "G2")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "t.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ text := spreadsheetText(res)
+ if !strings.Contains(text, "<tr><th>Group</th><th>Score</th></tr>") {
+ t.Fatalf("trailing labeled column was dropped, got:\n%s", text)
+ }
+ if !strings.Contains(text, "<tr><td>G1</td><td></td></tr>") {
+ t.Fatalf("dataless cell must render empty, got:\n%s", text)
+ }
+}
+
+// TestTrimTrailingEmptyColumns_DropsFullyBlankColumn pins the helper
+// directly: a trailing column blank in every row, header included (e.g. a
+// CSV with trailing commas), is truncated.
+func TestTrimTrailingEmptyColumns_DropsFullyBlankColumn(t *testing.T) {
+ in := [][]string{
+ {"Group", "Score", ""},
+ {"G1", "", ""},
+ {"G2", "", ""},
+ }
+ got := trimTrailingEmptyColumns(in)
+ want := [][]string{
+ {"Group", "Score"},
+ {"G1", ""},
+ {"G2", ""},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("trimTrailingEmptyColumns = %q, want %q", got, want)
+ }
+}
+
// TestDetectHeaderRow_ListObject asserts a ListObject whose top row is > 1 is
// detected as the header row.
func TestDetectHeaderRow_ListObject(t *testing.T) {
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 0162aa0038..dfaf702cac 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -176,7 +176,8 @@ func readBIFF8Records(ctx context.Context, reader excel.Reader) ([][]string, []i
records = append(records, row.cells)
dataRows = append(dataRows, row.number)
}
- return cleanIllegalControlChars(records), dataRows, rows[0].number
+ records = trimTrailingEmptyColumns(cleanIllegalControlChars(records))
+ return records, dataRows, rows[0].number
}
func spreadsheetCellString(value any) string {