Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(chunker): count an escaped pipe as cell text when matching table widths (#3979)

Lukas98feef38de5c
diff --git a/internal/infrastructure/chunker/header_tracker.go b/internal/infrastructure/chunker/header_tracker.go
index 89dbdc84d..8b9e3940f 100644
--- a/internal/infrastructure/chunker/header_tracker.go
+++ b/internal/infrastructure/chunker/header_tracker.go
@@ -226,7 +226,7 @@ func tableRowColumnCount(line string) int {
if !strings.HasPrefix(line, "|") {
return 0
}
- parts := strings.Split(line, "|")
+ parts := splitTableCells(line)
if len(parts) > 0 && strings.TrimSpace(parts[0]) == "" {
parts = parts[1:]
}
@@ -236,6 +236,28 @@ func tableRowColumnCount(line string) int {
return len(parts)
}
+// splitTableCells splits a table row at its cell delimiters. As in GFM, a pipe
+// escaped as `\|` is text inside a cell, so a row whose cell holds one still
+// matches the width of its header.
+func splitTableCells(line string) []string {
+ var parts []string
+ start, backslashes := 0, 0
+ for i := 0; i < len(line); i++ {
+ switch line[i] {
+ case '\\':
+ backslashes++
+ continue
+ case '|':
+ if backslashes%2 == 0 {
+ parts = append(parts, line[start:i])
+ start = i + 1
+ }
+ }
+ backslashes = 0
+ }
+ return append(parts, line[start:])
+}
+
func firstTableRowColumnCount(text string) int {
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
diff --git a/internal/infrastructure/chunker/splitter_test.go b/internal/infrastructure/chunker/splitter_test.go
index 02d2f62a0..2781499c6 100644
--- a/internal/infrastructure/chunker/splitter_test.go
+++ b/internal/infrastructure/chunker/splitter_test.go
@@ -948,6 +948,69 @@ func TestHeaderTracker_ColumnMismatchEndsTable(t *testing.T) {
}
}
+func TestTableRowColumnCount_EscapedPipeIsCellText(t *testing.T) {
+ cases := []struct {
+ row string
+ want int
+ }{
+ {"| a | b | c |", 3},
+ {`| a \| b | c |`, 2},
+ {"| `x \\| y` | c |", 2},
+ {`| a \\| b | c |`, 3}, // an escaped backslash leaves the pipe a delimiter
+ {`| a \\\| b | c |`, 2},
+ {`| C:\temp\x | c |`, 2}, // a backslash before another character escapes nothing
+ }
+ for _, tc := range cases {
+ if got := tableRowColumnCount(tc.row); got != tc.want {
+ t.Errorf("tableRowColumnCount(%q) = %d, want %d", tc.row, got, tc.want)
+ }
+ }
+}
+
+func TestHeaderTracker_EscapedPipeRowKeepsTableHeader(t *testing.T) {
+ ht := newHeaderTracker()
+ ht.update("| Name | Note | City |\n| --- | --- | --- |\n")
+ ht.update("| Lee | x \\| y | Shanghai |\n")
+ if ht.getHeaders() == "" {
+ t.Fatal("a 3-col row with an escaped pipe in a cell ended the 3-col table header")
+ }
+}
+
+func TestSplitText_EscapedPipeRowKeepsTableHeaderInLaterChunks(t *testing.T) {
+ text := "" +
+ "前面的文字\n\n" +
+ "| 姓名 | 备注 | 城市 |\n" +
+ "| --- | --- | --- |\n" +
+ "| 张三 | a | 北京 |\n" +
+ "| 李四 | x \\| y | 上海 |\n" +
+ "| 王五 | b | 广州 |\n" +
+ "| 赵六 | c | 深圳 |\n" +
+ "| 孙七 | d | 杭州 |\n" +
+ "| 周八 | e | 成都 |\n" +
+ "\n后面的文字"
+ tableHeader := "| 姓名 | 备注 | 城市 |\n| --- | --- | --- |\n"
+
+ cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}
+ chunks := SplitText(text, cfg)
+
+ checked := 0
+ for _, c := range chunks {
+ for _, row := range []string{"| 王五", "| 赵六", "| 孙七", "| 周八"} {
+ if strings.Contains(c.Content, row) {
+ checked++
+ if !strings.HasPrefix(c.Content, tableHeader) {
+ t.Errorf("chunk (seq=%d) with rows after the escaped-pipe row is missing the header:\n%s",
+ c.Seq, c.Content)
+ }
+ break
+ }
+ }
+ }
+ if checked == 0 {
+ t.Fatal("expected chunks holding the rows after the escaped-pipe row")
+ }
+}
+
func TestHeaderTracker_ParagraphBreakEndsOnNextUnit(t *testing.T) {
ht := newHeaderTracker()
ht.update("| Name | Game | Fame | Blame |\n| --- | --- | --- | --- |\n")