Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(chunker): count an escaped pipe as cell text when matching table widths (#3979)
Lukas98feef38de5c
diff --git a/internal/infrastructure/chunker/header_tracker.go b/internal/infrastructure/chunker/header_tracker.goindex 89dbdc84d..8b9e3940f 100644--- a/internal/infrastructure/chunker/header_tracker.go+++ b/internal/infrastructure/chunker/header_tracker.go@@ -226,7 +226,7 @@ func tableRowColumnCount(line string) int {if !strings.HasPrefix(line, "|") {return 0}- parts := strings.Split(line, "|")+ parts := splitTableCells(line)if len(parts) > 0 && strings.TrimSpace(parts[0]) == "" {parts = parts[1:]}@@ -236,6 +236,28 @@ func tableRowColumnCount(line string) int {return len(parts)}+// splitTableCells splits a table row at its cell delimiters. As in GFM, a pipe+// escaped as `\|` is text inside a cell, so a row whose cell holds one still+// matches the width of its header.+func splitTableCells(line string) []string {+ var parts []string+ start, backslashes := 0, 0+ for i := 0; i < len(line); i++ {+ switch line[i] {+ case '\\':+ backslashes+++ continue+ case '|':+ if backslashes%2 == 0 {+ parts = append(parts, line[start:i])+ start = i + 1+ }+ }+ backslashes = 0+ }+ return append(parts, line[start:])+}+func firstTableRowColumnCount(text string) int {for _, line := range strings.Split(text, "\n") {line = strings.TrimSpace(line)diff --git a/internal/infrastructure/chunker/splitter_test.go b/internal/infrastructure/chunker/splitter_test.goindex 02d2f62a0..2781499c6 100644--- a/internal/infrastructure/chunker/splitter_test.go+++ b/internal/infrastructure/chunker/splitter_test.go@@ -948,6 +948,69 @@ func TestHeaderTracker_ColumnMismatchEndsTable(t *testing.T) {}}+func TestTableRowColumnCount_EscapedPipeIsCellText(t *testing.T) {+ cases := []struct {+ row string+ want int+ }{+ {"| a | b | c |", 3},+ {`| a \| b | c |`, 2},+ {"| `x \\| y` | c |", 2},+ {`| a \\| b | c |`, 3}, // an escaped backslash leaves the pipe a delimiter+ {`| a \\\| b | c |`, 2},+ {`| C:\temp\x | c |`, 2}, // a backslash before another character escapes nothing+ }+ for _, tc := range cases {+ if got := tableRowColumnCount(tc.row); got != tc.want {+ t.Errorf("tableRowColumnCount(%q) = %d, want %d", tc.row, got, tc.want)+ }+ }+}++func TestHeaderTracker_EscapedPipeRowKeepsTableHeader(t *testing.T) {+ ht := newHeaderTracker()+ ht.update("| Name | Note | City |\n| --- | --- | --- |\n")+ ht.update("| Lee | x \\| y | Shanghai |\n")+ if ht.getHeaders() == "" {+ t.Fatal("a 3-col row with an escaped pipe in a cell ended the 3-col table header")+ }+}++func TestSplitText_EscapedPipeRowKeepsTableHeaderInLaterChunks(t *testing.T) {+ text := "" ++ "前面的文字\n\n" ++ "| 姓名 | 备注 | 城市 |\n" ++ "| --- | --- | --- |\n" ++ "| 张三 | a | 北京 |\n" ++ "| 李四 | x \\| y | 上海 |\n" ++ "| 王五 | b | 广州 |\n" ++ "| 赵六 | c | 深圳 |\n" ++ "| 孙七 | d | 杭州 |\n" ++ "| 周八 | e | 成都 |\n" ++ "\n后面的文字"+ tableHeader := "| 姓名 | 备注 | 城市 |\n| --- | --- | --- |\n"++ cfg := SplitterConfig{ChunkSize: 60, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}}+ chunks := SplitText(text, cfg)++ checked := 0+ for _, c := range chunks {+ for _, row := range []string{"| 王五", "| 赵六", "| 孙七", "| 周八"} {+ if strings.Contains(c.Content, row) {+ checked+++ if !strings.HasPrefix(c.Content, tableHeader) {+ t.Errorf("chunk (seq=%d) with rows after the escaped-pipe row is missing the header:\n%s",+ c.Seq, c.Content)+ }+ break+ }+ }+ }+ if checked == 0 {+ t.Fatal("expected chunks holding the rows after the escaped-pipe row")+ }+}+func TestHeaderTracker_ParagraphBreakEndsOnNextUnit(t *testing.T) {ht := newHeaderTracker()ht.update("| Name | Game | Fame | Blame |\n| --- | --- | --- | --- |\n")