Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(chunker): preserve source offsets for table child chunks (#3969)

Yeager0204e915adb147d7
diff --git a/internal/infrastructure/chunker/parent_child_offsets_test.go b/internal/infrastructure/chunker/parent_child_offsets_test.go
new file mode 100644
index 000000000..f87a7a25c
--- /dev/null
+++ b/internal/infrastructure/chunker/parent_child_offsets_test.go
@@ -0,0 +1,148 @@
+package chunker
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/sourceloc"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestParentChildTableSourceOffsets(t *testing.T) {
+ splitters := []struct {
+ name string
+ split func(string, SplitterConfig, SplitterConfig) ParentChildResult
+ }{
+ {"legacy", SplitTextParentChild},
+ {"adaptive", SplitParentChild},
+ {"diagnostics", func(text string, parent, child SplitterConfig) ParentChildResult {
+ result, _ := SplitParentChildWithDiagnostics(text, parent, child)
+ return result
+ }},
+ }
+ for _, fixture := range []struct {
+ name, newline, row string
+ }{
+ {"ascii", "\n", "| apple | red |"},
+ {"unicode", "\n", "| 苹果🍎 | 北京 |"},
+ {"crlf", "\r\n", "| 苹果🍎 | 北京 |"},
+ } {
+ header := "| Item | City |" + fixture.newline + "| --- | --- |" + fixture.newline
+ // Repeated rows also catch attempts to recover positions with a text search.
+ text := "Intro" + fixture.newline + header +
+ strings.Repeat(fixture.row+fixture.newline, 12) + "| last | row |" + fixture.newline
+ parentCfg := SplitterConfig{
+ ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy,
+ }
+ // Ensure this exercises a parent with a synthetic header, not merely
+ // children with headers inside a single source-backed parent.
+ parents := SplitText(text, parentCfg)
+ sawSyntheticParent := false
+ for _, parent := range parents {
+ if len([]rune(parent.Content)) > parent.End-parent.Start {
+ sawSyntheticParent = true
+ }
+ }
+ if !sawSyntheticParent {
+ t.Fatalf("%s fixture did not produce a parent with a synthetic header", fixture.name)
+ }
+ for _, splitter := range splitters {
+ // 12 isolates the virtual header as a child; 50 re-splits table
+ // rows; 160 keeps the parent intact (ParentIndex == -1).
+ for _, childSize := range []int{12, 50, 160} {
+ t.Run(fmt.Sprintf("%s/%s/child=%d", fixture.name, splitter.name, childSize), func(t *testing.T) {
+ childCfg := parentCfg
+ childCfg.ChunkSize = childSize
+ result := splitter.split(text, parentCfg, childCfg)
+ if len(result.Children) == 0 {
+ t.Fatal("expected children")
+ }
+ textRunes := []rune(text)
+ covered := make([]bool, len(textRunes))
+ var chunks []Chunk
+ for i, child := range result.Children {
+ if child.Seq != i {
+ t.Errorf("child %d has sequence %d", i, child.Seq)
+ }
+ if child.Start < 0 || child.End > len(textRunes) || child.Start >= child.End {
+ t.Fatalf("child %d has invalid source span [%d, %d), document length %d",
+ i, child.Start, child.End, len(textRunes))
+ }
+ content := []rune(child.Content)
+ span := child.End - child.Start
+ if span > len(content) {
+ t.Fatalf("child %d source span exceeds content length", i)
+ }
+ want := string(textRunes[child.Start:child.End])
+ if got := string(content[len(content)-span:]); got != want {
+ t.Fatalf("child %d source span [%d, %d): got suffix %q, want %q",
+ i, child.Start, child.End, got, want)
+ }
+ if child.ParentIndex >= 0 {
+ if child.ParentIndex >= len(result.Parents) {
+ t.Fatalf("child %d has invalid parent index %d", i, child.ParentIndex)
+ }
+ parent := result.Parents[child.ParentIndex]
+ if child.Start < parent.Start || child.End > parent.End {
+ t.Errorf("child %d source span escapes its parent", i)
+ }
+ } else if childSize == 160 && strings.Contains(child.Content, fixture.row) {
+ if !strings.Contains(child.Content, "| Item | City |") {
+ t.Errorf("child %d lost table header context", i)
+ }
+ }
+ for pos := child.Start; pos < child.End; pos++ {
+ covered[pos] = true
+ }
+ chunks = append(chunks, child.Chunk)
+ }
+ for pos, ok := range covered {
+ if !ok {
+ t.Fatalf("source rune %d is missing from children", pos)
+ }
+ }
+ if got := restoreTextFromChunks(chunks); got != text {
+ t.Fatalf("source reconstruction mismatch: got %q, want %q", got, text)
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestParentChildTableCitationUsesSourceRow(t *testing.T) {
+ prefix := "| Item | City |\n| --- | --- |\n" + strings.Repeat("| 苹果🍎 | 北京 |\n", 12)
+ lastRow := "| last | row |\n"
+ text := prefix + lastRow
+ index := sourceloc.NewIndex(text, []types.SourceBlock{
+ {Start: 0, End: len([]rune(prefix)), Locator: types.SourceLocator{
+ Type: types.SourceLocatorSheet, Sheet: "Sheet1", RowStart: 1, RowEnd: 13,
+ }},
+ {Start: len([]rune(prefix)), End: len([]rune(text)), Locator: types.SourceLocator{
+ Type: types.SourceLocatorSheet, Sheet: "Sheet1", RowStart: 14, RowEnd: 14,
+ }},
+ })
+ parentCfg := SplitterConfig{
+ ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy,
+ }
+ childCfg := parentCfg
+ childCfg.ChunkSize = 12
+ result := SplitParentChild(text, parentCfg, childCfg)
+ for _, child := range result.Children {
+ if child.Content != lastRow {
+ continue
+ }
+ // ProcessDocument assigns source evidence with these exact offsets.
+ locators := index.Locators(child.Start, child.End)
+ if len(locators) != 1 {
+ t.Fatalf("last row has %d source locators, want one", len(locators))
+ }
+ loc := locators[0]
+ if loc.Sheet != "Sheet1" || loc.RowStart != 14 || loc.RowEnd != 14 || loc.Quote != "| last | row |" {
+ t.Fatalf("last row points to the wrong source: %+v", loc)
+ }
+ return
+ }
+ t.Fatal("last table row was not retained as a child")
+}
diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go
index 69ff38e43..23b6596c7 100644
--- a/internal/infrastructure/chunker/splitter.go
+++ b/internal/infrastructure/chunker/splitter.go
@@ -874,7 +874,7 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
var children []ChildChunk
childSeq := 0
for _, parent := range parents {
- subs := SplitText(parent.Content, childCfg)
+ subs := mapChildrenToSource(parent, SplitText(parent.Content, childCfg))
parentIndex := -1
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
@@ -883,13 +883,7 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
}
for _, sub := range subs {
- // Adjust offsets: sub positions are relative to parent content,
- // shift to document-level offsets.
- // Use additive shift (not Content-length based) so that chunks with
- // prepended context headers keep correct positional tracking.
sub.Seq = childSeq
- sub.Start += parent.Start
- sub.End += parent.Start
children = append(children, ChildChunk{
Chunk: sub,
ParentIndex: parentIndex,
@@ -900,6 +894,24 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
return ParentChildResult{Parents: newParents, Children: children}
}
+// mapChildrenToSource translates offsets in parent.Content to document offsets.
+// A parent may start with a synthetic table header that occupies no source
+// positions. Keep that context in Content, but exclude it from the mapped span
+// and discard children that contain only this synthetic prefix.
+func mapChildrenToSource(parent Chunk, children []Chunk) []Chunk {
+ offset := parent.End - runeLen(parent.Content)
+ out := children[:0]
+ for _, child := range children {
+ child.Start = max(parent.Start, offset+child.Start)
+ child.End += offset
+ if child.End <= child.Start {
+ continue
+ }
+ out = append(out, child)
+ }
+ return out
+}
+
// ExtractImageRefs extracts markdown image references from text.
// The URL group supports one level of balanced parentheses so that URLs
// like https://example.com/item_(abc)/123 are captured in full.
diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go
index 4766462a7..578c98b8f 100644
--- a/internal/infrastructure/chunker/strategy.go
+++ b/internal/infrastructure/chunker/strategy.go
@@ -166,7 +166,7 @@ func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagn
var children []ChildChunk
childSeq := 0
for _, parent := range parents {
- subs := Split(parent.Content, childCfg)
+ subs := mapChildrenToSource(parent, Split(parent.Content, childCfg))
parentIndex := -1
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
@@ -175,8 +175,6 @@ func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagn
}
for _, sub := range subs {
sub.Seq = childSeq
- sub.Start += parent.Start
- sub.End += parent.Start
sub.ContextHeader = mergeBreadcrumbs(parent.ContextHeader, sub.ContextHeader)
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
childSeq++