Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(chunker): preserve source offsets for table child chunks (#3969)
Yeager0204e915adb147d7
diff --git a/internal/infrastructure/chunker/parent_child_offsets_test.go b/internal/infrastructure/chunker/parent_child_offsets_test.gonew file mode 100644index 000000000..f87a7a25c--- /dev/null+++ b/internal/infrastructure/chunker/parent_child_offsets_test.go@@ -0,0 +1,148 @@+package chunker++import (+ "fmt"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/sourceloc"+ "github.com/Tencent/WeKnora/internal/types"+)++func TestParentChildTableSourceOffsets(t *testing.T) {+ splitters := []struct {+ name string+ split func(string, SplitterConfig, SplitterConfig) ParentChildResult+ }{+ {"legacy", SplitTextParentChild},+ {"adaptive", SplitParentChild},+ {"diagnostics", func(text string, parent, child SplitterConfig) ParentChildResult {+ result, _ := SplitParentChildWithDiagnostics(text, parent, child)+ return result+ }},+ }+ for _, fixture := range []struct {+ name, newline, row string+ }{+ {"ascii", "\n", "| apple | red |"},+ {"unicode", "\n", "| 苹果🍎 | 北京 |"},+ {"crlf", "\r\n", "| 苹果🍎 | 北京 |"},+ } {+ header := "| Item | City |" + fixture.newline + "| --- | --- |" + fixture.newline+ // Repeated rows also catch attempts to recover positions with a text search.+ text := "Intro" + fixture.newline + header ++ strings.Repeat(fixture.row+fixture.newline, 12) + "| last | row |" + fixture.newline+ parentCfg := SplitterConfig{+ ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy,+ }+ // Ensure this exercises a parent with a synthetic header, not merely+ // children with headers inside a single source-backed parent.+ parents := SplitText(text, parentCfg)+ sawSyntheticParent := false+ for _, parent := range parents {+ if len([]rune(parent.Content)) > parent.End-parent.Start {+ sawSyntheticParent = true+ }+ }+ if !sawSyntheticParent {+ t.Fatalf("%s fixture did not produce a parent with a synthetic header", fixture.name)+ }+ for _, splitter := range splitters {+ // 12 isolates the virtual header as a child; 50 re-splits table+ // rows; 160 keeps the parent intact (ParentIndex == -1).+ for _, childSize := range []int{12, 50, 160} {+ t.Run(fmt.Sprintf("%s/%s/child=%d", fixture.name, splitter.name, childSize), func(t *testing.T) {+ childCfg := parentCfg+ childCfg.ChunkSize = childSize+ result := splitter.split(text, parentCfg, childCfg)+ if len(result.Children) == 0 {+ t.Fatal("expected children")+ }+ textRunes := []rune(text)+ covered := make([]bool, len(textRunes))+ var chunks []Chunk+ for i, child := range result.Children {+ if child.Seq != i {+ t.Errorf("child %d has sequence %d", i, child.Seq)+ }+ if child.Start < 0 || child.End > len(textRunes) || child.Start >= child.End {+ t.Fatalf("child %d has invalid source span [%d, %d), document length %d",+ i, child.Start, child.End, len(textRunes))+ }+ content := []rune(child.Content)+ span := child.End - child.Start+ if span > len(content) {+ t.Fatalf("child %d source span exceeds content length", i)+ }+ want := string(textRunes[child.Start:child.End])+ if got := string(content[len(content)-span:]); got != want {+ t.Fatalf("child %d source span [%d, %d): got suffix %q, want %q",+ i, child.Start, child.End, got, want)+ }+ if child.ParentIndex >= 0 {+ if child.ParentIndex >= len(result.Parents) {+ t.Fatalf("child %d has invalid parent index %d", i, child.ParentIndex)+ }+ parent := result.Parents[child.ParentIndex]+ if child.Start < parent.Start || child.End > parent.End {+ t.Errorf("child %d source span escapes its parent", i)+ }+ } else if childSize == 160 && strings.Contains(child.Content, fixture.row) {+ if !strings.Contains(child.Content, "| Item | City |") {+ t.Errorf("child %d lost table header context", i)+ }+ }+ for pos := child.Start; pos < child.End; pos++ {+ covered[pos] = true+ }+ chunks = append(chunks, child.Chunk)+ }+ for pos, ok := range covered {+ if !ok {+ t.Fatalf("source rune %d is missing from children", pos)+ }+ }+ if got := restoreTextFromChunks(chunks); got != text {+ t.Fatalf("source reconstruction mismatch: got %q, want %q", got, text)+ }+ })+ }+ }+ }+}++func TestParentChildTableCitationUsesSourceRow(t *testing.T) {+ prefix := "| Item | City |\n| --- | --- |\n" + strings.Repeat("| 苹果🍎 | 北京 |\n", 12)+ lastRow := "| last | row |\n"+ text := prefix + lastRow+ index := sourceloc.NewIndex(text, []types.SourceBlock{+ {Start: 0, End: len([]rune(prefix)), Locator: types.SourceLocator{+ Type: types.SourceLocatorSheet, Sheet: "Sheet1", RowStart: 1, RowEnd: 13,+ }},+ {Start: len([]rune(prefix)), End: len([]rune(text)), Locator: types.SourceLocator{+ Type: types.SourceLocatorSheet, Sheet: "Sheet1", RowStart: 14, RowEnd: 14,+ }},+ })+ parentCfg := SplitterConfig{+ ChunkSize: 80, ChunkOverlap: 5, Separators: []string{"\n\n", "\n"}, Strategy: StrategyLegacy,+ }+ childCfg := parentCfg+ childCfg.ChunkSize = 12+ result := SplitParentChild(text, parentCfg, childCfg)+ for _, child := range result.Children {+ if child.Content != lastRow {+ continue+ }+ // ProcessDocument assigns source evidence with these exact offsets.+ locators := index.Locators(child.Start, child.End)+ if len(locators) != 1 {+ t.Fatalf("last row has %d source locators, want one", len(locators))+ }+ loc := locators[0]+ if loc.Sheet != "Sheet1" || loc.RowStart != 14 || loc.RowEnd != 14 || loc.Quote != "| last | row |" {+ t.Fatalf("last row points to the wrong source: %+v", loc)+ }+ return+ }+ t.Fatal("last table row was not retained as a child")+}diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.goindex 69ff38e43..23b6596c7 100644--- a/internal/infrastructure/chunker/splitter.go+++ b/internal/infrastructure/chunker/splitter.go@@ -874,7 +874,7 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Parenvar children []ChildChunkchildSeq := 0for _, parent := range parents {- subs := SplitText(parent.Content, childCfg)+ subs := mapChildrenToSource(parent, SplitText(parent.Content, childCfg))parentIndex := -1if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {@@ -883,13 +883,7 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren}for _, sub := range subs {- // Adjust offsets: sub positions are relative to parent content,- // shift to document-level offsets.- // Use additive shift (not Content-length based) so that chunks with- // prepended context headers keep correct positional tracking.sub.Seq = childSeq- sub.Start += parent.Start- sub.End += parent.Startchildren = append(children, ChildChunk{Chunk: sub,ParentIndex: parentIndex,@@ -900,6 +894,24 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Parenreturn ParentChildResult{Parents: newParents, Children: children}}+// mapChildrenToSource translates offsets in parent.Content to document offsets.+// A parent may start with a synthetic table header that occupies no source+// positions. Keep that context in Content, but exclude it from the mapped span+// and discard children that contain only this synthetic prefix.+func mapChildrenToSource(parent Chunk, children []Chunk) []Chunk {+ offset := parent.End - runeLen(parent.Content)+ out := children[:0]+ for _, child := range children {+ child.Start = max(parent.Start, offset+child.Start)+ child.End += offset+ if child.End <= child.Start {+ continue+ }+ out = append(out, child)+ }+ return out+}+// ExtractImageRefs extracts markdown image references from text.// The URL group supports one level of balanced parentheses so that URLs// like https://example.com/item_(abc)/123 are captured in full.diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.goindex 4766462a7..578c98b8f 100644--- a/internal/infrastructure/chunker/strategy.go+++ b/internal/infrastructure/chunker/strategy.go@@ -166,7 +166,7 @@ func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagnvar children []ChildChunkchildSeq := 0for _, parent := range parents {- subs := Split(parent.Content, childCfg)+ subs := mapChildrenToSource(parent, Split(parent.Content, childCfg))parentIndex := -1if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {@@ -175,8 +175,6 @@ func splitParentChild(text string, parentCfg, childCfg SplitterConfig, withDiagn}for _, sub := range subs {sub.Seq = childSeq- sub.Start += parent.Start- sub.End += parent.Startsub.ContextHeader = mergeBreadcrumbs(parent.ContextHeader, sub.ContextHeader)children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})childSeq++