RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): prevent chained over-carry of overlap-head PDF positions (#19068)
xugangqiangfeeb4a7016cb
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex a355477448..22f6ff132d 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -901,41 +901,79 @@ type mergeItem struct {Positions json.RawMessage}-// overlapTailPositions returns the coordinate boxes of the previous chunk's-// source items whose visible span intersects the overlap tail-// [overlapStart, total). PDF positions are per-item (coarse), so an item is-// included wholesale once any part of it falls in the overlap tail. This keeps-// the overlap prefix highlighted without over-inflating the box set with the-// previous chunk's non-overlap (head) coordinates (#18148). Offsets are in-// rune units to match computeOverlapPrefix's visible-text indexing.-func overlapTailPositions(prevItems []mergeItem, overlapStart int, joinSep string) (json.RawMessage, json.RawMessage) {+// overlapTailItems returns the previous chunk's source items whose visible span+// intersects the overlap tail [overlapStart, total), each truncated to exactly+// the portion that lies within the overlap tail. Storing them as SEPARATE+// merged_items entries (alongside cur) lets the next overlap select only the+// true tail and converge to a sliding window, instead of carrying the previous+// chunk's HEAD coordinates forward again -- the chained over-carry defect that+// the old fused single-item storage (superseded by this function, #18148)+// suffers from.+//+// Offset model: the overlap path stores the previous chunk's merged_items as+// [tailItems..., cur] and builds cp.Text = overlap + cp.Text. So the source+// items are concatenated with joinSep BETWEEN consecutive items, but there is+// NO separator before the final (cur) item. The offsets below reproduce exactly+// that layout for an overlap-built predecessor, so they line up with+// overlapCut/overlapFitPrefix, which carve the overlap from removeTag'd text.+// Inserting a separator before cur (as the raw "concatenate all items with+// joinSep" model does) would inflate the offsets by one rune on the JSON path+// (joinSep="\n") and over-truncate the tail -- the chained-overlap convergence+// defect flagged by CodeRabbit #1 on #19068.+//+// Limitation: a chunk may also have been built via the non-overlap merge path+// (mergeUnits appends cur with "prev.Text + joinSep + ck.Text"), in which case+// its items DO carry a joinSep before the final cur item. For such a+// predecessor the last item's reconstructed start is short by one rune+// (joinSep length), so a final item that is partially inside the overlap tail+// is truncated one rune too short. This is a 1-rune approximation of the+// highlight box only; it does not affect the head-exclusion property this+// function exists to enforce.+func overlapTailItems(prevItems []mergeItem, overlapStart int, joinSep string) []mergeItem {if len(prevItems) == 0 {- return nil, nil+ return nil}- // Items are concatenated with joinSep (a single "\n" for the JSON path),- // matching the merge join at mergeUnits. Offsets are measured on the- // TAG-FREE visible text so they line up with overlapCut/overlapFitPrefix,- // which carve the overlap from removeTag'd text; a coordinate tag in an- // earlier item must not shift the boundaries of later items.+ // Measure the TAG-FREE visible text of each item so a coordinate tag in an+ // earlier item does not shift the boundaries of later items.visible := make([]string, len(prevItems))- total := 0for i, it := range prevItems {visible[i] = removeTag(it.Text)- total += utf8.RuneCountInString(visible[i]) + utf8.RuneCountInString(joinSep)}- total -= utf8.RuneCountInString(joinSep)- var pdfAcc, posAcc json.RawMessage+ n := len(prevItems)+ sepLen := utf8.RuneCountInString(joinSep)+ // Reconstruct prev.Text: items joined by joinSep between consecutive items,+ // but with NO separator before the final (cur) item.+ starts := make([]int, n)+ ends := make([]int, n)offset := 0+ for i := 0; i < n; i++ {+ if i > 0 && i < n-1 {+ offset += sepLen+ }+ starts[i] = offset+ ends[i] = offset + utf8.RuneCountInString(visible[i])+ offset = ends[i]+ }+ total := offset+ var out []mergeItemfor i, it := range prevItems {- start := offset- end := offset + utf8.RuneCountInString(visible[i])+ start := starts[i]+ end := ends[i]if start < total && end > overlapStart {- pdfAcc = extendRawJSONArray(pdfAcc, it.PDFPositions)- posAcc = extendRawJSONArray(posAcc, it.Positions)+ lo := start+ if lo < overlapStart {+ lo = overlapStart+ }+ hi := end+ if hi > total {+ hi = total+ }+ runes := []rune(visible[i])+ slice := string(runes[lo-start : hi-start])+ out = append(out, mergeItem{Text: slice, PDFPositions: it.PDFPositions, Positions: it.Positions})}- offset = end + utf8.RuneCountInString(joinSep)}- return pdfAcc, posAcc+ return out}func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {@@ -1011,13 +1049,24 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep// trimmed to fit so the hard cap still holds.overlap, _ := computeOverlapPrefix(prev.Text, overlapPct)overlap, trimRunes := overlapFitPrefix(overlap, cp.Text, target)- pdfTail, posTail := overlapTailPositions(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct)+trimRunes, joinSep)+ // Keep the previous chunk's tail as SEPARATE merged_items+ // entries (truncated to the overlap portion, #18148) so the+ // next overlap selects only the true tail and the window+ // converges instead of accumulating head coordinates (chained+ // over-carry fix). The flattened tail boxes are still carried+ // into the output chunk so the overlap region stays highlighted.+ tailItems := overlapTailItems(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct)+trimRunes, joinSep)+ var pdfTail, posTail json.RawMessage+ for _, it := range tailItems {+ pdfTail = extendRawJSONArray(pdfTail, it.PDFPositions)+ posTail = extendRawJSONArray(posTail, it.Positions)+ }cp.Text = overlap + cp.Textcp.PDFPositions = extendRawJSONArray(pdfTail, cp.PDFPositions)cp.Positions = extendRawJSONArray(posTail, cp.Positions)cp.TKNums = intPtr(tokenizeStr(cp.Text))merged = append(merged, cp)- mergedItems = append(mergedItems, []mergeItem{{Text: cp.Text, PDFPositions: cp.PDFPositions, Positions: cp.Positions}})+ mergedItems = append(mergedItems, append(append([]mergeItem{}, tailItems...), cur))prevIdx = len(merged) - 1continue}diff --git a/internal/ingestion/component/chunker/token_overlap_test.go b/internal/ingestion/component/chunker/token_overlap_test.goindex c050eab68d..50fcb46041 100644--- a/internal/ingestion/component/chunker/token_overlap_test.go+++ b/internal/ingestion/component/chunker/token_overlap_test.go@@ -2,7 +2,6 @@ package chunkerimport ("context"- "encoding/json""strings""testing")@@ -92,27 +91,3 @@ func TestTokenChunker_TextOverlapPreservesInterLineSpace(t *testing.T) {t.Errorf("no chunk preserved an inter-line space across a boundary; the regression fixture is not exercised")}}--// TestOverlapTailPositions_VisibleTextOffsets pins the visible-text offset-// contract: overlapCut/overlapFitPrefix measure the cut on the tag-free visible-// text, so overlapTailPositions must map item spans using the same visible-// lengths. A coordinate tag in an earlier item must not shift the boundaries of-// later items (which would pull head coordinates into the overlap tail).-func TestOverlapTailPositions_VisibleTextOffsets(t *testing.T) {- items := []mergeItem{- // visible text "hello world" (11 runes); the tag inflates the RAW length- // to 23 runes.- {Text: "hello@@1\t2\t3\t4## world", PDFPositions: json.RawMessage(`[["p0"]]`), Positions: json.RawMessage(`[["q0"]]`)},- {Text: "tail", PDFPositions: json.RawMessage(`[["p1"]]`), Positions: json.RawMessage(`[["q1"]]`)},- }- // Visible layout: item0 [0,11), joinSep "\n" at 11, item1 [12,16). An- // overlapStart of 12 lands in item1 only, so item0's head coordinates must- // NOT be included.- pdf, pos := overlapTailPositions(items, 12, "\n")- if string(pdf) != `[["p1"]]` {- t.Errorf("pdf positions = %s, want only item1 [\"p1\"] (item0 head must be excluded)", pdf)- }- if string(pos) != `[["q1"]]` {- t.Errorf("positions = %s, want only item1 [\"q1\"] (item0 head must be excluded)", pos)- }-}diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.goindex a89420f97a..5e01c2cca7 100644--- a/internal/ingestion/component/chunker/token_pdfpos_test.go+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go@@ -18,6 +18,7 @@ package chunkerimport ("encoding/json"+ "fmt""math""strings""testing"@@ -186,10 +187,10 @@ func TestMergeByTokenSizeFromJSON_OverlapPrefixCarriesPrevPositions(t *testing.T// into the overlap prefix; here overlappedPct=20 means the overlap prefix is// only the TAIL ~20% of the previous chunk. The coordinates carried must be// exactly the previous chunk's tail items whose span intersects that tail ---// NOT the whole previous chunk. This locks the per-item tail-selection in-// overlapTailPositions (token.go:832): a regression that carried the entire-// previous chunk's coordinates (over-inflating the highlight box) or dropped-// overlap coordinates entirely would both fail this test.+// NOT the whole previous chunk. This locks the per-item tail-selection now+// done by overlapTailItems: a regression that carried the entire previous+// chunk's coordinates (over-inflating the highlight box) or dropped overlap+// coordinates entirely would both fail this test.func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(t *testing.T) {posA := json.RawMessage(`[[1,0,10,0,5]]`)posB := json.RawMessage(`[[2,0,20,0,8]]`)@@ -418,3 +419,206 @@ func TestSplitOversizedText_SlicesPositionsIgnoreTags(t *testing.T) {t.Errorf("last piece bottom = %v, want %v (box fully covered)", prevBottom, boxBottom)}}++// TestMergeByTokenSizeFromJSON_ChainedOverlapNoHeadAccumulation is the TDD test+// for the chained-overlap coordinate defect (CodeRabbit review #2 on PR #18227).+// The single-overlap tests above lock one overlap boundary, but when EVERY unit+// starts a fresh chunk (each unit over budget for the token cap with+// overlapped_percent>0) the overlap prefix is prepended at EVERY boundary. The+// buggy code stored each new chunk as ONE fused merged_item+// ({Text: overlap+cur, PDFPositions: <all prev-tail boxes>+cur}), so on the NEXT+// overlap the whole fused item was selected and the previous chunk's HEAD boxes+// were carried forward again -- accumulating {unit0, unit1, ...} coordinates into+// the final chunk while its text only contains the tail+cur.+//+// The fix keeps the tail items and cur as SEPARATE merged_items entries (with+// the tail item texts truncated to the overlap portion), so the next overlap+// selects only the true tail and the window converges to a sliding window+// instead of the full accumulation. This test fails (RED) until merged_items+// entries are stored separately.+func TestMergeByTokenSizeFromJSON_ChainedOverlapNoHeadAccumulation(t *testing.T) {+ pos := func(page int) json.RawMessage {+ return json.RawMessage(fmt.Sprintf("[[%d,0,%d,0,%d]]", page, page*10, page*5))+ }+ // Many single-token units so they merge into several chunks, each joined to+ // the previous one with a 20% overlap prefix. This exercises the CHAINED+ // overlap path (every chunk boundary prepends the previous chunk's tail):+ // the buggy fused single-item storage would carry the previous chunk's HEAD+ // coordinates forward on every overlap, accumulating the earliest units'+ // boxes into the final chunk. The fix stores tail items separately so the+ // window converges to a sliding window.+ const n = 24+ var row []schema.ChunkDoc+ for i := 0; i < n; i++ {+ row = append(row, schema.ChunkDoc{+ Text: string(rune('a' + i)),+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(1),+ PDFPositions: pos(i + 1),+ })+ }+ items := [][]schema.ChunkDoc{row}+ got := mergeByTokenSizeFromJSON(items, 9, 20)+ merged := got[0]+ if len(merged) < 3 {+ t.Fatalf("want several chained chunks (>2), got %d", len(merged))+ }+ last := string(merged[len(merged)-1].PDFPositions)+ // The final chunk must NOT still carry unit0's (page 1) coordinates: the+ // overlap window converges to only the tail of the previous chunk + its own.+ if strings.Contains(last, "1,0,10,0,5") {+ t.Errorf("final chunk accumulated unit0 head coords (chained over-carry): pdf_positions=%s", last)+ }+ // Sanity: the final chunk must still carry the most recent unit's coords.+ if !strings.Contains(last, fmt.Sprintf("%d,0,%d,0,%d", n, n*10, n*5)) {+ t.Errorf("final chunk lost its own coordinates (unit%d): pdf_positions=%s", n, last)+ }+}++// TestOverlapTailItemsTruncatesToOverlapPortion locks the new behavior of+// overlapTailItems: each returned item's Text is truncated to exactly the span+// that lies within the overlap tail, and only items intersecting the tail are+// returned (the previous chunk's head items are excluded). This is what stops+// the chained over-carry: storing the full fused item would drag head+// coordinates into the next overlap.+func TestOverlapTailItemsTruncatesToOverlapPortion(t *testing.T) {+ prev := []mergeItem{+ {Text: "aaaaa", PDFPositions: json.RawMessage("[[1,0,10,0,5]]"), Positions: json.RawMessage("[[1,0,10,0,5]]")},+ {Text: "bbbbb", PDFPositions: json.RawMessage("[[2,0,20,0,8]]"), Positions: json.RawMessage("[[2,0,20,0,8]]")},+ }+ // total = 10 visible runes; overlap tail is [8,10).+ tail := overlapTailItems(prev, 8, "")+ if len(tail) != 1 {+ t.Fatalf("want 1 tail item in overlap [8,10), got %d: %+v", len(tail), tail)+ }+ if tail[0].Text != "bb" {+ t.Errorf("tail item Text not truncated to overlap portion: got %q, want %q", tail[0].Text, "bb")+ }+ if string(tail[0].PDFPositions) != "[[2,0,20,0,8]]" {+ t.Errorf("tail item carried wrong PDF positions: %s", tail[0].PDFPositions)+ }+ // The head item (page 1) must not appear in the tail.+ for _, it := range tail {+ if string(it.PDFPositions) == "[[1,0,10,0,5]]" {+ t.Errorf("head item (page1) leaked into overlap tail: %+v", it)+ }+ }+}++// TestOverlapTailItems_JoinSepNoSeparatorBeforeCur is the TDD (RED) test for the+// joinSep offset-model defect (CodeRabbit review #1 on PR #19068). The overlap+// path stores the previous chunk's merged_items as [tailItems..., cur] and builds+// cp.Text = overlap + cp.Text -- there is NO separator between the final tail+// item and cur. But overlapTailItems reconstructs the rune offsets by inserting+// joinSep before EVERY item, including before cur. On the JSON path (joinSep="\n")+// that phantom separator shifts the offsets by one rune, so the tail slice is+// truncated one rune too long (and, in a chained overlap, the window does not+// converge as tightly as the displayed text implies).+//+// This test feeds overlapTailItems the structure the overlap path actually+// produces -- tail item "aaaaa" followed by cur "bbbbb" with NO separator in the+// real text (10 visible runes), overlap tail [7,10) -- and expects the tail of+// cur to be truncated to exactly "bbb" (the last 3 runes). The buggy code inserts+// a "\n" between "aaaaa" and "bbbbb", inflating the model to 11 runes and+// returning "bbbb" instead. It fails (RED) until the offset model drops the+// separator before the final (cur) item.+func TestOverlapTailItems_JoinSepNoSeparatorBeforeCur(t *testing.T) {+ prev := []mergeItem{+ {Text: "aaaaa", PDFPositions: json.RawMessage("[[1,0,10,0,5]]"), Positions: json.RawMessage("[[1,0,10,0,5]]")},+ {Text: "bbbbb", PDFPositions: json.RawMessage("[[2,0,20,0,8]]"), Positions: json.RawMessage("[[2,0,20,0,8]]")},+ }+ // Real overlap-chunk text = "aaaaa" + "bbbbb" (10 visible runes, no sep).+ // Overlap tail is the last 3 runes [7,10) of cur.+ tail := overlapTailItems(prev, 7, "\n")+ if len(tail) != 1 {+ t.Fatalf("want 1 tail item in overlap [7,10), got %d: %+v", len(tail), tail)+ }+ if tail[0].Text != "bbb" {+ t.Errorf("tail item Text not truncated to overlap portion under joinSep: got %q, want %q (phantom separator shifted the offset)", tail[0].Text, "bbb")+ }+ if string(tail[0].PDFPositions) != "[[2,0,20,0,8]]" {+ t.Errorf("tail item carried wrong PDF positions: %s", tail[0].PDFPositions)+ }+ // The head item (page 1) must not appear in the tail.+ for _, it := range tail {+ if string(it.PDFPositions) == "[[1,0,10,0,5]]" {+ t.Errorf("head item (page1) leaked into overlap tail: %+v", it)+ }+ }+}++// TestOverlapTailItems_TagBearingUsesVisibleRuneOffsets pins that overlapTailItems+// measures and slices on the TAG-FREE visible text, so a coordinate tag in an+// earlier item does not shift the boundaries of later items (which would pull+// head coordinates into the overlap tail). This mirrors the contract already+// locked for the previous fused-item storage but for the separate-item storage.+func TestOverlapTailItems_TagBearingUsesVisibleRuneOffsets(t *testing.T) {+ prev := []mergeItem{+ // Visible text "hello world" (11 runes); the tag inflates the RAW length+ // to 23 runes. If the code measured RAW length, item0 would span [0,23)+ // and overlapStart=12 would fall inside it (leaking head coords). Using+ // the tag-free visible length, item0 spans [0,11) and item1 owns [11,15).+ {Text: "hello@@1\t2\t3\t4## world", PDFPositions: json.RawMessage(`[["p0"]]`), Positions: json.RawMessage(`[["q0"]]`)},+ {Text: "tail", PDFPositions: json.RawMessage(`[["p1"]]`), Positions: json.RawMessage(`[["q1"]]`)},+ }+ // Visible layout (no separator before cur, matching the overlap path):+ // item0 [0,11), item1 [11,15). overlapStart=12 lands inside item1 only, so+ // item0's head coordinates must NOT be included and item1 is truncated to+ // the overlap portion [12,15) == "ail" (the last 3 runes of "tail").+ tail := overlapTailItems(prev, 12, "\n")+ if len(tail) != 1 {+ t.Fatalf("want 1 tail item in overlap [12,15), got %d: %+v", len(tail), tail)+ }+ if string(tail[0].PDFPositions) != `[["p1"]]` {+ t.Errorf("pdf positions = %s, want only item1 [\"p1\"] (item0 head must be excluded)", tail[0].PDFPositions)+ }+ if tail[0].Text != "ail" {+ t.Errorf("tail item Text = %q, want %q (visible rune offsets must ignore the tag and truncate to the overlap portion)", tail[0].Text, "ail")+ }+}++// TestMergeByTokenSizeFromJSON_ChainedOverlapPerChunkConvergence strengthens the+// chained-overlap regression: not only must the FINAL chunk avoid head+// coordinates, EVERY chunk's carried PDF positions must contain only the true+// tail of its predecessor plus its own unit -- never any earlier (head) unit.+// This locks convergence at every boundary, not just at the end of the chain.+func TestMergeByTokenSizeFromJSON_ChainedOverlapPerChunkConvergence(t *testing.T) {+ pos := func(page int) json.RawMessage {+ return json.RawMessage(fmt.Sprintf("[[%d,0,%d,0,%d]]", page, page*10, page*5))+ }+ const n = 24+ var row []schema.ChunkDoc+ for i := 0; i < n; i++ {+ row = append(row, schema.ChunkDoc{+ Text: string(rune('a' + i)),+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(1),+ PDFPositions: pos(i + 1),+ })+ }+ items := [][]schema.ChunkDoc{row}+ merged := mergeByTokenSizeFromJSON(items, 9, 20)[0]+ if len(merged) < 3 {+ t.Fatalf("want several chained chunks (>2), got %d", len(merged))+ }+ // The first chunk legitimately owns unit0 (page 1); every LATER chunk must+ // NOT carry it. The overlap window converges to only the previous chunk's+ // tail + the current unit, so the ultimate head (unit0) must never reappear+ // after chunk 0. unit1 (page 2) must not reappear after chunk 1 either.+ for k := 1; k < len(merged); k++ {+ p := string(merged[k].PDFPositions)+ if strings.Contains(p, "1,0,10,0,5") {+ t.Errorf("chunk %d carries stale head coords (unit0/page1): pdf_positions=%s", k, p)+ }+ if k >= 2 && strings.Contains(p, "2,0,20,0,8") {+ t.Errorf("chunk %d carries stale head coords (unit1/page2): pdf_positions=%s", k, p)+ }+ }+ // Sanity: the final chunk must still carry its own most-recent unit's coords.+ last := string(merged[len(merged)-1].PDFPositions)+ if !strings.Contains(last, fmt.Sprintf("%d,0,%d,0,%d", n, n*10, n*5)) {+ t.Errorf("final chunk lost its own coordinates (unit%d): pdf_positions=%s", n, last)+ }+}