Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): prevent chained over-carry of overlap-head PDF positions (#19068)

xugangqiangfeeb4a7016cb
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index a355477448..22f6ff132d 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -901,41 +901,79 @@ type mergeItem struct {
Positions json.RawMessage
}
-// overlapTailPositions returns the coordinate boxes of the previous chunk's
-// source items whose visible span intersects the overlap tail
-// [overlapStart, total). PDF positions are per-item (coarse), so an item is
-// included wholesale once any part of it falls in the overlap tail. This keeps
-// the overlap prefix highlighted without over-inflating the box set with the
-// previous chunk's non-overlap (head) coordinates (#18148). Offsets are in
-// rune units to match computeOverlapPrefix's visible-text indexing.
-func overlapTailPositions(prevItems []mergeItem, overlapStart int, joinSep string) (json.RawMessage, json.RawMessage) {
+// overlapTailItems returns the previous chunk's source items whose visible span
+// intersects the overlap tail [overlapStart, total), each truncated to exactly
+// the portion that lies within the overlap tail. Storing them as SEPARATE
+// merged_items entries (alongside cur) lets the next overlap select only the
+// true tail and converge to a sliding window, instead of carrying the previous
+// chunk's HEAD coordinates forward again -- the chained over-carry defect that
+// the old fused single-item storage (superseded by this function, #18148)
+// suffers from.
+//
+// Offset model: the overlap path stores the previous chunk's merged_items as
+// [tailItems..., cur] and builds cp.Text = overlap + cp.Text. So the source
+// items are concatenated with joinSep BETWEEN consecutive items, but there is
+// NO separator before the final (cur) item. The offsets below reproduce exactly
+// that layout for an overlap-built predecessor, so they line up with
+// overlapCut/overlapFitPrefix, which carve the overlap from removeTag'd text.
+// Inserting a separator before cur (as the raw "concatenate all items with
+// joinSep" model does) would inflate the offsets by one rune on the JSON path
+// (joinSep="\n") and over-truncate the tail -- the chained-overlap convergence
+// defect flagged by CodeRabbit #1 on #19068.
+//
+// Limitation: a chunk may also have been built via the non-overlap merge path
+// (mergeUnits appends cur with "prev.Text + joinSep + ck.Text"), in which case
+// its items DO carry a joinSep before the final cur item. For such a
+// predecessor the last item's reconstructed start is short by one rune
+// (joinSep length), so a final item that is partially inside the overlap tail
+// is truncated one rune too short. This is a 1-rune approximation of the
+// highlight box only; it does not affect the head-exclusion property this
+// function exists to enforce.
+func overlapTailItems(prevItems []mergeItem, overlapStart int, joinSep string) []mergeItem {
if len(prevItems) == 0 {
- return nil, nil
+ return nil
}
- // Items are concatenated with joinSep (a single "\n" for the JSON path),
- // matching the merge join at mergeUnits. Offsets are measured on the
- // TAG-FREE visible text so they line up with overlapCut/overlapFitPrefix,
- // which carve the overlap from removeTag'd text; a coordinate tag in an
- // earlier item must not shift the boundaries of later items.
+ // Measure the TAG-FREE visible text of each item so a coordinate tag in an
+ // earlier item does not shift the boundaries of later items.
visible := make([]string, len(prevItems))
- total := 0
for i, it := range prevItems {
visible[i] = removeTag(it.Text)
- total += utf8.RuneCountInString(visible[i]) + utf8.RuneCountInString(joinSep)
}
- total -= utf8.RuneCountInString(joinSep)
- var pdfAcc, posAcc json.RawMessage
+ n := len(prevItems)
+ sepLen := utf8.RuneCountInString(joinSep)
+ // Reconstruct prev.Text: items joined by joinSep between consecutive items,
+ // but with NO separator before the final (cur) item.
+ starts := make([]int, n)
+ ends := make([]int, n)
offset := 0
+ for i := 0; i < n; i++ {
+ if i > 0 && i < n-1 {
+ offset += sepLen
+ }
+ starts[i] = offset
+ ends[i] = offset + utf8.RuneCountInString(visible[i])
+ offset = ends[i]
+ }
+ total := offset
+ var out []mergeItem
for i, it := range prevItems {
- start := offset
- end := offset + utf8.RuneCountInString(visible[i])
+ start := starts[i]
+ end := ends[i]
if start < total && end > overlapStart {
- pdfAcc = extendRawJSONArray(pdfAcc, it.PDFPositions)
- posAcc = extendRawJSONArray(posAcc, it.Positions)
+ lo := start
+ if lo < overlapStart {
+ lo = overlapStart
+ }
+ hi := end
+ if hi > total {
+ hi = total
+ }
+ runes := []rune(visible[i])
+ slice := string(runes[lo-start : hi-start])
+ out = append(out, mergeItem{Text: slice, PDFPositions: it.PDFPositions, Positions: it.Positions})
}
- offset = end + utf8.RuneCountInString(joinSep)
}
- return pdfAcc, posAcc
+ return out
}
func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
@@ -1011,13 +1049,24 @@ func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep
// trimmed to fit so the hard cap still holds.
overlap, _ := computeOverlapPrefix(prev.Text, overlapPct)
overlap, trimRunes := overlapFitPrefix(overlap, cp.Text, target)
- pdfTail, posTail := overlapTailPositions(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct)+trimRunes, joinSep)
+ // Keep the previous chunk's tail as SEPARATE merged_items
+ // entries (truncated to the overlap portion, #18148) so the
+ // next overlap selects only the true tail and the window
+ // converges instead of accumulating head coordinates (chained
+ // over-carry fix). The flattened tail boxes are still carried
+ // into the output chunk so the overlap region stays highlighted.
+ tailItems := overlapTailItems(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct)+trimRunes, joinSep)
+ var pdfTail, posTail json.RawMessage
+ for _, it := range tailItems {
+ pdfTail = extendRawJSONArray(pdfTail, it.PDFPositions)
+ posTail = extendRawJSONArray(posTail, it.Positions)
+ }
cp.Text = overlap + cp.Text
cp.PDFPositions = extendRawJSONArray(pdfTail, cp.PDFPositions)
cp.Positions = extendRawJSONArray(posTail, cp.Positions)
cp.TKNums = intPtr(tokenizeStr(cp.Text))
merged = append(merged, cp)
- mergedItems = append(mergedItems, []mergeItem{{Text: cp.Text, PDFPositions: cp.PDFPositions, Positions: cp.Positions}})
+ mergedItems = append(mergedItems, append(append([]mergeItem{}, tailItems...), cur))
prevIdx = len(merged) - 1
continue
}
diff --git a/internal/ingestion/component/chunker/token_overlap_test.go b/internal/ingestion/component/chunker/token_overlap_test.go
index c050eab68d..50fcb46041 100644
--- a/internal/ingestion/component/chunker/token_overlap_test.go
+++ b/internal/ingestion/component/chunker/token_overlap_test.go
@@ -2,7 +2,6 @@ package chunker
import (
"context"
- "encoding/json"
"strings"
"testing"
)
@@ -92,27 +91,3 @@ func TestTokenChunker_TextOverlapPreservesInterLineSpace(t *testing.T) {
t.Errorf("no chunk preserved an inter-line space across a boundary; the regression fixture is not exercised")
}
}
-
-// TestOverlapTailPositions_VisibleTextOffsets pins the visible-text offset
-// contract: overlapCut/overlapFitPrefix measure the cut on the tag-free visible
-// text, so overlapTailPositions must map item spans using the same visible
-// lengths. A coordinate tag in an earlier item must not shift the boundaries of
-// later items (which would pull head coordinates into the overlap tail).
-func TestOverlapTailPositions_VisibleTextOffsets(t *testing.T) {
- items := []mergeItem{
- // visible text "hello world" (11 runes); the tag inflates the RAW length
- // to 23 runes.
- {Text: "hello@@1\t2\t3\t4## world", PDFPositions: json.RawMessage(`[["p0"]]`), Positions: json.RawMessage(`[["q0"]]`)},
- {Text: "tail", PDFPositions: json.RawMessage(`[["p1"]]`), Positions: json.RawMessage(`[["q1"]]`)},
- }
- // Visible layout: item0 [0,11), joinSep "\n" at 11, item1 [12,16). An
- // overlapStart of 12 lands in item1 only, so item0's head coordinates must
- // NOT be included.
- pdf, pos := overlapTailPositions(items, 12, "\n")
- if string(pdf) != `[["p1"]]` {
- t.Errorf("pdf positions = %s, want only item1 [\"p1\"] (item0 head must be excluded)", pdf)
- }
- if string(pos) != `[["q1"]]` {
- t.Errorf("positions = %s, want only item1 [\"q1\"] (item0 head must be excluded)", pos)
- }
-}
diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.go
index a89420f97a..5e01c2cca7 100644
--- a/internal/ingestion/component/chunker/token_pdfpos_test.go
+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go
@@ -18,6 +18,7 @@ package chunker
import (
"encoding/json"
+ "fmt"
"math"
"strings"
"testing"
@@ -186,10 +187,10 @@ func TestMergeByTokenSizeFromJSON_OverlapPrefixCarriesPrevPositions(t *testing.T
// into the overlap prefix; here overlappedPct=20 means the overlap prefix is
// only the TAIL ~20% of the previous chunk. The coordinates carried must be
// exactly the previous chunk's tail items whose span intersects that tail --
-// NOT the whole previous chunk. This locks the per-item tail-selection in
-// overlapTailPositions (token.go:832): a regression that carried the entire
-// previous chunk's coordinates (over-inflating the highlight box) or dropped
-// overlap coordinates entirely would both fail this test.
+// NOT the whole previous chunk. This locks the per-item tail-selection now
+// done by overlapTailItems: a regression that carried the entire previous
+// chunk's coordinates (over-inflating the highlight box) or dropped overlap
+// coordinates entirely would both fail this test.
func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(t *testing.T) {
posA := json.RawMessage(`[[1,0,10,0,5]]`)
posB := json.RawMessage(`[[2,0,20,0,8]]`)
@@ -418,3 +419,206 @@ func TestSplitOversizedText_SlicesPositionsIgnoreTags(t *testing.T) {
t.Errorf("last piece bottom = %v, want %v (box fully covered)", prevBottom, boxBottom)
}
}
+
+// TestMergeByTokenSizeFromJSON_ChainedOverlapNoHeadAccumulation is the TDD test
+// for the chained-overlap coordinate defect (CodeRabbit review #2 on PR #18227).
+// The single-overlap tests above lock one overlap boundary, but when EVERY unit
+// starts a fresh chunk (each unit over budget for the token cap with
+// overlapped_percent>0) the overlap prefix is prepended at EVERY boundary. The
+// buggy code stored each new chunk as ONE fused merged_item
+// ({Text: overlap+cur, PDFPositions: <all prev-tail boxes>+cur}), so on the NEXT
+// overlap the whole fused item was selected and the previous chunk's HEAD boxes
+// were carried forward again -- accumulating {unit0, unit1, ...} coordinates into
+// the final chunk while its text only contains the tail+cur.
+//
+// The fix keeps the tail items and cur as SEPARATE merged_items entries (with
+// the tail item texts truncated to the overlap portion), so the next overlap
+// selects only the true tail and the window converges to a sliding window
+// instead of the full accumulation. This test fails (RED) until merged_items
+// entries are stored separately.
+func TestMergeByTokenSizeFromJSON_ChainedOverlapNoHeadAccumulation(t *testing.T) {
+ pos := func(page int) json.RawMessage {
+ return json.RawMessage(fmt.Sprintf("[[%d,0,%d,0,%d]]", page, page*10, page*5))
+ }
+ // Many single-token units so they merge into several chunks, each joined to
+ // the previous one with a 20% overlap prefix. This exercises the CHAINED
+ // overlap path (every chunk boundary prepends the previous chunk's tail):
+ // the buggy fused single-item storage would carry the previous chunk's HEAD
+ // coordinates forward on every overlap, accumulating the earliest units'
+ // boxes into the final chunk. The fix stores tail items separately so the
+ // window converges to a sliding window.
+ const n = 24
+ var row []schema.ChunkDoc
+ for i := 0; i < n; i++ {
+ row = append(row, schema.ChunkDoc{
+ Text: string(rune('a' + i)),
+ DocType: "text",
+ CKType: "text",
+ TKNums: intPtr(1),
+ PDFPositions: pos(i + 1),
+ })
+ }
+ items := [][]schema.ChunkDoc{row}
+ got := mergeByTokenSizeFromJSON(items, 9, 20)
+ merged := got[0]
+ if len(merged) < 3 {
+ t.Fatalf("want several chained chunks (>2), got %d", len(merged))
+ }
+ last := string(merged[len(merged)-1].PDFPositions)
+ // The final chunk must NOT still carry unit0's (page 1) coordinates: the
+ // overlap window converges to only the tail of the previous chunk + its own.
+ if strings.Contains(last, "1,0,10,0,5") {
+ t.Errorf("final chunk accumulated unit0 head coords (chained over-carry): pdf_positions=%s", last)
+ }
+ // Sanity: the final chunk must still carry the most recent unit's coords.
+ if !strings.Contains(last, fmt.Sprintf("%d,0,%d,0,%d", n, n*10, n*5)) {
+ t.Errorf("final chunk lost its own coordinates (unit%d): pdf_positions=%s", n, last)
+ }
+}
+
+// TestOverlapTailItemsTruncatesToOverlapPortion locks the new behavior of
+// overlapTailItems: each returned item's Text is truncated to exactly the span
+// that lies within the overlap tail, and only items intersecting the tail are
+// returned (the previous chunk's head items are excluded). This is what stops
+// the chained over-carry: storing the full fused item would drag head
+// coordinates into the next overlap.
+func TestOverlapTailItemsTruncatesToOverlapPortion(t *testing.T) {
+ prev := []mergeItem{
+ {Text: "aaaaa", PDFPositions: json.RawMessage("[[1,0,10,0,5]]"), Positions: json.RawMessage("[[1,0,10,0,5]]")},
+ {Text: "bbbbb", PDFPositions: json.RawMessage("[[2,0,20,0,8]]"), Positions: json.RawMessage("[[2,0,20,0,8]]")},
+ }
+ // total = 10 visible runes; overlap tail is [8,10).
+ tail := overlapTailItems(prev, 8, "")
+ if len(tail) != 1 {
+ t.Fatalf("want 1 tail item in overlap [8,10), got %d: %+v", len(tail), tail)
+ }
+ if tail[0].Text != "bb" {
+ t.Errorf("tail item Text not truncated to overlap portion: got %q, want %q", tail[0].Text, "bb")
+ }
+ if string(tail[0].PDFPositions) != "[[2,0,20,0,8]]" {
+ t.Errorf("tail item carried wrong PDF positions: %s", tail[0].PDFPositions)
+ }
+ // The head item (page 1) must not appear in the tail.
+ for _, it := range tail {
+ if string(it.PDFPositions) == "[[1,0,10,0,5]]" {
+ t.Errorf("head item (page1) leaked into overlap tail: %+v", it)
+ }
+ }
+}
+
+// TestOverlapTailItems_JoinSepNoSeparatorBeforeCur is the TDD (RED) test for the
+// joinSep offset-model defect (CodeRabbit review #1 on PR #19068). The overlap
+// path stores the previous chunk's merged_items as [tailItems..., cur] and builds
+// cp.Text = overlap + cp.Text -- there is NO separator between the final tail
+// item and cur. But overlapTailItems reconstructs the rune offsets by inserting
+// joinSep before EVERY item, including before cur. On the JSON path (joinSep="\n")
+// that phantom separator shifts the offsets by one rune, so the tail slice is
+// truncated one rune too long (and, in a chained overlap, the window does not
+// converge as tightly as the displayed text implies).
+//
+// This test feeds overlapTailItems the structure the overlap path actually
+// produces -- tail item "aaaaa" followed by cur "bbbbb" with NO separator in the
+// real text (10 visible runes), overlap tail [7,10) -- and expects the tail of
+// cur to be truncated to exactly "bbb" (the last 3 runes). The buggy code inserts
+// a "\n" between "aaaaa" and "bbbbb", inflating the model to 11 runes and
+// returning "bbbb" instead. It fails (RED) until the offset model drops the
+// separator before the final (cur) item.
+func TestOverlapTailItems_JoinSepNoSeparatorBeforeCur(t *testing.T) {
+ prev := []mergeItem{
+ {Text: "aaaaa", PDFPositions: json.RawMessage("[[1,0,10,0,5]]"), Positions: json.RawMessage("[[1,0,10,0,5]]")},
+ {Text: "bbbbb", PDFPositions: json.RawMessage("[[2,0,20,0,8]]"), Positions: json.RawMessage("[[2,0,20,0,8]]")},
+ }
+ // Real overlap-chunk text = "aaaaa" + "bbbbb" (10 visible runes, no sep).
+ // Overlap tail is the last 3 runes [7,10) of cur.
+ tail := overlapTailItems(prev, 7, "\n")
+ if len(tail) != 1 {
+ t.Fatalf("want 1 tail item in overlap [7,10), got %d: %+v", len(tail), tail)
+ }
+ if tail[0].Text != "bbb" {
+ t.Errorf("tail item Text not truncated to overlap portion under joinSep: got %q, want %q (phantom separator shifted the offset)", tail[0].Text, "bbb")
+ }
+ if string(tail[0].PDFPositions) != "[[2,0,20,0,8]]" {
+ t.Errorf("tail item carried wrong PDF positions: %s", tail[0].PDFPositions)
+ }
+ // The head item (page 1) must not appear in the tail.
+ for _, it := range tail {
+ if string(it.PDFPositions) == "[[1,0,10,0,5]]" {
+ t.Errorf("head item (page1) leaked into overlap tail: %+v", it)
+ }
+ }
+}
+
+// TestOverlapTailItems_TagBearingUsesVisibleRuneOffsets pins that overlapTailItems
+// measures and slices on the TAG-FREE visible text, so a coordinate tag in an
+// earlier item does not shift the boundaries of later items (which would pull
+// head coordinates into the overlap tail). This mirrors the contract already
+// locked for the previous fused-item storage but for the separate-item storage.
+func TestOverlapTailItems_TagBearingUsesVisibleRuneOffsets(t *testing.T) {
+ prev := []mergeItem{
+ // Visible text "hello world" (11 runes); the tag inflates the RAW length
+ // to 23 runes. If the code measured RAW length, item0 would span [0,23)
+ // and overlapStart=12 would fall inside it (leaking head coords). Using
+ // the tag-free visible length, item0 spans [0,11) and item1 owns [11,15).
+ {Text: "hello@@1\t2\t3\t4## world", PDFPositions: json.RawMessage(`[["p0"]]`), Positions: json.RawMessage(`[["q0"]]`)},
+ {Text: "tail", PDFPositions: json.RawMessage(`[["p1"]]`), Positions: json.RawMessage(`[["q1"]]`)},
+ }
+ // Visible layout (no separator before cur, matching the overlap path):
+ // item0 [0,11), item1 [11,15). overlapStart=12 lands inside item1 only, so
+ // item0's head coordinates must NOT be included and item1 is truncated to
+ // the overlap portion [12,15) == "ail" (the last 3 runes of "tail").
+ tail := overlapTailItems(prev, 12, "\n")
+ if len(tail) != 1 {
+ t.Fatalf("want 1 tail item in overlap [12,15), got %d: %+v", len(tail), tail)
+ }
+ if string(tail[0].PDFPositions) != `[["p1"]]` {
+ t.Errorf("pdf positions = %s, want only item1 [\"p1\"] (item0 head must be excluded)", tail[0].PDFPositions)
+ }
+ if tail[0].Text != "ail" {
+ t.Errorf("tail item Text = %q, want %q (visible rune offsets must ignore the tag and truncate to the overlap portion)", tail[0].Text, "ail")
+ }
+}
+
+// TestMergeByTokenSizeFromJSON_ChainedOverlapPerChunkConvergence strengthens the
+// chained-overlap regression: not only must the FINAL chunk avoid head
+// coordinates, EVERY chunk's carried PDF positions must contain only the true
+// tail of its predecessor plus its own unit -- never any earlier (head) unit.
+// This locks convergence at every boundary, not just at the end of the chain.
+func TestMergeByTokenSizeFromJSON_ChainedOverlapPerChunkConvergence(t *testing.T) {
+ pos := func(page int) json.RawMessage {
+ return json.RawMessage(fmt.Sprintf("[[%d,0,%d,0,%d]]", page, page*10, page*5))
+ }
+ const n = 24
+ var row []schema.ChunkDoc
+ for i := 0; i < n; i++ {
+ row = append(row, schema.ChunkDoc{
+ Text: string(rune('a' + i)),
+ DocType: "text",
+ CKType: "text",
+ TKNums: intPtr(1),
+ PDFPositions: pos(i + 1),
+ })
+ }
+ items := [][]schema.ChunkDoc{row}
+ merged := mergeByTokenSizeFromJSON(items, 9, 20)[0]
+ if len(merged) < 3 {
+ t.Fatalf("want several chained chunks (>2), got %d", len(merged))
+ }
+ // The first chunk legitimately owns unit0 (page 1); every LATER chunk must
+ // NOT carry it. The overlap window converges to only the previous chunk's
+ // tail + the current unit, so the ultimate head (unit0) must never reappear
+ // after chunk 0. unit1 (page 2) must not reappear after chunk 1 either.
+ for k := 1; k < len(merged); k++ {
+ p := string(merged[k].PDFPositions)
+ if strings.Contains(p, "1,0,10,0,5") {
+ t.Errorf("chunk %d carries stale head coords (unit0/page1): pdf_positions=%s", k, p)
+ }
+ if k >= 2 && strings.Contains(p, "2,0,20,0,8") {
+ t.Errorf("chunk %d carries stale head coords (unit1/page2): pdf_positions=%s", k, p)
+ }
+ }
+ // Sanity: the final chunk must still carry its own most-recent unit's coords.
+ last := string(merged[len(merged)-1].PDFPositions)
+ if !strings.Contains(last, fmt.Sprintf("%d,0,%d,0,%d", n, n*10, n*5)) {
+ t.Errorf("final chunk lost its own coordinates (unit%d): pdf_positions=%s", n, last)
+ }
+}