RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): proportionally slice PDF positions so sub-chunk previews match text (#18864)
jay777216b673e613581
diff --git a/internal/ingestion/component/chunker/positions_slice.go b/internal/ingestion/component/chunker/positions_slice.gonew file mode 100644index 0000000000..8462348b21--- /dev/null+++ b/internal/ingestion/component/chunker/positions_slice.go@@ -0,0 +1,181 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "encoding/json"+ "unicode/utf8"+)++// toFloat converts a JSON number (int, int64, float64) to float64.+func toFloat(v any) (float64, bool) {+ switch x := v.(type) {+ case float64:+ return x, true+ case float32:+ return float64(x), true+ case int:+ return float64(x), true+ case int64:+ return float64(x), true+ case int32:+ return float64(x), true+ case json.Number:+ f, err := x.Float64()+ if err != nil {+ return 0, false+ }+ return f, true+ default:+ return 0, false+ }+}++// slicePositionsByTextRatio proportionally slices a PDF position matrix+// (JSON array of [page, left, right, top, bottom] rows) by vertical ratio.+//+// Each row is [page, left, right, top, bottom]. When a chunk's text is split+// into sub-pieces, the original bbox is sliced vertically so each sub-piece's+// screenshot crops only its own region instead of the whole paragraph.+//+// startRatio and endRatio are in [0,1] of the total vertical span. For a+// single-row matrix the slice is a simple vertical interpolation. For a+// multi-row matrix the total height is the sum of row heights; the slice is+// distributed sequentially across rows so a piece spanning a page boundary+// correctly covers the tail of one row and the head of the next.+func slicePositionsByTextRatio(raw json.RawMessage, startRatio, endRatio float64) json.RawMessage {+ if len(raw) == 0 {+ return nil+ }+ if startRatio < 0 {+ startRatio = 0+ }+ if endRatio > 1 {+ endRatio = 1+ }+ if startRatio >= endRatio {+ return nil+ }+ var matrix [][]any+ if err := json.Unmarshal(raw, &matrix); err != nil || len(matrix) == 0 {+ return nil+ }+ // Compute height per row. Any malformed row (short, non-numeric,+ // or non-positive height) invalidates the whole matrix so callers+ // can fall back to the original bbox rather than silently losing a+ // page region.+ heights := make([]float64, len(matrix))+ total := 0.0+ for i, row := range matrix {+ if len(row) < 5 {+ return nil+ }+ top, ok1 := toFloat(row[3])+ bottom, ok2 := toFloat(row[4])+ if !ok1 || !ok2 || bottom <= top {+ return nil+ }+ h := bottom - top+ heights[i] = h+ total += h+ }+ if total <= 0 {+ return nil+ }+ targetStart := startRatio * total+ targetEnd := endRatio * total+ var out [][]any+ cum := 0.0+ for i, row := range matrix {+ h := heights[i]+ if h <= 0 {+ continue+ }+ segStart := cum+ segEnd := cum + h+ cum += h+ interStart := segStart+ if targetStart > segStart {+ interStart = targetStart+ }+ interEnd := segEnd+ if targetEnd < segEnd {+ interEnd = targetEnd+ }+ if interStart >= interEnd {+ continue+ }+ localStartRatio := (interStart - segStart) / h+ localEndRatio := (interEnd - segStart) / h+ oldTop, _ := toFloat(row[3])+ oldBottom, _ := toFloat(row[4])+ oldH := oldBottom - oldTop+ newTop := oldTop + localStartRatio*oldH+ newBottom := oldTop + localEndRatio*oldH+ newRow := make([]any, len(row))+ copy(newRow, row)+ newRow[3] = newTop+ newRow[4] = newBottom+ out = append(out, newRow)+ }+ if len(out) == 0 {+ return nil+ }+ b, err := json.Marshal(out)+ if err != nil {+ return nil+ }+ return json.RawMessage(b)+}++// sliceAnyPositions is the map-value variant used by the title chunker+// (GroupTitle/HierarchyTitle) where positions are stored as any+// (typically [][]float64). It round-trips through JSON so the same+// vertical slicing logic applies regardless of the in-memory representation.+func sliceAnyPositions(pos any, startRatio, endRatio float64) any {+ if pos == nil {+ return nil+ }+ b, err := json.Marshal(pos)+ if err != nil || len(b) == 0 || string(b) == "null" {+ return nil+ }+ sliced := slicePositionsByTextRatio(json.RawMessage(b), startRatio, endRatio)+ if len(sliced) == 0 {+ return nil+ }+ // Prefer restoring as [][]float64 when possible (the title chunker's+ // native type). Fall back to a generic decoded value.+ var mat [][]float64+ if err := json.Unmarshal(sliced, &mat); err == nil {+ return mat+ }+ var generic any+ if err := json.Unmarshal(sliced, &generic); err == nil {+ return generic+ }+ return nil+}++// totalRunes returns the sum of rune counts for the given strings.+func totalRunes(strs []string) int {+ n := 0+ for _, s := range strs {+ n += utf8.RuneCountInString(s)+ }+ return n+}diff --git a/internal/ingestion/component/chunker/positions_slice_test.go b/internal/ingestion/component/chunker/positions_slice_test.gonew file mode 100644index 0000000000..1b6541c80a--- /dev/null+++ b/internal/ingestion/component/chunker/positions_slice_test.go@@ -0,0 +1,199 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "encoding/json"+ "math"+ "testing"+)++const posSliceEpsilon = 1e-9++func assertMatrixNear(t *testing.T, got json.RawMessage, want [][]float64) {+ t.Helper()+ if len(got) == 0 {+ t.Fatalf("got nil matrix, want %v", want)+ }+ m := matrixOfRaw(t, got)+ if len(m) != len(want) {+ t.Fatalf("matrix rows: got %d (%v), want %d (%v)", len(m), m, len(want), want)+ }+ for i := range want {+ if len(m[i]) != len(want[i]) {+ t.Fatalf("row %d width: got %d, want %d", i, len(m[i]), len(want[i]))+ }+ for j := range want[i] {+ if math.Abs(m[i][j]-want[i][j]) > posSliceEpsilon {+ t.Fatalf("m[%d][%d] = %v, want %v", i, j, m[i][j], want[i][j])+ }+ }+ }+}++func matrixOfRaw(t *testing.T, raw json.RawMessage) [][]float64 {+ t.Helper()+ var m [][]float64+ if err := json.Unmarshal(raw, &m); err != nil {+ t.Fatalf("unmarshal %s: %v", raw, err)+ }+ return m+}++func TestSlicePositionsByTextRatio_SingleRowHalves(t *testing.T) {+ raw := json.RawMessage(`[[1,10,200,0,100]]`)+ assertMatrixNear(t, slicePositionsByTextRatio(raw, 0, 0.5), [][]float64{{1, 10, 200, 0, 50}})+ assertMatrixNear(t, slicePositionsByTextRatio(raw, 0.5, 1), [][]float64{{1, 10, 200, 50, 100}})+}++func TestSlicePositionsByTextRatio_AdjacentSlicesAbut(t *testing.T) {+ raw := json.RawMessage(`[[1,10,20,50,80]]`)+ first := slicePositionsByTextRatio(raw, 0, 0.4)+ second := slicePositionsByTextRatio(raw, 0.4, 1)+ m1, m2 := matrixOfRaw(t, first), matrixOfRaw(t, second)+ if len(m1) != 1 || len(m2) != 1 {+ t.Fatalf("single-row input must yield single-row slices: %v / %v", m1, m2)+ }+ if math.Abs(m1[0][4]-m2[0][3]) > posSliceEpsilon {+ t.Errorf("adjacent slices must abut: first bottom=%v second top=%v", m1[0][4], m2[0][3])+ }+ if m1[0][3] != 50 || m2[0][4] != 80 {+ t.Errorf("outer bounds changed: %v / %v", m1, m2)+ }+ if m1[0][1] != 10 || m1[0][2] != 20 || m2[0][1] != 10 || m2[0][2] != 20 {+ t.Errorf("left/right must be preserved: %v / %v", m1, m2)+ }+}++func TestSlicePositionsByTextRatio_MultiRowSequentialDistribution(t *testing.T) {+ // Two equal-height rows (page 1 and page 2), total height 20.+ raw := json.RawMessage(`[[1,10,20,0,10],[2,10,20,10,20]]`)+ // First quarter of total height lives entirely in row 1.+ assertMatrixNear(t, slicePositionsByTextRatio(raw, 0, 0.25), [][]float64{{1, 10, 20, 0, 5}})+ // Last quarter lives entirely in row 2.+ assertMatrixNear(t, slicePositionsByTextRatio(raw, 0.75, 1), [][]float64{{2, 10, 20, 15, 20}})+}++func TestSlicePositionsByTextRatio_CrossPagePieceSpansBothRows(t *testing.T) {+ raw := json.RawMessage(`[[1,10,20,0,10],[2,10,20,10,20]]`)+ got := matrixOfRaw(t, slicePositionsByTextRatio(raw, 0.4, 0.6))+ if len(got) != 2 {+ t.Fatalf("a piece spanning the row boundary must emit both tail and head rows, got %v", got)+ }+ if math.Abs(got[0][3]-8) > posSliceEpsilon || math.Abs(got[0][4]-10) > posSliceEpsilon {+ t.Errorf("tail of row 1 wrong: %v", got[0])+ }+ if math.Abs(got[1][3]-10) > posSliceEpsilon || math.Abs(got[1][4]-12) > posSliceEpsilon {+ t.Errorf("head of row 2 wrong: %v", got[1])+ }+}++func TestSlicePositionsByTextRatio_PreservesPageNumbersAndLeftRight(t *testing.T) {+ raw := json.RawMessage(`[[7,3.5,44.25,0,40],[9,3.5,44.25,40,80]]`)+ got := matrixOfRaw(t, slicePositionsByTextRatio(raw, 0.25, 0.75))+ if len(got) == 0 {+ t.Fatal("expected a non-empty sliced matrix")+ }+ if got[0][0] != 7 || got[len(got)-1][0] != 9 {+ t.Errorf("page numbers not preserved per row: %v", got)+ }+ for _, row := range got {+ if row[1] != 3.5 || row[2] != 44.25 {+ t.Errorf("left/right altered: %v", row)+ }+ }+}++func TestSlicePositionsByTextRatio_MalformedRowsReturnNil(t *testing.T) {+ // Any malformed row (short, zero-height) invalidates the whole matrix+ // so callers fall back to the original bbox rather than silently losing+ // a page region.+ for name, raw := range map[string]json.RawMessage{+ "short row": json.RawMessage(`[[1,10,20],[1,10,20,0,10]]`),+ "zero height": json.RawMessage(`[[1,10,20,30,30],[1,10,20,0,10]]`),+ "mixed": json.RawMessage(`[[1,10,20],[1,10,20,30,30],[1,10,20,0,10]]`),+ } {+ if got := slicePositionsByTextRatio(raw, 0, 0.5); got != nil {+ t.Errorf("%s: expected nil for malformed matrix, got %s", name, got)+ }+ }+}++func TestSlicePositionsByTextRatio_ZeroTotalReturnsNil(t *testing.T) {+ if got := slicePositionsByTextRatio(json.RawMessage(`[[1,10,20,5,5]]`), 0, 1); got != nil {+ t.Errorf("zero-height matrix must return nil, got %s", got)+ }+}++func TestSlicePositionsByTextRatio_EmptyOrGarbageReturnsNil(t *testing.T) {+ for name, raw := range map[string]json.RawMessage{+ "empty": {},+ "garbage": json.RawMessage(`not-json`),+ "not-array": json.RawMessage(`{"page":1}`),+ "null": json.RawMessage(`null`),+ } {+ if got := slicePositionsByTextRatio(raw, 0, 1); got != nil {+ t.Errorf("%s: expected nil, got %s", name, got)+ }+ }+}++func TestSlicePositionsByTextRatio_InvalidRatiosReturnNil(t *testing.T) {+ raw := json.RawMessage(`[[1,10,20,0,100]]`)+ cases := []struct{ s, e float64 }{{0.5, 0.5}, {0.8, 0.2}, {-1, -2}}+ for _, c := range cases {+ if got := slicePositionsByTextRatio(raw, c.s, c.e); got != nil && c.s >= c.e {+ t.Errorf("start=%v >= end=%v must return nil, got %s", c.s, c.e, got)+ }+ }+ // Clamping still works for slight out-of-range values.+ assertMatrixNear(t, slicePositionsByTextRatio(raw, -0.1, 1.5), [][]float64{{1, 10, 20, 0, 100}})+}++func TestSliceAnyPositions_Float64MatrixRoundTrip(t *testing.T) {+ pos := [][]float64{{1, 0, 100, 0, 100}}+ sliced := sliceAnyPositions(pos, 0, 0.5)+ mat, ok := sliced.([][]float64)+ if !ok {+ t.Fatalf("sliceAnyPositions returned %T, want [][]float64", sliced)+ }+ if len(mat) != 1 || mat[0][3] != 0 || math.Abs(mat[0][4]-50) > posSliceEpsilon {+ t.Errorf("sliced positions = %v, want [[1 0 100 0 50]]", mat)+ }+}++func TestSliceAnyPositions_NilAndUnknownTypesReturnNil(t *testing.T) {+ if got := sliceAnyPositions(nil, 0, 1); got != nil {+ t.Errorf("nil input must return nil, got %v", got)+ }+ if got := sliceAnyPositions("not-a-matrix", 0, 1); got != nil {+ t.Errorf("non-matrix value must return nil, got %v", got)+ }+ // A value with no position-like rows yields no usable slicing either.+ if got := sliceAnyPositions([]any{map[string]any{"page": 1}}, 0, 1); got != nil {+ t.Errorf("malformed matrix must return nil, got %v", got)+ }+}++func TestTotalRunes(t *testing.T) {+ if got := totalRunes([]string{"a", "中中", ""}); got != 3 {+ t.Errorf("totalRunes = %d, want 3", got)+ }+ if got := totalRunes(nil); got != 0 {+ t.Errorf("totalRunes(nil) = %d, want 0", got)+ }+}diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.goindex 7a272fde5b..a3da43b66b 100644--- a/internal/ingestion/component/chunker/title_cap.go+++ b/internal/ingestion/component/chunker/title_cap.go@@ -203,11 +203,50 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {return []map[string]any{chunk}}+ // Proportionally slice PDF positions so each sub-chunk's screenshot+ // crops only its own vertical region (fix for chunk-screenshot mismatch).+ // When no positions are present the old shallow-copy behaviour is kept.+ _, hasPDF := chunk["_pdf_positions"]+ _, hasPos := chunk["positions"]+ if !hasPDF && !hasPos {+ out := make([]map[string]any, 0, len(finalGroups))+ for _, g := range finalGroups {+ sub := shallowCopyChunk(chunk)+ sub["text"] = g+ out = append(out, sub)+ }+ return out+ }+ total := totalRunes(finalGroups)+ if total == 0 {+ out := make([]map[string]any, 0, len(finalGroups))+ for _, g := range finalGroups {+ sub := shallowCopyChunk(chunk)+ sub["text"] = g+ out = append(out, sub)+ }+ return out+ }out := make([]map[string]any, 0, len(finalGroups))+ cum := 0for _, g := range finalGroups {+ pcRunes := utf8.RuneCountInString(g)+ startRatio := float64(cum) / float64(total)+ endRatio := float64(cum+pcRunes) / float64(total)sub := shallowCopyChunk(chunk)sub["text"] = g+ if hasPDF {+ if sliced := sliceAnyPositions(chunk["_pdf_positions"], startRatio, endRatio); sliced != nil {+ sub["_pdf_positions"] = sliced+ }+ }+ if hasPos {+ if sliced := sliceAnyPositions(chunk["positions"], startRatio, endRatio); sliced != nil {+ sub["positions"] = sliced+ }+ }out = append(out, sub)+ cum += pcRunes}return out}diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.goindex 5939f96963..fac568ba47 100644--- a/internal/ingestion/component/chunker/title_token_cap_test.go+++ b/internal/ingestion/component/chunker/title_token_cap_test.go@@ -18,6 +18,8 @@ import ("context""encoding/json""fmt"+ "math"+ "reflect""strings""testing""unicode/utf8"@@ -273,28 +275,61 @@ func TestHardSplitByTokens_EnglishRemainderStaysWhole(t *testing.T) {}}-func TestEnforceTitleTokenCap_SubChunksKeepPositions(t *testing.T) {+func TestEnforceTitleTokenCap_SubChunksHaveSlicedPositions(t *testing.T) {charTokenizer()defer restoreTokenizer()+ // Single 5-tuple row [page,left,right,top,bottom] with a height of 30.pos := [][]float64{{1, 10, 200, 50, 80}}- body := strings.Join(joinSentences(12), "")+ body := strings.Join(joinSentences(12), "") // 48 runeschunks := []map[string]any{{"text": body, "positions": pos},}got := enforceTitleTokenCap(chunks, 20)+ if len(got) != 3 { // 20 + 20 + 8 runes+ t.Fatalf("expected split into 3 chunks, got %d", len(got))+ }+ // Each sub-chunk's positions must be vertically sliced by its rune share+ // of the whole body (heights 12.5 / 12.5 / 5 on the 30-high box).+ wantBounds := [][2]float64{{50, 62.5}, {62.5, 75}, {75, 80}}+ for i := range got {+ v, ok := got[i]["positions"].([][]float64)+ if !ok || len(v) != 1 {+ t.Fatalf("sub-chunk %d positions = %#v, want one sliced row", i, got[i]["positions"])+ }+ row := v[0]+ if row[0] != 1 || row[1] != 10 || row[2] != 200 {+ t.Errorf("sub-chunk %d kept columns wrong: %v", i, row)+ }+ if math.Abs(row[3]-wantBounds[i][0]) > 1e-9 || math.Abs(row[4]-wantBounds[i][1]) > 1e-9 {+ t.Errorf("sub-chunk %d bounds = [%v,%v], want [%v,%v]",+ i, row[3], row[4], wantBounds[i][0], wantBounds[i][1])+ }+ }+}++// TestEnforceTitleTokenCap_MalformedPositionsFallbackKeepsOriginal pins the+// fallback contract: a position matrix that cannot be sliced (no valid+// [page,left,right,top,bottom] rows) is carried to every sub-chunk verbatim,+// so the preview is degraded to the shared coarse bbox instead of being lost.+func TestEnforceTitleTokenCap_MalformedPositionsFallbackKeepsOriginal(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ src := []any{map[string]any{"page": float64(1)}}+ chunks := []map[string]any{+ {"text": body, "_pdf_positions": src},+ }+ got := enforceTitleTokenCap(chunks, 20)if len(got) <= 1 {t.Fatalf("expected split, got %d", len(got))}- // Every sub-chunk keeps the original position matrix: the on-demand crop- // pass and the position index derive from it.for i := range got {- v, ok := got[i]["positions"].([][]float64)+ v, ok := got[i]["_pdf_positions"].([]any)if !ok || len(v) == 0 {- t.Errorf("sub-chunk %d positions = %#v, want [[1 10 200 50 80]]", i, got[i]["positions"])- continue+ t.Fatalf("sub-chunk %d _pdf_positions = %#v, want the original matrix", i, got[i]["_pdf_positions"])}- if v[0][0] != 1 {- t.Errorf("sub-chunk %d positions = %#v, want the original matrix", i, v)+ if !reflect.DeepEqual(v[0], src[0]) {+ t.Errorf("sub-chunk %d _pdf_positions = %#v, want the source value verbatim", i, v[0])}}}@@ -521,16 +556,21 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}- // Every sub-chunk keeps the source position matrix verbatim so the crop- // pass can attach a preview image to each of them.+ // Every sub-chunk carries a sliced position row: the on-demand crop pass+ // must attach a preview of its own vertical region, not the whole box.for i := range chunks {v, ok := chunks[i]["positions"]if !ok {t.Errorf("sub-chunk %d missing positions", i)continue}- if vm, ok := v.([][]float64); !ok || len(vm) == 0 || vm[0][0] != 1 {- t.Errorf("sub-chunk %d positions = %#v, want [[1 10 200 50 80]]", i, v)+ vm, ok := v.([][]float64)+ if !ok || len(vm) == 0 || vm[0][0] != 1 {+ t.Errorf("sub-chunk %d positions = %#v, want a sliced [[1 ...]] matrix", i, v)+ continue+ }+ if vm[0][4] > 80+1e-9 || vm[0][3] < 50-1e-9 {+ t.Errorf("sub-chunk %d bounds out of source span: %v", i, vm[0])}}}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 670a161774..ff362f11fc 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -625,16 +625,59 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp) []schema.Chuif !delimPattern.MatchString(txt) {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}- out := make([]schema.ChunkDoc, 0, len(parts))+ // Collect non-empty parts first so we can slice positions proportionally.+ var kept []stringfor _, p := range parts {if strings.TrimSpace(p) == "" {continue}- out = append(out, buildChunkDoc(it, "text", p, "", ""))+ kept = append(kept, p)}- if len(out) == 0 {+ if len(kept) == 0 {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}+ // If the item carries PDF positions, slice them proportionally so each+ // delimiter-split piece's screenshot crops only its own region instead of+ // the whole item bbox (fix for chunk-screenshot mismatch).+ if len(it.PDFPositions) == 0 && len(it.Positions) == 0 {+ out := make([]schema.ChunkDoc, 0, len(kept))+ for _, p := range kept {+ out = append(out, buildChunkDoc(it, "text", p, "", ""))+ }+ return out+ }+ // Visible-text ratio: parser tags (@@...##) carry no visual height+ // and must not shift crop boundaries. Mirrors the overlap logic+ // which already counts on removeTag'd text.+ runCount := 0+ for _, p := range kept {+ runCount += utf8.RuneCountInString(removeTag(p))+ }+ if runCount == 0 {+ out := make([]schema.ChunkDoc, 0, len(kept))+ for _, p := range kept {+ out = append(out, buildChunkDoc(it, "text", p, "", ""))+ }+ return out+ }+ out := make([]schema.ChunkDoc, 0, len(kept))+ cumRunes := 0+ for _, p := range kept {+ pcRunes := utf8.RuneCountInString(removeTag(p))+ startRatio := float64(cumRunes) / float64(runCount)+ endRatio := float64(cumRunes+pcRunes) / float64(runCount)+ ck := buildChunkDoc(it, "text", p, "", "")+ ck.PDFPositions = slicePositionsByTextRatio(it.PDFPositions, startRatio, endRatio)+ ck.Positions = slicePositionsByTextRatio(it.Positions, startRatio, endRatio)+ if len(ck.PDFPositions) == 0 && len(it.PDFPositions) > 0 {+ ck.PDFPositions = it.PDFPositions+ }+ if len(ck.Positions) == 0 && len(it.Positions) > 0 {+ ck.Positions = it.Positions+ }+ out = append(out, ck)+ cumRunes += pcRunes+ }return out}@@ -1133,13 +1176,67 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {}// Every sub-piece inherits the source unit's metadata (coarse positions +// item attributes); each is built from a clone so every ChunkDoc field- // survives the split and each piece keeps its page-region preview.- for i := range pieces {+ // survives the split. PDF positions are proportionally sliced vertically so+ // each piece's screenshot crops only its own region instead of the whole+ // paragraph (fix for chunk-screenshot mismatch).+ if len(ck.PDFPositions) == 0 && len(ck.Positions) == 0 {+ for i := range pieces {+ inherited := cloneChunkDoc(ck)+ inherited.Text = pieces[i].Text+ inherited.TKNums = pieces[i].TKNums+ inherited.CKType = "text"+ pieces[i] = inherited+ }+ return pieces+ }+ // Ratio basis: rune counts of the real source content. The synthetic+ // leading "\n" glue (text-path units are built as "\n"+paragraph) carries+ // no visual height, so it is excluded from the first piece's count to keep+ // the slice proportions aligned with the original text. Parser tags+ // (@@...##) also carry no visual height and are stripped before counting.+ counts := make([]int, len(pieces))+ runCount := 0+ for i, p := range pieces {+ n := utf8.RuneCountInString(removeTag(p.Text))+ if i == 0 && lead != "" {+ n -= utf8.RuneCountInString(lead)+ if n < 0 {+ n = 0+ }+ }+ counts[i] = n+ runCount += n+ }+ if runCount == 0 {+ for i := range pieces {+ inherited := cloneChunkDoc(ck)+ inherited.Text = pieces[i].Text+ inherited.TKNums = pieces[i].TKNums+ inherited.CKType = "text"+ pieces[i] = inherited+ }+ return pieces+ }+ cumRunes := 0+ for i, p := range pieces {+ startRatio := float64(cumRunes) / float64(runCount)+ endRatio := float64(cumRunes+counts[i]) / float64(runCount)inherited := cloneChunkDoc(ck)- inherited.Text = pieces[i].Text- inherited.TKNums = pieces[i].TKNums+ inherited.Text = p.Text+ inherited.TKNums = p.TKNumsinherited.CKType = "text"+ inherited.PDFPositions = slicePositionsByTextRatio(ck.PDFPositions, startRatio, endRatio)+ inherited.Positions = slicePositionsByTextRatio(ck.Positions, startRatio, endRatio)+ // Fall back to original positions if slicing produced nothing (e.g.+ // malformed matrix) so the piece still gets a preview rather than none.+ if len(inherited.PDFPositions) == 0 && len(ck.PDFPositions) > 0 {+ inherited.PDFPositions = ck.PDFPositions+ }+ if len(inherited.Positions) == 0 && len(ck.Positions) > 0 {+ inherited.Positions = ck.Positions+ }pieces[i] = inherited+ cumRunes += counts[i]}return pieces}diff --git a/internal/ingestion/component/chunker/token_pdfpos_test.go b/internal/ingestion/component/chunker/token_pdfpos_test.goindex 2d6dc1e5e9..a89420f97a 100644--- a/internal/ingestion/component/chunker/token_pdfpos_test.go+++ b/internal/ingestion/component/chunker/token_pdfpos_test.go@@ -18,8 +18,10 @@ package chunkerimport ("encoding/json"+ "math""strings""testing"+ "unicode/utf8""ragflow/internal/ingestion/component/schema")@@ -240,3 +242,179 @@ func TestMergeByTokenSizeFromJSON_PartialOverlapPrefixCarriesOnlyTailPositions(t}}}++// TestChunkFromItem_SlicesPositionsAcrossDelimiterPieces pins the+// chunk-screenshot-mismatch fix on the JSON delimiter-split path+// (chunkFromItem): an item whose text contains the active delimiter must NOT+// copy its whole bbox to every piece. Each piece's positions must be a+// vertical slice proportional to its rune share, and adjacent slices must+// abut so the pieces jointly tile the original box.+func TestChunkFromItem_SlicesPositionsAcrossDelimiterPieces(t *testing.T) {+ it := schema.ChunkDoc{+ Text: "AAAA\nBBBB",+ DocType: "text",+ CKType: "text",+ PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),+ }+ got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}))+ if len(got) != 2 {+ t.Fatalf("want 2 delimiter-split pieces, got %d", len(got))+ }+ p0 := matrixOfRaw(t, got[0].PDFPositions)+ p1 := matrixOfRaw(t, got[1].PDFPositions)+ if len(p0) != 1 || len(p1) != 1 {+ t.Fatalf("single-row input must yield single-row slices: %v / %v", p0, p1)+ }+ if p0[0][3] != 0 || math.Abs(p0[0][4]-20) > 1e-9 {+ t.Errorf("piece 0 bounds = [%v,%v], want [0,20]", p0[0][3], p0[0][4])+ }+ if math.Abs(p1[0][3]-20) > 1e-9 || p1[0][4] != 40 {+ t.Errorf("piece 1 bounds = [%v,%v], want [20,40]", p1[0][3], p1[0][4])+ }+}++// TestSplitOversizedText_SlicedPositionsTrackPieceTextShare pins the core+// user-facing invariant of the screenshot-mismatch fix: each hard-split+// piece's cropped region height is proportional to its OWN text share (its+// rune count relative to all siblings), so the thumbnail matches the chunk+// text instead of showing the whole paragraph. The synthetic leading "\n"+// glue carries no visual height and must not inflate the first slice.+//+// This test uses REAL tokenizer behavior only to drive WHERE the oversized+// unit splits; the ratio assertions are derived from the emitted piece texts,+// so they hold regardless of cut points.+func TestSplitOversizedText_SlicedPositionsTrackPieceTextShare(t *testing.T) {+ const boxTop, boxBottom = 50.0, 80.0 // height 30+ build := func(text string) schema.ChunkDoc {+ return schema.ChunkDoc{+ Text: text,+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(tokenizeStr(text)),+ PDFPositions: json.RawMessage(`[[1,10,200,50,80]]`),+ }+ }+ run := func(ck schema.ChunkDoc, lead bool) {+ t.Helper()+ got := splitOversizedText(ck, 30)+ if len(got) < 2 {+ t.Fatalf("oversized unit must be split, got %d piece(s)", len(got))+ }+ counts := make([]int, len(got))+ total := 0+ for i, p := range got {+ n := utf8.RuneCountInString(p.Text)+ if i == 0 && lead {+ n -= utf8.RuneCountInString("\n") // leading glue: no visual height+ if n < 0 {+ n = 0+ }+ }+ counts[i] = n+ total += n+ }+ prevBottom := boxTop+ for i, p := range got {+ m := matrixOfRaw(t, p.PDFPositions)+ if len(m) != 1 {+ t.Fatalf("piece %d: want one sliced row, got %v", i, m)+ }+ wantHeight := float64(counts[i]) / float64(total) * (boxBottom - boxTop)+ if math.Abs((m[0][4]-m[0][3])-wantHeight) > 1e-6*float64(len(got)) {+ t.Errorf("piece %d height = %v, want %v (text share %d/%d)",+ i, m[0][4]-m[0][3], wantHeight, counts[i], total)+ }+ if math.Abs(m[0][3]-prevBottom) > 1e-9 {+ t.Errorf("piece %d top = %v, want the previous bottom %v (slices must abut)", i, m[0][3], prevBottom)+ }+ prevBottom = m[0][4]+ }+ if math.Abs(prevBottom-boxBottom) > 1e-9 {+ t.Errorf("last piece bottom = %v, want %v (box fully covered)", prevBottom, boxBottom)+ }+ }++ body := strings.Repeat("word ", 100) // ~100 tokens, no sentence boundary+ // No lead glue: raw rune shares apply.+ run(build(body), false)+ // Text-path units are built as "\n"+paragraph (mergeByTokenSize); the+ // leading glue must be excluded from the first piece's share.+ run(build("\n"+body), true)+}++// TestChunkFromItem_SlicesPositionsIgnoresPositionTags ensures the+// position-slicing ratio is based on visible text. Parser tags+// (@@...##) carry no visual height and must not shift crop boundaries.+// This is the regression for the tag-aware fix (token.go:649).+func TestChunkFromItem_SlicesPositionsIgnoresPositionTags(t *testing.T) {+ // Visible "AAAA"(4) vs "CCCC"(4) => 0.5 split => [0,20]/[20,40].+ // Raw first piece contains a 16-rune tag; if counted it would shift+ // the boundary to ~33.3, reproducing the screenshot mismatch.+ it := schema.ChunkDoc{+ Text: "AAAA@@1\t0\t200\t0\t40##\nCCCC",+ DocType: "text",+ CKType: "text",+ PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),+ }+ got := chunkFromItem(it, compileDelimPattern([]string{"`\n`"}))+ if len(got) != 2 {+ t.Fatalf("want 2 delimiter-split pieces, got %d", len(got))+ }+ p0 := matrixOfRaw(t, got[0].PDFPositions)+ p1 := matrixOfRaw(t, got[1].PDFPositions)+ if len(p0) != 1 || len(p1) != 1 {+ t.Fatalf("single-row input must yield single-row slices: %v / %v", p0, p1)+ }+ if math.Abs(p0[0][4]-20) > 1e-9 {+ t.Errorf("piece 0 bottom = %v, want 20 (tag runes must not shift ratio)", p0[0][4])+ }+ if math.Abs(p1[0][3]-20) > 1e-9 {+ t.Errorf("piece 1 top = %v, want 20 (tag runes must not shift ratio)", p1[0][3])+ }+}++// TestSplitOversizedText_SlicesPositionsIgnoreTags verifies the hard-cap+// path also uses tag-free visible runes for its ratio, mirroring the+// delimiter path. The leading "\n" glue and tags both carry no height.+func TestSplitOversizedText_SlicesPositionsIgnoreTags(t *testing.T) {+ const boxTop, boxBottom = 0.0, 40.0+ tag := "@@1\t0\t200\t0\t40##"+ body := strings.Repeat("word ", 30) + tag + strings.Repeat("word ", 30)+ ck := schema.ChunkDoc{+ Text: body,+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(tokenizeStr(body)),+ PDFPositions: json.RawMessage(`[[1,0,200,0,40]]`),+ }+ got := splitOversizedText(ck, 30)+ if len(got) < 2 {+ t.Fatalf("oversized unit must be split, got %d piece(s)", len(got))+ }+ // Ratio must be computed from visible (tag-free) text.+ totalVisible := 0+ counts := make([]int, len(got))+ for i, p := range got {+ n := utf8.RuneCountInString(removeTag(p.Text))+ counts[i] = n+ totalVisible += n+ }+ prevBottom := boxTop+ for i, p := range got {+ m := matrixOfRaw(t, p.PDFPositions)+ if len(m) != 1 {+ t.Fatalf("piece %d: want one sliced row, got %v", i, m)+ }+ wantHeight := float64(counts[i]) / float64(totalVisible) * (boxBottom - boxTop)+ if math.Abs((m[0][4]-m[0][3])-wantHeight) > 1e-6*float64(len(got)) {+ t.Errorf("piece %d height = %v, want %v (visible share %d/%d)", i, m[0][4]-m[0][3], wantHeight, counts[i], totalVisible)+ }+ if math.Abs(m[0][3]-prevBottom) > 1e-9 {+ t.Errorf("piece %d top = %v, want previous bottom %v (slices must abut)", i, m[0][3], prevBottom)+ }+ prevBottom = m[0][4]+ }+ if math.Abs(prevBottom-boxBottom) > 1e-9 {+ t.Errorf("last piece bottom = %v, want %v (box fully covered)", prevBottom, boxBottom)+ }+}