Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): slice General split positions by visible-text ratio (#20827)

jay777210ae367476bbf
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index e921f4bad1..51816d2c6a 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -31,6 +31,7 @@ import (
"regexp"
"sort"
"strings"
+ "unicode/utf8"
"go.uber.org/zap"
"gorm.io/gorm"
@@ -1239,13 +1240,15 @@ func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDeli
if keepDelimiter {
parts = foldWhitespaceSegments(parts)
}
+ filtered := make([]string, 0, len(parts))
for _, part := range parts {
if !keepDelimiter && strings.TrimSpace(part) == "" {
continue
}
- piece := cloneChunkDoc(unit)
- piece.Text = part
- piece.TKNums = intPtr(tokenizeStr(part))
+ filtered = append(filtered, part)
+ }
+ parts = filtered
+ for _, piece := range sliceGeneralSplitPieces(unit, parts) {
result = append(result, piece)
}
}
@@ -1257,6 +1260,32 @@ func normalizeGeneralNewlines(text string) string {
return strings.ReplaceAll(text, "\r", "\n")
}
+// sliceGeneralSplitPieces builds one output piece per text part, slicing the
+// source paragraph bbox via slicePositionsByCounts so adjacent pieces abut
+// instead of each carrying the full paragraph box. Parser position tags
+// (@@...##) carry no visual height and are excluded from the ratio, mirroring
+// the Token path (chunkFromItem/splitOversizedText).
+func sliceGeneralSplitPieces(unit schema.ChunkDoc, parts []string) []schema.ChunkDoc {
+ out := make([]schema.ChunkDoc, 0, len(parts))
+ if len(parts) == 0 {
+ return out
+ }
+ counts := make([]int, len(parts))
+ for i, part := range parts {
+ counts[i] = utf8.RuneCountInString(removeTag(part))
+ }
+ pdfSliced, posSliced := slicePositionsByCounts(unit.PDFPositions, unit.Positions, counts)
+ for i, part := range parts {
+ piece := cloneChunkDoc(unit)
+ piece.Text = part
+ piece.TKNums = intPtr(tokenizeStr(part))
+ piece.PDFPositions = pdfSliced[i]
+ piece.Positions = posSliced[i]
+ out = append(out, piece)
+ }
+ return out
+}
+
func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Regexp) []schema.ChunkDoc {
visible := make([]schema.ChunkDoc, 0, len(chunks))
for _, chunk := range chunks {
@@ -1293,12 +1322,19 @@ func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []sc
continue
}
mom := strings.TrimPrefix(chunk.Text, "\n")
- for _, part := range splitKeepingGeneralDelimiter(chunk.Text, pattern) {
+ raw := splitKeepingGeneralDelimiter(chunk.Text, pattern)
+ parts := make([]string, 0, len(raw))
+ for _, part := range raw {
if strings.TrimSpace(part) == "" {
continue
}
- piece := cloneChunkDoc(chunk)
- piece.Text = part
+ parts = append(parts, part)
+ }
+ if len(parts) == 0 {
+ result = append(result, chunk)
+ continue
+ }
+ for _, piece := range sliceGeneralSplitPieces(chunk, parts) {
piece.Mom = mom
result = append(result, piece)
}
diff --git a/internal/ingestion/component/chunker/general_block_delimiter_test.go b/internal/ingestion/component/chunker/general_block_delimiter_test.go
index 2cb87e718e..db51ad6ad8 100644
--- a/internal/ingestion/component/chunker/general_block_delimiter_test.go
+++ b/internal/ingestion/component/chunker/general_block_delimiter_test.go
@@ -60,7 +60,13 @@ func TestGeneralChunkerPDFPunctuationPreservesMediaOrderAndContext(t *testing.T)
t.Fatalf("chunks = %#v", chunks)
}
assertMaterializedMediaContext(t, chunks[1], "前文。还有一句。<table>A</table>后文。还有一句。")
- if got, want := chunks[0]["positions"], [][]float64{{1, 0, 10, 10, 15}}; !reflect.DeepEqual(got, want) {
+ // The unit "前文。还有一句。" is delimiter-split into "前文。" (3 visible
+ // runes) and "还有一句。" (5 visible runes) whose bboxes partition the
+ // source [10,15] box proportionally (3:5), then merged back into one
+ // chunk. The two rows cover exactly the source span — the Token path
+ // likewise retains one row per merged item — so expect the partition
+ // rather than a single full-span row.
+ if got, want := chunks[0]["positions"], [][]float64{{1, 0, 10, 10, 11.875}, {1, 0, 10, 11.875, 15}}; !reflect.DeepEqual(got, want) {
t.Fatalf("positions = %#v, want %#v", got, want)
}
}
diff --git a/internal/ingestion/component/chunker/general_positions_slice_test.go b/internal/ingestion/component/chunker/general_positions_slice_test.go
new file mode 100644
index 0000000000..489e0bc964
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_positions_slice_test.go
@@ -0,0 +1,161 @@
+package chunker
+
+import (
+ "encoding/json"
+ "math"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func decodeGeneralMatrix(t *testing.T, raw json.RawMessage) [][]float64 {
+ t.Helper()
+ var m [][]float64
+ if err := json.Unmarshal(raw, &m); err != nil {
+ t.Fatalf("unmarshal %s: %v", raw, err)
+ }
+ return m
+}
+
+func TestSplitGeneralUnitsSlicesPositionsAbut(t *testing.T) {
+ unit := schema.ChunkDoc{
+ Text: "第一句。第二句。",
+ DocType: "text",
+ PDFPositions: json.RawMessage(`[[1,10,200,0,100]]`),
+ Positions: json.RawMessage(`[[1,10,200,0,100]]`),
+ }
+ pattern := compileDelimPattern([]string{"。"})
+ got := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, false)
+ if len(got) != 2 {
+ t.Fatalf("split pieces = %d, want 2 (%q)", len(got), generalChunkTexts(got))
+ }
+ first := decodeGeneralMatrix(t, got[0].PDFPositions)
+ second := decodeGeneralMatrix(t, got[1].PDFPositions)
+ if len(first) != 1 || len(second) != 1 {
+ t.Fatalf("single-row input must yield single-row slices: %v / %v", first, second)
+ }
+ if first[0][3] != 0 || second[0][4] != 100 {
+ t.Errorf("outer bounds changed: %v / %v", first[0], second[0])
+ }
+ if math.Abs(first[0][4]-second[0][3]) > posSliceEpsilon {
+ t.Errorf("adjacent General slices must abut: first bottom=%v second top=%v", first[0][4], second[0][3])
+ }
+ if math.Abs(first[0][4]-50) > 1e-6 {
+ t.Errorf("equal 3+3 rune split must cut at mid height 50, got bottom=%v", first[0][4])
+ }
+}
+
+func TestSplitGeneralChildrenSlicesPositionsAbut(t *testing.T) {
+ chunk := schema.ChunkDoc{
+ Text: "aaa;bbb",
+ DocType: "text",
+ CKType: "text",
+ PDFPositions: json.RawMessage(`[[1,10,20,0,100]]`),
+ Positions: json.RawMessage(`[[1,10,20,0,100]]`),
+ }
+ pattern := compileChildrenPattern([]string{";"})
+ got := splitGeneralChildren([]schema.ChunkDoc{chunk}, pattern)
+ if len(got) != 2 {
+ t.Fatalf("children pieces = %d, want 2", len(got))
+ }
+ first := decodeGeneralMatrix(t, got[0].PDFPositions)
+ second := decodeGeneralMatrix(t, got[1].PDFPositions)
+ if math.Abs(first[0][4]-second[0][3]) > posSliceEpsilon {
+ t.Errorf("adjacent children slices must abut: %v / %v", first[0], second[0])
+ }
+ if first[0][4] == 100 && second[0][3] == 0 {
+ t.Errorf("children slices still carry the full box: %v / %v", first[0], second[0])
+ }
+}
+
+func TestSplitGeneralUnitsKeepDelimiterSlicesPositions(t *testing.T) {
+ unit := schema.ChunkDoc{
+ Text: "aaa。bbb。",
+ DocType: "text",
+ PDFPositions: json.RawMessage(`[[1,10,200,0,100]]`),
+ Positions: json.RawMessage(`[[1,10,200,0,100]]`),
+ }
+ pattern := compileDelimPattern([]string{"。"})
+ got := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, true)
+ if len(got) != 2 {
+ t.Fatalf("split pieces = %d, want 2 (%q)", len(got), generalChunkTexts(got))
+ }
+ if got[0].Text != "aaa。" || got[1].Text != "bbb。" {
+ t.Errorf("keepDelimiter must retain delimiters, got %q", generalChunkTexts(got))
+ }
+ first := decodeGeneralMatrix(t, got[0].PDFPositions)
+ second := decodeGeneralMatrix(t, got[1].PDFPositions)
+ if math.Abs(first[0][4]-second[0][3]) > posSliceEpsilon {
+ t.Errorf("keepDelimiter slices must abut: first bottom=%v second top=%v", first[0][4], second[0][3])
+ }
+}
+
+func TestSplitGeneralUnitsMultiRowSplitAcrossPages(t *testing.T) {
+ unit := schema.ChunkDoc{
+ Text: "上。下。",
+ DocType: "text",
+ PDFPositions: json.RawMessage(`[[1,10,20,0,10],[2,10,20,10,20]]`),
+ Positions: json.RawMessage(`[[1,10,20,0,10],[2,10,20,10,20]]`),
+ }
+ pattern := compileDelimPattern([]string{"。"})
+ got := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, false)
+ if len(got) != 2 {
+ t.Fatalf("split pieces = %d, want 2 (%q)", len(got), generalChunkTexts(got))
+ }
+ first := decodeGeneralMatrix(t, got[0].PDFPositions)
+ second := decodeGeneralMatrix(t, got[1].PDFPositions)
+ if len(first) != 1 || first[0][0] != 1 || first[0][3] != 0 || first[0][4] != 10 {
+ t.Errorf("first piece must cover page-1 row exactly, got %v", first)
+ }
+ if len(second) != 1 || second[0][0] != 2 || second[0][3] != 10 || second[0][4] != 20 {
+ t.Errorf("second piece must cover page-2 row exactly, got %v", second)
+ }
+}
+
+func TestSplitGeneralUnitsMalformedMatrixKeepsFullBox(t *testing.T) {
+ raw := json.RawMessage(`[[1,10,20,30,30]]`)
+ unit := schema.ChunkDoc{
+ Text: "第一句。第二句。",
+ DocType: "text",
+ PDFPositions: raw,
+ Positions: raw,
+ }
+ pattern := compileDelimPattern([]string{"。"})
+ got := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, false)
+ if len(got) != 2 {
+ t.Fatalf("split pieces = %d, want 2", len(got))
+ }
+ for i, piece := range got {
+ if string(piece.PDFPositions) != string(raw) || string(piece.Positions) != string(raw) {
+ t.Errorf("piece %d must keep the source box on malformed input, got %s / %s",
+ i, piece.PDFPositions, piece.Positions)
+ }
+ }
+}
+
+func TestSplitGeneralUnitsTagOnlyPieceKeepsFullBox(t *testing.T) {
+ // Contract lock (see TestSlicePositionsByCounts_ZeroWeightPieceKeepsFullBox):
+ // a tag-only segment between delimiters has no visible height yet keeps
+ // the full source box so it retains its preview. Changing this to empty
+ // positions requires checking ImgID-carrying media first.
+ raw := json.RawMessage(`[[1,10,200,0,100]]`)
+ unit := schema.ChunkDoc{
+ Text: "A。@@1\t10\t20\t30\t40##。B",
+ DocType: "text",
+ PDFPositions: raw,
+ Positions: raw,
+ }
+ pattern := compileDelimPattern([]string{"。"})
+ got := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, false)
+ if len(got) != 3 {
+ t.Fatalf("split pieces = %d, want 3 (%q)", len(got), generalChunkTexts(got))
+ }
+ if string(got[1].PDFPositions) != string(raw) {
+ t.Errorf("tag-only piece must keep the full box, got %s", got[1].PDFPositions)
+ }
+ first := decodeGeneralMatrix(t, got[0].PDFPositions)
+ third := decodeGeneralMatrix(t, got[2].PDFPositions)
+ if math.Abs(first[0][4]-50) > 1e-6 || math.Abs(third[0][3]-50) > 1e-6 {
+ t.Errorf("visible pieces must split around the tag-only segment, got %v / %v", first, third)
+ }
+}
diff --git a/internal/ingestion/component/chunker/positions_slice.go b/internal/ingestion/component/chunker/positions_slice.go
index 59515f54d2..7a9806bc68 100644
--- a/internal/ingestion/component/chunker/positions_slice.go
+++ b/internal/ingestion/component/chunker/positions_slice.go
@@ -119,6 +119,50 @@ func slicePositionsByTextRatio(raw json.RawMessage, startRatio, endRatio float64
return json.RawMessage(b)
}
+// slicePositionsByCounts slices both position matrices by per-piece visible
+// counts so adjacent pieces abut instead of each carrying the full source
+// box. counts[i] is the visible-text weight of piece i; parser tags and
+// synthetic glue must already be excluded by the caller (see removeTag).
+// A piece whose slice is empty (zero-weight piece or malformed matrix)
+// falls back to the source matrix so it keeps its preview instead of none.
+// When the total weight is not positive every piece falls back the same way.
+// Shared by the Token path (chunkFromItem/splitOversizedText) and the General
+// path (sliceGeneralSplitPieces) so the three stay mirrored.
+func slicePositionsByCounts(pdfSrc, posSrc json.RawMessage, counts []int) ([]json.RawMessage, []json.RawMessage) {
+ pdfOut := make([]json.RawMessage, len(counts))
+ posOut := make([]json.RawMessage, len(counts))
+ total := 0
+ for _, c := range counts {
+ total += c
+ }
+ if total <= 0 {
+ for i := range counts {
+ if len(pdfSrc) > 0 {
+ pdfOut[i] = pdfSrc
+ }
+ if len(posSrc) > 0 {
+ posOut[i] = posSrc
+ }
+ }
+ return pdfOut, posOut
+ }
+ cum := 0
+ for i, c := range counts {
+ startRatio := float64(cum) / float64(total)
+ endRatio := float64(cum+c) / float64(total)
+ pdfOut[i] = slicePositionsByTextRatio(pdfSrc, startRatio, endRatio)
+ posOut[i] = slicePositionsByTextRatio(posSrc, startRatio, endRatio)
+ if len(pdfOut[i]) == 0 && len(pdfSrc) > 0 {
+ pdfOut[i] = pdfSrc
+ }
+ if len(posOut[i]) == 0 && len(posSrc) > 0 {
+ posOut[i] = posSrc
+ }
+ cum += c
+ }
+ return pdfOut, posOut
+}
+
// sliceAnyPositions is the map-value variant used by the title chunker
// (GroupTitle/HierarchyTitle) where positions are stored as any
// (typically [][]float64). It round-trips through JSON so the same
diff --git a/internal/ingestion/component/chunker/positions_slice_test.go b/internal/ingestion/component/chunker/positions_slice_test.go
index 1b6541c80a..5fd57d35e0 100644
--- a/internal/ingestion/component/chunker/positions_slice_test.go
+++ b/internal/ingestion/component/chunker/positions_slice_test.go
@@ -197,3 +197,63 @@ func TestTotalRunes(t *testing.T) {
t.Errorf("totalRunes(nil) = %d, want 0", got)
}
}
+
+func TestSlicePositionsByCounts_EqualSplitAbuts(t *testing.T) {
+ raw := json.RawMessage(`[[1,10,200,0,100]]`)
+ pdf, pos := slicePositionsByCounts(raw, raw, []int{3, 3})
+ if len(pdf) != 2 || len(pos) != 2 {
+ t.Fatalf("expected 2+2 slices, got %d+%d", len(pdf), len(pos))
+ }
+ assertMatrixNear(t, pdf[0], [][]float64{{1, 10, 200, 0, 50}})
+ assertMatrixNear(t, pdf[1], [][]float64{{1, 10, 200, 50, 100}})
+ assertMatrixNear(t, pos[0], [][]float64{{1, 10, 200, 0, 50}})
+ assertMatrixNear(t, pos[1], [][]float64{{1, 10, 200, 50, 100}})
+}
+
+func TestSlicePositionsByCounts_ZeroWeightPieceKeepsFullBox(t *testing.T) {
+ // Contract lock: a zero-visible-weight piece (e.g. a tag-only segment
+ // between delimiters) falls back to the full source box so it keeps its
+ // preview. This mirrors the Token path; switching zero-weight pieces to
+ // empty positions must be a deliberate, three-call-site change with a
+ // preview-impact check for ImgID-carrying media, not a drive-by.
+ raw := json.RawMessage(`[[1,10,200,0,100]]`)
+ pdf, _ := slicePositionsByCounts(raw, nil, []int{1, 0, 1})
+ if len(pdf) != 3 {
+ t.Fatalf("expected 3 slices, got %d", len(pdf))
+ }
+ assertMatrixNear(t, pdf[0], [][]float64{{1, 10, 200, 0, 50}})
+ assertMatrixNear(t, pdf[1], [][]float64{{1, 10, 200, 0, 100}})
+ assertMatrixNear(t, pdf[2], [][]float64{{1, 10, 200, 50, 100}})
+}
+
+func TestSlicePositionsByCounts_MalformedMatrixFallsBack(t *testing.T) {
+ raw := json.RawMessage(`[[1,10,20,30,30]]`)
+ pdf, pos := slicePositionsByCounts(raw, raw, []int{1, 1})
+ for i := range pdf {
+ if string(pdf[i]) != string(raw) || string(pos[i]) != string(raw) {
+ t.Errorf("piece %d must fall back to the source box, got %s / %s", i, pdf[i], pos[i])
+ }
+ }
+}
+
+func TestSlicePositionsByCounts_NonPositiveTotalFallsBack(t *testing.T) {
+ raw := json.RawMessage(`[[1,10,200,0,100]]`)
+ pdf, pos := slicePositionsByCounts(raw, raw, []int{0, 0})
+ for i := range pdf {
+ if string(pdf[i]) != string(raw) || string(pos[i]) != string(raw) {
+ t.Errorf("piece %d must fall back to the source box, got %s / %s", i, pdf[i], pos[i])
+ }
+ }
+}
+
+func TestSlicePositionsByCounts_EmptySourcesStayEmpty(t *testing.T) {
+ pdf, pos := slicePositionsByCounts(nil, nil, []int{2, 2})
+ if len(pdf) != 2 || len(pos) != 2 {
+ t.Fatalf("expected 2+2 slices, got %d+%d", len(pdf), len(pos))
+ }
+ for i := range pdf {
+ if len(pdf[i]) != 0 || len(pos[i]) != 0 {
+ t.Errorf("piece %d must stay empty, got %s / %s", i, pdf[i], pos[i])
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index f710e1f177..9cbfa38610 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -755,35 +755,20 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bo
}
// Visible-text ratio: parser tags (@@...##) carry no visual height
// and must not shift crop boundaries. Mirrors the overlap logic
- // which already counts on removeTag'd text.
- runCount := 0
- for _, p := range kept {
- runCount += utf8.RuneCountInString(removeTag(p))
- }
- if runCount == 0 {
- out := make([]schema.ChunkDoc, 0, len(kept))
- for _, p := range kept {
- out = append(out, buildChunkDoc(it, "text", p, "", ""))
- }
- return out
- }
+ // which already counts on removeTag'd text. Positions are sliced via
+ // the shared slicePositionsByCounts helper (mirrored by the General
+ // path) so each delimiter-split piece crops only its own region.
+ counts := make([]int, len(kept))
+ for i, p := range kept {
+ counts[i] = utf8.RuneCountInString(removeTag(p))
+ }
+ pdfSliced, posSliced := slicePositionsByCounts(it.PDFPositions, it.Positions, counts)
out := make([]schema.ChunkDoc, 0, len(kept))
- cumRunes := 0
- for _, p := range kept {
- pcRunes := utf8.RuneCountInString(removeTag(p))
- startRatio := float64(cumRunes) / float64(runCount)
- endRatio := float64(cumRunes+pcRunes) / float64(runCount)
+ for i, p := range kept {
ck := buildChunkDoc(it, "text", p, "", "")
- ck.PDFPositions = slicePositionsByTextRatio(it.PDFPositions, startRatio, endRatio)
- ck.Positions = slicePositionsByTextRatio(it.Positions, startRatio, endRatio)
- if len(ck.PDFPositions) == 0 && len(it.PDFPositions) > 0 {
- ck.PDFPositions = it.PDFPositions
- }
- if len(ck.Positions) == 0 && len(it.Positions) > 0 {
- ck.Positions = it.Positions
- }
+ ck.PDFPositions = pdfSliced[i]
+ ck.Positions = posSliced[i]
out = append(out, ck)
- cumRunes += pcRunes
}
return out
}
@@ -1231,6 +1216,21 @@ func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc
return out
}
+// pieceVisibleCount returns the visible-text weight of a split piece: the
+// rune count after stripping parser tags (@@...##), excluding the synthetic
+// leading "\n" glue on the first piece (text-path units are built as
+// "\n"+paragraph, which carries no visual height).
+func pieceVisibleCount(text string, isFirst bool, lead string) int {
+ n := utf8.RuneCountInString(removeTag(text))
+ if isFirst && lead != "" {
+ n -= utf8.RuneCountInString(lead)
+ if n < 0 {
+ n = 0
+ }
+ }
+ return n
+}
+
// splitOversizedText splits one oversized text unit into <= target pieces.
// Sentence boundaries are preferred; a boundary-less run that still exceeds
// target is hard token-split. Delimiters are preserved so the concatenated
@@ -1304,67 +1304,23 @@ func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
}
// Every sub-piece inherits the source unit's metadata (coarse positions +
// item attributes); each is built from a clone so every ChunkDoc field
- // survives the split. PDF positions are proportionally sliced vertically so
- // each piece's screenshot crops only its own region instead of the whole
- // paragraph (fix for chunk-screenshot mismatch).
- if len(ck.PDFPositions) == 0 && len(ck.Positions) == 0 {
- for i := range pieces {
- inherited := cloneChunkDoc(ck)
- inherited.Text = pieces[i].Text
- inherited.TKNums = pieces[i].TKNums
- inherited.CKType = "text"
- pieces[i] = inherited
- }
- return pieces
- }
- // Ratio basis: rune counts of the real source content. The synthetic
- // leading "\n" glue (text-path units are built as "\n"+paragraph) carries
- // no visual height, so it is excluded from the first piece's count to keep
- // the slice proportions aligned with the original text. Parser tags
- // (@@...##) also carry no visual height and are stripped before counting.
+ // survives the split. PDF positions are sliced via the shared
+ // slicePositionsByCounts helper so each piece's screenshot crops only
+ // its own region instead of the whole paragraph (fix for
+ // chunk-screenshot mismatch).
counts := make([]int, len(pieces))
- runCount := 0
for i, p := range pieces {
- n := utf8.RuneCountInString(removeTag(p.Text))
- if i == 0 && lead != "" {
- n -= utf8.RuneCountInString(lead)
- if n < 0 {
- n = 0
- }
- }
- counts[i] = n
- runCount += n
+ counts[i] = pieceVisibleCount(p.Text, i == 0, lead)
}
- if runCount == 0 {
- for i := range pieces {
- inherited := cloneChunkDoc(ck)
- inherited.Text = pieces[i].Text
- inherited.TKNums = pieces[i].TKNums
- inherited.CKType = "text"
- pieces[i] = inherited
- }
- return pieces
- }
- cumRunes := 0
+ pdfSliced, posSliced := slicePositionsByCounts(ck.PDFPositions, ck.Positions, counts)
for i, p := range pieces {
- startRatio := float64(cumRunes) / float64(runCount)
- endRatio := float64(cumRunes+counts[i]) / float64(runCount)
inherited := cloneChunkDoc(ck)
inherited.Text = p.Text
inherited.TKNums = p.TKNums
inherited.CKType = "text"
- inherited.PDFPositions = slicePositionsByTextRatio(ck.PDFPositions, startRatio, endRatio)
- inherited.Positions = slicePositionsByTextRatio(ck.Positions, startRatio, endRatio)
- // Fall back to original positions if slicing produced nothing (e.g.
- // malformed matrix) so the piece still gets a preview rather than none.
- if len(inherited.PDFPositions) == 0 && len(ck.PDFPositions) > 0 {
- inherited.PDFPositions = ck.PDFPositions
- }
- if len(inherited.Positions) == 0 && len(ck.Positions) > 0 {
- inherited.Positions = ck.Positions
- }
+ inherited.PDFPositions = pdfSliced[i]
+ inherited.Positions = posSliced[i]
pieces[i] = inherited
- cumRunes += counts[i]
}
return pieces
}