Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): align PDF child previews with source positions (#21853)

jay777218e0ed2bc151f
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index a9276e3cb0..44a6c69741 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -125,21 +125,9 @@ func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string
return []string{text}
}
if !keepDelim {
- // Drop mode: keep only the text between delimiters, skipping any
- // delimiter that sits at the very start (no preceding text).
var out []string
- cursor := 0
- for _, idx := range idxs {
- start, end := idx[0], idx[1]
- if start == cursor {
- cursor = end
- continue
- }
- out = append(out, text[cursor:start])
- cursor = end
- }
- if cursor < len(text) {
- out = append(out, text[cursor:])
+ for _, part := range splitDroppedDelimiterRanges(len(text), idxs) {
+ out = append(out, text[part[0]:part[1]])
}
return out
}
@@ -159,6 +147,21 @@ func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string
return out
}
+func splitDroppedDelimiterRanges(length int, matches [][]int) [][2]int {
+ var ranges [][2]int
+ cursor := 0
+ for _, match := range matches {
+ if match[0] > cursor {
+ ranges = append(ranges, [2]int{cursor, match[0]})
+ }
+ cursor = match[1]
+ }
+ if cursor < length {
+ ranges = append(ranges, [2]int{cursor, length})
+ }
+ return ranges
+}
+
// foldWhitespaceSegments attaches blank runs to neighboring text without
// changing their position in the reconstructed source.
func foldWhitespaceSegments(parts []string) []string {
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index ab6ff4d21e..0caa64aa55 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -233,11 +233,19 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
+ var unitSources [][]pdfSourceSpan
+ if childrenPattern != nil {
+ unitSources = pdfUnitSources(units)
+ }
customDelimiter := hasCustomDelim(c.param.Delimiters)
if customDelimiter {
- units = splitGeneralUnits(units, primaryPattern, false)
+ units, unitSources = splitPDFUnits(units, unitSources, primaryPattern, false)
+ }
+ if unitSources != nil {
+ units, unitSources = sortPDFUnitsWithSources(units, unitSources)
+ } else {
+ units = sortPDFUnits(units)
}
- units = sortPDFUnits(units)
if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {
common.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")
}
@@ -245,6 +253,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
chunks := make([]schema.ChunkDoc, 0, len(units))
body := make([]schema.ChunkDoc, 0, len(units))
+ var chunkSources, bodySources [][]pdfSourceSpan
flushBody := func() {
if len(body) == 0 {
return
@@ -252,21 +261,30 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
joinSep := "\n"
if !customDelimiter {
body = separateGeneralTextBlocks(body)
- body = splitGeneralUnits(body, primaryPattern, true)
+ body, bodySources = splitPDFUnits(body, bodySources, primaryPattern, true)
joinSep = ""
}
- chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, joinSep)...)
+ merged, mergedSources := mergeGeneralUnitsWithSources(body, bodySources, c.param.ChunkTokenSize, c.param.OverlappedPercent, joinSep)
+ chunks = append(chunks, merged...)
+ chunkSources = append(chunkSources, mergedSources...)
body = body[:0]
+ bodySources = bodySources[:0]
}
- for _, unit := range units {
+ for index, unit := range units {
if itemDocType(unit) == "text" {
body = append(body, unit)
+ if unitSources != nil {
+ bodySources = append(bodySources, unitSources[index])
+ }
} else {
flushBody()
media := cloneChunkDoc(unit)
media.Text = normalizeGeneralNewlines(media.Text)
media.DocType = itemDocType(media)
chunks = append(chunks, media)
+ if unitSources != nil {
+ chunkSources = append(chunkSources, nil)
+ }
}
}
flushBody()
@@ -276,7 +294,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
chunks[i].Text = removeTag(chunks[i].Text)
}
}
- chunks = splitGeneralChildren(chunks, childrenPattern)
+ chunks = splitGeneralChildrenWithSources(chunks, chunkSources, childrenPattern)
engine, err := newPDFEngineFromUpstream(ctx, db, upstream)
if err != nil {
common.Warn("GeneralChunker: could not open PDF for on-demand cropping", zap.Error(err))
@@ -300,15 +318,25 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
// order. Units with explicit parser order or without coordinates are barriers:
// they stay at their parser positions and preserve their relative order.
func sortPDFUnits(units []schema.ChunkDoc) []schema.ChunkDoc {
+ ordered, _ := sortPDFUnitsWithSources(units, nil)
+ return ordered
+}
+
+func sortPDFUnitsWithSources(units []schema.ChunkDoc, sources [][]pdfSourceSpan) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
type positionedUnit struct {
unit schema.ChunkDoc
row []float64
positioned bool
+ sources []pdfSourceSpan
}
ordered := make([]positionedUnit, 0, len(units))
- for _, unit := range units {
+ for index, unit := range units {
row, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions})
- ordered = append(ordered, positionedUnit{unit: cloneChunkDoc(unit), row: row, positioned: ok && !unit.PreserveOrder})
+ item := positionedUnit{unit: cloneChunkDoc(unit), row: row, positioned: ok && !unit.PreserveOrder}
+ if sources != nil {
+ item.sources = sources[index]
+ }
+ ordered = append(ordered, item)
}
runStart := -1
flush := func(end int) {
@@ -331,10 +359,14 @@ func sortPDFUnits(units []schema.ChunkDoc) []schema.ChunkDoc {
flush(i)
}
result := make([]schema.ChunkDoc, 0, len(ordered))
+ var orderedSources [][]pdfSourceSpan
for _, item := range ordered {
result = append(result, item.unit)
+ if sources != nil {
+ orderedSources = append(orderedSources, item.sources)
+ }
}
- return result
+ return result, orderedSources
}
func hasPDFPositions(units []schema.ChunkDoc) bool {
@@ -1181,14 +1213,24 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch
// appended while the current chunk is at or below the overlap-scaled target;
// the append itself may cross the target. Oversized units remain indivisible.
func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
+ merged, _ := mergeGeneralUnitsWithSources(units, nil, target, overlapPct, joinSep)
+ return merged
+}
+
+func mergeGeneralUnitsWithSources(units []schema.ChunkDoc, sources [][]pdfSourceSpan, target int, overlapPct float64, joinSep string) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
overlapPct = max(0, min(100, overlapPct))
threshold := float64(target) * (100 - overlapPct) / 100
merged := make([]schema.ChunkDoc, 0, len(units))
+ var mergedSources [][]pdfSourceSpan
+ tracked := sources != nil
current := -1
- for _, unit := range units {
+ for index, unit := range units {
if itemDocType(unit) != "text" {
merged = append(merged, cloneChunkDoc(unit))
+ if tracked {
+ mergedSources = append(mergedSources, sources[index])
+ }
current = -1
continue
}
@@ -1202,19 +1244,40 @@ func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64,
unit.TKNums = intPtr(count)
if count > target {
merged = append(merged, cloneChunkDoc(unit))
+ if tracked {
+ mergedSources = append(mergedSources, sources[index])
+ }
current = -1
continue
}
if current < 0 || float64(intValue(merged[current].TKNums)) > threshold {
merged = append(merged, cloneChunkDoc(unit))
+ if tracked {
+ mergedSources = append(mergedSources, sources[index])
+ }
current = len(merged) - 1
continue
}
+ if tracked {
+ if mergedSources[current] == nil || sources[index] == nil {
+ mergedSources[current] = nil
+ } else {
+ offset := len(removeTag(merged[current].Text))
+ if merged[current].Text != "" && unit.Text != "" {
+ offset += len(removeTag(joinSep))
+ }
+ for _, span := range sources[index] {
+ span.start += offset
+ span.end += offset
+ mergedSources[current] = append(mergedSources[current], span)
+ }
+ }
+ }
mergeGeneralChunk(&merged[current], unit, joinSep)
}
- return applyGeneralOverlap(merged, overlapPct, joinSep)
+ return applyGeneralOverlapWithSources(merged, mergedSources, overlapPct, joinSep)
}
func sameSpreadsheetTable(first, second schema.ChunkDoc) bool {
@@ -1300,8 +1363,13 @@ func mergeGeneralPositions(first, second json.RawMessage) json.RawMessage {
}
func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep string) []schema.ChunkDoc {
+ result, _ := applyGeneralOverlapWithSources(chunks, nil, overlapPct, joinSep)
+ return result
+}
+
+func applyGeneralOverlapWithSources(chunks []schema.ChunkDoc, sources [][]pdfSourceSpan, overlapPct float64, joinSep string) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
if overlapPct <= 0 {
- return chunks
+ return chunks, sources
}
previousText := -1
for i := range chunks {
@@ -1313,6 +1381,24 @@ func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep s
prefix, _ := computeOverlapPrefix(chunks[previousText].Text, overlapPct)
if prefix != "" {
current := cloneChunkDoc(chunks[i])
+ if sources != nil {
+ previous := chunks[previousText]
+ previousVisible := removeTag(previous.Text)
+ if sources[previousText] == nil || sources[i] == nil || !strings.HasSuffix(previousVisible, prefix) {
+ sources[i] = nil
+ } else {
+ start := len(previousVisible) - len(prefix)
+ spans := slicePDFSourceSpans(sources[previousText], previousVisible, start, len(previousVisible))
+ joined := joinGeneralOverlapText(prefix, current.Text, joinSep)
+ offset := len(removeTag(joined)) - len(removeTag(current.Text))
+ for _, span := range sources[i] {
+ span.start += offset
+ span.end += offset
+ spans = append(spans, span)
+ }
+ sources[i] = spans
+ }
+ }
current.Text = joinGeneralOverlapText(prefix, current.Text, joinSep)
current.TKNums = intPtr(tokenizeStr(current.Text))
chunks[i] = current
@@ -1320,7 +1406,7 @@ func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep s
}
previousText = i
}
- return chunks
+ return chunks, sources
}
func joinGeneralOverlapText(prefix, text, separator string) string {
@@ -1441,21 +1527,43 @@ func normalizeGeneralNewlines(text string) string {
// (@@...##) carry no visual height and are excluded from the ratio, mirroring
// the Token path (chunkFromItem/splitOversizedText).
func sliceGeneralSplitPieces(unit schema.ChunkDoc, parts []string) []schema.ChunkDoc {
+ return sliceGeneralSplitPiecesWithSources(unit, parts, nil)
+}
+
+func sliceGeneralSplitPiecesWithSources(unit schema.ChunkDoc, parts []string, sources []pdfSourceSpan) []schema.ChunkDoc {
out := make([]schema.ChunkDoc, 0, len(parts))
if len(parts) == 0 {
return out
}
- counts := make([]int, len(parts))
- for i, part := range parts {
- counts[i] = utf8.RuneCountInString(removeTag(part))
- }
- pdfSliced, posSliced := slicePositionsByCounts(unit.PDFPositions, unit.Positions, counts)
+ var pdfSliced, posSliced []json.RawMessage
+ cursor := 0
for i, part := range parts {
piece := cloneChunkDoc(unit)
piece.Text = part
piece.TKNums = intPtr(tokenizeStr(part))
- piece.PDFPositions = pdfSliced[i]
- piece.Positions = posSliced[i]
+ positioned := false
+ if sources != nil {
+ if offset := strings.Index(unit.Text[cursor:], part); offset >= 0 {
+ start := cursor + offset
+ cursor = start + len(part)
+ spans := slicePDFSourceSpans(sources, unit.Text, start, cursor)
+ pdfPositions, positions := pdfSourcePositions(spans)
+ if len(pdfPositions) > 0 || len(positions) > 0 {
+ piece.PDFPositions, piece.Positions = pdfPositions, positions
+ positioned = true
+ }
+ }
+ }
+ if !positioned {
+ if pdfSliced == nil {
+ counts := make([]int, len(parts))
+ for index, text := range parts {
+ counts[index] = utf8.RuneCountInString(removeTag(text))
+ }
+ pdfSliced, posSliced = slicePositionsByCounts(unit.PDFPositions, unit.Positions, counts)
+ }
+ piece.PDFPositions, piece.Positions = pdfSliced[i], posSliced[i]
+ }
out = append(out, piece)
}
return out
@@ -1491,11 +1599,15 @@ func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Reg
// TokenChunker splitByChildren helper intentionally drops delimiters and is
// therefore not reused here.
func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
+ return splitGeneralChildrenWithSources(chunks, nil, pattern)
+}
+
+func splitGeneralChildrenWithSources(chunks []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *regexp.Regexp) []schema.ChunkDoc {
if pattern == nil {
return chunks
}
result := make([]schema.ChunkDoc, 0, len(chunks))
- for _, chunk := range chunks {
+ for index, chunk := range chunks {
if itemDocType(chunk) != "text" {
result = append(result, chunk)
continue
@@ -1513,7 +1625,11 @@ func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []sc
result = append(result, chunk)
continue
}
- for _, piece := range sliceGeneralSplitPieces(chunk, parts) {
+ var spans []pdfSourceSpan
+ if sources != nil {
+ spans = sources[index]
+ }
+ for _, piece := range sliceGeneralSplitPiecesWithSources(chunk, parts, spans) {
piece.Mom = mom
result = append(result, piece)
}
diff --git a/internal/ingestion/component/chunker/general_pdf_source_positions.go b/internal/ingestion/component/chunker/general_pdf_source_positions.go
new file mode 100644
index 0000000000..d6f66a0f8c
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_source_positions.go
@@ -0,0 +1,122 @@
+package chunker
+
+import (
+ "encoding/json"
+ "regexp"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+type pdfSourceSpan struct {
+ start, end int
+ pdfPositions, positions json.RawMessage
+}
+
+func pdfUnitSources(units []schema.ChunkDoc) [][]pdfSourceSpan {
+ sources := make([][]pdfSourceSpan, len(units))
+ for index, unit := range units {
+ text := removeTag(normalizeGeneralNewlines(unit.Text))
+ start := len(text) - len(strings.TrimLeftFunc(text, unicode.IsSpace))
+ sources[index] = []pdfSourceSpan{{start: start, end: start + len(strings.TrimSpace(text)), pdfPositions: unit.PDFPositions, positions: unit.Positions}}
+ }
+ return sources
+}
+
+func splitPDFUnits(units []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *regexp.Regexp, keepDelimiter bool) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
+ if sources == nil {
+ return splitGeneralUnits(units, pattern, keepDelimiter), nil
+ }
+ var result []schema.ChunkDoc
+ var resultSources [][]pdfSourceSpan
+ for index, unit := range units {
+ pieces := splitGeneralUnits([]schema.ChunkDoc{unit}, pattern, keepDelimiter)
+ rawText := normalizeGeneralNewlines(unit.Text)
+ text := removeTag(rawText)
+ var ranges [][2]int
+ if !keepDelimiter && pattern != nil && itemDocType(unit) == "text" && pattern.MatchString(rawText) {
+ for _, part := range splitDroppedDelimiterRanges(len(rawText), pattern.FindAllStringIndex(rawText, -1)) {
+ if strings.TrimSpace(rawText[part[0]:part[1]]) != "" {
+ ranges = append(ranges, part)
+ }
+ }
+ }
+ cursor := 0
+ for pieceIndex, piece := range pieces {
+ pieceText := removeTag(piece.Text)
+ var spans []pdfSourceSpan
+ if ranges != nil && len(ranges) == len(pieces) && sources[index] != nil {
+ part := ranges[pieceIndex]
+ prefix, selected, suffix := removeTag(rawText[:part[0]]), removeTag(rawText[part[0]:part[1]]), removeTag(rawText[part[1]:])
+ if selected == pieceText && prefix+selected+suffix == text {
+ spans = slicePDFSourceSpans(sources[index], text, len(prefix), len(prefix)+len(selected))
+ }
+ } else if ranges == nil {
+ if offset := strings.Index(text[cursor:], pieceText); offset >= 0 && sources[index] != nil {
+ start := cursor + offset
+ end := start + len(pieceText)
+ spans = slicePDFSourceSpans(sources[index], text, start, end)
+ cursor = end
+ }
+ }
+ result = append(result, piece)
+ resultSources = append(resultSources, spans)
+ }
+ }
+ return result, resultSources
+}
+
+func slicePDFSourceSpans(spans []pdfSourceSpan, text string, start, end int) []pdfSourceSpan {
+ result := make([]pdfSourceSpan, 0, len(spans))
+ for _, span := range spans {
+ if span.start < 0 || span.end > len(text) || span.start >= span.end {
+ continue
+ }
+ left, right := max(start, span.start), min(end, span.end)
+ if left >= right {
+ continue
+ }
+ if left != span.start || right != span.end {
+ total := utf8.RuneCountInString(text[span.start:span.end])
+ if total > 0 {
+ begin := float64(utf8.RuneCountInString(text[span.start:left])) / float64(total)
+ finish := float64(utf8.RuneCountInString(text[span.start:right])) / float64(total)
+ if sliced := slicePositionsByTextRatio(span.pdfPositions, begin, finish); len(sliced) > 0 {
+ span.pdfPositions = sliced
+ }
+ if sliced := slicePositionsByTextRatio(span.positions, begin, finish); len(sliced) > 0 {
+ span.positions = sliced
+ }
+ }
+ }
+ span.start = left - start
+ span.end = right - start
+ result = append(result, span)
+ }
+ return result
+}
+
+func pdfSourcePositions(spans []pdfSourceSpan) (json.RawMessage, json.RawMessage) {
+ pdf := make([]json.RawMessage, 0, len(spans))
+ positions := make([]json.RawMessage, 0, len(spans))
+ for _, span := range spans {
+ pdf = append(pdf, span.pdfPositions)
+ positions = append(positions, span.positions)
+ }
+ return mergePDFSourcePositions(pdf), mergePDFSourcePositions(positions)
+}
+
+func mergePDFSourcePositions(sources []json.RawMessage) json.RawMessage {
+ if rows := mergePositionMatrix(sources...); rows != nil {
+ if encoded, err := json.Marshal(rows); err == nil {
+ return encoded
+ }
+ }
+ var positions json.RawMessage
+ for _, source := range sources {
+ positions = extendRawJSONArray(positions, source)
+ }
+ return positions
+}
diff --git a/internal/ingestion/component/chunker/general_pdf_source_positions_test.go b/internal/ingestion/component/chunker/general_pdf_source_positions_test.go
new file mode 100644
index 0000000000..27af8adca1
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_source_positions_test.go
@@ -0,0 +1,280 @@
+package chunker
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func TestGeneralPDFChildPositionsFollowSourceText(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"#"},
+ "children_delimiters": []string{"#"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ items := []map[string]any{
+ {"text": "上一段的长长长长长长长长长长长长长长长长长长长长长长长长长长长句。", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 10, 40}}},
+ {"text": "#", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 20, 45, 55}}},
+ {"text": "本段第一句。", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 60, 75}}},
+ {"text": "本段第二句。", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 80, 95}}},
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "source.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want 2: %#v", len(chunks), chunks)
+ }
+ for index, want := range [][][]float64{
+ {{1, 10, 200, 10, 40}, {1, 10, 20, 45, 55}},
+ {{1, 10, 200, 60, 75}, {1, 10, 200, 80, 95}},
+ } {
+ raw, err := json.Marshal(chunks[index]["positions"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got [][]float64
+ if err := json.Unmarshal(raw, &got); err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("chunk %d positions = %v, want %v", index, got, want)
+ }
+ }
+}
+
+func TestGeneralPDFOverlapChildPositionsIncludeSourceTail(t *testing.T) {
+ firstPosition := json.RawMessage(`[[1,10,20,0,100]]`)
+ secondPosition := json.RawMessage(`[[1,10,20,200,300]]`)
+ chunks := []schema.ChunkDoc{
+ {Text: "ABCDE", DocType: "text", Positions: firstPosition},
+ {Text: "FGHIJ", DocType: "text", Positions: secondPosition},
+ }
+ sources := [][]pdfSourceSpan{{{start: 0, end: 5, positions: firstPosition}}, {{start: 0, end: 5, positions: secondPosition}}}
+ chunks, sources = applyGeneralOverlapWithSources(chunks, sources, 40, "\n")
+ children := splitGeneralChildrenWithSources(chunks, sources, compileDelimPattern([]string{"#"}))
+ if len(children) != 2 || children[1].Text != "DE\nFGHIJ" {
+ t.Fatalf("unexpected overlapped children: %#v", children)
+ }
+ got := decodeGeneralMatrix(t, children[1].Positions)
+ want := [][]float64{{1, 10, 20, 60, 100}, {1, 10, 20, 200, 300}}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("overlap positions = %v, want %v", got, want)
+ }
+}
+
+func TestGeneralPDFChildrenWithinOneSourceUseProportionalPositions(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"#"},
+ "children_delimiters": []string{"#"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "source.pdf", "file_type": "pdf", "output_format": "json",
+ "json": []map[string]any{{"text": "one#two", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 20, 0, 70}}}},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want 2", len(chunks))
+ }
+ for index, want := range [][][]float64{{{1, 10, 20, 0, 40}}, {{1, 10, 20, 40, 70}}} {
+ raw, err := json.Marshal(chunks[index]["positions"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got [][]float64
+ if err := json.Unmarshal(raw, &got); err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("chunk %d positions = %v, want %v", index, got, want)
+ }
+ }
+}
+
+func TestGeneralPDFChildPositionsFollowPageOrder(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"#"}, "children_delimiters": []string{"#"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ items := []map[string]any{
+ {"text": "second page", "doc_type_kwd": "text", "positions": [][]float64{{2, 10, 100, 10, 20}}},
+ {"text": "first page", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 100, 10, 20}}},
+ {"text": "#", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 20, 25, 35}}},
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "source.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want 2: %#v", len(chunks), chunks)
+ }
+ for index, want := range [][][]float64{{{1, 10, 100, 10, 20}, {1, 10, 20, 25, 35}}, {{2, 10, 100, 10, 20}}} {
+ raw, err := json.Marshal(chunks[index]["positions"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got [][]float64
+ if err := json.Unmarshal(raw, &got); err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("chunk %d positions = %v, want %v", index, got, want)
+ }
+ }
+}
+
+func TestPDFSourceOffsetsSkipCustomDelimiter(t *testing.T) {
+ for _, testCase := range []struct {
+ name string
+ text string
+ wantTexts []string
+ wantBounds [][2]float64
+ }{
+ {"delimiter substring", "a###", []string{"a", "#"}, [][2]float64{{0, 25}, {75, 100}}},
+ {"leading and repeated", "##a####b", []string{"a", "b"}, [][2]float64{{25, 37.5}, {87.5, 100}}},
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ unit := schema.ChunkDoc{Text: testCase.text, DocType: "text", Positions: json.RawMessage(`[[1,10,200,0,100]]`)}
+ pieces, sources := splitPDFUnits([]schema.ChunkDoc{unit}, pdfUnitSources([]schema.ChunkDoc{unit}), compileDelimPattern([]string{"`##`"}), false)
+ if len(pieces) != len(testCase.wantTexts) {
+ t.Fatalf("pieces = %v, want %v", generalChunkTexts(pieces), testCase.wantTexts)
+ }
+ for index, piece := range pieces {
+ if piece.Text != testCase.wantTexts[index] {
+ t.Errorf("piece %d text = %q, want %q", index, piece.Text, testCase.wantTexts[index])
+ }
+ _, raw := pdfSourcePositions(sources[index])
+ positions := decodeGeneralMatrix(t, raw)
+ if len(positions) != 1 || positions[0][3] != testCase.wantBounds[index][0] || positions[0][4] != testCase.wantBounds[index][1] {
+ t.Errorf("piece %d positions = %v, want bounds %v", index, positions, testCase.wantBounds[index])
+ }
+ }
+ })
+ }
+}
+
+func TestGeneralPDFCustomDelimiterPreservesSourceBounds(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512, "delimiters": []string{"`##`"}, "children_delimiters": []string{"。"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "source.pdf", "file_type": "pdf", "output_format": "json",
+ "json": []map[string]any{{"text": "a###", "doc_type_kwd": "text", "positions": [][]float64{{1, 10, 200, 0, 100}}}},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 || chunks[0]["text"] != "a\n#" {
+ t.Fatalf("unexpected chunks: %#v", chunks)
+ }
+ raw, err := json.Marshal(chunks[0]["positions"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got [][]float64
+ if err := json.Unmarshal(raw, &got); err != nil {
+ t.Fatal(err)
+ }
+ want := [][]float64{{1, 10, 200, 0, 25}, {1, 10, 200, 75, 100}}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("positions = %v, want %v", got, want)
+ }
+}
+
+func TestPDFSourceOffsetsIgnorePositionTags(t *testing.T) {
+ unit := schema.ChunkDoc{
+ Text: "a@@1\t2\t3\t4\t5##xyyb", DocType: "text",
+ Positions: json.RawMessage(`[[1,10,200,0,100]]`),
+ }
+ pieces, sources := splitPDFUnits([]schema.ChunkDoc{unit}, pdfUnitSources([]schema.ChunkDoc{unit}), compileDelimPattern([]string{"`xy`"}), false)
+ if len(pieces) != 2 || removeTag(pieces[1].Text) != "yb" {
+ t.Fatalf("unexpected pieces: %v", generalChunkTexts(pieces))
+ }
+ _, raw := pdfSourcePositions(sources[1])
+ got := decodeGeneralMatrix(t, raw)
+ want := [][]float64{{1, 10, 200, 60, 100}}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("positions = %v, want %v", got, want)
+ }
+}
+
+func TestPDFSourceOffsetsWithWhitespaceDelimiters(t *testing.T) {
+ for _, testCase := range []struct {
+ name string
+ text string
+ delimiter string
+ bounds [][2]float64
+ }{
+ {"repeated spaces", "a b", "` `", [][2]float64{{0, 25}, {75, 100}}},
+ {"repeated tabs", "a\t\tb", "`\t`", [][2]float64{{0, 25}, {75, 100}}},
+ {"spaces in delimiter", "a / b", "` / `", [][2]float64{{0, 20}, {80, 100}}},
+ {"position tag", "a@@1\t2\t3\t4\t5## b", "` `", [][2]float64{{0, 25}, {75, 100}}},
+ } {
+ t.Run(testCase.name, func(t *testing.T) {
+ unit := schema.ChunkDoc{Text: testCase.text, DocType: "text", Positions: json.RawMessage(`[[1,10,200,0,100]]`)}
+ pieces, sources := splitPDFUnits([]schema.ChunkDoc{unit}, pdfUnitSources([]schema.ChunkDoc{unit}), compileDelimPattern([]string{testCase.delimiter}), false)
+ if len(pieces) != 2 || removeTag(pieces[0].Text) != "a" || removeTag(pieces[1].Text) != "b" {
+ t.Fatalf("unexpected pieces: %v", generalChunkTexts(pieces))
+ }
+ for index, want := range testCase.bounds {
+ _, raw := pdfSourcePositions(sources[index])
+ positions := decodeGeneralMatrix(t, raw)
+ if len(positions) != 1 || positions[0][3] != want[0] || positions[0][4] != want[1] {
+ t.Errorf("piece %d positions = %v, want bounds %v", index, positions, want)
+ }
+ }
+ })
+ }
+}
+
+func TestGeneralPDFWrappedChildExcludesPreviousParagraph(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512, "delimiters": []string{"#"}, "children_delimiters": []string{"#"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ items := []map[string]any{
+ {"text": "上一段很长,不能算进下一段。", "doc_type_kwd": "text", "_pdf_positions": [][]float64{{1, 10, 200, 10, 40}}},
+ {"text": "#", "doc_type_kwd": "text", "_pdf_positions": [][]float64{{1, 10, 20, 45, 55}}},
+ {"text": "将军向宠,性行淑均,\n晓畅军事。\n愚以为营中之事。", "doc_type_kwd": "text", "_pdf_positions": [][]float64{{1, 10, 200, 60, 100}}},
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{"name": "source.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want 2", len(chunks))
+ }
+ raw, err := json.Marshal(chunks[1]["_pdf_positions"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got [][]float64
+ if err := json.Unmarshal(raw, &got); err != nil {
+ t.Fatal(err)
+ }
+ want := [][]float64{{1, 10, 200, 60, 100}}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("wrapped child positions = %v, want %v", got, want)
+ }
+}