Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): preserve PDF sentences across page boundaries (#21982)

jay77721aed68a269758
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 0caa64aa55..d871355d56 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -260,6 +260,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
}
joinSep := "\n"
if !customDelimiter {
+ body, bodySources = joinPDFContinuations(body, bodySources)
body = separateGeneralTextBlocks(body)
body, bodySources = splitPDFUnits(body, bodySources, primaryPattern, true)
joinSep = ""
@@ -268,7 +269,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
chunks = append(chunks, merged...)
chunkSources = append(chunkSources, mergedSources...)
body = body[:0]
- bodySources = bodySources[:0]
+ bodySources = nil
}
for index, unit := range units {
if itemDocType(unit) == "text" {
diff --git a/internal/ingestion/component/chunker/general_pdf_continuation.go b/internal/ingestion/component/chunker/general_pdf_continuation.go
new file mode 100644
index 0000000000..33f1044335
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_continuation.go
@@ -0,0 +1,139 @@
+package chunker
+
+import (
+ "math"
+ "regexp"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
+)
+
+var pdfListStart = regexp.MustCompile(`^(?:[((]?[0-9一二三四五六七八九十百]+[))..、]|[a-zA-Z]\)|[a-zA-Z]\.(?:\s|$)|[-*•]\s)`)
+
+// joinPDFContinuations removes physical page breaks within continuing prose
+// before delimiter splitting turns source regions into independent merge units.
+// Each source keeps its own text range and page rectangle for later splitting.
+func joinPDFContinuations(units []schema.ChunkDoc, sources [][]pdfSourceSpan) ([]schema.ChunkDoc, [][]pdfSourceSpan) {
+ first := 1
+ for first < len(units) && !canJoinPDFContinuation(units[first-1], units[first]) {
+ first++
+ }
+ if first >= len(units) {
+ return units, sources
+ }
+ if len(sources) != len(units) {
+ sources = pdfUnitSources(units)
+ }
+ joined := append([]schema.ChunkDoc(nil), units[:first]...)
+ joinedSources := append([][]pdfSourceSpan(nil), sources[:first]...)
+ for i := first; i < len(units); i++ {
+ unit := units[i]
+ if !canJoinPDFContinuation(joined[len(joined)-1], unit) {
+ joined = append(joined, unit)
+ joinedSources = append(joinedSources, sources[i])
+ continue
+ }
+ index := len(joined) - 1
+ joined[index] = cloneChunkDoc(joined[index])
+ previous := &joined[index]
+ left := removeTag(normalizeGeneralNewlines(previous.Text))
+ right := removeTag(normalizeGeneralNewlines(unit.Text))
+ previous.Text = strings.TrimRightFunc(left, unicode.IsSpace)
+ unit.Text = strings.TrimLeftFunc(right, unicode.IsSpace)
+ leftSources := slicePDFSourceSpans(joinedSources[index], left, 0, len(previous.Text))
+ rightSources := slicePDFSourceSpans(sources[i], right, len(right)-len(unit.Text), len(right))
+ separator := pdfContinuationSeparator(previous.Text, unit.Text)
+ offset := len(previous.Text) + len(separator)
+ for j := range rightSources {
+ rightSources[j].start += offset
+ rightSources[j].end += offset
+ }
+ joinedSources[index] = append(leftSources, rightSources...)
+ previous.Text += separator + unit.Text
+ previous.PDFPositions = mergeGeneralPositions(previous.PDFPositions, unit.PDFPositions)
+ previous.Positions = mergeGeneralPositions(previous.Positions, unit.Positions)
+ mergeGeneralMetadata(previous, unit)
+ previous.TKNums = nil // Delimiter splitting counts the resulting units.
+ }
+ return joined, joinedSources
+}
+
+func canJoinPDFContinuation(previous, next schema.ChunkDoc) bool {
+ for _, unit := range []schema.ChunkDoc{previous, next} {
+ if itemDocType(unit) != "text" || unit.PreserveOrder {
+ return false
+ }
+ for _, layout := range []string{unit.Layout, unit.LayoutType} {
+ if layout != "" && layout != "text" {
+ return false
+ }
+ }
+ }
+ left := normalizeGeneralNewlines(removeTag(previous.Text))
+ right := normalizeGeneralNewlines(removeTag(next.Text))
+ // A blank line is an explicit paragraph break, including whitespace-only lines.
+ leftEnd := len(strings.TrimRightFunc(left, unicode.IsSpace))
+ rightStart := len(right) - len(strings.TrimLeftFunc(right, unicode.IsSpace))
+ if strings.Count(left[leftEnd:], "\n") > 1 || strings.Count(right[:rightStart], "\n") > 1 {
+ return false
+ }
+ left, right = strings.TrimSpace(left), strings.TrimSpace(right)
+ if left == "" || right == "" || pdfListStart.MatchString(right) {
+ return false
+ }
+ ending := strings.TrimRight(left, `"'”’))]】》`)
+ if ending == "" || strings.ContainsRune("。!?.!?;;::", lastPDFRune(ending)) {
+ return false
+ }
+ first, _ := utf8.DecodeRuneInString(right)
+ if unicode.IsUpper(first) || (!unicode.IsLetter(first) && !strings.ContainsRune(",、,", first)) {
+ return false
+ }
+ positions := func(unit schema.ChunkDoc) [][]float64 {
+ if len(unit.PDFPositions) > 0 {
+ return parser.PDFPositionRowsFromJSON(unit.PDFPositions)
+ }
+ return parser.PDFPositionRowsFromJSON(unit.Positions)
+ }
+ before, after := positions(previous), positions(next)
+ if len(before) == 0 || len(after) == 0 {
+ return false
+ }
+ a, b := before[len(before)-1], after[0]
+ // A continuation starts higher on the next page than the previous page's
+ // last region ended; a region at or below that point is unrelated.
+ if b[0] != a[0]+1 || b[3] >= a[4] {
+ return false
+ }
+ height := math.Min(a[4]-a[3], b[4]-b[3])
+ // Regions may cover whole paragraphs: their height must not make a
+ // first-line indent look like harmless alignment noise (PDF-point units).
+ tolerance := math.Min(height*0.6, 12)
+ return height > 0 && a[2] > a[1] && b[2] > b[1] && math.Abs(a[1]-b[1]) <= tolerance
+}
+
+func lastPDFRune(text string) rune {
+ r, _ := utf8.DecodeLastRuneInString(text)
+ return r
+}
+
+func pdfContinuationSeparator(left, right string) string {
+ a := lastPDFRune(left)
+ b, _ := utf8.DecodeRuneInString(right)
+ if pdfNoSpaceRune(a) && pdfNoSpaceRune(b) || strings.ContainsRune("-\u2010\u2011", a) || unicode.IsPunct(b) {
+ return ""
+ }
+ return " "
+}
+
+// pdfNoSpaceRune reports scripts that write words without separating spaces.
+// A boundary keeps the space when only one side is in this set because the
+// other side is space-delimited. The prolonged sound mark is Common script, so
+// it is listed explicitly; Hangul is excluded because Korean separates words
+// with spaces.
+func pdfNoSpaceRune(r rune) bool {
+ return r == '\u30fc' || unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana)
+}
diff --git a/internal/ingestion/component/chunker/general_pdf_continuation_test.go b/internal/ingestion/component/chunker/general_pdf_continuation_test.go
new file mode 100644
index 0000000000..a03830e761
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_continuation_test.go
@@ -0,0 +1,291 @@
+package chunker
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func TestGeneralPDFJoinsCrossPageSentenceBeforeTokenMerge(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ delimiters []string
+ }{
+ {"newline", []string{"\n"}}, {"punctuation", []string{"。"}}, {"none", nil},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ first := "为确保工作有序推进,领导小组下设综合工作组、物资保障"
+ items := []map[string]any{
+ {"text": first, "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 619, 718}}},
+ {"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 197, 115, 130}}},
+ }
+ // Force the budget boundary to fall between the two source regions.
+ out := invokePDFContinuation(t, items, tokenizeStr(first)-1, tc.delimiters, nil)
+ if got, want := outputTexts(t, out), []string{first + "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts = %q, want %q", got, want)
+ }
+ chunks := outputChunks(t, out)
+ if got, want := chunks[0]["positions"], [][]float64{{3, 80, 520, 619, 718}, {4, 80, 197, 115, 130}}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("positions = %#v, want %#v", got, want)
+ }
+ })
+ }
+}
+
+func TestGeneralPDFCrossPageSentenceSplitKeepsSourcePositions(t *testing.T) {
+ items := []map[string]any{
+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},
+ {"text": "组。下一段。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 200, 100, 160}}},
+ }
+ out := invokePDFContinuation(t, items, 512, []string{"。"}, []string{"。"})
+ if got, want := outputTexts(t, out), []string{"物资保障组。", "下一段。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts = %q, want %q", got, want)
+ }
+ chunks := outputChunks(t, out)
+ for i, want := range [][][]float64{
+ {{3, 80, 520, 700, 715}, {4, 80, 200, 100, 120}},
+ {{4, 80, 200, 120, 160}},
+ } {
+ if got := chunks[i]["positions"]; !reflect.DeepEqual(got, want) {
+ t.Errorf("chunk %d positions = %#v, want %#v", i, got, want)
+ }
+ }
+}
+
+func TestGeneralPDFCrossPageParentPositionsFollowSourceText(t *testing.T) {
+ items := []map[string]any{
+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},
+ {"text": "组。下一段。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 200, 100, 160}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"。"}, nil)
+ if got, want := outputTexts(t, out), []string{"物资保障组。", "下一段。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts = %q, want %q", got, want)
+ }
+ chunks := outputChunks(t, out)
+ if got, want := chunks[1]["positions"], [][]float64{{4, 80, 200, 120, 160}}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("positions = %#v, want %#v", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageEnglishKeepsWordSpace(t *testing.T) {
+ items := []map[string]any{
+ {"text": "work with the\r\n", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "team to finish.", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"work with the team to finish."}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageKanaJoinsWithoutSpace(t *testing.T) {
+ items := []map[string]any{
+ {"text": "これは", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "テストです。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"これはテストです。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageMixedScriptKeepsSpace(t *testing.T) {
+ items := []map[string]any{
+ {"text": "The word is", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "ひらがな", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"The word is ひらがな"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageHanLatinKeepsSpace(t *testing.T) {
+ items := []map[string]any{
+ {"text": "使用", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "python 进行开发。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"使用 python 进行开发。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageHanKanaJoinsWithoutSpace(t *testing.T) {
+ items := []map[string]any{
+ {"text": "日本語", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "の文章。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"日本語の文章。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageProlongedSoundMarkJoinsWithoutSpace(t *testing.T) {
+ items := []map[string]any{
+ {"text": "コンピュー", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "ター。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"コンピューター。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageAbbreviationIsNotListMarker(t *testing.T) {
+ items := []map[string]any{
+ {"text": "references,", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "e.g. this article.", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"references, e.g. this article."}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageRejectsRegionBelowPreviousBottom(t *testing.T) {
+ items := []map[string]any{
+ {"text": "负责物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 100, 115}}},
+ {"text": "下一段内容", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 520, 500, 515}}},
+ }
+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"负责物资保障\n下一段内容"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFContinuationRebuildsMismatchedSources(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "物资保障", DocType: "text", Positions: json.RawMessage(`[[3,80,520,700,715]]`)},
+ {Text: "组、宣传工作组。", DocType: "text", Positions: json.RawMessage(`[[4,80,200,100,115]]`)},
+ }
+ joined, sources := joinPDFContinuations(units, [][]pdfSourceSpan{})
+ if len(joined) != 1 {
+ t.Fatalf("joined units = %d, want 1", len(joined))
+ }
+ if len(sources) != 1 || len(sources[0]) == 0 {
+ t.Fatalf("sources = %#v, want one populated source list", sources)
+ }
+}
+
+func TestGeneralPDFContinuationAcrossThreePagesKeepsEachSource(t *testing.T) {
+ items := []map[string]any{
+ {"text": "为保障工作", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "有序推进,设立物资保障", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 520, 100, 700}}},
+ {"text": "组。", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"为保障工作有序推进,设立物资保障组。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+ chunks := outputChunks(t, out)
+ if got, want := chunks[0]["positions"], [][]float64{{1, 80, 520, 700, 715}, {2, 80, 520, 100, 700}, {3, 80, 200, 100, 115}}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("positions=%#v, want %#v", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageContinuationHonorsCustomDelimiter(t *testing.T) {
+ items := []map[string]any{
+ {"text": "物资保障SEP", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 1, []string{"`SEP`"}, nil)
+ if got, want := outputTexts(t, out), []string{"物资保障", "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageContinuationDoesNotCrossMedia(t *testing.T) {
+ for _, kind := range []string{"image", "table"} {
+ t.Run(kind, func(t *testing.T) {
+ items := []map[string]any{
+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "插图或表格", "doc_type_kwd": kind, "positions": [][]float64{{1, 80, 520, 720, 760}}},
+ {"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ }
+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"物资保障", "插图或表格", "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+ })
+ }
+}
+
+func TestGeneralPDFCrossPageSourcesResetAfterMedia(t *testing.T) {
+ items := []map[string]any{
+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},
+ {"text": "组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},
+ {"text": "插图", "doc_type_kwd": "image", "positions": [][]float64{{2, 80, 520, 200, 300}}},
+ {"text": "后文。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 520, 400, 415}}},
+ }
+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"物资保障组。", "插图", "后文。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageTallParagraphKeepsIndentation(t *testing.T) {
+ items := []map[string]any{
+ {"text": "上一段没有结束标点", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 600, 720}}},
+ {"text": "新的缩进段落。", "doc_type_kwd": "text", "positions": [][]float64{{2, 112, 520, 100, 250}}},
+ }
+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+ if got, want := outputTexts(t, out), []string{"上一段没有结束标点\n新的缩进段落。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+}
+
+func TestGeneralPDFCrossPageContinuationKeepsParagraphBoundaries(t *testing.T) {
+ for _, tc := range []struct {
+ name, first, second, layout string
+ page, left float64
+ }{
+ {"sentence ended", "第一段。", "下一段。", "", 4, 80},
+ {"numbered item", "负责物资保障", "(二)宣传工作组", "", 4, 80},
+ {"title", "负责物资保障", "管理办法", "title", 4, 80},
+ {"new indentation", "负责物资保障", "下一段内容", "", 4, 112},
+ {"same page", "负责物资保障", "下一段内容", "", 3, 80},
+ {"missing page", "负责物资保障", "下一段内容", "", 5, 80},
+ {"blank line", "负责物资保障\n\n", "下一段内容", "", 4, 80},
+ {"spaced blank line", "负责物资保障\n \t\n", "下一段内容", "", 4, 80},
+ {"lettered item", "负责物资保障", "b) 宣传工作组", "", 4, 80},
+ {"lettered item no space", "负责物资保障", "b)宣传工作组", "", 4, 80},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ top := 100.0
+ if tc.page == 3 {
+ top = 740
+ }
+ items := []map[string]any{
+ {"text": tc.first, "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},
+ {"text": tc.second, "doc_type_kwd": "text", "layout_type": tc.layout, "positions": [][]float64{{tc.page, tc.left, 520, top, top + 15}}},
+ }
+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)
+ blankLine := tc.name == "blank line" || tc.name == "spaced blank line"
+ if got, want := outputTexts(t, out), []string{tc.first + "\n" + tc.second}; !blankLine && !reflect.DeepEqual(got, want) {
+ t.Fatalf("texts=%q, want %q", got, want)
+ }
+ if blankLine {
+ if got := outputTexts(t, out); len(got) != 1 || got[0] != tc.first+tc.second {
+ t.Fatalf("lost paragraph boundary: %q", got)
+ }
+ }
+ })
+ }
+}
+
+func invokePDFContinuation(t *testing.T, items []map[string]any, budget int, delimiters, children []string) map[string]any {
+ t.Helper()
+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": budget, "delimiters": delimiters, "children_delimiters": children})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "continuation.pdf", "file_type": "pdf", "output_format": "json", "json": items})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return out
+}
diff --git a/internal/ingestion/component/chunker/general_pdf_source_positions.go b/internal/ingestion/component/chunker/general_pdf_source_positions.go
index d6f66a0f8c..da3ca21956 100644
--- a/internal/ingestion/component/chunker/general_pdf_source_positions.go
+++ b/internal/ingestion/component/chunker/general_pdf_source_positions.go
@@ -61,6 +61,9 @@ func splitPDFUnits(units []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *
cursor = end
}
}
+ if pdfPositions, positions := pdfSourcePositions(spans); len(pdfPositions) > 0 || len(positions) > 0 {
+ piece.PDFPositions, piece.Positions = pdfPositions, positions
+ }
result = append(result, piece)
resultSources = append(resultSources, spans)
}