RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): preserve PDF sentences across page boundaries (#21982)
jay77721aed68a269758
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 0caa64aa55..d871355d56 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -260,6 +260,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups}joinSep := "\n"if !customDelimiter {+ body, bodySources = joinPDFContinuations(body, bodySources)body = separateGeneralTextBlocks(body)body, bodySources = splitPDFUnits(body, bodySources, primaryPattern, true)joinSep = ""@@ -268,7 +269,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upschunks = append(chunks, merged...)chunkSources = append(chunkSources, mergedSources...)body = body[:0]- bodySources = bodySources[:0]+ bodySources = nil}for index, unit := range units {if itemDocType(unit) == "text" {diff --git a/internal/ingestion/component/chunker/general_pdf_continuation.go b/internal/ingestion/component/chunker/general_pdf_continuation.gonew file mode 100644index 0000000000..33f1044335--- /dev/null+++ b/internal/ingestion/component/chunker/general_pdf_continuation.go@@ -0,0 +1,139 @@+package chunker++import (+ "math"+ "regexp"+ "strings"+ "unicode"+ "unicode/utf8"++ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser"+)++var pdfListStart = regexp.MustCompile(`^(?:[((]?[0-9一二三四五六七八九十百]+[))..、]|[a-zA-Z]\)|[a-zA-Z]\.(?:\s|$)|[-*•]\s)`)++// joinPDFContinuations removes physical page breaks within continuing prose+// before delimiter splitting turns source regions into independent merge units.+// Each source keeps its own text range and page rectangle for later splitting.+func joinPDFContinuations(units []schema.ChunkDoc, sources [][]pdfSourceSpan) ([]schema.ChunkDoc, [][]pdfSourceSpan) {+ first := 1+ for first < len(units) && !canJoinPDFContinuation(units[first-1], units[first]) {+ first+++ }+ if first >= len(units) {+ return units, sources+ }+ if len(sources) != len(units) {+ sources = pdfUnitSources(units)+ }+ joined := append([]schema.ChunkDoc(nil), units[:first]...)+ joinedSources := append([][]pdfSourceSpan(nil), sources[:first]...)+ for i := first; i < len(units); i++ {+ unit := units[i]+ if !canJoinPDFContinuation(joined[len(joined)-1], unit) {+ joined = append(joined, unit)+ joinedSources = append(joinedSources, sources[i])+ continue+ }+ index := len(joined) - 1+ joined[index] = cloneChunkDoc(joined[index])+ previous := &joined[index]+ left := removeTag(normalizeGeneralNewlines(previous.Text))+ right := removeTag(normalizeGeneralNewlines(unit.Text))+ previous.Text = strings.TrimRightFunc(left, unicode.IsSpace)+ unit.Text = strings.TrimLeftFunc(right, unicode.IsSpace)+ leftSources := slicePDFSourceSpans(joinedSources[index], left, 0, len(previous.Text))+ rightSources := slicePDFSourceSpans(sources[i], right, len(right)-len(unit.Text), len(right))+ separator := pdfContinuationSeparator(previous.Text, unit.Text)+ offset := len(previous.Text) + len(separator)+ for j := range rightSources {+ rightSources[j].start += offset+ rightSources[j].end += offset+ }+ joinedSources[index] = append(leftSources, rightSources...)+ previous.Text += separator + unit.Text+ previous.PDFPositions = mergeGeneralPositions(previous.PDFPositions, unit.PDFPositions)+ previous.Positions = mergeGeneralPositions(previous.Positions, unit.Positions)+ mergeGeneralMetadata(previous, unit)+ previous.TKNums = nil // Delimiter splitting counts the resulting units.+ }+ return joined, joinedSources+}++func canJoinPDFContinuation(previous, next schema.ChunkDoc) bool {+ for _, unit := range []schema.ChunkDoc{previous, next} {+ if itemDocType(unit) != "text" || unit.PreserveOrder {+ return false+ }+ for _, layout := range []string{unit.Layout, unit.LayoutType} {+ if layout != "" && layout != "text" {+ return false+ }+ }+ }+ left := normalizeGeneralNewlines(removeTag(previous.Text))+ right := normalizeGeneralNewlines(removeTag(next.Text))+ // A blank line is an explicit paragraph break, including whitespace-only lines.+ leftEnd := len(strings.TrimRightFunc(left, unicode.IsSpace))+ rightStart := len(right) - len(strings.TrimLeftFunc(right, unicode.IsSpace))+ if strings.Count(left[leftEnd:], "\n") > 1 || strings.Count(right[:rightStart], "\n") > 1 {+ return false+ }+ left, right = strings.TrimSpace(left), strings.TrimSpace(right)+ if left == "" || right == "" || pdfListStart.MatchString(right) {+ return false+ }+ ending := strings.TrimRight(left, `"'”’))]】》`)+ if ending == "" || strings.ContainsRune("。!?.!?;;::", lastPDFRune(ending)) {+ return false+ }+ first, _ := utf8.DecodeRuneInString(right)+ if unicode.IsUpper(first) || (!unicode.IsLetter(first) && !strings.ContainsRune(",、,", first)) {+ return false+ }+ positions := func(unit schema.ChunkDoc) [][]float64 {+ if len(unit.PDFPositions) > 0 {+ return parser.PDFPositionRowsFromJSON(unit.PDFPositions)+ }+ return parser.PDFPositionRowsFromJSON(unit.Positions)+ }+ before, after := positions(previous), positions(next)+ if len(before) == 0 || len(after) == 0 {+ return false+ }+ a, b := before[len(before)-1], after[0]+ // A continuation starts higher on the next page than the previous page's+ // last region ended; a region at or below that point is unrelated.+ if b[0] != a[0]+1 || b[3] >= a[4] {+ return false+ }+ height := math.Min(a[4]-a[3], b[4]-b[3])+ // Regions may cover whole paragraphs: their height must not make a+ // first-line indent look like harmless alignment noise (PDF-point units).+ tolerance := math.Min(height*0.6, 12)+ return height > 0 && a[2] > a[1] && b[2] > b[1] && math.Abs(a[1]-b[1]) <= tolerance+}++func lastPDFRune(text string) rune {+ r, _ := utf8.DecodeLastRuneInString(text)+ return r+}++func pdfContinuationSeparator(left, right string) string {+ a := lastPDFRune(left)+ b, _ := utf8.DecodeRuneInString(right)+ if pdfNoSpaceRune(a) && pdfNoSpaceRune(b) || strings.ContainsRune("-\u2010\u2011", a) || unicode.IsPunct(b) {+ return ""+ }+ return " "+}++// pdfNoSpaceRune reports scripts that write words without separating spaces.+// A boundary keeps the space when only one side is in this set because the+// other side is space-delimited. The prolonged sound mark is Common script, so+// it is listed explicitly; Hangul is excluded because Korean separates words+// with spaces.+func pdfNoSpaceRune(r rune) bool {+ return r == '\u30fc' || unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana)+}diff --git a/internal/ingestion/component/chunker/general_pdf_continuation_test.go b/internal/ingestion/component/chunker/general_pdf_continuation_test.gonew file mode 100644index 0000000000..a03830e761--- /dev/null+++ b/internal/ingestion/component/chunker/general_pdf_continuation_test.go@@ -0,0 +1,291 @@+package chunker++import (+ "encoding/json"+ "reflect"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++func TestGeneralPDFJoinsCrossPageSentenceBeforeTokenMerge(t *testing.T) {+ for _, tc := range []struct {+ name string+ delimiters []string+ }{+ {"newline", []string{"\n"}}, {"punctuation", []string{"。"}}, {"none", nil},+ } {+ t.Run(tc.name, func(t *testing.T) {+ first := "为确保工作有序推进,领导小组下设综合工作组、物资保障"+ items := []map[string]any{+ {"text": first, "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 619, 718}}},+ {"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 197, 115, 130}}},+ }+ // Force the budget boundary to fall between the two source regions.+ out := invokePDFContinuation(t, items, tokenizeStr(first)-1, tc.delimiters, nil)+ if got, want := outputTexts(t, out), []string{first + "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts = %q, want %q", got, want)+ }+ chunks := outputChunks(t, out)+ if got, want := chunks[0]["positions"], [][]float64{{3, 80, 520, 619, 718}, {4, 80, 197, 115, 130}}; !reflect.DeepEqual(got, want) {+ t.Fatalf("positions = %#v, want %#v", got, want)+ }+ })+ }+}++func TestGeneralPDFCrossPageSentenceSplitKeepsSourcePositions(t *testing.T) {+ items := []map[string]any{+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},+ {"text": "组。下一段。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 200, 100, 160}}},+ }+ out := invokePDFContinuation(t, items, 512, []string{"。"}, []string{"。"})+ if got, want := outputTexts(t, out), []string{"物资保障组。", "下一段。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts = %q, want %q", got, want)+ }+ chunks := outputChunks(t, out)+ for i, want := range [][][]float64{+ {{3, 80, 520, 700, 715}, {4, 80, 200, 100, 120}},+ {{4, 80, 200, 120, 160}},+ } {+ if got := chunks[i]["positions"]; !reflect.DeepEqual(got, want) {+ t.Errorf("chunk %d positions = %#v, want %#v", i, got, want)+ }+ }+}++func TestGeneralPDFCrossPageParentPositionsFollowSourceText(t *testing.T) {+ items := []map[string]any{+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},+ {"text": "组。下一段。", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 200, 100, 160}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"。"}, nil)+ if got, want := outputTexts(t, out), []string{"物资保障组。", "下一段。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts = %q, want %q", got, want)+ }+ chunks := outputChunks(t, out)+ if got, want := chunks[1]["positions"], [][]float64{{4, 80, 200, 120, 160}}; !reflect.DeepEqual(got, want) {+ t.Fatalf("positions = %#v, want %#v", got, want)+ }+}++func TestGeneralPDFCrossPageEnglishKeepsWordSpace(t *testing.T) {+ items := []map[string]any{+ {"text": "work with the\r\n", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "team to finish.", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"work with the team to finish."}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageKanaJoinsWithoutSpace(t *testing.T) {+ items := []map[string]any{+ {"text": "これは", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "テストです。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"これはテストです。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageMixedScriptKeepsSpace(t *testing.T) {+ items := []map[string]any{+ {"text": "The word is", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "ひらがな", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"The word is ひらがな"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageHanLatinKeepsSpace(t *testing.T) {+ items := []map[string]any{+ {"text": "使用", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "python 进行开发。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"使用 python 进行开发。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageHanKanaJoinsWithoutSpace(t *testing.T) {+ items := []map[string]any{+ {"text": "日本語", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "の文章。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"日本語の文章。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageProlongedSoundMarkJoinsWithoutSpace(t *testing.T) {+ items := []map[string]any{+ {"text": "コンピュー", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "ター。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"コンピューター。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageAbbreviationIsNotListMarker(t *testing.T) {+ items := []map[string]any{+ {"text": "references,", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "e.g. this article.", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"references, e.g. this article."}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageRejectsRegionBelowPreviousBottom(t *testing.T) {+ items := []map[string]any{+ {"text": "负责物资保障", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 100, 115}}},+ {"text": "下一段内容", "doc_type_kwd": "text", "positions": [][]float64{{4, 80, 520, 500, 515}}},+ }+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"负责物资保障\n下一段内容"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFContinuationRebuildsMismatchedSources(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "物资保障", DocType: "text", Positions: json.RawMessage(`[[3,80,520,700,715]]`)},+ {Text: "组、宣传工作组。", DocType: "text", Positions: json.RawMessage(`[[4,80,200,100,115]]`)},+ }+ joined, sources := joinPDFContinuations(units, [][]pdfSourceSpan{})+ if len(joined) != 1 {+ t.Fatalf("joined units = %d, want 1", len(joined))+ }+ if len(sources) != 1 || len(sources[0]) == 0 {+ t.Fatalf("sources = %#v, want one populated source list", sources)+ }+}++func TestGeneralPDFContinuationAcrossThreePagesKeepsEachSource(t *testing.T) {+ items := []map[string]any{+ {"text": "为保障工作", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "有序推进,设立物资保障", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 520, 100, 700}}},+ {"text": "组。", "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"为保障工作有序推进,设立物资保障组。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+ chunks := outputChunks(t, out)+ if got, want := chunks[0]["positions"], [][]float64{{1, 80, 520, 700, 715}, {2, 80, 520, 100, 700}, {3, 80, 200, 100, 115}}; !reflect.DeepEqual(got, want) {+ t.Fatalf("positions=%#v, want %#v", got, want)+ }+}++func TestGeneralPDFCrossPageContinuationHonorsCustomDelimiter(t *testing.T) {+ items := []map[string]any{+ {"text": "物资保障SEP", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 1, []string{"`SEP`"}, nil)+ if got, want := outputTexts(t, out), []string{"物资保障", "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageContinuationDoesNotCrossMedia(t *testing.T) {+ for _, kind := range []string{"image", "table"} {+ t.Run(kind, func(t *testing.T) {+ items := []map[string]any{+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "插图或表格", "doc_type_kwd": kind, "positions": [][]float64{{1, 80, 520, 720, 760}}},+ {"text": "组、宣传工作组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ }+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"物资保障", "插图或表格", "组、宣传工作组。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+ })+ }+}++func TestGeneralPDFCrossPageSourcesResetAfterMedia(t *testing.T) {+ items := []map[string]any{+ {"text": "物资保障", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 700, 715}}},+ {"text": "组。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 200, 100, 115}}},+ {"text": "插图", "doc_type_kwd": "image", "positions": [][]float64{{2, 80, 520, 200, 300}}},+ {"text": "后文。", "doc_type_kwd": "text", "positions": [][]float64{{2, 80, 520, 400, 415}}},+ }+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"物资保障组。", "插图", "后文。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageTallParagraphKeepsIndentation(t *testing.T) {+ items := []map[string]any{+ {"text": "上一段没有结束标点", "doc_type_kwd": "text", "positions": [][]float64{{1, 80, 520, 600, 720}}},+ {"text": "新的缩进段落。", "doc_type_kwd": "text", "positions": [][]float64{{2, 112, 520, 100, 250}}},+ }+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)+ if got, want := outputTexts(t, out), []string{"上一段没有结束标点\n新的缩进段落。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+}++func TestGeneralPDFCrossPageContinuationKeepsParagraphBoundaries(t *testing.T) {+ for _, tc := range []struct {+ name, first, second, layout string+ page, left float64+ }{+ {"sentence ended", "第一段。", "下一段。", "", 4, 80},+ {"numbered item", "负责物资保障", "(二)宣传工作组", "", 4, 80},+ {"title", "负责物资保障", "管理办法", "title", 4, 80},+ {"new indentation", "负责物资保障", "下一段内容", "", 4, 112},+ {"same page", "负责物资保障", "下一段内容", "", 3, 80},+ {"missing page", "负责物资保障", "下一段内容", "", 5, 80},+ {"blank line", "负责物资保障\n\n", "下一段内容", "", 4, 80},+ {"spaced blank line", "负责物资保障\n \t\n", "下一段内容", "", 4, 80},+ {"lettered item", "负责物资保障", "b) 宣传工作组", "", 4, 80},+ {"lettered item no space", "负责物资保障", "b)宣传工作组", "", 4, 80},+ } {+ t.Run(tc.name, func(t *testing.T) {+ top := 100.0+ if tc.page == 3 {+ top = 740+ }+ items := []map[string]any{+ {"text": tc.first, "doc_type_kwd": "text", "positions": [][]float64{{3, 80, 520, 700, 715}}},+ {"text": tc.second, "doc_type_kwd": "text", "layout_type": tc.layout, "positions": [][]float64{{tc.page, tc.left, 520, top, top + 15}}},+ }+ out := invokePDFContinuation(t, items, 512, []string{"\n"}, nil)+ blankLine := tc.name == "blank line" || tc.name == "spaced blank line"+ if got, want := outputTexts(t, out), []string{tc.first + "\n" + tc.second}; !blankLine && !reflect.DeepEqual(got, want) {+ t.Fatalf("texts=%q, want %q", got, want)+ }+ if blankLine {+ if got := outputTexts(t, out); len(got) != 1 || got[0] != tc.first+tc.second {+ t.Fatalf("lost paragraph boundary: %q", got)+ }+ }+ })+ }+}++func invokePDFContinuation(t *testing.T, items []map[string]any, budget int, delimiters, children []string) map[string]any {+ t.Helper()+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": budget, "delimiters": delimiters, "children_delimiters": children})+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "continuation.pdf", "file_type": "pdf", "output_format": "json", "json": items})+ if err != nil {+ t.Fatal(err)+ }+ return out+}diff --git a/internal/ingestion/component/chunker/general_pdf_source_positions.go b/internal/ingestion/component/chunker/general_pdf_source_positions.goindex d6f66a0f8c..da3ca21956 100644--- a/internal/ingestion/component/chunker/general_pdf_source_positions.go+++ b/internal/ingestion/component/chunker/general_pdf_source_positions.go@@ -61,6 +61,9 @@ func splitPDFUnits(units []schema.ChunkDoc, sources [][]pdfSourceSpan, pattern *cursor = end}}+ if pdfPositions, positions := pdfSourcePositions(spans); len(pdfPositions) > 0 || len(positions) > 0 {+ piece.PDFPositions, piece.Positions = pdfPositions, positions+ }result = append(result, piece)resultSources = append(resultSources, spans)}