Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): include the image/table context window in the chunk text (#19678)

jay777215dda6fd999df
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index cbdfe9fae0..049e6c3b67 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -208,11 +208,43 @@ func emptyOutputs() map[string]any {
// model resolution) is NOT re-emitted here — it lives in the workflow-wide
// CanvasState.Globals bag (seeded at pipeline start, published by the File
// component) and read directly from ctx. See runtime.CanvasState.Globals.
+//
+// Media context is materialized into the chunk body here, the chunker's last
+// step: every variant passes through this builder, so the folded text is what
+// the chunk id, the extractor, the tokenizer and the index write all see.
func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {
+ materialized := make([]schema.ChunkDoc, len(chunks))
+ for i := range chunks {
+ materialized[i] = materializeMediaContext(chunks[i])
+ }
return map[string]any{
"output_format": "chunks",
- "chunks": schema.ChunkDocsToMaps(chunks),
+ "chunks": schema.ChunkDocsToMaps(materialized),
+ }
+}
+
+// materializeMediaContext folds a media chunk's surrounding context into its
+// body and clears the two fields that carried it. Python's chunker emits the
+// same shape — its finalize builds remove_tag(context_above + text +
+// context_below) and drops the fields (rag/flow/chunker/token_chunker.py:343-
+// 359) — which is why Python persists the context inside the chunk body.
+// Folding here also puts the context into the chunk id (ChunkID hashes the
+// body), matching Python's id, which hashes the context-bearing body.
+//
+// Tag stripping runs after the merge, as in Python: the payload was already
+// stripped by the chunker, so this only covers the context, which is collected
+// from neighbouring units.
+//
+// Only media chunks carry context (attachMediaContext and
+// attachGeneralMediaContext write it), so text chunks pass through untouched.
+func materializeMediaContext(ck schema.ChunkDoc) schema.ChunkDoc {
+ if ck.ContextAbove == "" && ck.ContextBelow == "" {
+ return ck
}
+ ck.Text = removeTag(schema.ContextualText(ck))
+ ck.ContextAbove = ""
+ ck.ContextBelow = ""
+ return ck
}
// withName returns a shallow copy of inputs with name set, so a component can
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index ee3938ef49..e6898490f1 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -28,9 +28,12 @@ import (
"reflect"
"strings"
"testing"
+ "time"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/ingestion/task/indexdoc"
)
func TestGeneralChunkerRegistered(t *testing.T) {
@@ -516,6 +519,23 @@ func TestGeneralChunkerDOCXMediaDoesNotBreakTextMerge(t *testing.T) {
}
}
+// assertMaterializedMediaContext pins the chunker output contract for media
+// chunks: the surrounding context is folded into the chunk body and the
+// retrieval-only context fields are gone — the shape Python's chunker emits
+// (rag/flow/chunker/token_chunker.py:343-359) and the shape the persisted
+// content_with_weight and the chunk id are built from.
+func assertMaterializedMediaContext(t *testing.T, chunk map[string]any, wantText string) {
+ t.Helper()
+ if got := chunk["text"]; got != wantText {
+ t.Errorf("media chunk text = %q, want %q", got, wantText)
+ }
+ for _, key := range []string{"context_above", "context_below"} {
+ if _, exists := chunk[key]; exists {
+ t.Errorf("media chunk must not carry %s after materialization: %+v", key, chunk)
+ }
+ }
+}
+
func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {
component, err := NewGeneralChunker(map[string]any{
"chunk_token_size": 10,
@@ -544,9 +564,7 @@ func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {
if chunks[1]["doc_type_kwd"] != "table" {
t.Fatalf("table chunk = %+v", chunks[1])
}
- if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {
- t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
- }
+ assertMaterializedMediaContext(t, chunks[1], "before<table><tr><td>A</td></tr></table>after")
}
func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {
@@ -701,9 +719,7 @@ func TestGeneralChunkerPDFUsesPositionOrderForMediaContext(t *testing.T) {
if len(chunks) != 3 {
t.Fatalf("chunks = %#v, want text, media, text", chunks)
}
- if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {
- t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
- }
+ assertMaterializedMediaContext(t, chunks[1], "before<table>A</table>after")
if chunks[0]["text"] != "before" || chunks[2]["text"] != "after" {
t.Errorf("position-ordered text = [%v, %v]", chunks[0]["text"], chunks[2]["text"])
}
@@ -783,9 +799,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
if chunks[1]["ck_type"] != "image" {
t.Fatalf("image chunk = %#v", chunks[1])
}
- if chunks[1]["context_above"] != "Revenue" || chunks[1]["context_below"] != "Growth" {
- t.Fatalf("image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
- }
+ assertMaterializedMediaContext(t, chunks[1], "RevenueB2Growth")
}
func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T) {
@@ -810,9 +824,95 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)
if len(chunks) != 3 {
t.Fatalf("chunks = %#v, want row, image, row", chunks)
}
- if chunks[1]["context_above"] != "Sheet one" || chunks[1]["context_below"] != nil {
- t.Fatalf("cross-sheet image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
+ assertMaterializedMediaContext(t, chunks[1], "Sheet oneB2")
+}
+
+// TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent pins the
+// downstream half of the fold: once the context is part of the chunk body, it
+// is also part of the persisted content_with_weight and of the chunk id
+// (ChunkID hashes the body). Python's chunker emits the merged body, so its
+// id and stored content already carry the context; before the fold Go hashed
+// and stored the bare media payload instead, which made the configured window
+// invisible to both the chunk list and the index.
+func TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 2,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := indexdoc.NormalizeChunks(out)
+ if _, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "document.docx", time.Now()); err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+
+ const wantText = "before<table><tr><td>A</td></tr></table>after"
+ var table map[string]any
+ for _, ck := range chunks {
+ if ck["ck_type"] == "table" {
+ table = ck
+ break
+ }
+ }
+ if table == nil {
+ t.Fatalf("no table chunk in %#v", chunks)
+ }
+ if got := table["content_with_weight"]; got != wantText {
+ t.Errorf("persisted content_with_weight = %q, want %q", got, wantText)
+ }
+ if got, want := table["id"], common.ChunkID("doc-1", wantText); got != want {
+ t.Errorf("chunk id = %v, want %v (context must drive chunk identity)", got, want)
+ }
+}
+
+// TestGeneralChunkerMediaContextStripsPositionTags pins the tag-stripping
+// order Python uses: remove_tag runs on the merged body, so a position tag
+// carried by a neighbouring text unit never reaches the stored media chunk.
+// (The parser keeps boxes in the positions field, so this is a guard, not a
+// re-strip of the media payload.)
+func TestGeneralChunkerMediaContextStripsPositionTags(t *testing.T) {
+ position := func(top float64) []any {
+ return []any{[]any{1.0, 0.0, 10.0, top, top + 5}}
+ }
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "table_context_size": 20,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before@@1\t0.0\t10.0\t10.0\t20.0##", "doc_type_kwd": "text", "positions": position(10)},
+ {"text": "<table>A</table>", "doc_type_kwd": "table", "positions": position(20)},
+ {"text": "after@@1\t0.0\t10.0\t10.0\t30.0\t35.0##", "doc_type_kwd": "text", "positions": position(30)},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 3 {
+ t.Fatalf("chunks = %#v, want text, media, text", chunks)
}
+ assertMaterializedMediaContext(t, chunks[1], "before<table>A</table>after")
}
func TestGeneralChunkerPDFAttachesOutlineOnce(t *testing.T) {
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 91269b6af8..f0b2a30699 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -588,25 +588,36 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
}
}
- // Attach surrounding media context (token_chunker.py:358).
- attached := attachMediaContext(perItem, c.param.TableContextSize, c.param.ImageContextSize)
+ // Surrounding media context is attached on the flattened chunk list,
+ // mirroring Python's _attach_context_to_media_chunks (token_chunker.py:537,
+ // :545): the units are flat there, so a media block collects the text units
+ // around it across item boundaries. Attaching per upstream item instead
+ // leaves every media chunk without neighbours, because one item rarely
+ // yields more than one chunk.
+ flat := flatten(perItem)
// Python's naive_merge: custom (backtick) delimiters produce one
// chunk per segment — no token-size merge (naive_merge:1194-1213).
// Otherwise split-then-merge: delimiter-split segments are greedily
// merged to chunk_token_size with optional overlap.
- if !hasCustomDelim(c.param.Delimiters) {
+ customDelim := hasCustomDelim(c.param.Delimiters)
+ if !customDelim {
+ // Attach before the merge — Python's non-delimiter branch collects the
+ // context from the pre-merge text units. The merge only joins text
+ // chunks, so a media chunk keeps what it collected.
+ flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)
// Python _merge_text_chunks_by_token_size merges adjacent text
- // chunks across JSON items into one global token budget. Flatten the
- // per-item structure into a single sequence first so the merge is
- // global; non-text chunks still break the merge via their CKType.
- attached = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flatten(attached)}, c.param.ChunkTokenSize, c.param.OverlappedPercent)
+ // chunks across JSON items into one global token budget.
+ flat = flatten(mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flat}, c.param.ChunkTokenSize, c.param.OverlappedPercent))
}
- flat := flatten(attached)
if childrenPattern != nil {
flat = splitByChildren(flat, childrenPattern)
}
+ if customDelim {
+ // Python's delimiter branch splits by children first, then attaches.
+ flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)
+ }
// Crop image/table chunks on demand when a PDF engine is available.
flat = cropImageChunks(ctx, engine, flat)
@@ -618,7 +629,15 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// indexed/embedded chunk text. Crop above reads positions, not text,
// so the ordering is safe.
m.Text = removeTag(m.Text)
- if m.Text == "" {
+ // Drop a chunk only when nothing about it is retrievable. A media chunk
+ // may carry no body of its own and still be indexed through its
+ // surrounding context: Python's _finalize_json_chunks strips the merged
+ // text and applies the check to it — remove_tag(context_above + text +
+ // context_below) then .strip() — so a context that carries nothing but
+ // a position tag counts as empty. The context is folded into Text once,
+ // at chunkOutputs, so the decision is made on that merged text here
+ // without duplicating the fold.
+ if strings.TrimSpace(removeTag(schema.ContextualText(m))) == "" {
continue
}
out = append(out, m)
@@ -818,32 +837,30 @@ func partition(n, parts int) []lane {
return out
}
-func attachMediaContext(perItem [][]schema.ChunkDoc, tableCtx, imageCtx int) [][]schema.ChunkDoc {
+// attachMediaContext writes the surrounding context onto the media chunks of a
+// flat chunk list. Mirrors token_chunker.py:_attach_context_to_media_chunks,
+// which runs on the flat chunk list as well: a media block collects the text
+// units around it, up to the token budget, across upstream item boundaries.
+func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []schema.ChunkDoc {
if tableCtx <= 0 && imageCtx <= 0 {
- return perItem
+ return chunks
}
- for idx := range perItem {
- chunks := perItem[idx]
- if len(chunks) == 0 {
+ for i, ck := range chunks {
+ ckType := ck.CKType
+ if ckType != "table" && ckType != "image" {
continue
}
- for i, ck := range chunks {
- ckType := ck.CKType
- if ckType != "table" && ckType != "image" {
- continue
- }
- ctx := imageCtx
- if ckType == "table" {
- ctx = tableCtx
- }
- if ctx <= 0 {
- continue
- }
- chunks[i].ContextAbove = collectContext(chunks, i, ctx, true)
- chunks[i].ContextBelow = collectContext(chunks, i, ctx, false)
+ ctx := imageCtx
+ if ckType == "table" {
+ ctx = tableCtx
}
+ if ctx <= 0 {
+ continue
+ }
+ chunks[i].ContextAbove = collectContext(chunks, i, ctx, true)
+ chunks[i].ContextBelow = collectContext(chunks, i, ctx, false)
}
- return perItem
+ return chunks
}
// collectContext walks chunks around `i` (above when direction==true,
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index ed0053003c..42927b0414 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -425,6 +425,100 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {
}
}
+// TestTokenChunkerMediaContextSpansUpstreamItems pins the flat attach contract
+// on the canvas path: Python collects a media chunk's context from the flat
+// chunk list (token_chunker.py:537, :545), so the text units of *other*
+// upstream items are in scope. The context is also what keeps a caption-less
+// media chunk alive — its own body is empty — and what that chunk indexes.
+func TestTokenChunkerMediaContextSpansUpstreamItems(t *testing.T) {
+ cases := []struct {
+ name string
+ params map[string]any
+ mediaItem map[string]any
+ mediaType string
+ wantText string
+ }{
+ {
+ name: "image without a body",
+ params: map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 512, "image_context_size": 20},
+ mediaItem: map[string]any{"text": "", "image": "figure-bytes", "doc_type_kwd": "image"},
+ mediaType: "image",
+ wantText: "abovebelow",
+ },
+ {
+ name: "table",
+ params: map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 512, "table_context_size": 20},
+ mediaItem: map[string]any{"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ mediaType: "table",
+ wantText: "above<table><tr><td>A</td></tr></table>below",
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c, err := NewTokenChunker(tc.params)
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "fig.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "above", "doc_type_kwd": "text"},
+ tc.mediaItem,
+ {"text": "below", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ var media map[string]any
+ for _, ck := range chunks {
+ if ck["ck_type"] == tc.mediaType {
+ media = ck
+ break
+ }
+ }
+ if media == nil {
+ t.Fatalf("%s chunk missing from %d chunks: %+v", tc.mediaType, len(chunks), chunks)
+ }
+ assertMaterializedMediaContext(t, media, tc.wantText)
+ })
+ }
+}
+
+// TestTokenChunkerDropsMediaChunkWithTagOnlyContext pins the drop filter
+// against Python's finalize, which strips parser position tags from the merged
+// body before the empty check (token_chunker.py:343). A media chunk whose only
+// surrounding context is a position tag has nothing retrievable left, so it
+// must not survive — the fold would otherwise emit a chunk with an empty body.
+func TestTokenChunkerDropsMediaChunkWithTagOnlyContext(t *testing.T) {
+ component, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "chunk_token_size": 512,
+ "table_context_size": 20,
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "fig.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "@@1\t0.0\t10.0\t10.0\t20.0##", "doc_type_kwd": "text"},
+ {"text": "", "doc_type_kwd": "table"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if chunks := outputChunks(t, out); len(chunks) != 0 {
+ t.Fatalf("chunks = %+v, want none: the media chunk's only context is a position tag", chunks)
+ }
+}
+
// TestTokenChunker_InvokeDeterministic runs a 20-item structured
// payload 10 times under the race detector and asserts the chunk
// list is identical every time.
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index f4d3372b35..3c73b42ade 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -269,6 +269,15 @@ func ChunkDocsToMaps(in []ChunkDoc) []map[string]any {
return out
}
+// ContextualText joins a chunk's body with its surrounding media context in
+// the order Python materializes it (rag/flow/chunker/token_chunker.py:343):
+// context_above, text, context_below, concatenated as they are. No separator
+// is inserted: Python's finalize concatenates directly, and the context
+// collectors keep their own boundary punctuation and newlines.
+func ContextualText(d ChunkDoc) string {
+ return d.ContextAbove + d.Text + d.ContextBelow
+}
+
func (d *ChunkDoc) SetExtraValue(key string, value any) error {
if d.Extra == nil {
d.Extra = make(map[string]json.RawMessage)
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index eb1e745a79..4cf37c12e1 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -585,6 +585,36 @@ func TestExtractorOutputsJSONRoundTrip(t *testing.T) {
}
}
+// ---------------------------------------------------------------------------
+// ContextualText
+// ---------------------------------------------------------------------------
+
+// TestContextualTextConcatenatesMediaContext pins the single join rule shared
+// by the chunker's output fold and the tokenizer's retrieval text: the context
+// boundaries are concatenated as-is, mirroring Python's finalize
+// (rag/flow/chunker/token_chunker.py:343) — no separator is inserted.
+func TestContextualTextConcatenatesMediaContext(t *testing.T) {
+ cases := []struct {
+ name string
+ doc ChunkDoc
+ want string
+ }{
+ {"body only", ChunkDoc{Text: "body"}, "body"},
+ {"both sides", ChunkDoc{ContextAbove: "before", Text: "<table/>", ContextBelow: "after"}, "before<table/>after"},
+ {"keeps producer whitespace", ChunkDoc{ContextAbove: "before ", Text: "body", ContextBelow: " after"}, "before body after"},
+ {"above only", ChunkDoc{ContextAbove: "before", Text: "body"}, "beforebody"},
+ {"below only", ChunkDoc{Text: "body", ContextBelow: "after"}, "bodyafter"},
+ {"empty chunk", ChunkDoc{}, ""},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := ContextualText(tc.doc); got != tc.want {
+ t.Errorf("ContextualText() = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index e580db68af..527047283e 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -791,7 +791,7 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)
smt = st
}
ck.ContentSmLtks = smt
- } else if t := contextualChunkText(*ck); strings.TrimSpace(t) != "" {
+ } else if t := schema.ContextualText(*ck); strings.TrimSpace(t) != "" {
tt, err := tok.Tokenize(t)
if err != nil {
return fmt.Errorf("tokenizer: text tokenize: %w", err)
@@ -821,7 +821,7 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
for _, f := range fields {
switch f {
case "text":
- b.WriteString(contextualChunkText(ck))
+ b.WriteString(schema.ContextualText(ck))
case "questions":
b.WriteString(ck.Questions)
case "keywords":
@@ -841,34 +841,6 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
return b.String()
}
-// contextualChunkText is the retrieval representation of a chunk's body.
-// ContextAbove/ContextBelow are generated by GeneralChunker for media blocks
-// and must participate in both full-text tokenization and embeddings, while
-// ChunkDoc.Text remains the display/storage payload (for example, the table
-// HTML or image caption). Context values normally carry their own punctuation
-// or whitespace, matching Python's doc_tokenize_chunks_with_images contract.
-// If a producer trims both sides of a boundary, contextualChunkText inserts a
-// newline so retrieval and embedding do not fuse adjacent English words.
-func contextualChunkText(ck schema.ChunkDoc) string {
- text := ""
- for _, part := range []string{ck.ContextAbove, ck.Text, ck.ContextBelow} {
- if part == "" {
- continue
- }
- if text != "" && !contextBoundaryHasWhitespace(text, part) {
- text += "\n"
- }
- text += part
- }
- return text
-}
-
-func contextBoundaryHasWhitespace(left, right string) bool {
- leftRune, _ := utf8.DecodeLastRuneInString(left)
- rightRune, _ := utf8.DecodeRuneInString(right)
- return unicode.IsSpace(leftRune) || unicode.IsSpace(rightRune)
-}
-
// shouldHaveEmbedding reports whether the tokenizer must attach embedding
// vectors: the search method requests embedding AND a KB is present.
//
@@ -904,7 +876,7 @@ func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []
}
func requiresFullTextTokens(ck schema.ChunkDoc) bool {
- return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(contextualChunkText(ck)) != ""
+ return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(schema.ContextualText(ck)) != ""
}
func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index ae05a28a9f..9118ddf10b 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -304,17 +304,6 @@ func TestTokenizerComponent_EmbeddingIncludesMediaContext(t *testing.T) {
}
}
-func TestContextualChunkTextSeparatesTightMediaBoundaries(t *testing.T) {
- ck := schema.ChunkDoc{
- ContextAbove: "See results in table:",
- Text: "Revenue grew by 20%.",
- ContextBelow: "Conclusion follows.",
- }
- if got, want := contextualChunkText(ck), "See results in table:\nRevenue grew by 20%.\nConclusion follows."; got != want {
- t.Fatalf("contextual text = %q, want %q", got, want)
- }
-}
-
// TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero uses an
// empty chunk list, so tokenizeChunks is a no-op and the C++ pool is not needed.
func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *testing.T) {