RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): include the image/table context window in the chunk text (#19678)
jay777215dda6fd999df
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex cbdfe9fae0..049e6c3b67 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -208,11 +208,43 @@ func emptyOutputs() map[string]any {// model resolution) is NOT re-emitted here — it lives in the workflow-wide// CanvasState.Globals bag (seeded at pipeline start, published by the File// component) and read directly from ctx. See runtime.CanvasState.Globals.+//+// Media context is materialized into the chunk body here, the chunker's last+// step: every variant passes through this builder, so the folded text is what+// the chunk id, the extractor, the tokenizer and the index write all see.func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {+ materialized := make([]schema.ChunkDoc, len(chunks))+ for i := range chunks {+ materialized[i] = materializeMediaContext(chunks[i])+ }return map[string]any{"output_format": "chunks",- "chunks": schema.ChunkDocsToMaps(chunks),+ "chunks": schema.ChunkDocsToMaps(materialized),+ }+}++// materializeMediaContext folds a media chunk's surrounding context into its+// body and clears the two fields that carried it. Python's chunker emits the+// same shape — its finalize builds remove_tag(context_above + text ++// context_below) and drops the fields (rag/flow/chunker/token_chunker.py:343-+// 359) — which is why Python persists the context inside the chunk body.+// Folding here also puts the context into the chunk id (ChunkID hashes the+// body), matching Python's id, which hashes the context-bearing body.+//+// Tag stripping runs after the merge, as in Python: the payload was already+// stripped by the chunker, so this only covers the context, which is collected+// from neighbouring units.+//+// Only media chunks carry context (attachMediaContext and+// attachGeneralMediaContext write it), so text chunks pass through untouched.+func materializeMediaContext(ck schema.ChunkDoc) schema.ChunkDoc {+ if ck.ContextAbove == "" && ck.ContextBelow == "" {+ return ck}+ ck.Text = removeTag(schema.ContextualText(ck))+ ck.ContextAbove = ""+ ck.ContextBelow = ""+ return ck}// withName returns a shallow copy of inputs with name set, so a component candiff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex ee3938ef49..e6898490f1 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -28,9 +28,12 @@ import ("reflect""strings""testing"+ "time""ragflow/internal/agent/runtime"+ "ragflow/internal/common""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/ingestion/task/indexdoc")func TestGeneralChunkerRegistered(t *testing.T) {@@ -516,6 +519,23 @@ func TestGeneralChunkerDOCXMediaDoesNotBreakTextMerge(t *testing.T) {}}+// assertMaterializedMediaContext pins the chunker output contract for media+// chunks: the surrounding context is folded into the chunk body and the+// retrieval-only context fields are gone — the shape Python's chunker emits+// (rag/flow/chunker/token_chunker.py:343-359) and the shape the persisted+// content_with_weight and the chunk id are built from.+func assertMaterializedMediaContext(t *testing.T, chunk map[string]any, wantText string) {+ t.Helper()+ if got := chunk["text"]; got != wantText {+ t.Errorf("media chunk text = %q, want %q", got, wantText)+ }+ for _, key := range []string{"context_above", "context_below"} {+ if _, exists := chunk[key]; exists {+ t.Errorf("media chunk must not carry %s after materialization: %+v", key, chunk)+ }+ }+}+func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10,@@ -544,9 +564,7 @@ func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {if chunks[1]["doc_type_kwd"] != "table" {t.Fatalf("table chunk = %+v", chunks[1])}- if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {- t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])- }+ assertMaterializedMediaContext(t, chunks[1], "before<table><tr><td>A</td></tr></table>after")}func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {@@ -701,9 +719,7 @@ func TestGeneralChunkerPDFUsesPositionOrderForMediaContext(t *testing.T) {if len(chunks) != 3 {t.Fatalf("chunks = %#v, want text, media, text", chunks)}- if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {- t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])- }+ assertMaterializedMediaContext(t, chunks[1], "before<table>A</table>after")if chunks[0]["text"] != "before" || chunks[2]["text"] != "after" {t.Errorf("position-ordered text = [%v, %v]", chunks[0]["text"], chunks[2]["text"])}@@ -783,9 +799,7 @@ func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {if chunks[1]["ck_type"] != "image" {t.Fatalf("image chunk = %#v", chunks[1])}- if chunks[1]["context_above"] != "Revenue" || chunks[1]["context_below"] != "Growth" {- t.Fatalf("image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])- }+ assertMaterializedMediaContext(t, chunks[1], "RevenueB2Growth")}func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T) {@@ -810,9 +824,95 @@ func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T)if len(chunks) != 3 {t.Fatalf("chunks = %#v, want row, image, row", chunks)}- if chunks[1]["context_above"] != "Sheet one" || chunks[1]["context_below"] != nil {- t.Fatalf("cross-sheet image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])+ assertMaterializedMediaContext(t, chunks[1], "Sheet oneB2")+}++// TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent pins the+// downstream half of the fold: once the context is part of the chunk body, it+// is also part of the persisted content_with_weight and of the chunk id+// (ChunkID hashes the body). Python's chunker emits the merged body, so its+// id and stored content already carry the context; before the fold Go hashed+// and stored the bare media payload instead, which made the configured window+// invisible to both the chunk list and the index.+func TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 2,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := indexdoc.NormalizeChunks(out)+ if _, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "document.docx", time.Now()); err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }++ const wantText = "before<table><tr><td>A</td></tr></table>after"+ var table map[string]any+ for _, ck := range chunks {+ if ck["ck_type"] == "table" {+ table = ck+ break+ }+ }+ if table == nil {+ t.Fatalf("no table chunk in %#v", chunks)+ }+ if got := table["content_with_weight"]; got != wantText {+ t.Errorf("persisted content_with_weight = %q, want %q", got, wantText)+ }+ if got, want := table["id"], common.ChunkID("doc-1", wantText); got != want {+ t.Errorf("chunk id = %v, want %v (context must drive chunk identity)", got, want)+ }+}++// TestGeneralChunkerMediaContextStripsPositionTags pins the tag-stripping+// order Python uses: remove_tag runs on the merged body, so a position tag+// carried by a neighbouring text unit never reaches the stored media chunk.+// (The parser keeps boxes in the positions field, so this is a guard, not a+// re-strip of the media payload.)+func TestGeneralChunkerMediaContextStripsPositionTags(t *testing.T) {+ position := func(top float64) []any {+ return []any{[]any{1.0, 0.0, 10.0, top, top + 5}}+ }+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "table_context_size": 20,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before@@1\t0.0\t10.0\t10.0\t20.0##", "doc_type_kwd": "text", "positions": position(10)},+ {"text": "<table>A</table>", "doc_type_kwd": "table", "positions": position(20)},+ {"text": "after@@1\t0.0\t10.0\t10.0\t30.0\t35.0##", "doc_type_kwd": "text", "positions": position(30)},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v, want text, media, text", chunks)}+ assertMaterializedMediaContext(t, chunks[1], "before<table>A</table>after")}func TestGeneralChunkerPDFAttachesOutlineOnce(t *testing.T) {diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 91269b6af8..f0b2a30699 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -588,25 +588,36 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s}}- // Attach surrounding media context (token_chunker.py:358).- attached := attachMediaContext(perItem, c.param.TableContextSize, c.param.ImageContextSize)+ // Surrounding media context is attached on the flattened chunk list,+ // mirroring Python's _attach_context_to_media_chunks (token_chunker.py:537,+ // :545): the units are flat there, so a media block collects the text units+ // around it across item boundaries. Attaching per upstream item instead+ // leaves every media chunk without neighbours, because one item rarely+ // yields more than one chunk.+ flat := flatten(perItem)// Python's naive_merge: custom (backtick) delimiters produce one// chunk per segment — no token-size merge (naive_merge:1194-1213).// Otherwise split-then-merge: delimiter-split segments are greedily// merged to chunk_token_size with optional overlap.- if !hasCustomDelim(c.param.Delimiters) {+ customDelim := hasCustomDelim(c.param.Delimiters)+ if !customDelim {+ // Attach before the merge — Python's non-delimiter branch collects the+ // context from the pre-merge text units. The merge only joins text+ // chunks, so a media chunk keeps what it collected.+ flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)// Python _merge_text_chunks_by_token_size merges adjacent text- // chunks across JSON items into one global token budget. Flatten the- // per-item structure into a single sequence first so the merge is- // global; non-text chunks still break the merge via their CKType.- attached = mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flatten(attached)}, c.param.ChunkTokenSize, c.param.OverlappedPercent)+ // chunks across JSON items into one global token budget.+ flat = flatten(mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flat}, c.param.ChunkTokenSize, c.param.OverlappedPercent))}- flat := flatten(attached)if childrenPattern != nil {flat = splitByChildren(flat, childrenPattern)}+ if customDelim {+ // Python's delimiter branch splits by children first, then attaches.+ flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)+ }// Crop image/table chunks on demand when a PDF engine is available.flat = cropImageChunks(ctx, engine, flat)@@ -618,7 +629,15 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s// indexed/embedded chunk text. Crop above reads positions, not text,// so the ordering is safe.m.Text = removeTag(m.Text)- if m.Text == "" {+ // Drop a chunk only when nothing about it is retrievable. A media chunk+ // may carry no body of its own and still be indexed through its+ // surrounding context: Python's _finalize_json_chunks strips the merged+ // text and applies the check to it — remove_tag(context_above + text ++ // context_below) then .strip() — so a context that carries nothing but+ // a position tag counts as empty. The context is folded into Text once,+ // at chunkOutputs, so the decision is made on that merged text here+ // without duplicating the fold.+ if strings.TrimSpace(removeTag(schema.ContextualText(m))) == "" {continue}out = append(out, m)@@ -818,32 +837,30 @@ func partition(n, parts int) []lane {return out}-func attachMediaContext(perItem [][]schema.ChunkDoc, tableCtx, imageCtx int) [][]schema.ChunkDoc {+// attachMediaContext writes the surrounding context onto the media chunks of a+// flat chunk list. Mirrors token_chunker.py:_attach_context_to_media_chunks,+// which runs on the flat chunk list as well: a media block collects the text+// units around it, up to the token budget, across upstream item boundaries.+func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []schema.ChunkDoc {if tableCtx <= 0 && imageCtx <= 0 {- return perItem+ return chunks}- for idx := range perItem {- chunks := perItem[idx]- if len(chunks) == 0 {+ for i, ck := range chunks {+ ckType := ck.CKType+ if ckType != "table" && ckType != "image" {continue}- for i, ck := range chunks {- ckType := ck.CKType- if ckType != "table" && ckType != "image" {- continue- }- ctx := imageCtx- if ckType == "table" {- ctx = tableCtx- }- if ctx <= 0 {- continue- }- chunks[i].ContextAbove = collectContext(chunks, i, ctx, true)- chunks[i].ContextBelow = collectContext(chunks, i, ctx, false)+ ctx := imageCtx+ if ckType == "table" {+ ctx = tableCtx}+ if ctx <= 0 {+ continue+ }+ chunks[i].ContextAbove = collectContext(chunks, i, ctx, true)+ chunks[i].ContextBelow = collectContext(chunks, i, ctx, false)}- return perItem+ return chunks}// collectContext walks chunks around `i` (above when direction==true,diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex ed0053003c..42927b0414 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -425,6 +425,100 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {}}+// TestTokenChunkerMediaContextSpansUpstreamItems pins the flat attach contract+// on the canvas path: Python collects a media chunk's context from the flat+// chunk list (token_chunker.py:537, :545), so the text units of *other*+// upstream items are in scope. The context is also what keeps a caption-less+// media chunk alive — its own body is empty — and what that chunk indexes.+func TestTokenChunkerMediaContextSpansUpstreamItems(t *testing.T) {+ cases := []struct {+ name string+ params map[string]any+ mediaItem map[string]any+ mediaType string+ wantText string+ }{+ {+ name: "image without a body",+ params: map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 512, "image_context_size": 20},+ mediaItem: map[string]any{"text": "", "image": "figure-bytes", "doc_type_kwd": "image"},+ mediaType: "image",+ wantText: "abovebelow",+ },+ {+ name: "table",+ params: map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 512, "table_context_size": 20},+ mediaItem: map[string]any{"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ mediaType: "table",+ wantText: "above<table><tr><td>A</td></tr></table>below",+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ c, err := NewTokenChunker(tc.params)+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "fig.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "above", "doc_type_kwd": "text"},+ tc.mediaItem,+ {"text": "below", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ var media map[string]any+ for _, ck := range chunks {+ if ck["ck_type"] == tc.mediaType {+ media = ck+ break+ }+ }+ if media == nil {+ t.Fatalf("%s chunk missing from %d chunks: %+v", tc.mediaType, len(chunks), chunks)+ }+ assertMaterializedMediaContext(t, media, tc.wantText)+ })+ }+}++// TestTokenChunkerDropsMediaChunkWithTagOnlyContext pins the drop filter+// against Python's finalize, which strips parser position tags from the merged+// body before the empty check (token_chunker.py:343). A media chunk whose only+// surrounding context is a position tag has nothing retrievable left, so it+// must not survive — the fold would otherwise emit a chunk with an empty body.+func TestTokenChunkerDropsMediaChunkWithTagOnlyContext(t *testing.T) {+ component, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "chunk_token_size": 512,+ "table_context_size": 20,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "fig.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "@@1\t0.0\t10.0\t10.0\t20.0##", "doc_type_kwd": "text"},+ {"text": "", "doc_type_kwd": "table"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if chunks := outputChunks(t, out); len(chunks) != 0 {+ t.Fatalf("chunks = %+v, want none: the media chunk's only context is a position tag", chunks)+ }+}+// TestTokenChunker_InvokeDeterministic runs a 20-item structured// payload 10 times under the race detector and asserts the chunk// list is identical every time.diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex f4d3372b35..3c73b42ade 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -269,6 +269,15 @@ func ChunkDocsToMaps(in []ChunkDoc) []map[string]any {return out}+// ContextualText joins a chunk's body with its surrounding media context in+// the order Python materializes it (rag/flow/chunker/token_chunker.py:343):+// context_above, text, context_below, concatenated as they are. No separator+// is inserted: Python's finalize concatenates directly, and the context+// collectors keep their own boundary punctuation and newlines.+func ContextualText(d ChunkDoc) string {+ return d.ContextAbove + d.Text + d.ContextBelow+}+func (d *ChunkDoc) SetExtraValue(key string, value any) error {if d.Extra == nil {d.Extra = make(map[string]json.RawMessage)diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex eb1e745a79..4cf37c12e1 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -585,6 +585,36 @@ func TestExtractorOutputsJSONRoundTrip(t *testing.T) {}}+// ---------------------------------------------------------------------------+// ContextualText+// ---------------------------------------------------------------------------++// TestContextualTextConcatenatesMediaContext pins the single join rule shared+// by the chunker's output fold and the tokenizer's retrieval text: the context+// boundaries are concatenated as-is, mirroring Python's finalize+// (rag/flow/chunker/token_chunker.py:343) — no separator is inserted.+func TestContextualTextConcatenatesMediaContext(t *testing.T) {+ cases := []struct {+ name string+ doc ChunkDoc+ want string+ }{+ {"body only", ChunkDoc{Text: "body"}, "body"},+ {"both sides", ChunkDoc{ContextAbove: "before", Text: "<table/>", ContextBelow: "after"}, "before<table/>after"},+ {"keeps producer whitespace", ChunkDoc{ContextAbove: "before ", Text: "body", ContextBelow: " after"}, "before body after"},+ {"above only", ChunkDoc{ContextAbove: "before", Text: "body"}, "beforebody"},+ {"below only", ChunkDoc{Text: "body", ContextBelow: "after"}, "bodyafter"},+ {"empty chunk", ChunkDoc{}, ""},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := ContextualText(tc.doc); got != tc.want {+ t.Errorf("ContextualText() = %q, want %q", got, tc.want)+ }+ })+ }+}+// ---------------------------------------------------------------------------// helpers// ---------------------------------------------------------------------------diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex e580db68af..527047283e 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -791,7 +791,7 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)smt = st}ck.ContentSmLtks = smt- } else if t := contextualChunkText(*ck); strings.TrimSpace(t) != "" {+ } else if t := schema.ContextualText(*ck); strings.TrimSpace(t) != "" {tt, err := tok.Tokenize(t)if err != nil {return fmt.Errorf("tokenizer: text tokenize: %w", err)@@ -821,7 +821,7 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {for _, f := range fields {switch f {case "text":- b.WriteString(contextualChunkText(ck))+ b.WriteString(schema.ContextualText(ck))case "questions":b.WriteString(ck.Questions)case "keywords":@@ -841,34 +841,6 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {return b.String()}-// contextualChunkText is the retrieval representation of a chunk's body.-// ContextAbove/ContextBelow are generated by GeneralChunker for media blocks-// and must participate in both full-text tokenization and embeddings, while-// ChunkDoc.Text remains the display/storage payload (for example, the table-// HTML or image caption). Context values normally carry their own punctuation-// or whitespace, matching Python's doc_tokenize_chunks_with_images contract.-// If a producer trims both sides of a boundary, contextualChunkText inserts a-// newline so retrieval and embedding do not fuse adjacent English words.-func contextualChunkText(ck schema.ChunkDoc) string {- text := ""- for _, part := range []string{ck.ContextAbove, ck.Text, ck.ContextBelow} {- if part == "" {- continue- }- if text != "" && !contextBoundaryHasWhitespace(text, part) {- text += "\n"- }- text += part- }- return text-}--func contextBoundaryHasWhitespace(left, right string) bool {- leftRune, _ := utf8.DecodeLastRuneInString(left)- rightRune, _ := utf8.DecodeRuneInString(right)- return unicode.IsSpace(leftRune) || unicode.IsSpace(rightRune)-}-// shouldHaveEmbedding reports whether the tokenizer must attach embedding// vectors: the search method requests embedding AND a KB is present.//@@ -904,7 +876,7 @@ func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []}func requiresFullTextTokens(ck schema.ChunkDoc) bool {- return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(contextualChunkText(ck)) != ""+ return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(schema.ContextualText(ck)) != ""}func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex ae05a28a9f..9118ddf10b 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -304,17 +304,6 @@ func TestTokenizerComponent_EmbeddingIncludesMediaContext(t *testing.T) {}}-func TestContextualChunkTextSeparatesTightMediaBoundaries(t *testing.T) {- ck := schema.ChunkDoc{- ContextAbove: "See results in table:",- Text: "Revenue grew by 20%.",- ContextBelow: "Conclusion follows.",- }- if got, want := contextualChunkText(ck), "See results in table:\nRevenue grew by 20%.\nConclusion follows."; got != want {- t.Fatalf("contextual text = %q, want %q", got, want)- }-}-// TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero uses an// empty chunk list, so tokenizeChunks is a no-op and the C++ pool is not needed.func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *testing.T) {