RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): count child tokens and cut media context on sentences (#19696)
jay777212a610c732f34
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 049e6c3b67..4fb3c426ed 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -181,6 +181,16 @@ func itemTextOrFallback(it schema.ChunkDoc) string {// swizzle the count strategy in one place if needed.func tokenizeStr(s string) int { return tokenizer.NumTokensFromString(s) }+// setChunkText replaces a chunk's body together with the token count that+// describes it, so a caller cannot leave the two out of sync. TKNums is read as+// a budget by the media window walk and by the merge thresholds, and it is+// emitted as tk_nums, so a body replaced on its own silently misreports the+// chunk's size. Merge paths that join several units keep their summed count.+func setChunkText(ck *schema.ChunkDoc, text string) {+ ck.Text = text+ ck.TKNums = intPtr(tokenizeStr(text))+}+// toString normalises a chunk-map field to a string. Empty strings// for missing fields.func toString(v any) string {@@ -235,13 +245,16 @@ func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {// stripped by the chunker, so this only covers the context, which is collected// from neighbouring units.//+// The body goes through setChunkText, so TKNums keeps describing the body the+// chunk carries now instead of the bare payload it replaced.+//// Only media chunks carry context (attachMediaContext and// attachGeneralMediaContext write it), so text chunks pass through untouched.func materializeMediaContext(ck schema.ChunkDoc) schema.ChunkDoc {if ck.ContextAbove == "" && ck.ContextBelow == "" {return ck}- ck.Text = removeTag(schema.ContextualText(ck))+ setChunkText(&ck, removeTag(schema.ContextualText(ck)))ck.ContextAbove = ""ck.ContextBelow = ""return ckdiff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex d945232c75..9a7b53cf2a 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -42,7 +42,7 @@ import (const ComponentNameGeneralChunker = "GeneralChunker"-var generalSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)+var contextSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)type generalChunkerParam struct {ChunkTokenSize int@@ -467,7 +467,7 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abovtext := units[cursor].Texttokens := generalUnitTokens(units[cursor])if tokens >= remaining {- piece := takeGeneralContextSentence(text, remaining, above)+ piece := takeContextSentences(text, remaining, above)if above {parts = append([]string{piece}, parts...)} else {@@ -496,8 +496,13 @@ func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {return doc.TableID != "" || doc.Sheet != "" || doc.SheetIndex != nil}-func takeGeneralContextSentence(text string, budget int, fromEnd bool) string {- sentences := splitGeneralContextSentences(text)+// takeContextSentences returns the whole sentences at one end of text whose+// combined token count reaches the budget, taking the last sentence even when+// it overshoots: a sentence is the smallest unit a context window may cut at,+// so a text without a sentence boundary comes back whole. Shared by the media+// context collectors of both chunkers (general.go, token.go).+func takeContextSentences(text string, budget int, fromEnd bool) string {+ sentences := splitContextSentences(text)if len(sentences) == 0 {return text}@@ -526,8 +531,8 @@ func takeGeneralContextSentence(text string, budget int, fromEnd bool) string {return text[:end]}-func splitGeneralContextSentences(text string) []string {- indices := generalSentencePattern.FindAllStringIndex(text, -1)+func splitContextSentences(text string) []string {+ indices := contextSentencePattern.FindAllStringIndex(text, -1)if len(indices) == 0 {if text == "" {return nildiff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex c8717c32ce..b8bf8d5702 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -318,12 +318,12 @@ func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T)}}-func TestTakeGeneralContextSentencePreservesBoundarySelection(t *testing.T) {+func TestTakeContextSentencesPreservesBoundarySelection(t *testing.T) {text := "one!two!three!"- if got, want := takeGeneralContextSentence(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {+ if got, want := takeContextSentences(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {t.Fatalf("suffix context = %q, want %q", got, want)}- if got, want := takeGeneralContextSentence(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {+ if got, want := takeContextSentences(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {t.Fatalf("prefix context = %q, want %q", got, want)}}diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex e6898490f1..1638718900 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -534,6 +534,11 @@ func assertMaterializedMediaContext(t *testing.T, chunk map[string]any, wantTextt.Errorf("media chunk must not carry %s after materialization: %+v", key, chunk)}}+ // The folded body and its token count must agree: a media chunk is never+ // merged, so TKNums is a count of exactly this text.+ if count, ok := chunk["tk_nums"].(float64); !ok || count != float64(tokenizeStr(wantText)) {+ t.Errorf("media chunk tk_nums = %v, want %d (its own text)", chunk["tk_nums"], tokenizeStr(wantText))+ }}func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex e4e65ae43e..03e33f861b 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -519,7 +519,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil// Strip parser position tags from the final text:// the merge paths may carry @@...## markers that must not leak into// indexed/embedded chunk text.- ch.Text = removeTag(strings.TrimSpace(ch.Text))+ setChunkText(&ch, removeTag(strings.TrimSpace(ch.Text)))if ch.Text == "" {continue}@@ -628,7 +628,7 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s// the merge paths may carry @@...## markers that must not leak into// indexed/embedded chunk text. Crop above reads positions, not text,// so the ordering is safe.- m.Text = removeTag(m.Text)+ setChunkText(&m, removeTag(m.Text))// Drop a chunk only when nothing about it is retrievable. A media chunk// may carry no body of its own and still be indexed through its// surrounding context: Python's _finalize_json_chunks strips the merged@@ -866,6 +866,8 @@ func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []sche// collectContext walks chunks around `i` (above when direction==true,// below when false), pulling text chunks while remaining token budget// stays positive. Matches token_chunker.py:_attach_context_to_media_chunks.+// A neighbour that does not fit is truncated on a sentence boundary+// (takeContextSentences), not at an arbitrary rune.func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) string {var parts []stringremain := ctxTokens@@ -877,7 +879,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stritk := intValue(chunks[pos].TKNums)txt := chunks[pos].Textif tk >= remain {- parts = append([]string{takeFromEnd(txt, remain)}, parts...)+ parts = append([]string{takeContextSentences(txt, remain, true)}, parts...)remain = 0break}@@ -893,7 +895,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stritk := intValue(chunks[pos].TKNums)txt := chunks[pos].Textif tk >= remain {- parts = append(parts, takeFromStart(txt, remain))+ parts = append(parts, takeContextSentences(txt, remain, false))remain = 0break}@@ -906,39 +908,6 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) strireturn strings.Join(parts, "")}-// takeFromEnd returns the smallest tail of text whose token count is >=-// tokens, counted exactly via tokenizeStr The previous-// 4-bytes-per-token heuristic over-counted for CJK text.-func takeFromEnd(text string, tokens int) string {- runes := []rune(text)- // The tail runes[i:] grows as i decreases, so the first (largest i,- // i.e. smallest tail) that meets the budget is the answer.- for i := len(runes); i > 0; i-- {- cand := string(runes[i:])- if tokenizeStr(cand) >= tokens {- return cand- }- }- return text-}--// takeFromStart returns the smallest prefix of text whose token count is >=-// tokens, counted exactly via tokenizeStr-func takeFromStart(text string, tokens int) string {- runes := []rune(text)- best := text- // Prefix grows as i increases; the first (smallest) qualifying prefix- // is the answer.- for i := 1; i <= len(runes); i++ {- cand := string(runes[:i])- if tokenizeStr(cand) >= tokens {- best = cand- break- }- }- return best-}-// mergeUnits is the single, unified token-merge core shared by BOTH the text// path (mergeByTokenSize) and the JSON path (mergeByTokenSizeFromJSON). It is// a faithful port of Python rag/flow/chunker/token_chunker.py:@@ -1700,7 +1669,11 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.continue}cp := cloneChunkDoc(ck)- cp.Text = p+ // The count describes the child's own text. The delimiter branch+ // attaches the media context after this split, and that walk is+ // charged with TKNums, so an inherited parent count would spend the+ // configured window on the first neighbour.+ setChunkText(&cp, p)cp.Mom = momout = append(out, cp)}@@ -1736,15 +1709,13 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []scif strings.TrimSpace(t) == "" {continue}- for _, child := range splitDroppingDelim(t, pattern) {- if strings.TrimSpace(child) == "" {+ for _, text := range splitDroppingDelim(t, pattern) {+ if strings.TrimSpace(text) == "" {continue}- out = append(out, schema.ChunkDoc{- Text: child,- CKType: d.CKType,- Mom: strings.TrimPrefix(t, "\n"),- })+ child := schema.ChunkDoc{CKType: d.CKType, Mom: strings.TrimPrefix(t, "\n")}+ setChunkText(&child, text)+ out = append(out, child)}}return outdiff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex 4515d63186..ac745ddb27 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -301,33 +301,37 @@ func TestMergeByTokenSizeFromJSON_EmptyPrevKeepsChunk(t *testing.T) {}}-// TestTakeFromEndRespectsTokenCount and TestTakeFromStartRespectsTokenCount-// covers takeFromEnd/takeFromStart used a-// fixed 4-bytes-per-token heuristic which badly over-counts for CJK text-// (≈3 bytes/char, 1-2 tokens/char). They must now count tokens exactly via-// tokenizeStr so the returned slice is close to the requested token budget.-func TestTakeFromEndRespectsTokenCount(t *testing.T) {- const target = 20- s := strings.Repeat("中", 60)- got := takeFromEnd(s, target)- if !strings.HasSuffix(s, got) {- t.Fatalf("takeFromEnd result must be a suffix of input")+// TestTakeContextSentencesCoversBudgetOnSentenceBoundaries pins the truncation+// rule the token path shares with the general path: whole sentences are taken+// from the requested end until the budget is reached, so a cut never lands+// inside a sentence and the run is the smallest one that covers the budget.+func TestTakeContextSentencesCoversBudgetOnSentenceBoundaries(t *testing.T) {+ const text = "alpha beta. gamma delta. epsilon zeta."+ budget := tokenizeStr("gamma delta. epsilon zeta.")++ if got, want := takeContextSentences(text, budget, true), "gamma delta. epsilon zeta."; got != want {+ t.Errorf("tail context = %q, want %q", got, want)+ }+ if got, want := takeContextSentences(text, budget, false), "alpha beta. gamma delta. "; got != want {+ t.Errorf("prefix context = %q, want %q", got, want)}- n := tokenizeStr(got)- if n < target-3 || n > target+3 {- t.Errorf("takeFromEnd(%d tokens) returned slice with %d tokens (want ~%d)", target, n, target)+ // One sentence less falls short of the budget, so the walk stopped at the+ // first sentence run that covers it — the counting itself is exact.+ if short := tokenizeStr("epsilon zeta."); short >= budget {+ t.Fatalf("fixture invalid: dropping a sentence still meets the budget (%d >= %d)", short, budget)}}-func TestTakeFromStartRespectsTokenCount(t *testing.T) {- const target = 20+// TestTakeContextSentencesKeepsBoundarylessText pins the documented overshoot:+// a sentence is the smallest unit the window may cut at, so text without a+// boundary — the CJK case the removed rune-level helper used to slice mid-text+// — comes back whole.+func TestTakeContextSentencesKeepsBoundarylessText(t *testing.T) {s := strings.Repeat("中", 60)- got := takeFromStart(s, target)- if !strings.HasPrefix(s, got) {- t.Fatalf("takeFromStart result must be a prefix of input")+ if got := takeContextSentences(s, 20, true); got != s {+ t.Fatalf("context = %q, want the whole boundary-less text", got)}- n := tokenizeStr(got)- if n < target-3 || n > target+3 {- t.Errorf("takeFromStart(%d tokens) returned slice with %d tokens (want ~%d)", target, n, target)+ if got := takeContextSentences(s, 20, false); got != s {+ t.Fatalf("context = %q, want the whole boundary-less text", got)}}diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 42927b0414..62ac929da9 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -519,6 +519,111 @@ func TestTokenChunkerDropsMediaChunkWithTagOnlyContext(t *testing.T) {}}+// TestTokenChunkerDelimiterWindowUsesChildTokenCounts pins the effect the+// children's token counts have on the media window: the delimiter branch+// attaches the context after the children split, so the budget walk is charged+// with whatever count the children carry. A child that inherits its parent's+// count spends the whole window on the first neighbour and the configured+// window silently under-collects.+func TestTokenChunkerDelimiterWindowUsesChildTokenCounts(t *testing.T) {+ component, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"`|`"},+ "children_delimiters": []string{". "},+ "chunk_token_size": 512,+ "table_context_size": 7,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "fig.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "gamma delta. alpha beta.", "doc_type_kwd": "text"},+ {"text": "", "doc_type_kwd": "table"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ var media map[string]any+ for _, ck := range outputChunks(t, out) {+ if ck["ck_type"] == "table" {+ media = ck+ }+ }+ if media == nil {+ t.Fatalf("table chunk missing: %+v", out)+ }+ // Each child is 3 tokens, so the 7-token window holds both of them.+ assertMaterializedMediaContext(t, media, "gamma deltaalpha beta.")+}++// TestMaterializeMediaContextKeepsTokenCountInSync pins the invariant the fold+// must preserve: once the context is folded into the body, TKNums describes+// that body rather than the media payload it replaced. The count is read as a+// budget by the merge and window walks and emitted as tk_nums on the chunk.+func TestMaterializeMediaContextKeepsTokenCountInSync(t *testing.T) {+ const body = "<table><tr><td>A</td></tr></table>"+ ck := schema.ChunkDoc{+ Text: body,+ DocType: "table",+ CKType: "table",+ TKNums: intPtr(tokenizeStr(body)),+ ContextAbove: "above",+ ContextBelow: "below",+ }++ got := materializeMediaContext(ck)+ if want := "above" + body + "below"; got.Text != want {+ t.Fatalf("folded text = %q, want %q", got.Text, want)+ }+ if want := tokenizeStr(got.Text); intValue(got.TKNums) != want {+ t.Errorf("tk_nums = %d, want %d (the folded body)", intValue(got.TKNums), want)+ }+}++// TestTokenChunkerWindowTruncatesOnSentenceBoundary pins that a neighbour+// larger than the remaining window is trimmed on a sentence boundary. A cut+// inside the sentence would hand the index a fragment: the removed rune-level+// helper returned "zeta." for this fixture, a piece of "epsilon zeta.".+func TestTokenChunkerWindowTruncatesOnSentenceBoundary(t *testing.T) {+ component, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "chunk_token_size": 512,+ "table_context_size": 3,+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "fig.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "epsilon zeta.", "doc_type_kwd": "text"},+ {"text": "", "doc_type_kwd": "table"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ var media map[string]any+ for _, ck := range outputChunks(t, out) {+ if ck["ck_type"] == "table" {+ media = ck+ }+ }+ if media == nil {+ t.Fatalf("table chunk missing: %+v", out)+ }+ // "epsilon zeta." is one sentence of 4 tokens, so the 3-token window takes+ // it whole rather than cutting into it.+ assertMaterializedMediaContext(t, media, "epsilon zeta.")+}+// TestTokenChunker_InvokeDeterministic runs a 20-item structured// payload 10 times under the race detector and asserts the chunk// list is identical every time.@@ -988,6 +1093,50 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {}}+// TestSplitByChildrenRecomputesTokenCounts pins that a child carries the count+// of its own text. The count is a budget input for the media window in the+// delimiter branch, where the attach runs after this split.+func TestSplitByChildrenRecomputesTokenCounts(t *testing.T) {+ const parentText = "gamma delta. alpha beta."+ children := splitByChildren([]schema.ChunkDoc{{+ Text: parentText,+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(tokenizeStr(parentText)),+ }}, regexp.MustCompile(`\. `))++ if len(children) != 2 {+ t.Fatalf("children = %d, want 2", len(children))+ }+ for _, child := range children {+ if got, want := intValue(child.TKNums), tokenizeStr(child.Text); got != want {+ t.Errorf("child %q tk_nums = %d, want %d (its own text)", child.Text, got, want)+ }+ }+}++// TestApplyChildrenDelimText_RecomputesTokenCounts pins that the text-path+// children carry their own count, so a consumer that budgets with TKNums does+// not read a stale or missing value.+func TestApplyChildrenDelimText_RecomputesTokenCounts(t *testing.T) {+ const parentText = "gamma delta. alpha beta."+ out := applyChildrenDelimText([]schema.ChunkDoc{{+ Text: parentText,+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(tokenizeStr(parentText)),+ }}, regexp.MustCompile(`\. `))++ if len(out) != 2 {+ t.Fatalf("children = %d, want 2", len(out))+ }+ for _, child := range out {+ if got, want := intValue(child.TKNums), tokenizeStr(child.Text); got != want {+ t.Errorf("child %q tk_nums = %d, want %d (its own text)", child.Text, got, want)+ }+ }+}+// TestApplyChildrenDelimText_DefaultsMomToCurrentChunk verifies that when an// incoming chunk has no Mom, the children fall back to the current chunk's// text (the historical behaviour preserved by the fix for #17876).