Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): count child tokens and cut media context on sentences (#19696)

jay777212a610c732f34
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 049e6c3b67..4fb3c426ed 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -181,6 +181,16 @@ func itemTextOrFallback(it schema.ChunkDoc) string {
// swizzle the count strategy in one place if needed.
func tokenizeStr(s string) int { return tokenizer.NumTokensFromString(s) }
+// setChunkText replaces a chunk's body together with the token count that
+// describes it, so a caller cannot leave the two out of sync. TKNums is read as
+// a budget by the media window walk and by the merge thresholds, and it is
+// emitted as tk_nums, so a body replaced on its own silently misreports the
+// chunk's size. Merge paths that join several units keep their summed count.
+func setChunkText(ck *schema.ChunkDoc, text string) {
+ ck.Text = text
+ ck.TKNums = intPtr(tokenizeStr(text))
+}
+
// toString normalises a chunk-map field to a string. Empty strings
// for missing fields.
func toString(v any) string {
@@ -235,13 +245,16 @@ func chunkOutputs(chunks []schema.ChunkDoc) map[string]any {
// stripped by the chunker, so this only covers the context, which is collected
// from neighbouring units.
//
+// The body goes through setChunkText, so TKNums keeps describing the body the
+// chunk carries now instead of the bare payload it replaced.
+//
// Only media chunks carry context (attachMediaContext and
// attachGeneralMediaContext write it), so text chunks pass through untouched.
func materializeMediaContext(ck schema.ChunkDoc) schema.ChunkDoc {
if ck.ContextAbove == "" && ck.ContextBelow == "" {
return ck
}
- ck.Text = removeTag(schema.ContextualText(ck))
+ setChunkText(&ck, removeTag(schema.ContextualText(ck)))
ck.ContextAbove = ""
ck.ContextBelow = ""
return ck
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index d945232c75..9a7b53cf2a 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -42,7 +42,7 @@ import (
const ComponentNameGeneralChunker = "GeneralChunker"
-var generalSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)
+var contextSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)
type generalChunkerParam struct {
ChunkTokenSize int
@@ -467,7 +467,7 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abov
text := units[cursor].Text
tokens := generalUnitTokens(units[cursor])
if tokens >= remaining {
- piece := takeGeneralContextSentence(text, remaining, above)
+ piece := takeContextSentences(text, remaining, above)
if above {
parts = append([]string{piece}, parts...)
} else {
@@ -496,8 +496,13 @@ func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {
return doc.TableID != "" || doc.Sheet != "" || doc.SheetIndex != nil
}
-func takeGeneralContextSentence(text string, budget int, fromEnd bool) string {
- sentences := splitGeneralContextSentences(text)
+// takeContextSentences returns the whole sentences at one end of text whose
+// combined token count reaches the budget, taking the last sentence even when
+// it overshoots: a sentence is the smallest unit a context window may cut at,
+// so a text without a sentence boundary comes back whole. Shared by the media
+// context collectors of both chunkers (general.go, token.go).
+func takeContextSentences(text string, budget int, fromEnd bool) string {
+ sentences := splitContextSentences(text)
if len(sentences) == 0 {
return text
}
@@ -526,8 +531,8 @@ func takeGeneralContextSentence(text string, budget int, fromEnd bool) string {
return text[:end]
}
-func splitGeneralContextSentences(text string) []string {
- indices := generalSentencePattern.FindAllStringIndex(text, -1)
+func splitContextSentences(text string) []string {
+ indices := contextSentencePattern.FindAllStringIndex(text, -1)
if len(indices) == 0 {
if text == "" {
return nil
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
index c8717c32ce..b8bf8d5702 100644
--- a/internal/ingestion/component/chunker/general_merge_test.go
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -318,12 +318,12 @@ func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T)
}
}
-func TestTakeGeneralContextSentencePreservesBoundarySelection(t *testing.T) {
+func TestTakeContextSentencesPreservesBoundarySelection(t *testing.T) {
text := "one!two!three!"
- if got, want := takeGeneralContextSentence(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {
+ if got, want := takeContextSentences(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {
t.Fatalf("suffix context = %q, want %q", got, want)
}
- if got, want := takeGeneralContextSentence(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {
+ if got, want := takeContextSentences(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {
t.Fatalf("prefix context = %q, want %q", got, want)
}
}
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index e6898490f1..1638718900 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -534,6 +534,11 @@ func assertMaterializedMediaContext(t *testing.T, chunk map[string]any, wantText
t.Errorf("media chunk must not carry %s after materialization: %+v", key, chunk)
}
}
+ // The folded body and its token count must agree: a media chunk is never
+ // merged, so TKNums is a count of exactly this text.
+ if count, ok := chunk["tk_nums"].(float64); !ok || count != float64(tokenizeStr(wantText)) {
+ t.Errorf("media chunk tk_nums = %v, want %d (its own text)", chunk["tk_nums"], tokenizeStr(wantText))
+ }
}
func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index e4e65ae43e..03e33f861b 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -519,7 +519,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil
// Strip parser position tags from the final text:
// the merge paths may carry @@...## markers that must not leak into
// indexed/embedded chunk text.
- ch.Text = removeTag(strings.TrimSpace(ch.Text))
+ setChunkText(&ch, removeTag(strings.TrimSpace(ch.Text)))
if ch.Text == "" {
continue
}
@@ -628,7 +628,7 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
// the merge paths may carry @@...## markers that must not leak into
// indexed/embedded chunk text. Crop above reads positions, not text,
// so the ordering is safe.
- m.Text = removeTag(m.Text)
+ setChunkText(&m, removeTag(m.Text))
// Drop a chunk only when nothing about it is retrievable. A media chunk
// may carry no body of its own and still be indexed through its
// surrounding context: Python's _finalize_json_chunks strips the merged
@@ -866,6 +866,8 @@ func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []sche
// collectContext walks chunks around `i` (above when direction==true,
// below when false), pulling text chunks while remaining token budget
// stays positive. Matches token_chunker.py:_attach_context_to_media_chunks.
+// A neighbour that does not fit is truncated on a sentence boundary
+// (takeContextSentences), not at an arbitrary rune.
func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) string {
var parts []string
remain := ctxTokens
@@ -877,7 +879,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
tk := intValue(chunks[pos].TKNums)
txt := chunks[pos].Text
if tk >= remain {
- parts = append([]string{takeFromEnd(txt, remain)}, parts...)
+ parts = append([]string{takeContextSentences(txt, remain, true)}, parts...)
remain = 0
break
}
@@ -893,7 +895,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
tk := intValue(chunks[pos].TKNums)
txt := chunks[pos].Text
if tk >= remain {
- parts = append(parts, takeFromStart(txt, remain))
+ parts = append(parts, takeContextSentences(txt, remain, false))
remain = 0
break
}
@@ -906,39 +908,6 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stri
return strings.Join(parts, "")
}
-// takeFromEnd returns the smallest tail of text whose token count is >=
-// tokens, counted exactly via tokenizeStr The previous
-// 4-bytes-per-token heuristic over-counted for CJK text.
-func takeFromEnd(text string, tokens int) string {
- runes := []rune(text)
- // The tail runes[i:] grows as i decreases, so the first (largest i,
- // i.e. smallest tail) that meets the budget is the answer.
- for i := len(runes); i > 0; i-- {
- cand := string(runes[i:])
- if tokenizeStr(cand) >= tokens {
- return cand
- }
- }
- return text
-}
-
-// takeFromStart returns the smallest prefix of text whose token count is >=
-// tokens, counted exactly via tokenizeStr
-func takeFromStart(text string, tokens int) string {
- runes := []rune(text)
- best := text
- // Prefix grows as i increases; the first (smallest) qualifying prefix
- // is the answer.
- for i := 1; i <= len(runes); i++ {
- cand := string(runes[:i])
- if tokenizeStr(cand) >= tokens {
- best = cand
- break
- }
- }
- return best
-}
-
// mergeUnits is the single, unified token-merge core shared by BOTH the text
// path (mergeByTokenSize) and the JSON path (mergeByTokenSizeFromJSON). It is
// a faithful port of Python rag/flow/chunker/token_chunker.py:
@@ -1700,7 +1669,11 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.
continue
}
cp := cloneChunkDoc(ck)
- cp.Text = p
+ // The count describes the child's own text. The delimiter branch
+ // attaches the media context after this split, and that walk is
+ // charged with TKNums, so an inherited parent count would spend the
+ // configured window on the first neighbour.
+ setChunkText(&cp, p)
cp.Mom = mom
out = append(out, cp)
}
@@ -1736,15 +1709,13 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc
if strings.TrimSpace(t) == "" {
continue
}
- for _, child := range splitDroppingDelim(t, pattern) {
- if strings.TrimSpace(child) == "" {
+ for _, text := range splitDroppingDelim(t, pattern) {
+ if strings.TrimSpace(text) == "" {
continue
}
- out = append(out, schema.ChunkDoc{
- Text: child,
- CKType: d.CKType,
- Mom: strings.TrimPrefix(t, "\n"),
- })
+ child := schema.ChunkDoc{CKType: d.CKType, Mom: strings.TrimPrefix(t, "\n")}
+ setChunkText(&child, text)
+ out = append(out, child)
}
}
return out
diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.go
index 4515d63186..ac745ddb27 100644
--- a/internal/ingestion/component/chunker/token_batch1_test.go
+++ b/internal/ingestion/component/chunker/token_batch1_test.go
@@ -301,33 +301,37 @@ func TestMergeByTokenSizeFromJSON_EmptyPrevKeepsChunk(t *testing.T) {
}
}
-// TestTakeFromEndRespectsTokenCount and TestTakeFromStartRespectsTokenCount
-// covers takeFromEnd/takeFromStart used a
-// fixed 4-bytes-per-token heuristic which badly over-counts for CJK text
-// (≈3 bytes/char, 1-2 tokens/char). They must now count tokens exactly via
-// tokenizeStr so the returned slice is close to the requested token budget.
-func TestTakeFromEndRespectsTokenCount(t *testing.T) {
- const target = 20
- s := strings.Repeat("中", 60)
- got := takeFromEnd(s, target)
- if !strings.HasSuffix(s, got) {
- t.Fatalf("takeFromEnd result must be a suffix of input")
+// TestTakeContextSentencesCoversBudgetOnSentenceBoundaries pins the truncation
+// rule the token path shares with the general path: whole sentences are taken
+// from the requested end until the budget is reached, so a cut never lands
+// inside a sentence and the run is the smallest one that covers the budget.
+func TestTakeContextSentencesCoversBudgetOnSentenceBoundaries(t *testing.T) {
+ const text = "alpha beta. gamma delta. epsilon zeta."
+ budget := tokenizeStr("gamma delta. epsilon zeta.")
+
+ if got, want := takeContextSentences(text, budget, true), "gamma delta. epsilon zeta."; got != want {
+ t.Errorf("tail context = %q, want %q", got, want)
+ }
+ if got, want := takeContextSentences(text, budget, false), "alpha beta. gamma delta. "; got != want {
+ t.Errorf("prefix context = %q, want %q", got, want)
}
- n := tokenizeStr(got)
- if n < target-3 || n > target+3 {
- t.Errorf("takeFromEnd(%d tokens) returned slice with %d tokens (want ~%d)", target, n, target)
+ // One sentence less falls short of the budget, so the walk stopped at the
+ // first sentence run that covers it — the counting itself is exact.
+ if short := tokenizeStr("epsilon zeta."); short >= budget {
+ t.Fatalf("fixture invalid: dropping a sentence still meets the budget (%d >= %d)", short, budget)
}
}
-func TestTakeFromStartRespectsTokenCount(t *testing.T) {
- const target = 20
+// TestTakeContextSentencesKeepsBoundarylessText pins the documented overshoot:
+// a sentence is the smallest unit the window may cut at, so text without a
+// boundary — the CJK case the removed rune-level helper used to slice mid-text
+// — comes back whole.
+func TestTakeContextSentencesKeepsBoundarylessText(t *testing.T) {
s := strings.Repeat("中", 60)
- got := takeFromStart(s, target)
- if !strings.HasPrefix(s, got) {
- t.Fatalf("takeFromStart result must be a prefix of input")
+ if got := takeContextSentences(s, 20, true); got != s {
+ t.Fatalf("context = %q, want the whole boundary-less text", got)
}
- n := tokenizeStr(got)
- if n < target-3 || n > target+3 {
- t.Errorf("takeFromStart(%d tokens) returned slice with %d tokens (want ~%d)", target, n, target)
+ if got := takeContextSentences(s, 20, false); got != s {
+ t.Fatalf("context = %q, want the whole boundary-less text", got)
}
}
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 42927b0414..62ac929da9 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -519,6 +519,111 @@ func TestTokenChunkerDropsMediaChunkWithTagOnlyContext(t *testing.T) {
}
}
+// TestTokenChunkerDelimiterWindowUsesChildTokenCounts pins the effect the
+// children's token counts have on the media window: the delimiter branch
+// attaches the context after the children split, so the budget walk is charged
+// with whatever count the children carry. A child that inherits its parent's
+// count spends the whole window on the first neighbour and the configured
+// window silently under-collects.
+func TestTokenChunkerDelimiterWindowUsesChildTokenCounts(t *testing.T) {
+ component, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{"`|`"},
+ "children_delimiters": []string{". "},
+ "chunk_token_size": 512,
+ "table_context_size": 7,
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "fig.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "gamma delta. alpha beta.", "doc_type_kwd": "text"},
+ {"text": "", "doc_type_kwd": "table"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ var media map[string]any
+ for _, ck := range outputChunks(t, out) {
+ if ck["ck_type"] == "table" {
+ media = ck
+ }
+ }
+ if media == nil {
+ t.Fatalf("table chunk missing: %+v", out)
+ }
+ // Each child is 3 tokens, so the 7-token window holds both of them.
+ assertMaterializedMediaContext(t, media, "gamma deltaalpha beta.")
+}
+
+// TestMaterializeMediaContextKeepsTokenCountInSync pins the invariant the fold
+// must preserve: once the context is folded into the body, TKNums describes
+// that body rather than the media payload it replaced. The count is read as a
+// budget by the merge and window walks and emitted as tk_nums on the chunk.
+func TestMaterializeMediaContextKeepsTokenCountInSync(t *testing.T) {
+ const body = "<table><tr><td>A</td></tr></table>"
+ ck := schema.ChunkDoc{
+ Text: body,
+ DocType: "table",
+ CKType: "table",
+ TKNums: intPtr(tokenizeStr(body)),
+ ContextAbove: "above",
+ ContextBelow: "below",
+ }
+
+ got := materializeMediaContext(ck)
+ if want := "above" + body + "below"; got.Text != want {
+ t.Fatalf("folded text = %q, want %q", got.Text, want)
+ }
+ if want := tokenizeStr(got.Text); intValue(got.TKNums) != want {
+ t.Errorf("tk_nums = %d, want %d (the folded body)", intValue(got.TKNums), want)
+ }
+}
+
+// TestTokenChunkerWindowTruncatesOnSentenceBoundary pins that a neighbour
+// larger than the remaining window is trimmed on a sentence boundary. A cut
+// inside the sentence would hand the index a fragment: the removed rune-level
+// helper returned "zeta." for this fixture, a piece of "epsilon zeta.".
+func TestTokenChunkerWindowTruncatesOnSentenceBoundary(t *testing.T) {
+ component, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "chunk_token_size": 512,
+ "table_context_size": 3,
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "fig.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "epsilon zeta.", "doc_type_kwd": "text"},
+ {"text": "", "doc_type_kwd": "table"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ var media map[string]any
+ for _, ck := range outputChunks(t, out) {
+ if ck["ck_type"] == "table" {
+ media = ck
+ }
+ }
+ if media == nil {
+ t.Fatalf("table chunk missing: %+v", out)
+ }
+ // "epsilon zeta." is one sentence of 4 tokens, so the 3-token window takes
+ // it whole rather than cutting into it.
+ assertMaterializedMediaContext(t, media, "epsilon zeta.")
+}
+
// TestTokenChunker_InvokeDeterministic runs a 20-item structured
// payload 10 times under the race detector and asserts the chunk
// list is identical every time.
@@ -988,6 +1093,50 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {
}
}
+// TestSplitByChildrenRecomputesTokenCounts pins that a child carries the count
+// of its own text. The count is a budget input for the media window in the
+// delimiter branch, where the attach runs after this split.
+func TestSplitByChildrenRecomputesTokenCounts(t *testing.T) {
+ const parentText = "gamma delta. alpha beta."
+ children := splitByChildren([]schema.ChunkDoc{{
+ Text: parentText,
+ DocType: "text",
+ CKType: "text",
+ TKNums: intPtr(tokenizeStr(parentText)),
+ }}, regexp.MustCompile(`\. `))
+
+ if len(children) != 2 {
+ t.Fatalf("children = %d, want 2", len(children))
+ }
+ for _, child := range children {
+ if got, want := intValue(child.TKNums), tokenizeStr(child.Text); got != want {
+ t.Errorf("child %q tk_nums = %d, want %d (its own text)", child.Text, got, want)
+ }
+ }
+}
+
+// TestApplyChildrenDelimText_RecomputesTokenCounts pins that the text-path
+// children carry their own count, so a consumer that budgets with TKNums does
+// not read a stale or missing value.
+func TestApplyChildrenDelimText_RecomputesTokenCounts(t *testing.T) {
+ const parentText = "gamma delta. alpha beta."
+ out := applyChildrenDelimText([]schema.ChunkDoc{{
+ Text: parentText,
+ DocType: "text",
+ CKType: "text",
+ TKNums: intPtr(tokenizeStr(parentText)),
+ }}, regexp.MustCompile(`\. `))
+
+ if len(out) != 2 {
+ t.Fatalf("children = %d, want 2", len(out))
+ }
+ for _, child := range out {
+ if got, want := intValue(child.TKNums), tokenizeStr(child.Text); got != want {
+ t.Errorf("child %q tk_nums = %d, want %d (its own text)", child.Text, got, want)
+ }
+ }
+}
+
// TestApplyChildrenDelimText_DefaultsMomToCurrentChunk verifies that when an
// incoming chunk has no Mom, the children fall back to the current chunk's
// text (the historical behaviour preserved by the fix for #17876).