RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): one BPE pass per hardSplitPiece iteration instead of two (#18648)
changshenhan78d426abda2c
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 22f6ff132d..040f874563 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -83,7 +83,6 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema"- "ragflow/internal/tokenizer""ragflow/internal/parser/chunk")@@ -1325,9 +1324,18 @@ func splitSentencesLossless(text string) []string {func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {var out []schema.ChunkDocrest := text- for tokenizeStr(rest) > target {- head := tokenizer.TrimContentToTokenLimit(rest, target)- if head == "" || head == rest {+ for {+ // One BPE pass per iteration instead of two: trimToTokenLimit (the+ // package-level tokenizer seam, see title_cap.go) returns rest+ // unchanged exactly when rest fits within target tokens, so its+ // result doubles as the loop-exit condition. The shrinking remainder+ // is still re-encoded from scratch each round — one encode per+ // iteration, not two.+ head := trimToTokenLimit(rest, target)+ if head == rest {+ break // rest already fits; the trailing append emits it+ }+ if head == "" {break // cannot shrink further; avoid an infinite loop}// TrimContentToTokenLimit decodes a token prefix; if that lands on adiff --git a/internal/ingestion/component/chunker/token_encoder_missing_test.go b/internal/ingestion/component/chunker/token_encoder_missing_test.gonew file mode 100644index 0000000000..7e01566ab0--- /dev/null+++ b/internal/ingestion/component/chunker/token_encoder_missing_test.go@@ -0,0 +1,73 @@+package chunker++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// byteTrimStub reproduces tokenizer.TrimContentToTokenLimit's encoder-missing+// fallback (UTF-8-safe byte-length trim) verbatim, minus the global encoder+// lookup. Swapping it into the trimToTokenLimit seam pins the encoder-missing+// path of hardSplitPiece deterministically: no global tiktoken state is+// touched, so the test is order-independent in full-package runs (tiktoken-go+// caches the loaded encoding in a package-level map that no reset can evict).+func byteTrimStub(s string, limit int) string {+ if limit < 0 {+ limit = 0+ }+ if limit <= 0 {+ return ""+ }+ b := []byte(s)+ if len(b) <= limit {+ return s+ }+ for limit > 0 && !utf8.Valid(b[:limit]) {+ limit--+ }+ return string(b[:limit])+}++// TestHardSplitPiece_EncoderMissingFallback pins the encoder-missing fallback+// path of hardSplitPiece. With the encoder unavailable, trimToTokenLimit falls+// back to a UTF-8-safe byte-length trim. The pre-optimization loop guard+// (tokenizeStr(rest) > target) never fired in that state, so the whole+// over-limit remainder was emitted as ONE piece; the current loop drives off+// the trim result and keeps emitting bounded pieces instead — strictly more+// correct, and the behavior this test locks in so the fallback stays a+// first-class, tested path.+func TestHardSplitPiece_EncoderMissingFallback(t *testing.T) {+ orig := trimToTokenLimit+ trimToTokenLimit = byteTrimStub+ t.Cleanup(func() { trimToTokenLimit = orig })++ // Pure ASCII+CJK text with no @@...## tags: adjustCutPastTag extends a cut+ // past a tag when tokenizeStr(head) <= target, which a dead encoder makes+ // always true and would swallow the byte-bounded shape under test.+ text := hardSplitBenchText()+ const target = 64++ got := hardSplitPiece(text, "", target)++ // The fallback must emit bounded pieces, not one giant remainder (the old+ // guard-never-fires behavior).+ if len(got) < 2 {+ t.Fatalf("encoder-missing fallback produced %d piece(s); want bounded pieces (>1)", len(got))+ }+ // Lossless: the pieces concatenate back to the input exactly, each within+ // the byte budget and valid UTF-8 (the byte-trim fallback's guarantees).+ var sb strings.Builder+ for i, p := range got {+ if len(p.Text) > target {+ t.Fatalf("piece %d is %d bytes; byte-trim fallback must keep pieces <= %d bytes", i, len(p.Text), target)+ }+ if !utf8.ValidString(p.Text) {+ t.Fatalf("piece %d is not valid UTF-8", i)+ }+ sb.WriteString(p.Text)+ }+ if sb.String() != text {+ t.Fatalf("pieces do not concatenate back to the input (got %d bytes, want %d)", sb.Len(), len(text))+ }+}diff --git a/internal/ingestion/component/chunker/token_helpers_test.go b/internal/ingestion/component/chunker/token_helpers_test.gonew file mode 100644index 0000000000..d0acc36623--- /dev/null+++ b/internal/ingestion/component/chunker/token_helpers_test.go@@ -0,0 +1,22 @@+package chunker++import (+ "strings"+)++// hardSplitBenchText builds a realistic oversized boundary-less run: a long+// unbroken sequence without sentence delimiters, mixing CJK and ASCII like a+// dense paragraph of a scanned/OCR'd document. This is exactly the input that+// hardSplitPiece must token-split, and the reason the function exists. Sized+// so a single hardSplitPiece call takes on the order of milliseconds: large+// enough for the per-iteration encode work to dominate a benchmark, small+// enough that every test sharing it stays fast. Shared by benchmarks and+// fallback-path tests alike.+func hardSplitBenchText() string {+ var b strings.Builder+ const sentence = "企业知识库检索系统需要高效处理海量非结构化文档的解析切分与向量化入库RAGFlow整合了文档理解文本切分混合检索与重排等多个环节Knowledgebase ingestion splits long passages into token-bounded chunks for embedding. "+ for b.Len() < 12_000 {+ b.WriteString(sentence)+ }+ return b.String()+}diff --git a/internal/ingestion/component/chunker/token_invariant_test.go b/internal/ingestion/component/chunker/token_invariant_test.gonew file mode 100644index 0000000000..ebe4806913--- /dev/null+++ b/internal/ingestion/component/chunker/token_invariant_test.go@@ -0,0 +1,80 @@+package chunker++import (+ "strings"+ "testing"+ "unicode/utf8"+)++// tagSlackTokens bounds how far past target a piece may legitimately go when+// adjustCutPastTag swallows a leading @@...## coordinate tag whole instead of+// splitting it: the trim result is within target, and the extension past the+// closing "##" adds at most the tag's own tokens (corpus tags measure <= 6).+const tagSlackTokens = 16++// TestHardSplitPieceInvariants pins the output contract of hardSplitPiece that+// callers rely on. It replaces the retired differential-against-reference+// harness: a second implementation of the same algorithm is a permanent+// maintenance liability, while these invariants hold for any correct+// token-splitting of this shape.+//+// 1. Bounded pieces: each piece is at most target tokens plus the documented+// one-token drift a re-encoded continuation piece can carry (cl100k is+// byte-level; a cut trimmed off a mid-multibyte boundary is re-absorbed by+// the next piece). Inputs without coordinate tags get the strict bound;+// tag inputs additionally allow the leading-tag-whole extension slack.+// 2. Lossless at the byte level: the pieces concatenate back to the input+// exactly. Pieces are byte-prefixes, not rune-prefixes — a cut may land+// mid-multibyte and leave raw continuation bytes at a piece's tail+// (observed with emoji/CJK corpora), repaired by the next piece's head;+// the downstream JSON layer replaces any residual invalid tail.+// 3. Monotone progress: pieces are non-empty and each is a true byte prefix+// of the not-yet-emitted remainder — the split never revisits or reorders+// bytes.+func TestHardSplitPieceInvariants(t *testing.T) {+ inputs := []string{ // plain, CJK, coordinate tags, emoji, degenerate+ strings.Repeat("boundary-less run of text with no sentence delimiter ", 60),+ strings.Repeat("中文中文中文无句读", 200),+ strings.Repeat("prelude @@12,34## body @@56,78## tail ", 80),+ strings.Repeat("mid-char🎉emoji🎉run-", 80),+ strings.Repeat("a", 600),+ "single-token",+ "",+ "\n\n\n",+ }+ for _, target := range []int{32, 64, 128, 512} {+ for _, in := range inputs {+ hasTags := strings.Contains(in, "@@")+ got := hardSplitPiece(in, "", target)++ var joined strings.Builder+ progress := 0+ for i, p := range got {+ if p.Text == "" {+ t.Fatalf("target=%d in=%d bytes: piece %d is empty", target, len(in), i)+ }+ if !strings.HasPrefix(in[progress:], p.Text) {+ t.Fatalf("target=%d in=%d bytes: piece %d is not a prefix of the remainder (monotone progress broken)", target, len(in), i)+ }+ progress += len(p.Text)++ // +utf8.UTFMax: a cut landing mid-multibyte leaves up to+ // UTFMax-1 dangling raw bytes at the tail, which re-encode as+ // individual single-byte tokens (cl100k is byte-level), plus+ // the one-token continuation drift.+ bound := target + utf8.UTFMax+ if hasTags {+ bound += tagSlackTokens+ }+ if n := tokenizeStr(p.Text); n > bound {+ t.Fatalf("target=%d in=%d bytes: piece %d has %d tokens; bound is %d", target, len(in), i, n, bound)+ }+ joined.WriteString(p.Text)+ }+ if joined.String() != in {+ t.Fatalf("target=%d in=%d bytes: pieces do not reproduce input (%d != %d bytes)",+ target, len(in), joined.Len(), len(in))+ }+ }+ }+}diff --git a/internal/ingestion/component/chunker/token_perf_test.go b/internal/ingestion/component/chunker/token_perf_test.gonew file mode 100644index 0000000000..775bc57f3d--- /dev/null+++ b/internal/ingestion/component/chunker/token_perf_test.go@@ -0,0 +1,32 @@+package chunker++import (+ "strings"+ "testing"+)++// benchHardSplit runs hardSplitPiece and reports both the elapsed time and the+// number of produced pieces, so the benchmark stays honest about output size+// (a regression that halves the pieces would mask a speedup).+func benchHardSplit(b *testing.B, text string) {+ b.Helper()+ var pieces int+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ pieces = len(hardSplitPiece(text, "", 512))+ }+ b.ReportMetric(float64(pieces), "pieces/op")+}++func BenchmarkHardSplitPiece_Text(b *testing.B) {+ text := hardSplitBenchText()+ if n := tokenizeStr(text); n <= 512 {+ b.Fatalf("benchmark input only has %d tokens; it must exceed the 512-token target", n)+ }+ benchHardSplit(b, text)+}++func BenchmarkHardSplitPiece_CJK(b *testing.B) {+ text := strings.Repeat("无句读长文知识库检索切分处理效率优化", 250)+ benchHardSplit(b, text)+}