RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
perf(chunker): make hardSplitPiece O(L) with a single-encode token walk (#19712)
changshenhan598e7297eb63
diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.goindex b049c66160..e97ae6e22e 100644--- a/internal/ingestion/component/chunker/title_cap.go+++ b/internal/ingestion/component/chunker/title_cap.go@@ -33,13 +33,18 @@ import (// realNumTokens / realTrimToTokenLimit are the production tokenizer functions.// They are referenced through package-level vars (numTokens /-// trimToTokenLimit) so tests can swap in a deterministic stub.+// trimToTokenLimit, and encodeTokens / decodeTokens for the raw token-id+// API the O(L) hard-split walk drives on) so tests can swap in a+// deterministic stub.var (realNumTokens = tokenizer.NumTokensFromStringrealTrimToTokenLimit = tokenizer.TrimContentToTokenLimitnumTokens = realNumTokenstrimToTokenLimit = realTrimToTokenLimit++ encodeTokens = tokenizer.EncodeCL100KTokens+ decodeTokens = tokenizer.DecodeCL100KTokens)// titleTokenCount counts tokens for text. tokenizer.NumTokensFromString nowdiff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex f0b2a30699..e4e65ae43e 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -1399,12 +1399,123 @@ func splitSentencesLossless(text string) []string {}// hardSplitPiece hard token-splits a boundary-less run into <= target pieces.-// Each piece is a true character prefix of the remaining text (U+FFFD tail-// trimmed), so the concatenated pieces reproduce the input exactly. A token-// cut never lands inside a @@...## coordinate tag: the cut is extended past-// the closing "##" when the budget allows, otherwise backed off to before the-// "@@" — so a tag is never split across two pieces.+// The run is encoded once and the token array walked in target-sized steps,+// each slice decoded straight back to its exact byte range — O(L) overall,+// where the former loop re-encoded a shrinking remainder every round and went+// quadratic on long boundary-less inputs. Pieces concatenate back to the input+// exactly, and every cut lands on an original token boundary, so a piece is a+// true byte prefix of the remainder (a cut may leave raw UTF-8 continuation+// bytes at the tail when the boundary splits a multibyte rune; the next+// piece's head repairs them). A token cut never lands inside a @@...##+// coordinate tag: the cut is extended past the closing "##" when the budget+// allows, otherwise backed off to before the "@@" — so a tag is never split+// across two pieces.+//+// When the encoder is unavailable, hardSplitPieceByteLoop (the former+// per-round-trim loop) takes over and degrades to byte-length trimming.func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {+ if text == "" {+ return nil+ }+ tokens, ok := encodeTokens(text)+ if !ok {+ return hardSplitPieceByteLoop(text, docType, target)+ }+ if target <= 0 || len(tokens) <= target {+ // target <= 0 is unreachable from validated configs; the byte loop+ // would emit the whole text as one piece, so mirror that here.+ return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text)), CKType: "text"}}+ }++ // offsets[i] = byte length of the first i tokens decoded. Token ids repeat+ // heavily in natural text, so per-id byte lengths are memoized.+ offsets := make([]int, len(tokens)+1)+ lens := make(map[int]int, len(tokens)/2)+ for i, tok := range tokens {+ l, seen := lens[tok]+ if !seen {+ l = len(decodeTokens(tokens[i : i+1]))+ lens[tok] = l+ }+ offsets[i+1] = offsets[i] + l+ }+ if offsets[len(tokens)] != len(text) {+ // Unreachable with a byte-level BPE (decode∘encode is the identity on+ // the byte string); a mismatch here would corrupt every piece, so+ // degrade to the byte loop instead of emitting it.+ return hardSplitPieceByteLoop(text, docType, target)+ }++ // @@...## tag spans in byte offsets. The leftmost-first scan with the+ // resume index mirrors adjustCutPastTag's paired-tag search; spans come+ // out sorted and non-overlapping.+ type tagSpan struct{ start, end int }+ var tags []tagSpan+ for i := 0; i < len(text); {+ s := strings.Index(text[i:], "@@")+ if s < 0 {+ break+ }+ s += i+ eRel := strings.Index(text[s+2:], "##")+ if eRel < 0 {+ break+ }+ tags = append(tags, tagSpan{s, s + 2 + eRel + 2})+ i = tags[len(tags)-1].end+ }++ var out []schema.ChunkDoc+ p, tagIdx := 0, 0+ for p < len(tokens) {+ q := p + target+ if q > len(tokens) {+ q = len(tokens)+ }+ // Consume tags that end at or before the cut; they can no longer+ // matter. A tag the cut lands inside is resolved below and+ // deliberately not consumed: after a back-off the next cut can+ // re-enter the same tag.+ for tagIdx < len(tags) && tags[tagIdx].end <= offsets[q] {+ tagIdx+++ }+ if tagIdx < len(tags) && tags[tagIdx].start < offsets[q] && offsets[q] < tags[tagIdx].end {+ tag := tags[tagIdx]+ endTok := q+ for endTok < len(tokens) && offsets[endTok] < tag.end {+ endTok+++ }+ startTok := q+ for startTok > p && offsets[startTok] > tag.start {+ startTok--+ }+ // Extend past the tag when the piece stays within budget, or when+ // the tag starts the piece and backing off would stall; otherwise+ // back off to the last token boundary at or before the "@@".+ // startTok == p exactly when the piece begins at (or inside) the+ // tag, so the forced extension of adjustCutPastTag's leading-tag+ // case is preserved.+ if endTok-p <= target || startTok == p {+ q = endTok+ } else {+ q = startTok+ }+ }+ if q <= p {+ q = p + 1 // never stall (defensive; unreachable above)+ }+ piece := text[offsets[p]:offsets[q]]+ out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece)), CKType: "text"})+ p = q+ }+ return out+}++// hardSplitPieceByteLoop is the pre-O(L) body of hardSplitPiece, kept as the+// encoder-unavailable fallback. With the encoder dead, trimToTokenLimit+// degrades to a UTF-8-safe byte-length trim and the loop still emits bounded,+// lossless pieces.+func hardSplitPieceByteLoop(text string, docType string, target int) []schema.ChunkDoc {var out []schema.ChunkDocrest := textfor {diff --git a/internal/ingestion/component/chunker/token_encoder_missing_test.go b/internal/ingestion/component/chunker/token_encoder_missing_test.goindex 7e01566ab0..4908b12e40 100644--- a/internal/ingestion/component/chunker/token_encoder_missing_test.go+++ b/internal/ingestion/component/chunker/token_encoder_missing_test.go@@ -8,10 +8,10 @@ import (// byteTrimStub reproduces tokenizer.TrimContentToTokenLimit's encoder-missing// fallback (UTF-8-safe byte-length trim) verbatim, minus the global encoder-// lookup. Swapping it into the trimToTokenLimit seam pins the encoder-missing-// path of hardSplitPiece deterministically: no global tiktoken state is-// touched, so the test is order-independent in full-package runs (tiktoken-go-// caches the loaded encoding in a package-level map that no reset can evict).+// lookup. Swapping it into the trimToTokenLimit seam pins the byte loop+// deterministically: no global tiktoken state is touched, so the test is+// order-independent in full-package runs (tiktoken-go caches the loaded+// encoding in a package-level map that no reset can evict).func byteTrimStub(s string, limit int) string {if limit < 0 {limit = 0@@ -29,22 +29,29 @@ func byteTrimStub(s string, limit int) string {return string(b[:limit])}+// deadEncodeStub reports the encoder as unavailable, the same degraded world+// production hits when InitCL100KEncoder failed, without touching any global+// tiktoken state.+func deadEncodeStub(string) ([]int, bool) { return nil, false }+// TestHardSplitPiece_EncoderMissingFallback pins the encoder-missing fallback-// path of hardSplitPiece. With the encoder unavailable, trimToTokenLimit falls-// back to a UTF-8-safe byte-length trim. The pre-optimization loop guard+// path of hardSplitPiece. With the encoder unavailable, the token-space walk+// is skipped and the byte loop drives off trimToTokenLimit, which falls back+// to a UTF-8-safe byte-length trim. The pre-optimization loop guard// (tokenizeStr(rest) > target) never fired in that state, so the whole// over-limit remainder was emitted as ONE piece; the current loop drives off// the trim result and keeps emitting bounded pieces instead — strictly more// correct, and the behavior this test locks in so the fallback stays a// first-class, tested path.func TestHardSplitPiece_EncoderMissingFallback(t *testing.T) {- orig := trimToTokenLimit- trimToTokenLimit = byteTrimStub- t.Cleanup(func() { trimToTokenLimit = orig })+ origEncode, origTrim := encodeTokens, trimToTokenLimit+ encodeTokens, trimToTokenLimit = deadEncodeStub, byteTrimStub+ t.Cleanup(func() { encodeTokens, trimToTokenLimit = origEncode, origTrim })- // Pure ASCII+CJK text with no @@...## tags: adjustCutPastTag extends a cut- // past a tag when tokenizeStr(head) <= target, which a dead encoder makes- // always true and would swallow the byte-bounded shape under test.+ // Pure ASCII+CJK text with no @@...## tags: the byte loop's+ // adjustCutPastTag extends a cut past a tag when tokenizeStr(head) <=+ // target, which a dead encoder makes always true and would swallow the+ // byte-bounded shape under test.text := hardSplitBenchText()const target = 64diff --git a/internal/ingestion/component/chunker/token_invariant_test.go b/internal/ingestion/component/chunker/token_invariant_test.goindex ebe4806913..a37e5841d3 100644--- a/internal/ingestion/component/chunker/token_invariant_test.go+++ b/internal/ingestion/component/chunker/token_invariant_test.go@@ -3,13 +3,12 @@ package chunkerimport ("strings""testing"- "unicode/utf8")-// tagSlackTokens bounds how far past target a piece may legitimately go when-// adjustCutPastTag swallows a leading @@...## coordinate tag whole instead of-// splitting it: the trim result is within target, and the extension past the-// closing "##" adds at most the tag's own tokens (corpus tags measure <= 6).+// tagSlackTokens bounds how far past target a piece may legitimately go when a+// leading @@...## coordinate tag is swallowed whole instead of split: the+// extension past the closing "##" adds at most the tag's own tokens (corpus+// tags measure <= 6).const tagSlackTokens = 16// TestHardSplitPieceInvariants pins the output contract of hardSplitPiece that@@ -18,11 +17,12 @@ const tagSlackTokens = 16// maintenance liability, while these invariants hold for any correct// token-splitting of this shape.//-// 1. Bounded pieces: each piece is at most target tokens plus the documented-// one-token drift a re-encoded continuation piece can carry (cl100k is-// byte-level; a cut trimmed off a mid-multibyte boundary is re-absorbed by-// the next piece). Inputs without coordinate tags get the strict bound;-// tag inputs additionally allow the leading-tag-whole extension slack.+// 1. Bounded pieces: every cut lands on an original token boundary and pieces+// are at most target original tokens (plus the leading-tag-whole+// extension slack for tag inputs). Re-encoded piece token counts can+// overshoot only at the two boundary tokens, whose BPE merges depended on+// now-removed neighbors; the test self-calibrates that overshoot from the+// longest token in the input instead of hardcoding a vocabulary fact.// 2. Lossless at the byte level: the pieces concatenate back to the input// exactly. Pieces are byte-prefixes, not rune-prefixes — a cut may land// mid-multibyte and leave raw continuation bytes at a piece's tail@@ -31,6 +31,8 @@ const tagSlackTokens = 16// 3. Monotone progress: pieces are non-empty and each is a true byte prefix// of the not-yet-emitted remainder — the split never revisits or reorders// bytes.+// 4. Tags stay whole: every complete @@...## span lies entirely inside one+// piece — never split across a piece boundary.func TestHardSplitPieceInvariants(t *testing.T) {inputs := []string{ // plain, CJK, coordinate tags, emoji, degeneratestrings.Repeat("boundary-less run of text with no sentence delimiter ", 60),@@ -47,6 +49,17 @@ func TestHardSplitPieceInvariants(t *testing.T) {hasTags := strings.Contains(in, "@@")got := hardSplitPiece(in, "", target)+ // Longest single token in the input bounds the re-encode overshoot+ // at a piece's two boundary tokens (see invariant 1).+ maxTokBytes := 1+ if toks, ok := encodeTokens(in); ok {+ for i := range toks {+ if n := len(decodeTokens(toks[i : i+1])); n > maxTokBytes {+ maxTokBytes = n+ }+ }+ }+var joined strings.Builderprogress := 0for i, p := range got {@@ -58,11 +71,11 @@ func TestHardSplitPieceInvariants(t *testing.T) {}progress += len(p.Text)- // +utf8.UTFMax: a cut landing mid-multibyte leaves up to- // UTFMax-1 dangling raw bytes at the tail, which re-encode as- // individual single-byte tokens (cl100k is byte-level), plus- // the one-token continuation drift.- bound := target + utf8.UTFMax+ // +2*(maxTokBytes-1): the piece's first and last tokens were+ // BPE-merged with neighbors outside the slice; re-encoded+ // alone they can decompose into up to maxTokBytes single-byte+ // tokens each.+ bound := target + 2*(maxTokBytes-1)if hasTags {bound += tagSlackTokens}@@ -75,6 +88,33 @@ func TestHardSplitPieceInvariants(t *testing.T) {t.Fatalf("target=%d in=%d bytes: pieces do not reproduce input (%d != %d bytes)",target, len(in), joined.Len(), len(in))}++ // Every complete @@...## span sits inside a single piece.+ for i := 0; i < len(in); {+ sRel := strings.Index(in[i:], "@@")+ if sRel < 0 {+ break+ }+ s := i + sRel+ eRel := strings.Index(in[s+2:], "##")+ if eRel < 0 {+ break+ }+ e := s + 2 + eRel + 2+ whole := false+ pos := 0+ for _, p := range got {+ if pos <= s && e <= pos+len(p.Text) {+ whole = true+ break+ }+ pos += len(p.Text)+ }+ if !whole {+ t.Fatalf("target=%d in=%d bytes: tag span [%d,%d) is split across pieces", target, len(in), s, e)+ }+ i = e+ }}}}diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.goindex b85a9963c4..095bdca655 100644--- a/internal/tokenizer/tokenizer.go+++ b/internal/tokenizer/tokenizer.go@@ -644,3 +644,30 @@ func TrimContentToTokenLimit(s string, limit int) string {}return enc.Decode(tokens[:limit])}++// EncodeCL100KTokens returns the cl100k_base token ids of s, from the same+// cached encoder NumTokensFromString and TrimContentToTokenLimit use. ok is+// false when the encoder is unavailable — the same degraded world where those+// two degrade — so callers can run their own fallback instead of mistaking an+// empty result for "zero tokens".+func EncodeCL100KTokens(s string) (tokens []int, ok bool) {+ enc, err := getCL100KEncoder()+ if err != nil || enc == nil {+ return nil, false+ }+ return enc.Encode(s, nil, nil), true+}++// DecodeCL100KTokens concatenates the raw byte sequences of tokens. Decoding+// is a plain vocabulary-table concat (no re-segmentation), so+// Decode(Encode(s)) == s and Decode(tokens[a:b]) is exactly the corresponding+// byte slice of s — including a slice whose ends cut a multibyte rune, which+// comes back as raw continuation bytes rather than U+FFFD. Returns "" when+// the encoder is unavailable (callers gate on EncodeCL100KTokens first).+func DecodeCL100KTokens(tokens []int) string {+ enc, err := getCL100KEncoder()+ if err != nil || enc == nil {+ return ""+ }+ return enc.Decode(tokens)+}