Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

perf(chunker): make hardSplitPiece O(L) with a single-encode token walk (#19712)

changshenhan598e7297eb63
diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.go
index b049c66160..e97ae6e22e 100644
--- a/internal/ingestion/component/chunker/title_cap.go
+++ b/internal/ingestion/component/chunker/title_cap.go
@@ -33,13 +33,18 @@ import (
// realNumTokens / realTrimToTokenLimit are the production tokenizer functions.
// They are referenced through package-level vars (numTokens /
-// trimToTokenLimit) so tests can swap in a deterministic stub.
+// trimToTokenLimit, and encodeTokens / decodeTokens for the raw token-id
+// API the O(L) hard-split walk drives on) so tests can swap in a
+// deterministic stub.
var (
realNumTokens = tokenizer.NumTokensFromString
realTrimToTokenLimit = tokenizer.TrimContentToTokenLimit
numTokens = realNumTokens
trimToTokenLimit = realTrimToTokenLimit
+
+ encodeTokens = tokenizer.EncodeCL100KTokens
+ decodeTokens = tokenizer.DecodeCL100KTokens
)
// titleTokenCount counts tokens for text. tokenizer.NumTokensFromString now
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index f0b2a30699..e4e65ae43e 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -1399,12 +1399,123 @@ func splitSentencesLossless(text string) []string {
}
// hardSplitPiece hard token-splits a boundary-less run into <= target pieces.
-// Each piece is a true character prefix of the remaining text (U+FFFD tail
-// trimmed), so the concatenated pieces reproduce the input exactly. A token
-// cut never lands inside a @@...## coordinate tag: the cut is extended past
-// the closing "##" when the budget allows, otherwise backed off to before the
-// "@@" — so a tag is never split across two pieces.
+// The run is encoded once and the token array walked in target-sized steps,
+// each slice decoded straight back to its exact byte range — O(L) overall,
+// where the former loop re-encoded a shrinking remainder every round and went
+// quadratic on long boundary-less inputs. Pieces concatenate back to the input
+// exactly, and every cut lands on an original token boundary, so a piece is a
+// true byte prefix of the remainder (a cut may leave raw UTF-8 continuation
+// bytes at the tail when the boundary splits a multibyte rune; the next
+// piece's head repairs them). A token cut never lands inside a @@...##
+// coordinate tag: the cut is extended past the closing "##" when the budget
+// allows, otherwise backed off to before the "@@" — so a tag is never split
+// across two pieces.
+//
+// When the encoder is unavailable, hardSplitPieceByteLoop (the former
+// per-round-trim loop) takes over and degrades to byte-length trimming.
func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {
+ if text == "" {
+ return nil
+ }
+ tokens, ok := encodeTokens(text)
+ if !ok {
+ return hardSplitPieceByteLoop(text, docType, target)
+ }
+ if target <= 0 || len(tokens) <= target {
+ // target <= 0 is unreachable from validated configs; the byte loop
+ // would emit the whole text as one piece, so mirror that here.
+ return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text)), CKType: "text"}}
+ }
+
+ // offsets[i] = byte length of the first i tokens decoded. Token ids repeat
+ // heavily in natural text, so per-id byte lengths are memoized.
+ offsets := make([]int, len(tokens)+1)
+ lens := make(map[int]int, len(tokens)/2)
+ for i, tok := range tokens {
+ l, seen := lens[tok]
+ if !seen {
+ l = len(decodeTokens(tokens[i : i+1]))
+ lens[tok] = l
+ }
+ offsets[i+1] = offsets[i] + l
+ }
+ if offsets[len(tokens)] != len(text) {
+ // Unreachable with a byte-level BPE (decode∘encode is the identity on
+ // the byte string); a mismatch here would corrupt every piece, so
+ // degrade to the byte loop instead of emitting it.
+ return hardSplitPieceByteLoop(text, docType, target)
+ }
+
+ // @@...## tag spans in byte offsets. The leftmost-first scan with the
+ // resume index mirrors adjustCutPastTag's paired-tag search; spans come
+ // out sorted and non-overlapping.
+ type tagSpan struct{ start, end int }
+ var tags []tagSpan
+ for i := 0; i < len(text); {
+ s := strings.Index(text[i:], "@@")
+ if s < 0 {
+ break
+ }
+ s += i
+ eRel := strings.Index(text[s+2:], "##")
+ if eRel < 0 {
+ break
+ }
+ tags = append(tags, tagSpan{s, s + 2 + eRel + 2})
+ i = tags[len(tags)-1].end
+ }
+
+ var out []schema.ChunkDoc
+ p, tagIdx := 0, 0
+ for p < len(tokens) {
+ q := p + target
+ if q > len(tokens) {
+ q = len(tokens)
+ }
+ // Consume tags that end at or before the cut; they can no longer
+ // matter. A tag the cut lands inside is resolved below and
+ // deliberately not consumed: after a back-off the next cut can
+ // re-enter the same tag.
+ for tagIdx < len(tags) && tags[tagIdx].end <= offsets[q] {
+ tagIdx++
+ }
+ if tagIdx < len(tags) && tags[tagIdx].start < offsets[q] && offsets[q] < tags[tagIdx].end {
+ tag := tags[tagIdx]
+ endTok := q
+ for endTok < len(tokens) && offsets[endTok] < tag.end {
+ endTok++
+ }
+ startTok := q
+ for startTok > p && offsets[startTok] > tag.start {
+ startTok--
+ }
+ // Extend past the tag when the piece stays within budget, or when
+ // the tag starts the piece and backing off would stall; otherwise
+ // back off to the last token boundary at or before the "@@".
+ // startTok == p exactly when the piece begins at (or inside) the
+ // tag, so the forced extension of adjustCutPastTag's leading-tag
+ // case is preserved.
+ if endTok-p <= target || startTok == p {
+ q = endTok
+ } else {
+ q = startTok
+ }
+ }
+ if q <= p {
+ q = p + 1 // never stall (defensive; unreachable above)
+ }
+ piece := text[offsets[p]:offsets[q]]
+ out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece)), CKType: "text"})
+ p = q
+ }
+ return out
+}
+
+// hardSplitPieceByteLoop is the pre-O(L) body of hardSplitPiece, kept as the
+// encoder-unavailable fallback. With the encoder dead, trimToTokenLimit
+// degrades to a UTF-8-safe byte-length trim and the loop still emits bounded,
+// lossless pieces.
+func hardSplitPieceByteLoop(text string, docType string, target int) []schema.ChunkDoc {
var out []schema.ChunkDoc
rest := text
for {
diff --git a/internal/ingestion/component/chunker/token_encoder_missing_test.go b/internal/ingestion/component/chunker/token_encoder_missing_test.go
index 7e01566ab0..4908b12e40 100644
--- a/internal/ingestion/component/chunker/token_encoder_missing_test.go
+++ b/internal/ingestion/component/chunker/token_encoder_missing_test.go
@@ -8,10 +8,10 @@ import (
// byteTrimStub reproduces tokenizer.TrimContentToTokenLimit's encoder-missing
// fallback (UTF-8-safe byte-length trim) verbatim, minus the global encoder
-// lookup. Swapping it into the trimToTokenLimit seam pins the encoder-missing
-// path of hardSplitPiece deterministically: no global tiktoken state is
-// touched, so the test is order-independent in full-package runs (tiktoken-go
-// caches the loaded encoding in a package-level map that no reset can evict).
+// lookup. Swapping it into the trimToTokenLimit seam pins the byte loop
+// deterministically: no global tiktoken state is touched, so the test is
+// order-independent in full-package runs (tiktoken-go caches the loaded
+// encoding in a package-level map that no reset can evict).
func byteTrimStub(s string, limit int) string {
if limit < 0 {
limit = 0
@@ -29,22 +29,29 @@ func byteTrimStub(s string, limit int) string {
return string(b[:limit])
}
+// deadEncodeStub reports the encoder as unavailable, the same degraded world
+// production hits when InitCL100KEncoder failed, without touching any global
+// tiktoken state.
+func deadEncodeStub(string) ([]int, bool) { return nil, false }
+
// TestHardSplitPiece_EncoderMissingFallback pins the encoder-missing fallback
-// path of hardSplitPiece. With the encoder unavailable, trimToTokenLimit falls
-// back to a UTF-8-safe byte-length trim. The pre-optimization loop guard
+// path of hardSplitPiece. With the encoder unavailable, the token-space walk
+// is skipped and the byte loop drives off trimToTokenLimit, which falls back
+// to a UTF-8-safe byte-length trim. The pre-optimization loop guard
// (tokenizeStr(rest) > target) never fired in that state, so the whole
// over-limit remainder was emitted as ONE piece; the current loop drives off
// the trim result and keeps emitting bounded pieces instead — strictly more
// correct, and the behavior this test locks in so the fallback stays a
// first-class, tested path.
func TestHardSplitPiece_EncoderMissingFallback(t *testing.T) {
- orig := trimToTokenLimit
- trimToTokenLimit = byteTrimStub
- t.Cleanup(func() { trimToTokenLimit = orig })
+ origEncode, origTrim := encodeTokens, trimToTokenLimit
+ encodeTokens, trimToTokenLimit = deadEncodeStub, byteTrimStub
+ t.Cleanup(func() { encodeTokens, trimToTokenLimit = origEncode, origTrim })
- // Pure ASCII+CJK text with no @@...## tags: adjustCutPastTag extends a cut
- // past a tag when tokenizeStr(head) <= target, which a dead encoder makes
- // always true and would swallow the byte-bounded shape under test.
+ // Pure ASCII+CJK text with no @@...## tags: the byte loop's
+ // adjustCutPastTag extends a cut past a tag when tokenizeStr(head) <=
+ // target, which a dead encoder makes always true and would swallow the
+ // byte-bounded shape under test.
text := hardSplitBenchText()
const target = 64
diff --git a/internal/ingestion/component/chunker/token_invariant_test.go b/internal/ingestion/component/chunker/token_invariant_test.go
index ebe4806913..a37e5841d3 100644
--- a/internal/ingestion/component/chunker/token_invariant_test.go
+++ b/internal/ingestion/component/chunker/token_invariant_test.go
@@ -3,13 +3,12 @@ package chunker
import (
"strings"
"testing"
- "unicode/utf8"
)
-// tagSlackTokens bounds how far past target a piece may legitimately go when
-// adjustCutPastTag swallows a leading @@...## coordinate tag whole instead of
-// splitting it: the trim result is within target, and the extension past the
-// closing "##" adds at most the tag's own tokens (corpus tags measure <= 6).
+// tagSlackTokens bounds how far past target a piece may legitimately go when a
+// leading @@...## coordinate tag is swallowed whole instead of split: the
+// extension past the closing "##" adds at most the tag's own tokens (corpus
+// tags measure <= 6).
const tagSlackTokens = 16
// TestHardSplitPieceInvariants pins the output contract of hardSplitPiece that
@@ -18,11 +17,12 @@ const tagSlackTokens = 16
// maintenance liability, while these invariants hold for any correct
// token-splitting of this shape.
//
-// 1. Bounded pieces: each piece is at most target tokens plus the documented
-// one-token drift a re-encoded continuation piece can carry (cl100k is
-// byte-level; a cut trimmed off a mid-multibyte boundary is re-absorbed by
-// the next piece). Inputs without coordinate tags get the strict bound;
-// tag inputs additionally allow the leading-tag-whole extension slack.
+// 1. Bounded pieces: every cut lands on an original token boundary and pieces
+// are at most target original tokens (plus the leading-tag-whole
+// extension slack for tag inputs). Re-encoded piece token counts can
+// overshoot only at the two boundary tokens, whose BPE merges depended on
+// now-removed neighbors; the test self-calibrates that overshoot from the
+// longest token in the input instead of hardcoding a vocabulary fact.
// 2. Lossless at the byte level: the pieces concatenate back to the input
// exactly. Pieces are byte-prefixes, not rune-prefixes — a cut may land
// mid-multibyte and leave raw continuation bytes at a piece's tail
@@ -31,6 +31,8 @@ const tagSlackTokens = 16
// 3. Monotone progress: pieces are non-empty and each is a true byte prefix
// of the not-yet-emitted remainder — the split never revisits or reorders
// bytes.
+// 4. Tags stay whole: every complete @@...## span lies entirely inside one
+// piece — never split across a piece boundary.
func TestHardSplitPieceInvariants(t *testing.T) {
inputs := []string{ // plain, CJK, coordinate tags, emoji, degenerate
strings.Repeat("boundary-less run of text with no sentence delimiter ", 60),
@@ -47,6 +49,17 @@ func TestHardSplitPieceInvariants(t *testing.T) {
hasTags := strings.Contains(in, "@@")
got := hardSplitPiece(in, "", target)
+ // Longest single token in the input bounds the re-encode overshoot
+ // at a piece's two boundary tokens (see invariant 1).
+ maxTokBytes := 1
+ if toks, ok := encodeTokens(in); ok {
+ for i := range toks {
+ if n := len(decodeTokens(toks[i : i+1])); n > maxTokBytes {
+ maxTokBytes = n
+ }
+ }
+ }
+
var joined strings.Builder
progress := 0
for i, p := range got {
@@ -58,11 +71,11 @@ func TestHardSplitPieceInvariants(t *testing.T) {
}
progress += len(p.Text)
- // +utf8.UTFMax: a cut landing mid-multibyte leaves up to
- // UTFMax-1 dangling raw bytes at the tail, which re-encode as
- // individual single-byte tokens (cl100k is byte-level), plus
- // the one-token continuation drift.
- bound := target + utf8.UTFMax
+ // +2*(maxTokBytes-1): the piece's first and last tokens were
+ // BPE-merged with neighbors outside the slice; re-encoded
+ // alone they can decompose into up to maxTokBytes single-byte
+ // tokens each.
+ bound := target + 2*(maxTokBytes-1)
if hasTags {
bound += tagSlackTokens
}
@@ -75,6 +88,33 @@ func TestHardSplitPieceInvariants(t *testing.T) {
t.Fatalf("target=%d in=%d bytes: pieces do not reproduce input (%d != %d bytes)",
target, len(in), joined.Len(), len(in))
}
+
+ // Every complete @@...## span sits inside a single piece.
+ for i := 0; i < len(in); {
+ sRel := strings.Index(in[i:], "@@")
+ if sRel < 0 {
+ break
+ }
+ s := i + sRel
+ eRel := strings.Index(in[s+2:], "##")
+ if eRel < 0 {
+ break
+ }
+ e := s + 2 + eRel + 2
+ whole := false
+ pos := 0
+ for _, p := range got {
+ if pos <= s && e <= pos+len(p.Text) {
+ whole = true
+ break
+ }
+ pos += len(p.Text)
+ }
+ if !whole {
+ t.Fatalf("target=%d in=%d bytes: tag span [%d,%d) is split across pieces", target, len(in), s, e)
+ }
+ i = e
+ }
}
}
}
diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go
index b85a9963c4..095bdca655 100644
--- a/internal/tokenizer/tokenizer.go
+++ b/internal/tokenizer/tokenizer.go
@@ -644,3 +644,30 @@ func TrimContentToTokenLimit(s string, limit int) string {
}
return enc.Decode(tokens[:limit])
}
+
+// EncodeCL100KTokens returns the cl100k_base token ids of s, from the same
+// cached encoder NumTokensFromString and TrimContentToTokenLimit use. ok is
+// false when the encoder is unavailable — the same degraded world where those
+// two degrade — so callers can run their own fallback instead of mistaking an
+// empty result for "zero tokens".
+func EncodeCL100KTokens(s string) (tokens []int, ok bool) {
+ enc, err := getCL100KEncoder()
+ if err != nil || enc == nil {
+ return nil, false
+ }
+ return enc.Encode(s, nil, nil), true
+}
+
+// DecodeCL100KTokens concatenates the raw byte sequences of tokens. Decoding
+// is a plain vocabulary-table concat (no re-segmentation), so
+// Decode(Encode(s)) == s and Decode(tokens[a:b]) is exactly the corresponding
+// byte slice of s — including a slice whose ends cut a multibyte rune, which
+// comes back as raw continuation bytes rather than U+FFFD. Returns "" when
+// the encoder is unavailable (callers gate on EncodeCL100KTokens first).
+func DecodeCL100KTokens(tokens []int) string {
+ enc, err := getCL100KEncoder()
+ if err != nil || enc == nil {
+ return ""
+ }
+ return enc.Decode(tokens)
+}