Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(title-chunker): enforce chunk_token_cap on the Go title-family chunks (#18455) (#18551)

Jackd76c15e2a5cb
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index 7a0572ec7e..fbfafaa7d3 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -120,18 +120,24 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
if len(records) == 0 {
return emptyOutputs(), nil
}
- return chunkFromRecords(parentCtx, db, inputs, p, records)
+ return chunkFromRecords(parentCtx, db, inputs, p, records, p.ChunkTokenCap)
}
// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over
// an already-extracted record list: resolve heading levels, split into
-// sections, merge adjacent text records, build chunks, and perform on-demand
-// PDF cropping. GroupTitleChunker feeds records in input order; ManualChunker
-// feeds them after a (page, top, left) resort. Sharing this body guarantees
-// both strategies emit byte-identical output for coordinate-free input (the
-// docx manual branch) — the no-regression contract locked by
-// TestManualChunker_NoPositionsEqualsGroupChunker.
-func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord) (map[string]any, error) {
+// sections, merge adjacent text records, build chunks, enforce the token cap
+// (title family only), and perform on-demand PDF cropping. GroupTitleChunker
+// feeds records in input order; ManualChunker feeds them after a (page, top,
+// left) resort. Sharing this body keeps both strategies byte-identical for
+// coordinate-free input as long as no built chunk exceeds the token cap
+// (TestManualChunker_NoPositionsEqualsGroupChunker); once one does, only
+// GroupTitleChunker re-splits it (ManualChunker is exempt).
+//
+// tokenCap is the title-family token ceiling (chunk_token_cap). ManualChunker
+// must stay exempt from #18455's cap, so it passes 0; GroupTitleChunker passes
+// p.ChunkTokenCap. The cap is applied right after build_chunks and BEFORE the
+// on-demand crop, mirroring Python's invoke() order (build -> cap -> set_chunks).
+func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord, tokenCap int) (map[string]any, error) {
ctx := newLevelContext(records, outlineFromInputs(inputs), p)
levels := ctx.Levels()
// Count heading level distribution for debugging.
@@ -169,6 +175,13 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]
zap.Int("groups", len(groups)),
)
chunks := buildChunksFromRecordGroups(groups, p, isPlainTextFormat(inputs))
+ // Enforce the title-family token ceiling (chunk_token_cap) right after
+ // build_chunks and BEFORE the on-demand crop, mirroring Python's
+ // invoke() order (build -> cap -> set_chunks). ManualChunker passes
+ // tokenCap=0, so this is a no-op there.
+ if tokenCap > 0 {
+ chunks = enforceTitleTokenCap(chunks, tokenCap)
+ }
common.Debug("chunker stage",
zap.String("component", "Chunker"),
zap.String("variant", "group"),
diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go
index c8dfd0d35c..a7ca007f40 100644
--- a/internal/ingestion/component/chunker/hierarchy.go
+++ b/internal/ingestion/component/chunker/hierarchy.go
@@ -241,6 +241,12 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
)
chunks := buildChunksFromRecordGroups(recordGroups, p, isPlainTextFormat(inputs))
+ // Enforce the title-family token ceiling (chunk_token_cap) right after
+ // build_chunks and BEFORE the on-demand crop, mirroring Python's
+ // invoke() order (build -> cap -> set_chunks).
+ if p.ChunkTokenCap > 0 {
+ chunks = enforceTitleTokenCap(chunks, p.ChunkTokenCap)
+ }
common.Debug("chunker stage",
zap.String("component", "Chunker"),
zap.String("variant", "hierarchy"),
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go
index b690230e85..d783f1932e 100644
--- a/internal/ingestion/component/chunker/manual.go
+++ b/internal/ingestion/component/chunker/manual.go
@@ -115,7 +115,9 @@ func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs
if hasPdfPositions(records) {
sortRecordsByPosition(records)
}
- return chunkFromRecords(ctx, db, inputs, &c.param, records)
+ // ManualChunker is exempt from the title-family token cap (#18455): it
+ // does not inherit BaseTitleChunker, so it always passes tokenCap=0.
+ return chunkFromRecords(ctx, db, inputs, &c.param, records, 0)
}
// hasPdfPositions reports whether any record carries a PDF coordinate matrix
diff --git a/internal/ingestion/component/chunker/sentence_boundary.go b/internal/ingestion/component/chunker/sentence_boundary.go
new file mode 100644
index 0000000000..1d0e654027
--- /dev/null
+++ b/internal/ingestion/component/chunker/sentence_boundary.go
@@ -0,0 +1,32 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except under the License.
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import "regexp"
+
+// sentenceBoundaryRe is the single sentence-boundary definition shared by the
+// title chunker family and the token chunker's JSON path. It mirrors Python's
+// rag/flow/chunker/_sentence_boundary.py:SENTENCE_BOUNDARY_RE exactly: the
+// Chinese/English period, exclamation mark, question mark, the Chinese
+// variants, the newline, and the English ". " (period + space) boundary. The
+// capturing group keeps the delimiter attached to the preceding sentence so
+// re-merged text preserves original boundaries.
+//
+// The token chunker's TEXT path (naive_merge port) uses a different delimiter
+// — sentenceDelimiter in token.go, mirroring Python naive_merge's production
+// delimiter "\n!?。;!?" (no ". "). The two constants mirror two distinct
+// Python constants and differ only in that English boundary, so they are
+// intentionally NOT merged into one.
+var sentenceBoundaryRe = regexp.MustCompile(`([。!??;!\n]|\. )`)
diff --git a/internal/ingestion/component/chunker/sentence_boundary_test.go b/internal/ingestion/component/chunker/sentence_boundary_test.go
new file mode 100644
index 0000000000..a8da922107
--- /dev/null
+++ b/internal/ingestion/component/chunker/sentence_boundary_test.go
@@ -0,0 +1,58 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except under the License.
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import "testing"
+
+// TestSentenceBoundary_SharedRegexMatchesPython pins sentenceBoundaryRe to the
+// exact Python SENTENCE_BOUNDARY_RE (rag/flow/chunker/_sentence_boundary.py),
+// including the English ". " boundary used by the title chunker and the token
+// chunker's JSON path.
+func TestSentenceBoundary_SharedRegexMatchesPython(t *testing.T) {
+ const want = `([。!??;!\n]|\. )`
+ if got := sentenceBoundaryRe.String(); got != want {
+ t.Errorf("sentenceBoundaryRe = %q, want %q (Python SENTENCE_BOUNDARY_RE)", got, want)
+ }
+ // English ". " boundary must be recognized.
+ if !sentenceBoundaryRe.MatchString("Hello. World") {
+ t.Error("sentenceBoundaryRe does not match the English '. ' boundary")
+ }
+ // CJK boundaries still recognized.
+ for _, p := range []string{"一。", "二!", "三?", "四;", "五\n"} {
+ if !sentenceBoundaryRe.MatchString(p) {
+ t.Errorf("sentenceBoundaryRe does not match boundary %q", p)
+ }
+ }
+}
+
+// TestSentenceBoundary_TokenTextPathDelimiterDistinct pins sentenceDelimiter
+// (the token-chunker TEXT path / naive_merge port) to Python naive_merge's
+// production delimiter "\n!?。;!?" — deliberately WITHOUT the English ". "
+// boundary. The two constants mirror two distinct Python delimiters; asserting
+// the exact difference documents that this is intentional, not a drift.
+func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {
+ const want = `(\n|[!?。;!?])`
+ if got := sentenceDelimiter.String(); got != want {
+ t.Errorf("sentenceDelimiter = %q, want %q (Python naive_merge delimiter)", got, want)
+ }
+ // The text-path delimiter must NOT split on the English ". " boundary
+ // (matching Python naive_merge), while the shared regex does.
+ if sentenceDelimiter.MatchString("Hello. World") {
+ t.Error("sentenceDelimiter split on '. ' but naive_merge's delimiter has no '. ' boundary")
+ }
+ if !sentenceBoundaryRe.MatchString("Hello. World") {
+ t.Error("sentenceBoundaryRe should split on '. '")
+ }
+}
diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.go
index 7484925a5f..87f8f38278 100644
--- a/internal/ingestion/component/chunker/title.go
+++ b/internal/ingestion/component/chunker/title.go
@@ -101,6 +101,9 @@ func (p *titleChunkerParam) Update(conf map[string]any) {
if v, ok := conf["root_chunk_as_heading"].(bool); ok {
p.TitleChunkerParam.RootChunkAsHeading = v
}
+ if v, ok := schema.NumericFromAny(conf["chunk_token_cap"]); ok {
+ p.TitleChunkerParam.ChunkTokenCap = int(v)
+ }
}
// parseLevels accepts a [[string]] representation — the natural
diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.go
new file mode 100644
index 0000000000..17b7c6b8c8
--- /dev/null
+++ b/internal/ingestion/component/chunker/title_cap.go
@@ -0,0 +1,266 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except under the License.
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+// Token-count cap for the title-chunker family (GroupTitle / HierarchyTitle),
+// mirroring Python title_chunker/common.py:_enforce_token_cap (#18455).
+//
+// A built chunk that exceeds `chunk_token_cap` tokens is re-split on sentence
+// boundaries into <= cap sub-chunks; a single boundary-less run that still
+// exceeds the cap is hard-split so the ceiling always holds. Table/image
+// chunks are atomic and never split. The first sub-chunk keeps the source
+// chunk's merged PDF position matrix (Plan A); subsequent sub-chunks carry an
+// empty array.
+
+import (
+ "encoding/json"
+ "strings"
+ "unicode/utf8"
+
+ "ragflow/internal/tokenizer"
+)
+
+// realNumTokens / realTrimToTokenLimit are the production tokenizer functions.
+// They are referenced through package-level vars (numTokens /
+// trimToTokenLimit) so tests can swap in a deterministic stub.
+var (
+ realNumTokens = tokenizer.NumTokensFromString
+ realTrimToTokenLimit = tokenizer.TrimContentToTokenLimit
+
+ numTokens = realNumTokens
+ trimToTokenLimit = realTrimToTokenLimit
+)
+
+// titleTokenCount counts tokens for text. numTokensFromString returns 0 when
+// the encoder is unavailable; in that case fall back to the rune count so the
+// cap is still enforced (mirrors Python #18455's character-count fallback).
+func titleTokenCount(text string) int {
+ if text == "" {
+ return 0
+ }
+ n := numTokens(text)
+ if n == 0 {
+ return utf8.RuneCountInString(text)
+ }
+ return n
+}
+
+// titleSentenceSplit splits text on sentence boundaries, keeping each
+// delimiter attached to the preceding sentence so the concatenation of the
+// returned pieces reproduces the original text exactly. It uses the shared
+// sentenceBoundaryRe (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE),
+// which includes the English ". " boundary.
+func titleSentenceSplit(text string) []string {
+ if text == "" {
+ return nil
+ }
+ idxs := sentenceBoundaryRe.FindAllStringIndex(text, -1)
+ var out []string
+ prev := 0
+ for _, idx := range idxs {
+ if idx[0] > prev {
+ out = append(out, text[prev:idx[1]])
+ } else {
+ out = append(out, text[idx[0]:idx[1]])
+ }
+ prev = idx[1]
+ }
+ if prev < len(text) {
+ out = append(out, text[prev:])
+ }
+ return out
+}
+
+// runePrefix returns the first n runes of s (a rune-safe prefix; never splits
+// a multi-byte character). Used by the offline/hard-split fallback.
+func runePrefix(s string, n int) string {
+ if n <= 0 {
+ return ""
+ }
+ count := 0
+ for i := range s {
+ if count >= n {
+ return s[:i]
+ }
+ count++
+ }
+ return s
+}
+
+// hardSplitByTokens hard-splits text into pieces whose token count is <= cap.
+// It mirrors Python's _hard_split_by_tokens (token_utils.truncate prefix).
+// When the tokenizer is unavailable (numTokens returns 0) it degrades to a
+// rune-prefix of length cap, keeping the ceiling intact (Python #18455
+// character-count fallback).
+func hardSplitByTokens(text string, cap int) []string {
+ if cap <= 0 {
+ if text == "" {
+ return nil
+ }
+ return []string{text}
+ }
+ var out []string
+ rest := text
+ for utf8.RuneCountInString(rest) > 0 {
+ // A remainder already within the token cap stays whole. This must be
+ // checked on TOKENS: for English text runes far exceed tokens, so
+ // comparing runes against the cap would over-fragment an in-cap tail.
+ if titleTokenCount(rest) <= cap {
+ out = append(out, rest)
+ break
+ }
+ head := trimToTokenLimit(rest, cap)
+ // No progress (tokenizer unavailable / cannot shrink): fall back to a
+ // rune prefix so the loop always makes progress and the ceiling holds.
+ if head == "" || head == rest || !strings.HasPrefix(rest, head) {
+ head = runePrefix(rest, cap)
+ if head == "" || head == rest {
+ out = append(out, rest)
+ break
+ }
+ }
+ out = append(out, head)
+ rest = rest[len(head):]
+ }
+ return out
+}
+
+// enforceTitleTokenCap applies the hard token ceiling to every text chunk
+// after build_chunks. Mirrors Python BaseTitleChunker._enforce_token_cap:
+// - cap <= 0 disables the ceiling (chunks returned unchanged);
+// - non-text chunks (table/image) are atomic and skipped;
+// - a text chunk already within cap is kept;
+// - an over-cap text chunk is re-split via splitTitleChunkByCap.
+func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {
+ if cap <= 0 {
+ return chunks
+ }
+ out := make([]map[string]any, 0, len(chunks))
+ for _, chunk := range chunks {
+ if toStringOrDefault(chunk["doc_type_kwd"], "text") != "text" {
+ out = append(out, chunk)
+ continue
+ }
+ text := toString(chunk["text"])
+ if text == "" || titleTokenCount(text) <= cap {
+ out = append(out, chunk)
+ continue
+ }
+ out = append(out, splitTitleChunkByCap(chunk, cap)...)
+ }
+ return out
+}
+
+// splitTitleChunkByCap re-splits one oversized text chunk into <= cap
+// sub-chunks. Sentences are tried first (greedy grouping); any remaining
+// over-cap segment is hard-split. The first sub-chunk keeps the source
+// chunk's merged PDF position matrix (Plan A); the rest carry an empty array.
+func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
+ text := toString(chunk["text"])
+ sentences := titleSentenceSplit(text)
+ if len(sentences) == 0 {
+ return []map[string]any{chunk}
+ }
+ groups := []string{}
+ current := ""
+ for _, s := range sentences {
+ cand := s
+ if current != "" {
+ cand = current + s
+ }
+ if current != "" && titleTokenCount(cand) > cap {
+ groups = append(groups, current)
+ current = s
+ } else {
+ current = cand
+ }
+ }
+ if current != "" {
+ groups = append(groups, current)
+ }
+ finalGroups := []string{}
+ for _, g := range groups {
+ if titleTokenCount(g) <= cap {
+ finalGroups = append(finalGroups, g)
+ } else {
+ finalGroups = append(finalGroups, hardSplitByTokens(g, cap)...)
+ }
+ }
+ if len(finalGroups) == 0 {
+ return []map[string]any{chunk}
+ }
+
+ out := make([]map[string]any, 0, len(finalGroups))
+ for i, g := range finalGroups {
+ sub := shallowCopyChunk(chunk)
+ sub["text"] = g
+ // Plan A: only the first sub-chunk keeps the position matrix; later
+ // sub-chunks carry an empty matrix of the same type, or drop the key
+ // when the source type is unknown.
+ if i > 0 {
+ if empty, ok := emptyPositionsLike(chunk["_pdf_positions"]); ok {
+ sub["_pdf_positions"] = empty
+ } else {
+ delete(sub, "_pdf_positions")
+ }
+ if empty, ok := emptyPositionsLike(chunk["positions"]); ok {
+ sub["positions"] = empty
+ } else {
+ delete(sub, "positions")
+ }
+ }
+ out = append(out, sub)
+ }
+ return out
+}
+
+// shallowCopyChunk returns a shallow copy of c. Coordinate matrices
+// ([][]float64) are shared by reference, but splitTitleChunkByCap replaces
+// (never mutates) them for sub-chunks, so the source chunk is never aliased.
+func shallowCopyChunk(c map[string]any) map[string]any {
+ m := make(map[string]any, len(c))
+ for k, v := range c {
+ m[k] = v
+ }
+ return m
+}
+
+// emptyPositionsLike returns an empty coordinate array of the same Go type as
+// the source value (mergePositionMatrix produces [][]float64; tests may feed
+// json.RawMessage), so a Plan-A sub-chunk's position field stays type-consistent
+// with the first sub-chunk's. The second result is false for unknown types;
+// the caller then DELETES the key instead of storing a nil-valued one.
+func emptyPositionsLike(v any) (any, bool) {
+ switch v.(type) {
+ case [][]float64:
+ return [][]float64{}, true
+ case json.RawMessage:
+ return json.RawMessage("[]"), true
+ default:
+ return nil, false
+ }
+}
+
+// toStringOrDefault returns the string value of v, or def when v is absent /
+// nil / not a string. Used to treat a missing doc_type_kwd as "text".
+func toStringOrDefault(v any, def string) string {
+ if v == nil {
+ return def
+ }
+ if s, ok := v.(string); ok {
+ return s
+ }
+ return def
+}
diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.go
new file mode 100644
index 0000000000..1c2b9a67e9
--- /dev/null
+++ b/internal/ingestion/component/chunker/title_token_cap_test.go
@@ -0,0 +1,598 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except under the License.
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func testCtx() context.Context { return context.Background() }
+
+// charTokenizer installs a deterministic "1 token == 1 rune" stub (mirroring
+// the Python test_token_cap suite, which fakes num_tokens_from_string as
+// len(text)). This makes the cap a hard rune ceiling so assertions are fully
+// reproducible. Callers must defer restoreTokenizer().
+func charTokenizer() {
+ numTokens = func(s string) int {
+ if s == "" {
+ return 0
+ }
+ return utf8.RuneCountInString(s)
+ }
+ trimToTokenLimit = func(s string, limit int) string {
+ if limit < 0 {
+ limit = 0
+ }
+ if utf8.RuneCountInString(s) <= limit {
+ return s
+ }
+ return runePrefix(s, limit)
+ }
+}
+
+// restoreTokenizer resets the tokenizer seam to the real implementation.
+func restoreTokenizer() {
+ numTokens = realNumTokens
+ trimToTokenLimit = realTrimToTokenLimit
+}
+
+// assertCapInvariants checks the post-_enforce_token_cap guarantees for a
+// slice of built chunks: every text piece is <= cap (re-tokenized), non-text
+// chunks are untouched, and the concatenated text reproduces the source
+// (lossless). The optional trailing newline that build_chunks appends is
+// stripped before comparison, matching Python's rstrip("\n").
+func assertCapInvariants(t *testing.T, chunks []map[string]any, cap int, source string) {
+ t.Helper()
+ var got strings.Builder
+ for _, ck := range chunks {
+ text := toString(ck["text"])
+ dt := toStringOrDefault(ck["doc_type_kwd"], "text")
+ if dt != "text" {
+ got.WriteString(text)
+ continue
+ }
+ if n := titleTokenCount(text); n > cap {
+ t.Errorf("chunk exceeds cap: tokens=%d (cap=%d) text=%q", n, cap, text)
+ }
+ got.WriteString(text)
+ }
+ if strings.TrimRight(got.String(), "\n") != strings.TrimRight(source, "\n") {
+ t.Errorf("lossless check failed:\n got=%q\nwant=%q", got.String(), source)
+ }
+}
+
+func TestTitleTokenCount_CharStub(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ if got := titleTokenCount("hello"); got != 5 {
+ t.Errorf("titleTokenCount(char) = %d, want 5", got)
+ }
+}
+
+func TestTitleTokenCount_OfflineFallback(t *testing.T) {
+ // Simulate an unavailable tokenizer (num_tokens_from_string returns 0).
+ saved := numTokens
+ numTokens = func(string) int { return 0 }
+ defer func() { numTokens = saved }()
+ // With the offline fallback, non-empty text counts as its rune length.
+ if got := titleTokenCount("hello世界"); got != 7 {
+ t.Errorf("offline fallback token count = %d, want 7", got)
+ }
+ if got := titleTokenCount(""); got != 0 {
+ t.Errorf("empty text token count = %d, want 0", got)
+ }
+}
+
+func TestTitleSentenceSplit_Boundaries(t *testing.T) {
+ // Chinese boundaries. The final fragment without a trailing delimiter is
+ // kept as its own sentence (matches Python re.split reassembly).
+ zh := "第一句。第二句!第三句?第四句;尾"
+ got := titleSentenceSplit(zh)
+ if len(got) != 5 {
+ t.Fatalf("zh split = %d sentences, want 5: %v", len(got), got)
+ }
+ wantEnds := []string{"。", "!", "?", ";"}
+ for i := 0; i < 4; i++ {
+ if !strings.HasSuffix(got[i], wantEnds[i]) {
+ t.Errorf("sentence %d = %q, want suffix %q", i, got[i], wantEnds[i])
+ }
+ }
+ if got[4] != "尾" {
+ t.Errorf("trailing sentence = %q, want \"尾\"", got[4])
+ }
+ // English ". " boundary (the Python #18455 regex includes `\. `).
+ en := "Hello. World. Foo"
+ eg := titleSentenceSplit(en)
+ if len(eg) != 3 {
+ t.Fatalf("en split = %d, want 3: %v", len(eg), eg)
+ }
+ if eg[0] != "Hello. " || eg[1] != "World. " || eg[2] != "Foo" {
+ t.Errorf("en split = %v, want [Hello. World. Foo]", eg)
+ }
+}
+
+func TestTitleSentenceSplit_Lossless(t *testing.T) {
+ zh := "第一句。第二句!第三句?第四句;尾"
+ if got := strings.Join(titleSentenceSplit(zh), ""); got != zh {
+ t.Errorf("sentence split not lossless: %q", got)
+ }
+ en := "Hello. World. Foo"
+ if got := strings.Join(titleSentenceSplit(en), ""); got != en {
+ t.Errorf("en sentence split not lossless: %q", got)
+ }
+}
+
+func TestEnforceTitleTokenCap_CapZeroNoop(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ chunks := []map[string]any{
+ {"text": strings.Repeat("x", 100)},
+ }
+ got := enforceTitleTokenCap(chunks, 0)
+ if len(got) != 1 {
+ t.Fatalf("cap=0 must keep 1 chunk, got %d", len(got))
+ }
+ if toString(got[0]["text"]) != strings.Repeat("x", 100) {
+ t.Errorf("cap=0 altered text: %q", toString(got[0]["text"]))
+ }
+}
+
+func TestEnforceTitleTokenCap_WithinCapUnchanged(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ chunks := []map[string]any{{"text": "S00。S01。"}}
+ got := enforceTitleTokenCap(chunks, 512)
+ if len(got) != 1 {
+ t.Fatalf("within-cap chunk split unexpectedly: %d chunks", len(got))
+ }
+ if toString(got[0]["text"]) != "S00。S01。" {
+ t.Errorf("within-cap text altered: %q", toString(got[0]["text"]))
+ }
+}
+
+func TestEnforceTitleTokenCap_OverCapResplits(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ chunks := []map[string]any{{"text": body}}
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) <= 1 {
+ t.Fatalf("expected oversized chunk to be split, got %d", len(got))
+ }
+ assertCapInvariants(t, got, 20, body)
+ for _, ck := range got {
+ if !strings.HasSuffix(strings.TrimRight(toString(ck["text"]), "\n"), "。") {
+ t.Errorf("chunk cut mid-sentence: %q", toString(ck["text"]))
+ }
+ }
+}
+
+func TestEnforceTitleTokenCap_NonTextAtomic(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ big := strings.Repeat("x", 200)
+ chunks := []map[string]any{{"text": big, "doc_type_kwd": "table"}}
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) != 1 {
+ t.Fatalf("table chunk must stay atomic, got %d", len(got))
+ }
+ if toString(got[0]["text"]) != big {
+ t.Errorf("table text altered: len=%d", len(toString(got[0]["text"])))
+ }
+ // image too
+ img := []map[string]any{{"text": big, "doc_type_kwd": "image"}}
+ if got := enforceTitleTokenCap(img, 20); len(got) != 1 {
+ t.Errorf("image chunk must stay atomic, got %d", len(got))
+ }
+}
+
+func TestEnforceTitleTokenCap_BoundarylessHardSplit(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Repeat("x", 100)
+ chunks := []map[string]any{{"text": body}}
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) <= 1 {
+ t.Fatalf("boundary-less run must be hard-split, got %d", len(got))
+ }
+ assertCapInvariants(t, got, 20, body)
+}
+
+func TestEnforceTitleTokenCap_TokenizerZeroFallback(t *testing.T) {
+ // numTokens reports 0 everywhere -> offline char fallback must still cap.
+ saved := numTokens
+ numTokens = func(string) int { return 0 }
+ defer func() { numTokens = saved }()
+ body := strings.Repeat("x", 100)
+ chunks := []map[string]any{{"text": body}}
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) <= 1 {
+ t.Fatalf("cap must apply even when tokenizer reports 0, got %d", len(got))
+ }
+ assertCapInvariants(t, got, 20, body)
+}
+
+// TestHardSplitByTokens_EnglishRemainderStaysWhole pins the over-fragmentation
+// fix: a remainder that already satisfies the TOKEN cap must be kept whole even
+// when its RUNE count exceeds the cap (English text is ~4 runes/token). The
+// char stub (1 rune == 1 token) makes the two counts identical and masks this,
+// so this test uses a 3-runes-per-token stub.
+func TestHardSplitByTokens_EnglishRemainderStaysWhole(t *testing.T) {
+ savedNum, savedTrim := numTokens, trimToTokenLimit
+ // 3 runes == 1 token.
+ numTokens = func(s string) int {
+ if s == "" {
+ return 0
+ }
+ return (utf8.RuneCountInString(s) + 2) / 3
+ }
+ trimToTokenLimit = func(s string, limit int) string {
+ maxRunes := limit * 3
+ if utf8.RuneCountInString(s) <= maxRunes {
+ return s
+ }
+ return runePrefix(s, maxRunes)
+ }
+ defer func() { numTokens, trimToTokenLimit = savedNum, savedTrim }()
+
+ const cap = 100
+ // 600 runes == 200 tokens == exactly 2 cap units. The second unit's
+ // remainder (300 runes == 100 tokens) is within the cap and must NOT be
+ // re-cut on runes.
+ body := strings.Repeat("ab", 300)
+ got := hardSplitByTokens(body, cap)
+ if len(got) != 2 {
+ t.Fatalf("hardSplitByTokens produced %d pieces, want 2 (in-cap remainder must stay whole)", len(got))
+ }
+ if strings.Join(got, "") != body {
+ t.Errorf("hard-split not lossless: %d runes vs %d", utf8.RuneCountInString(strings.Join(got, "")), utf8.RuneCountInString(body))
+ }
+ for i, p := range got {
+ if n := numTokens(p); n > cap {
+ t.Errorf("piece %d exceeds cap: tokens=%d (cap=%d)", i, n, cap)
+ }
+ }
+}
+
+func TestEnforceTitleTokenCap_PlanAPositions(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ pos := [][]float64{{1, 10, 200, 50, 80}}
+ body := strings.Join(joinSentences(12), "")
+ chunks := []map[string]any{
+ {"text": body, "positions": pos},
+ }
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) <= 1 {
+ t.Fatalf("expected split, got %d", len(got))
+ }
+ // First sub-chunk keeps the original position matrix.
+ first, ok := got[0]["positions"].([][]float64)
+ if !ok || len(first) == 0 || first[0][0] != 1 {
+ t.Errorf("first sub-chunk positions = %#v, want [[1 10 200 50 80]]", got[0]["positions"])
+ }
+ // Remaining sub-chunks carry an empty matrix (Plan A).
+ for i := 1; i < len(got); i++ {
+ v, ok := got[i]["positions"].([][]float64)
+ if !ok {
+ t.Errorf("sub-chunk %d positions type = %T, want [][]float64", i, got[i]["positions"])
+ continue
+ }
+ if len(v) != 0 {
+ t.Errorf("sub-chunk %d positions = %#v, want empty", i, v)
+ }
+ }
+}
+
+func TestEnforceTitleTokenCap_PlanA_UnknownPositionsType(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ // An unknown position value type (not [][]float64 / json.RawMessage): the
+ // key must be DROPPED from later sub-chunks, never stored as a nil value.
+ chunks := []map[string]any{
+ {"text": body, "positions": "not-a-matrix"},
+ }
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) <= 1 {
+ t.Fatalf("expected split, got %d", len(got))
+ }
+ // First sub-chunk keeps the value via shallow copy.
+ if _, ok := got[0]["positions"]; !ok {
+ t.Error("first sub-chunk lost its positions key")
+ }
+ for i := 1; i < len(got); i++ {
+ if _, ok := got[i]["positions"]; ok {
+ t.Errorf("sub-chunk %d kept unknown-type positions key, want deleted", i)
+ }
+ }
+}
+
+func TestEnforceTitleTokenCap_GreedyGrouping(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ // 12 sentences, 4 runes each, cap 20 -> 5 sentences/chunk (20), not 1 each.
+ body := strings.Join(joinSentences(12), "")
+ chunks := []map[string]any{{"text": body}}
+ got := enforceTitleTokenCap(chunks, 20)
+ if len(got) != 3 { // 5+5+2
+ t.Fatalf("greedy grouping produced %d chunks, want 3", len(got))
+ }
+ assertCapInvariants(t, got, 20, body)
+}
+
+// joinSentences builds 12 sentences "S00。".."S11。" mirroring the Python
+// test_hierarchy_oversized_chunk_respects_cap body.
+func joinSentences(n int) []string {
+ out := make([]string, 0, n)
+ for i := 0; i < n; i++ {
+ out = append(out, sprintfSentence(i))
+ }
+ return out
+}
+
+func sprintfSentence(i int) string {
+ return fmt.Sprintf("S%02d。", i)
+}
+
+// TestSprintfSentence_ThreeDigits pins the %02d formatting for 3-digit
+// indexes: the old hand-rolled twoDigit derived each digit from a single rune
+// addition and broke at i >= 100 (produced ":0").
+func TestSprintfSentence_ThreeDigits(t *testing.T) {
+ if got := sprintfSentence(100); got != "S100。" {
+ t.Errorf("sprintfSentence(100) = %q, want \"S100。\"", got)
+ }
+}
+
+func TestJoinSentences_Formatting(t *testing.T) {
+ got := joinSentences(15)
+ if len(got) != 15 {
+ t.Fatalf("joinSentences(15) = %d, want 15", len(got))
+ }
+ if got[14] != "S14。" {
+ t.Errorf("joinSentences(15)[14] = %q, want \"S14。\"", got[14])
+ }
+ if got[9] != "S09。" {
+ t.Errorf("joinSentences(15)[9] = %q, want \"S09。\"", got[9])
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline (integration) tests: cap applied through invokeGroup/invokeHierarchy
+// ---------------------------------------------------------------------------
+
+func newTitleParam(t *testing.T, method string, cap int, levels [][]string) titleChunkerParam {
+ t.Helper()
+ p := defaultsTitle()
+ conf := map[string]any{"method": method, "chunk_token_cap": cap}
+ if levels != nil {
+ lv := make([]any, 0, len(levels))
+ for _, g := range levels {
+ inner := make([]any, 0, len(g))
+ for _, s := range g {
+ inner = append(inner, s)
+ }
+ lv = append(lv, inner)
+ }
+ conf["levels"] = lv
+ }
+ if method == "hierarchy" {
+ conf["hierarchy"] = 1
+ }
+ p.Update(conf)
+ // NOTE: validation is intentionally skipped here — the char-stub tests use
+ // sub-128 caps (e.g. 20) that the production Validate() rejects, mirroring
+ // the Python suite which stubs out check(). Range validation is covered
+ // separately by TestTitleChunkerParam_ChunkTokenCapValidate.
+ return p
+}
+
+func TestTitleCap_GroupPipeline_RespectsCap(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "group", 20, [][]string{{`^# `}})
+ inputs := map[string]any{"output_format": "text", "text": body}
+ got, err := invokeGroup(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeGroup: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ assertCapInvariants(t, chunks, 20, body)
+}
+
+// TestTitleCap_GroupPipeline_ValidatedInRangeCap exercises the wired path with
+// a cap inside the production-validated range (128..8000). The char-stub suite
+// otherwise only uses sub-128 caps that production Validate() rejects, so this
+// is the only end-to-end coverage of an accepted configuration.
+func TestTitleCap_GroupPipeline_ValidatedInRangeCap(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Repeat("ab", 200) // 400 stub tokens > cap 128
+ p := newTitleParam(t, "group", 128, [][]string{{`^# `}})
+ if err := p.TitleChunkerParam.Validate(); err != nil {
+ t.Fatalf("cap=128 must pass production Validate: %v", err)
+ }
+ inputs := map[string]any{"output_format": "text", "text": body}
+ got, err := invokeGroup(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeGroup: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected cap=128 to re-split the oversized body, got %d", len(chunks))
+ }
+ assertCapInvariants(t, chunks, 128, body)
+}
+
+func TestTitleCap_HierarchyPipeline_RespectsCap(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})
+ inputs := map[string]any{"output_format": "text", "text": body}
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ assertCapInvariants(t, chunks, 20, body)
+}
+
+func TestTitleCap_GroupPipeline_CapZeroNoop(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "group", 0, [][]string{{`^# `}})
+ inputs := map[string]any{"output_format": "text", "text": body}
+ got, err := invokeGroup(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeGroup: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("cap=0 must keep 1 chunk, got %d", len(chunks))
+ }
+}
+
+func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ // 6 records "S00。\n...\nS05。" -> built text is each on its own line.
+ records := make([]string, 6)
+ for i := 0; i < 6; i++ {
+ records[i] = sprintfSentence(i)
+ }
+ body := strings.Join(records, "\n")
+ p := newTitleParam(t, "group", 10, [][]string{{`^# `}})
+ inputs := map[string]any{"output_format": "text", "text": body}
+ got, err := invokeGroup(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeGroup: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected merged split, got %d", len(chunks))
+ }
+ // The built text joins each record with "\n" and appends a trailing "\n".
+ var want strings.Builder
+ for _, r := range records {
+ want.WriteString(r)
+ want.WriteString("\n")
+ }
+ assertCapInvariants(t, chunks, 10, want.String())
+}
+
+func TestTitleCap_HierarchyPipeline_PlanAPositions(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})
+ inputs := map[string]any{
+ "output_format": "chunks",
+ "chunks": []schema.ChunkDoc{
+ {Text: body, DocType: "text", Positions: json.RawMessage(`[[1,10,200,50,80]]`)},
+ },
+ }
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ first := chunks[0]["positions"]
+ if first == nil {
+ t.Fatalf("first sub-chunk missing positions")
+ }
+ if fm, ok := first.([][]float64); !ok || len(fm) == 0 || fm[0][0] != 1 {
+ t.Errorf("first positions = %#v, want [[1 10 200 50 80]]", first)
+ }
+ for i := 1; i < len(chunks); i++ {
+ v, ok := chunks[i]["positions"]
+ if !ok {
+ t.Errorf("sub-chunk %d missing positions", i)
+ continue
+ }
+ if vm, ok := v.([][]float64); !ok || len(vm) != 0 {
+ t.Errorf("sub-chunk %d positions = %#v, want empty [][]float64", i, v)
+ }
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Schema validation
+// ---------------------------------------------------------------------------
+
+func TestTitleChunkerParam_ChunkTokenCapDefaults(t *testing.T) {
+ if got := (schema.TitleChunkerParam{}).Defaults().ChunkTokenCap; got != 512 {
+ t.Errorf("default ChunkTokenCap = %d, want 512", got)
+ }
+}
+
+func TestTitleChunkerParam_ChunkTokenCapValidate(t *testing.T) {
+ cases := []struct {
+ cap int
+ ok bool
+ }{
+ {0, true}, // disabled
+ {50, false}, // below 128
+ {127, false}, // below 128
+ {128, true}, // lower bound
+ {512, true}, // default
+ {8000, true}, // upper bound
+ {8001, false}, // above 8000
+ {9000, false}, // above 8000
+ }
+ for _, c := range cases {
+ p := schema.TitleChunkerParam{Method: "group", Levels: [][]string{{"^#"}}, ChunkTokenCap: c.cap}
+ err := p.Validate()
+ if c.ok && err != nil {
+ t.Errorf("cap=%d: unexpected error %v", c.cap, err)
+ }
+ if !c.ok && err == nil {
+ t.Errorf("cap=%d: expected error, got nil", c.cap)
+ }
+ }
+}
+
+// TestTitleChunkerParam_ChunkTokenCapValidate_EmptyMethodBypass pins the
+// validation-order fix: the cap range check must run even when Method is ""
+// (which otherwise early-returns nil), so an out-of-range cap cannot slip
+// through as an active ceiling.
+func TestTitleChunkerParam_ChunkTokenCapValidate_EmptyMethodBypass(t *testing.T) {
+ p := schema.TitleChunkerParam{Method: "", Levels: [][]string{{"^#"}}, ChunkTokenCap: 1}
+ if err := p.Validate(); err == nil {
+ t.Error(`method="" with cap=1 must be rejected (out of 128..8000)`)
+ }
+ pOK := schema.TitleChunkerParam{Method: "", Levels: [][]string{{"^#"}}, ChunkTokenCap: 512}
+ if err := pOK.Validate(); err != nil {
+ t.Errorf(`method="" with cap=512 must pass: %v`, err)
+ }
+}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index fd6b5bb148..97b835e85f 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -364,13 +364,15 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child
return chunkOutputs(docs)
}
-// sentenceDelimiter is the sentence/clause-boundary regex used to split
-// oversized sections. It mirrors the delimiter Python's chunker actually
-// uses in production: rag/app/naive.py:1285 passes "\n!?。;!?" to
-// naive_merge, which includes ASCII "!" and "?" as well as the CJK
-// punctuation "。;!?". It deliberately does NOT include an English
-// ". " fallback: Python's production delimiter has no "\.\s", so adding
-// it would diverge from Python's chunk boundaries.
+// sentenceDelimiter is the token-chunker TEXT-path sentence delimiter (the
+// naive_merge port). It mirrors the delimiter Python's naive_merge receives in
+// production: rag/app/naive.py passes "\n!?。;!?", which includes ASCII "!"
+// and "?" plus the CJK punctuation "。;!?" but NOT an English ". " boundary.
+//
+// The Title chunker and the token chunker's JSON path use the shared
+// sentenceBoundaryRe instead (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE,
+// which adds ". "). The two constants mirror two distinct Python delimiters and
+// differ only in that English boundary, exactly as Python does.
var sentenceDelimiter = regexp.MustCompile(`(\n|[!?。;!?])`)
// overlapCut returns the visible-text rune offset where the overlap prefix
diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go
index 0e1ee3e629..26c6be3884 100644
--- a/internal/ingestion/component/schema/chunker.go
+++ b/internal/ingestion/component/schema/chunker.go
@@ -417,17 +417,26 @@ type TitleChunkerParam struct {
// RootChunkAsHeading, when true, prepends the root chunk's text
// to every emitted chunk (and drops the root chunk itself).
RootChunkAsHeading bool `json:"root_chunk_as_heading"`
+
+ // ChunkTokenCap is the hard ceiling on each emitted text chunk's token
+ // count (mirrors Python title_chunker/common.py chunk_token_cap, added in
+ // #18455). A built text chunk exceeding it is re-split on sentence
+ // boundaries into <= cap sub-chunks. 0 disables the ceiling. The Python
+ // default is 512; valid range when non-zero is 128..8000.
+ ChunkTokenCap int `json:"chunk_token_cap"`
}
// Defaults returns the Python default TitleChunkerParam. `Method` is
// not initialized in the Python `__init__` (it is set externally); the
// default is left as the empty string and the component must supply it.
+// `ChunkTokenCap` defaults to 512, matching Python #18455.
func (TitleChunkerParam) Defaults() TitleChunkerParam {
return TitleChunkerParam{
Levels: [][]string{},
Hierarchy: nil,
IncludeHeadingContent: false,
RootChunkAsHeading: false,
+ ChunkTokenCap: 512,
}
}
@@ -435,6 +444,14 @@ func (TitleChunkerParam) Defaults() TitleChunkerParam {
// expressible in pure-data terms: when Method == "hierarchy" the
// hierarchy depth and level config must be present.
func (p *TitleChunkerParam) Validate() error {
+ // chunk_token_cap: 0 disables the ceiling; when set it must be a positive
+ // integer in [128, 8000] (mirrors Python title_chunker #18455). Checked
+ // BEFORE the method switch so an unset method cannot bypass the range.
+ if p.ChunkTokenCap != 0 {
+ if p.ChunkTokenCap < 128 || p.ChunkTokenCap > 8000 {
+ return errInvalidValue{Field: "chunk_token_cap", Value: fmt.Sprintf("%d", p.ChunkTokenCap)}
+ }
+ }
switch p.Method {
case "hierarchy", "group":
case "":