Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): keep ancestor headings on every cap-split hierarchy piece (#20953)

jay777219f94f13cd486
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index f839bfaee4..84d1ee8f1a 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -173,13 +173,13 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]
zap.String("variant", "group"),
zap.Int("groups", len(groups)),
)
- chunks := buildChunksFromRecordGroups(groups, p, isPlainTextFormat(inputs))
+ chunks, _ := buildChunksFromRecordGroups(groups, nil, p, isPlainTextFormat(inputs))
// Enforce the title-family token ceiling (chunk_token_cap) right after
// build_chunks and BEFORE the on-demand crop, mirroring Python's
// invoke() order (build -> cap -> set_chunks). ManualChunker passes
// tokenCap=0, so this is a no-op there.
if tokenCap > 0 {
- chunks = enforceTitleTokenCap(chunks, tokenCap)
+ chunks = enforceTitleTokenCap(chunks, nil, tokenCap)
}
common.Debug("chunker stage",
zap.String("component", "Chunker"),
@@ -294,9 +294,20 @@ func isPlainTextFormat(inputs map[string]any) bool {
// root_chunk_as_heading is applied here, exactly as python does (post
// materialisation): the root chunk's text is prepended to every
// following chunk and the root chunk is dropped.
-func buildChunksFromRecordGroups(groups [][]lineRecord, p *titleChunkerParam, plain bool) []map[string]any {
+//
+// prefixes is optional and, when non-nil, is aligned 1:1 with groups:
+// prefixes[i] is the ancestor heading chain group i starts with (the
+// hierarchy strategy sets it; group/manual pass nil). The returned prefixes
+// stay aligned with the returned chunks after empty groups are dropped and
+// root_chunk_as_heading is applied, so the token-cap pass can re-prepend a
+// chunk's chain to every piece it splits off.
+func buildChunksFromRecordGroups(groups [][]lineRecord, prefixes []string, p *titleChunkerParam, plain bool) ([]map[string]any, []string) {
chunks := make([]map[string]any, 0, len(groups))
- for _, g := range groups {
+ var outPrefixes []string
+ if prefixes != nil {
+ outPrefixes = make([]string, 0, len(groups))
+ }
+ for i, g := range groups {
if len(g) == 0 {
continue
}
@@ -332,15 +343,24 @@ func buildChunksFromRecordGroups(groups [][]lineRecord, p *titleChunkerParam, pl
chunk["positions"] = m
}
chunks = append(chunks, chunk)
+ if prefixes != nil {
+ outPrefixes = append(outPrefixes, prefixes[i])
+ }
}
if p.RootChunkAsHeading && len(chunks) > 1 {
rootText := toString(chunks[0]["text"])
for i := 1; i < len(chunks); i++ {
chunks[i]["text"] = rootText + "\n" + toString(chunks[i]["text"])
+ if prefixes != nil {
+ outPrefixes[i] = rootText + "\n" + outPrefixes[i]
+ }
}
chunks = chunks[1:]
+ if prefixes != nil {
+ outPrefixes = outPrefixes[1:]
+ }
}
- return chunks
+ return chunks, outPrefixes
}
// posTagRemove matches parser-emitted position tags of the form
diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go
index ea59cfc8ce..f590f68957 100644
--- a/internal/ingestion/component/chunker/hierarchy.go
+++ b/internal/ingestion/component/chunker/hierarchy.go
@@ -41,6 +41,7 @@ package chunker
import (
"context"
"fmt"
+ "strings"
"go.uber.org/zap"
"gorm.io/gorm"
@@ -180,7 +181,13 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
// resolve_target_level(text_levels, hierarchy) — the exact call
// python makes inside flush_text_records (Gap H: the hierarchy
// pointer is dereferenced defensively so a nil never panics).
+ // recordGroups and groupPrefixes stay aligned: groupPrefixes[i] is the
+ // leading segment (up to and including the last title, each line + "\n")
+ // that chunk i starts with, empty for groups without headings.
+ // enforceTitleTokenCap re-prepends it to every cap-split piece, so a
+ // split does not strip the parent headings from the continuation pieces.
var recordGroups [][]lineRecord
+ var groupPrefixes []string
var textRun []lineRecord
var textLevels []int
@@ -200,6 +207,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
runCopy := make([]lineRecord, len(textRun))
copy(runCopy, textRun)
recordGroups = append(recordGroups, runCopy)
+ groupPrefixes = append(groupPrefixes, "")
} else {
indexed := make([]indexedLine, len(textRun))
for j := range textRun {
@@ -216,7 +224,25 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
for k, idx := range path {
grp[k] = textRun[idx]
}
+ // The re-attached chain is the chunk text's leading segment:
+ // every path record up to and including the last title
+ // (level <= targetLevel). Ancestor body text folded in by
+ // getPaths sits between the titles, so a titles-only chain
+ // would not be a text prefix. Position tags are stripped to
+ // match the built chunk text.
+ lastTitle := -1
+ for k, idx := range path {
+ if textLevels[idx] <= targetLevel {
+ lastTitle = k
+ }
+ }
+ var chain strings.Builder
+ for k := 0; k <= lastTitle; k++ {
+ chain.WriteString(removeTag(textRun[path[k]].text))
+ chain.WriteByte('\n')
+ }
recordGroups = append(recordGroups, grp)
+ groupPrefixes = append(groupPrefixes, chain.String())
}
}
textRun = textRun[:0]
@@ -231,6 +257,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
}
flush()
recordGroups = append(recordGroups, []lineRecord{rec})
+ groupPrefixes = append(groupPrefixes, "")
}
flush()
common.Debug("chunker stage",
@@ -239,12 +266,13 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
zap.Int("record_groups", len(recordGroups)),
)
- chunks := buildChunksFromRecordGroups(recordGroups, p, isPlainTextFormat(inputs))
+ chunks, prefixes := buildChunksFromRecordGroups(recordGroups, groupPrefixes, p, isPlainTextFormat(inputs))
// Enforce the title-family token ceiling (chunk_token_cap) right after
// build_chunks and BEFORE the on-demand crop, mirroring Python's
- // invoke() order (build -> cap -> set_chunks).
+ // invoke() order (build -> cap -> set_chunks). prefixes re-attaches each
+ // chunk's ancestor heading chain to every piece the cap splits off.
if p.ChunkTokenCap > 0 {
- chunks = enforceTitleTokenCap(chunks, p.ChunkTokenCap)
+ chunks = enforceTitleTokenCap(chunks, prefixes, p.ChunkTokenCap)
}
common.Debug("chunker stage",
zap.String("component", "Chunker"),
diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.go
index e97ae6e22e..a440a3eb9c 100644
--- a/internal/ingestion/component/chunker/title_cap.go
+++ b/internal/ingestion/component/chunker/title_cap.go
@@ -23,6 +23,10 @@ package chunker
// chunks are atomic and never split. Every sub-chunk keeps the source chunk's
// merged PDF position matrix so each one still gets its page-region preview
// image and position highlight.
+//
+// Hierarchy sub-chunks additionally re-prepend the source chunk's leading
+// segment, so a split never strips the parent headings from the continuation
+// pieces (see splitTitleChunkByCap).
import (
"strings"
@@ -149,12 +153,22 @@ func hardSplitByTokens(text string, cap int) []string {
// - non-text chunks (table/image) are atomic and skipped;
// - a text chunk already within cap is kept;
// - an over-cap text chunk is re-split via splitTitleChunkByCap.
-func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {
+//
+// prefixes is aligned 1:1 with chunks (nil outside the hierarchy strategy):
+// prefixes[i] is the leading segment chunk i starts with (ancestor headings
+// and any ancestor body text folded in). Every sub-chunk of a split
+// re-prepends it (see splitTitleChunkByCap), so a split does not strip the
+// parent headings from the continuation pieces.
+func enforceTitleTokenCap(chunks []map[string]any, prefixes []string, cap int) []map[string]any {
if cap <= 0 {
return chunks
}
out := make([]map[string]any, 0, len(chunks))
- for _, chunk := range chunks {
+ for i, chunk := range chunks {
+ prefix := ""
+ if prefixes != nil {
+ prefix = prefixes[i]
+ }
if toStringOrDefault(chunk["doc_type_kwd"], "text") != "text" {
out = append(out, chunk)
continue
@@ -164,7 +178,7 @@ func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {
out = append(out, chunk)
continue
}
- out = append(out, splitTitleChunkByCap(chunk, cap)...)
+ out = append(out, splitTitleChunkByCap(chunk, prefix, cap)...)
}
return out
}
@@ -175,9 +189,33 @@ func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {
// merged PDF position matrix (the coordinates are coarse, source-chunk
// level), so the on-demand crop pass can attach a preview image to each
// sub-chunk.
-func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
+//
+// prefix is the leading segment the source chunk starts with (empty outside
+// the hierarchy strategy): the ancestor headings plus any ancestor body text
+// folded in, up to and including the leaf's own heading. It is detached from
+// the text before grouping and re-prepended to every piece: without that,
+// only the first piece would keep the parent headings and every continuation
+// piece would be a bare body fragment. The grouping budget subtracts the
+// prefix so each piece still fits the cap; a prefix that would leave the
+// body less than half the cap disables re-attachment (plain split) instead
+// of multiplying near-empty pieces that each repeat it.
+func splitTitleChunkByCap(chunk map[string]any, prefix string, cap int) []map[string]any {
text := toString(chunk["text"])
- sentences := titleSentenceSplit(text)
+ body := text
+ if prefix != "" {
+ // Re-attaching the prefix only pays off while the body keeps at
+ // least half the cap: a larger prefix would multiply near-empty
+ // pieces that each repeat it, so fall back to a plain split. The
+ // HasPrefix check is a safety net (the prefix is the text's leading
+ // segment by construction).
+ if !strings.HasPrefix(text, prefix) || titleTokenCount(prefix) > cap/2 {
+ prefix = ""
+ } else {
+ body = text[len(prefix):]
+ }
+ }
+ budget := cap - titleTokenCount(prefix)
+ sentences := titleSentenceSplit(body)
if len(sentences) == 0 {
return []map[string]any{chunk}
}
@@ -188,7 +226,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
if current != "" {
cand = current + s
}
- if current != "" && titleTokenCount(cand) > cap {
+ if current != "" && titleTokenCount(cand) > budget {
groups = append(groups, current)
current = s
} else {
@@ -200,10 +238,10 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
}
finalGroups := []string{}
for _, g := range groups {
- if titleTokenCount(g) <= cap {
+ if titleTokenCount(g) <= budget {
finalGroups = append(finalGroups, g)
} else {
- finalGroups = append(finalGroups, hardSplitByTokens(g, cap)...)
+ finalGroups = append(finalGroups, hardSplitByTokens(g, budget)...)
}
}
if len(finalGroups) == 0 {
@@ -219,7 +257,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
out := make([]map[string]any, 0, len(finalGroups))
for _, g := range finalGroups {
sub := shallowCopyChunk(chunk)
- sub["text"] = g
+ sub["text"] = prefix + g
out = append(out, sub)
}
return out
@@ -229,7 +267,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
out := make([]map[string]any, 0, len(finalGroups))
for _, g := range finalGroups {
sub := shallowCopyChunk(chunk)
- sub["text"] = g
+ sub["text"] = prefix + g
out = append(out, sub)
}
return out
@@ -241,7 +279,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {
startRatio := float64(cum) / float64(total)
endRatio := float64(cum+pcRunes) / float64(total)
sub := shallowCopyChunk(chunk)
- sub["text"] = g
+ sub["text"] = prefix + g
if hasPDF {
if sliced := sliceAnyPositions(chunk["_pdf_positions"], startRatio, endRatio); sliced != nil {
sub["_pdf_positions"] = sliced
diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.go
index fac568ba47..4772e6a014 100644
--- a/internal/ingestion/component/chunker/title_token_cap_test.go
+++ b/internal/ingestion/component/chunker/title_token_cap_test.go
@@ -149,7 +149,7 @@ func TestEnforceTitleTokenCap_CapZeroNoop(t *testing.T) {
chunks := []map[string]any{
{"text": strings.Repeat("x", 100)},
}
- got := enforceTitleTokenCap(chunks, 0)
+ got := enforceTitleTokenCap(chunks, nil, 0)
if len(got) != 1 {
t.Fatalf("cap=0 must keep 1 chunk, got %d", len(got))
}
@@ -162,7 +162,7 @@ func TestEnforceTitleTokenCap_WithinCapUnchanged(t *testing.T) {
charTokenizer()
defer restoreTokenizer()
chunks := []map[string]any{{"text": "S00。S01。"}}
- got := enforceTitleTokenCap(chunks, 512)
+ got := enforceTitleTokenCap(chunks, nil, 512)
if len(got) != 1 {
t.Fatalf("within-cap chunk split unexpectedly: %d chunks", len(got))
}
@@ -176,7 +176,7 @@ func TestEnforceTitleTokenCap_OverCapResplits(t *testing.T) {
defer restoreTokenizer()
body := strings.Join(joinSentences(12), "")
chunks := []map[string]any{{"text": body}}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) <= 1 {
t.Fatalf("expected oversized chunk to be split, got %d", len(got))
}
@@ -193,7 +193,7 @@ func TestEnforceTitleTokenCap_NonTextAtomic(t *testing.T) {
defer restoreTokenizer()
big := strings.Repeat("x", 200)
chunks := []map[string]any{{"text": big, "doc_type_kwd": "table"}}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) != 1 {
t.Fatalf("table chunk must stay atomic, got %d", len(got))
}
@@ -202,7 +202,7 @@ func TestEnforceTitleTokenCap_NonTextAtomic(t *testing.T) {
}
// image too
img := []map[string]any{{"text": big, "doc_type_kwd": "image"}}
- if got := enforceTitleTokenCap(img, 20); len(got) != 1 {
+ if got := enforceTitleTokenCap(img, nil, 20); len(got) != 1 {
t.Errorf("image chunk must stay atomic, got %d", len(got))
}
}
@@ -212,7 +212,7 @@ func TestEnforceTitleTokenCap_BoundarylessHardSplit(t *testing.T) {
defer restoreTokenizer()
body := strings.Repeat("x", 100)
chunks := []map[string]any{{"text": body}}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) <= 1 {
t.Fatalf("boundary-less run must be hard-split, got %d", len(got))
}
@@ -226,7 +226,7 @@ func TestEnforceTitleTokenCap_TokenizerZeroFallback(t *testing.T) {
defer func() { numTokens = saved }()
body := strings.Repeat("x", 100)
chunks := []map[string]any{{"text": body}}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) <= 1 {
t.Fatalf("cap must apply even when tokenizer reports 0, got %d", len(got))
}
@@ -284,7 +284,7 @@ func TestEnforceTitleTokenCap_SubChunksHaveSlicedPositions(t *testing.T) {
chunks := []map[string]any{
{"text": body, "positions": pos},
}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) != 3 { // 20 + 20 + 8 runes
t.Fatalf("expected split into 3 chunks, got %d", len(got))
}
@@ -319,7 +319,7 @@ func TestEnforceTitleTokenCap_MalformedPositionsFallbackKeepsOriginal(t *testing
chunks := []map[string]any{
{"text": body, "_pdf_positions": src},
}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) <= 1 {
t.Fatalf("expected split, got %d", len(got))
}
@@ -344,7 +344,7 @@ func TestEnforceTitleTokenCap_UnknownPositionsTypeCarriedAsIs(t *testing.T) {
chunks := []map[string]any{
{"text": body, "positions": "not-a-matrix"},
}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) <= 1 {
t.Fatalf("expected split, got %d", len(got))
}
@@ -362,7 +362,7 @@ func TestEnforceTitleTokenCap_GreedyGrouping(t *testing.T) {
// 12 sentences, 4 runes each, cap 20 -> 5 sentences/chunk (20), not 1 each.
body := strings.Join(joinSentences(12), "")
chunks := []map[string]any{{"text": body}}
- got := enforceTitleTokenCap(chunks, 20)
+ got := enforceTitleTokenCap(chunks, nil, 20)
if len(got) != 3 { // 5+5+2
t.Fatalf("greedy grouping produced %d chunks, want 3", len(got))
}
@@ -575,6 +575,211 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {
}
}
+// TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain pins the fix for
+// the bug where the cap split stripped the parent headings: every piece of a
+// re-split hierarchy chunk must re-prepend the ancestor heading chain, not
+// just the first piece.
+func TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 24, [][]string{{`^# `, `^## `}})
+ two := 2
+ p.Hierarchy = &two
+ inputText := "# H1\n## H2a\n" + body
+ inputs := map[string]any{"output_format": "text", "text": inputText}
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ const chain = "# H1\n## H2a\n"
+ var rebuilt strings.Builder
+ for i, ck := range chunks {
+ text := toString(ck["text"])
+ if !strings.HasPrefix(text, chain) {
+ t.Errorf("chunk %d lost its heading chain: %q", i, text)
+ continue
+ }
+ if n := titleTokenCount(text); n > 24 {
+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)
+ }
+ if n := strings.Count(text, "# H1\n"); n != 1 {
+ t.Errorf("chunk %d repeated the chain %d times: %q", i, n, text)
+ }
+ rebuilt.WriteString(strings.TrimPrefix(text, chain))
+ }
+ // Stripping the re-attached chain from every piece must reproduce the
+ // original text (build_chunks appends a trailing "\n" per record).
+ want := inputText + "\n"
+ if got := chain + rebuilt.String(); got != want {
+ t.Errorf("chain-stripped pieces not lossless:\n got=%q\nwant=%q", got, want)
+ }
+}
+
+// TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain pins the
+// root_chunk_as_heading interaction: after the root chunk's text is prepended
+// to every following chunk, a cap split must still re-prepend the full prefix
+// (root text + ancestor chain) to each piece.
+func TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 40, [][]string{{`^# `, `^## `}})
+ two := 2
+ p.Hierarchy = &two
+ p.RootChunkAsHeading = true
+ inputText := "前言。\n# H1\n## H2a\n" + body
+ inputs := map[string]any{"output_format": "text", "text": inputText}
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ // build_chunks joins each record with "\n" (the root chunk text is
+ // "前言。\n") and root_chunk_as_heading prepends it plus a separator to
+ // every chunk; the chain must survive on every piece of the split.
+ const chain = "前言。\n\n# H1\n## H2a\n"
+ for i, ck := range chunks {
+ text := toString(ck["text"])
+ if !strings.HasPrefix(text, chain) {
+ t.Errorf("chunk %d lost its heading chain: %q", i, text)
+ }
+ if n := titleTokenCount(text); n > 40 {
+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)
+ }
+ }
+}
+
+// TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain covers the case
+// where a non-leaf ancestor carries its own body text: getPaths folds that
+// text into the leaf's path before the leaf's own title, so the re-attached
+// chain must cover every path record up to and including the last title — not
+// just the title lines — or the prefix check fails and the split strips the
+// chain again.
+func TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 60, [][]string{{`^# `, `^## `, `^### `}})
+ three := 3
+ p.Hierarchy = &three
+ inputText := "# A\n前言A。\n## B\n前言B。\n### C\n" + body
+ inputs := map[string]any{"output_format": "text", "text": inputText}
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ const chain = "# A\n前言A。\n## B\n前言B。\n### C\n"
+ var rebuilt strings.Builder
+ for i, ck := range chunks {
+ text := toString(ck["text"])
+ if !strings.HasPrefix(text, chain) {
+ t.Errorf("chunk %d lost its leading chain: %q", i, text)
+ continue
+ }
+ if n := titleTokenCount(text); n > 60 {
+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)
+ }
+ rebuilt.WriteString(strings.TrimPrefix(text, chain))
+ }
+ if got := chain + rebuilt.String(); got != inputText+"\n" {
+ t.Errorf("chain-stripped pieces not lossless:\n got=%q\nwant=%q", got, inputText+"\n")
+ }
+}
+
+// TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles pins the
+// include_heading_content=true branch: the leading segment is the ancestor
+// titles only (bodies are not folded in), and a cap split of the leaf chunk
+// must still re-prepend it to every piece. The parent's own chunk is emitted
+// unchanged.
+func TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 24, [][]string{{`^# `, `^## `}})
+ two := 2
+ p.Hierarchy = &two
+ p.IncludeHeadingContent = true
+ inputText := "# H1\n短。\n## H2a\n" + body
+ inputs := map[string]any{"output_format": "text", "text": inputText}
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 2 {
+ t.Fatalf("expected parent chunk + split leaf pieces, got %d", len(chunks))
+ }
+ // The parent's own chunk is within cap and emitted as-is.
+ if got := toString(chunks[0]["text"]); got != "# H1\n短。\n" {
+ t.Errorf("parent chunk = %q, want %q", got, "# H1\n短。\n")
+ }
+ const chain = "# H1\n## H2a\n"
+ var rebuilt strings.Builder
+ for i := 1; i < len(chunks); i++ {
+ text := toString(chunks[i]["text"])
+ if !strings.HasPrefix(text, chain) {
+ t.Errorf("chunk %d lost its heading chain: %q", i, text)
+ continue
+ }
+ if n := titleTokenCount(text); n > 24 {
+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)
+ }
+ rebuilt.WriteString(strings.TrimPrefix(text, chain))
+ }
+ if got := rebuilt.String(); got != body+"\n" {
+ t.Errorf("chain-stripped pieces not lossless:\n got=%q\nwant=%q", got, body+"\n")
+ }
+}
+
+// TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack pins the minimum body
+// budget guard: when the repeated prefix would leave the body less than half
+// the cap, re-attachment is disabled (plain split) instead of multiplying
+// near-empty pieces that each repeat an oversized prefix.
+func TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack(t *testing.T) {
+ charTokenizer()
+ defer restoreTokenizer()
+ body := strings.Join(joinSentences(12), "")
+ p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `, `^## `}})
+ two := 2
+ p.Hierarchy = &two
+ inputText := "# H1\n## H2a\n" + body
+ inputs := map[string]any{"output_format": "text", "text": inputText}
+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
+ if err != nil {
+ t.Fatalf("invokeHierarchy: %v", err)
+ }
+ chunks := got["chunks"].([]map[string]any)
+ if len(chunks) <= 1 {
+ t.Fatalf("expected split, got %d", len(chunks))
+ }
+ // 61 stub-tokens of text at cap 20: a plain split yields 4 pieces; a
+ // prefix re-attachment with a 1-token body budget would yield ~9.
+ if len(chunks) > 5 {
+ t.Errorf("oversized prefix must not multiply pieces: got %d", len(chunks))
+ }
+ for i, ck := range chunks {
+ text := toString(ck["text"])
+ if n := titleTokenCount(text); n > 20 {
+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)
+ }
+ if i > 0 && strings.Contains(text, "# H1") {
+ t.Errorf("chunk %d must stay bare when the prefix is too large: %q", i, text)
+ }
+ }
+}
+
// ---------------------------------------------------------------------------
// Schema validation
// ---------------------------------------------------------------------------