RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): keep ancestor headings on every cap-split hierarchy piece (#20953)
jay777219f94f13cd486
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex f839bfaee4..84d1ee8f1a 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -173,13 +173,13 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]zap.String("variant", "group"),zap.Int("groups", len(groups)),)- chunks := buildChunksFromRecordGroups(groups, p, isPlainTextFormat(inputs))+ chunks, _ := buildChunksFromRecordGroups(groups, nil, p, isPlainTextFormat(inputs))// Enforce the title-family token ceiling (chunk_token_cap) right after// build_chunks and BEFORE the on-demand crop, mirroring Python's// invoke() order (build -> cap -> set_chunks). ManualChunker passes// tokenCap=0, so this is a no-op there.if tokenCap > 0 {- chunks = enforceTitleTokenCap(chunks, tokenCap)+ chunks = enforceTitleTokenCap(chunks, nil, tokenCap)}common.Debug("chunker stage",zap.String("component", "Chunker"),@@ -294,9 +294,20 @@ func isPlainTextFormat(inputs map[string]any) bool {// root_chunk_as_heading is applied here, exactly as python does (post// materialisation): the root chunk's text is prepended to every// following chunk and the root chunk is dropped.-func buildChunksFromRecordGroups(groups [][]lineRecord, p *titleChunkerParam, plain bool) []map[string]any {+//+// prefixes is optional and, when non-nil, is aligned 1:1 with groups:+// prefixes[i] is the ancestor heading chain group i starts with (the+// hierarchy strategy sets it; group/manual pass nil). The returned prefixes+// stay aligned with the returned chunks after empty groups are dropped and+// root_chunk_as_heading is applied, so the token-cap pass can re-prepend a+// chunk's chain to every piece it splits off.+func buildChunksFromRecordGroups(groups [][]lineRecord, prefixes []string, p *titleChunkerParam, plain bool) ([]map[string]any, []string) {chunks := make([]map[string]any, 0, len(groups))- for _, g := range groups {+ var outPrefixes []string+ if prefixes != nil {+ outPrefixes = make([]string, 0, len(groups))+ }+ for i, g := range groups {if len(g) == 0 {continue}@@ -332,15 +343,24 @@ func buildChunksFromRecordGroups(groups [][]lineRecord, p *titleChunkerParam, plchunk["positions"] = m}chunks = append(chunks, chunk)+ if prefixes != nil {+ outPrefixes = append(outPrefixes, prefixes[i])+ }}if p.RootChunkAsHeading && len(chunks) > 1 {rootText := toString(chunks[0]["text"])for i := 1; i < len(chunks); i++ {chunks[i]["text"] = rootText + "\n" + toString(chunks[i]["text"])+ if prefixes != nil {+ outPrefixes[i] = rootText + "\n" + outPrefixes[i]+ }}chunks = chunks[1:]+ if prefixes != nil {+ outPrefixes = outPrefixes[1:]+ }}- return chunks+ return chunks, outPrefixes}// posTagRemove matches parser-emitted position tags of the formdiff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.goindex ea59cfc8ce..f590f68957 100644--- a/internal/ingestion/component/chunker/hierarchy.go+++ b/internal/ingestion/component/chunker/hierarchy.go@@ -41,6 +41,7 @@ package chunkerimport ("context""fmt"+ "strings""go.uber.org/zap""gorm.io/gorm"@@ -180,7 +181,13 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a// resolve_target_level(text_levels, hierarchy) — the exact call// python makes inside flush_text_records (Gap H: the hierarchy// pointer is dereferenced defensively so a nil never panics).+ // recordGroups and groupPrefixes stay aligned: groupPrefixes[i] is the+ // leading segment (up to and including the last title, each line + "\n")+ // that chunk i starts with, empty for groups without headings.+ // enforceTitleTokenCap re-prepends it to every cap-split piece, so a+ // split does not strip the parent headings from the continuation pieces.var recordGroups [][]lineRecord+ var groupPrefixes []stringvar textRun []lineRecordvar textLevels []int@@ -200,6 +207,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]arunCopy := make([]lineRecord, len(textRun))copy(runCopy, textRun)recordGroups = append(recordGroups, runCopy)+ groupPrefixes = append(groupPrefixes, "")} else {indexed := make([]indexedLine, len(textRun))for j := range textRun {@@ -216,7 +224,25 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]afor k, idx := range path {grp[k] = textRun[idx]}+ // The re-attached chain is the chunk text's leading segment:+ // every path record up to and including the last title+ // (level <= targetLevel). Ancestor body text folded in by+ // getPaths sits between the titles, so a titles-only chain+ // would not be a text prefix. Position tags are stripped to+ // match the built chunk text.+ lastTitle := -1+ for k, idx := range path {+ if textLevels[idx] <= targetLevel {+ lastTitle = k+ }+ }+ var chain strings.Builder+ for k := 0; k <= lastTitle; k++ {+ chain.WriteString(removeTag(textRun[path[k]].text))+ chain.WriteByte('\n')+ }recordGroups = append(recordGroups, grp)+ groupPrefixes = append(groupPrefixes, chain.String())}}textRun = textRun[:0]@@ -231,6 +257,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a}flush()recordGroups = append(recordGroups, []lineRecord{rec})+ groupPrefixes = append(groupPrefixes, "")}flush()common.Debug("chunker stage",@@ -239,12 +266,13 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]azap.Int("record_groups", len(recordGroups)),)- chunks := buildChunksFromRecordGroups(recordGroups, p, isPlainTextFormat(inputs))+ chunks, prefixes := buildChunksFromRecordGroups(recordGroups, groupPrefixes, p, isPlainTextFormat(inputs))// Enforce the title-family token ceiling (chunk_token_cap) right after// build_chunks and BEFORE the on-demand crop, mirroring Python's- // invoke() order (build -> cap -> set_chunks).+ // invoke() order (build -> cap -> set_chunks). prefixes re-attaches each+ // chunk's ancestor heading chain to every piece the cap splits off.if p.ChunkTokenCap > 0 {- chunks = enforceTitleTokenCap(chunks, p.ChunkTokenCap)+ chunks = enforceTitleTokenCap(chunks, prefixes, p.ChunkTokenCap)}common.Debug("chunker stage",zap.String("component", "Chunker"),diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.goindex e97ae6e22e..a440a3eb9c 100644--- a/internal/ingestion/component/chunker/title_cap.go+++ b/internal/ingestion/component/chunker/title_cap.go@@ -23,6 +23,10 @@ package chunker// chunks are atomic and never split. Every sub-chunk keeps the source chunk's// merged PDF position matrix so each one still gets its page-region preview// image and position highlight.+//+// Hierarchy sub-chunks additionally re-prepend the source chunk's leading+// segment, so a split never strips the parent headings from the continuation+// pieces (see splitTitleChunkByCap).import ("strings"@@ -149,12 +153,22 @@ func hardSplitByTokens(text string, cap int) []string {// - non-text chunks (table/image) are atomic and skipped;// - a text chunk already within cap is kept;// - an over-cap text chunk is re-split via splitTitleChunkByCap.-func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {+//+// prefixes is aligned 1:1 with chunks (nil outside the hierarchy strategy):+// prefixes[i] is the leading segment chunk i starts with (ancestor headings+// and any ancestor body text folded in). Every sub-chunk of a split+// re-prepends it (see splitTitleChunkByCap), so a split does not strip the+// parent headings from the continuation pieces.+func enforceTitleTokenCap(chunks []map[string]any, prefixes []string, cap int) []map[string]any {if cap <= 0 {return chunks}out := make([]map[string]any, 0, len(chunks))- for _, chunk := range chunks {+ for i, chunk := range chunks {+ prefix := ""+ if prefixes != nil {+ prefix = prefixes[i]+ }if toStringOrDefault(chunk["doc_type_kwd"], "text") != "text" {out = append(out, chunk)continue@@ -164,7 +178,7 @@ func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {out = append(out, chunk)continue}- out = append(out, splitTitleChunkByCap(chunk, cap)...)+ out = append(out, splitTitleChunkByCap(chunk, prefix, cap)...)}return out}@@ -175,9 +189,33 @@ func enforceTitleTokenCap(chunks []map[string]any, cap int) []map[string]any {// merged PDF position matrix (the coordinates are coarse, source-chunk// level), so the on-demand crop pass can attach a preview image to each// sub-chunk.-func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {+//+// prefix is the leading segment the source chunk starts with (empty outside+// the hierarchy strategy): the ancestor headings plus any ancestor body text+// folded in, up to and including the leaf's own heading. It is detached from+// the text before grouping and re-prepended to every piece: without that,+// only the first piece would keep the parent headings and every continuation+// piece would be a bare body fragment. The grouping budget subtracts the+// prefix so each piece still fits the cap; a prefix that would leave the+// body less than half the cap disables re-attachment (plain split) instead+// of multiplying near-empty pieces that each repeat it.+func splitTitleChunkByCap(chunk map[string]any, prefix string, cap int) []map[string]any {text := toString(chunk["text"])- sentences := titleSentenceSplit(text)+ body := text+ if prefix != "" {+ // Re-attaching the prefix only pays off while the body keeps at+ // least half the cap: a larger prefix would multiply near-empty+ // pieces that each repeat it, so fall back to a plain split. The+ // HasPrefix check is a safety net (the prefix is the text's leading+ // segment by construction).+ if !strings.HasPrefix(text, prefix) || titleTokenCount(prefix) > cap/2 {+ prefix = ""+ } else {+ body = text[len(prefix):]+ }+ }+ budget := cap - titleTokenCount(prefix)+ sentences := titleSentenceSplit(body)if len(sentences) == 0 {return []map[string]any{chunk}}@@ -188,7 +226,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {if current != "" {cand = current + s}- if current != "" && titleTokenCount(cand) > cap {+ if current != "" && titleTokenCount(cand) > budget {groups = append(groups, current)current = s} else {@@ -200,10 +238,10 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {}finalGroups := []string{}for _, g := range groups {- if titleTokenCount(g) <= cap {+ if titleTokenCount(g) <= budget {finalGroups = append(finalGroups, g)} else {- finalGroups = append(finalGroups, hardSplitByTokens(g, cap)...)+ finalGroups = append(finalGroups, hardSplitByTokens(g, budget)...)}}if len(finalGroups) == 0 {@@ -219,7 +257,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {out := make([]map[string]any, 0, len(finalGroups))for _, g := range finalGroups {sub := shallowCopyChunk(chunk)- sub["text"] = g+ sub["text"] = prefix + gout = append(out, sub)}return out@@ -229,7 +267,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {out := make([]map[string]any, 0, len(finalGroups))for _, g := range finalGroups {sub := shallowCopyChunk(chunk)- sub["text"] = g+ sub["text"] = prefix + gout = append(out, sub)}return out@@ -241,7 +279,7 @@ func splitTitleChunkByCap(chunk map[string]any, cap int) []map[string]any {startRatio := float64(cum) / float64(total)endRatio := float64(cum+pcRunes) / float64(total)sub := shallowCopyChunk(chunk)- sub["text"] = g+ sub["text"] = prefix + gif hasPDF {if sliced := sliceAnyPositions(chunk["_pdf_positions"], startRatio, endRatio); sliced != nil {sub["_pdf_positions"] = sliceddiff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.goindex fac568ba47..4772e6a014 100644--- a/internal/ingestion/component/chunker/title_token_cap_test.go+++ b/internal/ingestion/component/chunker/title_token_cap_test.go@@ -149,7 +149,7 @@ func TestEnforceTitleTokenCap_CapZeroNoop(t *testing.T) {chunks := []map[string]any{{"text": strings.Repeat("x", 100)},}- got := enforceTitleTokenCap(chunks, 0)+ got := enforceTitleTokenCap(chunks, nil, 0)if len(got) != 1 {t.Fatalf("cap=0 must keep 1 chunk, got %d", len(got))}@@ -162,7 +162,7 @@ func TestEnforceTitleTokenCap_WithinCapUnchanged(t *testing.T) {charTokenizer()defer restoreTokenizer()chunks := []map[string]any{{"text": "S00。S01。"}}- got := enforceTitleTokenCap(chunks, 512)+ got := enforceTitleTokenCap(chunks, nil, 512)if len(got) != 1 {t.Fatalf("within-cap chunk split unexpectedly: %d chunks", len(got))}@@ -176,7 +176,7 @@ func TestEnforceTitleTokenCap_OverCapResplits(t *testing.T) {defer restoreTokenizer()body := strings.Join(joinSentences(12), "")chunks := []map[string]any{{"text": body}}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) <= 1 {t.Fatalf("expected oversized chunk to be split, got %d", len(got))}@@ -193,7 +193,7 @@ func TestEnforceTitleTokenCap_NonTextAtomic(t *testing.T) {defer restoreTokenizer()big := strings.Repeat("x", 200)chunks := []map[string]any{{"text": big, "doc_type_kwd": "table"}}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) != 1 {t.Fatalf("table chunk must stay atomic, got %d", len(got))}@@ -202,7 +202,7 @@ func TestEnforceTitleTokenCap_NonTextAtomic(t *testing.T) {}// image tooimg := []map[string]any{{"text": big, "doc_type_kwd": "image"}}- if got := enforceTitleTokenCap(img, 20); len(got) != 1 {+ if got := enforceTitleTokenCap(img, nil, 20); len(got) != 1 {t.Errorf("image chunk must stay atomic, got %d", len(got))}}@@ -212,7 +212,7 @@ func TestEnforceTitleTokenCap_BoundarylessHardSplit(t *testing.T) {defer restoreTokenizer()body := strings.Repeat("x", 100)chunks := []map[string]any{{"text": body}}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) <= 1 {t.Fatalf("boundary-less run must be hard-split, got %d", len(got))}@@ -226,7 +226,7 @@ func TestEnforceTitleTokenCap_TokenizerZeroFallback(t *testing.T) {defer func() { numTokens = saved }()body := strings.Repeat("x", 100)chunks := []map[string]any{{"text": body}}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) <= 1 {t.Fatalf("cap must apply even when tokenizer reports 0, got %d", len(got))}@@ -284,7 +284,7 @@ func TestEnforceTitleTokenCap_SubChunksHaveSlicedPositions(t *testing.T) {chunks := []map[string]any{{"text": body, "positions": pos},}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) != 3 { // 20 + 20 + 8 runest.Fatalf("expected split into 3 chunks, got %d", len(got))}@@ -319,7 +319,7 @@ func TestEnforceTitleTokenCap_MalformedPositionsFallbackKeepsOriginal(t *testingchunks := []map[string]any{{"text": body, "_pdf_positions": src},}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) <= 1 {t.Fatalf("expected split, got %d", len(got))}@@ -344,7 +344,7 @@ func TestEnforceTitleTokenCap_UnknownPositionsTypeCarriedAsIs(t *testing.T) {chunks := []map[string]any{{"text": body, "positions": "not-a-matrix"},}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) <= 1 {t.Fatalf("expected split, got %d", len(got))}@@ -362,7 +362,7 @@ func TestEnforceTitleTokenCap_GreedyGrouping(t *testing.T) {// 12 sentences, 4 runes each, cap 20 -> 5 sentences/chunk (20), not 1 each.body := strings.Join(joinSentences(12), "")chunks := []map[string]any{{"text": body}}- got := enforceTitleTokenCap(chunks, 20)+ got := enforceTitleTokenCap(chunks, nil, 20)if len(got) != 3 { // 5+5+2t.Fatalf("greedy grouping produced %d chunks, want 3", len(got))}@@ -575,6 +575,211 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {}}+// TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain pins the fix for+// the bug where the cap split stripped the parent headings: every piece of a+// re-split hierarchy chunk must re-prepend the ancestor heading chain, not+// just the first piece.+func TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 24, [][]string{{`^# `, `^## `}})+ two := 2+ p.Hierarchy = &two+ inputText := "# H1\n## H2a\n" + body+ inputs := map[string]any{"output_format": "text", "text": inputText}+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ const chain = "# H1\n## H2a\n"+ var rebuilt strings.Builder+ for i, ck := range chunks {+ text := toString(ck["text"])+ if !strings.HasPrefix(text, chain) {+ t.Errorf("chunk %d lost its heading chain: %q", i, text)+ continue+ }+ if n := titleTokenCount(text); n > 24 {+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)+ }+ if n := strings.Count(text, "# H1\n"); n != 1 {+ t.Errorf("chunk %d repeated the chain %d times: %q", i, n, text)+ }+ rebuilt.WriteString(strings.TrimPrefix(text, chain))+ }+ // Stripping the re-attached chain from every piece must reproduce the+ // original text (build_chunks appends a trailing "\n" per record).+ want := inputText + "\n"+ if got := chain + rebuilt.String(); got != want {+ t.Errorf("chain-stripped pieces not lossless:\n got=%q\nwant=%q", got, want)+ }+}++// TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain pins the+// root_chunk_as_heading interaction: after the root chunk's text is prepended+// to every following chunk, a cap split must still re-prepend the full prefix+// (root text + ancestor chain) to each piece.+func TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 40, [][]string{{`^# `, `^## `}})+ two := 2+ p.Hierarchy = &two+ p.RootChunkAsHeading = true+ inputText := "前言。\n# H1\n## H2a\n" + body+ inputs := map[string]any{"output_format": "text", "text": inputText}+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ // build_chunks joins each record with "\n" (the root chunk text is+ // "前言。\n") and root_chunk_as_heading prepends it plus a separator to+ // every chunk; the chain must survive on every piece of the split.+ const chain = "前言。\n\n# H1\n## H2a\n"+ for i, ck := range chunks {+ text := toString(ck["text"])+ if !strings.HasPrefix(text, chain) {+ t.Errorf("chunk %d lost its heading chain: %q", i, text)+ }+ if n := titleTokenCount(text); n > 40 {+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)+ }+ }+}++// TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain covers the case+// where a non-leaf ancestor carries its own body text: getPaths folds that+// text into the leaf's path before the leaf's own title, so the re-attached+// chain must cover every path record up to and including the last title — not+// just the title lines — or the prefix check fails and the split strips the+// chain again.+func TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 60, [][]string{{`^# `, `^## `, `^### `}})+ three := 3+ p.Hierarchy = &three+ inputText := "# A\n前言A。\n## B\n前言B。\n### C\n" + body+ inputs := map[string]any{"output_format": "text", "text": inputText}+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ const chain = "# A\n前言A。\n## B\n前言B。\n### C\n"+ var rebuilt strings.Builder+ for i, ck := range chunks {+ text := toString(ck["text"])+ if !strings.HasPrefix(text, chain) {+ t.Errorf("chunk %d lost its leading chain: %q", i, text)+ continue+ }+ if n := titleTokenCount(text); n > 60 {+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)+ }+ rebuilt.WriteString(strings.TrimPrefix(text, chain))+ }+ if got := chain + rebuilt.String(); got != inputText+"\n" {+ t.Errorf("chain-stripped pieces not lossless:\n got=%q\nwant=%q", got, inputText+"\n")+ }+}++// TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles pins the+// include_heading_content=true branch: the leading segment is the ancestor+// titles only (bodies are not folded in), and a cap split of the leaf chunk+// must still re-prepend it to every piece. The parent's own chunk is emitted+// unchanged.+func TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 24, [][]string{{`^# `, `^## `}})+ two := 2+ p.Hierarchy = &two+ p.IncludeHeadingContent = true+ inputText := "# H1\n短。\n## H2a\n" + body+ inputs := map[string]any{"output_format": "text", "text": inputText}+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 2 {+ t.Fatalf("expected parent chunk + split leaf pieces, got %d", len(chunks))+ }+ // The parent's own chunk is within cap and emitted as-is.+ if got := toString(chunks[0]["text"]); got != "# H1\n短。\n" {+ t.Errorf("parent chunk = %q, want %q", got, "# H1\n短。\n")+ }+ const chain = "# H1\n## H2a\n"+ var rebuilt strings.Builder+ for i := 1; i < len(chunks); i++ {+ text := toString(chunks[i]["text"])+ if !strings.HasPrefix(text, chain) {+ t.Errorf("chunk %d lost its heading chain: %q", i, text)+ continue+ }+ if n := titleTokenCount(text); n > 24 {+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)+ }+ rebuilt.WriteString(strings.TrimPrefix(text, chain))+ }+ if got := rebuilt.String(); got != body+"\n" {+ t.Errorf("chain-stripped pieces not lossless:\n got=%q\nwant=%q", got, body+"\n")+ }+}++// TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack pins the minimum body+// budget guard: when the repeated prefix would leave the body less than half+// the cap, re-attachment is disabled (plain split) instead of multiplying+// near-empty pieces that each repeat an oversized prefix.+func TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack(t *testing.T) {+ charTokenizer()+ defer restoreTokenizer()+ body := strings.Join(joinSentences(12), "")+ p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `, `^## `}})+ two := 2+ p.Hierarchy = &two+ inputText := "# H1\n## H2a\n" + body+ inputs := map[string]any{"output_format": "text", "text": inputText}+ got, err := invokeHierarchy(testCtx(), nil, inputs, &p)+ if err != nil {+ t.Fatalf("invokeHierarchy: %v", err)+ }+ chunks := got["chunks"].([]map[string]any)+ if len(chunks) <= 1 {+ t.Fatalf("expected split, got %d", len(chunks))+ }+ // 61 stub-tokens of text at cap 20: a plain split yields 4 pieces; a+ // prefix re-attachment with a 1-token body budget would yield ~9.+ if len(chunks) > 5 {+ t.Errorf("oversized prefix must not multiply pieces: got %d", len(chunks))+ }+ for i, ck := range chunks {+ text := toString(ck["text"])+ if n := titleTokenCount(text); n > 20 {+ t.Errorf("chunk %d exceeds cap: tokens=%d text=%q", i, n, text)+ }+ if i > 0 && strings.Contains(text, "# H1") {+ t.Errorf("chunk %d must stay bare when the prefix is too large: %q", i, text)+ }+ }+}+// ---------------------------------------------------------------------------// Schema validation// ---------------------------------------------------------------------------