Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

70.7 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

// SCOPE (honest) for token.go:
//
//   - WHITELIST: delimiter_mode ∈ {"delimiter"} (the
//     single-chunk "one" behaviour moved to OneChunker in one.go).
//     chunk_token_size > 0, overlapped_percent accepts a [0,1) fraction or a
//     [0,90] percentage (normalized to [0,90] by normalizeOverlappedPercent,
//     mirroring Python's normalize_overlapped_percent), table_context_size ≥ 0,
//     image_context_size ≥ 0. enum/range checks live in param.Check.
//
//   - DELIMITER PARSING for the TokenChunker list API mirrors Python
//     token_chunker: only entries wrapped in backticks (e.g. "`\\n\\n`")
//     produce an active split pattern. Plain list entries are not
//     compiled into the pattern. (The single-string parser_config.delimiter
//     field is not parsed in Go; only the []string list API is consumed.)
//
//   - CHILDREN DELIMITERS (the secondary split) is implemented via the
//     splitByDelim helper; emitted chunks carry the parent ("mom") and the
//     split child ("text") keys, with the delimiter retained (lossless).
//
//   - MODE "delimiter" uses the regex-aware delimiter pattern to split
//     text into segments; unlike token_size, these segments are NOT
//     merged — they become standalone chunks.
//
//   - MODE "token_size" implements Python's naive_merge split-then-
//     merge: segments are split by the configured delimiter pattern
//     (chunkFromItem), then greedily merged to chunk_token_size with
//     optional overlap (mergeByTokenSizeFromJSON). The JSON and text
//     payload paths share the same merge after splitting.
//
//   - JSON-STRUCTURED INPUT (output_format == "json", or the default
//     parser-style branch when output_format is unset) is normalized
//     into the same internal chunk shape via a parallel fan-out.
//     Media-context attachment is per-item sequential; merge is
//     index-deterministic.
//
//   - PDF text previews (Python `restore_pdf_text_previews`) are
//     generated on demand for text chunks that carry PDF positions:
//     cropImageChunks crops the text region and writes a preview image,
//     then imageUploadDecorator uploads it to img_id. See pdfcrop_cgo.go.
//
//   - OVER-BUDGET UNITS (contract #17799): a single item that exceeds
//     chunk_token_size is KEPT WHOLE as its own chunk and is NOT
//     atom-split; the embedding/rerank layer truncates it later. The
//     TokenChunker must never sub-split a single item (the naive_merge
//     invariant). If oversized-unit handling is ever needed to avoid a
//     single mega-chunk, it belongs at the CHUNKER side (or a dedicated
//     PreSplitter stage between Parser and Chunker), fed by an EXPLICIT
//     token budget + tokenizer — NOT in the parser, and NOT as a
//     char-window atom-split. The earlier splitOversizedUnit /
//     splitAtomByTokenBudget helpers were a misplaced (parser-layer logic
//     wrongly living in the chunker) and unwired vestige; they were
//     removed to align with this contract.
package chunker

import (
	"context"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
	"sync"
	"unicode"
	"unicode/utf8"

	"go.uber.org/zap"
	"gorm.io/gorm"
	"ragflow/internal/agent/runtime"
	"ragflow/internal/common"
	deepdoctype "ragflow/internal/deepdoc/parser/type"
	"ragflow/internal/ingestion/component/globals"
	"ragflow/internal/ingestion/component/schema"
	"ragflow/internal/parser/parser"

	"ragflow/internal/parser/chunk"
)

type tokenChunkerParam struct {
	schema.TokenChunkerParam
}

func (p *tokenChunkerParam) Update(conf map[string]any) {
	if conf == nil {
		return
	}
	if v, ok := conf["delimiter_mode"].(string); ok {
		p.TokenChunkerParam.DelimiterMode = v
	}
	if v, ok := schema.NumericFromAny(conf["chunk_token_size"]); ok {
		p.TokenChunkerParam.ChunkTokenSize = int(v)
	}
	if v, ok := conf["delimiters"].([]any); ok {
		p.TokenChunkerParam.Delimiters = stringListFromAny(v)
	} else if v, ok := conf["delimiters"].([]string); ok {
		p.TokenChunkerParam.Delimiters = append([]string(nil), v...)
	}
	if v, ok := conf["overlapped_percent"]; ok {
		p.TokenChunkerParam.OverlappedPercent = schema.NormalizeOverlappedPercent(v)
	}
	if v, ok := conf["children_delimiters"].([]any); ok {
		p.TokenChunkerParam.ChildrenDelimiters = stringListFromAny(v)
	} else if v, ok := conf["children_delimiters"].([]string); ok {
		p.TokenChunkerParam.ChildrenDelimiters = append([]string(nil), v...)
	}
	if v, ok := schema.NumericFromAny(conf["table_context_size"]); ok {
		p.TokenChunkerParam.TableContextSize = int(v)
	}
	if v, ok := schema.NumericFromAny(conf["image_context_size"]); ok {
		p.TokenChunkerParam.ImageContextSize = int(v)
	}
}

func defaultsToken(p tokenChunkerParam) tokenChunkerParam {
	p.TokenChunkerParam = schema.TokenChunkerParam{}.Defaults()
	return p
}

// TokenChunkerComponent implements the runtime.Component interface for
// the TokenChunker variant.
type TokenChunkerComponent struct {
	name  string
	param tokenChunkerParam
}

// NewTokenChunker constructs a TokenChunker from the DSL param map.
// Errors here surface as canvas compile failures (mirrors the
// python check() phase).
func NewTokenChunker(params map[string]any) (runtime.Component, error) {
	if err := schema.ValidateDelimiterEntries(params, false); err != nil {
		return nil, runtime.NewUserFacingError(fmt.Sprintf("TokenChunker: %s", err))
	}
	p := defaultsToken(tokenChunkerParam{})
	p.Update(params)
	if err := p.TokenChunkerParam.Validate(); err != nil {
		return nil, fmt.Errorf("TokenChunker: %w", err)
	}
	return &TokenChunkerComponent{
		name:  ComponentNameTokenChunker,
		param: p,
	}, nil
}

// Inputs is exposed so callers can introspect.
func (c *TokenChunkerComponent) Inputs() map[string]string { return ChunkerInputs }

// Outputs is exposed so callers can introspect.
func (c *TokenChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }

// Invoke runs the chunker against the input payload.
//
// Concurrency: text payloads are fanned across 4 goroutines by
// primary-delimiter segment; structured JSON/chunks payloads fan
// across items. Merge is by input index (plan §8 R8): the i-th
// goroutine's output occupies slot i, regardless of completion order.
//
// Timeout: honours ctx cancellation only — there is no inner @timeout
// decorator equivalent (plan §8 R1).
func (c *TokenChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
	return c.invoke(ctx, db, inputs)
}

func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
	if inputs == nil {
		return emptyOutputs(), nil
	}
	// `name` lives in the workflow-wide Globals bag (seeded at pipeline
	// start, published by the File component), not in the upstream output
	// map. decodeChunkerFromUpstream validates it, so carry the resolved
	// name into the decode input.
	name := globals.GlobalOrInput(ctx, inputs, "name", "")
	decInputs := inputs
	if name != "" {
		decInputs = cloneInputs(inputs)
		decInputs["name"] = name
	}
	upstream, err := decodeChunkerFromUpstream(decInputs)
	if err != nil {
		return inputErrorOutput(err), nil
	}

	delimPattern := compileDelimPattern(c.param.Delimiters)
	childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)

	switch upstream.OutputFormat {
	case schema.PayloadFormatMarkdown:
		if upstream.MarkdownResult == nil {
			return emptyOutputs(), nil
		}
		return c.invokeTextPayload(ctx, *upstream.MarkdownResult, delimPattern, childrenPattern), nil
	case schema.PayloadFormatText:
		if upstream.TextResult == nil {
			return emptyOutputs(), nil
		}
		return c.invokeTextPayload(ctx, *upstream.TextResult, delimPattern, childrenPattern), nil
	case schema.PayloadFormatHTML:
		if upstream.HTMLResult == nil {
			return emptyOutputs(), nil
		}
		return c.invokeTextPayload(ctx, *upstream.HTMLResult, delimPattern, childrenPattern), nil
	default:
		// Port of token_chunker.py:347 — when the upstream emitted
		// chunks (output_format == "chunks", e.g. a TitleChunker
		// feeding into this TokenChunker), consume those chunks rather
		// than the raw parser json_result. Otherwise fall back to the
		// structured json_result. This fixes #16812 where a
		// TitleChunker → TokenChunker chain silently discarded the
		// chapter-level chunks and re-chunked the raw parser output.
		var items []schema.ChunkDoc
		if upstream.OutputFormat == schema.PayloadFormatChunks {
			items = upstream.Chunks
		} else {
			items = upstream.JSONResult
		}

		// Re-acquire the source PDF (if the Parser forwarded storage
		// refs) so image/table sections are cropped on demand rather
		// than carried through the wire. Best-effort: a nil engine
		// simply skips cropping.
		engine, engErr := newPDFEngineFromUpstream(ctx, db, upstream)
		if engErr != nil {
			common.Warn("TokenChunker: could not open PDF for on-demand cropping", zap.Error(engErr))
		}
		if engine != nil {
			defer engine.Close()
		}
		return c.invokeJSONPayload(ctx, items, delimPattern, childrenPattern, engine, upstream.FileType), nil
	}
}

func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstream, error) {
	var out schema.ChunkerFromUpstream
	// Fast path: the parser emits typed []parser.DocItem. Convert it to
	// []ChunkDoc directly via schema.ChunkDocsFromDocItem (which projects each
	// item through its canonical ToMap wire shape), then let the JSON marshal
	// below serialize the already-typed ChunkDoc. This keeps the
	// chunker free of the historical map intermediate for the in-memory
	// Parser -> Chunker leg. When inputs["json"] is already []map[string]any
	// (legacy fixtures, round-tripped JSON, or absent), the original path runs.
	inputs = stripChunkerRuntimeTimestamps(inputs)
	if docs, ok := inputs["json"].([]parser.DocItem); ok && len(docs) > 0 {
		conv, _, err := schema.ChunkDocsFromDocItem(docs)
		if err != nil {
			return out, err
		}
		inputs["json"] = conv
	}
	data, err := json.Marshal(inputs)
	if err != nil {
		return out, err
	}
	if err := json.Unmarshal(data, &out); err != nil {
		return out, err
	}
	if err := out.Validate(); err != nil {
		return out, err
	}
	if err := validateUpstreamMediaRefs(out.JSONResult); err != nil {
		return out, err
	}
	if err := validateUpstreamMediaRefs(out.Chunks); err != nil {
		return out, err
	}
	materializeTableText(out.JSONResult)
	materializeTableText(out.Chunks)
	return out, nil
}

// validateUpstreamMediaRefs rejects any present but unusable reference at the
// chunker input boundary. Chunkers merge items and keep only the first media
// attachment, so validating when an item is consumed would make the failure
// depend on input order; every present reference must be checked first.
func validateUpstreamMediaRefs(items []schema.ChunkDoc) error {
	for i := range items {
		if ref := items[i].MediaRef; ref != nil && !ref.Valid() {
			return fmt.Errorf("chunker: item %d has an invalid media_ref", i)
		}
	}
	return nil
}

func stripChunkerRuntimeTimestamps(inputs map[string]any) map[string]any {
	out := make(map[string]any, len(inputs))
	for k, v := range inputs {
		if k == "_created_time" || k == "_elapsed_time" {
			continue
		}
		out[k] = v
	}
	return out
}

// cropTitleChunks crops image/table/text previews for chunks produced by
// the Title/Group/Hierarchy chunkers, mirroring the TokenChunker JSON path
// (cropImageChunks at token.go:513). A nil engine (or an undecodable payload)
// skips cropping, but the returned maps are still normalized so retired
// chunker-internal structural layout never leaves the component.
func cropTitleChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks []map[string]any) []map[string]any {
	if len(chunks) == 0 {
		return chunks
	}
	if engine == nil {
		normalizeTitleChunkMaps(chunks)
		return chunks
	}
	docs, _, err := schema.ChunkDocsFromAny(chunks)
	if err != nil || len(docs) == 0 {
		normalizeTitleChunkMaps(chunks)
		return chunks
	}
	for i := range docs {
		// Only this bridge receives IDs finalized by the title-family
		// builder. Mark them for the shared crop helper without extending
		// ID trust to General/Token chunks decoded from upstream input.
		if docs[i].ID != "" {
			if docs[i].Extra == nil {
				docs[i].Extra = make(map[string]json.RawMessage)
			}
			docs[i].Extra[titleMediaContextIDKey] = json.RawMessage("true")
		}
	}
	cropped := cropImageChunks(ctx, engine, docs)
	for i := range cropped {
		clearStructuralLayout(&cropped[i])
		delete(cropped[i].Extra, titleMediaContextIDKey)
	}
	return schema.ChunkDocsToMaps(cropped)
}

// normalizeTitleChunkMaps keeps the Title-family output shape independent of
// whether a PDF engine is available: chunker-internal structural layout
// never leaves the component.
func normalizeTitleChunkMaps(chunks []map[string]any) {
	for _, chunk := range chunks {
		if layout, _ := chunk["layout"].(string); isStructuralLayout(layout) {
			delete(chunk, "layout")
		}
	}
}

// invokeTextPayload handles plain-text input (output_format in
// {Markdown,text,html} on the python side).
func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
	if text == "" {
		return emptyOutputs()
	}

	// No active delimiter at all: single-section merge that only re-splits
	// oversized sections on sentence boundaries.
	if !hasActiveDelimiter(delimPattern) {
		return c.mergeByTokenSize(text, nil, childrenPattern)
	}

	// Custom (backtick-wrapped) delimiter: one chunk per segment, no token
	// merge — mirrors Python naive_merge's has_custom branch
	// (token_chunker.py:1194-1213).
	if hasCustomDelim(c.param.Delimiters) {
		return c.chunkPerSegment(text, delimPattern, childrenPattern)
	}

	// Bare delimiter: split into paragraphs (delimiter dropped), then merge
	// by token size — mirrors Python naive_merge's default branch. This is
	// the #17723 fix: bare delimiters were previously ignored entirely.
	return c.mergeByTokenSize(text, delimPattern, childrenPattern)
}

// chunkPerSegment splits text on a custom (backtick) delimiter and emits one
// chunk per segment with no token-size merge. Mirrors Python naive_merge's
// has_custom branch (token_chunker.py:1194-1213): the custom delimiter itself
// is DROPPED (and each segment .strip()ed). Only the built-in (non-backtick)
// delimiter set is retained losslessly (see mergeByTokenSize) — a backtick
// delimiter is an explicit split instruction, not text to preserve.
func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
	parts := splitByDelim(text, delimPattern, false)
	cleaned := make([]string, 0, len(parts))
	for _, p := range parts {
		trimmed := strings.TrimSpace(p)
		if trimmed == "" {
			continue
		}
		cleaned = append(cleaned, trimmed)
	}
	if len(cleaned) == 0 {
		return emptyOutputs()
	}
	textDocs := make([]schema.ChunkDoc, 0, len(cleaned))
	for _, s := range cleaned {
		// Python's unstructured text path emits only text here. Do not
		// synthesize structured-input metadata such as doc_type_kwd.
		textDocs = append(textDocs, schema.ChunkDoc{Text: s})
	}
	// A backtick children delimiter is also custom and is dropped; a bare
	// children delimiter (e.g. "。") is retained losslessly like the main
	// built-in set.
	keepChildren := !hasCustomDelim(c.param.ChildrenDelimiters)
	docs := applyChildrenDelimText(textDocs, childrenPattern, keepChildren)
	return chunkOutputs(docs)
}

// sentenceDelimiter is the token-chunker TEXT-path sentence delimiter (the
// naive_merge port). It mirrors the delimiter Python's naive_merge receives in
// production: rag/app/naive.py passes DEFAULT_DELIMITER ("\n!?;。;!?", defined
// once in rag/nlp/delim.py), which includes ASCII "!", "?" and ";" plus the CJK
// punctuation "。;!?" but NOT an English ". " boundary.
//
// The Title chunker and the token chunker's JSON path use the shared
// sentenceBoundaryRe instead (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE,
// which adds ". "). The two constants mirror two distinct Python delimiters and
// differ only in that English boundary, exactly as Python does.
var sentenceDelimiter = regexp.MustCompile(`(\n|[!?;。;!?])`)

// overlapCut returns the visible-text rune offset where the overlap prefix
// begins, mirroring the cut computed inside computeOverlapPrefix. It is split
// out so the coordinate-carrying path reuses the exact same cut as the text
// path (#18148).
func overlapCut(prevText string, overlappedPct float64) int {
	visible := removeTag(prevText)
	runes := []rune(visible)
	cut := int(float64(len(runes)) * (100.0 - overlappedPct) / 100.0)
	if cut < 0 {
		cut = 0
	}
	if cut >= len(runes) {
		return len(runes)
	}
	return cut
}

// computeOverlapPrefix returns (overlapText, overlapTokenCount) carved from
// the tail of prevText after stripping parser tags. overlappedPct is a
// percentage in [0, 100]. Mirrors Python rag/nlp._compute_overlap_prefix.
func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int) {
	visible := removeTag(prevText)
	if visible == "" {
		return "", 0
	}
	runes := []rune(visible)
	cut := overlapCut(prevText, overlappedPct)
	if cut >= len(runes) {
		return "", 0
	}
	overlap := string(runes[cut:])
	return overlap, tokenizeStr(overlap)
}

// mergeByTokenSize implements exact token-based chunk merging that mirrors
// Python naive_merge (rag/nlp/__init__.py) after the strict chunk_token_num
// hard-cap fix.
// It uses tokenizeStr for precise token counting, treats the payload as a
// single section, and splits oversized sections on production sentence
// delimiters. Sections are merged with the unified core (hard-cap expansion +
// UNDER_CAP merge): oversized units are re-split (sentence boundaries first,
// hard token-split fallback) so no text chunk exceeds chunk_token_size. When
// overlap>0 the previous chunk's tail is prepended and trimmed to fit, so the
// hard cap still holds.
func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
	target := c.param.ChunkTokenSize
	overlapPct := c.param.OverlappedPercent
	// Clamp to [0,100] so the merge math below never produces a
	// negative/inverted threshold for an out-of-range value (review:
	// yuzhichang, PR #17396). c.param.OverlappedPercent is already in
	// [0,90] via Update/Validate, so this is a defensive no-op in
	// normal operation.
	if overlapPct < 0 {
		overlapPct = 0
	} else if overlapPct > 100 {
		overlapPct = 100
	}

	// Normalize line endings to LF before any splitting. Python's
	// naive_merge runs text.replace("\r\n", "\n").replace("\r", "\n"),
	// then treats the input string as one section.
	text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")

	// Build the merge units.
	//
	// When a bare (non-custom) delimiter is active we split the text into
	// paragraphs, each paragraph KEEPING its trailing delimiter (lossless), and
	// use it VERBATIM as a unit. Because the delimiter stays attached, the
	// merged text reproduces the source exactly — no "\n" is synthesized, and a
	// sentence period such as "。" is preserved rather than deleted. An
	// oversized paragraph is re-split by the hard-cap expansion in mergeUnits.
	//
	// Otherwise (no active delimiter) the whole text is a single section and
	// oversized sections are re-split on production sentence delimiters, which
	// injects "\n" to mirror Python's sentence-boundary handling. This is the
	// historical behavior and must stay byte-for-byte identical.
	useDelimSplit := hasActiveDelimiter(delimPattern) && !hasCustomDelim(c.param.Delimiters)
	var units []schema.ChunkDoc
	if useDelimSplit {
		// Split on the delimiter and KEEP it attached to the preceding
		// paragraph (lossless). The paragraph retains its original surrounding
		// whitespace; only a truly empty segment is skipped. No "\n" is
		// prepended — that used to compensate for the dropped delimiter and is
		// now wrong (it would double a "\n" or inject a stray newline before a
		// "。").
		for _, sub := range splitByDelim(text, delimPattern, true) {
			if sub == "" {
				continue
			}
			units = append(units, schema.ChunkDoc{Text: sub, TKNums: intPtr(tokenizeStr(sub))})
		}
	} else {
		sections := []string{text}
		for _, sec := range sections {
			sec = strings.TrimSpace(sec)
			if sec == "" {
				continue
			}
			t := "\n" + sec
			tk := tokenizeStr(t)
			if tk <= target {
				units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk)})
				continue
			}
			// Oversized section: split on production sentence delimiters into
			// units. A unit that still exceeds the budget is re-split by the
			// hard-cap expansion in mergeUnits (sentence boundaries first,
			// hard token-split fallback), so no chunk ever exceeds the target.
			parts := sentenceDelimiter.Split(sec, -1)
			hadPart := false
			for _, part := range parts {
				// Keep the raw split fragment, including any inter-line
				// trailing whitespace (Python's naive_merge builds each unit
				// from "\n" + sub_sec with its trailing space, never TrimSpaced).
				// Only genuinely empty fragments are skipped.
				if part == "" {
					continue
				}
				hadPart = true
				seg := "\n" + part
				units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg))})
			}
			if !hadPart {
				units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t))})
			}
		}
	}

	if len(units) == 0 {
		return emptyOutputs()
	}

	// Merge with the unified hard-cap core (oversized-unit expansion + UNDER_CAP
	// merge + overlap prefix trimmed to fit). joinSep is "" for the text path,
	// mirroring Python naive_merge's concatenation of adjacent paragraphs.
	merged := mergeUnits(units, target, overlapPct, "")
	docs := make([]schema.ChunkDoc, 0, len(merged))
	for _, ch := range merged {
		// Strip parser position tags from the final text:
		// the merge paths may carry @@...## markers that must not leak into
		// indexed/embedded chunk text. Only the tags are removed — the source
		// whitespace (including any retained delimiter) is preserved, so the
		// emitted chunk reproduces the source exactly (lossless). Trimming the
		// chunk text would drop a retained delimiter that happens to sit at a
		// chunk boundary.
		setChunkText(&ch, removeTag(ch.Text))
		if ch.Text == "" {
			continue
		}
		docs = append(docs, ch)
	}
	final := applyChildrenDelimText(docs, childrenPattern, !hasCustomDelim(c.param.ChildrenDelimiters))
	return chunkOutputs(final)
}

// invokeJSONPayload handles structured upstream input. Items fan
// across 4 goroutines; merge is by input index.
func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []schema.ChunkDoc, delimPattern, childrenPattern *regexp.Regexp, engine deepdoctype.PDFEngine, fileType string) map[string]any {
	if len(items) == 0 {
		return emptyOutputs()
	}
	workers := 4
	if workers < 1 {
		workers = 1
	}
	if workers > len(items) {
		workers = len(items)
	}
	lanes := partition(len(items), workers)
	perItem := make([][]schema.ChunkDoc, len(items))

	// Computed once before the goroutines so the split path can decide whether
	// the main delimiter is a custom (backtick) one — those are dropped, bare
	// delimiters are retained losslessly.
	customDelim := hasCustomDelim(c.param.Delimiters)

	var wg sync.WaitGroup
	for w := 0; w < workers; w++ {
		lane := lanes[w]
		wg.Add(1)
		go func(start, end int) {
			defer wg.Done()
			for i := start; i < end; i++ {
				if err := ctx.Err(); err != nil {
					perItem[i] = nil
					continue
				}
				if isTextParserSentenceFallback(fileType, c.param.Delimiters, items[i]) {
					perItem[i] = splitTextParserSentences(items[i])
				} else {
					// A custom (backtick) delimiter is an explicit split
					// instruction, not text to preserve: drop it like Python's
					// _split_text_by_pattern. Bare delimiters are retained losslessly.
					perItem[i] = chunkFromItem(items[i], delimPattern, !customDelim)
				}
			}
		}(lane.start, lane.end)
	}
	wg.Wait()
	if err := ctx.Err(); err != nil {
		return map[string]any{
			"output_format": "chunks",
			"chunks":        []map[string]any{},
			"_ERROR":        fmt.Sprintf("TokenChunker: %v", err),
		}
	}
	if !customDelim {
		separateItemBoundaries(perItem)
	}

	// Surrounding media context is attached on the flattened chunk list,
	// mirroring Python's _attach_context_to_media_chunks (token_chunker.py:537,
	// :545): the units are flat there, so a media block collects the text units
	// around it across item boundaries. Attaching per upstream item instead
	// leaves every media chunk without neighbours, because one item rarely
	// yields more than one chunk.
	flat := flatten(perItem)
	// perItem is no longer needed once flattened: drop the [][]ChunkDoc
	// scaffolding (its sub-slice backing arrays) so it can be collected. The
	// ChunkDoc values now live solely in flat.
	perItem = nil

	// Python's naive_merge: custom (backtick) delimiters produce one
	// chunk per segment — no token-size merge (naive_merge:1194-1213).
	// Otherwise split-then-merge: delimiter-split segments are greedily
	// merged to chunk_token_size with optional overlap.
	if !customDelim {
		// Attach before the merge — Python's non-delimiter branch collects the
		// context from the pre-merge text units. The merge only joins text
		// chunks, so a media chunk keeps what it collected.
		flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)
		// Python _merge_text_chunks_by_token_size merges adjacent text
		// chunks across JSON items into one global token budget. The join
		// separator stays empty because delimiters within each item are
		// retained. separateItemBoundaries adds separators only between
		// adjacent text items that have none at their edges.
		flat = flatten(mergeByTokenSizeFromJSON([][]schema.ChunkDoc{flat}, c.param.ChunkTokenSize, c.param.OverlappedPercent))
	}

	if childrenPattern != nil {
		flat = splitByChildren(flat, childrenPattern, !hasCustomDelim(c.param.ChildrenDelimiters))
	}
	if customDelim {
		// Python's delimiter branch splits by children first, then attaches.
		flat = attachMediaContext(flat, c.param.TableContextSize, c.param.ImageContextSize)
	}

	// Crop image/table chunks on demand when a PDF engine is available.
	flat = cropImageChunks(ctx, engine, flat)

	// Fuse the keep-filter and the media-context materialisation into a single
	// pass. The previous code built `out` (a full []ChunkDoc copy of the kept
	// chunks) and then chunkOutputs copied it again into `materialized`. Building
	// `materialized` directly removes one full-document-scale slice allocation
	// from the chunker's peak (the 3266-page PDF holds millions of fine-grained
	// ChunkDocs at this point).
	materialized := make([]schema.ChunkDoc, 0, len(flat))
	for _, m := range flat {
		// Strip parser position tags from the final text:
		// the merge paths may carry @@...## markers that must not leak into
		// indexed/embedded chunk text. Crop above reads positions, not text,
		// so the ordering is safe.
		setChunkText(&m, removeTag(m.Text))
		// Drop a chunk only when nothing about it is retrievable. A media chunk
		// may carry no body of its own and still be indexed through its
		// surrounding context: Python's _finalize_json_chunks strips the merged
		// text and applies the check to it — remove_tag(context_above + text +
		// context_below) then .strip() — so a context that carries nothing but
		// a position tag counts as empty. The context is folded into Text once,
		// at chunkOutputs, so the decision is made on that merged text here
		// without duplicating the fold. Inline bytes, an uploaded ID or a
		// deferred reference also make the chunk retrievable; dropping it here
		// would lose the image before the materializer can resolve or reject it.
		// Position-only media without a crop result is dropped here; crop above
		// already attempted the on-demand PDF handoff.
		if strings.TrimSpace(removeTag(schema.ContextualText(m))) == "" && !hasMediaPayload(m) {
			continue
		}
		materialized = append(materialized, materializeMediaContext(m))
	}
	// flat has been consumed; release it before we build the output map.
	flat = nil
	if len(materialized) == 0 {
		return emptyOutputs()
	}
	return map[string]any{
		"output_format": "chunks",
		"chunks":        schema.ChunkDocsToMaps(materialized),
	}
}

// isTextParserSentenceFallback restores the semantic sentence units that the
// shared TextParser used to emit. Parser boundaries remain owned by the
// chunker: this only applies the text/code parser's default sentence grammar
// before TokenChunker merges units by its configured token budget.
func isTextParserSentenceFallback(fileType string, delimiters []string, item schema.ChunkDoc) bool {
	if itemDocType(item) != "text" || len(delimiters) != 1 || delimiters[0] != "\n" {
		return false
	}
	switch strings.ToLower(strings.TrimSpace(fileType)) {
	case "txt", "py", "js", "java", "c", "cpp", "h", "php", "go", "ts", "sh", "cs", "kt", "sql":
		return true
	default:
		return false
	}
}

func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {
	text := itemTextOrFallback(item)
	if !sentenceDelimiter.MatchString(text) {
		return []schema.ChunkDoc{buildChunkDoc(item, "text", text, "", "")}
	}
	parts := splitSentencesLossless(text)
	out := make([]schema.ChunkDoc, 0, len(parts))
	for _, part := range parts {
		part = strings.TrimSpace(part)
		if part == "" || strings.TrimSpace(sentenceDelimiter.ReplaceAllString(part, "")) == "" {
			continue
		}
		out = append(out, buildChunkDoc(item, "text", part, "", ""))
	}
	if len(out) == 0 {
		return []schema.ChunkDoc{buildChunkDoc(item, "text", text, "", "")}
	}
	return out
}

// ---------------------------------------------------------------------------
// JSON-payload internals
// ---------------------------------------------------------------------------

func separateItemBoundaries(perItem [][]schema.ChunkDoc) {
	var previous *schema.ChunkDoc
	for i := range perItem {
		units := perItem[i]
		if len(units) == 0 {
			continue
		}
		firstIndex, lastIndex := 0, len(units)-1
		for firstIndex <= lastIndex && itemDocType(units[firstIndex]) == "text" && units[firstIndex].Text == "" {
			firstIndex++
		}
		for lastIndex >= firstIndex && itemDocType(units[lastIndex]) == "text" && units[lastIndex].Text == "" {
			lastIndex--
		}
		if firstIndex > lastIndex {
			continue
		}
		first := &units[firstIndex]
		if previous != nil && itemDocType(*previous) == "text" && itemDocType(*first) == "text" &&
			strings.TrimRightFunc(previous.Text, unicode.IsSpace) == previous.Text &&
			strings.TrimLeftFunc(first.Text, unicode.IsSpace) == first.Text {
			first.Text = "\n" + first.Text
			first.TKNums = intPtr(tokenizeStr(first.Text))
		}
		previous = &units[lastIndex]
	}
}

// chunkFromItem mirrors _build_json_chunks for a single item. keepDelim
// follows the same policy as the text path: a bare delimiter is retained so
// the chunk reproduces the source exactly, but a custom (backtick) delimiter
// is dropped (Python-compatible split instruction).
func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {
	ckType := itemDocType(it)
	txt := itemTextOrFallback(it)
	if ckType != "text" {
		return []schema.ChunkDoc{buildChunkDoc(it, ckType, txt, "", "")}
	}
	if !hasActiveDelimiter(delimPattern) {
		return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}
	}
	parts := splitByDelim(txt, delimPattern, keepDelim)
	if !delimPattern.MatchString(txt) {
		return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}
	}
	kept := foldWhitespaceSegments(parts)
	if len(kept) == 0 {
		return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}
	}
	// If the item carries PDF positions, slice them proportionally so each
	// delimiter-split piece's screenshot crops only its own region instead of
	// the whole item bbox (fix for chunk-screenshot mismatch).
	if len(it.PDFPositions) == 0 && len(it.Positions) == 0 {
		out := make([]schema.ChunkDoc, 0, len(kept))
		for _, p := range kept {
			out = append(out, buildChunkDoc(it, "text", p, "", ""))
		}
		return out
	}
	// Visible-text ratio: parser tags (@@...##) carry no visual height
	// and must not shift crop boundaries. Mirrors the overlap logic
	// which already counts on removeTag'd text. Positions are sliced via
	// the shared slicePositionsByCounts helper (mirrored by the General
	// path) so each delimiter-split piece crops only its own region.
	counts := make([]int, len(kept))
	for i, p := range kept {
		counts[i] = utf8.RuneCountInString(removeTag(p))
	}
	pdfSliced, posSliced := slicePositionsByCounts(it.PDFPositions, it.Positions, counts)
	out := make([]schema.ChunkDoc, 0, len(kept))
	for i, p := range kept {
		ck := buildChunkDoc(it, "text", p, "", "")
		ck.PDFPositions = pdfSliced[i]
		ck.Positions = posSliced[i]
		out = append(out, ck)
	}
	return out
}

// buildChunkMap constructs the python-compatible chunk payload.
//
// The chunker output carries the basic text+doc_type_kwd
// fields plus the per-chunk meta fields the python
// rag/flow/chunker/token_chunker.py emits:
//
//   - tk_nums       — tokenized list (used downstream by Tokenizer)
//   - mom           — parent-section identifier (title / hierarchy
//     chunkers populate; TokenChunker pass-through)
//   - img_id        — image attachment identifier
//   - layout        — layout classification (text / table / image / figure)
//   - _pdf_positions — PDF bbox coordinates when the parser path
//     emitted them on the upstream item
//   - context_above / context_below — surrounding media context
//     when attachMediaContext was invoked
//
// Pass-through fields are sourced from the input item map. Missing
// fields are simply absent from the output (the python side does
// the same — see python `_build_json_chunks`).
func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string) schema.ChunkDoc {
	out := schema.ChunkDoc{
		Text:         text,
		DocType:      ckType,
		TKNums:       intPtr(tokenizeStr(text)),
		Mom:          it.Mom,
		ImgID:        it.ImgID,
		Layout:       it.Layout,
		PDFPositions: it.PDFPositions,
		Positions:    it.Positions,
		Image:        it.Image,
		MediaRef:     it.MediaRef,
		PageNumber:   it.PageNumber,
		Sheet:        it.Sheet,
		SheetIndex:   it.SheetIndex,
		Extra:        cloneExtra(it.Extra),
	}
	// buildChunkDoc rewrites every unit to its content type, so the parser's
	// block-kind assertion no longer applies to the emitted chunk.
	clearStructuralLayout(&out)
	if ctxAbove != "" {
		out.ContextAbove = ctxAbove
	}
	if ctxBelow != "" {
		out.ContextBelow = ctxBelow
	}
	return out
}

type lane struct{ start, end int }

func partition(n, parts int) []lane {
	if parts < 1 {
		parts = 1
	}
	if n < parts {
		parts = n
	}
	out := make([]lane, 0, parts)
	size := n / parts
	rem := n % parts
	cursor := 0
	for i := 0; i < parts; i++ {
		end := cursor + size
		if i < rem {
			end++
		}
		if end > n {
			end = n
		}
		if cursor < end {
			out = append(out, lane{start: cursor, end: end})
		}
		cursor = end
	}
	return out
}

// attachMediaContext writes the surrounding context onto the media chunks of a
// flat chunk list. Mirrors token_chunker.py:_attach_context_to_media_chunks,
// which runs on the flat chunk list as well: a media block collects the text
// units around it, up to the token budget, across upstream item boundaries.
func attachMediaContext(chunks []schema.ChunkDoc, tableCtx, imageCtx int) []schema.ChunkDoc {
	if tableCtx <= 0 && imageCtx <= 0 {
		return chunks
	}
	for i, ck := range chunks {
		docType := itemDocType(ck)
		if docType != "table" && docType != "image" {
			continue
		}
		ctx := imageCtx
		if docType == "table" {
			ctx = tableCtx
		}
		if ctx <= 0 {
			continue
		}
		chunks[i].ContextAbove = collectContext(chunks, i, ctx, true)
		chunks[i].ContextBelow = collectContext(chunks, i, ctx, false)
	}
	return chunks
}

// collectContext walks chunks around `i` (above when direction==true,
// below when false), pulling text chunks while remaining token budget
// stays positive. Matches token_chunker.py:_attach_context_to_media_chunks.
// A neighbour that does not fit is truncated on a sentence boundary
// (takeContextSentences), not at an arbitrary rune.
func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) string {
	var parts []string
	remain := ctxTokens
	var pos int
	if above {
		pos = i - 1
		for pos >= 0 && remain > 0 {
			if itemDocType(chunks[pos]) == "text" {
				tk := intValue(chunks[pos].TKNums)
				txt := chunks[pos].Text
				if tk >= remain {
					parts = append([]string{takeContextSentences(txt, remain, true, contextSentencePattern)}, parts...)
					remain = 0
					break
				}
				parts = append([]string{txt}, parts...)
				remain -= tk
			}
			pos--
		}
	} else {
		pos = i + 1
		for pos < len(chunks) && remain > 0 {
			if itemDocType(chunks[pos]) == "text" {
				tk := intValue(chunks[pos].TKNums)
				txt := chunks[pos].Text
				if tk >= remain {
					parts = append(parts, takeContextSentences(txt, remain, false, contextSentencePattern))
					remain = 0
					break
				}
				parts = append(parts, txt)
				remain -= tk
			}
			pos++
		}
	}
	return strings.Join(parts, "")
}

// mergeUnits is the single, unified token-merge core shared by BOTH the text
// path (mergeByTokenSize) and the JSON path (mergeByTokenSizeFromJSON). It is
// a faithful port of Python rag/flow/chunker/token_chunker.py:
// _merge_text_chunks_by_token_size (the JSON strategy) and is also the target
// the Python text path (rag/nlp naive_merge) is migrating to, so the two
// languages and the two paths converge on ONE algorithm.
//
// Hard-cap contract:
//   - Every emitted text chunk is <= target tokens. Oversized units (a single
//     unit whose token count exceeds target) are expanded FIRST: sentence
//     boundaries are tried, then a hard token-split fallback guarantees each
//     piece fits the target. The merge itself is UNDER_CAP: a projected join
//     that would push the running sum over target starts a fresh chunk, so no
//     chunk ever exceeds target.
//   - The merge threshold is SCALED to reserve room for overlap:
//     threshold = target * (100 - overlap) / 100. A chunk keeps receiving
//     units while its running token sum stays <= threshold; once it exceeds
//     threshold the next unit starts a fresh chunk.
//   - When a fresh chunk starts and overlap>0, the tail of the previous chunk
//     is UNCONDITIONALLY prepended (computeOverlapPrefix already strips parser
//     tags). If the overlap prefix would push the fresh chunk over target it is
//     trimmed to fit, so the hard cap still holds with overlap enabled.
//
// Token counts use the RUNNING SUM of per-unit counts (never re-tokenizing the
// joined string), matching Python's tk_nums += current["tk_nums"] (#17948).
// As a hard-cap safety net the merge ALSO re-checks the actual joined text via
// tokenizeStr before accepting a merge: joinSep (e.g. "\n" on the JSON path)
// can add tokens beyond the running sum, so a candidate whose joined text
// would exceed target starts a fresh chunk. The running-sum bookkeeping (and
// therefore chunk TKNums) stays as-is — it matches Python and can under-report
// the joinSep tokens by a few — while the emitted text never exceeds target.
// Non-text units pass through unchanged and reset the merge run. joinSep is
// "\n" for the JSON path and "" for the text path.
// mergeItem records one source unit that contributed to a merged chunk: its
// visible text and its coordinate box groups. Tracking items (not just the
// flattened position list) lets the overlap path map a visible-text offset
// range back to the exact boxes that belong to it, so the overlap prefix
// carries only the previous chunk's tail coordinates (#18148).
type mergeItem struct {
	Text         string
	PDFPositions json.RawMessage
	Positions    json.RawMessage
}

// overlapTailItems returns the previous chunk's source items whose visible span
// intersects the overlap tail [overlapStart, total), each truncated to exactly
// the portion that lies within the overlap tail. Storing them as SEPARATE
// merged_items entries (alongside cur) lets the next overlap select only the
// true tail and converge to a sliding window, instead of carrying the previous
// chunk's HEAD coordinates forward again -- the chained over-carry defect that
// the old fused single-item storage (superseded by this function, #18148)
// suffers from.
//
// Offset model: the overlap path stores the previous chunk's merged_items as
// [tailItems..., cur] and builds cp.Text = overlap + cp.Text. So the source
// items are concatenated with joinSep BETWEEN consecutive items, but there is
// NO separator before the final (cur) item. The offsets below reproduce exactly
// that layout for an overlap-built predecessor, so they line up with
// overlapCut/overlapFitPrefix, which carve the overlap from removeTag'd text.
// Inserting a separator before cur (as the raw "concatenate all items with
// joinSep" model does) would inflate the offsets by one rune on the JSON path
// (joinSep="\n") and over-truncate the tail -- the chained-overlap convergence
// defect flagged by CodeRabbit #1 on #19068.
//
// Limitation: a chunk may also have been built via the non-overlap merge path
// (mergeUnits appends cur with "prev.Text + joinSep + ck.Text"), in which case
// its items DO carry a joinSep before the final cur item. For such a
// predecessor the last item's reconstructed start is short by one rune
// (joinSep length), so a final item that is partially inside the overlap tail
// is truncated one rune too short. This is a 1-rune approximation of the
// highlight box only; it does not affect the head-exclusion property this
// function exists to enforce.
func overlapTailItems(prevItems []mergeItem, overlapStart int, joinSep string) []mergeItem {
	if len(prevItems) == 0 {
		return nil
	}
	// Measure the TAG-FREE visible text of each item so a coordinate tag in an
	// earlier item does not shift the boundaries of later items.
	visible := make([]string, len(prevItems))
	for i, it := range prevItems {
		visible[i] = removeTag(it.Text)
	}
	n := len(prevItems)
	sepLen := utf8.RuneCountInString(joinSep)
	// Reconstruct prev.Text: items joined by joinSep between consecutive items,
	// but with NO separator before the final (cur) item.
	starts := make([]int, n)
	ends := make([]int, n)
	offset := 0
	for i := 0; i < n; i++ {
		if i > 0 && i < n-1 {
			offset += sepLen
		}
		starts[i] = offset
		ends[i] = offset + utf8.RuneCountInString(visible[i])
		offset = ends[i]
	}
	total := offset
	var out []mergeItem
	for i, it := range prevItems {
		start := starts[i]
		end := ends[i]
		if start < total && end > overlapStart {
			lo := start
			if lo < overlapStart {
				lo = overlapStart
			}
			hi := end
			if hi > total {
				hi = total
			}
			runes := []rune(visible[i])
			slice := string(runes[lo-start : hi-start])
			out = append(out, mergeItem{Text: slice, PDFPositions: it.PDFPositions, Positions: it.Positions})
		}
	}
	return out
}

func mergeUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
	if overlapPct < 0 {
		overlapPct = 0
	} else if overlapPct > 100 {
		overlapPct = 100
	}
	// Expand oversized text units into <= target pieces BEFORE merging so the
	// UNDER_CAP merge can never exceed target (hard cap). Non-text units pass
	// through unchanged.
	units = expandOversizedUnits(units, target)
	// Scaled threshold reserves room for the unconditional overlap prefix.
	threshold := float64(target) * (100.0 - overlapPct) / 100.0

	merged := make([]schema.ChunkDoc, 0, len(units))
	// mergedItems parallels merged: for each merged chunk, the source items it
	// was built from. Tracking items (not just the flattened position list)
	// lets the overlap path map a visible-text offset range back to the exact
	// boxes that belong to it, so the overlap prefix carries only the previous
	// chunk's tail coordinates (#18148).
	mergedItems := make([][]mergeItem, 0, len(units))
	prevIdx := -1
	for i := range units {
		ck := units[i]
		if itemDocType(ck) != "text" {
			merged = append(merged, cloneChunkDoc(ck))
			mergedItems = append(mergedItems, nil)
			prevIdx = -1
			continue
		}
		tk := intValue(ck.TKNums)
		if tk <= 0 {
			tk = tokenizeStr(ck.Text)
		}
		cur := mergeItem{Text: ck.Text, PDFPositions: ck.PDFPositions, Positions: ck.Positions}
		if prevIdx < 0 {
			// First text chunk (or first after a non-text chunk): no prior
			// text to overlap with.
			cp := cloneChunkDoc(ck)
			cp.TKNums = intPtr(tk)
			merged = append(merged, cp)
			mergedItems = append(mergedItems, []mergeItem{cur})
			prevIdx = len(merged) - 1
			continue
		}
		prev := &merged[prevIdx]
		// UNDER_CAP: a projected join that would exceed target starts a fresh
		// chunk. The scaled threshold reserves room for the overlap prefix.
		startNew := float64(intValue(prev.TKNums)) > threshold || intValue(prev.TKNums)+tk > target
		if !startNew {
			// The hard cap must hold on the ACTUAL joined text: joinSep (e.g.
			// "\n" on the JSON path) can add tokens beyond the running sum, so
			// re-check the candidate. The running-sum decision and bookkeeping
			// stay unchanged (Python parity #17948); this guard only splits
			// when the joined text itself would exceed target.
			candidate := prev.Text + ck.Text
			if prev.Text != "" && ck.Text != "" {
				candidate = prev.Text + joinSep + ck.Text
			}
			if tokenizeStr(candidate) > target {
				startNew = true
			}
		}
		if startNew {
			cp := cloneChunkDoc(ck)
			if overlapPct > 0 && prev.Text != "" {
				// Unconditional overlap prefix (mirrors Python JSON). The
				// prefix is a duplicate of prev's tail; carry prev's tail
				// coordinates so the overlap region is highlighted (#18148),
				// then keep only cur's coordinates for the non-overlap part.
				// If the prefix would push the fresh chunk over target it is
				// trimmed to fit so the hard cap still holds.
				overlap, _ := computeOverlapPrefix(prev.Text, overlapPct)
				overlap, trimRunes := overlapFitPrefix(overlap, cp.Text, target)
				// Keep the previous chunk's tail as SEPARATE merged_items
				// entries (truncated to the overlap portion, #18148) so the
				// next overlap selects only the true tail and the window
				// converges instead of accumulating head coordinates (chained
				// over-carry fix). The flattened tail boxes are still carried
				// into the output chunk so the overlap region stays highlighted.
				tailItems := overlapTailItems(mergedItems[prevIdx], overlapCut(prev.Text, overlapPct)+trimRunes, joinSep)
				var pdfTail, posTail json.RawMessage
				for _, it := range tailItems {
					pdfTail = extendRawJSONArray(pdfTail, it.PDFPositions)
					posTail = extendRawJSONArray(posTail, it.Positions)
				}
				cp.Text = overlap + cp.Text
				cp.PDFPositions = extendRawJSONArray(pdfTail, cp.PDFPositions)
				cp.Positions = extendRawJSONArray(posTail, cp.Positions)
				cp.TKNums = intPtr(tokenizeStr(cp.Text))
				merged = append(merged, cp)
				mergedItems = append(mergedItems, append(append([]mergeItem{}, tailItems...), cur))
				prevIdx = len(merged) - 1
				continue
			}
			cp.TKNums = intPtr(tk)
			merged = append(merged, cp)
			mergedItems = append(mergedItems, []mergeItem{cur})
			prevIdx = len(merged) - 1
			continue
		}
		// Merge into the previous chunk, maintaining the running token sum.
		if prev.Text != "" && ck.Text != "" {
			prev.Text = prev.Text + joinSep + ck.Text
		} else {
			prev.Text = prev.Text + ck.Text
		}
		prev.TKNums = intPtr(intValue(prev.TKNums) + tk)
		prev.PDFPositions = extendRawJSONArray(prev.PDFPositions, ck.PDFPositions)
		prev.Positions = extendRawJSONArray(prev.Positions, ck.Positions)
		mergedItems[prevIdx] = append(mergedItems[prevIdx], cur)
	}
	return merged
}

// mergeByTokenSizeFromJSON merges the text units of each upstream item using
// the unified mergeUnits core (hard-cap expansion + UNDER_CAP merge),
// mirroring Python token_chunker.py:_merge_text_chunks_by_token_size. Non-text
// units pass through and reset the merge run.
func mergeByTokenSizeFromJSON(perItem [][]schema.ChunkDoc, chunkTokens int, overlappedPct float64) [][]schema.ChunkDoc {
	// overlappedPct is a [0,100] percentage. Clamp defensively because this
	// helper is also exercised directly by tests.
	if overlappedPct < 0 {
		overlappedPct = 0
	} else if overlappedPct > 100 {
		overlappedPct = 100
	}
	for idx := range perItem {
		if len(perItem[idx]) == 0 {
			continue
		}
		// All text units in the sequence are merged with the unified
		// JSON-strategy core. Non-text units pass through and reset the merge
		// run (see mergeUnits). Join separator is "" because each unit already
		// carries its own trailing delimiter (splitByDelim keeps it), so an
		// empty separator reproduces the source exactly. A "\n" separator would
		// have been needed when the delimiter was dropped, but would now
		// double-insert it.
		perItem[idx] = mergeUnits(perItem[idx], chunkTokens, overlappedPct, "")
	}
	return perItem
}

// ---------------------------------------------------------------------------
// Hard-cap expansion
// ---------------------------------------------------------------------------

// expandOversizedUnits re-splits text units whose token count exceeds target
// so every returned text unit is <= target tokens. Sentence boundaries are
// tried first (delimiters preserved, lossless); a piece that still exceeds
// target is hard-split on token count (char-prefix via TrimContentToTokenLimit,
// also lossless). Every sub-piece keeps the source unit's (coarse) PDF
// positions so each piece still gets its page-region preview image and
// highlight. Non-text units pass through unchanged.
func expandOversizedUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {
	if target <= 0 {
		return units
	}
	out := make([]schema.ChunkDoc, 0, len(units))
	for _, ck := range units {
		if itemDocType(ck) != "text" {
			out = append(out, ck)
			continue
		}
		tk := intValue(ck.TKNums)
		if tk <= 0 {
			tk = tokenizeStr(ck.Text)
		}
		if tk <= target {
			if ck.TKNums == nil {
				ck.TKNums = intPtr(tk)
			}
			out = append(out, ck)
			continue
		}
		out = append(out, splitOversizedText(ck, target)...)
	}
	return out
}

// pieceVisibleCount returns the visible-text weight of a split piece: the
// rune count after stripping parser tags (@@...##), excluding the synthetic
// leading "\n" glue on the first piece (text-path units are built as
// "\n"+paragraph, which carries no visual height).
func pieceVisibleCount(text string, isFirst bool, lead string) int {
	n := utf8.RuneCountInString(removeTag(text))
	if isFirst && lead != "" {
		n -= utf8.RuneCountInString(lead)
		if n < 0 {
			n = 0
		}
	}
	return n
}

// splitOversizedText splits one oversized text unit into <= target pieces.
// Sentence boundaries are preferred; a boundary-less run that still exceeds
// target is hard token-split. Delimiters are preserved so the concatenated
// pieces reproduce the unit text exactly (lossless). Every piece keeps the
// source unit's DocType and (coarse) PDF positions, built from a clone of the
// source so all ChunkDoc fields survive the split.
func splitOversizedText(ck schema.ChunkDoc, target int) []schema.ChunkDoc {
	text := ck.Text
	// Preserve a leading "\n" glue (text-path units are built as "\n"+part).
	lead := ""
	if strings.HasPrefix(text, "\n") {
		lead = "\n"
		text = strings.TrimPrefix(text, "\n")
	}
	// Split into sentences (delimiters attached). A whitespace-only fragment
	// (a bare delimiter from a repeated run, e.g. "\n\n") is prepended to the
	// NEXT non-whitespace piece so the concatenated pieces reproduce the unit
	// text exactly; trailing whitespace attaches to the last piece. If
	// attaching whitespace pushes a piece over the target, that piece is
	// hard-split so the hard cap still holds.
	var pieces []schema.ChunkDoc
	var wsPending string
	for _, part := range splitSentencesLossless(text) {
		if strings.TrimSpace(part) == "" {
			wsPending += part
			continue
		}
		p := schema.ChunkDoc{Text: wsPending + part, DocType: ck.DocType}
		wsPending = ""
		if n := tokenizeStr(p.Text); n > target {
			pieces = append(pieces, hardSplitPiece(p.Text, ck.DocType, target)...)
		} else {
			p.TKNums = intPtr(n)
			pieces = append(pieces, p)
		}
	}
	if wsPending != "" {
		if len(pieces) > 0 {
			last := &pieces[len(pieces)-1]
			last.Text += wsPending
			if n := tokenizeStr(last.Text); n > target {
				replaced := hardSplitPiece(last.Text, ck.DocType, target)
				pieces = append(pieces[:len(pieces)-1], replaced...)
			} else {
				last.TKNums = intPtr(n)
			}
		} else {
			// All-whitespace unit: keep it lossless but still within the cap —
			// hard-split like any other oversized text so a whitespace-only run
			// whose token count exceeds the target cannot emit an over-cap chunk.
			if n := tokenizeStr(wsPending); n > target {
				pieces = append(pieces, hardSplitPiece(wsPending, ck.DocType, target)...)
			} else {
				pieces = append(pieces, schema.ChunkDoc{Text: wsPending, DocType: ck.DocType, TKNums: intPtr(n)})
			}
		}
	}
	if len(pieces) == 0 {
		return nil
	}
	if lead != "" {
		pieces[0].Text = lead + pieces[0].Text
		// The leading "\n" adds a token; recompute so the running sum stays
		// faithful. Re-split if it pushes the piece over the target.
		if n := tokenizeStr(pieces[0].Text); n > target {
			replaced := hardSplitPiece(pieces[0].Text, ck.DocType, target)
			pieces = append(replaced, pieces[1:]...)
		} else {
			pieces[0].TKNums = intPtr(n)
		}
	}
	// Every sub-piece inherits the source unit's metadata (coarse positions +
	// item attributes); each is built from a clone so every ChunkDoc field
	// survives the split. PDF positions are sliced via the shared
	// slicePositionsByCounts helper so each piece's screenshot crops only
	// its own region instead of the whole paragraph (fix for
	// chunk-screenshot mismatch).
	counts := make([]int, len(pieces))
	for i, p := range pieces {
		counts[i] = pieceVisibleCount(p.Text, i == 0, lead)
	}
	pdfSliced, posSliced := slicePositionsByCounts(ck.PDFPositions, ck.Positions, counts)
	for i, p := range pieces {
		inherited := cloneChunkDoc(ck)
		inherited.Text = p.Text
		inherited.TKNums = p.TKNums
		clearStructuralLayout(&inherited)
		inherited.PDFPositions = pdfSliced[i]
		inherited.Positions = posSliced[i]
		pieces[i] = inherited
	}
	return pieces
}

// splitSentencesLossless splits text on sentenceDelimiter, keeping each
// delimiter attached to its preceding sentence, so the concatenated parts
// reproduce the input exactly (lossless). Mirrors Python's
// _split_text_by_sentences (rag/flow/chunker/title_chunker/common.py).
func splitSentencesLossless(text string) []string {
	if text == "" {
		return nil
	}
	idxs := sentenceDelimiter.FindAllStringIndex(text, -1)
	var out []string
	prev := 0
	for _, idx := range idxs {
		if idx[0] > prev {
			out = append(out, text[prev:idx[1]])
		} else {
			out = append(out, text[idx[0]:idx[1]])
		}
		prev = idx[1]
	}
	if prev < len(text) {
		out = append(out, text[prev:])
	}
	return out
}

// hardSplitPiece hard token-splits a boundary-less run into <= target pieces.
// The run is encoded once and the token array walked in target-sized steps,
// each slice decoded straight back to its exact byte range — O(L) overall,
// where the former loop re-encoded a shrinking remainder every round and went
// quadratic on long boundary-less inputs. Pieces concatenate back to the input
// exactly, and every cut lands on an original token boundary, so a piece is a
// true byte prefix of the remainder (a cut may leave raw UTF-8 continuation
// bytes at the tail when the boundary splits a multibyte rune; the next
// piece's head repairs them). A token cut never lands inside a @@...##
// coordinate tag: the cut is extended past the closing "##" when the budget
// allows, otherwise backed off to before the "@@" — so a tag is never split
// across two pieces.
//
// When the encoder is unavailable, hardSplitPieceByteLoop (the former
// per-round-trim loop) takes over and degrades to byte-length trimming.
func hardSplitPiece(text string, docType string, target int) []schema.ChunkDoc {
	if text == "" {
		return nil
	}
	tokens, ok := encodeTokens(text)
	if !ok {
		return hardSplitPieceByteLoop(text, docType, target)
	}
	if target <= 0 || len(tokens) <= target {
		// target <= 0 is unreachable from validated configs; the byte loop
		// would emit the whole text as one piece, so mirror that here.
		return []schema.ChunkDoc{{Text: text, DocType: docType, TKNums: intPtr(tokenizeStr(text))}}
	}

	// offsets[i] = byte length of the first i tokens decoded. Token ids repeat
	// heavily in natural text, so per-id byte lengths are memoized.
	offsets := make([]int, len(tokens)+1)
	lens := make(map[int]int, len(tokens)/2)
	for i, tok := range tokens {
		l, seen := lens[tok]
		if !seen {
			l = len(decodeTokens(tokens[i : i+1]))
			lens[tok] = l
		}
		offsets[i+1] = offsets[i] + l
	}
	if offsets[len(tokens)] != len(text) {
		// Unreachable with a byte-level BPE (decode∘encode is the identity on
		// the byte string); a mismatch here would corrupt every piece, so
		// degrade to the byte loop instead of emitting it.
		return hardSplitPieceByteLoop(text, docType, target)
	}

	// @@...## tag spans in byte offsets. The leftmost-first scan with the
	// resume index mirrors adjustCutPastTag's paired-tag search; spans come
	// out sorted and non-overlapping.
	type tagSpan struct{ start, end int }
	var tags []tagSpan
	for i := 0; i < len(text); {
		s := strings.Index(text[i:], "@@")
		if s < 0 {
			break
		}
		s += i
		eRel := strings.Index(text[s+2:], "##")
		if eRel < 0 {
			break
		}
		tags = append(tags, tagSpan{s, s + 2 + eRel + 2})
		i = tags[len(tags)-1].end
	}

	var out []schema.ChunkDoc
	p, tagIdx := 0, 0
	for p < len(tokens) {
		q := p + target
		if q > len(tokens) {
			q = len(tokens)
		}
		// Consume tags that end at or before the cut; they can no longer
		// matter. A tag the cut lands inside is resolved below and
		// deliberately not consumed: after a back-off the next cut can
		// re-enter the same tag.
		for tagIdx < len(tags) && tags[tagIdx].end <= offsets[q] {
			tagIdx++
		}
		if tagIdx < len(tags) && tags[tagIdx].start < offsets[q] && offsets[q] < tags[tagIdx].end {
			tag := tags[tagIdx]
			endTok := q
			for endTok < len(tokens) && offsets[endTok] < tag.end {
				endTok++
			}
			startTok := q
			for startTok > p && offsets[startTok] > tag.start {
				startTok--
			}
			// Extend past the tag when the piece stays within budget, or when
			// the tag starts the piece and backing off would stall; otherwise
			// back off to the last token boundary at or before the "@@".
			// startTok == p exactly when the piece begins at (or inside) the
			// tag, so the forced extension of adjustCutPastTag's leading-tag
			// case is preserved.
			if endTok-p <= target || startTok == p {
				q = endTok
			} else {
				q = startTok
			}
		}
		if q <= p {
			q = p + 1 // never stall (defensive; unreachable above)
		}
		piece := text[offsets[p]:offsets[q]]
		out = append(out, schema.ChunkDoc{Text: piece, DocType: docType, TKNums: intPtr(tokenizeStr(piece))})
		p = q
	}
	return out
}

// hardSplitPieceByteLoop is the pre-O(L) body of hardSplitPiece, kept as the
// encoder-unavailable fallback. With the encoder dead, trimToTokenLimit
// degrades to a UTF-8-safe byte-length trim and the loop still emits bounded,
// lossless pieces.
func hardSplitPieceByteLoop(text string, docType string, target int) []schema.ChunkDoc {
	var out []schema.ChunkDoc
	rest := text
	for {
		// One BPE pass per iteration instead of two: trimToTokenLimit (the
		// package-level tokenizer seam, see title_cap.go) returns rest
		// unchanged exactly when rest fits within target tokens, so its
		// result doubles as the loop-exit condition. The shrinking remainder
		// is still re-encoded from scratch each round — one encode per
		// iteration, not two.
		head := trimToTokenLimit(rest, target)
		if head == rest {
			break // rest already fits; the trailing append emits it
		}
		if head == "" {
			break // cannot shrink further; avoid an infinite loop
		}
		// TrimContentToTokenLimit decodes a token prefix; if that lands on a
		// mid-multibyte boundary it can emit a trailing U+FFFD which is not a
		// true prefix of rest. Trim it so the rest[len(head):] advance stays
		// lossless.
		if !strings.HasPrefix(rest, head) {
			head = strings.TrimRight(head, "\uFFFD")
			if head == "" {
				break
			}
		}
		cut := adjustCutPastTag(rest, len(head), target)
		if cut <= 0 || cut >= len(rest) {
			// Safety: never stall. An empty cut or a whole-text cut means no
			// progress is possible; emit the remaining text as the last piece.
			if rest != "" {
				out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest))})
			}
			rest = ""
			break
		}
		head = rest[:cut]
		out = append(out, schema.ChunkDoc{Text: head, DocType: docType, TKNums: intPtr(tokenizeStr(head))})
		rest = rest[cut:]
	}
	if rest != "" {
		out = append(out, schema.ChunkDoc{Text: rest, DocType: docType, TKNums: intPtr(tokenizeStr(rest))})
	}
	return out
}

// adjustCutPastTag shifts a character cut so it never lands inside a
// @@...## coordinate tag: extending past the closing "##" is preferred when
// the token budget allows; otherwise the cut backs off to just before the
// "@@". Tags with no closing "##" (unterminated) are ignored — they have no
// paired span to protect.
func adjustCutPastTag(text string, cut, target int) int {
	searchFrom := 0
	for {
		s := strings.Index(text[searchFrom:], "@@")
		if s < 0 {
			return cut
		}
		s += searchFrom
		eRel := strings.Index(text[s+2:], "##")
		if eRel < 0 {
			return cut // unterminated tag; nothing to protect
		}
		e := s + 2 + eRel + 2
		if cut > s && cut < e {
			// The cut lands inside a tag: prefer extending past the tag if it
			// stays within the token budget, else back off to before it. A
			// leading tag (s == 0) cannot back off to an empty piece, so it is
			// kept whole by extending past it.
			if e <= len(text) && tokenizeStr(text[:e]) <= target {
				return e
			}
			if s > 0 {
				return s
			}
			return e
		}
		searchFrom = e
	}
}

// overlapFitPrefix trims the overlap prefix so overlap+current fits within
// target tokens (hard cap with overlap enabled). It returns the trimmed
// overlap and the number of runes removed from its front, so callers can
// adjust the position-tail cut to the actual overlap region.
func overlapFitPrefix(overlap, current string, target int) (string, int) {
	if tokenizeStr(overlap+current) <= target {
		return overlap, 0
	}
	runes := []rune(overlap)
	for c := 1; c <= len(runes); c++ {
		suffix := string(runes[c:])
		if tokenizeStr(suffix+current) <= target {
			return suffix, c
		}
	}
	// current alone should never exceed target (units are pre-expanded); if it
	// somehow does, drop the overlap entirely.
	return "", len(runes)
}

func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {
	out := in
	if in.TKNums != nil {
		v := *in.TKNums
		out.TKNums = &v
	}
	if in.ChunkOrderInt != nil {
		v := *in.ChunkOrderInt
		out.ChunkOrderInt = &v
	}
	if in.PageNumber != nil {
		v := *in.PageNumber
		out.PageNumber = &v
	}
	// Deep-copy the coordinate byte slices so the clone does not alias
	// the source's backing array (diff 2.5 defensive fix).
	if in.PDFPositions != nil {
		out.PDFPositions = append(json.RawMessage(nil), in.PDFPositions...)
	}
	if in.Positions != nil {
		out.Positions = append(json.RawMessage(nil), in.Positions...)
	}
	out.Extra = cloneExtra(in.Extra)
	return out
}

func cloneExtra(in map[string]json.RawMessage) map[string]json.RawMessage {
	if in == nil {
		return nil
	}
	out := make(map[string]json.RawMessage, len(in))
	for key, value := range in {
		out[key] = append(json.RawMessage(nil), value...)
	}
	return out
}

// extendRawJSONArray concatenates two JSON array payloads, mirroring
// Python's `merged[prev][KEY].extend(current[KEY])`. Either operand may be
// empty; the result is always a valid JSON array (or an empty raw message).
//
// It is used to accumulate PDF coordinate lists (`_pdf_positions`,
// `positions`) when text chunks are merged (diffs 2.5 / 2.3).
//
// Fast path (compact arrays): the positions that flow through here are always
// json.Marshal'd parser output, i.e. compact `[...]` arrays with no internal
// whitespace. Instead of the old per-step json.Unmarshal + json.Marshal
// (which re-serialised the ENTIRE growing array on every merge step — O(n^2)
// bytes plus reflection — and dominated the chunker's wall time / heap peak on
// large PDFs, see the 10k-unit benchmark: ~11 GB allocated per merge), we strip
// the outer brackets and append the element bytes directly. Appending into `a`'s
// (re)growing buffer is O(n) total. The merged coordinate list is semantically
// identical (extend, not replace) and byte-for-byte equal to the old path for
// compact input.
//
// Slow path: for any non-compact-array / malformed operand we fall back to the
// exact prior behaviour (unmarshal + marshal), so output never diverges from the
// historical contract.
//
// Mutation contract (append-like): the fast path grows `a`'s backing buffer in
// place to accumulate the result, so the returned slice may alias / reuse `a`'s
// storage and `a` must be treated as consumed after the call. Callers that pass
// a shared or reused buffer as `a` must copy it first; the merge path satisfies
// this naturally because `pdfTail`/`posTail` are dedicated accumulators that own
// their buffers. When `a` is empty the result is a fresh COPY of `b` (never `b`
// itself) so the first extension never aliases the caller's source coordinate
// bytes.
func extendRawJSONArray(a, b json.RawMessage) json.RawMessage {
	if len(a) == 0 {
		// Return a COPY of b, not b itself: the caller stores the result and
		// the fast path may later mutate its backing array in place. Returning
		// b directly would alias the caller's (possibly shared) source bytes,
		// e.g. the overlap branch re-extends the same growing buffer and would
		// corrupt the original unit coordinate list (see #18148 overlap chain).
		return append(json.RawMessage(nil), b...)
	}
	if len(b) == 0 {
		return append(json.RawMessage(nil), a...)
	}
	if a[0] == '[' && a[len(a)-1] == ']' && b[0] == '[' && b[len(b)-1] == ']' {
		innerA := a[1 : len(a)-1]
		innerB := b[1 : len(b)-1]
		// Drop a's trailing ']' and append b's elements followed by a fresh
		// ']'. append reuses/grows a's buffer (amortised O(n) total).
		a = a[:len(a)-1]
		if len(innerA) > 0 && len(innerB) > 0 {
			a = append(a, ',')
		}
		a = append(a, innerB...)
		a = append(a, ']')
		return a
	}
	// Slow path (unchanged semantics).
	var arrA, arrB []json.RawMessage
	if err := json.Unmarshal(a, &arrA); err != nil {
		return b
	}
	if err := json.Unmarshal(b, &arrB); err != nil {
		return a
	}
	arrA = append(arrA, arrB...)
	out, err := json.Marshal(arrA)
	if err != nil {
		return a
	}
	return out
}

func flatten(perItem [][]schema.ChunkDoc) []schema.ChunkDoc {
	var out []schema.ChunkDoc
	for _, cs := range perItem {
		out = append(out, cs...)
	}
	return out
}

func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {
	if pattern == nil {
		return chunks
	}
	var out []schema.ChunkDoc
	for _, ck := range chunks {
		if ck.DocType != "text" {
			out = append(out, ck)
			continue
		}
		mom := strings.TrimPrefix(ck.Text, "\n")
		parts := splitByDelim(ck.Text, pattern, keepDelim)
		// Whitespace-only pieces from consecutive delimiters ("A\n\nB") must be
		// folded into an adjacent child, not discarded — otherwise the repeated
		// newline disappears from the child text. A leading whitespace-only
		// piece (no preceding child of this parent yet) is buffered and
		// prepended to the first real child. This only matters when the
		// delimiter is retained (keepDelim); a dropped custom delimiter leaves
		// no whitespace-only pieces behind.
		var leading string
		for _, p := range parts {
			if strings.TrimSpace(p) == "" {
				if n := len(out); n > 0 && out[n-1].Mom == mom {
					prev := out[n-1]
					setChunkText(&prev, prev.Text+p)
					out[n-1] = prev
					continue
				}
				leading += p
				continue
			}
			if leading != "" {
				p = leading + p
				leading = ""
			}
			cp := cloneChunkDoc(ck)
			// The count describes the child's own text. The delimiter branch
			// attaches the media context after this split, and that walk is
			// charged with TKNums, so an inherited parent count would spend the
			// configured window on the first neighbour.
			setChunkText(&cp, p)
			cp.Mom = mom
			out = append(out, cp)
		}
	}
	return out
}

// ---------------------------------------------------------------------------
// shared text-payload helpers (used by TitleChunker et al.)
// ---------------------------------------------------------------------------

// hasActiveDelimiter reports whether a compiled delimiter pattern is
// present (non-nil). compileDelimPattern returns nil when no active
// pattern exists, so a nil check is sufficient.
func hasActiveDelimiter(p *regexp.Regexp) bool {
	return p != nil
}

// hasCustomDelim reports whether any delimiter uses backtick syntax
// (`pattern`). Python's naive_merge skips token-size merging when
// custom delimiters are present. Delegates to the canonical helper.
func hasCustomDelim(delims []string) bool {
	return chunk.HasCustomDelimiterList(delims)
}

func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp, keepDelim bool) []schema.ChunkDoc {
	if pattern == nil {
		return docs
	}
	var out []schema.ChunkDoc
	for _, d := range docs {
		t := d.Text
		if strings.TrimSpace(t) == "" {
			continue
		}
		mom := strings.TrimPrefix(t, "\n")
		// Whitespace-only pieces from consecutive delimiters must be folded into
		// an adjacent child (or prepended to the first child when leading), so a
		// repeated newline in the source is not silently dropped. This only
		// matters when the delimiter is retained (keepDelim); a dropped custom
		// delimiter leaves no whitespace-only pieces behind.
		//
		// Folding is scoped to the CURRENT parent only: parentStart records how
		// many children existed before this parent's pieces, so a whitespace
		// piece may only attach to this parent's own last child. Comparing
		// out[n-1].Mom would be wrong because Mom holds the parent TEXT
		// (TrimPrefix(t, "\n")), not a parent identity; two distinct parents
		// with identical text would then cross-fold, corrupting one child and
		// dropping the other's delimiter.
		parentStart := len(out)
		var leading string
		for _, text := range splitByDelim(t, pattern, keepDelim) {
			if strings.TrimSpace(text) == "" {
				if n := len(out); n > parentStart {
					prev := out[n-1]
					setChunkText(&prev, prev.Text+text)
					out[n-1] = prev
					continue
				}
				leading += text
				continue
			}
			if leading != "" {
				text = leading + text
				leading = ""
			}
			child := schema.ChunkDoc{DocType: d.DocType, Layout: d.Layout, Mom: mom}
			setChunkText(&child, text)
			out = append(out, child)
		}
	}
	return out
}

func intValue(v *int) int {
	if v == nil {
		return 0
	}
	return *v
}

func intPtr(v int) *int { return &v }

// init registers TokenChunker under CategoryIngestion.
func init() {
	MustRegisterChunker(ComponentNameTokenChunker)
}