RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): keep structured tables and images in the title family (#21040)
jay777210ad97e70af66
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 50b112f5ac..ea99778323 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -17,6 +17,8 @@package chunkerimport (+ "crypto/sha256"+ "encoding/hex""fmt""regexp""strings"@@ -25,6 +27,7 @@ import ("ragflow/internal/common""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/chunk"+ "ragflow/internal/parser/tableutil""ragflow/internal/tokenizer")@@ -233,6 +236,20 @@ func itemTextOrFallback(it schema.ChunkDoc) string {return ""}+// materializeTableText fills Text with the HTML rendering of a structured+// TableData when the item carries no text of its own — the shape every parser+// emits for tables. Every consumer then works on a real table it can store and+// display, and the chunk id (a hash of Text) matches the stored content.+// RenderTableHTML marks the leading HeaderRows rows as <th>. Idempotent: an+// item that already carries text is left untouched.+func materializeTableText(docs []schema.ChunkDoc) {+ for i := range docs {+ if docs[i].TableData != nil && docs[i].Text == "" {+ docs[i].Text = tableutil.RenderTableHTML(docs[i].TableData)+ }+ }+}+// tokenizeStr is the shared NumTokensFromString wrapper used by// Table/Image context attachment. Lives here so we can centrally// swizzle the count strategy in one place if needed.@@ -268,6 +285,16 @@ func emptyOutputs() map[string]any {}}+// inputErrorOutput is the canonical short-circuit for an undecodable upstream+// payload, shared by every chunker variant's decode step.+func inputErrorOutput(err error) map[string]any {+ return map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{},+ "_ERROR": fmt.Sprintf("Input error: %v", err),+ }+}+// chunkOutputs builds the canonical chunker output (output_format="chunks" +// chunks). The Go runtime passes only this explicit output to the next node,// so the run-level metadata that downstream components still need (e.g.@@ -303,17 +330,51 @@ func canonicalChunkText(ck schema.ChunkDoc) string {return removeTag(materializeMediaContext(ck).Text)}+// mediaChunkIDPrefix marks the identity source of a media chunk whose+// canonical text is empty. The NUL byte keeps the source from colliding with+// any real chunk text.+const mediaChunkIDPrefix = "\x00media:"+// canonicalChunkID returns the deterministic chunk id: the single identity// used both as the MinIO object key and as ck["id"]. The id formula (ChunkID-// over canonicalChunkText) is centralized here, so the text normalization-// that feeds the hash lives in exactly one place. The decorator in-// register.go re-derives ck["id"] from the same already-finalized text as a-// fallback for chunks that bypassed the streamed crop-upload path; the two-// routes cannot disagree because a chunk is either stamped by crop-upload-// (and the decorator reuses that id) or computed by the decorator fallback —-// never both — and both hash the normalized chunk body.+// over chunkIDSource) is centralized here, so the identity that feeds the+// hash lives in exactly one place. The decorator in register.go re-derives+// ck["id"] from the same already-finalized text as a fallback for chunks that+// bypassed the streamed crop-upload path; the two routes cannot disagree+// because a chunk is either stamped by crop-upload (and the decorator reuses+// that id) or computed by the decorator fallback — never both — and both hash+// the same identity.func canonicalChunkID(docID string, ck schema.ChunkDoc) string {- return common.ChunkID(docID, canonicalChunkText(ck))+ return common.ChunkID(docID, chunkIDSource(ck))+}++// chunkIDSource returns the identity string hashed into a chunk id. For a+// media chunk whose canonical text is empty — a captionless image — hashing+// the text alone would give every such chunk in a document the same id+// (ChunkID(docID, "")), silently overwriting images in MinIO and merging+// index entries, so a stable per-image identity is folded in instead.+// Non-empty text hashes unchanged.+func chunkIDSource(ck schema.ChunkDoc) string {+ if text := canonicalChunkText(ck); text != "" {+ return text+ }+ return mediaChunkIdentity(ck.Image, ck.ImgID)+}++// mediaChunkIdentity returns the stable identity of a text-less media chunk:+// a digest of the image payload when present (identical images collapse onto+// one id, which is harmless — they carry identical bytes), else the image+// reference. Empty when the chunk has neither, leaving the id the plain+// empty-text hash.+func mediaChunkIdentity(image, imgID string) string {+ if image != "" {+ sum := sha256.Sum256([]byte(image))+ return mediaChunkIDPrefix + hex.EncodeToString(sum[:])+ }+ if imgID != "" {+ return mediaChunkIDPrefix + imgID+ }+ return ""}// materializeMediaContext folds a media chunk's surrounding context into itsdiff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex 84d1ee8f1a..2a48eca1ef 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -110,7 +110,11 @@ func buildSectionIDs(levels []int, targetLevel int) []int {// shared chunkFromRecords pipeline (which ManualChunker also uses,// after a physical-position resort).func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {- records := extractLineRecords(inputs)+ upstream, err := decodeChunkerFromUpstream(inputs)+ if err != nil {+ return inputErrorOutput(err), nil+ }+ records := extractLineRecords(upstream)common.Debug("chunker stage",zap.String("component", "Chunker"),zap.String("variant", "group"),@@ -119,7 +123,7 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,if len(records) == 0 {return emptyOutputs(), nil}- return chunkFromRecords(parentCtx, db, inputs, p, records, p.ChunkTokenCap)+ return chunkFromRecords(parentCtx, db, upstream, p, records, p.ChunkTokenCap)}// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over@@ -136,8 +140,8 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,// must stay exempt from #18455's cap, so it passes 0; GroupTitleChunker passes// p.ChunkTokenCap. The cap is applied right after build_chunks and BEFORE the// on-demand crop, mirroring Python's invoke() order (build -> cap -> set_chunks).-func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord, tokenCap int) (map[string]any, error) {- ctx := newLevelContext(records, outlineFromInputs(inputs), p)+func chunkFromRecords(parentCtx context.Context, db *gorm.DB, upstream schema.ChunkerFromUpstream, p *titleChunkerParam, records []lineRecord, tokenCap int) (map[string]any, error) {+ ctx := newLevelContext(records, outlineFromUpstream(upstream), p)levels := ctx.Levels()// Count heading level distribution for debugging.headingCounts := make(map[int]int)@@ -173,7 +177,7 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]zap.String("variant", "group"),zap.Int("groups", len(groups)),)- chunks, _ := buildChunksFromRecordGroups(groups, nil, p, isPlainTextFormat(inputs))+ chunks, _ := buildChunksFromRecordGroups(groups, nil, p, isPlainTextFormat(upstream))// Enforce the title-family token ceiling (chunk_token_cap) right after// build_chunks and BEFORE the on-demand crop, mirroring Python's// invoke() order (build -> cap -> set_chunks). ManualChunker passes@@ -185,21 +189,19 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]zap.String("component", "Chunker"),zap.String("variant", "group"),zap.Int("chunks", len(chunks)),- zap.Bool("plain_text", isPlainTextFormat(inputs)),+ zap.Bool("plain_text", isPlainTextFormat(upstream)),)// On-demand PDF preview cropping for image/table/text chunks,// mirroring the TokenChunker JSON path (token.go:513). Best-effort:// a missing or unreadable PDF simply skips cropping.- if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {- engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)- if eErr != nil {- common.Warn("chunker: could not open PDF for on-demand cropping", zap.Error(eErr))- }- if engine != nil {- defer engine.Close()- chunks = cropTitleChunks(parentCtx, engine, chunks)- }+ engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)+ if eErr != nil {+ common.Warn("chunker: could not open PDF for on-demand cropping", zap.Error(eErr))+ }+ if engine != nil {+ defer engine.Close()+ chunks = cropTitleChunks(parentCtx, engine, chunks)}if len(chunks) == 0 {@@ -278,9 +280,10 @@ func joinGroupText(g []lineRecord) string {// "html"]` branch of common.py:build_chunks_from_record_groups. Plain// payloads emit only "text"; structured payloads (chunks/json) also// carry doc_type_kwd and img_id.-func isPlainTextFormat(inputs map[string]any) bool {- if f, ok := inputs["output_format"].(string); ok {- return f == "markdown" || f == "text" || f == "html"+func isPlainTextFormat(upstream schema.ChunkerFromUpstream) bool {+ switch upstream.OutputFormat {+ case schema.PayloadFormatMarkdown, schema.PayloadFormatText, schema.PayloadFormatHTML:+ return true}return false}@@ -311,10 +314,17 @@ func buildChunksFromRecordGroups(groups [][]lineRecord, prefixes []string, p *tiif len(g) == 0 {continue}- // Strip parser-emitted position tags (`@@...##`) from the- // joined text (diff 1.6 / 2.8). Mirrors common.py:255- // `RAGFlowPdfParser.remove_tag("".join(...))`.- chunk := map[string]any{"text": removeTag(joinGroupText(g))}+ // Strip parser-emitted position tags (`@@...##`) from the joined text+ // (diff 1.6 / 2.8). Mirrors common.py:255+ // `RAGFlowPdfParser.remove_tag("".join(...))`. A non-text group is a+ // single atomic record (table/image): its text is taken verbatim —+ // Python emits records[0]["text"] unchanged (common.py:419) — instead+ // of the group join, which would append a trailing newline.+ chunkText := removeTag(joinGroupText(g))+ if !g[0].isText() {+ chunkText = removeTag(g[0].text)+ }+ chunk := map[string]any{"text": chunkText}if !plain {first := g[0]if first.docType != "" {@@ -323,6 +333,18 @@ func buildChunksFromRecordGroups(groups [][]lineRecord, prefixes []string, p *tiif first.imgID != nil {chunk["img_id"] = *first.imgID}+ if first.image != "" {+ chunk["image"] = first.image+ }+ if first.sheet != "" {+ chunk["sheet"] = first.sheet+ }+ if first.sheetIndex != nil {+ // Numbers on the chunk wire are float64 (every other+ // chunker's maps pass through a JSON round trip);+ // indexdoc.isSpreadsheetChunk keys on presence, not type.+ chunk["sheet_index"] = float64(*first.sheetIndex)+ }}// Merge PDF coordinate matrices across the merged records// instead of keeping only the leading record's (diff 1.6).@@ -427,37 +449,75 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {return out}-// extractLineRecords reads the chunker inputs in the same order the-// python BaseTitleChunker.extract_line_records uses:+// extractLineRecords converts the decoded upstream payload into records in the+// same order the python BaseTitleChunker.extract_line_records uses: the field+// named by output_format selects the payload (chunks / json / markdown / text+// / html), mirroring python's branch on output_format rather than key+// presence alone — a declared "markdown" payload is read even when a stray+// empty "text" key is also present. When output_format is unset the first+// present payload wins (chunks, json, then text/markdown/html), preserving+// the legacy key scan.//-// 1. If upstream emitted chunks (output_format == "chunks") OR-// upstream emitted JSON, normalize from the list payload.-// 2. Otherwise, treat text/markdown/html as a "one record per line"-// stream (preserving indentation for non-text formats, strip-only-// for the text format).-func extractLineRecords(inputs map[string]any) []lineRecord {- if docs := chunksFromInputs(inputs); docs != nil {- return recordsFromStructured(docs)- }- text, _ := stringFromInputs(inputs, "text", "content", "markdown", "html")- if text == "" {+// The payload is consumed from the decoded upstream — never from the raw+// inputs — so every chunker variant shares the one normalization entry+// (decodeChunkerFromUpstream).+func extractLineRecords(upstream schema.ChunkerFromUpstream) []lineRecord {+ switch upstream.OutputFormat {+ case schema.PayloadFormatChunks:+ return recordsFromStructured(upstream.Chunks)+ case schema.PayloadFormatJSON:+ return recordsFromStructured(upstream.JSONResult)+ case schema.PayloadFormatMarkdown:+ return lineRecordsFromPayload(upstream.MarkdownResult)+ case schema.PayloadFormatText:+ return lineRecordsFromPayload(upstream.TextResult)+ case schema.PayloadFormatHTML:+ return lineRecordsFromPayload(upstream.HTMLResult)+ }+ // output_format unset: first present payload wins.+ if upstream.Chunks != nil {+ return recordsFromStructured(upstream.Chunks)+ }+ if upstream.JSONResult != nil {+ return recordsFromStructured(upstream.JSONResult)+ }+ for _, candidate := range []*string{upstream.TextResult, upstream.MarkdownResult, upstream.HTMLResult} {+ if candidate != nil {+ return lineRecordsFromPayload(candidate)+ }+ }+ return nil+}++// lineRecordsFromPayload splits a plain-text payload into one record per line.+// An absent or empty payload yields no records.+func lineRecordsFromPayload(payload *string) []lineRecord {+ if payload == nil || *payload == "" {return nil}- return lineRecordsFromText(text)+ return lineRecordsFromText(*payload)}+// recordsFromStructured converts parser-shaped items into records: text items+// keep their text, and media items — tables whose TableData has already been+// rendered to HTML by materializeTableText, images carrying an image payload+// or reference — pass through as single records. Only an item with none of+// text, image and img_id is dropped.func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {out := make([]lineRecord, 0, len(items))for _, it := range items {text := itemTextOrFallback(it)- if text == "" {+ // A structured table arrives with Text already rendered from TableData+ // (materializeTableText); an image item legitimately carries no text —+ // its payload is the image field. Only a truly empty item is skipped.+ if text == "" && it.Image == "" && it.ImgID == "" {continue}dt := it.DocTypeif dt == "" {dt = "text"}- appendRecord := func(recordText string) {+ appendRecord := func(recordText, image string) {var imgID *stringif it.ImgID != "" {img := it.ImgID@@ -483,28 +543,32 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {text: recordText,docType: dt,imgID: imgID,+ image: image,layout: it.Layout,ckType: it.CKType,pdfPositions: it.PDFPositions,positions: it.Positions,+ sheet: it.Sheet,+ sheetIndex: it.SheetIndex,parentMeta: meta,})}// TextParser intentionally emits one coordinate-free text item for the// whole document. Title-family chunkers still need one record per line// so heading regexes can see headings after the first line. Structured- // media/table items and positioned text remain atomic.+ // media/table items and positioned text remain atomic, and the image+ // payload stays out of per-line records so a blob is never duplicated.plainText := itemDocType(it) == "text" && (it.CKType == "" || it.CKType == "text")unpositioned := len(it.PDFPositions) == 0 && len(it.Positions) == 0if plainText && unpositioned && strings.Contains(text, "\n") {for _, line := range strings.Split(normalizeGeneralNewlines(text), "\n") {if trim(line) != "" {- appendRecord(line)+ appendRecord(line, "")}}continue}- appendRecord(text)+ appendRecord(text, it.Image)}return out}diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.goindex a9edcea927..c14d4e8bbb 100644--- a/internal/ingestion/component/chunker/group_test.go+++ b/internal/ingestion/component/chunker/group_test.go@@ -65,12 +65,24 @@ func TestGroupTitleChunker_NewRejectsHierarchyWithoutHierarchyParam(t *testing.T}}+// decodedRecords decodes a raw chunker payload the way every component does+// (decodeChunkerFromUpstream) and extracts the line records from the result.+func decodedRecords(t *testing.T, payload map[string]any) []lineRecord {+ t.Helper()+ upstream, err := decodeChunkerFromUpstream(payload)+ if err != nil {+ t.Fatalf("decodeChunkerFromUpstream: %v", err)+ }+ return extractLineRecords(upstream)+}+// TestExtractLineRecords_MarkdownKey verifies extractLineRecords reads// the "markdown" payload key. Before this fix extractLineRecords only-// looked at "text"/"content" and silently returned nil for parser+// looked at "text" and silently returned nil for parser// output that carried output_format="markdown".func TestExtractLineRecords_MarkdownKey(t *testing.T) {- records := extractLineRecords(map[string]any{+ records := decodedRecords(t, map[string]any{+ "name": "t.md","output_format": "markdown","markdown": "line1\nline2\nline3",})@@ -84,10 +96,27 @@ func TestExtractLineRecords_MarkdownKey(t *testing.T) {}}+// TestExtractLineRecords_DeclaredFormatBeatsStrayEmptyText pins the+// output_format-driven payload selection (python parity): a declared+// "markdown" payload is read even when a stray empty "text" key is present,+// so an empty alias key cannot silently swallow the whole document.+func TestExtractLineRecords_DeclaredFormatBeatsStrayEmptyText(t *testing.T) {+ records := decodedRecords(t, map[string]any{+ "name": "t.md",+ "output_format": "markdown",+ "markdown": "line1\nline2",+ "text": "",+ })+ if len(records) != 2 {+ t.Fatalf("got %d records, want 2 from the declared markdown payload", len(records))+ }+}+// TestExtractLineRecords_HTMLKey verifies extractLineRecords reads the// "html" payload key, same safety-net rationale as TestMarkdownKey.func TestExtractLineRecords_HTMLKey(t *testing.T) {- records := extractLineRecords(map[string]any{+ records := decodedRecords(t, map[string]any{+ "name": "t.html","output_format": "html","html": "<p>first</p>\n<p>second</p>",})@@ -99,7 +128,8 @@ func TestExtractLineRecords_HTMLKey(t *testing.T) {// TestExtractLineRecords_TextKeyStillWorks ensures the existing "text"// key path is not broken by the addition of "markdown"/"html".func TestExtractLineRecords_TextKeyStillWorks(t *testing.T) {- records := extractLineRecords(map[string]any{+ records := decodedRecords(t, map[string]any{+ "name": "t.txt","text": "hello\nworld",})if len(records) != 2 {@@ -113,7 +143,7 @@ func TestExtractLineRecords_ParserStructuredTextSplitsLines(t *testing.T) {t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)}- records := extractLineRecords(map[string]any{+ records := decodedRecords(t, map[string]any{"name": "notes.txt","output_format": "json","json": parsed.JSON,diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.goindex f590f68957..2c4e0f817c 100644--- a/internal/ingestion/component/chunker/hierarchy.go+++ b/internal/ingestion/component/chunker/hierarchy.go@@ -141,7 +141,11 @@ func (n *chunkNode) getPaths(paths *[][]int, titles []int, depth int, includeHea// invokeHierarchy runs the HierarchyTitleChunker strategy.func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {- records := extractLineRecords(inputs)+ upstream, err := decodeChunkerFromUpstream(inputs)+ if err != nil {+ return inputErrorOutput(err), nil+ }+ records := extractLineRecords(upstream)common.Debug("chunker stage",zap.String("component", "Chunker"),zap.String("variant", "hierarchy"),@@ -150,7 +154,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]aif len(records) == 0 {return emptyOutputs(), nil}- ctx := newLevelContext(records, outlineFromInputs(inputs), p)+ ctx := newLevelContext(records, outlineFromUpstream(upstream), p)levels := ctx.Levels()// Count heading level distribution for debugging.headingCounts := make(map[int]int)@@ -266,7 +270,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]azap.Int("record_groups", len(recordGroups)),)- chunks, prefixes := buildChunksFromRecordGroups(recordGroups, groupPrefixes, p, isPlainTextFormat(inputs))+ chunks, prefixes := buildChunksFromRecordGroups(recordGroups, groupPrefixes, p, isPlainTextFormat(upstream))// Enforce the title-family token ceiling (chunk_token_cap) right after// build_chunks and BEFORE the on-demand crop, mirroring Python's// invoke() order (build -> cap -> set_chunks). prefixes re-attaches each@@ -278,21 +282,19 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]azap.String("component", "Chunker"),zap.String("variant", "hierarchy"),zap.Int("chunks", len(chunks)),- zap.Bool("plain_text", isPlainTextFormat(inputs)),+ zap.Bool("plain_text", isPlainTextFormat(upstream)),)// On-demand PDF preview cropping for image/table/text chunks,// mirroring the TokenChunker JSON path (token.go:513). Best-effort:// a missing or unreadable PDF simply skips cropping.- if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {- engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)- if eErr != nil {- common.Warn("HierarchyTitleChunker: could not open PDF for on-demand cropping", zap.Error(eErr))- }- if engine != nil {- defer engine.Close()- chunks = cropTitleChunks(parentCtx, engine, chunks)- }+ engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)+ if eErr != nil {+ common.Warn("HierarchyTitleChunker: could not open PDF for on-demand cropping", zap.Error(eErr))+ }+ if engine != nil {+ defer engine.Close()+ chunks = cropTitleChunks(parentCtx, engine, chunks)}if len(chunks) == 0 {diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.goindex d783f1932e..3225493489 100644--- a/internal/ingestion/component/chunker/manual.go+++ b/internal/ingestion/component/chunker/manual.go@@ -105,8 +105,14 @@ func (c *ManualChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputsreturn c.invoke(ctx, db, withName(inputs, name))}+// invoke decodes the upstream payload once, extracts the records, and runs+// the shared group pipeline after a physical-position resort.func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {- records := extractLineRecords(inputs)+ upstream, err := decodeChunkerFromUpstream(inputs)+ if err != nil {+ return inputErrorOutput(err), nil+ }+ records := extractLineRecords(upstream)if len(records) == 0 {return emptyOutputs(), nil}@@ -117,7 +123,7 @@ func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs}// ManualChunker is exempt from the title-family token cap (#18455): it// does not inherit BaseTitleChunker, so it always passes tokenCap=0.- return chunkFromRecords(ctx, db, inputs, &c.param, records, 0)+ return chunkFromRecords(ctx, db, upstream, &c.param, records, 0)}// hasPdfPositions reports whether any record carries a PDF coordinate matrixdiff --git a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go b/internal/ingestion/component/chunker/manual_pdf_cgo_test.goindex 897928fd72..7d377af73e 100644--- a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go+++ b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go@@ -79,10 +79,14 @@ func TestManualChunker_RealPDFResort(t *testing.T) {// Same shape the chunker receives in production.input := map[string]any{"name": "Doc1.pdf",- "output_format": "json",+ "output_format": "chunks","chunks": res.JSON,}- records := extractLineRecords(input)+ upstream, err := decodeChunkerFromUpstream(input)+ if err != nil {+ t.Fatalf("decodeChunkerFromUpstream: %v", err)+ }+ records := extractLineRecords(upstream)if len(records) == 0 {t.Fatal("extractLineRecords produced no records from real parser output")}diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.goindex 5fb2f9dd5f..10bf71c4e5 100644--- a/internal/ingestion/component/chunker/manual_test.go+++ b/internal/ingestion/component/chunker/manual_test.go@@ -322,7 +322,7 @@ func TestHasPdfPositions(t *testing.T) {// with 1-based page numbers, accompanied by layout / page_number /// doc_type_kwd. Using [][]any — not [][]float64 — matters: that is the concrete// in-memory type the parser produces, and recordsFromStructured must consume it-// after the chunksFromInputs JSON round-trip.+// after the decodeChunkerFromUpstream JSON round-trip.func parserShapedItem(text string, page, left, right, top, bottom float64) map[string]any {return map[string]any{"text": text,diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.goindex 9f21d806e2..53ae5fd1fe 100644--- a/internal/ingestion/component/chunker/one.go+++ b/internal/ingestion/component/chunker/one.go@@ -29,7 +29,6 @@ package chunkerimport ("context"- "fmt""strings""ragflow/internal/agent/runtime"@@ -72,17 +71,14 @@ func (c *OneChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mareturn c.invoke(ctx, inputs)}+// invoke collapses the decoded upstream payload into a single chunk.func (c *OneChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {if inputs == nil {return emptyOutputs(), nil}upstream, err := decodeChunkerFromUpstream(inputs)if err != nil {- return map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{},- "_ERROR": fmt.Sprintf("Input error: %v", err),- }, nil+ return inputErrorOutput(err), nil}switch upstream.OutputFormat {diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.goindex a59e2aeadb..10ae52dac1 100644--- a/internal/ingestion/component/chunker/page.go+++ b/internal/ingestion/component/chunker/page.go@@ -30,7 +30,6 @@ package chunkerimport ("context"- "fmt""ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"@@ -74,17 +73,14 @@ func (c *PageChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mreturn c.invoke(ctx, inputs)}+// invoke emits one chunk per decoded upstream item (slide or page).func (c *PageChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {if inputs == nil {return emptyOutputs(), nil}upstream, err := decodeChunkerFromUpstream(inputs)if err != nil {- return map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{},- "_ERROR": fmt.Sprintf("Input error: %v", err),- }, nil+ return inputErrorOutput(err), nil}// The Page chunker preserves one incoming JSON item per output chunk.diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex f381120aa4..fb09b86f3b 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -88,17 +88,14 @@ func (c *QAChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapreturn c.invoke(ctx, inputs)}+// invoke extracts question-answer pairs from the decoded upstream payload.func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {if inputs == nil {return emptyOutputs(), nil}upstream, err := decodeChunkerFromUpstream(inputs)if err != nil {- return map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{},- "_ERROR": fmt.Sprintf("Input error: %v", err),- }, nil+ return inputErrorOutput(err), nil}qPrefix, aPrefix := "问题:", "回答:"diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.goindex dc875ce72c..661c30aeef 100644--- a/internal/ingestion/component/chunker/register.go+++ b/internal/ingestion/component/chunker/register.go@@ -96,10 +96,12 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m//// A chunk id may already be stamped (e.g. by the streamed crop-upload// path, which keys MinIO by the canonical id). Reuse it so the id is- // computed exactly once; both routes normalize the text through- // canonicalChunkText, so the crop-time key and this id are identical- // and cannot diverge. For chunks that bypassed crop, fall back to- // deriving it here from the already-finalized text.+ // computed exactly once; both routes derive the identity through the+ // shared helpers, so the crop-time key and this id are identical and+ // cannot diverge. For chunks that bypassed crop, fall back to deriving+ // it here from the already-finalized text — a captionless image folds+ // its payload identity in (mediaChunkIdentity), so distinct images in+ // one document never share an id.for _, ck := range chunks {if id, ok := ck["id"]; ok && id != "" {continue@@ -108,6 +110,11 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs mif err != nil {return nil, err}+ if text == "" {+ image, _ := ck["image"].(string)+ imgID, _ := ck["img_id"].(string)+ text = mediaChunkIdentity(image, imgID)+ }ck["id"] = common.ChunkID(docID, text)}@@ -132,7 +139,6 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m// by all chunker variants.var ChunkerInputs = map[string]string{"text": "Plain-text input. The chunker slices this into downstream chunks.",- "content": "Alias for \"text\".","chunks": "Optional upstream chunk list (structured JSON form).","name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.","file_type": "Canonical parser routing extension used by GeneralChunker.",diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.goindex 64a7709391..0580d496ef 100644--- a/internal/ingestion/component/chunker/sheet_identity_test.go+++ b/internal/ingestion/component/chunker/sheet_identity_test.go@@ -46,8 +46,19 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {}for _, tc := range []struct {name string+ params map[string]anyfactory func(map[string]any) (runtime.Component, error)- }{{"QA", NewQAChunker}, {"Token", NewTokenChunker}, {"One", NewOneChunker}} {+ }{+ {"QA", nil, NewQAChunker},+ {"Token", nil, NewTokenChunker},+ {"One", nil, NewOneChunker},+ // The title-family chunkers share extractLineRecords; a+ // spreadsheet item must keep its sheet identity there too, or its+ // selections would be decoded as PDF boxes.+ {"Group", map[string]any{"levels": [][]string{{`^# `}}}, NewGroupTitleChunker},+ {"Hierarchy", map[string]any{"hierarchy": 1, "levels": [][]string{{`^# `}}}, NewHierarchyTitleChunker},+ {"Manual", map[string]any{"levels": [][]string{{`^# `}}}, NewManualChunker},+ } {t.Run(source.name+"/"+tc.name, func(t *testing.T) {batches := [][]map[string]any{parser.ItemsToMaps(source.result.JSON)}// One preserves positions only for a single source item.@@ -58,7 +69,7 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {}}for _, items := range batches {- comp, err := tc.factory(nil)+ comp, err := tc.factory(tc.params)if err != nil {t.Fatal(err)}diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 1e674a1289..cf455aba8e 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -31,7 +31,6 @@ package chunkerimport ("context""encoding/json"- "fmt""strings""ragflow/internal/agent/runtime"@@ -76,17 +75,14 @@ func (c *TableChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputsreturn c.invoke(ctx, inputs)}+// invoke expands the decoded upstream tables into one chunk per data row.func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {if inputs == nil {return emptyOutputs(), nil}upstream, err := decodeChunkerFromUpstream(inputs)if err != nil {- return map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{},- "_ERROR": fmt.Sprintf("Input error: %v", err),- }, nil+ return inputErrorOutput(err), nil}switch upstream.OutputFormat {diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.goindex 87f8f38278..4bf5400223 100644--- a/internal/ingestion/component/chunker/title.go+++ b/internal/ingestion/component/chunker/title.go@@ -39,7 +39,7 @@// fallback_level = len(selected_group) + 1. Strategy (1) —// PDF-outline detection — is also ported: newLevelContext// (title.go) reads the Parser-supplied file.outline via-// outlineFromInputs / resolveOutlineLevels / outlineSimilarity+// outlineFromUpstream / resolveOutlineLevels / outlineSimilarity// (port of common.py:_outline_similarity) before falling back to// the regex/layout branch, wired into group.go / hierarchy.go// (Chunker omission 1.5, Gap C closed).@@ -557,27 +557,27 @@ func resolveOutlineLevels(records []lineRecord, outline []outlineEntry) (levelsreturn levels, max(1, maxLevel), true}-// outlineFromInputs reads the parser-supplied PDF outline from the upstream-// file metadata (file.outline, written by the ingestion PDF parser's-// outlinesToFileMeta) and normalizes it into the chunker's outlineEntry-// shape. Returns nil when no outline is present, so callers fall back to-// frequency-based title detection. Numbers are coerced from int/float64-// because the runtime may hand the chunker a JSON-decoded payload.-func outlineFromInputs(inputs map[string]any) []outlineEntry {- file, _ := inputs["file"].(map[string]any)- if file == nil {+// outlineFromUpstream reads the parser-supplied PDF outline from the decoded+// upstream file metadata (file.outline, written by the ingestion PDF parser's+// outlinesToFileMeta and kept verbatim in ChunkerFileMeta.Extra) and normalizes+// it into the chunker's outlineEntry shape. Returns nil when no outline is+// present, so callers fall back to frequency-based title detection. Numbers+// are coerced from int/float64 because the runtime may hand the chunker a+// JSON-decoded payload.+func outlineFromUpstream(upstream schema.ChunkerFromUpstream) []outlineEntry {+ if upstream.File == nil {return nil}- raw, _ := file["outline"].([]any)- if len(raw) == 0 {+ raw, ok := upstream.File.Extra["outline"]+ if !ok || len(raw) == 0 {return nil}- out := make([]outlineEntry, 0, len(raw))- for _, item := range raw {- m, ok := item.(map[string]any)- if !ok {- continue- }+ var items []map[string]any+ if err := json.Unmarshal(raw, &items); err != nil {+ return nil+ }+ out := make([]outlineEntry, 0, len(items))+ for _, m := range items {title, _ := m["title"].(string)if title == "" {continue@@ -638,10 +638,13 @@ type lineRecord struct {text stringdocType stringimgID *string+ image stringlayout stringckType stringpdfPositions json.RawMessagepositions json.RawMessage+ sheet string+ sheetIndex *intparentMeta map[string]any}@@ -773,9 +776,9 @@ func (c *TitleChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs}switch c.param.Method {case "hierarchy":- return invokeHierarchy(ctx, db, inputs, &c.param)+ return invokeHierarchy(ctx, db, withName(inputs, name), &c.param)case "group":- return invokeGroup(ctx, db, inputs, &c.param)+ return invokeGroup(ctx, db, withName(inputs, name), &c.param)default:return map[string]any{"output_format": "chunks",diff --git a/internal/ingestion/component/chunker/title_structured_media_test.go b/internal/ingestion/component/chunker/title_structured_media_test.gonew file mode 100644index 0000000000..34a2529f98--- /dev/null+++ b/internal/ingestion/component/chunker/title_structured_media_test.go@@ -0,0 +1,351 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "context"+ "encoding/json"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil"+)++const titleTestTableHTML = "<table><tr><th>name</th><th>age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"++// pngBase64Red is a second valid 1x1 PNG (red), distinct from pngBase64, used+// to pin that captionless images keep distinct chunk ids.+const pngBase64Red = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="++// titleFamilyCase pairs a title-family entry point with its display name.+type titleFamilyCase struct {+ name string+ comp runtime.Component+}++// titleFamilyComponents builds the three entry points behind the shared+// extractLineRecords record path. TitleChunker's group/hierarchy methods+// dispatch into the same invokeGroup/invokeHierarchy implementations, so the+// standalone components cover every title-family strategy.+func titleFamilyComponents(t *testing.T) []titleFamilyCase {+ t.Helper()+ levels := [][]string{{`^# `}}+ group, err := NewGroupTitleChunker(map[string]any{"levels": levels})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ hierarchy, err := NewHierarchyTitleChunker(map[string]any{"hierarchy": 1, "levels": levels})+ if err != nil {+ t.Fatalf("NewHierarchyTitleChunker: %v", err)+ }+ manual, err := NewManualChunker(map[string]any{"levels": levels})+ if err != nil {+ t.Fatalf("NewManualChunker: %v", err)+ }+ return []titleFamilyCase{+ {"group", group},+ {"hierarchy", hierarchy},+ {"manual", manual},+ }+}++// titleFamilyChunks invokes one title-family component over a json payload and+// returns the emitted chunks.+func titleFamilyChunks(t *testing.T, comp runtime.Component, items []map[string]any) []map[string]any {+ t.Helper()+ out, err := comp.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.pdf",+ "output_format": "json",+ "json": items,+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ return chunks+}++// renderedChunkText returns the HTML rendering the chunker derives from the+// item's structured TableData, through the same map round trip the chunker+// performs.+func renderedChunkText(t *testing.T, item map[string]any) string {+ t.Helper()+ raw, err := json.Marshal(item["table"])+ if err != nil {+ t.Fatalf("marshal table: %v", err)+ }+ var typed entity.TableData+ if err := json.Unmarshal(raw, &typed); err != nil {+ t.Fatalf("unmarshal table: %v", err)+ }+ return tableutil.RenderTableHTML(&typed)+}++// TestTitleFamilyRendersStructuredTableAtomic: a parser-shaped table item+// (structured TableData, no text) must survive the title-family record path as+// one atomic chunk carrying the verbatim HTML rendering — header rows rendered+// as <th> per HeaderRows — with its positions kept and no neighbor text glued+// in.+func TestTitleFamilyRendersStructuredTableAtomic(t *testing.T) {+ item := tableItem(t, titleTestTableHTML, map[string]any{+ "_pdf_positions": [][]float64{{1, 10, 200, 10, 40}},+ })+ want := renderedChunkText(t, item)+ if !strings.Contains(want, "<th>name</th>") || !strings.Contains(want, "<td>Alice</td>") {+ t.Fatalf("fixture rendering lost header/cell markup: %q", want)+ }++ for _, tc := range titleFamilyComponents(t) {+ t.Run(tc.name, func(t *testing.T) {+ chunks := titleFamilyChunks(t, tc.comp, []map[string]any{+ {"text": "# H1", "doc_type_kwd": "text"},+ item,+ {"text": "body paragraph", "doc_type_kwd": "text"},+ })+ var tableChunks []map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "table" {+ tableChunks = append(tableChunks, ck)+ }+ }+ if len(tableChunks) != 1 {+ t.Fatalf("table chunks = %d, want exactly 1 (chunks: %#v)", len(tableChunks), chunks)+ }+ ck := tableChunks[0]+ if ck["text"] != want {+ t.Errorf("table chunk text = %q, want the verbatim rendering %q", ck["text"], want)+ }+ if _, ok := ck["table"]; ok {+ t.Errorf("structured TableData must not leak onto the chunk: %#v", ck["table"])+ }+ if got := ck["_pdf_positions"]; !reflect.DeepEqual(got, [][]float64{{1, 10, 200, 10, 40}}) {+ t.Errorf("_pdf_positions = %#v, want the source matrix", got)+ }+ for _, other := range chunks {+ if other["doc_type_kwd"] != "table" && strings.Contains(toString(other["text"]), "<table>") {+ t.Errorf("table markup leaked into a text chunk: %#v", other)+ }+ }+ })+ }+}++// TestTitleFamilyKeepsEmptyTextImageItem: an image item legitimately carries no+// text — its payload is the image field — and must survive as a chunk instead+// of being dropped by the empty-text skip.+func TestTitleFamilyKeepsEmptyTextImageItem(t *testing.T) {+ payload := "data:image/png;base64," + pngBase64+ for _, tc := range titleFamilyComponents(t) {+ t.Run(tc.name, func(t *testing.T) {+ chunks := titleFamilyChunks(t, tc.comp, []map[string]any{+ {"text": "", "image": payload, "doc_type_kwd": "image"},+ })+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want the image item kept as 1 chunk", len(chunks))+ }+ ck := chunks[0]+ if ck["doc_type_kwd"] != "image" {+ t.Errorf("doc_type_kwd = %v, want image", ck["doc_type_kwd"])+ }+ if ck["image"] != payload {+ t.Errorf("image payload lost: %v", ck["image"])+ }+ if ck["text"] != "" {+ t.Errorf("text = %q, want empty", ck["text"])+ }+ })+ }+}++// TestTitleFamilyEmptyTextImageUploadsThroughDecorator: the decorated chunker+// uploads the payload at the chunk id, writes img_id, and drops the raw bytes —+// the empty-text image chunk must flow through the same decorator path as+// every other chunker's media chunk.+func TestTitleFamilyEmptyTextImageUploadsThroughDecorator(t *testing.T) {+ prev := ChunkImageUploader+ ChunkImageUploader = func(_ context.Context, kbID, chunkID string, _ []byte) (string, error) {+ return kbID + "-" + chunkID, nil+ }+ t.Cleanup(func() { ChunkImageUploader = prev })++ comp, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ decorated := &imageUploadDecorator{inner: comp}++ payload := "data:image/png;base64," + pngBase64+ out, err := decorated.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.docx",+ "kb_id": testKBID,+ "output_format": "json",+ "json": []map[string]any{+ {"text": "", "image": payload, "doc_type_kwd": "image"},+ },+ })+ if err != nil {+ t.Fatalf("decorated Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want 1", len(chunks))+ }+ ck := chunks[0]+ // No doc_id in the payload (it would trigger the on-demand PDF crop path,+ // which needs a database), so the id hashes the empty document id plus the+ // captionless image's own identity (mediaChunkIdentity).+ wantChunkID := common.ChunkID("", mediaChunkIdentity(payload, ""))+ if ck["id"] != wantChunkID {+ t.Errorf("id = %v, want %q", ck["id"], wantChunkID)+ }+ if ck["img_id"] != testKBID+"-"+wantChunkID {+ t.Errorf("img_id = %v, want %q", ck["img_id"], testKBID+"-"+wantChunkID)+ }+ if _, ok := ck["image"]; ok {+ t.Errorf("image not dropped after upload: %v", ck["image"])+ }+}++// TestTitleFamilyCaptionlessImagesGetDistinctIDs: two different captionless+// images in one document must not collapse onto the same chunk id — a shared+// id would make the second upload overwrite the first image in MinIO and merge+// the two index entries.+func TestTitleFamilyCaptionlessImagesGetDistinctIDs(t *testing.T) {+ uploaded := map[string]int{}+ prev := ChunkImageUploader+ ChunkImageUploader = func(_ context.Context, kbID, chunkID string, _ []byte) (string, error) {+ uploaded[chunkID]+++ return kbID + "-" + chunkID, nil+ }+ t.Cleanup(func() { ChunkImageUploader = prev })++ comp, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ decorated := &imageUploadDecorator{inner: comp}++ out, err := decorated.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.docx",+ "kb_id": testKBID,+ "output_format": "json",+ "json": []map[string]any{+ {"text": "", "image": "data:image/png;base64," + pngBase64, "doc_type_kwd": "image"},+ {"text": "", "image": "data:image/png;base64," + pngBase64Red, "doc_type_kwd": "image"},+ },+ })+ if err != nil {+ t.Fatalf("decorated Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %d, want 2", len(chunks))+ }+ idA, _ := chunks[0]["id"].(string)+ idB, _ := chunks[1]["id"].(string)+ if idA == "" || idB == "" || idA == idB {+ t.Fatalf("captionless image ids must be distinct, got %q and %q", idA, idB)+ }+ if idA == common.ChunkID("", "") || idB == common.ChunkID("", "") {+ t.Fatalf("id still hashes the empty text alone: %q / %q", idA, idB)+ }+ if len(uploaded) != 2 || uploaded[idA]+uploaded[idB] != 2 {+ t.Fatalf("uploads must land on two distinct keys, got %v", uploaded)+ }+}++// TestTitleFamilyReportsDecodeFailure: a payload that declares a format but+// carries no matching payload fails decode and short-circuits through the+// family-wide input-error shape instead of silently producing nothing.+func TestTitleFamilyReportsDecodeFailure(t *testing.T) {+ for _, tc := range titleFamilyComponents(t) {+ t.Run(tc.name, func(t *testing.T) {+ out, err := tc.comp.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.pdf",+ "output_format": "json", // no "json" payload → decode fails+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if msg, _ := out["_ERROR"].(string); !strings.HasPrefix(msg, "Input error:") {+ t.Fatalf("_ERROR = %v, want an input error", out["_ERROR"])+ }+ if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 0 {+ t.Fatalf("chunks = %#v, want none", chunks)+ }+ })+ }+}++// TestTitleFamilyDropsFullyEmptyItems: an item with no text, no table and no+// image has nothing retrievable and stays dropped.+func TestTitleFamilyDropsFullyEmptyItems(t *testing.T) {+ for _, tc := range titleFamilyComponents(t) {+ t.Run(tc.name, func(t *testing.T) {+ chunks := titleFamilyChunks(t, tc.comp, []map[string]any{+ {"text": "", "doc_type_kwd": "text"},+ })+ if len(chunks) != 0 {+ t.Fatalf("chunks = %d, want 0 for a fully empty item: %#v", len(chunks), chunks)+ }+ chunks = titleFamilyChunks(t, tc.comp, []map[string]any{+ {"text": "", "doc_type_kwd": "text"},+ {"text": "real body", "doc_type_kwd": "text"},+ })+ if len(chunks) != 1 || !strings.Contains(toString(chunks[0]["text"]), "real body") {+ t.Fatalf("chunks = %#v, want only the retrievable text chunk", chunks)+ }+ })+ }+}++// TestTitleFamilyPreservesSheetIdentity: a spreadsheet item routed through a+// title chunker keeps its sheet identity so its positions matrix stays in the+// spreadsheet vocabulary instead of being decoded as PDF boxes.+func TestTitleFamilyPreservesSheetIdentity(t *testing.T) {+ item := tableItem(t, "<table><tr><th>Q</th><th>A</th></tr><tr><td>q1</td><td>a1</td></tr></table>",+ map[string]any{+ "sheet": "Orders",+ "sheet_index": 1,+ "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}},+ })+ comp, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ chunks := titleFamilyChunks(t, comp, []map[string]any{item})+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want 1", len(chunks))+ }+ ck := chunks[0]+ if ck["sheet"] != "Orders" {+ t.Errorf("sheet = %v, want Orders", ck["sheet"])+ }+ if ck["sheet_index"] != float64(1) {+ t.Errorf("sheet_index = %#v, want 1", ck["sheet_index"])+ }+ if _, ok := ck["_pdf_positions"]; ok {+ t.Errorf("spreadsheet selections leaked into _pdf_positions: %#v", ck["_pdf_positions"])+ }+ if got := ck["positions"]; !reflect.DeepEqual(got, [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}}) {+ t.Errorf("positions = %#v, want the source matrix", got)+ }+}diff --git a/internal/ingestion/component/chunker/title_test.go b/internal/ingestion/component/chunker/title_test.goindex 114ab1c5b5..a1b0a4c083 100644--- a/internal/ingestion/component/chunker/title_test.go+++ b/internal/ingestion/component/chunker/title_test.go@@ -439,18 +439,23 @@ func TestNewLevelContext_OutlineBranch(t *testing.T) {}}-// TestOutlineFromInputs verifies the parser-supplied file.outline is parsed-// into outlineEntry, tolerating float64-encoded levels (runtime JSON path).-func TestOutlineFromInputs(t *testing.T) {- inputs := map[string]any{+// TestOutlineFromUpstream verifies the parser-supplied file.outline survives+// the decode path and is parsed into outlineEntry, tolerating float64-encoded+// levels (runtime JSON path).+func TestOutlineFromUpstream(t *testing.T) {+ up, err := decodeChunkerFromUpstream(map[string]any{+ "name": "t.pdf","file": map[string]any{"outline": []any{map[string]any{"title": "第一章", "level": float64(0), "page_number": float64(1)},map[string]any{"title": "第二章", "level": float64(1), "page_number": float64(3)},},},+ })+ if err != nil {+ t.Fatalf("decodeChunkerFromUpstream: %v", err)}- out := outlineFromInputs(inputs)+ out := outlineFromUpstream(up)if len(out) != 2 {t.Fatalf("outline len = %d, want 2", len(out))}@@ -462,8 +467,12 @@ func TestOutlineFromInputs(t *testing.T) {}// No file / no outline -> nil (frequency fallback).- if got := outlineFromInputs(map[string]any{}); got != nil {- t.Errorf("empty inputs outline = %v, want nil", got)+ empty, err := decodeChunkerFromUpstream(map[string]any{"name": "t.pdf"})+ if err != nil {+ t.Fatalf("decodeChunkerFromUpstream: %v", err)+ }+ if got := outlineFromUpstream(empty); got != nil {+ t.Errorf("empty upstream outline = %v, want nil", got)}}diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.goindex 4772e6a014..d35cf8fb2a 100644--- a/internal/ingestion/component/chunker/title_token_cap_test.go+++ b/internal/ingestion/component/chunker/title_token_cap_test.go@@ -440,7 +440,7 @@ func TestTitleCap_GroupPipeline_RespectsCap(t *testing.T) {defer restoreTokenizer()body := strings.Join(joinSentences(12), "")p := newTitleParam(t, "group", 20, [][]string{{`^# `}})- inputs := map[string]any{"output_format": "text", "text": body}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}got, err := invokeGroup(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeGroup: %v", err)@@ -464,7 +464,7 @@ func TestTitleCap_GroupPipeline_ValidatedInRangeCap(t *testing.T) {if err := p.TitleChunkerParam.Validate(); err != nil {t.Fatalf("cap=128 must pass production Validate: %v", err)}- inputs := map[string]any{"output_format": "text", "text": body}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}got, err := invokeGroup(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeGroup: %v", err)@@ -481,7 +481,7 @@ func TestTitleCap_HierarchyPipeline_RespectsCap(t *testing.T) {defer restoreTokenizer()body := strings.Join(joinSentences(12), "")p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})- inputs := map[string]any{"output_format": "text", "text": body}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}got, err := invokeHierarchy(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeHierarchy: %v", err)@@ -498,7 +498,7 @@ func TestTitleCap_GroupPipeline_CapZeroNoop(t *testing.T) {defer restoreTokenizer()body := strings.Join(joinSentences(12), "")p := newTitleParam(t, "group", 0, [][]string{{`^# `}})- inputs := map[string]any{"output_format": "text", "text": body}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}got, err := invokeGroup(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeGroup: %v", err)@@ -519,7 +519,7 @@ func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {}body := strings.Join(records, "\n")p := newTitleParam(t, "group", 10, [][]string{{`^# `}})- inputs := map[string]any{"output_format": "text", "text": body}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}got, err := invokeGroup(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeGroup: %v", err)@@ -543,6 +543,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {body := strings.Join(joinSentences(12), "")p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})inputs := map[string]any{+ "name": "t.txt","output_format": "chunks","chunks": []schema.ChunkDoc{{Text: body, DocType: "text", Positions: json.RawMessage(`[[1,10,200,50,80]]`)},@@ -587,7 +588,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain(t *testing.T) {two := 2p.Hierarchy = &twoinputText := "# H1\n## H2a\n" + body- inputs := map[string]any{"output_format": "text", "text": inputText}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}got, err := invokeHierarchy(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeHierarchy: %v", err)@@ -633,7 +634,7 @@ func TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain(t *testing.T) {p.Hierarchy = &twop.RootChunkAsHeading = trueinputText := "前言。\n# H1\n## H2a\n" + body- inputs := map[string]any{"output_format": "text", "text": inputText}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}got, err := invokeHierarchy(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeHierarchy: %v", err)@@ -671,7 +672,7 @@ func TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain(t *testing.T)three := 3p.Hierarchy = &threeinputText := "# A\n前言A。\n## B\n前言B。\n### C\n" + body- inputs := map[string]any{"output_format": "text", "text": inputText}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}got, err := invokeHierarchy(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeHierarchy: %v", err)@@ -712,7 +713,7 @@ func TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles(t *testing.p.Hierarchy = &twop.IncludeHeadingContent = trueinputText := "# H1\n短。\n## H2a\n" + body- inputs := map[string]any{"output_format": "text", "text": inputText}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}got, err := invokeHierarchy(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeHierarchy: %v", err)@@ -755,7 +756,7 @@ func TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack(t *testing.T) {two := 2p.Hierarchy = &twoinputText := "# H1\n## H2a\n" + body- inputs := map[string]any{"output_format": "text", "text": inputText}+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}got, err := invokeHierarchy(testCtx(), nil, inputs, &p)if err != nil {t.Fatalf("invokeHierarchy: %v", err)diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex be1ca2a73a..4703f186b1 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -86,7 +86,6 @@ import ("ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser"- "ragflow/internal/parser/tableutil""ragflow/internal/parser/chunk")@@ -190,11 +189,7 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs}upstream, err := decodeChunkerFromUpstream(decInputs)if err != nil {- return map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{},- "_ERROR": fmt.Sprintf("Input error: %v", err),- }, nil+ return inputErrorOutput(err), nil}delimPattern := compileDelimPattern(c.param.Delimiters)@@ -273,20 +268,8 @@ func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstreaif err := out.Validate(); err != nil {return out, err}- // A structured table (e.g. the office/xlsx producer) carries no Text of its- // own. Derive its HTML <table> rendering on Text so every chunker that- // consumes it (token/group/hierarchy/one/page/qa) stores and displays a real- // table, and the chunk id (a hash of Text) matches the stored content.- for i := range out.JSONResult {- if out.JSONResult[i].TableData != nil && out.JSONResult[i].Text == "" {- out.JSONResult[i].Text = tableutil.RenderTableHTML(out.JSONResult[i].TableData)- }- }- for i := range out.Chunks {- if out.Chunks[i].TableData != nil && out.Chunks[i].Text == "" {- out.Chunks[i].Text = tableutil.RenderTableHTML(out.Chunks[i].TableData)- }- }+ materializeTableText(out.JSONResult)+ materializeTableText(out.Chunks)return out, nil}@@ -1857,43 +1840,6 @@ func compileChildrenPattern(delims []string) *regexp.Regexp {return chunk.CompileDelimiterPatternList(delims, true)}-// stringFromInputs returns the string value at the first matching key-// in `keys`, or ("", false) when none is set.-func stringFromInputs(inputs map[string]any, keys ...string) (string, bool) {- for _, k := range keys {- if v, ok := inputs[k].(string); ok {- return v, true- }- }- return "", false-}--// chunksFromInputs returns the chunk list from inputs as a uniform-// []map[string]any, or nil when absent. Both []map[string]any (the-// JSON-decoded form) and []any (the slice-of-mixed form) are handled.-//-// Two upstream keys are accepted, in priority order:-//-// - "chunks" — canonical post-chunker shape (chunker → chunker-// re-entry, test fixtures, downstream stages).-// - "json" — the parser-structured-output key (Parser-// component emits under "json"; we accept it-// so a token-chunker can run directly after-// a parser without an intermediate reshape).-func chunksFromInputs(inputs map[string]any) []schema.ChunkDoc {- for _, key := range []string{"chunks", "json"} {- v, ok := inputs[key]- if !ok {- continue- }- chunks, found, err := schema.ChunkDocsFromAny(v)- if err == nil && found {- return chunks- }- }- return nil-}-func intValue(v *int) int {if v == nil {return 0