Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): keep structured tables and images in the title family (#21040)

jay777210ad97e70af66
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 50b112f5ac..ea99778323 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -17,6 +17,8 @@
package chunker
import (
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
"regexp"
"strings"
@@ -25,6 +27,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/chunk"
+ "ragflow/internal/parser/tableutil"
"ragflow/internal/tokenizer"
)
@@ -233,6 +236,20 @@ func itemTextOrFallback(it schema.ChunkDoc) string {
return ""
}
+// materializeTableText fills Text with the HTML rendering of a structured
+// TableData when the item carries no text of its own — the shape every parser
+// emits for tables. Every consumer then works on a real table it can store and
+// display, and the chunk id (a hash of Text) matches the stored content.
+// RenderTableHTML marks the leading HeaderRows rows as <th>. Idempotent: an
+// item that already carries text is left untouched.
+func materializeTableText(docs []schema.ChunkDoc) {
+ for i := range docs {
+ if docs[i].TableData != nil && docs[i].Text == "" {
+ docs[i].Text = tableutil.RenderTableHTML(docs[i].TableData)
+ }
+ }
+}
+
// tokenizeStr is the shared NumTokensFromString wrapper used by
// Table/Image context attachment. Lives here so we can centrally
// swizzle the count strategy in one place if needed.
@@ -268,6 +285,16 @@ func emptyOutputs() map[string]any {
}
}
+// inputErrorOutput is the canonical short-circuit for an undecodable upstream
+// payload, shared by every chunker variant's decode step.
+func inputErrorOutput(err error) map[string]any {
+ return map[string]any{
+ "output_format": "chunks",
+ "chunks": []map[string]any{},
+ "_ERROR": fmt.Sprintf("Input error: %v", err),
+ }
+}
+
// chunkOutputs builds the canonical chunker output (output_format="chunks" +
// chunks). The Go runtime passes only this explicit output to the next node,
// so the run-level metadata that downstream components still need (e.g.
@@ -303,17 +330,51 @@ func canonicalChunkText(ck schema.ChunkDoc) string {
return removeTag(materializeMediaContext(ck).Text)
}
+// mediaChunkIDPrefix marks the identity source of a media chunk whose
+// canonical text is empty. The NUL byte keeps the source from colliding with
+// any real chunk text.
+const mediaChunkIDPrefix = "\x00media:"
+
// canonicalChunkID returns the deterministic chunk id: the single identity
// used both as the MinIO object key and as ck["id"]. The id formula (ChunkID
-// over canonicalChunkText) is centralized here, so the text normalization
-// that feeds the hash lives in exactly one place. The decorator in
-// register.go re-derives ck["id"] from the same already-finalized text as a
-// fallback for chunks that bypassed the streamed crop-upload path; the two
-// routes cannot disagree because a chunk is either stamped by crop-upload
-// (and the decorator reuses that id) or computed by the decorator fallback —
-// never both — and both hash the normalized chunk body.
+// over chunkIDSource) is centralized here, so the identity that feeds the
+// hash lives in exactly one place. The decorator in register.go re-derives
+// ck["id"] from the same already-finalized text as a fallback for chunks that
+// bypassed the streamed crop-upload path; the two routes cannot disagree
+// because a chunk is either stamped by crop-upload (and the decorator reuses
+// that id) or computed by the decorator fallback — never both — and both hash
+// the same identity.
func canonicalChunkID(docID string, ck schema.ChunkDoc) string {
- return common.ChunkID(docID, canonicalChunkText(ck))
+ return common.ChunkID(docID, chunkIDSource(ck))
+}
+
+// chunkIDSource returns the identity string hashed into a chunk id. For a
+// media chunk whose canonical text is empty — a captionless image — hashing
+// the text alone would give every such chunk in a document the same id
+// (ChunkID(docID, "")), silently overwriting images in MinIO and merging
+// index entries, so a stable per-image identity is folded in instead.
+// Non-empty text hashes unchanged.
+func chunkIDSource(ck schema.ChunkDoc) string {
+ if text := canonicalChunkText(ck); text != "" {
+ return text
+ }
+ return mediaChunkIdentity(ck.Image, ck.ImgID)
+}
+
+// mediaChunkIdentity returns the stable identity of a text-less media chunk:
+// a digest of the image payload when present (identical images collapse onto
+// one id, which is harmless — they carry identical bytes), else the image
+// reference. Empty when the chunk has neither, leaving the id the plain
+// empty-text hash.
+func mediaChunkIdentity(image, imgID string) string {
+ if image != "" {
+ sum := sha256.Sum256([]byte(image))
+ return mediaChunkIDPrefix + hex.EncodeToString(sum[:])
+ }
+ if imgID != "" {
+ return mediaChunkIDPrefix + imgID
+ }
+ return ""
}
// materializeMediaContext folds a media chunk's surrounding context into its
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index 84d1ee8f1a..2a48eca1ef 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -110,7 +110,11 @@ func buildSectionIDs(levels []int, targetLevel int) []int {
// shared chunkFromRecords pipeline (which ManualChunker also uses,
// after a physical-position resort).
func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {
- records := extractLineRecords(inputs)
+ upstream, err := decodeChunkerFromUpstream(inputs)
+ if err != nil {
+ return inputErrorOutput(err), nil
+ }
+ records := extractLineRecords(upstream)
common.Debug("chunker stage",
zap.String("component", "Chunker"),
zap.String("variant", "group"),
@@ -119,7 +123,7 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
if len(records) == 0 {
return emptyOutputs(), nil
}
- return chunkFromRecords(parentCtx, db, inputs, p, records, p.ChunkTokenCap)
+ return chunkFromRecords(parentCtx, db, upstream, p, records, p.ChunkTokenCap)
}
// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over
@@ -136,8 +140,8 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
// must stay exempt from #18455's cap, so it passes 0; GroupTitleChunker passes
// p.ChunkTokenCap. The cap is applied right after build_chunks and BEFORE the
// on-demand crop, mirroring Python's invoke() order (build -> cap -> set_chunks).
-func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord, tokenCap int) (map[string]any, error) {
- ctx := newLevelContext(records, outlineFromInputs(inputs), p)
+func chunkFromRecords(parentCtx context.Context, db *gorm.DB, upstream schema.ChunkerFromUpstream, p *titleChunkerParam, records []lineRecord, tokenCap int) (map[string]any, error) {
+ ctx := newLevelContext(records, outlineFromUpstream(upstream), p)
levels := ctx.Levels()
// Count heading level distribution for debugging.
headingCounts := make(map[int]int)
@@ -173,7 +177,7 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]
zap.String("variant", "group"),
zap.Int("groups", len(groups)),
)
- chunks, _ := buildChunksFromRecordGroups(groups, nil, p, isPlainTextFormat(inputs))
+ chunks, _ := buildChunksFromRecordGroups(groups, nil, p, isPlainTextFormat(upstream))
// Enforce the title-family token ceiling (chunk_token_cap) right after
// build_chunks and BEFORE the on-demand crop, mirroring Python's
// invoke() order (build -> cap -> set_chunks). ManualChunker passes
@@ -185,21 +189,19 @@ func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]
zap.String("component", "Chunker"),
zap.String("variant", "group"),
zap.Int("chunks", len(chunks)),
- zap.Bool("plain_text", isPlainTextFormat(inputs)),
+ zap.Bool("plain_text", isPlainTextFormat(upstream)),
)
// On-demand PDF preview cropping for image/table/text chunks,
// mirroring the TokenChunker JSON path (token.go:513). Best-effort:
// a missing or unreadable PDF simply skips cropping.
- if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {
- engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)
- if eErr != nil {
- common.Warn("chunker: could not open PDF for on-demand cropping", zap.Error(eErr))
- }
- if engine != nil {
- defer engine.Close()
- chunks = cropTitleChunks(parentCtx, engine, chunks)
- }
+ engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)
+ if eErr != nil {
+ common.Warn("chunker: could not open PDF for on-demand cropping", zap.Error(eErr))
+ }
+ if engine != nil {
+ defer engine.Close()
+ chunks = cropTitleChunks(parentCtx, engine, chunks)
}
if len(chunks) == 0 {
@@ -278,9 +280,10 @@ func joinGroupText(g []lineRecord) string {
// "html"]` branch of common.py:build_chunks_from_record_groups. Plain
// payloads emit only "text"; structured payloads (chunks/json) also
// carry doc_type_kwd and img_id.
-func isPlainTextFormat(inputs map[string]any) bool {
- if f, ok := inputs["output_format"].(string); ok {
- return f == "markdown" || f == "text" || f == "html"
+func isPlainTextFormat(upstream schema.ChunkerFromUpstream) bool {
+ switch upstream.OutputFormat {
+ case schema.PayloadFormatMarkdown, schema.PayloadFormatText, schema.PayloadFormatHTML:
+ return true
}
return false
}
@@ -311,10 +314,17 @@ func buildChunksFromRecordGroups(groups [][]lineRecord, prefixes []string, p *ti
if len(g) == 0 {
continue
}
- // Strip parser-emitted position tags (`@@...##`) from the
- // joined text (diff 1.6 / 2.8). Mirrors common.py:255
- // `RAGFlowPdfParser.remove_tag("".join(...))`.
- chunk := map[string]any{"text": removeTag(joinGroupText(g))}
+ // Strip parser-emitted position tags (`@@...##`) from the joined text
+ // (diff 1.6 / 2.8). Mirrors common.py:255
+ // `RAGFlowPdfParser.remove_tag("".join(...))`. A non-text group is a
+ // single atomic record (table/image): its text is taken verbatim —
+ // Python emits records[0]["text"] unchanged (common.py:419) — instead
+ // of the group join, which would append a trailing newline.
+ chunkText := removeTag(joinGroupText(g))
+ if !g[0].isText() {
+ chunkText = removeTag(g[0].text)
+ }
+ chunk := map[string]any{"text": chunkText}
if !plain {
first := g[0]
if first.docType != "" {
@@ -323,6 +333,18 @@ func buildChunksFromRecordGroups(groups [][]lineRecord, prefixes []string, p *ti
if first.imgID != nil {
chunk["img_id"] = *first.imgID
}
+ if first.image != "" {
+ chunk["image"] = first.image
+ }
+ if first.sheet != "" {
+ chunk["sheet"] = first.sheet
+ }
+ if first.sheetIndex != nil {
+ // Numbers on the chunk wire are float64 (every other
+ // chunker's maps pass through a JSON round trip);
+ // indexdoc.isSpreadsheetChunk keys on presence, not type.
+ chunk["sheet_index"] = float64(*first.sheetIndex)
+ }
}
// Merge PDF coordinate matrices across the merged records
// instead of keeping only the leading record's (diff 1.6).
@@ -427,37 +449,75 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {
return out
}
-// extractLineRecords reads the chunker inputs in the same order the
-// python BaseTitleChunker.extract_line_records uses:
+// extractLineRecords converts the decoded upstream payload into records in the
+// same order the python BaseTitleChunker.extract_line_records uses: the field
+// named by output_format selects the payload (chunks / json / markdown / text
+// / html), mirroring python's branch on output_format rather than key
+// presence alone — a declared "markdown" payload is read even when a stray
+// empty "text" key is also present. When output_format is unset the first
+// present payload wins (chunks, json, then text/markdown/html), preserving
+// the legacy key scan.
//
-// 1. If upstream emitted chunks (output_format == "chunks") OR
-// upstream emitted JSON, normalize from the list payload.
-// 2. Otherwise, treat text/markdown/html as a "one record per line"
-// stream (preserving indentation for non-text formats, strip-only
-// for the text format).
-func extractLineRecords(inputs map[string]any) []lineRecord {
- if docs := chunksFromInputs(inputs); docs != nil {
- return recordsFromStructured(docs)
- }
- text, _ := stringFromInputs(inputs, "text", "content", "markdown", "html")
- if text == "" {
+// The payload is consumed from the decoded upstream — never from the raw
+// inputs — so every chunker variant shares the one normalization entry
+// (decodeChunkerFromUpstream).
+func extractLineRecords(upstream schema.ChunkerFromUpstream) []lineRecord {
+ switch upstream.OutputFormat {
+ case schema.PayloadFormatChunks:
+ return recordsFromStructured(upstream.Chunks)
+ case schema.PayloadFormatJSON:
+ return recordsFromStructured(upstream.JSONResult)
+ case schema.PayloadFormatMarkdown:
+ return lineRecordsFromPayload(upstream.MarkdownResult)
+ case schema.PayloadFormatText:
+ return lineRecordsFromPayload(upstream.TextResult)
+ case schema.PayloadFormatHTML:
+ return lineRecordsFromPayload(upstream.HTMLResult)
+ }
+ // output_format unset: first present payload wins.
+ if upstream.Chunks != nil {
+ return recordsFromStructured(upstream.Chunks)
+ }
+ if upstream.JSONResult != nil {
+ return recordsFromStructured(upstream.JSONResult)
+ }
+ for _, candidate := range []*string{upstream.TextResult, upstream.MarkdownResult, upstream.HTMLResult} {
+ if candidate != nil {
+ return lineRecordsFromPayload(candidate)
+ }
+ }
+ return nil
+}
+
+// lineRecordsFromPayload splits a plain-text payload into one record per line.
+// An absent or empty payload yields no records.
+func lineRecordsFromPayload(payload *string) []lineRecord {
+ if payload == nil || *payload == "" {
return nil
}
- return lineRecordsFromText(text)
+ return lineRecordsFromText(*payload)
}
+// recordsFromStructured converts parser-shaped items into records: text items
+// keep their text, and media items — tables whose TableData has already been
+// rendered to HTML by materializeTableText, images carrying an image payload
+// or reference — pass through as single records. Only an item with none of
+// text, image and img_id is dropped.
func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {
out := make([]lineRecord, 0, len(items))
for _, it := range items {
text := itemTextOrFallback(it)
- if text == "" {
+ // A structured table arrives with Text already rendered from TableData
+ // (materializeTableText); an image item legitimately carries no text —
+ // its payload is the image field. Only a truly empty item is skipped.
+ if text == "" && it.Image == "" && it.ImgID == "" {
continue
}
dt := it.DocType
if dt == "" {
dt = "text"
}
- appendRecord := func(recordText string) {
+ appendRecord := func(recordText, image string) {
var imgID *string
if it.ImgID != "" {
img := it.ImgID
@@ -483,28 +543,32 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {
text: recordText,
docType: dt,
imgID: imgID,
+ image: image,
layout: it.Layout,
ckType: it.CKType,
pdfPositions: it.PDFPositions,
positions: it.Positions,
+ sheet: it.Sheet,
+ sheetIndex: it.SheetIndex,
parentMeta: meta,
})
}
// TextParser intentionally emits one coordinate-free text item for the
// whole document. Title-family chunkers still need one record per line
// so heading regexes can see headings after the first line. Structured
- // media/table items and positioned text remain atomic.
+ // media/table items and positioned text remain atomic, and the image
+ // payload stays out of per-line records so a blob is never duplicated.
plainText := itemDocType(it) == "text" && (it.CKType == "" || it.CKType == "text")
unpositioned := len(it.PDFPositions) == 0 && len(it.Positions) == 0
if plainText && unpositioned && strings.Contains(text, "\n") {
for _, line := range strings.Split(normalizeGeneralNewlines(text), "\n") {
if trim(line) != "" {
- appendRecord(line)
+ appendRecord(line, "")
}
}
continue
}
- appendRecord(text)
+ appendRecord(text, it.Image)
}
return out
}
diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.go
index a9edcea927..c14d4e8bbb 100644
--- a/internal/ingestion/component/chunker/group_test.go
+++ b/internal/ingestion/component/chunker/group_test.go
@@ -65,12 +65,24 @@ func TestGroupTitleChunker_NewRejectsHierarchyWithoutHierarchyParam(t *testing.T
}
}
+// decodedRecords decodes a raw chunker payload the way every component does
+// (decodeChunkerFromUpstream) and extracts the line records from the result.
+func decodedRecords(t *testing.T, payload map[string]any) []lineRecord {
+ t.Helper()
+ upstream, err := decodeChunkerFromUpstream(payload)
+ if err != nil {
+ t.Fatalf("decodeChunkerFromUpstream: %v", err)
+ }
+ return extractLineRecords(upstream)
+}
+
// TestExtractLineRecords_MarkdownKey verifies extractLineRecords reads
// the "markdown" payload key. Before this fix extractLineRecords only
-// looked at "text"/"content" and silently returned nil for parser
+// looked at "text" and silently returned nil for parser
// output that carried output_format="markdown".
func TestExtractLineRecords_MarkdownKey(t *testing.T) {
- records := extractLineRecords(map[string]any{
+ records := decodedRecords(t, map[string]any{
+ "name": "t.md",
"output_format": "markdown",
"markdown": "line1\nline2\nline3",
})
@@ -84,10 +96,27 @@ func TestExtractLineRecords_MarkdownKey(t *testing.T) {
}
}
+// TestExtractLineRecords_DeclaredFormatBeatsStrayEmptyText pins the
+// output_format-driven payload selection (python parity): a declared
+// "markdown" payload is read even when a stray empty "text" key is present,
+// so an empty alias key cannot silently swallow the whole document.
+func TestExtractLineRecords_DeclaredFormatBeatsStrayEmptyText(t *testing.T) {
+ records := decodedRecords(t, map[string]any{
+ "name": "t.md",
+ "output_format": "markdown",
+ "markdown": "line1\nline2",
+ "text": "",
+ })
+ if len(records) != 2 {
+ t.Fatalf("got %d records, want 2 from the declared markdown payload", len(records))
+ }
+}
+
// TestExtractLineRecords_HTMLKey verifies extractLineRecords reads the
// "html" payload key, same safety-net rationale as TestMarkdownKey.
func TestExtractLineRecords_HTMLKey(t *testing.T) {
- records := extractLineRecords(map[string]any{
+ records := decodedRecords(t, map[string]any{
+ "name": "t.html",
"output_format": "html",
"html": "<p>first</p>\n<p>second</p>",
})
@@ -99,7 +128,8 @@ func TestExtractLineRecords_HTMLKey(t *testing.T) {
// TestExtractLineRecords_TextKeyStillWorks ensures the existing "text"
// key path is not broken by the addition of "markdown"/"html".
func TestExtractLineRecords_TextKeyStillWorks(t *testing.T) {
- records := extractLineRecords(map[string]any{
+ records := decodedRecords(t, map[string]any{
+ "name": "t.txt",
"text": "hello\nworld",
})
if len(records) != 2 {
@@ -113,7 +143,7 @@ func TestExtractLineRecords_ParserStructuredTextSplitsLines(t *testing.T) {
t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
}
- records := extractLineRecords(map[string]any{
+ records := decodedRecords(t, map[string]any{
"name": "notes.txt",
"output_format": "json",
"json": parsed.JSON,
diff --git a/internal/ingestion/component/chunker/hierarchy.go b/internal/ingestion/component/chunker/hierarchy.go
index f590f68957..2c4e0f817c 100644
--- a/internal/ingestion/component/chunker/hierarchy.go
+++ b/internal/ingestion/component/chunker/hierarchy.go
@@ -141,7 +141,11 @@ func (n *chunkNode) getPaths(paths *[][]int, titles []int, depth int, includeHea
// invokeHierarchy runs the HierarchyTitleChunker strategy.
func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {
- records := extractLineRecords(inputs)
+ upstream, err := decodeChunkerFromUpstream(inputs)
+ if err != nil {
+ return inputErrorOutput(err), nil
+ }
+ records := extractLineRecords(upstream)
common.Debug("chunker stage",
zap.String("component", "Chunker"),
zap.String("variant", "hierarchy"),
@@ -150,7 +154,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
if len(records) == 0 {
return emptyOutputs(), nil
}
- ctx := newLevelContext(records, outlineFromInputs(inputs), p)
+ ctx := newLevelContext(records, outlineFromUpstream(upstream), p)
levels := ctx.Levels()
// Count heading level distribution for debugging.
headingCounts := make(map[int]int)
@@ -266,7 +270,7 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
zap.Int("record_groups", len(recordGroups)),
)
- chunks, prefixes := buildChunksFromRecordGroups(recordGroups, groupPrefixes, p, isPlainTextFormat(inputs))
+ chunks, prefixes := buildChunksFromRecordGroups(recordGroups, groupPrefixes, p, isPlainTextFormat(upstream))
// Enforce the title-family token ceiling (chunk_token_cap) right after
// build_chunks and BEFORE the on-demand crop, mirroring Python's
// invoke() order (build -> cap -> set_chunks). prefixes re-attaches each
@@ -278,21 +282,19 @@ func invokeHierarchy(parentCtx context.Context, db *gorm.DB, inputs map[string]a
zap.String("component", "Chunker"),
zap.String("variant", "hierarchy"),
zap.Int("chunks", len(chunks)),
- zap.Bool("plain_text", isPlainTextFormat(inputs)),
+ zap.Bool("plain_text", isPlainTextFormat(upstream)),
)
// On-demand PDF preview cropping for image/table/text chunks,
// mirroring the TokenChunker JSON path (token.go:513). Best-effort:
// a missing or unreadable PDF simply skips cropping.
- if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {
- engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)
- if eErr != nil {
- common.Warn("HierarchyTitleChunker: could not open PDF for on-demand cropping", zap.Error(eErr))
- }
- if engine != nil {
- defer engine.Close()
- chunks = cropTitleChunks(parentCtx, engine, chunks)
- }
+ engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)
+ if eErr != nil {
+ common.Warn("HierarchyTitleChunker: could not open PDF for on-demand cropping", zap.Error(eErr))
+ }
+ if engine != nil {
+ defer engine.Close()
+ chunks = cropTitleChunks(parentCtx, engine, chunks)
}
if len(chunks) == 0 {
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go
index d783f1932e..3225493489 100644
--- a/internal/ingestion/component/chunker/manual.go
+++ b/internal/ingestion/component/chunker/manual.go
@@ -105,8 +105,14 @@ func (c *ManualChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs
return c.invoke(ctx, db, withName(inputs, name))
}
+// invoke decodes the upstream payload once, extracts the records, and runs
+// the shared group pipeline after a physical-position resort.
func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
- records := extractLineRecords(inputs)
+ upstream, err := decodeChunkerFromUpstream(inputs)
+ if err != nil {
+ return inputErrorOutput(err), nil
+ }
+ records := extractLineRecords(upstream)
if len(records) == 0 {
return emptyOutputs(), nil
}
@@ -117,7 +123,7 @@ func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs
}
// ManualChunker is exempt from the title-family token cap (#18455): it
// does not inherit BaseTitleChunker, so it always passes tokenCap=0.
- return chunkFromRecords(ctx, db, inputs, &c.param, records, 0)
+ return chunkFromRecords(ctx, db, upstream, &c.param, records, 0)
}
// hasPdfPositions reports whether any record carries a PDF coordinate matrix
diff --git a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go
index 897928fd72..7d377af73e 100644
--- a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go
+++ b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go
@@ -79,10 +79,14 @@ func TestManualChunker_RealPDFResort(t *testing.T) {
// Same shape the chunker receives in production.
input := map[string]any{
"name": "Doc1.pdf",
- "output_format": "json",
+ "output_format": "chunks",
"chunks": res.JSON,
}
- records := extractLineRecords(input)
+ upstream, err := decodeChunkerFromUpstream(input)
+ if err != nil {
+ t.Fatalf("decodeChunkerFromUpstream: %v", err)
+ }
+ records := extractLineRecords(upstream)
if len(records) == 0 {
t.Fatal("extractLineRecords produced no records from real parser output")
}
diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.go
index 5fb2f9dd5f..10bf71c4e5 100644
--- a/internal/ingestion/component/chunker/manual_test.go
+++ b/internal/ingestion/component/chunker/manual_test.go
@@ -322,7 +322,7 @@ func TestHasPdfPositions(t *testing.T) {
// with 1-based page numbers, accompanied by layout / page_number /
// doc_type_kwd. Using [][]any — not [][]float64 — matters: that is the concrete
// in-memory type the parser produces, and recordsFromStructured must consume it
-// after the chunksFromInputs JSON round-trip.
+// after the decodeChunkerFromUpstream JSON round-trip.
func parserShapedItem(text string, page, left, right, top, bottom float64) map[string]any {
return map[string]any{
"text": text,
diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.go
index 9f21d806e2..53ae5fd1fe 100644
--- a/internal/ingestion/component/chunker/one.go
+++ b/internal/ingestion/component/chunker/one.go
@@ -29,7 +29,6 @@ package chunker
import (
"context"
- "fmt"
"strings"
"ragflow/internal/agent/runtime"
@@ -72,17 +71,14 @@ func (c *OneChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs ma
return c.invoke(ctx, inputs)
}
+// invoke collapses the decoded upstream payload into a single chunk.
func (c *OneChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
if inputs == nil {
return emptyOutputs(), nil
}
upstream, err := decodeChunkerFromUpstream(inputs)
if err != nil {
- return map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{},
- "_ERROR": fmt.Sprintf("Input error: %v", err),
- }, nil
+ return inputErrorOutput(err), nil
}
switch upstream.OutputFormat {
diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.go
index a59e2aeadb..10ae52dac1 100644
--- a/internal/ingestion/component/chunker/page.go
+++ b/internal/ingestion/component/chunker/page.go
@@ -30,7 +30,6 @@ package chunker
import (
"context"
- "fmt"
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
@@ -74,17 +73,14 @@ func (c *PageChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs m
return c.invoke(ctx, inputs)
}
+// invoke emits one chunk per decoded upstream item (slide or page).
func (c *PageChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
if inputs == nil {
return emptyOutputs(), nil
}
upstream, err := decodeChunkerFromUpstream(inputs)
if err != nil {
- return map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{},
- "_ERROR": fmt.Sprintf("Input error: %v", err),
- }, nil
+ return inputErrorOutput(err), nil
}
// The Page chunker preserves one incoming JSON item per output chunk.
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index f381120aa4..fb09b86f3b 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -88,17 +88,14 @@ func (c *QAChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
return c.invoke(ctx, inputs)
}
+// invoke extracts question-answer pairs from the decoded upstream payload.
func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
if inputs == nil {
return emptyOutputs(), nil
}
upstream, err := decodeChunkerFromUpstream(inputs)
if err != nil {
- return map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{},
- "_ERROR": fmt.Sprintf("Input error: %v", err),
- }, nil
+ return inputErrorOutput(err), nil
}
qPrefix, aPrefix := "问题:", "回答:"
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index dc875ce72c..661c30aeef 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -96,10 +96,12 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
//
// A chunk id may already be stamped (e.g. by the streamed crop-upload
// path, which keys MinIO by the canonical id). Reuse it so the id is
- // computed exactly once; both routes normalize the text through
- // canonicalChunkText, so the crop-time key and this id are identical
- // and cannot diverge. For chunks that bypassed crop, fall back to
- // deriving it here from the already-finalized text.
+ // computed exactly once; both routes derive the identity through the
+ // shared helpers, so the crop-time key and this id are identical and
+ // cannot diverge. For chunks that bypassed crop, fall back to deriving
+ // it here from the already-finalized text — a captionless image folds
+ // its payload identity in (mediaChunkIdentity), so distinct images in
+ // one document never share an id.
for _, ck := range chunks {
if id, ok := ck["id"]; ok && id != "" {
continue
@@ -108,6 +110,11 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
if err != nil {
return nil, err
}
+ if text == "" {
+ image, _ := ck["image"].(string)
+ imgID, _ := ck["img_id"].(string)
+ text = mediaChunkIdentity(image, imgID)
+ }
ck["id"] = common.ChunkID(docID, text)
}
@@ -132,7 +139,6 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
// by all chunker variants.
var ChunkerInputs = map[string]string{
"text": "Plain-text input. The chunker slices this into downstream chunks.",
- "content": "Alias for \"text\".",
"chunks": "Optional upstream chunk list (structured JSON form).",
"name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.",
"file_type": "Canonical parser routing extension used by GeneralChunker.",
diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.go
index 64a7709391..0580d496ef 100644
--- a/internal/ingestion/component/chunker/sheet_identity_test.go
+++ b/internal/ingestion/component/chunker/sheet_identity_test.go
@@ -46,8 +46,19 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {
}
for _, tc := range []struct {
name string
+ params map[string]any
factory func(map[string]any) (runtime.Component, error)
- }{{"QA", NewQAChunker}, {"Token", NewTokenChunker}, {"One", NewOneChunker}} {
+ }{
+ {"QA", nil, NewQAChunker},
+ {"Token", nil, NewTokenChunker},
+ {"One", nil, NewOneChunker},
+ // The title-family chunkers share extractLineRecords; a
+ // spreadsheet item must keep its sheet identity there too, or its
+ // selections would be decoded as PDF boxes.
+ {"Group", map[string]any{"levels": [][]string{{`^# `}}}, NewGroupTitleChunker},
+ {"Hierarchy", map[string]any{"hierarchy": 1, "levels": [][]string{{`^# `}}}, NewHierarchyTitleChunker},
+ {"Manual", map[string]any{"levels": [][]string{{`^# `}}}, NewManualChunker},
+ } {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
batches := [][]map[string]any{parser.ItemsToMaps(source.result.JSON)}
// One preserves positions only for a single source item.
@@ -58,7 +69,7 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {
}
}
for _, items := range batches {
- comp, err := tc.factory(nil)
+ comp, err := tc.factory(tc.params)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index 1e674a1289..cf455aba8e 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -31,7 +31,6 @@ package chunker
import (
"context"
"encoding/json"
- "fmt"
"strings"
"ragflow/internal/agent/runtime"
@@ -76,17 +75,14 @@ func (c *TableChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs
return c.invoke(ctx, inputs)
}
+// invoke expands the decoded upstream tables into one chunk per data row.
func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
if inputs == nil {
return emptyOutputs(), nil
}
upstream, err := decodeChunkerFromUpstream(inputs)
if err != nil {
- return map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{},
- "_ERROR": fmt.Sprintf("Input error: %v", err),
- }, nil
+ return inputErrorOutput(err), nil
}
switch upstream.OutputFormat {
diff --git a/internal/ingestion/component/chunker/title.go b/internal/ingestion/component/chunker/title.go
index 87f8f38278..4bf5400223 100644
--- a/internal/ingestion/component/chunker/title.go
+++ b/internal/ingestion/component/chunker/title.go
@@ -39,7 +39,7 @@
// fallback_level = len(selected_group) + 1. Strategy (1) —
// PDF-outline detection — is also ported: newLevelContext
// (title.go) reads the Parser-supplied file.outline via
-// outlineFromInputs / resolveOutlineLevels / outlineSimilarity
+// outlineFromUpstream / resolveOutlineLevels / outlineSimilarity
// (port of common.py:_outline_similarity) before falling back to
// the regex/layout branch, wired into group.go / hierarchy.go
// (Chunker omission 1.5, Gap C closed).
@@ -557,27 +557,27 @@ func resolveOutlineLevels(records []lineRecord, outline []outlineEntry) (levels
return levels, max(1, maxLevel), true
}
-// outlineFromInputs reads the parser-supplied PDF outline from the upstream
-// file metadata (file.outline, written by the ingestion PDF parser's
-// outlinesToFileMeta) and normalizes it into the chunker's outlineEntry
-// shape. Returns nil when no outline is present, so callers fall back to
-// frequency-based title detection. Numbers are coerced from int/float64
-// because the runtime may hand the chunker a JSON-decoded payload.
-func outlineFromInputs(inputs map[string]any) []outlineEntry {
- file, _ := inputs["file"].(map[string]any)
- if file == nil {
+// outlineFromUpstream reads the parser-supplied PDF outline from the decoded
+// upstream file metadata (file.outline, written by the ingestion PDF parser's
+// outlinesToFileMeta and kept verbatim in ChunkerFileMeta.Extra) and normalizes
+// it into the chunker's outlineEntry shape. Returns nil when no outline is
+// present, so callers fall back to frequency-based title detection. Numbers
+// are coerced from int/float64 because the runtime may hand the chunker a
+// JSON-decoded payload.
+func outlineFromUpstream(upstream schema.ChunkerFromUpstream) []outlineEntry {
+ if upstream.File == nil {
return nil
}
- raw, _ := file["outline"].([]any)
- if len(raw) == 0 {
+ raw, ok := upstream.File.Extra["outline"]
+ if !ok || len(raw) == 0 {
return nil
}
- out := make([]outlineEntry, 0, len(raw))
- for _, item := range raw {
- m, ok := item.(map[string]any)
- if !ok {
- continue
- }
+ var items []map[string]any
+ if err := json.Unmarshal(raw, &items); err != nil {
+ return nil
+ }
+ out := make([]outlineEntry, 0, len(items))
+ for _, m := range items {
title, _ := m["title"].(string)
if title == "" {
continue
@@ -638,10 +638,13 @@ type lineRecord struct {
text string
docType string
imgID *string
+ image string
layout string
ckType string
pdfPositions json.RawMessage
positions json.RawMessage
+ sheet string
+ sheetIndex *int
parentMeta map[string]any
}
@@ -773,9 +776,9 @@ func (c *TitleChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs
}
switch c.param.Method {
case "hierarchy":
- return invokeHierarchy(ctx, db, inputs, &c.param)
+ return invokeHierarchy(ctx, db, withName(inputs, name), &c.param)
case "group":
- return invokeGroup(ctx, db, inputs, &c.param)
+ return invokeGroup(ctx, db, withName(inputs, name), &c.param)
default:
return map[string]any{
"output_format": "chunks",
diff --git a/internal/ingestion/component/chunker/title_structured_media_test.go b/internal/ingestion/component/chunker/title_structured_media_test.go
new file mode 100644
index 0000000000..34a2529f98
--- /dev/null
+++ b/internal/ingestion/component/chunker/title_structured_media_test.go
@@ -0,0 +1,351 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "context"
+ "encoding/json"
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/tableutil"
+)
+
+const titleTestTableHTML = "<table><tr><th>name</th><th>age</th></tr><tr><td>Alice</td><td>30</td></tr></table>"
+
+// pngBase64Red is a second valid 1x1 PNG (red), distinct from pngBase64, used
+// to pin that captionless images keep distinct chunk ids.
+const pngBase64Red = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+
+// titleFamilyCase pairs a title-family entry point with its display name.
+type titleFamilyCase struct {
+ name string
+ comp runtime.Component
+}
+
+// titleFamilyComponents builds the three entry points behind the shared
+// extractLineRecords record path. TitleChunker's group/hierarchy methods
+// dispatch into the same invokeGroup/invokeHierarchy implementations, so the
+// standalone components cover every title-family strategy.
+func titleFamilyComponents(t *testing.T) []titleFamilyCase {
+ t.Helper()
+ levels := [][]string{{`^# `}}
+ group, err := NewGroupTitleChunker(map[string]any{"levels": levels})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ hierarchy, err := NewHierarchyTitleChunker(map[string]any{"hierarchy": 1, "levels": levels})
+ if err != nil {
+ t.Fatalf("NewHierarchyTitleChunker: %v", err)
+ }
+ manual, err := NewManualChunker(map[string]any{"levels": levels})
+ if err != nil {
+ t.Fatalf("NewManualChunker: %v", err)
+ }
+ return []titleFamilyCase{
+ {"group", group},
+ {"hierarchy", hierarchy},
+ {"manual", manual},
+ }
+}
+
+// titleFamilyChunks invokes one title-family component over a json payload and
+// returns the emitted chunks.
+func titleFamilyChunks(t *testing.T, comp runtime.Component, items []map[string]any) []map[string]any {
+ t.Helper()
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.pdf",
+ "output_format": "json",
+ "json": items,
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ return chunks
+}
+
+// renderedChunkText returns the HTML rendering the chunker derives from the
+// item's structured TableData, through the same map round trip the chunker
+// performs.
+func renderedChunkText(t *testing.T, item map[string]any) string {
+ t.Helper()
+ raw, err := json.Marshal(item["table"])
+ if err != nil {
+ t.Fatalf("marshal table: %v", err)
+ }
+ var typed entity.TableData
+ if err := json.Unmarshal(raw, &typed); err != nil {
+ t.Fatalf("unmarshal table: %v", err)
+ }
+ return tableutil.RenderTableHTML(&typed)
+}
+
+// TestTitleFamilyRendersStructuredTableAtomic: a parser-shaped table item
+// (structured TableData, no text) must survive the title-family record path as
+// one atomic chunk carrying the verbatim HTML rendering — header rows rendered
+// as <th> per HeaderRows — with its positions kept and no neighbor text glued
+// in.
+func TestTitleFamilyRendersStructuredTableAtomic(t *testing.T) {
+ item := tableItem(t, titleTestTableHTML, map[string]any{
+ "_pdf_positions": [][]float64{{1, 10, 200, 10, 40}},
+ })
+ want := renderedChunkText(t, item)
+ if !strings.Contains(want, "<th>name</th>") || !strings.Contains(want, "<td>Alice</td>") {
+ t.Fatalf("fixture rendering lost header/cell markup: %q", want)
+ }
+
+ for _, tc := range titleFamilyComponents(t) {
+ t.Run(tc.name, func(t *testing.T) {
+ chunks := titleFamilyChunks(t, tc.comp, []map[string]any{
+ {"text": "# H1", "doc_type_kwd": "text"},
+ item,
+ {"text": "body paragraph", "doc_type_kwd": "text"},
+ })
+ var tableChunks []map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "table" {
+ tableChunks = append(tableChunks, ck)
+ }
+ }
+ if len(tableChunks) != 1 {
+ t.Fatalf("table chunks = %d, want exactly 1 (chunks: %#v)", len(tableChunks), chunks)
+ }
+ ck := tableChunks[0]
+ if ck["text"] != want {
+ t.Errorf("table chunk text = %q, want the verbatim rendering %q", ck["text"], want)
+ }
+ if _, ok := ck["table"]; ok {
+ t.Errorf("structured TableData must not leak onto the chunk: %#v", ck["table"])
+ }
+ if got := ck["_pdf_positions"]; !reflect.DeepEqual(got, [][]float64{{1, 10, 200, 10, 40}}) {
+ t.Errorf("_pdf_positions = %#v, want the source matrix", got)
+ }
+ for _, other := range chunks {
+ if other["doc_type_kwd"] != "table" && strings.Contains(toString(other["text"]), "<table>") {
+ t.Errorf("table markup leaked into a text chunk: %#v", other)
+ }
+ }
+ })
+ }
+}
+
+// TestTitleFamilyKeepsEmptyTextImageItem: an image item legitimately carries no
+// text — its payload is the image field — and must survive as a chunk instead
+// of being dropped by the empty-text skip.
+func TestTitleFamilyKeepsEmptyTextImageItem(t *testing.T) {
+ payload := "data:image/png;base64," + pngBase64
+ for _, tc := range titleFamilyComponents(t) {
+ t.Run(tc.name, func(t *testing.T) {
+ chunks := titleFamilyChunks(t, tc.comp, []map[string]any{
+ {"text": "", "image": payload, "doc_type_kwd": "image"},
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %d, want the image item kept as 1 chunk", len(chunks))
+ }
+ ck := chunks[0]
+ if ck["doc_type_kwd"] != "image" {
+ t.Errorf("doc_type_kwd = %v, want image", ck["doc_type_kwd"])
+ }
+ if ck["image"] != payload {
+ t.Errorf("image payload lost: %v", ck["image"])
+ }
+ if ck["text"] != "" {
+ t.Errorf("text = %q, want empty", ck["text"])
+ }
+ })
+ }
+}
+
+// TestTitleFamilyEmptyTextImageUploadsThroughDecorator: the decorated chunker
+// uploads the payload at the chunk id, writes img_id, and drops the raw bytes —
+// the empty-text image chunk must flow through the same decorator path as
+// every other chunker's media chunk.
+func TestTitleFamilyEmptyTextImageUploadsThroughDecorator(t *testing.T) {
+ prev := ChunkImageUploader
+ ChunkImageUploader = func(_ context.Context, kbID, chunkID string, _ []byte) (string, error) {
+ return kbID + "-" + chunkID, nil
+ }
+ t.Cleanup(func() { ChunkImageUploader = prev })
+
+ comp, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ decorated := &imageUploadDecorator{inner: comp}
+
+ payload := "data:image/png;base64," + pngBase64
+ out, err := decorated.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.docx",
+ "kb_id": testKBID,
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "", "image": payload, "doc_type_kwd": "image"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("decorated Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %d, want 1", len(chunks))
+ }
+ ck := chunks[0]
+ // No doc_id in the payload (it would trigger the on-demand PDF crop path,
+ // which needs a database), so the id hashes the empty document id plus the
+ // captionless image's own identity (mediaChunkIdentity).
+ wantChunkID := common.ChunkID("", mediaChunkIdentity(payload, ""))
+ if ck["id"] != wantChunkID {
+ t.Errorf("id = %v, want %q", ck["id"], wantChunkID)
+ }
+ if ck["img_id"] != testKBID+"-"+wantChunkID {
+ t.Errorf("img_id = %v, want %q", ck["img_id"], testKBID+"-"+wantChunkID)
+ }
+ if _, ok := ck["image"]; ok {
+ t.Errorf("image not dropped after upload: %v", ck["image"])
+ }
+}
+
+// TestTitleFamilyCaptionlessImagesGetDistinctIDs: two different captionless
+// images in one document must not collapse onto the same chunk id — a shared
+// id would make the second upload overwrite the first image in MinIO and merge
+// the two index entries.
+func TestTitleFamilyCaptionlessImagesGetDistinctIDs(t *testing.T) {
+ uploaded := map[string]int{}
+ prev := ChunkImageUploader
+ ChunkImageUploader = func(_ context.Context, kbID, chunkID string, _ []byte) (string, error) {
+ uploaded[chunkID]++
+ return kbID + "-" + chunkID, nil
+ }
+ t.Cleanup(func() { ChunkImageUploader = prev })
+
+ comp, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ decorated := &imageUploadDecorator{inner: comp}
+
+ out, err := decorated.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.docx",
+ "kb_id": testKBID,
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "", "image": "data:image/png;base64," + pngBase64, "doc_type_kwd": "image"},
+ {"text": "", "image": "data:image/png;base64," + pngBase64Red, "doc_type_kwd": "image"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("decorated Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %d, want 2", len(chunks))
+ }
+ idA, _ := chunks[0]["id"].(string)
+ idB, _ := chunks[1]["id"].(string)
+ if idA == "" || idB == "" || idA == idB {
+ t.Fatalf("captionless image ids must be distinct, got %q and %q", idA, idB)
+ }
+ if idA == common.ChunkID("", "") || idB == common.ChunkID("", "") {
+ t.Fatalf("id still hashes the empty text alone: %q / %q", idA, idB)
+ }
+ if len(uploaded) != 2 || uploaded[idA]+uploaded[idB] != 2 {
+ t.Fatalf("uploads must land on two distinct keys, got %v", uploaded)
+ }
+}
+
+// TestTitleFamilyReportsDecodeFailure: a payload that declares a format but
+// carries no matching payload fails decode and short-circuits through the
+// family-wide input-error shape instead of silently producing nothing.
+func TestTitleFamilyReportsDecodeFailure(t *testing.T) {
+ for _, tc := range titleFamilyComponents(t) {
+ t.Run(tc.name, func(t *testing.T) {
+ out, err := tc.comp.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.pdf",
+ "output_format": "json", // no "json" payload → decode fails
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if msg, _ := out["_ERROR"].(string); !strings.HasPrefix(msg, "Input error:") {
+ t.Fatalf("_ERROR = %v, want an input error", out["_ERROR"])
+ }
+ if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 0 {
+ t.Fatalf("chunks = %#v, want none", chunks)
+ }
+ })
+ }
+}
+
+// TestTitleFamilyDropsFullyEmptyItems: an item with no text, no table and no
+// image has nothing retrievable and stays dropped.
+func TestTitleFamilyDropsFullyEmptyItems(t *testing.T) {
+ for _, tc := range titleFamilyComponents(t) {
+ t.Run(tc.name, func(t *testing.T) {
+ chunks := titleFamilyChunks(t, tc.comp, []map[string]any{
+ {"text": "", "doc_type_kwd": "text"},
+ })
+ if len(chunks) != 0 {
+ t.Fatalf("chunks = %d, want 0 for a fully empty item: %#v", len(chunks), chunks)
+ }
+ chunks = titleFamilyChunks(t, tc.comp, []map[string]any{
+ {"text": "", "doc_type_kwd": "text"},
+ {"text": "real body", "doc_type_kwd": "text"},
+ })
+ if len(chunks) != 1 || !strings.Contains(toString(chunks[0]["text"]), "real body") {
+ t.Fatalf("chunks = %#v, want only the retrievable text chunk", chunks)
+ }
+ })
+ }
+}
+
+// TestTitleFamilyPreservesSheetIdentity: a spreadsheet item routed through a
+// title chunker keeps its sheet identity so its positions matrix stays in the
+// spreadsheet vocabulary instead of being decoded as PDF boxes.
+func TestTitleFamilyPreservesSheetIdentity(t *testing.T) {
+ item := tableItem(t, "<table><tr><th>Q</th><th>A</th></tr><tr><td>q1</td><td>a1</td></tr></table>",
+ map[string]any{
+ "sheet": "Orders",
+ "sheet_index": 1,
+ "positions": [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}},
+ })
+ comp, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ chunks := titleFamilyChunks(t, comp, []map[string]any{item})
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %d, want 1", len(chunks))
+ }
+ ck := chunks[0]
+ if ck["sheet"] != "Orders" {
+ t.Errorf("sheet = %v, want Orders", ck["sheet"])
+ }
+ if ck["sheet_index"] != float64(1) {
+ t.Errorf("sheet_index = %#v, want 1", ck["sheet_index"])
+ }
+ if _, ok := ck["_pdf_positions"]; ok {
+ t.Errorf("spreadsheet selections leaked into _pdf_positions: %#v", ck["_pdf_positions"])
+ }
+ if got := ck["positions"]; !reflect.DeepEqual(got, [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}}) {
+ t.Errorf("positions = %#v, want the source matrix", got)
+ }
+}
diff --git a/internal/ingestion/component/chunker/title_test.go b/internal/ingestion/component/chunker/title_test.go
index 114ab1c5b5..a1b0a4c083 100644
--- a/internal/ingestion/component/chunker/title_test.go
+++ b/internal/ingestion/component/chunker/title_test.go
@@ -439,18 +439,23 @@ func TestNewLevelContext_OutlineBranch(t *testing.T) {
}
}
-// TestOutlineFromInputs verifies the parser-supplied file.outline is parsed
-// into outlineEntry, tolerating float64-encoded levels (runtime JSON path).
-func TestOutlineFromInputs(t *testing.T) {
- inputs := map[string]any{
+// TestOutlineFromUpstream verifies the parser-supplied file.outline survives
+// the decode path and is parsed into outlineEntry, tolerating float64-encoded
+// levels (runtime JSON path).
+func TestOutlineFromUpstream(t *testing.T) {
+ up, err := decodeChunkerFromUpstream(map[string]any{
+ "name": "t.pdf",
"file": map[string]any{
"outline": []any{
map[string]any{"title": "第一章", "level": float64(0), "page_number": float64(1)},
map[string]any{"title": "第二章", "level": float64(1), "page_number": float64(3)},
},
},
+ })
+ if err != nil {
+ t.Fatalf("decodeChunkerFromUpstream: %v", err)
}
- out := outlineFromInputs(inputs)
+ out := outlineFromUpstream(up)
if len(out) != 2 {
t.Fatalf("outline len = %d, want 2", len(out))
}
@@ -462,8 +467,12 @@ func TestOutlineFromInputs(t *testing.T) {
}
// No file / no outline -> nil (frequency fallback).
- if got := outlineFromInputs(map[string]any{}); got != nil {
- t.Errorf("empty inputs outline = %v, want nil", got)
+ empty, err := decodeChunkerFromUpstream(map[string]any{"name": "t.pdf"})
+ if err != nil {
+ t.Fatalf("decodeChunkerFromUpstream: %v", err)
+ }
+ if got := outlineFromUpstream(empty); got != nil {
+ t.Errorf("empty upstream outline = %v, want nil", got)
}
}
diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.go
index 4772e6a014..d35cf8fb2a 100644
--- a/internal/ingestion/component/chunker/title_token_cap_test.go
+++ b/internal/ingestion/component/chunker/title_token_cap_test.go
@@ -440,7 +440,7 @@ func TestTitleCap_GroupPipeline_RespectsCap(t *testing.T) {
defer restoreTokenizer()
body := strings.Join(joinSentences(12), "")
p := newTitleParam(t, "group", 20, [][]string{{`^# `}})
- inputs := map[string]any{"output_format": "text", "text": body}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}
got, err := invokeGroup(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeGroup: %v", err)
@@ -464,7 +464,7 @@ func TestTitleCap_GroupPipeline_ValidatedInRangeCap(t *testing.T) {
if err := p.TitleChunkerParam.Validate(); err != nil {
t.Fatalf("cap=128 must pass production Validate: %v", err)
}
- inputs := map[string]any{"output_format": "text", "text": body}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}
got, err := invokeGroup(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeGroup: %v", err)
@@ -481,7 +481,7 @@ func TestTitleCap_HierarchyPipeline_RespectsCap(t *testing.T) {
defer restoreTokenizer()
body := strings.Join(joinSentences(12), "")
p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})
- inputs := map[string]any{"output_format": "text", "text": body}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}
got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
@@ -498,7 +498,7 @@ func TestTitleCap_GroupPipeline_CapZeroNoop(t *testing.T) {
defer restoreTokenizer()
body := strings.Join(joinSentences(12), "")
p := newTitleParam(t, "group", 0, [][]string{{`^# `}})
- inputs := map[string]any{"output_format": "text", "text": body}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}
got, err := invokeGroup(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeGroup: %v", err)
@@ -519,7 +519,7 @@ func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {
}
body := strings.Join(records, "\n")
p := newTitleParam(t, "group", 10, [][]string{{`^# `}})
- inputs := map[string]any{"output_format": "text", "text": body}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": body}
got, err := invokeGroup(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeGroup: %v", err)
@@ -543,6 +543,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {
body := strings.Join(joinSentences(12), "")
p := newTitleParam(t, "hierarchy", 20, [][]string{{`^# `}})
inputs := map[string]any{
+ "name": "t.txt",
"output_format": "chunks",
"chunks": []schema.ChunkDoc{
{Text: body, DocType: "text", Positions: json.RawMessage(`[[1,10,200,50,80]]`)},
@@ -587,7 +588,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain(t *testing.T) {
two := 2
p.Hierarchy = &two
inputText := "# H1\n## H2a\n" + body
- inputs := map[string]any{"output_format": "text", "text": inputText}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}
got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
@@ -633,7 +634,7 @@ func TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain(t *testing.T) {
p.Hierarchy = &two
p.RootChunkAsHeading = true
inputText := "前言。\n# H1\n## H2a\n" + body
- inputs := map[string]any{"output_format": "text", "text": inputText}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}
got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
@@ -671,7 +672,7 @@ func TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain(t *testing.T)
three := 3
p.Hierarchy = &three
inputText := "# A\n前言A。\n## B\n前言B。\n### C\n" + body
- inputs := map[string]any{"output_format": "text", "text": inputText}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}
got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
@@ -712,7 +713,7 @@ func TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles(t *testing.
p.Hierarchy = &two
p.IncludeHeadingContent = true
inputText := "# H1\n短。\n## H2a\n" + body
- inputs := map[string]any{"output_format": "text", "text": inputText}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}
got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
@@ -755,7 +756,7 @@ func TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack(t *testing.T) {
two := 2
p.Hierarchy = &two
inputText := "# H1\n## H2a\n" + body
- inputs := map[string]any{"output_format": "text", "text": inputText}
+ inputs := map[string]any{"name": "t.txt", "output_format": "text", "text": inputText}
got, err := invokeHierarchy(testCtx(), nil, inputs, &p)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index be1ca2a73a..4703f186b1 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -86,7 +86,6 @@ import (
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
- "ragflow/internal/parser/tableutil"
"ragflow/internal/parser/chunk"
)
@@ -190,11 +189,7 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs
}
upstream, err := decodeChunkerFromUpstream(decInputs)
if err != nil {
- return map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{},
- "_ERROR": fmt.Sprintf("Input error: %v", err),
- }, nil
+ return inputErrorOutput(err), nil
}
delimPattern := compileDelimPattern(c.param.Delimiters)
@@ -273,20 +268,8 @@ func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstrea
if err := out.Validate(); err != nil {
return out, err
}
- // A structured table (e.g. the office/xlsx producer) carries no Text of its
- // own. Derive its HTML <table> rendering on Text so every chunker that
- // consumes it (token/group/hierarchy/one/page/qa) stores and displays a real
- // table, and the chunk id (a hash of Text) matches the stored content.
- for i := range out.JSONResult {
- if out.JSONResult[i].TableData != nil && out.JSONResult[i].Text == "" {
- out.JSONResult[i].Text = tableutil.RenderTableHTML(out.JSONResult[i].TableData)
- }
- }
- for i := range out.Chunks {
- if out.Chunks[i].TableData != nil && out.Chunks[i].Text == "" {
- out.Chunks[i].Text = tableutil.RenderTableHTML(out.Chunks[i].TableData)
- }
- }
+ materializeTableText(out.JSONResult)
+ materializeTableText(out.Chunks)
return out, nil
}
@@ -1857,43 +1840,6 @@ func compileChildrenPattern(delims []string) *regexp.Regexp {
return chunk.CompileDelimiterPatternList(delims, true)
}
-// stringFromInputs returns the string value at the first matching key
-// in `keys`, or ("", false) when none is set.
-func stringFromInputs(inputs map[string]any, keys ...string) (string, bool) {
- for _, k := range keys {
- if v, ok := inputs[k].(string); ok {
- return v, true
- }
- }
- return "", false
-}
-
-// chunksFromInputs returns the chunk list from inputs as a uniform
-// []map[string]any, or nil when absent. Both []map[string]any (the
-// JSON-decoded form) and []any (the slice-of-mixed form) are handled.
-//
-// Two upstream keys are accepted, in priority order:
-//
-// - "chunks" — canonical post-chunker shape (chunker → chunker
-// re-entry, test fixtures, downstream stages).
-// - "json" — the parser-structured-output key (Parser
-// component emits under "json"; we accept it
-// so a token-chunker can run directly after
-// a parser without an intermediate reshape).
-func chunksFromInputs(inputs map[string]any) []schema.ChunkDoc {
- for _, key := range []string{"chunks", "json"} {
- v, ok := inputs[key]
- if !ok {
- continue
- }
- chunks, found, err := schema.ChunkDocsFromAny(v)
- if err == nil && found {
- return chunks
- }
- }
- return nil
-}
-
func intValue(v *int) int {
if v == nil {
return 0