RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(chunker): replace QAChunker/PresentationChunker with PairChunker/PageChunker, drop TagChunker (#18523)
Jack84bed4dec587
diff --git a/internal/agent/dsl/normalize.go b/internal/agent/dsl/normalize.goindex f6187bcdce..0579d36a13 100644--- a/internal/agent/dsl/normalize.go+++ b/internal/agent/dsl/normalize.go@@ -588,40 +588,40 @@ func stringsToAny(s []string) []any {// type is still rendered (just possibly in a generic shape). The user// can re-pick a type from the operator palette to refine.var componentNameToNodeTypeMap = map[string]string{- "Begin": "beginNode",- "Retrieval": "ragNode",- "Categorize": "categorizeNode",- "Message": "messageNode",- "Answer": "messageNode",- "RewriteQuestion": "rewriteNode",- "ExeSQL": "toolNode",- "Switch": "switchNode",- "Agent": "agentNode",- "Tool": "toolNode",- "File": "fileNode",- "Parser": "parserNode",- "Tokenizer": "tokenizerNode",- "TokenChunker": "chunkerNode",- "TitleChunker": "chunkerNode",- "OneChunker": "chunkerNode",- "TagChunker": "chunkerNode",- "TableChunker": "chunkerNode",- "PresentationChunker": "chunkerNode",- "Extractor": "contextNode",- "Loop": "loopNode",- "LoopStart": "loopStartNode",- "ExitLoop": "exitLoopNode",- "Iteration": "iterationNode",- "IterationStart": "iterationStartNode",- "Parallel": "parallelNode",- "DataOperations": "dataOperationsNode",- "ListOperations": "listOperationsNode",- "VariableAssigner": "variableAssignerNode",- "VariableAggregator": "variableAggregatorNode",- "Keyword": "keywordNode",- "Note": "noteNode",- "Placeholder": "placeholderNode",- "Code": "toolNode",+ "Begin": "beginNode",+ "Retrieval": "ragNode",+ "Categorize": "categorizeNode",+ "Message": "messageNode",+ "Answer": "messageNode",+ "RewriteQuestion": "rewriteNode",+ "ExeSQL": "toolNode",+ "Switch": "switchNode",+ "Agent": "agentNode",+ "Tool": "toolNode",+ "File": "fileNode",+ "Parser": "parserNode",+ "Tokenizer": "tokenizerNode",+ "TokenChunker": "chunkerNode",+ "TitleChunker": "chunkerNode",+ "OneChunker": "chunkerNode",+ "QAChunker": "chunkerNode",+ "TableChunker": "chunkerNode",+ "PageChunker": "chunkerNode",+ "Extractor": "contextNode",+ "Loop": "loopNode",+ "LoopStart": "loopStartNode",+ "ExitLoop": "exitLoopNode",+ "Iteration": "iterationNode",+ "IterationStart": "iterationStartNode",+ "Parallel": "parallelNode",+ "DataOperations": "dataOperationsNode",+ "ListOperations": "listOperationsNode",+ "VariableAssigner": "variableAssignerNode",+ "VariableAggregator": "variableAggregatorNode",+ "Keyword": "keywordNode",+ "Note": "noteNode",+ "Placeholder": "placeholderNode",+ "Code": "toolNode",}func componentNameToNodeType(name string) string {diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.goindex cfac9b7b46..c22f0de118 100644--- a/internal/handler/components_testpkg/components_handler_test.go+++ b/internal/handler/components_testpkg/components_handler_test.go@@ -147,7 +147,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {wantNames := []string{"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker","titlechunker", "tokenchunker", "tokenizer",}assertNameSet(t, "ingestion", data, wantNames)@@ -173,7 +173,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {wantNames := []string{"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker","titlechunker", "tokenchunker", "tokenizer",}assertNameSet(t, "ingestion,shared", data, wantNames)@@ -274,7 +274,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker","titlechunker", "tokenchunker", "tokenizer",}assertNameSet(t, "INGESTION (case-folded)", data, wantNames)diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 230df7fbf4..17bcecaa76 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -55,12 +55,10 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, erreturn NewQAChunker(params)case ComponentNameOneChunker:return NewOneChunker(params)- case ComponentNameTagChunker:- return NewTagChunker(params)case ComponentNameTableChunker:return NewTableChunker(params)- case ComponentNamePresentationChunker:- return NewPresentationChunker(params)+ case ComponentNamePageChunker:+ return NewPageChunker(params)default:return nil, fmt.Errorf("chunker: unknown component %q", name)}diff --git a/internal/ingestion/component/chunker/presentation.go b/internal/ingestion/component/chunker/page.gosimilarity index 67%rename from internal/ingestion/component/chunker/presentation.gorename to internal/ingestion/component/chunker/page.goindex 78abeff1d4..f9d8d0621a 100644--- a/internal/ingestion/component/chunker/presentation.go+++ b/internal/ingestion/component/chunker/page.go@@ -14,13 +14,13 @@// limitations under the License.//-// PresentationChunker emits one chunk per upstream slide or page.+// PageChunker emits one chunk per upstream slide or page.// It is the faithful Go port of the Python `presentation` chunk method// (rag/app/presentation.py), whose docstring states: "Every page will// be treated as a chunk."//// Unlike TokenChunker (which merges slides into a single chunk) or-// OneChunker (which collapses many slides into one), PresentationChunker+// OneChunker (which collapses many slides into one), PageChunker// keeps each slide as the unit of chunking. The upstream parser produces// one record per slide with text and slide_number; this chunker passes// each through unchanged. Note that the PPTX/PPT path does not emit image@@ -38,43 +38,43 @@ import ("gorm.io/gorm")-const ComponentNamePresentationChunker = "PresentationChunker"+const ComponentNamePageChunker = "PageChunker"-type presentationChunkerParam struct{}+type pageChunkerParam struct{}-func (p *presentationChunkerParam) Update(conf map[string]any) {}+func (p *pageChunkerParam) Update(conf map[string]any) {}-func (presentationChunkerParam) Defaults() presentationChunkerParam {- return presentationChunkerParam{}+func (pageChunkerParam) Defaults() pageChunkerParam {+ return pageChunkerParam{}}-func (presentationChunkerParam) Validate() error { return nil }+func (pageChunkerParam) Validate() error { return nil }-type PresentationChunkerComponent struct {+type PageChunkerComponent struct {name string- param presentationChunkerParam+ param pageChunkerParam}-func NewPresentationChunker(params map[string]any) (runtime.Component, error) {- p := presentationChunkerParam{}.Defaults()+func NewPageChunker(params map[string]any) (runtime.Component, error) {+ p := pageChunkerParam{}.Defaults()(&p).Update(params)if err := p.Validate(); err != nil {return nil, err}- return &PresentationChunkerComponent{- name: ComponentNamePresentationChunker,+ return &PageChunkerComponent{+ name: ComponentNamePageChunker,param: p,}, nil}-func (c *PresentationChunkerComponent) Inputs() map[string]string { return ChunkerInputs }+func (c *PageChunkerComponent) Inputs() map[string]string { return ChunkerInputs }-func (c *PresentationChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }+func (c *PageChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }-func (c *PresentationChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {+func (c *PageChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {return c.invoke(ctx, inputs)}-func (c *PresentationChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {+func (c *PageChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {if inputs == nil {return emptyOutputs(), nil}@@ -108,5 +108,5 @@ func slideItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {}func init() {- MustRegisterChunker(ComponentNamePresentationChunker)+ MustRegisterChunker(ComponentNamePageChunker)}diff --git a/internal/ingestion/component/chunker/presentation_test.go b/internal/ingestion/component/chunker/page_test.gosimilarity index 83%rename from internal/ingestion/component/chunker/presentation_test.gorename to internal/ingestion/component/chunker/page_test.goindex 17a3d518ac..337e1248cf 100644--- a/internal/ingestion/component/chunker/presentation_test.go+++ b/internal/ingestion/component/chunker/page_test.go@@ -21,16 +21,16 @@ import ("testing")-// slideChunksOf drives PresentationChunker.Invoke and returns the emitted maps.+// slideChunksOf drives PageChunker.Invoke and returns the emitted maps.func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any {t.Helper()- comp, err := NewPresentationChunker(nil)+ comp, err := NewPageChunker(nil)if err != nil {- t.Fatalf("NewPresentationChunker: %v", err)+ t.Fatalf("NewPageChunker: %v", err)}out, err := comp.Invoke(context.Background(), nil, inputs)if err != nil {- t.Fatalf("PresentationChunker.Invoke: %v", err)+ t.Fatalf("PageChunker.Invoke: %v", err)}chunks, ok := out["chunks"].([]map[string]any)if !ok {@@ -39,10 +39,10 @@ func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any {return chunks}-// TestPresentationChunker_OneChunkPerSlide ports rag/app/presentation.py's+// TestPageChunker_OneChunkPerSlide ports rag/app/presentation.py's// contract: each slide becomes exactly one chunk, and per-slide metadata// (page_number, image) survives the chunker round-trip.-func TestPresentationChunker_OneChunkPerSlide(t *testing.T) {+func TestPageChunker_OneChunkPerSlide(t *testing.T) {chunks := slideChunksOf(t, map[string]any{"name": "deck.pptx","output_format": "json",@@ -98,8 +98,8 @@ func TestPresentationChunker_OneChunkPerSlide(t *testing.T) {}}-// TestPresentationChunker_Empty yields no chunks when there is no data.-func TestPresentationChunker_Empty(t *testing.T) {+// TestPageChunker_Empty yields no chunks when there is no data.+func TestPageChunker_Empty(t *testing.T) {chunks := slideChunksOf(t, map[string]any{"name": "deck.pptx","output_format": "json",@@ -110,8 +110,8 @@ func TestPresentationChunker_Empty(t *testing.T) {}}-// TestPresentationChunker_NilInput yields no chunks.-func TestPresentationChunker_NilInput(t *testing.T) {+// TestPageChunker_NilInput yields no chunks.+func TestPageChunker_NilInput(t *testing.T) {chunks := slideChunksOf(t, nil)if len(chunks) != 0 {t.Fatalf("expected 0 chunks for nil input, got %d", len(chunks))diff --git a/internal/ingestion/component/chunker/qa_batch2_test.go b/internal/ingestion/component/chunker/qa_batch2_test.goindex 1bfb72bbbd..b6730bbaf4 100644--- a/internal/ingestion/component/chunker/qa_batch2_test.go+++ b/internal/ingestion/component/chunker/qa_batch2_test.go@@ -22,7 +22,7 @@ import ("testing")-// qaInvoke is a small helper that runs the QA chunker on a upstream-style+// qaInvoke is a small helper that runs the QA chunker on an upstream-style// input map and returns the produced chunks as generic maps.func qaInvoke(t *testing.T, inputs map[string]any) []map[string]any {t.Helper()@@ -41,9 +41,8 @@ func qaInvoke(t *testing.T, inputs map[string]any) []map[string]any {return chunks}-// TestQAChunker_DefaultLangIsChinese exercises migration :-// when no language is supplied, Python defaults to Chinese prefixes-// ("问题:"/"回答:"); the legacy Go code defaulted to English.+// TestQAChunker_DefaultLangIsChinese verifies that when no language is+// supplied the chunker uses Chinese prefixes ("问题:"/"回答:"), not English.func TestQAChunker_DefaultLangIsChinese(t *testing.T) {inputs := map[string]any{"name": "test.txt",@@ -63,10 +62,9 @@ func TestQAChunker_DefaultLangIsChinese(t *testing.T) {}}-// TestRmQAPrefixStripsMultipleSeparators exercises migration diff-// Chunker-2.12: the prefix regex must allow one-or-more separator chars-// (Python uses `[\t:: ]+`), so "Q:: answer" is fully stripped. The legacy-// Go pattern only matched a single separator, leaving ": answer".+// TestRmQAPrefixStripsMultipleSeparators pins the prefix regex: it must+// allow one-or-more separator chars (Python uses `[\t:: ]+`), so+// "Q:: answer" is fully stripped rather than leaving ": answer".func TestRmQAPrefixStripsMultipleSeparators(t *testing.T) {if got := rmQAPrefix("Q:: answer"); got != "answer" {t.Errorf("multi-separator prefix not fully stripped: got %q", got)@@ -79,9 +77,8 @@ func TestRmQAPrefixStripsMultipleSeparators(t *testing.T) {}}-// TestQAChunker_SetsTopInt exercises migration (top_int):-// each QA chunk must carry the source row index in `top_int`, matching-// Python beAdoc(..., row_num=i).+// TestQAChunker_SetsTopInt verifies that each QA chunk carries the source+// row index in `top_int`, matching Python beAdoc(..., row_num=i).func TestQAChunker_SetsTopInt(t *testing.T) {inputs := map[string]any{"name": "test.txt",@@ -107,10 +104,9 @@ func TestQAChunker_SetsTopInt(t *testing.T) {}}-// TestQAChunker_CarriesImageAndPositions exercises migration diff-// Chunker-1.8 (image / positions): when the upstream JSON item already-// carries an image id and pdf positions, the QA chunk must preserve them-// (Python beAdocPdf sets d["image"] and add_positions).+// TestQAChunker_CarriesImageAndPositions verifies that when the upstream+// JSON item already carries an image id and pdf positions, the QA chunk+// preserves them (Python beAdocPdf sets d["image"] and add_positions).func TestQAChunker_CarriesImageAndPositions(t *testing.T) {inputs := map[string]any{"name": "test.pdf",diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex e4ecefdf5c..478e218a92 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -213,7 +213,7 @@ func TestQAChunker_PrefixSpaceSeparatorStrips(t *testing.T) {}cww, _ := chunks[0]["content_with_weight"].(string)// Python qa.py:241 uses `[\t:: ]+`, so a space is a valid separator:- // a leading "A"/"Q" followed by a space is stripped. .+ // a leading "A"/"Q" followed by a space is stripped.if cww != "Question: language model is useful\tAnswer: How does it work" {t.Fatalf("space-separator prefix not stripped: %q", cww)}diff --git a/internal/ingestion/component/chunker/register_test.go b/internal/ingestion/component/chunker/register_test.goindex ae63c02649..4ea0fb3ebb 100644--- a/internal/ingestion/component/chunker/register_test.go+++ b/internal/ingestion/component/chunker/register_test.go@@ -22,13 +22,13 @@ import ("ragflow/internal/agent/runtime")-// TestNewChunkersRegistered pins that OneChunker, TagChunker and-// TableChunker are registered under CategoryIngestion (the exact+// TestNewChunkersRegistered pins that OneChunker, QAChunker, TableChunker+// and PageChunker are registered under CategoryIngestion (the exact// invariant the handler/service component-list tests assert). It lives// here so the check runs without the parser package, which has an// unrelated, pre-existing build break on the diverged branch.func TestNewChunkersRegistered(t *testing.T) {- for _, name := range []string{ComponentNameOneChunker, ComponentNameTagChunker, ComponentNameTableChunker, ComponentNamePresentationChunker} {+ for _, name := range []string{ComponentNameOneChunker, ComponentNameQAChunker, ComponentNameTableChunker, ComponentNamePageChunker} {_, cat, _, ok := runtime.DefaultRegistry.Lookup(name)if !ok {t.Fatalf("%s: not registered", name)diff --git a/internal/ingestion/component/chunker/tag.go b/internal/ingestion/component/chunker/tag.godeleted file mode 100644index d1f341b987..0000000000--- a/internal/ingestion/component/chunker/tag.go+++ /dev/null@@ -1,292 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--// TagChunker ports the Python `tag` chunk method (rag/app/tag.py).-//-// The Python chunk() reads an Excel/csv/txt file with two logical-// columns — content and tags (no header) — and emits ONE chunk per-// row, each carrying:-//-// content_with_weight = content (verbatim)-// tag_kwd = [t for t in tags.split(",") if t] (".\" -> "_")-//-// The Go ingestion pipeline feeds the chunker a parsed payload rather-// than the raw file, so TagChunker mirrors the sibling QAChunker and-// extracts (content, tags) pairs from every accepted upstream format:-//-// - Text / Markdown / CSV-style txt → delimiter-based pairing-// (tab wins over comma, matching tag.py's detectDelimiter).-// - HTML (spreadsheet → table) → first two <td> columns.-// - JSON (parsed doc items) → per-item text pairing.-//-// Every pair becomes one chunk with content_with_weight = content and-// tag_kwd = the comma-split, dot-sanitised tag list.-package chunker--import (- "context"- "encoding/csv"- "fmt"- "html"- "strings"-- "ragflow/internal/agent/runtime"- "ragflow/internal/ingestion/component/schema"- "ragflow/internal/tokenizer"-- "gorm.io/gorm"-)--const ComponentNameTagChunker = "TagChunker"--type tagChunkerParam struct{}--func (p *tagChunkerParam) Update(conf map[string]any) {}--func (tagChunkerParam) Defaults() tagChunkerParam { return tagChunkerParam{} }--func (tagChunkerParam) Validate() error { return nil }--type TagChunkerComponent struct {- name string- param tagChunkerParam-}--func NewTagChunker(params map[string]any) (runtime.Component, error) {- p := tagChunkerParam{}.Defaults()- (&p).Update(params)- if err := p.Validate(); err != nil {- return nil, err- }- return &TagChunkerComponent{- name: ComponentNameTagChunker,- param: p,- }, nil-}-func (c *TagChunkerComponent) Inputs() map[string]string { return ChunkerInputs }--func (c *TagChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }--func (c *TagChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {- return c.invoke(ctx, inputs)-}--func (c *TagChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {- if inputs == nil {- return emptyOutputs(), nil- }- upstream, err := decodeChunkerFromUpstream(inputs)- if err != nil {- return map[string]any{- "output_format": "chunks",- "chunks": []map[string]any{},- "_ERROR": fmt.Sprintf("Input error: %v", err),- }, nil- }-- var pairs []tagPair- switch upstream.OutputFormat {- case schema.PayloadFormatHTML:- pairs = extractTagTable(stringPtrVal(upstream.HTMLResult))- case schema.PayloadFormatMarkdown:- pairs = extractTagText(stringPtrVal(upstream.MarkdownResult))- case schema.PayloadFormatText:- pairs = extractTagText(stringPtrVal(upstream.TextResult))- default:- pairs = extractTagJSON(upstream.JSONResult)- }-- chunks := make([]schema.ChunkDoc, 0, len(pairs))- lang, _ := inputs["lang"].(string)- tok := tokenizer.New(lang)- for _, pair := range pairs {- content := strings.TrimSpace(pair.Content)- if content == "" {- continue- }- contentLTKS, _ := tok.Tokenize(content)- contentSMLTKS, _ := tok.FineGrainedTokenize(contentLTKS)- chunk := schema.ChunkDoc{- ContentWithWeight: content,- DocType: "text",- ContentLtks: contentLTKS,- ContentSmLtks: contentSMLTKS,- TagKwd: splitTagKwd(pair.Tags),- TopInt: []int{pair.RowNum},- }- chunks = append(chunks, chunk)- }-- return chunkOutputs(chunks), nil-}--// tagPair is a (content, tags) row extracted from the upstream payload.-// RowNum is the 0-based record START line (the first physical source-// line of the record), used by every extractor consistently and mapped-// to Python's top_int in beAdoc(tag.py:33).-type tagPair struct {- Content string- Tags string- RowNum int-}--// extractTagText ports tag.py:60-89 (txt) and tag.py:91-113 (csv).-func extractTagText(text string) []tagPair {- if text == "" {- return nil- }- lines := strings.Split(text, "\n")- delimiter := detectDelimiter(lines)-- if delimiter == "\t" {- return extractTagTextTab(lines)- }- return extractTagTextCSV(text, lines)-}--// extractTagTextTab handles tab-delimited text where no CSV quoting rules-// apply; one physical line always maps to one record.-func extractTagTextTab(lines []string) []tagPair {- var pairs []tagPair- content := ""- contentStart := -1- for i, line := range lines {- if strings.TrimSpace(line) == "" {- continue- }- if contentStart < 0 {- contentStart = i- }- parts := strings.Split(line, "\t")- if len(parts) != 2 {- content += "\n" + line- continue- }- content += "\n" + parts[0]- pairs = append(pairs, tagPair{Content: content, Tags: parts[1], RowNum: contentStart})- content = ""- contentStart = -1- }- return pairs-}--// extractTagTextCSV uses a full-text csv.Reader so that quoted fields-// spanning multiple physical lines are parsed correctly (see #16881).-func extractTagTextCSV(text string, lines []string) []tagPair {- lineStarts := make([]int, len(lines)+1)- off := 0- for i, l := range lines {- lineStarts[i] = off- off += len(l) + 1- }- lineStarts[len(lines)] = off-- r := csv.NewReader(strings.NewReader(text))- r.LazyQuotes = true- r.FieldsPerRecord = -1-- var pairs []tagPair- content := ""- prevLine := 0-- for {- record, err := r.Read()- if err != nil {- break- }- endOff := int(r.InputOffset())- curLine := prevLine- for curLine < len(lineStarts) && lineStarts[curLine] < endOff {- curLine++- }- startLine := prevLine- raw := strings.Join(lines[prevLine:curLine], "\n")- prevLine = curLine-- if len(record) != 2 {- content += "\n" + raw- continue- }- content += "\n" + record[0]- // RowNum is the 0-based record START line, kept consistent with- // extractTagTextTab (line i) and extractTagTable (<tr> i) so every- // tag-pair source uses the same row-index convention.- pairs = append(pairs, tagPair{Content: content, Tags: record[1], RowNum: startLine})- content = ""- }- return pairs-}--// extractTagTable ports the spreadsheet (xlsx/csv → html table) path.-// Python tag.py reads these via the Excel parser's (q, a) pairs, which-// are exactly the first two columns of each row — so we take the same-// two leading cells per <tr>.-func extractTagTable(htmlStr string) []tagPair {- if htmlStr == "" {- return nil- }- rows := htmlTR.FindAllStringSubmatch(htmlStr, -1)- pairs := make([]tagPair, 0, len(rows))- for i, row := range rows {- cells := htmlTD.FindAllStringSubmatch(row[1], -1)- var texts []string- for _, cell := range cells {- t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], ""))- t = strings.TrimSpace(t)- if t != "" {- texts = append(texts, t)- }- }- if len(texts) >= 2 {- pairs = append(pairs, tagPair{Content: texts[0], Tags: texts[1], RowNum: i})- }- }- return pairs-}--// extractTagJSON ports the JSON/structured upstream path: each item's-// text is run through the same delimiter pairing as the text path.-func extractTagJSON(items []schema.ChunkDoc) []tagPair {- var pairs []tagPair- for _, item := range items {- txt, _ := itemText(item)- if txt == "" {- continue- }- pairs = append(pairs, extractTagText(txt)...)- }- return pairs-}--// splitTagKwd ports tag.py:beAdoc's tag_kwd construction: split on-// comma, drop empties, and replace "." with "_" (the storage layer-// rejects dots in keyword values).-func splitTagKwd(s string) []string {- parts := strings.Split(s, ",")- out := make([]string, 0, len(parts))- for _, p := range parts {- p = strings.TrimSpace(p)- p = strings.ReplaceAll(p, ".", "_")- if p != "" {- out = append(out, p)- }- }- return out-}--func init() {- MustRegisterChunker(ComponentNameTagChunker)-}diff --git a/internal/ingestion/component/chunker/tag_test.go b/internal/ingestion/component/chunker/tag_test.godeleted file mode 100644index 515e0b6979..0000000000--- a/internal/ingestion/component/chunker/tag_test.go+++ /dev/null@@ -1,202 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package chunker--import (- "context"- "testing"-)--// tagChunksOf drives TagChunker.Invoke and returns the emitted chunk maps.-func tagChunksOf(t *testing.T, inputs map[string]any) []map[string]any {- t.Helper()- comp, err := NewTagChunker(nil)- if err != nil {- t.Fatalf("NewTagChunker: %v", err)- }- out, err := comp.Invoke(context.Background(), nil, inputs)- if err != nil {- t.Fatalf("TagChunker.Invoke: %v", err)- }- chunks, ok := out["chunks"].([]map[string]any)- if !ok {- t.Fatalf("chunks not []map[string]any: %T", out["chunks"])- }- return chunks-}--// tagKwdOf reads the tag_kwd field from a chunk map. After the-// ChunkDoc -> JSON -> map round-trip, the string slice arrives as-// []any, so we normalise it here.-func tagKwdOf(t *testing.T, m map[string]any) []string {- t.Helper()- raw, ok := m["tag_kwd"]- if !ok {- t.Fatalf("tag_kwd missing: %v", m)- }- arr, ok := raw.([]any)- if !ok {- t.Fatalf("tag_kwd not []any: %T", raw)- }- out := make([]string, 0, len(arr))- for _, v := range arr {- s, ok := v.(string)- if !ok {- t.Fatalf("tag_kwd entry not string: %T", v)- }- out = append(out, s)- }- return out-}--// TestTagChunker_TextTabDelimited ports tag.py's txt path: one chunk per-// content+tags row, content_with_weight = content, tag_kwd = comma-split-// (dots sanitsed) tags.-func TestTagChunker_TextTabDelimited(t *testing.T) {- chunks := tagChunksOf(t, map[string]any{- "name": "tags.txt",- "output_format": "text",- "text": "what is ragflow\tRAG,LLM\nhow to deploy\tGUIDE,ops.example",- })- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }-- c0 := chunks[0]- if got := c0["content_with_weight"]; got != "what is ragflow" {- t.Errorf("chunk0 content_with_weight = %q", got)- }- tags0 := tagKwdOf(t, c0)- if len(tags0) != 2 || tags0[0] != "RAG" || tags0[1] != "LLM" {- t.Errorf("chunk0 tag_kwd = %v, want [RAG LLM]", tags0)- }-- c1 := chunks[1]- tags1 := tagKwdOf(t, c1)- // "ops.example" -> "ops_example": dots are sanitsed for keyword storage.- if len(tags1) != 2 || tags1[0] != "GUIDE" || tags1[1] != "ops_example" {- t.Errorf("chunk1 tag_kwd = %v, want [GUIDE ops_example]", tags1)- }-}--// TestTagChunker_TextCommaDelimited verifies comma wins when the file has-// no tab-structured rows (mirrors tag.py:detectDelimiter).-func TestTagChunker_TextCommaDelimited(t *testing.T) {- chunks := tagChunksOf(t, map[string]any{- "name": "tags.csv",- "output_format": "text",- "text": "what is ragflow,RAG\nhow to deploy,GUIDE",- })- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- if got := chunks[0]["content_with_weight"]; got != "what is ragflow" {- t.Errorf("chunk0 content_with_weight = %q", got)- }- tags0 := tagKwdOf(t, chunks[0])- if len(tags0) != 1 || tags0[0] != "RAG" {- t.Errorf("chunk0 tag_kwd = %v, want [RAG]", tags0)- }-}--// TestTagChunker_HTMLTable ports the spreadsheet (xlsx/csv -> html table)-// path: first two <td> columns become content + tags.-func TestTagChunker_HTMLTable(t *testing.T) {- html := "<table><tr><td>q1</td><td>a1,b1</td></tr><tr><td>q2</td><td>a2</td></tr></table>"- chunks := tagChunksOf(t, map[string]any{- "name": "tags.xlsx",- "output_format": "html",- "html": html,- })- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- if got := chunks[0]["content_with_weight"]; got != "q1" {- t.Errorf("chunk0 content_with_weight = %q", got)- }- tags0 := tagKwdOf(t, chunks[0])- if len(tags0) != 2 || tags0[0] != "a1" || tags0[1] != "b1" {- t.Errorf("chunk0 tag_kwd = %v, want [a1 b1]", tags0)- }-}--// TestTagChunk_TopInt verifies each tag-pair row carries the 0-based-// source row index as top_int, matching Python tag.py:33 beAdoc(row_num=i).-func TestTagChunk_TopInt(t *testing.T) {- chunks := tagChunksOf(t, map[string]any{- "name": "tags.txt",- "output_format": "text",- "text": "tag1\tcontent1\ntag2\tcontent2",- })- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- if ti, ok := chunks[0]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(0) {- t.Errorf("chunk 0: top_int = %v, want [0]", chunks[0]["top_int"])- }- if ti, ok := chunks[1]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(1) {- t.Errorf("chunk 1: top_int = %v, want [1]", chunks[1]["top_int"])- }-}--// TestTagChunk_CSVTopIntStartLine locks the RowNum convention from-// review finding #4: a multi-line quoted CSV record must report the-// record's START line, not the line after it ends. The first record-// spans physical lines 0-1, so its top_int is [0]; the second record-// is on line 2, so its top_int is [2].-func TestTagChunk_CSVTopIntStartLine(t *testing.T) {- chunks := tagChunksOf(t, map[string]any{- "name": "tags.csv",- "output_format": "text",- "text": "\"multi\nline content\",TAG\nsecond,OTHER",- })- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- if ti, ok := chunks[0]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(0) {- t.Errorf("chunk 0: top_int = %v, want [0]", chunks[0]["top_int"])- }- if ti, ok := chunks[1]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(2) {- t.Errorf("chunk 1: top_int = %v, want [2]", chunks[1]["top_int"])- }-}--// TestTagChunk_TSVMultiLineRowNumStart pins the RowNum fix for TSV-// multi-line records: when a record spans multiple physical lines-// (because the first line does not contain a tab delimiter), the-// resulting chunk's top_int must report the first non-empty CONTENT-// start line, not the line where the tab-delimiter eventually-// terminates the record (PR #17419 review #1).-func TestTagChunk_TSVMultiLineRowNumStart(t *testing.T) {- chunks := tagChunksOf(t, map[string]any{- "name": "tags.txt",- "output_format": "text",- // Line 0 is empty, line 1 has no tab (accumulated as prefix- // of the first record), line 2 is valid TSV.- // The prefix starts at line 1 → RowNum must be 1, not 2.- "text": "\nprefix\nrecord1\tTAG1\nmulti\nline-prefix\nrecord2\tTAG2",- })- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- if ti, ok := chunks[0]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(1) {- t.Errorf("chunk 0: top_int = %v, want [1] (content start)", chunks[0]["top_int"])- }- if ti, ok := chunks[1]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(3) {- t.Errorf("chunk 1: top_int = %v, want [3] (content start after blank line)", chunks[1]["top_int"])- }-}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.jsonindex c10eab6733..7f6290847c 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json@@ -26,7 +26,7 @@},"Parser:HipSignsRhyme": {"downstream": [- "PresentationChunker:HappyHillsGlow"+ "PageChunker:HappyHillsGlow"],"obj": {"component_name": "Parser",@@ -96,12 +96,12 @@"Extractor:AutoExtractDefault"]},- "PresentationChunker:HappyHillsGlow": {+ "PageChunker:HappyHillsGlow": {"downstream": ["Extractor:AutoExtractDefault"],"obj": {- "component_name": "PresentationChunker",+ "component_name": "PageChunker","params": {"outputs": {"chunks": {@@ -117,7 +117,7 @@},"Extractor:AutoExtractDefault": {"upstream": [- "PresentationChunker:HappyHillsGlow"+ "PageChunker:HappyHillsGlow"],"downstream": ["Tokenizer:TallTreesDance"@@ -161,15 +161,15 @@"targetHandle": "end"},{- "id": "xy-edge__Parser:HipSignsRhymestart-PresentationChunker:HappyHillsGlowend",+ "id": "xy-edge__Parser:HipSignsRhymestart-PageChunker:HappyHillsGlowend","source": "Parser:HipSignsRhyme","sourceHandle": "start",- "target": "PresentationChunker:HappyHillsGlow",+ "target": "PageChunker:HappyHillsGlow","targetHandle": "end"},{- "id": "xy-edge__PresentationChunker:HappyHillsGlowstart-Extractor:AutoExtractDefaultend",- "source": "PresentationChunker:HappyHillsGlow",+ "id": "xy-edge__PageChunker:HappyHillsGlowstart-Extractor:AutoExtractDefaultend",+ "source": "PageChunker:HappyHillsGlow","sourceHandle": "start","target": "Extractor:AutoExtractDefault","targetHandle": "end"@@ -271,10 +271,10 @@}}},- "label": "PresentationChunker",+ "label": "PageChunker","name": "Presentation Chunker_0"},- "id": "PresentationChunker:HappyHillsGlow",+ "id": "PageChunker:HappyHillsGlow","measured": {"height": 74,"width": 200diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex 01daffa143..2900ed15c8 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -722,9 +722,6 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {if templateUsesComponent(t, templateBytes, "QAChunker") {t.Skip("template uses QAChunker which requires Q&A-structured content; covered separately")}- if templateUsesComponent(t, templateBytes, "TagChunker") {- t.Skip("template uses TagChunker which requires tag-structured content and parser setups not available for generic .md input; covered separately")- }if templateUsesComponent(t, templateBytes, "Compiler") {t.Skip("template uses Compiler which requires LLM/embedder/ES wiring not available in the headless smoke run; covered by the knowledge_compiler component E2E tests")}diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex 0330babc95..b9dbb572a6 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -47,7 +47,7 @@ var builtinComponentParamsGolden = map[string]string{"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PresentationChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}","table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}","knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"compilation_template_group_id\": \"\", \"llm_id\": \"\", \"plan\": false}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.goindex 41b4c284d3..b45653b6e5 100644--- a/internal/service/components_service_test.go+++ b/internal/service/components_service_test.go@@ -63,7 +63,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {}wantNames := []string{"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",}assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)}@@ -86,7 +86,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {}wantNames := []string{"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",}assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)}@@ -105,7 +105,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {}wantNames := []string{"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",}if len(got) != len(wantNames) {t.Errorf("duplicate category produced %d rows, want %d", len(got), len(wantNames))