Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(chunker): replace QAChunker/PresentationChunker with PairChunker/PageChunker, drop TagChunker (#18523)

Jack84bed4dec587
diff --git a/internal/agent/dsl/normalize.go b/internal/agent/dsl/normalize.go
index f6187bcdce..0579d36a13 100644
--- a/internal/agent/dsl/normalize.go
+++ b/internal/agent/dsl/normalize.go
@@ -588,40 +588,40 @@ func stringsToAny(s []string) []any {
// type is still rendered (just possibly in a generic shape). The user
// can re-pick a type from the operator palette to refine.
var componentNameToNodeTypeMap = map[string]string{
- "Begin": "beginNode",
- "Retrieval": "ragNode",
- "Categorize": "categorizeNode",
- "Message": "messageNode",
- "Answer": "messageNode",
- "RewriteQuestion": "rewriteNode",
- "ExeSQL": "toolNode",
- "Switch": "switchNode",
- "Agent": "agentNode",
- "Tool": "toolNode",
- "File": "fileNode",
- "Parser": "parserNode",
- "Tokenizer": "tokenizerNode",
- "TokenChunker": "chunkerNode",
- "TitleChunker": "chunkerNode",
- "OneChunker": "chunkerNode",
- "TagChunker": "chunkerNode",
- "TableChunker": "chunkerNode",
- "PresentationChunker": "chunkerNode",
- "Extractor": "contextNode",
- "Loop": "loopNode",
- "LoopStart": "loopStartNode",
- "ExitLoop": "exitLoopNode",
- "Iteration": "iterationNode",
- "IterationStart": "iterationStartNode",
- "Parallel": "parallelNode",
- "DataOperations": "dataOperationsNode",
- "ListOperations": "listOperationsNode",
- "VariableAssigner": "variableAssignerNode",
- "VariableAggregator": "variableAggregatorNode",
- "Keyword": "keywordNode",
- "Note": "noteNode",
- "Placeholder": "placeholderNode",
- "Code": "toolNode",
+ "Begin": "beginNode",
+ "Retrieval": "ragNode",
+ "Categorize": "categorizeNode",
+ "Message": "messageNode",
+ "Answer": "messageNode",
+ "RewriteQuestion": "rewriteNode",
+ "ExeSQL": "toolNode",
+ "Switch": "switchNode",
+ "Agent": "agentNode",
+ "Tool": "toolNode",
+ "File": "fileNode",
+ "Parser": "parserNode",
+ "Tokenizer": "tokenizerNode",
+ "TokenChunker": "chunkerNode",
+ "TitleChunker": "chunkerNode",
+ "OneChunker": "chunkerNode",
+ "QAChunker": "chunkerNode",
+ "TableChunker": "chunkerNode",
+ "PageChunker": "chunkerNode",
+ "Extractor": "contextNode",
+ "Loop": "loopNode",
+ "LoopStart": "loopStartNode",
+ "ExitLoop": "exitLoopNode",
+ "Iteration": "iterationNode",
+ "IterationStart": "iterationStartNode",
+ "Parallel": "parallelNode",
+ "DataOperations": "dataOperationsNode",
+ "ListOperations": "listOperationsNode",
+ "VariableAssigner": "variableAssignerNode",
+ "VariableAggregator": "variableAggregatorNode",
+ "Keyword": "keywordNode",
+ "Note": "noteNode",
+ "Placeholder": "placeholderNode",
+ "Code": "toolNode",
}
func componentNameToNodeType(name string) string {
diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.go
index cfac9b7b46..c22f0de118 100644
--- a/internal/handler/components_testpkg/components_handler_test.go
+++ b/internal/handler/components_testpkg/components_handler_test.go
@@ -147,7 +147,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {
wantNames := []string{
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker",
"titlechunker", "tokenchunker", "tokenizer",
}
assertNameSet(t, "ingestion", data, wantNames)
@@ -173,7 +173,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {
wantNames := []string{
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker",
"titlechunker", "tokenchunker", "tokenizer",
}
assertNameSet(t, "ingestion,shared", data, wantNames)
@@ -274,7 +274,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
"compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker",
"titlechunker", "tokenchunker", "tokenizer",
}
assertNameSet(t, "INGESTION (case-folded)", data, wantNames)
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 230df7fbf4..17bcecaa76 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -55,12 +55,10 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, er
return NewQAChunker(params)
case ComponentNameOneChunker:
return NewOneChunker(params)
- case ComponentNameTagChunker:
- return NewTagChunker(params)
case ComponentNameTableChunker:
return NewTableChunker(params)
- case ComponentNamePresentationChunker:
- return NewPresentationChunker(params)
+ case ComponentNamePageChunker:
+ return NewPageChunker(params)
default:
return nil, fmt.Errorf("chunker: unknown component %q", name)
}
diff --git a/internal/ingestion/component/chunker/presentation.go b/internal/ingestion/component/chunker/page.go
similarity index 67%
rename from internal/ingestion/component/chunker/presentation.go
rename to internal/ingestion/component/chunker/page.go
index 78abeff1d4..f9d8d0621a 100644
--- a/internal/ingestion/component/chunker/presentation.go
+++ b/internal/ingestion/component/chunker/page.go
@@ -14,13 +14,13 @@
// limitations under the License.
//
-// PresentationChunker emits one chunk per upstream slide or page.
+// PageChunker emits one chunk per upstream slide or page.
// It is the faithful Go port of the Python `presentation` chunk method
// (rag/app/presentation.py), whose docstring states: "Every page will
// be treated as a chunk."
//
// Unlike TokenChunker (which merges slides into a single chunk) or
-// OneChunker (which collapses many slides into one), PresentationChunker
+// OneChunker (which collapses many slides into one), PageChunker
// keeps each slide as the unit of chunking. The upstream parser produces
// one record per slide with text and slide_number; this chunker passes
// each through unchanged. Note that the PPTX/PPT path does not emit image
@@ -38,43 +38,43 @@ import (
"gorm.io/gorm"
)
-const ComponentNamePresentationChunker = "PresentationChunker"
+const ComponentNamePageChunker = "PageChunker"
-type presentationChunkerParam struct{}
+type pageChunkerParam struct{}
-func (p *presentationChunkerParam) Update(conf map[string]any) {}
+func (p *pageChunkerParam) Update(conf map[string]any) {}
-func (presentationChunkerParam) Defaults() presentationChunkerParam {
- return presentationChunkerParam{}
+func (pageChunkerParam) Defaults() pageChunkerParam {
+ return pageChunkerParam{}
}
-func (presentationChunkerParam) Validate() error { return nil }
+func (pageChunkerParam) Validate() error { return nil }
-type PresentationChunkerComponent struct {
+type PageChunkerComponent struct {
name string
- param presentationChunkerParam
+ param pageChunkerParam
}
-func NewPresentationChunker(params map[string]any) (runtime.Component, error) {
- p := presentationChunkerParam{}.Defaults()
+func NewPageChunker(params map[string]any) (runtime.Component, error) {
+ p := pageChunkerParam{}.Defaults()
(&p).Update(params)
if err := p.Validate(); err != nil {
return nil, err
}
- return &PresentationChunkerComponent{
- name: ComponentNamePresentationChunker,
+ return &PageChunkerComponent{
+ name: ComponentNamePageChunker,
param: p,
}, nil
}
-func (c *PresentationChunkerComponent) Inputs() map[string]string { return ChunkerInputs }
+func (c *PageChunkerComponent) Inputs() map[string]string { return ChunkerInputs }
-func (c *PresentationChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }
+func (c *PageChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }
-func (c *PresentationChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+func (c *PageChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
return c.invoke(ctx, inputs)
}
-func (c *PresentationChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
+func (c *PageChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
if inputs == nil {
return emptyOutputs(), nil
}
@@ -108,5 +108,5 @@ func slideItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
}
func init() {
- MustRegisterChunker(ComponentNamePresentationChunker)
+ MustRegisterChunker(ComponentNamePageChunker)
}
diff --git a/internal/ingestion/component/chunker/presentation_test.go b/internal/ingestion/component/chunker/page_test.go
similarity index 83%
rename from internal/ingestion/component/chunker/presentation_test.go
rename to internal/ingestion/component/chunker/page_test.go
index 17a3d518ac..337e1248cf 100644
--- a/internal/ingestion/component/chunker/presentation_test.go
+++ b/internal/ingestion/component/chunker/page_test.go
@@ -21,16 +21,16 @@ import (
"testing"
)
-// slideChunksOf drives PresentationChunker.Invoke and returns the emitted maps.
+// slideChunksOf drives PageChunker.Invoke and returns the emitted maps.
func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
t.Helper()
- comp, err := NewPresentationChunker(nil)
+ comp, err := NewPageChunker(nil)
if err != nil {
- t.Fatalf("NewPresentationChunker: %v", err)
+ t.Fatalf("NewPageChunker: %v", err)
}
out, err := comp.Invoke(context.Background(), nil, inputs)
if err != nil {
- t.Fatalf("PresentationChunker.Invoke: %v", err)
+ t.Fatalf("PageChunker.Invoke: %v", err)
}
chunks, ok := out["chunks"].([]map[string]any)
if !ok {
@@ -39,10 +39,10 @@ func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
return chunks
}
-// TestPresentationChunker_OneChunkPerSlide ports rag/app/presentation.py's
+// TestPageChunker_OneChunkPerSlide ports rag/app/presentation.py's
// contract: each slide becomes exactly one chunk, and per-slide metadata
// (page_number, image) survives the chunker round-trip.
-func TestPresentationChunker_OneChunkPerSlide(t *testing.T) {
+func TestPageChunker_OneChunkPerSlide(t *testing.T) {
chunks := slideChunksOf(t, map[string]any{
"name": "deck.pptx",
"output_format": "json",
@@ -98,8 +98,8 @@ func TestPresentationChunker_OneChunkPerSlide(t *testing.T) {
}
}
-// TestPresentationChunker_Empty yields no chunks when there is no data.
-func TestPresentationChunker_Empty(t *testing.T) {
+// TestPageChunker_Empty yields no chunks when there is no data.
+func TestPageChunker_Empty(t *testing.T) {
chunks := slideChunksOf(t, map[string]any{
"name": "deck.pptx",
"output_format": "json",
@@ -110,8 +110,8 @@ func TestPresentationChunker_Empty(t *testing.T) {
}
}
-// TestPresentationChunker_NilInput yields no chunks.
-func TestPresentationChunker_NilInput(t *testing.T) {
+// TestPageChunker_NilInput yields no chunks.
+func TestPageChunker_NilInput(t *testing.T) {
chunks := slideChunksOf(t, nil)
if len(chunks) != 0 {
t.Fatalf("expected 0 chunks for nil input, got %d", len(chunks))
diff --git a/internal/ingestion/component/chunker/qa_batch2_test.go b/internal/ingestion/component/chunker/qa_batch2_test.go
index 1bfb72bbbd..b6730bbaf4 100644
--- a/internal/ingestion/component/chunker/qa_batch2_test.go
+++ b/internal/ingestion/component/chunker/qa_batch2_test.go
@@ -22,7 +22,7 @@ import (
"testing"
)
-// qaInvoke is a small helper that runs the QA chunker on a upstream-style
+// qaInvoke is a small helper that runs the QA chunker on an upstream-style
// input map and returns the produced chunks as generic maps.
func qaInvoke(t *testing.T, inputs map[string]any) []map[string]any {
t.Helper()
@@ -41,9 +41,8 @@ func qaInvoke(t *testing.T, inputs map[string]any) []map[string]any {
return chunks
}
-// TestQAChunker_DefaultLangIsChinese exercises migration :
-// when no language is supplied, Python defaults to Chinese prefixes
-// ("问题:"/"回答:"); the legacy Go code defaulted to English.
+// TestQAChunker_DefaultLangIsChinese verifies that when no language is
+// supplied the chunker uses Chinese prefixes ("问题:"/"回答:"), not English.
func TestQAChunker_DefaultLangIsChinese(t *testing.T) {
inputs := map[string]any{
"name": "test.txt",
@@ -63,10 +62,9 @@ func TestQAChunker_DefaultLangIsChinese(t *testing.T) {
}
}
-// TestRmQAPrefixStripsMultipleSeparators exercises migration diff
-// Chunker-2.12: the prefix regex must allow one-or-more separator chars
-// (Python uses `[\t:: ]+`), so "Q:: answer" is fully stripped. The legacy
-// Go pattern only matched a single separator, leaving ": answer".
+// TestRmQAPrefixStripsMultipleSeparators pins the prefix regex: it must
+// allow one-or-more separator chars (Python uses `[\t:: ]+`), so
+// "Q:: answer" is fully stripped rather than leaving ": answer".
func TestRmQAPrefixStripsMultipleSeparators(t *testing.T) {
if got := rmQAPrefix("Q:: answer"); got != "answer" {
t.Errorf("multi-separator prefix not fully stripped: got %q", got)
@@ -79,9 +77,8 @@ func TestRmQAPrefixStripsMultipleSeparators(t *testing.T) {
}
}
-// TestQAChunker_SetsTopInt exercises migration (top_int):
-// each QA chunk must carry the source row index in `top_int`, matching
-// Python beAdoc(..., row_num=i).
+// TestQAChunker_SetsTopInt verifies that each QA chunk carries the source
+// row index in `top_int`, matching Python beAdoc(..., row_num=i).
func TestQAChunker_SetsTopInt(t *testing.T) {
inputs := map[string]any{
"name": "test.txt",
@@ -107,10 +104,9 @@ func TestQAChunker_SetsTopInt(t *testing.T) {
}
}
-// TestQAChunker_CarriesImageAndPositions exercises migration diff
-// Chunker-1.8 (image / positions): when the upstream JSON item already
-// carries an image id and pdf positions, the QA chunk must preserve them
-// (Python beAdocPdf sets d["image"] and add_positions).
+// TestQAChunker_CarriesImageAndPositions verifies that when the upstream
+// JSON item already carries an image id and pdf positions, the QA chunk
+// preserves them (Python beAdocPdf sets d["image"] and add_positions).
func TestQAChunker_CarriesImageAndPositions(t *testing.T) {
inputs := map[string]any{
"name": "test.pdf",
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index e4ecefdf5c..478e218a92 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -213,7 +213,7 @@ func TestQAChunker_PrefixSpaceSeparatorStrips(t *testing.T) {
}
cww, _ := chunks[0]["content_with_weight"].(string)
// Python qa.py:241 uses `[\t:: ]+`, so a space is a valid separator:
- // a leading "A"/"Q" followed by a space is stripped. .
+ // a leading "A"/"Q" followed by a space is stripped.
if cww != "Question: language model is useful\tAnswer: How does it work" {
t.Fatalf("space-separator prefix not stripped: %q", cww)
}
diff --git a/internal/ingestion/component/chunker/register_test.go b/internal/ingestion/component/chunker/register_test.go
index ae63c02649..4ea0fb3ebb 100644
--- a/internal/ingestion/component/chunker/register_test.go
+++ b/internal/ingestion/component/chunker/register_test.go
@@ -22,13 +22,13 @@ import (
"ragflow/internal/agent/runtime"
)
-// TestNewChunkersRegistered pins that OneChunker, TagChunker and
-// TableChunker are registered under CategoryIngestion (the exact
+// TestNewChunkersRegistered pins that OneChunker, QAChunker, TableChunker
+// and PageChunker are registered under CategoryIngestion (the exact
// invariant the handler/service component-list tests assert). It lives
// here so the check runs without the parser package, which has an
// unrelated, pre-existing build break on the diverged branch.
func TestNewChunkersRegistered(t *testing.T) {
- for _, name := range []string{ComponentNameOneChunker, ComponentNameTagChunker, ComponentNameTableChunker, ComponentNamePresentationChunker} {
+ for _, name := range []string{ComponentNameOneChunker, ComponentNameQAChunker, ComponentNameTableChunker, ComponentNamePageChunker} {
_, cat, _, ok := runtime.DefaultRegistry.Lookup(name)
if !ok {
t.Fatalf("%s: not registered", name)
diff --git a/internal/ingestion/component/chunker/tag.go b/internal/ingestion/component/chunker/tag.go
deleted file mode 100644
index d1f341b987..0000000000
--- a/internal/ingestion/component/chunker/tag.go
+++ /dev/null
@@ -1,292 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// TagChunker ports the Python `tag` chunk method (rag/app/tag.py).
-//
-// The Python chunk() reads an Excel/csv/txt file with two logical
-// columns — content and tags (no header) — and emits ONE chunk per
-// row, each carrying:
-//
-// content_with_weight = content (verbatim)
-// tag_kwd = [t for t in tags.split(",") if t] (".\" -> "_")
-//
-// The Go ingestion pipeline feeds the chunker a parsed payload rather
-// than the raw file, so TagChunker mirrors the sibling QAChunker and
-// extracts (content, tags) pairs from every accepted upstream format:
-//
-// - Text / Markdown / CSV-style txt → delimiter-based pairing
-// (tab wins over comma, matching tag.py's detectDelimiter).
-// - HTML (spreadsheet → table) → first two <td> columns.
-// - JSON (parsed doc items) → per-item text pairing.
-//
-// Every pair becomes one chunk with content_with_weight = content and
-// tag_kwd = the comma-split, dot-sanitised tag list.
-package chunker
-
-import (
- "context"
- "encoding/csv"
- "fmt"
- "html"
- "strings"
-
- "ragflow/internal/agent/runtime"
- "ragflow/internal/ingestion/component/schema"
- "ragflow/internal/tokenizer"
-
- "gorm.io/gorm"
-)
-
-const ComponentNameTagChunker = "TagChunker"
-
-type tagChunkerParam struct{}
-
-func (p *tagChunkerParam) Update(conf map[string]any) {}
-
-func (tagChunkerParam) Defaults() tagChunkerParam { return tagChunkerParam{} }
-
-func (tagChunkerParam) Validate() error { return nil }
-
-type TagChunkerComponent struct {
- name string
- param tagChunkerParam
-}
-
-func NewTagChunker(params map[string]any) (runtime.Component, error) {
- p := tagChunkerParam{}.Defaults()
- (&p).Update(params)
- if err := p.Validate(); err != nil {
- return nil, err
- }
- return &TagChunkerComponent{
- name: ComponentNameTagChunker,
- param: p,
- }, nil
-}
-func (c *TagChunkerComponent) Inputs() map[string]string { return ChunkerInputs }
-
-func (c *TagChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }
-
-func (c *TagChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
- return c.invoke(ctx, inputs)
-}
-
-func (c *TagChunkerComponent) invoke(_ context.Context, inputs map[string]any) (map[string]any, error) {
- if inputs == nil {
- return emptyOutputs(), nil
- }
- upstream, err := decodeChunkerFromUpstream(inputs)
- if err != nil {
- return map[string]any{
- "output_format": "chunks",
- "chunks": []map[string]any{},
- "_ERROR": fmt.Sprintf("Input error: %v", err),
- }, nil
- }
-
- var pairs []tagPair
- switch upstream.OutputFormat {
- case schema.PayloadFormatHTML:
- pairs = extractTagTable(stringPtrVal(upstream.HTMLResult))
- case schema.PayloadFormatMarkdown:
- pairs = extractTagText(stringPtrVal(upstream.MarkdownResult))
- case schema.PayloadFormatText:
- pairs = extractTagText(stringPtrVal(upstream.TextResult))
- default:
- pairs = extractTagJSON(upstream.JSONResult)
- }
-
- chunks := make([]schema.ChunkDoc, 0, len(pairs))
- lang, _ := inputs["lang"].(string)
- tok := tokenizer.New(lang)
- for _, pair := range pairs {
- content := strings.TrimSpace(pair.Content)
- if content == "" {
- continue
- }
- contentLTKS, _ := tok.Tokenize(content)
- contentSMLTKS, _ := tok.FineGrainedTokenize(contentLTKS)
- chunk := schema.ChunkDoc{
- ContentWithWeight: content,
- DocType: "text",
- ContentLtks: contentLTKS,
- ContentSmLtks: contentSMLTKS,
- TagKwd: splitTagKwd(pair.Tags),
- TopInt: []int{pair.RowNum},
- }
- chunks = append(chunks, chunk)
- }
-
- return chunkOutputs(chunks), nil
-}
-
-// tagPair is a (content, tags) row extracted from the upstream payload.
-// RowNum is the 0-based record START line (the first physical source
-// line of the record), used by every extractor consistently and mapped
-// to Python's top_int in beAdoc(tag.py:33).
-type tagPair struct {
- Content string
- Tags string
- RowNum int
-}
-
-// extractTagText ports tag.py:60-89 (txt) and tag.py:91-113 (csv).
-func extractTagText(text string) []tagPair {
- if text == "" {
- return nil
- }
- lines := strings.Split(text, "\n")
- delimiter := detectDelimiter(lines)
-
- if delimiter == "\t" {
- return extractTagTextTab(lines)
- }
- return extractTagTextCSV(text, lines)
-}
-
-// extractTagTextTab handles tab-delimited text where no CSV quoting rules
-// apply; one physical line always maps to one record.
-func extractTagTextTab(lines []string) []tagPair {
- var pairs []tagPair
- content := ""
- contentStart := -1
- for i, line := range lines {
- if strings.TrimSpace(line) == "" {
- continue
- }
- if contentStart < 0 {
- contentStart = i
- }
- parts := strings.Split(line, "\t")
- if len(parts) != 2 {
- content += "\n" + line
- continue
- }
- content += "\n" + parts[0]
- pairs = append(pairs, tagPair{Content: content, Tags: parts[1], RowNum: contentStart})
- content = ""
- contentStart = -1
- }
- return pairs
-}
-
-// extractTagTextCSV uses a full-text csv.Reader so that quoted fields
-// spanning multiple physical lines are parsed correctly (see #16881).
-func extractTagTextCSV(text string, lines []string) []tagPair {
- lineStarts := make([]int, len(lines)+1)
- off := 0
- for i, l := range lines {
- lineStarts[i] = off
- off += len(l) + 1
- }
- lineStarts[len(lines)] = off
-
- r := csv.NewReader(strings.NewReader(text))
- r.LazyQuotes = true
- r.FieldsPerRecord = -1
-
- var pairs []tagPair
- content := ""
- prevLine := 0
-
- for {
- record, err := r.Read()
- if err != nil {
- break
- }
- endOff := int(r.InputOffset())
- curLine := prevLine
- for curLine < len(lineStarts) && lineStarts[curLine] < endOff {
- curLine++
- }
- startLine := prevLine
- raw := strings.Join(lines[prevLine:curLine], "\n")
- prevLine = curLine
-
- if len(record) != 2 {
- content += "\n" + raw
- continue
- }
- content += "\n" + record[0]
- // RowNum is the 0-based record START line, kept consistent with
- // extractTagTextTab (line i) and extractTagTable (<tr> i) so every
- // tag-pair source uses the same row-index convention.
- pairs = append(pairs, tagPair{Content: content, Tags: record[1], RowNum: startLine})
- content = ""
- }
- return pairs
-}
-
-// extractTagTable ports the spreadsheet (xlsx/csv → html table) path.
-// Python tag.py reads these via the Excel parser's (q, a) pairs, which
-// are exactly the first two columns of each row — so we take the same
-// two leading cells per <tr>.
-func extractTagTable(htmlStr string) []tagPair {
- if htmlStr == "" {
- return nil
- }
- rows := htmlTR.FindAllStringSubmatch(htmlStr, -1)
- pairs := make([]tagPair, 0, len(rows))
- for i, row := range rows {
- cells := htmlTD.FindAllStringSubmatch(row[1], -1)
- var texts []string
- for _, cell := range cells {
- t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], ""))
- t = strings.TrimSpace(t)
- if t != "" {
- texts = append(texts, t)
- }
- }
- if len(texts) >= 2 {
- pairs = append(pairs, tagPair{Content: texts[0], Tags: texts[1], RowNum: i})
- }
- }
- return pairs
-}
-
-// extractTagJSON ports the JSON/structured upstream path: each item's
-// text is run through the same delimiter pairing as the text path.
-func extractTagJSON(items []schema.ChunkDoc) []tagPair {
- var pairs []tagPair
- for _, item := range items {
- txt, _ := itemText(item)
- if txt == "" {
- continue
- }
- pairs = append(pairs, extractTagText(txt)...)
- }
- return pairs
-}
-
-// splitTagKwd ports tag.py:beAdoc's tag_kwd construction: split on
-// comma, drop empties, and replace "." with "_" (the storage layer
-// rejects dots in keyword values).
-func splitTagKwd(s string) []string {
- parts := strings.Split(s, ",")
- out := make([]string, 0, len(parts))
- for _, p := range parts {
- p = strings.TrimSpace(p)
- p = strings.ReplaceAll(p, ".", "_")
- if p != "" {
- out = append(out, p)
- }
- }
- return out
-}
-
-func init() {
- MustRegisterChunker(ComponentNameTagChunker)
-}
diff --git a/internal/ingestion/component/chunker/tag_test.go b/internal/ingestion/component/chunker/tag_test.go
deleted file mode 100644
index 515e0b6979..0000000000
--- a/internal/ingestion/component/chunker/tag_test.go
+++ /dev/null
@@ -1,202 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package chunker
-
-import (
- "context"
- "testing"
-)
-
-// tagChunksOf drives TagChunker.Invoke and returns the emitted chunk maps.
-func tagChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
- t.Helper()
- comp, err := NewTagChunker(nil)
- if err != nil {
- t.Fatalf("NewTagChunker: %v", err)
- }
- out, err := comp.Invoke(context.Background(), nil, inputs)
- if err != nil {
- t.Fatalf("TagChunker.Invoke: %v", err)
- }
- chunks, ok := out["chunks"].([]map[string]any)
- if !ok {
- t.Fatalf("chunks not []map[string]any: %T", out["chunks"])
- }
- return chunks
-}
-
-// tagKwdOf reads the tag_kwd field from a chunk map. After the
-// ChunkDoc -> JSON -> map round-trip, the string slice arrives as
-// []any, so we normalise it here.
-func tagKwdOf(t *testing.T, m map[string]any) []string {
- t.Helper()
- raw, ok := m["tag_kwd"]
- if !ok {
- t.Fatalf("tag_kwd missing: %v", m)
- }
- arr, ok := raw.([]any)
- if !ok {
- t.Fatalf("tag_kwd not []any: %T", raw)
- }
- out := make([]string, 0, len(arr))
- for _, v := range arr {
- s, ok := v.(string)
- if !ok {
- t.Fatalf("tag_kwd entry not string: %T", v)
- }
- out = append(out, s)
- }
- return out
-}
-
-// TestTagChunker_TextTabDelimited ports tag.py's txt path: one chunk per
-// content+tags row, content_with_weight = content, tag_kwd = comma-split
-// (dots sanitsed) tags.
-func TestTagChunker_TextTabDelimited(t *testing.T) {
- chunks := tagChunksOf(t, map[string]any{
- "name": "tags.txt",
- "output_format": "text",
- "text": "what is ragflow\tRAG,LLM\nhow to deploy\tGUIDE,ops.example",
- })
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
-
- c0 := chunks[0]
- if got := c0["content_with_weight"]; got != "what is ragflow" {
- t.Errorf("chunk0 content_with_weight = %q", got)
- }
- tags0 := tagKwdOf(t, c0)
- if len(tags0) != 2 || tags0[0] != "RAG" || tags0[1] != "LLM" {
- t.Errorf("chunk0 tag_kwd = %v, want [RAG LLM]", tags0)
- }
-
- c1 := chunks[1]
- tags1 := tagKwdOf(t, c1)
- // "ops.example" -> "ops_example": dots are sanitsed for keyword storage.
- if len(tags1) != 2 || tags1[0] != "GUIDE" || tags1[1] != "ops_example" {
- t.Errorf("chunk1 tag_kwd = %v, want [GUIDE ops_example]", tags1)
- }
-}
-
-// TestTagChunker_TextCommaDelimited verifies comma wins when the file has
-// no tab-structured rows (mirrors tag.py:detectDelimiter).
-func TestTagChunker_TextCommaDelimited(t *testing.T) {
- chunks := tagChunksOf(t, map[string]any{
- "name": "tags.csv",
- "output_format": "text",
- "text": "what is ragflow,RAG\nhow to deploy,GUIDE",
- })
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- if got := chunks[0]["content_with_weight"]; got != "what is ragflow" {
- t.Errorf("chunk0 content_with_weight = %q", got)
- }
- tags0 := tagKwdOf(t, chunks[0])
- if len(tags0) != 1 || tags0[0] != "RAG" {
- t.Errorf("chunk0 tag_kwd = %v, want [RAG]", tags0)
- }
-}
-
-// TestTagChunker_HTMLTable ports the spreadsheet (xlsx/csv -> html table)
-// path: first two <td> columns become content + tags.
-func TestTagChunker_HTMLTable(t *testing.T) {
- html := "<table><tr><td>q1</td><td>a1,b1</td></tr><tr><td>q2</td><td>a2</td></tr></table>"
- chunks := tagChunksOf(t, map[string]any{
- "name": "tags.xlsx",
- "output_format": "html",
- "html": html,
- })
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- if got := chunks[0]["content_with_weight"]; got != "q1" {
- t.Errorf("chunk0 content_with_weight = %q", got)
- }
- tags0 := tagKwdOf(t, chunks[0])
- if len(tags0) != 2 || tags0[0] != "a1" || tags0[1] != "b1" {
- t.Errorf("chunk0 tag_kwd = %v, want [a1 b1]", tags0)
- }
-}
-
-// TestTagChunk_TopInt verifies each tag-pair row carries the 0-based
-// source row index as top_int, matching Python tag.py:33 beAdoc(row_num=i).
-func TestTagChunk_TopInt(t *testing.T) {
- chunks := tagChunksOf(t, map[string]any{
- "name": "tags.txt",
- "output_format": "text",
- "text": "tag1\tcontent1\ntag2\tcontent2",
- })
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- if ti, ok := chunks[0]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(0) {
- t.Errorf("chunk 0: top_int = %v, want [0]", chunks[0]["top_int"])
- }
- if ti, ok := chunks[1]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(1) {
- t.Errorf("chunk 1: top_int = %v, want [1]", chunks[1]["top_int"])
- }
-}
-
-// TestTagChunk_CSVTopIntStartLine locks the RowNum convention from
-// review finding #4: a multi-line quoted CSV record must report the
-// record's START line, not the line after it ends. The first record
-// spans physical lines 0-1, so its top_int is [0]; the second record
-// is on line 2, so its top_int is [2].
-func TestTagChunk_CSVTopIntStartLine(t *testing.T) {
- chunks := tagChunksOf(t, map[string]any{
- "name": "tags.csv",
- "output_format": "text",
- "text": "\"multi\nline content\",TAG\nsecond,OTHER",
- })
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- if ti, ok := chunks[0]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(0) {
- t.Errorf("chunk 0: top_int = %v, want [0]", chunks[0]["top_int"])
- }
- if ti, ok := chunks[1]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(2) {
- t.Errorf("chunk 1: top_int = %v, want [2]", chunks[1]["top_int"])
- }
-}
-
-// TestTagChunk_TSVMultiLineRowNumStart pins the RowNum fix for TSV
-// multi-line records: when a record spans multiple physical lines
-// (because the first line does not contain a tab delimiter), the
-// resulting chunk's top_int must report the first non-empty CONTENT
-// start line, not the line where the tab-delimiter eventually
-// terminates the record (PR #17419 review #1).
-func TestTagChunk_TSVMultiLineRowNumStart(t *testing.T) {
- chunks := tagChunksOf(t, map[string]any{
- "name": "tags.txt",
- "output_format": "text",
- // Line 0 is empty, line 1 has no tab (accumulated as prefix
- // of the first record), line 2 is valid TSV.
- // The prefix starts at line 1 → RowNum must be 1, not 2.
- "text": "\nprefix\nrecord1\tTAG1\nmulti\nline-prefix\nrecord2\tTAG2",
- })
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- if ti, ok := chunks[0]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(1) {
- t.Errorf("chunk 0: top_int = %v, want [1] (content start)", chunks[0]["top_int"])
- }
- if ti, ok := chunks[1]["top_int"].([]any); !ok || len(ti) != 1 || ti[0] != float64(3) {
- t.Errorf("chunk 1: top_int = %v, want [3] (content start after blank line)", chunks[1]["top_int"])
- }
-}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json
index c10eab6733..7f6290847c 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_presentation.json
@@ -26,7 +26,7 @@
},
"Parser:HipSignsRhyme": {
"downstream": [
- "PresentationChunker:HappyHillsGlow"
+ "PageChunker:HappyHillsGlow"
],
"obj": {
"component_name": "Parser",
@@ -96,12 +96,12 @@
"Extractor:AutoExtractDefault"
]
},
- "PresentationChunker:HappyHillsGlow": {
+ "PageChunker:HappyHillsGlow": {
"downstream": [
"Extractor:AutoExtractDefault"
],
"obj": {
- "component_name": "PresentationChunker",
+ "component_name": "PageChunker",
"params": {
"outputs": {
"chunks": {
@@ -117,7 +117,7 @@
},
"Extractor:AutoExtractDefault": {
"upstream": [
- "PresentationChunker:HappyHillsGlow"
+ "PageChunker:HappyHillsGlow"
],
"downstream": [
"Tokenizer:TallTreesDance"
@@ -161,15 +161,15 @@
"targetHandle": "end"
},
{
- "id": "xy-edge__Parser:HipSignsRhymestart-PresentationChunker:HappyHillsGlowend",
+ "id": "xy-edge__Parser:HipSignsRhymestart-PageChunker:HappyHillsGlowend",
"source": "Parser:HipSignsRhyme",
"sourceHandle": "start",
- "target": "PresentationChunker:HappyHillsGlow",
+ "target": "PageChunker:HappyHillsGlow",
"targetHandle": "end"
},
{
- "id": "xy-edge__PresentationChunker:HappyHillsGlowstart-Extractor:AutoExtractDefaultend",
- "source": "PresentationChunker:HappyHillsGlow",
+ "id": "xy-edge__PageChunker:HappyHillsGlowstart-Extractor:AutoExtractDefaultend",
+ "source": "PageChunker:HappyHillsGlow",
"sourceHandle": "start",
"target": "Extractor:AutoExtractDefault",
"targetHandle": "end"
@@ -271,10 +271,10 @@
}
}
},
- "label": "PresentationChunker",
+ "label": "PageChunker",
"name": "Presentation Chunker_0"
},
- "id": "PresentationChunker:HappyHillsGlow",
+ "id": "PageChunker:HappyHillsGlow",
"measured": {
"height": 74,
"width": 200
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index 01daffa143..2900ed15c8 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -722,9 +722,6 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {
if templateUsesComponent(t, templateBytes, "QAChunker") {
t.Skip("template uses QAChunker which requires Q&A-structured content; covered separately")
}
- if templateUsesComponent(t, templateBytes, "TagChunker") {
- t.Skip("template uses TagChunker which requires tag-structured content and parser setups not available for generic .md input; covered separately")
- }
if templateUsesComponent(t, templateBytes, "Compiler") {
t.Skip("template uses Compiler which requires LLM/embedder/ES wiring not available in the headless smoke run; covered by the knowledge_compiler component E2E tests")
}
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 0330babc95..b9dbb572a6 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -47,7 +47,7 @@ var builtinComponentParamsGolden = map[string]string{
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PresentationChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "presentation": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"slides\": {\"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pptx\", \"ppt\"]}}, \"Tokenizer:TallTreesDance\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"PageChunker:HappyHillsGlow\": {}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}",
"table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",
"knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"compilation_template_group_id\": \"\", \"llm_id\": \"\", \"plan\": false}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",
diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.go
index 41b4c284d3..b45653b6e5 100644
--- a/internal/service/components_service_test.go
+++ b/internal/service/components_service_test.go
@@ -63,7 +63,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {
}
wantNames := []string{
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",
}
assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)
}
@@ -86,7 +86,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {
}
wantNames := []string{
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",
}
assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)
}
@@ -105,7 +105,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {
}
wantNames := []string{
"extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
- "onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
+ "onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",
}
if len(got) != len(wantNames) {
t.Errorf("duplicate category produced %d rows, want %d", len(got), len(wantNames))