Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(chunker): add ManualChunker for manual-layout PDFs (#18272)

Jackae256bcf59d5
diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.go
index 1fefbb7650..cfac9b7b46 100644
--- a/internal/handler/components_testpkg/components_handler_test.go
+++ b/internal/handler/components_testpkg/components_handler_test.go
@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
"titlechunker", "tokenchunker", "tokenizer",
}
@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
"titlechunker", "tokenchunker", "tokenizer",
}
@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {
}
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker",
"titlechunker", "tokenchunker", "tokenizer",
}
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 92c32f5679..c0d57170bb 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -41,6 +41,8 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, er
return NewTitleChunker(params)
case ComponentNameGroupTitleChunker:
return NewGroupTitleChunker(params)
+ case ComponentNameManualChunker:
+ return NewManualChunker(params)
case ComponentNameHierarchyTitleChunker:
return NewHierarchyTitleChunker(params)
case ComponentNameQAChunker:
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index 6c87507300..7a0572ec7e 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -107,9 +107,9 @@ func buildSectionIDs(levels []int, targetLevel int) []int {
}
// invokeGroup runs the GroupTitleChunker strategy against the
-// supplied inputs. Detected headings + adjacent merges happen in two
-// goroutines (heading detection sequential, then a fan-out over
-// record-buckets for the merge pass).
+// supplied inputs. It extracts the line records and defers to the
+// shared chunkFromRecords pipeline (which ManualChunker also uses,
+// after a physical-position resort).
func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {
records := extractLineRecords(inputs)
common.Debug("chunker stage",
@@ -120,6 +120,18 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
if len(records) == 0 {
return emptyOutputs(), nil
}
+ return chunkFromRecords(parentCtx, db, inputs, p, records)
+}
+
+// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over
+// an already-extracted record list: resolve heading levels, split into
+// sections, merge adjacent text records, build chunks, and perform on-demand
+// PDF cropping. GroupTitleChunker feeds records in input order; ManualChunker
+// feeds them after a (page, top, left) resort. Sharing this body guarantees
+// both strategies emit byte-identical output for coordinate-free input (the
+// docx manual branch) — the no-regression contract locked by
+// TestManualChunker_NoPositionsEqualsGroupChunker.
+func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord) (map[string]any, error) {
ctx := newLevelContext(records, outlineFromInputs(inputs), p)
levels := ctx.Levels()
// Count heading level distribution for debugging.
@@ -170,7 +182,7 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,
if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {
engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)
if eErr != nil {
- slog.Warn("GroupTitleChunker: could not open PDF for on-demand cropping", "err", eErr)
+ slog.Warn("chunker: could not open PDF for on-demand cropping", "err", eErr)
}
if engine != nil {
defer engine.Close()
@@ -330,6 +342,21 @@ func removeTag(text string) string {
return posTagRemove.ReplaceAllString(text, "")
}
+// pdfPosRowLess orders two PDF coordinate 5-tuples [page,left,right,top,bottom]
+// by (page, top, left) — i.e. by indices (0, 3, 1). It is the single shared
+// (page, top, left) comparator used both when re-sorting chunker records
+// (ManualChunker) and when de-duplicating/merging position matrices
+// (mergePositionMatrix). Both rows must have length >= 5.
+func pdfPosRowLess(a, b []float64) bool {
+ if a[0] != b[0] {
+ return a[0] < b[0]
+ }
+ if a[3] != b[3] {
+ return a[3] < b[3]
+ }
+ return a[1] < b[1]
+}
+
// mergePositionMatrix aggregates multiple PDF coordinate matrices into a
// single de-duplicated, sorted matrix. Mirrors Python
// pdf_chunk_metadata.py:127 merge_pdf_positions: rows are 5-tuples
@@ -363,13 +390,7 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {
return nil
}
sort.Slice(out, func(i, j int) bool {
- if out[i][0] != out[j][0] {
- return out[i][0] < out[j][0]
- }
- if out[i][3] != out[j][3] {
- return out[i][3] < out[j][3]
- }
- return out[i][1] < out[j][1]
+ return pdfPosRowLess(out[i], out[j])
})
return out
}
diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.go
new file mode 100644
index 0000000000..b690230e85
--- /dev/null
+++ b/internal/ingestion/component/chunker/manual.go
@@ -0,0 +1,205 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// ManualChunker is the Go port of Python's `manual` doc-type chunk method
+// (rag/app/manual.py). Like GroupTitleChunker it merges adjacent text records
+// into heading-bounded groups, but it first re-sorts the records into physical
+// reading order before grouping.
+//
+// Why the resort matters: for multi-column / manually-laid-out PDFs the parser
+// emits records in READING order (column-by-column), so a naive grouping
+// interleaves the left column's later content with the right column's earlier
+// content. ManualChunker re-orders records by (page, top, left) — mirroring
+// Python's
+//
+// sorted(sections, key=lambda x: (x[-1][0][0], x[-1][0][3], x[-1][0][1]))
+//
+// so grouping then follows the true top-down, left-to-right layout.
+//
+// Crucially, docx (and any coordinate-free payload) emits records in
+// document-logical order with NO positions — exactly as Python's manual docx
+// branch, which performs no resort. When no record carries coordinates,
+// ManualChunker skips the sort entirely and reuses the identical grouping
+// pipeline as GroupTitleChunker, so its output is bit-for-bit equal (locked by
+// TestManualChunker_NoPositionsEqualsGroupChunker). No parser change is needed.
+package chunker
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sort"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/globals"
+
+ "gorm.io/gorm"
+)
+
+const ComponentNameManualChunker = "ManualChunker"
+
+// ManualChunkerComponent is the standalone manual-layout chunker. It shares
+// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by
+// the optional physical-position pre-sort.
+type ManualChunkerComponent struct {
+ name string
+ param titleChunkerParam
+}
+
+// NewManualChunker constructs the component. It accepts the same heading-level
+// params as TitleChunker (method is pinned to "group"); the position resort is
+// automatic based on whether the upstream payload carries coordinates.
+func NewManualChunker(params map[string]any) (runtime.Component, error) {
+ // method is pinned to "group": ManualChunker's entire value-add is the
+ // physical-position resort, which only fires inside the group path.
+ // A caller must not downgrade it to "naive"/"title" (that would skip the
+ // resort and silently diverge from Python's manual.py), so method is
+ // deliberately ignored here even if passed in params.
+ conf := map[string]any{"method": "group"}
+ for k, v := range params {
+ if k == "method" {
+ continue
+ }
+ conf[k] = v
+ }
+ p := defaultsTitle()
+ p.Update(conf)
+ if err := p.TitleChunkerParam.Validate(); err != nil {
+ return nil, fmt.Errorf("ManualChunker: %w", err)
+ }
+ return &ManualChunkerComponent{
+ name: ComponentNameManualChunker,
+ param: p,
+ }, nil
+}
+
+func (c *ManualChunkerComponent) Inputs() map[string]string { return ChunkerInputs }
+func (c *ManualChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }
+
+func (c *ManualChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+ if inputs == nil {
+ inputs = map[string]any{}
+ }
+ // `name` is read from the workflow-wide Globals bag (seeded at
+ // pipeline start, published by the File component), not from the
+ // upstream output map.
+ name := globals.GlobalOrInput(ctx, inputs, "name", "")
+ if name == "" {
+ return map[string]any{
+ "output_format": "chunks",
+ "chunks": []map[string]any{},
+ "_ERROR": "ManualChunker: missing required upstream field \"name\"",
+ }, nil
+ }
+ return c.invoke(ctx, db, withName(inputs, name))
+}
+
+func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+ records := extractLineRecords(inputs)
+ if len(records) == 0 {
+ return emptyOutputs(), nil
+ }
+ // Coordinate-free payloads (docx, plain text) need no resort — and
+ // skipping it keeps the output identical to GroupTitleChunker.
+ if hasPdfPositions(records) {
+ sortRecordsByPosition(records)
+ }
+ return chunkFromRecords(ctx, db, inputs, &c.param, records)
+}
+
+// hasPdfPositions reports whether any record carries a PDF coordinate matrix
+// (either the structured `_pdf_positions` or the legacy `positions` form).
+func hasPdfPositions(records []lineRecord) bool {
+ for _, r := range records {
+ if len(r.pdfPositions) > 0 || len(r.positions) > 0 {
+ return true
+ }
+ }
+ return false
+}
+
+// sortRecordsByPosition re-orders records into physical reading order
+// (page, then top, then left) using a stable sort, so records that share a
+// key keep their original relative order. Records without coordinates act as
+// immovable barriers: they keep their original relative order and are never
+// compared against positioned records.
+//
+// The slice is split into maximal contiguous runs of positioned records and
+// each run is sorted independently. This is required because a single
+// sort.SliceStable over a MIXED slice (some records with coordinates, some
+// without) is not a strict weak ordering: a coordinate-free record B sits
+// incomparable between two positioned records A and C, yet A and C remain
+// comparable — breaking equivalence-class transitivity and making the sort
+// undefined behaviour. Segmenting on coordinate-free records keeps each sort
+// a valid strict weak ordering while preserving the documented behaviour that
+// coordinate-free records keep their input order (see hasPdfPositions gate).
+func sortRecordsByPosition(records []lineRecord) {
+ runStart := -1
+ flush := func(end int) {
+ if runStart < 0 {
+ return
+ }
+ sort.SliceStable(records[runStart:end], func(i, j int) bool {
+ ar, _ := firstPositionRow(records[runStart+i])
+ br, _ := firstPositionRow(records[runStart+j])
+ return pdfPosRowLess(ar, br)
+ })
+ runStart = -1
+ }
+ for i := 0; i <= len(records); i++ {
+ positioned := i < len(records)
+ if positioned {
+ _, positioned = firstPositionRow(records[i])
+ }
+ if positioned {
+ if runStart < 0 {
+ runStart = i
+ }
+ } else {
+ flush(i)
+ }
+ }
+}
+
+// firstPositionRow returns the first PDF coordinate 5-tuple
+// [page,left,right,top,bottom] of a record's coordinate matrix, or
+// (nil, false) when the record carries no usable coordinates. It reads the
+// structured `_pdf_positions` key first and falls back to the legacy
+// `positions` key. Callers (pdfPosRowLess) index offsets 0/1/3, so we enforce
+// len(row) >= 5 here.
+func firstPositionRow(r lineRecord) ([]float64, bool) {
+ var raw json.RawMessage
+ switch {
+ case len(r.pdfPositions) > 0:
+ raw = r.pdfPositions
+ case len(r.positions) > 0:
+ raw = r.positions
+ default:
+ return nil, false
+ }
+ var mat [][]float64
+ if err := json.Unmarshal(raw, &mat); err != nil || len(mat) == 0 {
+ return nil, false
+ }
+ row := mat[0]
+ if len(row) < 5 {
+ return nil, false
+ }
+ return row, true
+}
+
+// init registers ManualChunker under CategoryIngestion.
+func init() {
+ MustRegisterChunker(ComponentNameManualChunker)
+}
diff --git a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go
new file mode 100644
index 0000000000..0367f85dda
--- /dev/null
+++ b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go
@@ -0,0 +1,97 @@
+//go:build cgo && integration
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+// TestManualChunker_RealPDFResort proves the (page, top, left) resort operates
+// on the ACTUAL records the Go PDF parser produces. This closes gap 1 from the
+// review: the template integration test only feeds a .txt fixture with no
+// coordinates, so nothing exercised the resort against real parser output.
+//
+// It parses a real PDF, normalizes its records through the SAME extractLineRecords
+// path the chunker uses in production, then asserts the resort yields a
+// non-decreasing physical order — i.e. the shared pdfPosRowLess comparator
+// genuinely consumes the parser-emitted _pdf_positions rather than silently
+// no-op'ing. The assertion is layout-independent: whether or not the reading
+// order differed from the physical order, the sorted result must be monotonic.
+func TestManualChunker_RealPDFResort(t *testing.T) {
+ t.Setenv("DEEPDOC_URL", "")
+ t.Setenv("OSSDEEPDOC_URL", "")
+
+ path := filepath.Join("..", "..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("ReadFile(%s): %v", path, err)
+ }
+ pdf := parser.NewPDFParser()
+ res := pdf.ParseWithResult(t.Context(), "Doc1.pdf", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatal("parser produced no JSON items")
+ }
+
+ // Same shape the chunker receives in production.
+ input := map[string]any{
+ "name": "Doc1.pdf",
+ "output_format": "json",
+ "chunks": res.JSON,
+ }
+ records := extractLineRecords(input)
+ if len(records) == 0 {
+ t.Fatal("extractLineRecords produced no records from real parser output")
+ }
+
+ positioned := 0
+ for _, r := range records {
+ if _, ok := firstPositionRow(r); ok {
+ positioned++
+ }
+ }
+ if positioned == 0 {
+ t.Fatal("no real parser records carried _pdf_positions; resort would be a no-op")
+ }
+
+ sortRecordsByPosition(records)
+
+ // After the resort, the physical (page, top, left) order must be
+ // non-decreasing. pdfPosRowLess(cur, prev) being true would mean a record
+ // physically precedes its predecessor — a resort violation. A coordinate-free
+ // record (no firstPositionRow) is skipped and never becomes prev, so prev may
+ // be nil until the first positioned record; guard against that.
+ var prev []float64
+ for i := 0; i < len(records); i++ {
+ cur, ok := firstPositionRow(records[i])
+ if !ok {
+ continue
+ }
+ if prev != nil && pdfPosRowLess(cur, prev) {
+ t.Fatalf("resort produced out-of-order records at %d: row %v precedes %v", i, cur, prev)
+ }
+ prev = cur
+ }
+}
diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.go
new file mode 100644
index 0000000000..5fb2f9dd5f
--- /dev/null
+++ b/internal/ingestion/component/chunker/manual_test.go
@@ -0,0 +1,431 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "encoding/json"
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+)
+
+// manualChunkerInput is a small builder for a structured (output_format=chunks)
+// upstream payload, mirroring the shape the Parser emits for PDF/JSON output.
+func manualChunkerInput(name string, items ...map[string]any) map[string]any {
+ return map[string]any{
+ "name": name,
+ "output_format": "chunks",
+ "chunks": items,
+ }
+}
+
+// posItem builds a chunk item carrying a PDF position matrix
+// [page,left,right,top,bottom] under the `_pdf_positions` key (matching
+// schema.ChunkDoc.PDFPositions).
+func posItem(text string, page, left, right, top, bottom float64) map[string]any {
+ return map[string]any{
+ "text": text,
+ "doc_type_kwd": "text",
+ "_pdf_positions": [][]float64{{page, left, right, top, bottom}},
+ }
+}
+
+// mustPosJSON marshals a single 5-tuple position matrix into the
+// json.RawMessage form used by lineRecord.pdfPositions / positions.
+func mustPosJSON(t *testing.T, page, left, right, top, bottom float64) json.RawMessage {
+ t.Helper()
+ b, err := json.Marshal([][]float64{{page, left, right, top, bottom}})
+ if err != nil {
+ t.Fatalf("marshal positions: %v", err)
+ }
+ return b
+}
+
+func plainItem(text string) map[string]any {
+ return map[string]any{"text": text, "doc_type_kwd": "text"}
+}
+
+func mustManual(t *testing.T, params map[string]any) *ManualChunkerComponent {
+ t.Helper()
+ c, err := NewManualChunker(params)
+ if err != nil {
+ t.Fatalf("NewManualChunker: %v", err)
+ }
+ mc, ok := c.(*ManualChunkerComponent)
+ if !ok {
+ t.Fatalf("NewManualChunker returned %T, want *ManualChunkerComponent", c)
+ }
+ return mc
+}
+
+// manualChunkTexts pulls the ordered chunk texts out of a chunker output map.
+func manualChunkTexts(out map[string]any) []string {
+ chunks, _ := out["chunks"].([]map[string]any)
+ out2 := make([]string, 0, len(chunks))
+ for _, ck := range chunks {
+ if s, _ := ck["text"].(string); s != "" {
+ out2 = append(out2, s)
+ }
+ }
+ return out2
+}
+
+// TestManualChunker_Registered pins that the component is discoverable via the
+// runtime registry under the CategoryIngestion category with non-empty
+// input/output metadata.
+func TestManualChunker_Registered(t *testing.T) {
+ factory, cat, meta, ok := runtime.DefaultRegistry.Lookup(ComponentNameManualChunker)
+ if !ok {
+ t.Fatal("ManualChunker: registry miss")
+ }
+ if cat != runtime.CategoryIngestion {
+ t.Errorf("category = %q, want %q", cat, runtime.CategoryIngestion)
+ }
+ if factory == nil {
+ t.Error("factory is nil")
+ }
+ if len(meta.Inputs) == 0 {
+ t.Error("inputs metadata empty")
+ }
+ if len(meta.Outputs) == 0 {
+ t.Error("outputs metadata empty")
+ }
+}
+
+// TestManualChunker_Empty is the trivial boundary: no records => empty outputs.
+func TestManualChunker_Empty(t *testing.T) {
+ c := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "doc"})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got, want := out["output_format"], "chunks"; got != want {
+ t.Errorf("output_format = %v, want %v", got, want)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 0 {
+ t.Errorf("chunks = %d, want 0", len(chunks))
+ }
+}
+
+// TestManualChunker_NoPositionsEqualsGroupChunker is the regression lock: when
+// the upstream payload carries NO physical coordinates (e.g. docx, which Go
+// emits in document-logical order — exactly as Python's manual docx branch,
+// which performs no coordinate resort), ManualChunker MUST behave bit-for-bit
+// like GroupTitleChunker. Any divergence here is an unintended regression.
+func TestManualChunker_NoPositionsEqualsGroupChunker(t *testing.T) {
+ var mkLevels = [][]string{{`^# `}, {`^## `}}
+ input := manualChunkerInput("doc",
+ plainItem("# H1"),
+ plainItem("body under h1 a"),
+ plainItem("body under h1 b"),
+ plainItem("## H2"),
+ plainItem("body under h2"),
+ plainItem("more body under h2"),
+ )
+
+ gc, err := NewGroupTitleChunker(map[string]any{"levels": mkLevels})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ gOut, err := gc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("group Invoke: %v", err)
+ }
+
+ mc := mustManual(t, map[string]any{"levels": mkLevels})
+ mOut, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("manual Invoke: %v", err)
+ }
+
+ gChunks, _ := gOut["chunks"].([]map[string]any)
+ mChunks, _ := mOut["chunks"].([]map[string]any)
+ if !reflect.DeepEqual(gChunks, mChunks) {
+ t.Fatalf("ManualChunker output diverges from GroupTitleChunker on no-coordinate input\n group: %#v\n manual: %#v", gChunks, mChunks)
+ }
+}
+
+// TestManualChunker_SortsByPageTopLeft is the core behavioural difference
+// vs. GroupTitleChunker. For a multi-column / manual PDF, the parser emits
+// records in READING order (column-by-column), not in PHYSICAL/top-down
+// order. ManualChunker must resort records by (page, top, left) before
+// grouping so that, e.g., the left column's content precedes the right
+// column's when they share a page band.
+func TestManualChunker_SortsByPageTopLeft(t *testing.T) {
+ // page 1, all body (no heading markers). Physical layout:
+ // top=100, left=10 -> "left col top"
+ // top=100, left=400 -> "right col"
+ // top=300, left=10 -> "left col bottom"
+ // top=400, left=200 -> "footer title"
+ // Parser reading order is scrambled relative to physical order.
+ input := manualChunkerInput("doc",
+ posItem("footer title", 1, 200, 400, 400, 500),
+ posItem("left col bottom", 1, 10, 200, 300, 400),
+ posItem("right col", 1, 400, 600, 100, 400),
+ posItem("left col top", 1, 10, 200, 100, 200),
+ )
+
+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ out, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ // Whether merged into one chunk or several, the concatenated text MUST
+ // follow physical order: left col top, right col, left col bottom,
+ // footer title.
+ var full strings.Builder
+ for _, s := range manualChunkTexts(out) {
+ full.WriteString(s)
+ full.WriteString("\n")
+ }
+ joined := full.String()
+ wantOrder := []string{"left col top", "right col", "left col bottom", "footer title"}
+ last := -1
+ for _, w := range wantOrder {
+ idx := strings.Index(joined, w)
+ if idx < 0 {
+ t.Fatalf("expected %q to appear in manual output %q", w, joined)
+ }
+ if idx <= last {
+ t.Fatalf("physical order violated: %q appears after a later element in %q", w, joined)
+ }
+ last = idx
+ }
+}
+
+// TestManualChunker_SortDiffersFromGroupWhenPositionsPresent proves the resort
+// actually fires (and is not a no-op): the same scrambled input must produce a
+// DIFFERENT text order from GroupTitleChunker, which groups in input order.
+func TestManualChunker_SortDiffersFromGroupWhenPositionsPresent(t *testing.T) {
+ input := manualChunkerInput("doc",
+ posItem("parser-reads-2nd-col-first", 1, 400, 600, 100, 400),
+ posItem("parser-reads-1st-col-first", 1, 10, 200, 100, 200),
+ )
+
+ gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ gOut, err := gc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("group Invoke: %v", err)
+ }
+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ mOut, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("manual Invoke: %v", err)
+ }
+ if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) {
+ t.Fatalf("ManualChunker did not reorder relative to GroupTitleChunker; both = %#v", mOut["chunks"])
+ }
+ // ManualChunker must place the 1st-column item first.
+ if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "1st-col") {
+ t.Fatalf("expected 1st-column text first in manual output, got %#v", texts)
+ }
+}
+
+// TestManualChunker_MergesPositions pins parity with GroupTitleChunker: when
+// adjacent text records are merged into one chunk, the PDF position matrices
+// are MERGED across the merged records (not dropped), and parser position tags
+// are stripped from the text.
+func TestManualChunker_MergesPositions(t *testing.T) {
+ input := manualChunkerInput("doc",
+ posItem("body one", 1, 10, 20, 30, 40),
+ posItem("body two", 2, 15, 25, 35, 45),
+ )
+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ out, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ found := false
+ for _, ck := range chunks {
+ text, _ := ck["text"].(string)
+ if !strings.Contains(text, "body one") || !strings.Contains(text, "body two") {
+ continue
+ }
+ found = true
+ if strings.Contains(text, "@@") {
+ t.Errorf("parser position tags leaked into chunk text: %q", text)
+ }
+ pos, ok := ck["_pdf_positions"].([][]float64)
+ if !ok {
+ t.Fatalf("_pdf_positions missing or wrong type %T on merged group chunk", ck["_pdf_positions"])
+ }
+ if len(pos) != 2 {
+ // Hard fail: the page/top/left checks below would index out of
+ // range if the merged matrix does not have exactly two rows.
+ t.Fatalf("merged _pdf_positions = %d rows, want 2 (both records)", len(pos))
+ }
+ // Merged matrix must itself be sorted by (page, top, left) — the
+ // same key pdfPosRowLess uses when merging.
+ if pdfPosRowLess(pos[1], pos[0]) {
+ t.Errorf("merged _pdf_positions not sorted by (page, top, left): %v", pos)
+ }
+ }
+ if !found {
+ t.Fatal("merged body group chunk not found in output")
+ }
+}
+
+// TestSortRecordsByPosition_Unit pins the comparator directly: (page, top,
+// left) ascending, stable for equal keys, and records without positions keep
+// their original relative order (so a no-coordinate payload is untouched).
+func TestSortRecordsByPosition_Unit(t *testing.T) {
+ recs := []lineRecord{
+ {text: "a", pdfPositions: mustPosJSON(t, 1, 200, 300, 400, 500)},
+ {text: "b", pdfPositions: mustPosJSON(t, 1, 10, 100, 300, 400)},
+ {text: "c", pdfPositions: mustPosJSON(t, 1, 10, 100, 100, 200)},
+ {text: "d"}, // no position
+ }
+ sortRecordsByPosition(recs)
+ got := []string{recs[0].text, recs[1].text, recs[2].text, recs[3].text}
+ want := []string{"c", "b", "a", "d"}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("sorted order = %v, want %v", got, want)
+ }
+}
+
+// TestHasPdfPositions is a tiny white-box check for the gate that decides
+// whether a resort is performed at all.
+func TestHasPdfPositions(t *testing.T) {
+ if hasPdfPositions([]lineRecord{{text: "x"}}) {
+ t.Error("plain record should report no positions")
+ }
+ if !hasPdfPositions([]lineRecord{{text: "x", pdfPositions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) {
+ t.Error("record with pdf_positions should report positions")
+ }
+ if !hasPdfPositions([]lineRecord{{text: "x", positions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) {
+ t.Error("record with positions should report positions")
+ }
+}
+
+// parserShapedItem builds a chunk item that mirrors EXACTLY the JSON shape the
+// Go PDF parser emits via pdfParseResultToJSON (see
+// TestPDFParseResultToJSON_NormalizesCoreFields): _pdf_positions is a [][]any
+// with 1-based page numbers, accompanied by layout / page_number /
+// doc_type_kwd. Using [][]any — not [][]float64 — matters: that is the concrete
+// in-memory type the parser produces, and recordsFromStructured must consume it
+// after the chunksFromInputs JSON round-trip.
+func parserShapedItem(text string, page, left, right, top, bottom float64) map[string]any {
+ return map[string]any{
+ "text": text,
+ "doc_type_kwd": "text",
+ "layout": "text",
+ "page_number": int(page),
+ "_pdf_positions": [][]any{{page, left, right, top, bottom}},
+ }
+}
+
+// legacyPosItem is like posItem but uses the legacy `positions` key instead of
+// the structured `_pdf_positions` key.
+func legacyPosItem(text string, page, left, right, top, bottom float64) map[string]any {
+ return map[string]any{
+ "text": text,
+ "doc_type_kwd": "text",
+ "positions": [][]float64{{page, left, right, top, bottom}},
+ }
+}
+
+// TestManualChunker_RealParserShapedPayload exercises the EXACT payload shape
+// the Go PDF parser delivers to the chunker (pdfParseResultToJSON ->
+// recordsFromStructured). It closes the coverage gap where the template
+// integration test only feeds a .txt fixture (no coordinates): here a
+// coordinate-bearing, parser-shaped multi-column doc proves the (page, top,
+// left) resort actually fires and produces a different order from
+// GroupTitleChunker (which groups in parser reading order).
+func TestManualChunker_RealParserShapedPayload(t *testing.T) {
+ input := manualChunkerInput("doc",
+ // Parser reading order (column-by-column) is scrambled vs physical.
+ parserShapedItem("right column, read first", 1, 400, 600, 100, 400),
+ parserShapedItem("left column, read second", 1, 10, 200, 100, 200),
+ )
+
+ gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ gOut, err := gc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("group Invoke: %v", err)
+ }
+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ mOut, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("manual Invoke: %v", err)
+ }
+ if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) {
+ t.Fatalf("ManualChunker did not reorder parser-shaped payload; both = %#v", mOut["chunks"])
+ }
+ // Physical (page, top, left) order must put the left column first.
+ if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "left column") {
+ t.Fatalf("expected left-column text first after resort, got %#v", texts)
+ }
+}
+
+// TestManualChunker_LegacyPositionsKeyTriggersSort pins that the resort fires
+// when records carry the LEGACY `positions` key rather than the structured
+// `_pdf_positions` key. firstPositionRow (and thus the shared pdfPosRowLess
+// comparator) must read both forms, otherwise a payload emitted with the old
+// key would silently skip the resort.
+func TestManualChunker_LegacyPositionsKeyTriggersSort(t *testing.T) {
+ input := manualChunkerInput("doc",
+ legacyPosItem("right column, read first", 1, 400, 600, 100, 400),
+ legacyPosItem("left column, read second", 1, 10, 200, 100, 200),
+ )
+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ out, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ // Without the legacy-key path the output would keep parser reading order
+ // (right column before left). The resort must place the left column first.
+ if texts := manualChunkTexts(out); len(texts) == 0 || !strings.Contains(texts[0], "left column") {
+ t.Fatalf("legacy `positions` key did not trigger resort; got %#v", texts)
+ }
+}
+
+// TestManualChunker_MixedPositionedAndPlain exercises the boundary where a
+// payload mixes coordinate-bearing records with coordinate-free records. A
+// single document type will not usually do this (production payloads are
+// homogeneously positioned-or-plain), but the sort must not corrupt order when
+// it happens: the contiguous positioned run reorders by (page, top, left) and
+// coordinate-free records keep their ORIGINAL relative order. (Go's stable
+// sort only reorders within a contiguous positioned run — a coordinate-free
+// record between two positioned records correctly stops the reorder from
+// crossing it, which is why the positioned records here are kept adjacent.)
+func TestManualChunker_MixedPositionedAndPlain(t *testing.T) {
+ input := manualChunkerInput("doc",
+ plainItem("plain A"),
+ posItem("right col", 1, 400, 600, 100, 400),
+ posItem("left col", 1, 10, 200, 100, 200),
+ plainItem("plain B"),
+ )
+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})
+ out, err := mc.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ joined := strings.Join(manualChunkTexts(out), "\n")
+ // The contiguous positioned run keeps physical order relative to itself.
+ if i, j := strings.Index(joined, "left col"), strings.Index(joined, "right col"); i < 0 || j < 0 || i > j {
+ t.Fatalf("positioned records not in physical order in %q", joined)
+ }
+ // Coordinate-free records keep their original relative order.
+ if i, j := strings.Index(joined, "plain A"), strings.Index(joined, "plain B"); i < 0 || j < 0 || i > j {
+ t.Fatalf("coordinate-free records lost original order in %q", joined)
+ }
+}
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
index e83e0d17e5..1faad567e9 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json
@@ -26,7 +26,7 @@
},
"Parser:HipSignsRhyme": {
"downstream": [
- "TitleChunker:NineInsectsFind"
+ "ManualChunker:NineInsectsFind"
],
"obj": {
"component_name": "Parser",
@@ -87,12 +87,12 @@
"File"
]
},
- "TitleChunker:NineInsectsFind": {
+ "ManualChunker:NineInsectsFind": {
"downstream": [
"Extractor:AutoExtractDefault"
],
"obj": {
- "component_name": "TitleChunker",
+ "component_name": "ManualChunker",
"params": {
"hierarchy": 0,
"include_heading_content": false,
@@ -158,7 +158,7 @@
},
"Extractor:AutoExtractDefault": {
"upstream": [
- "TitleChunker:NineInsectsFind"
+ "ManualChunker:NineInsectsFind"
],
"downstream": [
"Tokenizer:FunnyBalloonsGrin"
@@ -197,15 +197,15 @@
"targetHandle": "end"
},
{
- "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:NineInsectsFindend",
+ "id": "xy-edge__Parser:HipSignsRhymestart-ManualChunker:NineInsectsFindend",
"source": "Parser:HipSignsRhyme",
"sourceHandle": "start",
- "target": "TitleChunker:NineInsectsFind",
+ "target": "ManualChunker:NineInsectsFind",
"targetHandle": "end"
},
{
- "id": "xy-edge__TitleChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend",
- "source": "TitleChunker:NineInsectsFind",
+ "id": "xy-edge__ManualChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend",
+ "source": "ManualChunker:NineInsectsFind",
"sourceHandle": "start",
"target": "Extractor:AutoExtractDefault",
"targetHandle": "end"
@@ -407,10 +407,10 @@
}
]
},
- "label": "TitleChunker",
- "name": "Title Chunker_0"
+ "label": "ManualChunker",
+ "name": "Manual Chunker_0"
},
- "id": "TitleChunker:NineInsectsFind",
+ "id": "ManualChunker:NineInsectsFind",
"measured": {
"height": 74,
"width": 200
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index aa42b84cf1..01daffa143 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -424,9 +424,9 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
}
}
- chunkerState, ok := state["TitleChunker:NineInsectsFind"]
+ chunkerState, ok := state["ManualChunker:NineInsectsFind"]
if !ok {
- t.Fatal("missing TitleChunker:NineInsectsFind state")
+ t.Fatal("missing ManualChunker:NineInsectsFind state")
}
chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index 17459f27d4..b4148ed099 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -43,7 +43,7 @@ var builtinComponentParamsGolden = map[string]string{
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
- "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
+ "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
"picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",
diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.go
index 1466fac979..41b4c284d3 100644
--- a/internal/service/components_service_test.go
+++ b/internal/service/components_service_test.go
@@ -62,7 +62,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {
t.Fatalf("List(Ingestion) returned error: %v", err)
}
wantNames := []string{
- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
}
assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)
@@ -85,7 +85,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {
t.Fatalf("List(Ingestion,Shared) returned error: %v", err)
}
wantNames := []string{
- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
}
assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)
@@ -104,7 +104,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {
t.Fatalf("List(Ingestion x3) returned error: %v", err)
}
wantNames := []string{
- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",
+ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",
}
if len(got) != len(wantNames) {