RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(chunker): add ManualChunker for manual-layout PDFs (#18272)
Jackae256bcf59d5
diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.goindex 1fefbb7650..cfac9b7b46 100644--- a/internal/handler/components_testpkg/components_handler_test.go+++ b/internal/handler/components_testpkg/components_handler_test.go@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker","titlechunker", "tokenchunker", "tokenizer",}@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker","titlechunker", "tokenchunker", "tokenizer",}@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {}_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker","titlechunker", "tokenchunker", "tokenizer",}diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 92c32f5679..c0d57170bb 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -41,6 +41,8 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, erreturn NewTitleChunker(params)case ComponentNameGroupTitleChunker:return NewGroupTitleChunker(params)+ case ComponentNameManualChunker:+ return NewManualChunker(params)case ComponentNameHierarchyTitleChunker:return NewHierarchyTitleChunker(params)case ComponentNameQAChunker:diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex 6c87507300..7a0572ec7e 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -107,9 +107,9 @@ func buildSectionIDs(levels []int, targetLevel int) []int {}// invokeGroup runs the GroupTitleChunker strategy against the-// supplied inputs. Detected headings + adjacent merges happen in two-// goroutines (heading detection sequential, then a fan-out over-// record-buckets for the merge pass).+// supplied inputs. It extracts the line records and defers to the+// shared chunkFromRecords pipeline (which ManualChunker also uses,+// after a physical-position resort).func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam) (map[string]any, error) {records := extractLineRecords(inputs)common.Debug("chunker stage",@@ -120,6 +120,18 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,if len(records) == 0 {return emptyOutputs(), nil}+ return chunkFromRecords(parentCtx, db, inputs, p, records)+}++// chunkFromRecords runs the shared GroupTitle / Manual grouping pipeline over+// an already-extracted record list: resolve heading levels, split into+// sections, merge adjacent text records, build chunks, and perform on-demand+// PDF cropping. GroupTitleChunker feeds records in input order; ManualChunker+// feeds them after a (page, top, left) resort. Sharing this body guarantees+// both strategies emit byte-identical output for coordinate-free input (the+// docx manual branch) — the no-regression contract locked by+// TestManualChunker_NoPositionsEqualsGroupChunker.+func chunkFromRecords(parentCtx context.Context, db *gorm.DB, inputs map[string]any, p *titleChunkerParam, records []lineRecord) (map[string]any, error) {ctx := newLevelContext(records, outlineFromInputs(inputs), p)levels := ctx.Levels()// Count heading level distribution for debugging.@@ -170,7 +182,7 @@ func invokeGroup(parentCtx context.Context, db *gorm.DB, inputs map[string]any,if upstream, uErr := decodeChunkerFromUpstream(inputs); uErr == nil {engine, eErr := newPDFEngineFromUpstream(parentCtx, db, upstream)if eErr != nil {- slog.Warn("GroupTitleChunker: could not open PDF for on-demand cropping", "err", eErr)+ slog.Warn("chunker: could not open PDF for on-demand cropping", "err", eErr)}if engine != nil {defer engine.Close()@@ -330,6 +342,21 @@ func removeTag(text string) string {return posTagRemove.ReplaceAllString(text, "")}+// pdfPosRowLess orders two PDF coordinate 5-tuples [page,left,right,top,bottom]+// by (page, top, left) — i.e. by indices (0, 3, 1). It is the single shared+// (page, top, left) comparator used both when re-sorting chunker records+// (ManualChunker) and when de-duplicating/merging position matrices+// (mergePositionMatrix). Both rows must have length >= 5.+func pdfPosRowLess(a, b []float64) bool {+ if a[0] != b[0] {+ return a[0] < b[0]+ }+ if a[3] != b[3] {+ return a[3] < b[3]+ }+ return a[1] < b[1]+}+// mergePositionMatrix aggregates multiple PDF coordinate matrices into a// single de-duplicated, sorted matrix. Mirrors Python// pdf_chunk_metadata.py:127 merge_pdf_positions: rows are 5-tuples@@ -363,13 +390,7 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {return nil}sort.Slice(out, func(i, j int) bool {- if out[i][0] != out[j][0] {- return out[i][0] < out[j][0]- }- if out[i][3] != out[j][3] {- return out[i][3] < out[j][3]- }- return out[i][1] < out[j][1]+ return pdfPosRowLess(out[i], out[j])})return out}diff --git a/internal/ingestion/component/chunker/manual.go b/internal/ingestion/component/chunker/manual.gonew file mode 100644index 0000000000..b690230e85--- /dev/null+++ b/internal/ingestion/component/chunker/manual.go@@ -0,0 +1,205 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//+// ManualChunker is the Go port of Python's `manual` doc-type chunk method+// (rag/app/manual.py). Like GroupTitleChunker it merges adjacent text records+// into heading-bounded groups, but it first re-sorts the records into physical+// reading order before grouping.+//+// Why the resort matters: for multi-column / manually-laid-out PDFs the parser+// emits records in READING order (column-by-column), so a naive grouping+// interleaves the left column's later content with the right column's earlier+// content. ManualChunker re-orders records by (page, top, left) — mirroring+// Python's+//+// sorted(sections, key=lambda x: (x[-1][0][0], x[-1][0][3], x[-1][0][1]))+//+// so grouping then follows the true top-down, left-to-right layout.+//+// Crucially, docx (and any coordinate-free payload) emits records in+// document-logical order with NO positions — exactly as Python's manual docx+// branch, which performs no resort. When no record carries coordinates,+// ManualChunker skips the sort entirely and reuses the identical grouping+// pipeline as GroupTitleChunker, so its output is bit-for-bit equal (locked by+// TestManualChunker_NoPositionsEqualsGroupChunker). No parser change is needed.+package chunker++import (+ "context"+ "encoding/json"+ "fmt"+ "sort"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/globals"++ "gorm.io/gorm"+)++const ComponentNameManualChunker = "ManualChunker"++// ManualChunkerComponent is the standalone manual-layout chunker. It shares+// the GroupTitleChunker grouping body (chunkFromRecords) and only differs by+// the optional physical-position pre-sort.+type ManualChunkerComponent struct {+ name string+ param titleChunkerParam+}++// NewManualChunker constructs the component. It accepts the same heading-level+// params as TitleChunker (method is pinned to "group"); the position resort is+// automatic based on whether the upstream payload carries coordinates.+func NewManualChunker(params map[string]any) (runtime.Component, error) {+ // method is pinned to "group": ManualChunker's entire value-add is the+ // physical-position resort, which only fires inside the group path.+ // A caller must not downgrade it to "naive"/"title" (that would skip the+ // resort and silently diverge from Python's manual.py), so method is+ // deliberately ignored here even if passed in params.+ conf := map[string]any{"method": "group"}+ for k, v := range params {+ if k == "method" {+ continue+ }+ conf[k] = v+ }+ p := defaultsTitle()+ p.Update(conf)+ if err := p.TitleChunkerParam.Validate(); err != nil {+ return nil, fmt.Errorf("ManualChunker: %w", err)+ }+ return &ManualChunkerComponent{+ name: ComponentNameManualChunker,+ param: p,+ }, nil+}++func (c *ManualChunkerComponent) Inputs() map[string]string { return ChunkerInputs }+func (c *ManualChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }++func (c *ManualChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {+ if inputs == nil {+ inputs = map[string]any{}+ }+ // `name` is read from the workflow-wide Globals bag (seeded at+ // pipeline start, published by the File component), not from the+ // upstream output map.+ name := globals.GlobalOrInput(ctx, inputs, "name", "")+ if name == "" {+ return map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{},+ "_ERROR": "ManualChunker: missing required upstream field \"name\"",+ }, nil+ }+ return c.invoke(ctx, db, withName(inputs, name))+}++func (c *ManualChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {+ records := extractLineRecords(inputs)+ if len(records) == 0 {+ return emptyOutputs(), nil+ }+ // Coordinate-free payloads (docx, plain text) need no resort — and+ // skipping it keeps the output identical to GroupTitleChunker.+ if hasPdfPositions(records) {+ sortRecordsByPosition(records)+ }+ return chunkFromRecords(ctx, db, inputs, &c.param, records)+}++// hasPdfPositions reports whether any record carries a PDF coordinate matrix+// (either the structured `_pdf_positions` or the legacy `positions` form).+func hasPdfPositions(records []lineRecord) bool {+ for _, r := range records {+ if len(r.pdfPositions) > 0 || len(r.positions) > 0 {+ return true+ }+ }+ return false+}++// sortRecordsByPosition re-orders records into physical reading order+// (page, then top, then left) using a stable sort, so records that share a+// key keep their original relative order. Records without coordinates act as+// immovable barriers: they keep their original relative order and are never+// compared against positioned records.+//+// The slice is split into maximal contiguous runs of positioned records and+// each run is sorted independently. This is required because a single+// sort.SliceStable over a MIXED slice (some records with coordinates, some+// without) is not a strict weak ordering: a coordinate-free record B sits+// incomparable between two positioned records A and C, yet A and C remain+// comparable — breaking equivalence-class transitivity and making the sort+// undefined behaviour. Segmenting on coordinate-free records keeps each sort+// a valid strict weak ordering while preserving the documented behaviour that+// coordinate-free records keep their input order (see hasPdfPositions gate).+func sortRecordsByPosition(records []lineRecord) {+ runStart := -1+ flush := func(end int) {+ if runStart < 0 {+ return+ }+ sort.SliceStable(records[runStart:end], func(i, j int) bool {+ ar, _ := firstPositionRow(records[runStart+i])+ br, _ := firstPositionRow(records[runStart+j])+ return pdfPosRowLess(ar, br)+ })+ runStart = -1+ }+ for i := 0; i <= len(records); i++ {+ positioned := i < len(records)+ if positioned {+ _, positioned = firstPositionRow(records[i])+ }+ if positioned {+ if runStart < 0 {+ runStart = i+ }+ } else {+ flush(i)+ }+ }+}++// firstPositionRow returns the first PDF coordinate 5-tuple+// [page,left,right,top,bottom] of a record's coordinate matrix, or+// (nil, false) when the record carries no usable coordinates. It reads the+// structured `_pdf_positions` key first and falls back to the legacy+// `positions` key. Callers (pdfPosRowLess) index offsets 0/1/3, so we enforce+// len(row) >= 5 here.+func firstPositionRow(r lineRecord) ([]float64, bool) {+ var raw json.RawMessage+ switch {+ case len(r.pdfPositions) > 0:+ raw = r.pdfPositions+ case len(r.positions) > 0:+ raw = r.positions+ default:+ return nil, false+ }+ var mat [][]float64+ if err := json.Unmarshal(raw, &mat); err != nil || len(mat) == 0 {+ return nil, false+ }+ row := mat[0]+ if len(row) < 5 {+ return nil, false+ }+ return row, true+}++// init registers ManualChunker under CategoryIngestion.+func init() {+ MustRegisterChunker(ComponentNameManualChunker)+}diff --git a/internal/ingestion/component/chunker/manual_pdf_cgo_test.go b/internal/ingestion/component/chunker/manual_pdf_cgo_test.gonew file mode 100644index 0000000000..0367f85dda--- /dev/null+++ b/internal/ingestion/component/chunker/manual_pdf_cgo_test.go@@ -0,0 +1,97 @@+//go:build cgo && integration++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "os"+ "path/filepath"+ "testing"++ "ragflow/internal/parser/parser"+)++// TestManualChunker_RealPDFResort proves the (page, top, left) resort operates+// on the ACTUAL records the Go PDF parser produces. This closes gap 1 from the+// review: the template integration test only feeds a .txt fixture with no+// coordinates, so nothing exercised the resort against real parser output.+//+// It parses a real PDF, normalizes its records through the SAME extractLineRecords+// path the chunker uses in production, then asserts the resort yields a+// non-decreasing physical order — i.e. the shared pdfPosRowLess comparator+// genuinely consumes the parser-emitted _pdf_positions rather than silently+// no-op'ing. The assertion is layout-independent: whether or not the reading+// order differed from the physical order, the sorted result must be monotonic.+func TestManualChunker_RealPDFResort(t *testing.T) {+ t.Setenv("DEEPDOC_URL", "")+ t.Setenv("OSSDEEPDOC_URL", "")++ path := filepath.Join("..", "..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("ReadFile(%s): %v", path, err)+ }+ pdf := parser.NewPDFParser()+ res := pdf.ParseWithResult(t.Context(), "Doc1.pdf", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) == 0 {+ t.Fatal("parser produced no JSON items")+ }++ // Same shape the chunker receives in production.+ input := map[string]any{+ "name": "Doc1.pdf",+ "output_format": "json",+ "chunks": res.JSON,+ }+ records := extractLineRecords(input)+ if len(records) == 0 {+ t.Fatal("extractLineRecords produced no records from real parser output")+ }++ positioned := 0+ for _, r := range records {+ if _, ok := firstPositionRow(r); ok {+ positioned+++ }+ }+ if positioned == 0 {+ t.Fatal("no real parser records carried _pdf_positions; resort would be a no-op")+ }++ sortRecordsByPosition(records)++ // After the resort, the physical (page, top, left) order must be+ // non-decreasing. pdfPosRowLess(cur, prev) being true would mean a record+ // physically precedes its predecessor — a resort violation. A coordinate-free+ // record (no firstPositionRow) is skipped and never becomes prev, so prev may+ // be nil until the first positioned record; guard against that.+ var prev []float64+ for i := 0; i < len(records); i++ {+ cur, ok := firstPositionRow(records[i])+ if !ok {+ continue+ }+ if prev != nil && pdfPosRowLess(cur, prev) {+ t.Fatalf("resort produced out-of-order records at %d: row %v precedes %v", i, cur, prev)+ }+ prev = cur+ }+}diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.gonew file mode 100644index 0000000000..5fb2f9dd5f--- /dev/null+++ b/internal/ingestion/component/chunker/manual_test.go@@ -0,0 +1,431 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "encoding/json"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/agent/runtime"+)++// manualChunkerInput is a small builder for a structured (output_format=chunks)+// upstream payload, mirroring the shape the Parser emits for PDF/JSON output.+func manualChunkerInput(name string, items ...map[string]any) map[string]any {+ return map[string]any{+ "name": name,+ "output_format": "chunks",+ "chunks": items,+ }+}++// posItem builds a chunk item carrying a PDF position matrix+// [page,left,right,top,bottom] under the `_pdf_positions` key (matching+// schema.ChunkDoc.PDFPositions).+func posItem(text string, page, left, right, top, bottom float64) map[string]any {+ return map[string]any{+ "text": text,+ "doc_type_kwd": "text",+ "_pdf_positions": [][]float64{{page, left, right, top, bottom}},+ }+}++// mustPosJSON marshals a single 5-tuple position matrix into the+// json.RawMessage form used by lineRecord.pdfPositions / positions.+func mustPosJSON(t *testing.T, page, left, right, top, bottom float64) json.RawMessage {+ t.Helper()+ b, err := json.Marshal([][]float64{{page, left, right, top, bottom}})+ if err != nil {+ t.Fatalf("marshal positions: %v", err)+ }+ return b+}++func plainItem(text string) map[string]any {+ return map[string]any{"text": text, "doc_type_kwd": "text"}+}++func mustManual(t *testing.T, params map[string]any) *ManualChunkerComponent {+ t.Helper()+ c, err := NewManualChunker(params)+ if err != nil {+ t.Fatalf("NewManualChunker: %v", err)+ }+ mc, ok := c.(*ManualChunkerComponent)+ if !ok {+ t.Fatalf("NewManualChunker returned %T, want *ManualChunkerComponent", c)+ }+ return mc+}++// manualChunkTexts pulls the ordered chunk texts out of a chunker output map.+func manualChunkTexts(out map[string]any) []string {+ chunks, _ := out["chunks"].([]map[string]any)+ out2 := make([]string, 0, len(chunks))+ for _, ck := range chunks {+ if s, _ := ck["text"].(string); s != "" {+ out2 = append(out2, s)+ }+ }+ return out2+}++// TestManualChunker_Registered pins that the component is discoverable via the+// runtime registry under the CategoryIngestion category with non-empty+// input/output metadata.+func TestManualChunker_Registered(t *testing.T) {+ factory, cat, meta, ok := runtime.DefaultRegistry.Lookup(ComponentNameManualChunker)+ if !ok {+ t.Fatal("ManualChunker: registry miss")+ }+ if cat != runtime.CategoryIngestion {+ t.Errorf("category = %q, want %q", cat, runtime.CategoryIngestion)+ }+ if factory == nil {+ t.Error("factory is nil")+ }+ if len(meta.Inputs) == 0 {+ t.Error("inputs metadata empty")+ }+ if len(meta.Outputs) == 0 {+ t.Error("outputs metadata empty")+ }+}++// TestManualChunker_Empty is the trivial boundary: no records => empty outputs.+func TestManualChunker_Empty(t *testing.T) {+ c := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "doc"})+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got, want := out["output_format"], "chunks"; got != want {+ t.Errorf("output_format = %v, want %v", got, want)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 0 {+ t.Errorf("chunks = %d, want 0", len(chunks))+ }+}++// TestManualChunker_NoPositionsEqualsGroupChunker is the regression lock: when+// the upstream payload carries NO physical coordinates (e.g. docx, which Go+// emits in document-logical order — exactly as Python's manual docx branch,+// which performs no coordinate resort), ManualChunker MUST behave bit-for-bit+// like GroupTitleChunker. Any divergence here is an unintended regression.+func TestManualChunker_NoPositionsEqualsGroupChunker(t *testing.T) {+ var mkLevels = [][]string{{`^# `}, {`^## `}}+ input := manualChunkerInput("doc",+ plainItem("# H1"),+ plainItem("body under h1 a"),+ plainItem("body under h1 b"),+ plainItem("## H2"),+ plainItem("body under h2"),+ plainItem("more body under h2"),+ )++ gc, err := NewGroupTitleChunker(map[string]any{"levels": mkLevels})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ gOut, err := gc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("group Invoke: %v", err)+ }++ mc := mustManual(t, map[string]any{"levels": mkLevels})+ mOut, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("manual Invoke: %v", err)+ }++ gChunks, _ := gOut["chunks"].([]map[string]any)+ mChunks, _ := mOut["chunks"].([]map[string]any)+ if !reflect.DeepEqual(gChunks, mChunks) {+ t.Fatalf("ManualChunker output diverges from GroupTitleChunker on no-coordinate input\n group: %#v\n manual: %#v", gChunks, mChunks)+ }+}++// TestManualChunker_SortsByPageTopLeft is the core behavioural difference+// vs. GroupTitleChunker. For a multi-column / manual PDF, the parser emits+// records in READING order (column-by-column), not in PHYSICAL/top-down+// order. ManualChunker must resort records by (page, top, left) before+// grouping so that, e.g., the left column's content precedes the right+// column's when they share a page band.+func TestManualChunker_SortsByPageTopLeft(t *testing.T) {+ // page 1, all body (no heading markers). Physical layout:+ // top=100, left=10 -> "left col top"+ // top=100, left=400 -> "right col"+ // top=300, left=10 -> "left col bottom"+ // top=400, left=200 -> "footer title"+ // Parser reading order is scrambled relative to physical order.+ input := manualChunkerInput("doc",+ posItem("footer title", 1, 200, 400, 400, 500),+ posItem("left col bottom", 1, 10, 200, 300, 400),+ posItem("right col", 1, 400, 600, 100, 400),+ posItem("left col top", 1, 10, 200, 100, 200),+ )++ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ out, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }++ // Whether merged into one chunk or several, the concatenated text MUST+ // follow physical order: left col top, right col, left col bottom,+ // footer title.+ var full strings.Builder+ for _, s := range manualChunkTexts(out) {+ full.WriteString(s)+ full.WriteString("\n")+ }+ joined := full.String()+ wantOrder := []string{"left col top", "right col", "left col bottom", "footer title"}+ last := -1+ for _, w := range wantOrder {+ idx := strings.Index(joined, w)+ if idx < 0 {+ t.Fatalf("expected %q to appear in manual output %q", w, joined)+ }+ if idx <= last {+ t.Fatalf("physical order violated: %q appears after a later element in %q", w, joined)+ }+ last = idx+ }+}++// TestManualChunker_SortDiffersFromGroupWhenPositionsPresent proves the resort+// actually fires (and is not a no-op): the same scrambled input must produce a+// DIFFERENT text order from GroupTitleChunker, which groups in input order.+func TestManualChunker_SortDiffersFromGroupWhenPositionsPresent(t *testing.T) {+ input := manualChunkerInput("doc",+ posItem("parser-reads-2nd-col-first", 1, 400, 600, 100, 400),+ posItem("parser-reads-1st-col-first", 1, 10, 200, 100, 200),+ )++ gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ gOut, err := gc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("group Invoke: %v", err)+ }+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ mOut, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("manual Invoke: %v", err)+ }+ if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) {+ t.Fatalf("ManualChunker did not reorder relative to GroupTitleChunker; both = %#v", mOut["chunks"])+ }+ // ManualChunker must place the 1st-column item first.+ if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "1st-col") {+ t.Fatalf("expected 1st-column text first in manual output, got %#v", texts)+ }+}++// TestManualChunker_MergesPositions pins parity with GroupTitleChunker: when+// adjacent text records are merged into one chunk, the PDF position matrices+// are MERGED across the merged records (not dropped), and parser position tags+// are stripped from the text.+func TestManualChunker_MergesPositions(t *testing.T) {+ input := manualChunkerInput("doc",+ posItem("body one", 1, 10, 20, 30, 40),+ posItem("body two", 2, 15, 25, 35, 45),+ )+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ out, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ found := false+ for _, ck := range chunks {+ text, _ := ck["text"].(string)+ if !strings.Contains(text, "body one") || !strings.Contains(text, "body two") {+ continue+ }+ found = true+ if strings.Contains(text, "@@") {+ t.Errorf("parser position tags leaked into chunk text: %q", text)+ }+ pos, ok := ck["_pdf_positions"].([][]float64)+ if !ok {+ t.Fatalf("_pdf_positions missing or wrong type %T on merged group chunk", ck["_pdf_positions"])+ }+ if len(pos) != 2 {+ // Hard fail: the page/top/left checks below would index out of+ // range if the merged matrix does not have exactly two rows.+ t.Fatalf("merged _pdf_positions = %d rows, want 2 (both records)", len(pos))+ }+ // Merged matrix must itself be sorted by (page, top, left) — the+ // same key pdfPosRowLess uses when merging.+ if pdfPosRowLess(pos[1], pos[0]) {+ t.Errorf("merged _pdf_positions not sorted by (page, top, left): %v", pos)+ }+ }+ if !found {+ t.Fatal("merged body group chunk not found in output")+ }+}++// TestSortRecordsByPosition_Unit pins the comparator directly: (page, top,+// left) ascending, stable for equal keys, and records without positions keep+// their original relative order (so a no-coordinate payload is untouched).+func TestSortRecordsByPosition_Unit(t *testing.T) {+ recs := []lineRecord{+ {text: "a", pdfPositions: mustPosJSON(t, 1, 200, 300, 400, 500)},+ {text: "b", pdfPositions: mustPosJSON(t, 1, 10, 100, 300, 400)},+ {text: "c", pdfPositions: mustPosJSON(t, 1, 10, 100, 100, 200)},+ {text: "d"}, // no position+ }+ sortRecordsByPosition(recs)+ got := []string{recs[0].text, recs[1].text, recs[2].text, recs[3].text}+ want := []string{"c", "b", "a", "d"}+ if !reflect.DeepEqual(got, want) {+ t.Errorf("sorted order = %v, want %v", got, want)+ }+}++// TestHasPdfPositions is a tiny white-box check for the gate that decides+// whether a resort is performed at all.+func TestHasPdfPositions(t *testing.T) {+ if hasPdfPositions([]lineRecord{{text: "x"}}) {+ t.Error("plain record should report no positions")+ }+ if !hasPdfPositions([]lineRecord{{text: "x", pdfPositions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) {+ t.Error("record with pdf_positions should report positions")+ }+ if !hasPdfPositions([]lineRecord{{text: "x", positions: mustPosJSON(t, 1, 1, 1, 1, 1)}}) {+ t.Error("record with positions should report positions")+ }+}++// parserShapedItem builds a chunk item that mirrors EXACTLY the JSON shape the+// Go PDF parser emits via pdfParseResultToJSON (see+// TestPDFParseResultToJSON_NormalizesCoreFields): _pdf_positions is a [][]any+// with 1-based page numbers, accompanied by layout / page_number /+// doc_type_kwd. Using [][]any — not [][]float64 — matters: that is the concrete+// in-memory type the parser produces, and recordsFromStructured must consume it+// after the chunksFromInputs JSON round-trip.+func parserShapedItem(text string, page, left, right, top, bottom float64) map[string]any {+ return map[string]any{+ "text": text,+ "doc_type_kwd": "text",+ "layout": "text",+ "page_number": int(page),+ "_pdf_positions": [][]any{{page, left, right, top, bottom}},+ }+}++// legacyPosItem is like posItem but uses the legacy `positions` key instead of+// the structured `_pdf_positions` key.+func legacyPosItem(text string, page, left, right, top, bottom float64) map[string]any {+ return map[string]any{+ "text": text,+ "doc_type_kwd": "text",+ "positions": [][]float64{{page, left, right, top, bottom}},+ }+}++// TestManualChunker_RealParserShapedPayload exercises the EXACT payload shape+// the Go PDF parser delivers to the chunker (pdfParseResultToJSON ->+// recordsFromStructured). It closes the coverage gap where the template+// integration test only feeds a .txt fixture (no coordinates): here a+// coordinate-bearing, parser-shaped multi-column doc proves the (page, top,+// left) resort actually fires and produces a different order from+// GroupTitleChunker (which groups in parser reading order).+func TestManualChunker_RealParserShapedPayload(t *testing.T) {+ input := manualChunkerInput("doc",+ // Parser reading order (column-by-column) is scrambled vs physical.+ parserShapedItem("right column, read first", 1, 400, 600, 100, 400),+ parserShapedItem("left column, read second", 1, 10, 200, 100, 200),+ )++ gc, _ := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ gOut, err := gc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("group Invoke: %v", err)+ }+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ mOut, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("manual Invoke: %v", err)+ }+ if reflect.DeepEqual(gOut["chunks"], mOut["chunks"]) {+ t.Fatalf("ManualChunker did not reorder parser-shaped payload; both = %#v", mOut["chunks"])+ }+ // Physical (page, top, left) order must put the left column first.+ if texts := manualChunkTexts(mOut); len(texts) == 0 || !strings.Contains(texts[0], "left column") {+ t.Fatalf("expected left-column text first after resort, got %#v", texts)+ }+}++// TestManualChunker_LegacyPositionsKeyTriggersSort pins that the resort fires+// when records carry the LEGACY `positions` key rather than the structured+// `_pdf_positions` key. firstPositionRow (and thus the shared pdfPosRowLess+// comparator) must read both forms, otherwise a payload emitted with the old+// key would silently skip the resort.+func TestManualChunker_LegacyPositionsKeyTriggersSort(t *testing.T) {+ input := manualChunkerInput("doc",+ legacyPosItem("right column, read first", 1, 400, 600, 100, 400),+ legacyPosItem("left column, read second", 1, 10, 200, 100, 200),+ )+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ out, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ // Without the legacy-key path the output would keep parser reading order+ // (right column before left). The resort must place the left column first.+ if texts := manualChunkTexts(out); len(texts) == 0 || !strings.Contains(texts[0], "left column") {+ t.Fatalf("legacy `positions` key did not trigger resort; got %#v", texts)+ }+}++// TestManualChunker_MixedPositionedAndPlain exercises the boundary where a+// payload mixes coordinate-bearing records with coordinate-free records. A+// single document type will not usually do this (production payloads are+// homogeneously positioned-or-plain), but the sort must not corrupt order when+// it happens: the contiguous positioned run reorders by (page, top, left) and+// coordinate-free records keep their ORIGINAL relative order. (Go's stable+// sort only reorders within a contiguous positioned run — a coordinate-free+// record between two positioned records correctly stops the reorder from+// crossing it, which is why the positioned records here are kept adjacent.)+func TestManualChunker_MixedPositionedAndPlain(t *testing.T) {+ input := manualChunkerInput("doc",+ plainItem("plain A"),+ posItem("right col", 1, 400, 600, 100, 400),+ posItem("left col", 1, 10, 200, 100, 200),+ plainItem("plain B"),+ )+ mc := mustManual(t, map[string]any{"levels": [][]string{{`^# `}}})+ out, err := mc.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ joined := strings.Join(manualChunkTexts(out), "\n")+ // The contiguous positioned run keeps physical order relative to itself.+ if i, j := strings.Index(joined, "left col"), strings.Index(joined, "right col"); i < 0 || j < 0 || i > j {+ t.Fatalf("positioned records not in physical order in %q", joined)+ }+ // Coordinate-free records keep their original relative order.+ if i, j := strings.Index(joined, "plain A"), strings.Index(joined, "plain B"); i < 0 || j < 0 || i > j {+ t.Fatalf("coordinate-free records lost original order in %q", joined)+ }+}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.jsonindex e83e0d17e5..1faad567e9 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_manual.json@@ -26,7 +26,7 @@},"Parser:HipSignsRhyme": {"downstream": [- "TitleChunker:NineInsectsFind"+ "ManualChunker:NineInsectsFind"],"obj": {"component_name": "Parser",@@ -87,12 +87,12 @@"File"]},- "TitleChunker:NineInsectsFind": {+ "ManualChunker:NineInsectsFind": {"downstream": ["Extractor:AutoExtractDefault"],"obj": {- "component_name": "TitleChunker",+ "component_name": "ManualChunker","params": {"hierarchy": 0,"include_heading_content": false,@@ -158,7 +158,7 @@},"Extractor:AutoExtractDefault": {"upstream": [- "TitleChunker:NineInsectsFind"+ "ManualChunker:NineInsectsFind"],"downstream": ["Tokenizer:FunnyBalloonsGrin"@@ -197,15 +197,15 @@"targetHandle": "end"},{- "id": "xy-edge__Parser:HipSignsRhymestart-TitleChunker:NineInsectsFindend",+ "id": "xy-edge__Parser:HipSignsRhymestart-ManualChunker:NineInsectsFindend","source": "Parser:HipSignsRhyme","sourceHandle": "start",- "target": "TitleChunker:NineInsectsFind",+ "target": "ManualChunker:NineInsectsFind","targetHandle": "end"},{- "id": "xy-edge__TitleChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend",- "source": "TitleChunker:NineInsectsFind",+ "id": "xy-edge__ManualChunker:NineInsectsFindstart-Extractor:AutoExtractDefaultend",+ "source": "ManualChunker:NineInsectsFind","sourceHandle": "start","target": "Extractor:AutoExtractDefault","targetHandle": "end"@@ -407,10 +407,10 @@}]},- "label": "TitleChunker",- "name": "Title Chunker_0"+ "label": "ManualChunker",+ "name": "Manual Chunker_0"},- "id": "TitleChunker:NineInsectsFind",+ "id": "ManualChunker:NineInsectsFind","measured": {"height": 74,"width": 200diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex aa42b84cf1..01daffa143 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -424,9 +424,9 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)}}- chunkerState, ok := state["TitleChunker:NineInsectsFind"]+ chunkerState, ok := state["ManualChunker:NineInsectsFind"]if !ok {- t.Fatal("missing TitleChunker:NineInsectsFind state")+ t.Fatal("missing ManualChunker:NineInsectsFind state")}chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)if !ok || len(chunkerChunks) != len(wantChunkTexts) {diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex 17459f27d4..b4148ed099 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -43,7 +43,7 @@ var builtinComponentParamsGolden = map[string]string{"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",- "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",+ "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","paper": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"pdf\": {\"enable_multi_column\": true, \"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:SparklySchoolsTravel\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:GreatCarsWash\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}","picture": "{\"File\": {}, \"Parser:ViewsCaptureLight\": {\"image\": {\"output_format\": \"json\", \"parse_method\": \"ocr\", \"preprocess\": [\"main_content\"], \"suffix\": [\"bmp\", \"gif\", \"jpeg\", \"jpg\", \"png\", \"svg\", \"tif\", \"tiff\", \"webp\"]}, \"video\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"3gp\", \"3gpp\", \"avi\", \"flv\", \"mkv\", \"mov\", \"mp4\", \"mpeg\", \"mpg\", \"webm\", \"wmv\"]}}, \"TokenChunker:BrightColorsGlow\": {}, \"Tokenizer:SharpLensFocus\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}",diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.goindex 1466fac979..41b4c284d3 100644--- a/internal/service/components_service_test.go+++ b/internal/service/components_service_test.go@@ -62,7 +62,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {t.Fatalf("List(Ingestion) returned error: %v", err)}wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",}assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)@@ -85,7 +85,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {t.Fatalf("List(Ingestion,Shared) returned error: %v", err)}wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",}assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)@@ -104,7 +104,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {t.Fatalf("List(Ingestion x3) returned error: %v", err)}wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker", "titlechunker", "tokenchunker", "tokenizer",}if len(got) != len(wantNames) {