Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): pipeline cleanups (#21084)

xugangqiang14cf685b80b7
diff --git a/internal/ingestion/component/chunker/delim_compile_guard_test.go b/internal/ingestion/component/chunker/delim_compile_guard_test.go
new file mode 100644
index 0000000000..3f51bcb0fd
--- /dev/null
+++ b/internal/ingestion/component/chunker/delim_compile_guard_test.go
@@ -0,0 +1,124 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "testing"
+
+ "ragflow/internal/parser/chunk"
+)
+
+// TestCompileDelimPatternContract pins the shared delimiter-compiler contract
+// that both the main Delimiters list and the children_delimiters list go
+// through (compileChildrenPattern was merged into compileDelimPattern because
+// the two were byte-identical). The assertions follow
+// chunk.CompileDelimiterPatternList: empty entries are inactive, backtick
+// entries contribute their inner content, bare entries stay active, duplicates
+// collapse, and the alternation is longest-first and regex-escaped.
+func TestCompileDelimPatternContract(t *testing.T) {
+ cases := []struct {
+ name string
+ delims []string
+ want string // expected pattern string; "" means nil pattern
+ }{
+ {"nil list", nil, ""},
+ {"empty list", []string{}, ""},
+ {"only empty entry", []string{""}, ""},
+ {"only empty backticks", []string{"``"}, ""},
+ {"lone backtick is bare", []string{"`"}, "`"},
+ {"single bare", []string{"a"}, "a"},
+ {"backtick inner", []string{"`end`"}, "end"},
+ {"bare duplicates collapse", []string{"a", "a"}, "a"},
+ {"backtick and bare dedupe", []string{"`a`", "a"}, "a"},
+ {"empty among actives", []string{"", "b", ""}, "b"},
+ {"longest first", []string{"ab", "`abc`"}, `abc|ab`},
+ {"regex metacharacters escaped", []string{"a.b"}, `a\.b`},
+ {"unicode delimiters", []string{"。", ","}, `。|,`},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ got := compileDelimPattern(c.delims)
+ if c.want == "" {
+ if got != nil {
+ t.Fatalf("compileDelimPattern(%q) = %q, want nil", c.delims, got.String())
+ }
+ return
+ }
+ if got == nil {
+ t.Fatalf("compileDelimPattern(%q) = nil, want %q", c.delims, c.want)
+ }
+ if got.String() != c.want {
+ t.Fatalf("compileDelimPattern(%q) = %q, want %q", c.delims, got.String(), c.want)
+ }
+ })
+ }
+}
+
+// TestCompileDelimPatternMatchesLongestFirst guards the alternation order in
+// action: on overlapping text the longest delimiter must win, otherwise a
+// shorter prefix would silently shadow the longer delimiter.
+func TestCompileDelimPatternMatchesLongestFirst(t *testing.T) {
+ pattern := compileDelimPattern([]string{"ab", "`abc`"})
+ if pattern == nil {
+ t.Fatal("compileDelimPattern returned nil")
+ }
+ got := pattern.FindAllString("xxabcyy", -1)
+ if len(got) != 1 || got[0] != "abc" {
+ t.Fatalf("matches on \"xxabcyy\" = %v, want [abc] (longest first)", got)
+ }
+}
+
+// TestCompileDelimPatternIsCaseSensitive: the compiler must not emit (?i); the
+// chunker's delimiter matching is case-sensitive by contract.
+func TestCompileDelimPatternIsCaseSensitive(t *testing.T) {
+ pattern := compileDelimPattern([]string{"`end`"})
+ if pattern == nil {
+ t.Fatal("compileDelimPattern(`end`) returned nil")
+ }
+ if pattern.MatchString("END") {
+ t.Error("pattern matched uppercase END; delimiter matching must be case-sensitive")
+ }
+ if !pattern.MatchString("the end") {
+ t.Error("pattern did not match lowercase end")
+ }
+}
+
+// TestCompileDelimPatternDelegatesToChunk pins that the wrapper stays a thin
+// delegation to chunk.CompileDelimiterPatternList(..., keepBare=true): any
+// divergence would silently fork children_delimiters behaviour from the main
+// delimiter list.
+func TestCompileDelimPatternDelegatesToChunk(t *testing.T) {
+ inputs := [][]string{
+ nil,
+ {},
+ {""},
+ {"a"},
+ {"`end`", "b"},
+ {"。", ",", "`;`"},
+ {"a.b", "c*d"},
+ }
+ for _, delims := range inputs {
+ got := compileDelimPattern(delims)
+ want := chunk.CompileDelimiterPatternList(delims, true)
+ if (got == nil) != (want == nil) {
+ t.Fatalf("compileDelimPattern(%q) nil-ness = %v, want %v", delims, got == nil, want == nil)
+ }
+ if got != nil && got.String() != want.String() {
+ t.Fatalf("compileDelimPattern(%q) = %q, want %q", delims, got.String(), want.String())
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 5c61decece..a67709148c 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -231,7 +231,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
}
primaryPattern := compileDelimPattern(c.param.Delimiters)
- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
customDelimiter := hasCustomDelim(c.param.Delimiters)
if customDelimiter {
units = splitGeneralUnits(units, primaryPattern, false)
@@ -402,7 +402,7 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
return emptyOutputs(), nil
}
primaryPattern := compileDelimPattern(c.param.Delimiters)
- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
customDelimiter := hasCustomDelim(c.param.Delimiters)
if !customDelimiter {
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
@@ -627,7 +627,7 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream sc
return emptyOutputs(), nil
}
primaryPattern := compileDelimPattern(c.param.Delimiters)
- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
customDelimiter := hasCustomDelim(c.param.Delimiters)
if !customDelimiter {
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
@@ -969,7 +969,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
}
chunks = append(chunks, cloneChunkDoc(unit))
}
- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
chunks = finalizeGeneralChunks(chunks, childrenPattern)
if len(chunks) == 0 {
return emptyOutputs(), nil
@@ -1065,7 +1065,7 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch
return emptyOutputs(), nil
}
primaryPattern := compileDelimPattern(c.param.Delimiters)
- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
customDelimiter := hasCustomDelim(c.param.Delimiters)
units = splitGeneralUnits(units, primaryPattern, !customDelimiter)
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
diff --git a/internal/ingestion/component/chunker/general_positions_slice_test.go b/internal/ingestion/component/chunker/general_positions_slice_test.go
index 489e0bc964..e1f1c6b87e 100644
--- a/internal/ingestion/component/chunker/general_positions_slice_test.go
+++ b/internal/ingestion/component/chunker/general_positions_slice_test.go
@@ -53,7 +53,7 @@ func TestSplitGeneralChildrenSlicesPositionsAbut(t *testing.T) {
PDFPositions: json.RawMessage(`[[1,10,20,0,100]]`),
Positions: json.RawMessage(`[[1,10,20,0,100]]`),
}
- pattern := compileChildrenPattern([]string{";"})
+ pattern := compileDelimPattern([]string{";"})
got := splitGeneralChildren([]schema.ChunkDoc{chunk}, pattern)
if len(got) != 2 {
t.Fatalf("children pieces = %d, want 2", len(got))
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 4703f186b1..68cf9c0e4e 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -193,7 +193,7 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs
}
delimPattern := compileDelimPattern(c.param.Delimiters)
- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
switch upstream.OutputFormat {
case schema.PayloadFormatMarkdown:
@@ -1828,18 +1828,6 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp, keep
return out
}
-// compileChildrenPattern is the children_delimiters version of
-// compileDelimPattern. Returns nil when no delimiters exist.
-// compileChildrenPattern builds the children-split regex from a
-// `children_delimiters` list. Every non-empty entry is active (including bare
-// ones), and backtick-wrapped entries contribute their inner content — see
-// chunk.CompileDelimiterPatternList. Delegating keeps children splitting
-// consistent with the main delimiter list (backtick stripping + rune-descending
-// order) instead of re-implementing a divergent copy.
-func compileChildrenPattern(delims []string) *regexp.Regexp {
- return chunk.CompileDelimiterPatternList(delims, true)
-}
-
func intValue(v *int) int {
if v == nil {
return 0
diff --git a/internal/ingestion/component/document_storage.go b/internal/ingestion/component/document_storage.go
index 844acddcd4..cff4610c47 100644
--- a/internal/ingestion/component/document_storage.go
+++ b/internal/ingestion/component/document_storage.go
@@ -49,25 +49,13 @@ func FetchBinary(ctx context.Context, bucket, path string) ([]byte, error) {
if storageImpl == nil {
return nil, fmt.Errorf("no storage backend registered")
}
-
- type result struct {
- data []byte
- err error
- }
- done := make(chan result, 1)
- go func() {
- data, err := storageImpl.Get(ctx, bucket, path)
- done <- result{data: data, err: err}
- }()
- select {
- case <-ctx.Done():
- return nil, ctx.Err()
- case r := <-done:
- if r.err != nil {
- return nil, fmt.Errorf("storage.Get(%q, %q): %w", bucket, path, r.err)
- }
- return r.data, nil
+ // The storage layer takes ctx and honors it, so a cancellation surfaces
+ // through Get itself; no extra goroutine is needed to abort early.
+ data, err := storageImpl.Get(ctx, bucket, path)
+ if err != nil {
+ return nil, fmt.Errorf("storage.Get(%q, %q): %w", bucket, path, err)
}
+ return data, nil
}
func resolveStorage() storage.Storage {
diff --git a/internal/ingestion/component/document_storage_test.go b/internal/ingestion/component/document_storage_test.go
new file mode 100644
index 0000000000..8277d82650
--- /dev/null
+++ b/internal/ingestion/component/document_storage_test.go
@@ -0,0 +1,235 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/storage"
+)
+
+// fetchSpyStorage overrides only Get; every other Storage method delegates to
+// the embedded implementation (an in-memory one) so the spy stays small.
+type fetchSpyStorage struct {
+ storage.Storage
+ get func(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error)
+}
+
+func (s *fetchSpyStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
+ return s.get(ctx, bucket, fnm, tenantID...)
+}
+
+// withFetchStorage installs s as the storage backend for one test.
+func withFetchStorage(t *testing.T, s storage.Storage) {
+ t.Helper()
+ prev := SetStorageFactoryOverride
+ SetStorageFactoryOverride = func() storage.Storage { return s }
+ t.Cleanup(func() { SetStorageFactoryOverride = prev })
+}
+
+func TestFetchBinaryReturnsObjectAndForwardsContext(t *testing.T) {
+ payload := []byte("pdf-bytes-\x00\x01\xff")
+ type seen struct {
+ bucket, fnm string
+ ctx context.Context
+ }
+ var got seen
+ withFetchStorage(t, &fetchSpyStorage{
+ Storage: storage.NewMemoryStorage(),
+ get: func(ctx context.Context, bucket, fnm string, _ ...string) ([]byte, error) {
+ got = seen{bucket: bucket, fnm: fnm, ctx: ctx}
+ return payload, nil
+ },
+ })
+
+ ctx := context.Background()
+ data, err := FetchBinary(ctx, "kb-1", "docs/a.pdf")
+ if err != nil {
+ t.Fatalf("FetchBinary: %v", err)
+ }
+ if string(data) != string(payload) {
+ t.Fatalf("FetchBinary returned %q, want %q", data, payload)
+ }
+ if got.bucket != "kb-1" || got.fnm != "docs/a.pdf" {
+ t.Fatalf("Get called with (%q, %q), want (kb-1, docs/a.pdf)", got.bucket, got.fnm)
+ }
+ // FetchBinary must hand the caller's ctx to the storage layer so a
+ // cancellation reaches the in-flight download.
+ if got.ctx != ctx {
+ t.Error("FetchBinary did not forward the caller's ctx to storage.Get")
+ }
+}
+
+func TestFetchBinaryWrapsStorageError(t *testing.T) {
+ sentinel := errors.New("object missing")
+ withFetchStorage(t, &fetchSpyStorage{
+ Storage: storage.NewMemoryStorage(),
+ get: func(_ context.Context, _, _ string, _ ...string) ([]byte, error) {
+ return nil, sentinel
+ },
+ })
+
+ _, err := FetchBinary(context.Background(), "kb-1", "docs/missing.pdf")
+ if err == nil {
+ t.Fatal("FetchBinary succeeded despite a storage error")
+ }
+ if !errors.Is(err, sentinel) {
+ t.Fatalf("error %v does not wrap the storage error", err)
+ }
+ // The wrap must identify the object so operators can find it in logs.
+ for _, want := range []string{`"kb-1"`, `"docs/missing.pdf"`} {
+ if !strings.Contains(err.Error(), want) {
+ t.Errorf("error %q does not mention %q", err.Error(), want)
+ }
+ }
+}
+
+func TestFetchBinaryAlreadyCanceledContext(t *testing.T) {
+ called := false
+ withFetchStorage(t, &fetchSpyStorage{
+ Storage: storage.NewMemoryStorage(),
+ get: func(ctx context.Context, _, _ string, _ ...string) ([]byte, error) {
+ called = true
+ return nil, ctx.Err()
+ },
+ })
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, err := FetchBinary(ctx, "kb-1", "docs/a.pdf")
+ if err == nil {
+ t.Fatal("FetchBinary succeeded with a canceled ctx")
+ }
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("error %v does not wrap context.Canceled", err)
+ }
+ if !called {
+ t.Error("storage.Get was never reached; the cancellation must surface through it")
+ }
+}
+
+// TestFetchBinaryCancelDuringDownload pins that a cancellation while the
+// download is in flight surfaces promptly instead of waiting for the whole
+// object. The spy models a storage layer that honors ctx, as the real backends
+// do — which is why FetchBinary needs no goroutine of its own to abort early.
+func TestFetchBinaryCancelDuringDownload(t *testing.T) {
+ entered := make(chan struct{})
+ withFetchStorage(t, &fetchSpyStorage{
+ Storage: storage.NewMemoryStorage(),
+ get: func(ctx context.Context, _, _ string, _ ...string) ([]byte, error) {
+ close(entered)
+ <-ctx.Done()
+ return nil, ctx.Err()
+ },
+ })
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ <-entered
+ cancel()
+ }()
+
+ start := time.Now()
+ _, err := FetchBinary(ctx, "kb-1", "docs/slow.pdf")
+ elapsed := time.Since(start)
+ if err == nil || !errors.Is(err, context.Canceled) {
+ t.Fatalf("FetchBinary returned %v, want a context.Canceled wrap", err)
+ }
+ if elapsed > 2*time.Second {
+ t.Fatalf("FetchBinary took %v to observe the cancellation; it must not wait for the download", elapsed)
+ }
+}
+
+func TestFetchBinaryLargePayload(t *testing.T) {
+ payload := make([]byte, 4<<20) // 4 MiB
+ for i := range payload {
+ payload[i] = byte(i)
+ }
+ withFetchStorage(t, &fetchSpyStorage{
+ Storage: storage.NewMemoryStorage(),
+ get: func(_ context.Context, _, _ string, _ ...string) ([]byte, error) {
+ out := make([]byte, len(payload))
+ copy(out, payload)
+ return out, nil
+ },
+ })
+
+ data, err := FetchBinary(context.Background(), "kb-1", "docs/big.pdf")
+ if err != nil {
+ t.Fatalf("FetchBinary: %v", err)
+ }
+ if len(data) != len(payload) {
+ t.Fatalf("FetchBinary returned %d bytes, want %d", len(data), len(payload))
+ }
+ for i := range data {
+ if data[i] != payload[i] {
+ t.Fatalf("payload differs at byte %d: got %d, want %d", i, data[i], payload[i])
+ }
+ }
+}
+
+func TestFetchBinaryConcurrentCalls(t *testing.T) {
+ payload := []byte("shared-object")
+ withFetchStorage(t, &fetchSpyStorage{
+ Storage: storage.NewMemoryStorage(),
+ get: func(_ context.Context, _, _ string, _ ...string) ([]byte, error) {
+ return payload, nil
+ },
+ })
+
+ const callers = 16
+ var wg sync.WaitGroup
+ errs := make([]error, callers)
+ for i := 0; i < callers; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ data, err := FetchBinary(context.Background(), "kb-1", "docs/shared.pdf")
+ if err == nil && string(data) != string(payload) {
+ err = fmt.Errorf("got %q, want %q", data, payload)
+ }
+ errs[i] = err
+ }(i)
+ }
+ wg.Wait()
+ for i, err := range errs {
+ if err != nil {
+ t.Errorf("caller %d: %v", i, err)
+ }
+ }
+}
+
+func TestFetchBinaryWithoutStorageBackend(t *testing.T) {
+ if storage.GetStorageFactory().GetStorage() != nil {
+ t.Skip("a global storage backend is registered in this test binary; the nil-backend path cannot be observed")
+ }
+ withFetchStorage(t, nil)
+
+ _, err := FetchBinary(context.Background(), "kb-1", "docs/a.pdf")
+ if err == nil {
+ t.Fatal("FetchBinary succeeded without any storage backend")
+ }
+ if !strings.Contains(err.Error(), "no storage backend registered") {
+ t.Fatalf("error %q does not mention the missing storage backend", err.Error())
+ }
+}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index 38a3b4f775..8bfa54ca07 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -691,14 +691,24 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
}
}
-// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family default
-// output_format must be "json" (matching Python parser.py:212), not "text".
+// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family output
+// format must be "json" (matching Python parser.py:212), not "text".
// With "text" the structured email fields (from/to/subject/attachments/...) are
-// flattened into a blob and lost downstream.
+// flattened into a blob and lost downstream. The value is owned by
+// normalizeParserOutputFormats, so the assertion goes through the constructed
+// component rather than the raw defaults.
func TestDefaultEmailOutputFormatIsJSON(t *testing.T) {
- got, _ := defaultSetups()["email"]["output_format"].(string)
+ comp, err := NewParserComponent(nil)
+ if err != nil {
+ t.Fatalf("NewParserComponent: %v", err)
+ }
+ pc, ok := comp.(*ParserComponent)
+ if !ok {
+ t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)
+ }
+ got, _ := pc.setups["email"]["output_format"].(string)
if got != "json" {
- t.Fatalf("email default output_format = %q, want json", got)
+ t.Fatalf("email output_format = %q, want json", got)
}
}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 88c8fd0307..6282cad78d 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -299,58 +299,55 @@ func imageSetupRunsOCR(setup schema.ParserSetup) bool {
}
func defaultSetups() map[string]schema.ParserSetup {
- return map[string]schema.ParserSetup{
+ // Normalize up front so any caller (including tests that build a
+ // ParserComponent directly from these defaults) gets the single JSON
+ // output protocol. NewParserComponent re-normalizes idempotently; the
+ // per-family literals were removed because normalizeParserOutputFormats
+ // owns this key unconditionally.
+ s := map[string]schema.ParserSetup{
"pdf": {
"parse_method": "deepdoc",
"flatten_media_to_text": false,
"remove_toc": false,
"remove_header_footer": false,
"suffix": []string{"pdf"},
- "output_format": "json",
},
"spreadsheet": {
"parse_method": "deepdoc",
"flatten_media_to_text": false,
"html4excel": false,
- "output_format": "json",
"suffix": []string{"xls", "xlsx", "csv"},
},
"doc": {
"remove_toc": false,
"remove_header_footer": false,
"suffix": []string{"doc"},
- "output_format": "json",
},
"docx": {
"flatten_media_to_text": false,
"remove_toc": false,
"remove_header_footer": false,
"suffix": []string{"docx"},
- "output_format": "json",
},
"markdown": {
"flatten_media_to_text": false,
"suffix": []string{"md", "markdown", "mdx"},
"remove_toc": false,
- "output_format": "json",
},
"text&code": {
"suffix": []string{
"txt", "py", "js", "java", "c", "cpp", "h", "php",
"go", "ts", "sh", "cs", "kt", "sql",
},
- "output_format": "json",
},
"html": {
"suffix": []string{"htm", "html"},
"remove_toc": false,
"remove_header_footer": false,
- "output_format": "json",
},
"slides": {
- "parse_method": "deepdoc",
- "suffix": []string{"pptx", "ppt"},
- "output_format": "json",
+ "parse_method": "deepdoc",
+ "suffix": []string{"pptx", "ppt"},
},
"image": {
// Default stays on the legacy parse_method shape. ocr_enabled is
@@ -362,7 +359,6 @@ func defaultSetups() map[string]schema.ParserSetup {
"llm_id": "",
"system_prompt": "",
"suffix": []string{"jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp"},
- "output_format": "json",
},
"email": {
"suffix": []string{"eml", "msg"},
@@ -370,7 +366,6 @@ func defaultSetups() map[string]schema.ParserSetup {
"from", "to", "cc", "bcc", "date", "subject",
"body", "attachments", "metadata",
},
- "output_format": "json",
},
"audio": {
"suffix": []string{
@@ -378,22 +373,20 @@ func defaultSetups() map[string]schema.ParserSetup {
"aiff", "au", "midi", "wma", "realaudio", "vqf",
"oggvorbis", "ape",
},
- "output_format": "json",
},
"video": {
- "suffix": []string{"mp4", "avi", "mkv"},
- "output_format": "json",
- "prompt": "",
+ "suffix": []string{"mp4", "avi", "mkv"},
+ "prompt": "",
},
"epub": {
- "suffix": []string{"epub"},
- "output_format": "json",
+ "suffix": []string{"epub"},
},
"json": {
- "suffix": []string{"json", "jsonl", "ldjson"},
- "output_format": "json",
+ "suffix": []string{"json", "jsonl", "ldjson"},
},
}
+ normalizeParserOutputFormats(s)
+ return s
}
// Inputs returns the static parameter metadata. The component
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index bbad965b46..8cbfbf7e41 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -419,35 +419,6 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {
}
}
-func TestDefaultSetups_DOCX_OutputFormatJSON(t *testing.T) {
- setups := defaultSetups()
- docx, ok := setups["docx"]
- if !ok {
- t.Fatal("defaultSetups: docx key missing")
- }
- got, ok := docx["output_format"].(string)
- if !ok {
- t.Fatal("defaultSetups: docx.output_format missing or not a string")
- }
- if got != "json" {
- t.Errorf("docx.output_format = %q, want %q", got, "json")
- }
-}
-func TestDefaultSetups_Audio_OutputFormatJSON(t *testing.T) {
- setups := defaultSetups()
- audio, ok := setups["audio"]
- if !ok {
- t.Fatal("defaultSetups: audio key missing")
- }
- got, ok := audio["output_format"].(string)
- if !ok {
- t.Fatal("defaultSetups: audio.output_format missing or not a string")
- }
- if got != "json" {
- t.Errorf("audio.output_format = %q, want %q", got, "json")
- }
-}
-
func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) {
setups := defaultSetups()
got := &captureSetupConfigurer{}
@@ -2191,6 +2162,9 @@ func TestPythonFamilyName_FileTypeConstants(t *testing.T) {
// neither of which existed in defaultSetups.
func TestConfigureParserFromSetups_VisualAural(t *testing.T) {
setups := defaultSetups()
+ // The component normalizes output_format onto every family before the
+ // dispatch layer sees the setups, so mirror that here.
+ normalizeParserOutputFormats(setups)
t.Run("visual resolves to image setup", func(t *testing.T) {
got := &captureSetupConfigurer{}
diff --git a/internal/ingestion/component/parser_output_format_test.go b/internal/ingestion/component/parser_output_format_test.go
new file mode 100644
index 0000000000..d891751d02
--- /dev/null
+++ b/internal/ingestion/component/parser_output_format_test.go
@@ -0,0 +1,71 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "testing"
+)
+
+// TestDefaultSetupsNormalizedToJSON pins that the raw defaults already carry
+// the single JSON output protocol. defaultSetups() must not ship a per-family
+// output_format literal (normalizeParserOutputFormats owns that key), yet
+// every family it returns must end up on JSON — otherwise a caller building a
+// ParserComponent straight from defaultSetups() would silently get an
+// un-normalized setup (the fragility flagged in the PR #21084 review).
+func TestDefaultSetupsNormalizedToJSON(t *testing.T) {
+ for family, setup := range defaultSetups() {
+ if got := setup["output_format"]; got != "json" {
+ t.Errorf("defaultSetups()[%q].output_format = %v, want json", family, got)
+ }
+ }
+}
+
+// TestParserOutputFormatAlwaysJSON pins the observable contract that survives
+// the literal cleanup: however the component is constructed — defaults, an
+// empty param map, or a legacy per-family override — every family ends up on
+// the single JSON output protocol.
+func TestParserOutputFormatAlwaysJSON(t *testing.T) {
+ cases := []struct {
+ name string
+ params map[string]any
+ }{
+ {"defaults", nil},
+ {"empty params", map[string]any{}},
+ {"legacy markdown override", map[string]any{"pdf": map[string]any{"output_format": "markdown"}}},
+ {"legacy text override", map[string]any{"text&code": map[string]any{"output_format": "text"}}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ comp, err := NewParserComponent(c.params)
+ if err != nil {
+ t.Fatalf("NewParserComponent: %v", err)
+ }
+ pc, ok := comp.(*ParserComponent)
+ if !ok {
+ t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)
+ }
+ if len(pc.setups) == 0 {
+ t.Fatal("expected non-empty setups")
+ }
+ for family, setup := range pc.setups {
+ if got := setup["output_format"]; got != "json" {
+ t.Errorf("family %q output_format = %v, want json", family, got)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 529235ac9c..94b26834cb 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -1,7 +1,6 @@
package pipeline
import (
- "fmt"
"reflect"
"strings"
@@ -441,58 +440,3 @@ func IsChunkerComponent(componentID string) bool {
lowerID := strings.ToLower(componentID)
return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")
}
-
-// ResolveComponentParamsDefaults takes DSL JSON bytes and returns the
-// component params defaults as an entity.JSONMap {cpnID: {param: value}}.
-// This is a pure function — callers must load the DSL themselves.
-func ResolveComponentParamsDefaults(dslJSON []byte) (entity.JSONMap, error) {
- cp, err := ComponentParamsDefaults(dslJSON)
- if err != nil {
- return nil, err
- }
- out := make(entity.JSONMap, len(cp))
- for k, v := range cp {
- out[k] = v
- }
- return out, nil
-}
-
-// ResolveComponentParamsDefaultsFromIDs loads the DSL for the target pipeline
-// (builtin via parserID or custom canvas via pipelineID) and returns the
-// component params defaults. It is a convenience wrapper around
-// LoadPipelineDSL + ResolveComponentParamsDefaults; use the two-step form
-// when you already have the DSL bytes.
-//
-// Deprecated: prefer loading the DSL yourself and calling
-// ResolveComponentParamsDefaults directly, to keep DAO dependencies
-// out of the pipeline package.
-func ResolveComponentParamsDefaultsFromIDs(parserID string, pipelineID *string) (entity.JSONMap, error) {
- isCanvas := pipelineID != nil && strings.TrimSpace(*pipelineID) != ""
- var dslJSON []byte
- var err error
- if isCanvas {
- return nil, fmt.Errorf("ResolveComponentParamsDefaultsFromIDs: cannot load canvas DSL without DAO; " +
- "load the DSL via service.LoadPipelineDSL first")
- }
- registry, regErr := DefaultRegistry()
- if regErr != nil {
- return nil, fmt.Errorf("builtin registry: %w", regErr)
- }
- if !registry.IsValid(parserID) {
- return nil, fmt.Errorf("unknown builtin parser_id: %q", parserID)
- }
- dslStr, dslErr := LoadBuiltinDSL(parserID)
- if dslErr != nil {
- return nil, fmt.Errorf("load builtin DSL: %w", dslErr)
- }
- dslJSON = []byte(dslStr)
- cp, err := ComponentParamsDefaults(dslJSON)
- if err != nil {
- return nil, err
- }
- out := make(entity.JSONMap, len(cp))
- for k, v := range cp {
- out[k] = v
- }
- return out, nil
-}
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index b828df7bd8..4f2f6f4471 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -519,51 +519,51 @@ func TestBuildParserConfig_BuiltinExtractorKeepsTagFileID(t *testing.T) {
}
}
-// --- ResolveComponentParamsDefaults ---
+// --- ComponentParamsDefaults ---
-func TestResolveComponentParamsDefaults_Basic(t *testing.T) {
+func TestComponentParamsDefaults_Basic(t *testing.T) {
dslJSON := generalDSL(t)
- result, err := ResolveComponentParamsDefaults(dslJSON)
+ result, err := ComponentParamsDefaults(dslJSON)
if err != nil {
- t.Fatalf("ResolveComponentParamsDefaults: %v", err)
+ t.Fatalf("ComponentParamsDefaults: %v", err)
}
if len(result) == 0 {
t.Fatal("expected non-empty result")
}
// outputs should be stripped.
- parser := result["Parser:HipSignsRhyme"].(map[string]any)
+ parser := result["Parser:HipSignsRhyme"]
if _, ok := parser["outputs"]; ok {
t.Error("expected outputs to be stripped")
}
if _, ok := parser["pdf"]; !ok {
t.Error("expected pdf to be present")
}
- chunker := result["Chunker:LegalReadersDecide"].(map[string]any)
+ chunker := result["Chunker:LegalReadersDecide"]
if chunker["chunk_size"] != float64(512) {
t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"])
}
}
-func TestResolveComponentParamsDefaults_InvalidJSON(t *testing.T) {
- _, err := ResolveComponentParamsDefaults([]byte("not json"))
+func TestComponentParamsDefaults_InvalidJSON(t *testing.T) {
+ _, err := ComponentParamsDefaults([]byte("not json"))
if err == nil {
t.Error("expected error for invalid JSON")
}
}
-func TestResolveComponentParamsDefaults_ResultIsMutable(t *testing.T) {
- // Verify the returned map is a copy, not a reference to internal state.
+func TestComponentParamsDefaults_ResultIsMutable(t *testing.T) {
+ // Verify the returned map is a deep copy, not a reference to shared state.
dslJSON := generalDSL(t)
- result, err := ResolveComponentParamsDefaults(dslJSON)
+ result, err := ComponentParamsDefaults(dslJSON)
if err != nil {
- t.Fatalf("ResolveComponentParamsDefaults: %v", err)
+ t.Fatalf("ComponentParamsDefaults: %v", err)
}
// Mutate the result.
- parser := result["Parser:HipSignsRhyme"].(map[string]any)
+ parser := result["Parser:HipSignsRhyme"]
delete(parser, "pdf")
// Re-read: the second call should return a fresh copy unaffected by the mutation.
- result2, _ := ResolveComponentParamsDefaults(dslJSON)
- parser2 := result2["Parser:HipSignsRhyme"].(map[string]any)
+ result2, _ := ComponentParamsDefaults(dslJSON)
+ parser2 := result2["Parser:HipSignsRhyme"]
if _, ok := parser2["pdf"]; !ok {
t.Error("expected result to be independent copy (pdf preserved)")
}
diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.go
index 520625aa93..38c3dcef01 100644
--- a/internal/ingestion/task/embedder.go
+++ b/internal/ingestion/task/embedder.go
@@ -25,6 +25,7 @@ import (
"net"
"net/http"
"regexp"
+ "slices"
"strconv"
"strings"
"sync"
@@ -138,11 +139,11 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E
var lastErr error
for _, limit := range overLimitLadder(limiter.Limit(maxTokens)) {
- candidate := trimAll(texts, limiter, maxTokens, limit)
+ candidate, own := trimAll(texts, limiter.Counter(), limit)
embeds, usage, err := e.embedWithRetry(ctx, candidate)
if err == nil {
- e.recordUsage(cal, calKey, limiter, candidate, usage)
- return distributeTokenCount(embeds, usage, candidate, limiter), nil
+ e.recordUsage(cal, calKey, own, usage)
+ return distributeTokenCount(embeds, usage, own), nil
}
if !isOverLimitErr(err) {
return nil, err
@@ -153,7 +154,7 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E
// input's own count: a batch total is expected to exceed the window and
// would imply a ratio below 1, i.e. teach us nothing.
if cal != nil {
- cal.ObserveOverLimit(calKey, ownTokenMax(candidate, limiter), maxTokens)
+ cal.ObserveOverLimit(calKey, slices.Max(own), maxTokens)
}
common.Warn("embedding input over the model limit; shrinking and retrying",
zap.String("model", derefString(e.model.ModelName)),
@@ -208,38 +209,18 @@ func overLimitLadderToFloor(budget int) []int { return tokenizer.OverLimitLadder
// input genuinely broken rather than merely long.
const overLimitFloorTokens = tokenizer.OverLimitFloorTokens
-// trimAll re-trims every text to `limit` tokens. Texts handed in by the caller
-// are already trimmed to the full budget, and trimming a prefix again just makes
-// it shorter, so this needs no access to the untrimmed input.
-func trimAll(texts []string, limiter tokenizer.Limiter, maxTokens, limit int) []string {
+// trimAll re-trims every text to `limit` tokens and reports each trimmed
+// text's token count. Counting once here lets the distribute, calibration and
+// over-limit paths reuse the numbers instead of re-counting the same batch.
+func trimAll(texts []string, counter tokenizer.Counter, limit int) ([]string, []int) {
out := make([]string, len(texts))
+ own := make([]int, len(texts))
for i, t := range texts {
- out[i] = limiter.Counter().TrimToLimit(t, limit)
+ trimmed := counter.TrimToLimit(t, limit)
+ out[i] = trimmed
+ own[i] = counter.Count(trimmed)
}
- return out
-}
-
-// ownTokenTotal estimates what the batch costs in our own counter, used for the
-// usage-based calibration.
-func ownTokenTotal(texts []string, limiter tokenizer.Limiter) int {
- total := 0
- for _, t := range texts {
- total += limiter.Counter().Count(t)
- }
- return total
-}
-
-// ownTokenMax is the largest single input's cost in our own counter. The
-// provider's window bounds each input individually, so this - not the batch
-// total - is what an over-limit rejection tells us about.
-func ownTokenMax(texts []string, limiter tokenizer.Limiter) int {
- max := 0
- for _, t := range texts {
- if n := limiter.Counter().Count(t); n > max {
- max = n
- }
- }
- return max
+ return out, own
}
// embedWithRetry performs one embedding call, retrying rate limits exactly as
@@ -289,16 +270,14 @@ func (e *embedder) embedWithRetry(ctx context.Context, texts []string) ([]models
}
// distributeTokenCount attaches a token count to every input. Providers report
-// usage per request, so the total is spread over the inputs in proportion to our
-// own counts of exactly the texts we sent; the rounding remainder goes to the
-// largest input so the sum matches the provider's number.
-func distributeTokenCount(embeds []models.EmbeddingData, usageTotal int, texts []string, limiter tokenizer.Limiter) []componentpkg.EmbeddingResult {
+// usage per request, so the total is spread over the inputs in proportion to
+// our own counts of exactly the texts we sent (own, from trimAll); the rounding
+// remainder goes to the largest input so the sum matches the provider's number.
+func distributeTokenCount(embeds []models.EmbeddingData, usageTotal int, own []int) []componentpkg.EmbeddingResult {
vecs := make([]componentpkg.EmbeddingResult, len(embeds))
- own := make([]int, len(texts))
total := 0
- for i, t := range texts {
- own[i] = limiter.Counter().Count(t)
- total += own[i]
+ for _, n := range own {
+ total += n
}
if usageTotal <= 0 || total <= 0 {
// No usage reported by the provider: fall back to our own count rather
@@ -332,17 +311,20 @@ func distributeTokenCount(embeds []models.EmbeddingData, usageTotal int, texts [
return vecs
}
-// recordUsage feeds the calibration: the provider's total for this batch against
-// our own count of the same texts.
-func (e *embedder) recordUsage(cal *tokenizer.Calibration, key string, limiter tokenizer.Limiter, texts []string, usageTotal int) {
+// recordUsage feeds the calibration: the provider's total for this batch
+// against our own count of the same texts (own, from trimAll).
+func (e *embedder) recordUsage(cal *tokenizer.Calibration, key string, own []int, usageTotal int) {
if cal == nil || usageTotal <= 0 {
return
}
- own := ownTokenTotal(texts, limiter)
- if own <= 0 {
+ total := 0
+ for _, n := range own {
+ total += n
+ }
+ if total <= 0 {
return
}
- cal.ObserveUsage(key, own, usageTotal)
+ cal.ObserveUsage(key, total, usageTotal)
}
// encodeIsolating is the last resort: it embeds inputs one at a time so a single
@@ -352,6 +334,7 @@ func (e *embedder) recordUsage(cal *tokenizer.Calibration, key string, limiter t
func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiter tokenizer.Limiter, maxTokens int) ([]componentpkg.EmbeddingResult, error) {
out := make([]componentpkg.EmbeddingResult, len(texts))
cal, calKey := limiter.Calibration()
+ counter := limiter.Counter()
var lastErr error
for i, text := range texts {
if err := ctx.Err(); err != nil {
@@ -359,11 +342,13 @@ func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiter
}
embedded := false
for _, limit := range overLimitLadderToFloor(limiter.Limit(maxTokens)) {
- candidate := []string{limiter.Counter().TrimToLimit(text, limit)}
+ trimmed := counter.TrimToLimit(text, limit)
+ own := counter.Count(trimmed)
+ candidate := []string{trimmed}
embeds, usage, err := e.embedWithRetry(ctx, candidate)
if err == nil {
- e.recordUsage(cal, calKey, limiter, candidate, usage)
- out[i] = distributeTokenCount(embeds, usage, candidate, limiter)[0]
+ e.recordUsage(cal, calKey, []int{own}, usage)
+ out[i] = distributeTokenCount(embeds, usage, []int{own})[0]
embedded = true
break
}
@@ -371,7 +356,7 @@ func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiter
return nil, err
}
if cal != nil {
- cal.ObserveOverLimit(calKey, limiter.Counter().Count(candidate[0]), maxTokens)
+ cal.ObserveOverLimit(calKey, own, maxTokens)
}
lastErr = err
}
diff --git a/internal/ingestion/task/embedder_count_once_test.go b/internal/ingestion/task/embedder_count_once_test.go
new file mode 100644
index 0000000000..734fd0c90a
--- /dev/null
+++ b/internal/ingestion/task/embedder_count_once_test.go
@@ -0,0 +1,232 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package task
+
+import (
+ "context"
+ "sync"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity/models"
+ "ragflow/internal/tokenizer"
+)
+
+// countingSpy wraps a real counter and records how often the embedder path
+// counts and trims. The embedder must count each candidate text exactly once
+// per ladder attempt (in trimAll); usage distribution, calibration and the
+// over-limit observation reuse that number instead of re-running the
+// tokenizer over the same texts.
+type countingSpy struct {
+ tokenizer.Counter
+ mu sync.Mutex
+ counts int
+ trims int
+}
+
+func (c *countingSpy) Count(text string) int {
+ c.mu.Lock()
+ c.counts++
+ c.mu.Unlock()
+ return c.Counter.Count(text)
+}
+
+func (c *countingSpy) TrimToLimit(text string, limit int) string {
+ c.mu.Lock()
+ c.trims++
+ c.mu.Unlock()
+ return c.Counter.TrimToLimit(text, limit)
+}
+
+func (c *countingSpy) stats() (counts, trims int) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.counts, c.trims
+}
+
+// installCountingSpy swaps the process-wide cl100k counter for a spy. The
+// embedder under test declares no tokenizer of its own, so both the embedder
+// and the model layer resolve cl100k; routing it through the spy makes every
+// count observable. The original is restored on cleanup; if cl100k was never
+// registered the spy stays installed, which is behaviorally identical because
+// it delegates to the real counter.
+func installCountingSpy(t *testing.T) *countingSpy {
+ t.Helper()
+ spy := &countingSpy{Counter: tokenizer.CountCL100K()}
+ if orig, ok := tokenizer.CounterByID(tokenizer.CounterCL100K); ok {
+ t.Cleanup(func() { tokenizer.RegisterCounter(tokenizer.CounterCL100K, orig) })
+ }
+ tokenizer.RegisterCounter(tokenizer.CounterCL100K, spy)
+ return spy
+}
+
+// usageDriver reports a request-level usage like a real provider without
+// counting anything itself, so the spy only observes the embedder's counting.
+type usageDriver struct {
+ stubDriver
+ perText int
+}
+
+func (d *usageDriver) Embed(ctx context.Context, modelName *string, request models.EmbedRequest, apiConfig *models.APIConfig, embeddingConfig *models.EmbeddingConfig, usage *common.ModelUsage) ([]models.EmbeddingData, error) {
+ if usage != nil {
+ usage.InputTokens = d.perText * len(request.Texts)
+ }
+ return d.stubDriver.Embed(ctx, modelName, request, apiConfig, embeddingConfig, usage)
+}
+
+// TestEncodeSuccessPathCountsEachTextOnce is the headline count-once guard:
+// on the success path the embedder may run the tokenizer once per text (the
+// trim) and must count each trimmed text exactly once — the provider-usage
+// distribution and the usage-based calibration reuse that count. The model
+// layer's own trim of the same texts is out of scope here and not asserted.
+func TestEncodeSuccessPathCountsEachTextOnce(t *testing.T) {
+ spy := installCountingSpy(t)
+
+ emb := newLimitEmbedder(&usageDriver{perText: 7}, 8192)
+ texts := []string{"alpha beta", "gamma delta epsilon", "zeta"}
+ vecs, err := emb.Encode(context.Background(), texts)
+ if err != nil {
+ t.Fatalf("Encode: %v", err)
+ }
+ if len(vecs) != len(texts) {
+ t.Fatalf("got %d vectors, want %d", len(vecs), len(texts))
+ }
+ counts, _ := spy.stats()
+ if counts != len(texts) {
+ t.Errorf("Count called %d times, want %d (exactly one count per text on the success path)", counts, len(texts))
+ }
+}
+
+// TestTrimAllCountsEachTextOnce pins the extraction itself: trimAll trims and
+// counts every text exactly once and reports counts that match the trimmed
+// texts the distribute/calibration paths will reuse.
+func TestTrimAllCountsEachTextOnce(t *testing.T) {
+ cases := []struct {
+ name string
+ texts []string
+ limit int
+ }{
+ {"nil input", nil, 100},
+ {"empty input", []string{}, 100},
+ {"single text", []string{"alpha beta"}, 100},
+ {"multiple texts", []string{"alpha beta", "gamma delta epsilon zeta", "eta"}, 100},
+ {"empty string", []string{""}, 100},
+ {"zero limit", []string{"alpha beta", "gamma"}, 0},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ spy := &countingSpy{Counter: tokenizer.CountCL100K()}
+ out, own := trimAll(c.texts, spy, c.limit)
+
+ counts, trims := spy.stats()
+ if counts != len(c.texts) || trims != len(c.texts) {
+ t.Fatalf("Count called %d times, TrimToLimit %d times, want %d each", counts, trims, len(c.texts))
+ }
+ if len(out) != len(c.texts) || len(own) != len(c.texts) {
+ t.Fatalf("got (%d texts, %d counts), want %d each", len(out), len(own), len(c.texts))
+ }
+ for i := range c.texts {
+ want := tokenizer.CountCL100K().Count(out[i])
+ if own[i] != want {
+ t.Errorf("own[%d] = %d, want the count of the trimmed text %d", i, own[i], want)
+ }
+ }
+ })
+ }
+}
+
+// TestRecordUsageObservesSummedOwnCount pins that the calibration observation
+// uses the precomputed per-input counts: the provider total against their sum,
+// with the documented early returns preserved. The assertion is deliberately
+// tight (not just "> 1") so it catches a future change that stops summing or
+// silently no-ops.
+func TestRecordUsageObservesSummedOwnCount(t *testing.T) {
+ // Use an isolated calibration so the assertion is not masked by the
+ // process-wide DefaultCalibration's starting bound (1.05) nor by other
+ // tests observing the same key.
+ cal := tokenizer.NewCalibration(tokenizer.DefaultUncountedRatioUpper)
+ const key = "spy-record-usage-key"
+ emb := &embedder{}
+
+ // own=[2,3,1] sums to 6 against usage 21 -> ratio 3.5. A weak "> 1"
+ // assertion would also pass if recordUsage did nothing (default 1.05),
+ // so pin the specific ratio.
+ emb.recordUsage(cal, key, []int{2, 3, 1}, 21)
+ if got := cal.RatioUpper(key); got < 3 || got > 4 {
+ t.Fatalf("RatioUpper after observing own sum=6 vs usage=21 = %v, want ~3.5", got)
+ }
+
+ // Summation, not max/first element: the same total delivered as a single
+ // element must produce the identical ratio. This fails if recordUsage ever
+ // switches to max() or to the first element instead of the sum.
+ const keySingle = "spy-record-usage-key-single"
+ emb.recordUsage(cal, keySingle, []int{6}, 21)
+ if got, want := cal.RatioUpper(keySingle), cal.RatioUpper(key); got != want {
+ t.Fatalf("RatioUpper for summed [2,3,1] = %v but for [6] = %v; recordUsage must use the sum", want, got)
+ }
+
+ // Non-positive usage and all-zero own counts must not move the ratio.
+ before := cal.RatioUpper(key)
+ emb.recordUsage(cal, key, []int{2, 3, 1}, 0) // realCount <= 0 -> no-op
+ emb.recordUsage(cal, key, []int{0, 0}, 21) // ownCount <= 0 -> no-op
+ if after := cal.RatioUpper(key); after != before {
+ t.Fatalf("RatioUpper moved from %v to %v on no-op observations", before, after)
+ }
+}
+
+// TestEncodeTokenCountsFallBackToOwnCount pins that the count-once refactor
+// preserves the accounting contract: when the provider reports no usage, each
+// vector still carries our own count of the text that was actually sent.
+func TestEncodeTokenCountsFallBackToOwnCount(t *testing.T) {
+ installCountingSpy(t)
+
+ emb := newLimitEmbedder(&stubDriver{}, 8192) // stubDriver reports no usage
+ texts := []string{"alpha beta gamma", "delta epsilon"}
+ vecs, err := emb.Encode(context.Background(), texts)
+ if err != nil {
+ t.Fatalf("Encode: %v", err)
+ }
+ for i, v := range vecs {
+ want := tokenizer.CountCL100K().Count(texts[i])
+ if v.TokenCount != want {
+ t.Errorf("vector %d token count = %d, want our own count %d", i, v.TokenCount, want)
+ }
+ }
+}
+
+// TestEncodeTokenCountsSumToProviderUsage pins the distribution contract on
+// the refactored path: the per-input counts still sum to the provider total.
+func TestEncodeTokenCountsSumToProviderUsage(t *testing.T) {
+ installCountingSpy(t)
+
+ emb := newLimitEmbedder(&usageDriver{perText: 7}, 8192)
+ texts := []string{"alpha beta", "gamma delta epsilon", "zeta"}
+ vecs, err := emb.Encode(context.Background(), texts)
+ if err != nil {
+ t.Fatalf("Encode: %v", err)
+ }
+ sum := 0
+ for _, v := range vecs {
+ if v.TokenCount <= 0 {
+ t.Errorf("vector token count = %d, want positive", v.TokenCount)
+ }
+ sum += v.TokenCount
+ }
+ if want := 7 * len(texts); sum != want {
+ t.Errorf("token counts sum to %d, want the provider total %d", sum, want)
+ }
+}
diff --git a/internal/ingestion/task/embedder_limit_test.go b/internal/ingestion/task/embedder_limit_test.go
index e26b572d37..878b724814 100644
--- a/internal/ingestion/task/embedder_limit_test.go
+++ b/internal/ingestion/task/embedder_limit_test.go
@@ -147,8 +147,12 @@ func TestDistributeTokenCountSumsToProviderUsage(t *testing.T) {
limiter := tokenizer.NewExactLimiter(tokenizer.CountCL100K())
texts := []string{strings.Repeat("word ", 10), strings.Repeat("word ", 40), strings.Repeat("word ", 20)}
embeds := make([]models.EmbeddingData, len(texts))
+ own := make([]int, len(texts))
+ for i, t := range texts {
+ own[i] = limiter.Counter().Count(t)
+ }
- got := distributeTokenCount(embeds, 100, texts, limiter)
+ got := distributeTokenCount(embeds, 100, own)
sum := 0
largest, largestIdx := -1, -1
for i, v := range got {
@@ -169,11 +173,14 @@ func TestDistributeTokenCountFallsBackToOwnCountWithoutUsage(t *testing.T) {
limiter := tokenizer.NewExactLimiter(tokenizer.CountCL100K())
texts := []string{"alpha beta", "gamma delta epsilon zeta"}
embeds := make([]models.EmbeddingData, len(texts))
- got := distributeTokenCount(embeds, 0, texts, limiter)
+ own := make([]int, len(texts))
+ for i, t := range texts {
+ own[i] = limiter.Counter().Count(t)
+ }
+ got := distributeTokenCount(embeds, 0, own)
for i := range got {
- want := limiter.Counter().Count(texts[i])
- if got[i].TokenCount != want {
- t.Errorf("input %d token count = %d, want our own count %d", i, got[i].TokenCount, want)
+ if got[i].TokenCount != own[i] {
+ t.Errorf("input %d token count = %d, want our own count %d", i, got[i].TokenCount, own[i])
}
}
}
diff --git a/internal/ingestion/wire/wire.go b/internal/ingestion/wire/wire.go
index 004ef0f5ba..a9d2913f33 100644
--- a/internal/ingestion/wire/wire.go
+++ b/internal/ingestion/wire/wire.go
@@ -24,11 +24,10 @@
// fragile: a new entry that forgets the blank import sees
// "unknown component" at run time.
//
-// This package centralises the import list. cmd entries should
-// blank-import this package and call RegisterComponents once
-// at startup. The function is a no-op at run time — its only
-// purpose is to ensure the blank imports here are evaluated,
-// which in turn triggers init() in each component package.
+// This package centralises the import list: cmd entries
+// blank-import this package, and the Go toolchain evaluates
+// the blank imports at link time, triggering each component
+// package's init().
//
// LOCATION NOTE: this lives under ingestion/ (not
// ingestion/pipeline/) because pipeline.go imports the
@@ -44,33 +43,5 @@ import (
// Component registration: the blank imports trigger each
// package's init() which calls runtime.DefaultRegistry.Register.
_ "ragflow/internal/ingestion/component" // File / Parser / Tokenizer / Extractor
- _ "ragflow/internal/ingestion/component/chunker" // 4 chunker variants
+ _ "ragflow/internal/ingestion/component/chunker" // chunker variants
)
-
-// RegisterComponents is the single bootstrap entry point that
-// guarantees every ingestion component is registered before the
-// pipeline runner resolves a name. It is a no-op at run time
-// (the actual registration happens via the package-level init()
-// triggered by the blank imports above); the function exists
-// only so cmd entries have a deterministic symbol to call.
-//
-// Usage from a cmd entry:
-//
-// import _ "ragflow/internal/ingestion/wire"
-//
-// The blank import alone is sufficient — the Go toolchain will
-// evaluate the blank imports at link time. The function is
-// exported for callers that prefer a function-call style over a
-// blank import:
-//
-// import "ragflow/internal/ingestion/wire"
-// ...
-// wire.RegisterComponents()
-//
-// Both styles are equivalent; the blank-import form is the
-// convention in this repo today.
-func RegisterComponents() {
- // Intentional no-op. The init() functions in the blank-imported
- // packages above have already registered the components by
- // the time this function is callable.
-}
diff --git a/internal/ingestion/wire/wire_test.go b/internal/ingestion/wire/wire_test.go
new file mode 100644
index 0000000000..bcac76530b
--- /dev/null
+++ b/internal/ingestion/wire/wire_test.go
@@ -0,0 +1,51 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package wire
+
+import (
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+)
+
+// TestBlankImportRegistersIngestionComponents pins the registration contract
+// this package exists for: blank-importing wire must make every ingestion
+// component resolvable in the runtime registry. cmd entries rely on that
+// import alone, so a removed or renamed blank import has to fail here rather
+// than surface as "unknown component" at run time.
+//
+// The names are string literals on purpose: importing the component packages
+// directly would register them via the test's own imports and hide a broken
+// wire.go.
+func TestBlankImportRegistersIngestionComponents(t *testing.T) {
+ for _, name := range []string{
+ "File", "Parser", "Tokenizer", "Extractor",
+ "TokenChunker", "GeneralChunker", "PageChunker", "TableChunker",
+ } {
+ factory, category, _, ok := runtime.DefaultRegistry.Lookup(name)
+ if !ok {
+ t.Errorf("component %q is not registered; the blank imports in wire.go must trigger its init()", name)
+ continue
+ }
+ if factory == nil {
+ t.Errorf("component %q registered with a nil factory", name)
+ }
+ if category != runtime.CategoryIngestion {
+ t.Errorf("component %q category = %q, want %q", name, category, runtime.CategoryIngestion)
+ }
+ }
+}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index a67c20cea1..5313cdd175 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -226,11 +226,21 @@ func (s *S3Storage) Get(ctx context.Context, bucket, fnm string, tenantID ...str
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
for i := 0; i < 2; i++ {
+ // Honor cancellation up front so an already-canceled context returns
+ // promptly instead of running the reconnect/backoff loop.
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fnm),
})
if err != nil {
+ // The request may have failed because ctx was canceled mid-flight;
+ // surface that instead of sleeping and retrying with a dead context.
+ if cerr := ctx.Err(); cerr != nil {
+ return nil, cerr
+ }
common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
s.reconnect(ctx)
time.Sleep(time.Second)
@@ -244,6 +254,9 @@ func (s *S3Storage) Get(ctx context.Context, bucket, fnm string, tenantID ...str
return err
}()
if readErr != nil {
+ if cerr := ctx.Err(); cerr != nil {
+ return nil, cerr
+ }
common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(readErr))
s.reconnect(ctx)
time.Sleep(time.Second)
diff --git a/internal/storage/s3_cancel_test.go b/internal/storage/s3_cancel_test.go
new file mode 100644
index 0000000000..c9f13fa66d
--- /dev/null
+++ b/internal/storage/s3_cancel_test.go
@@ -0,0 +1,96 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "testing"
+ "time"
+)
+
+// TestS3GetSurfacesContextCancellation pins that Get does not swallow a
+// canceled context behind its reconnect/backoff loop. On a canceled context
+// it must return the context error promptly instead of sleeping for the
+// backoff window and returning a generic "after retries" error.
+//
+// This is the root fix for the FetchBinary cancellation regression flagged in
+// the review of PR #21084: the storage layer must honor ctx so cancellation
+// surfaces through Get itself, for every backend including AWS S3.
+func TestS3GetSurfacesContextCancellation(t *testing.T) {
+ // Any transport error triggers the reconnect/backoff branch of Get.
+ s := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ return nil, fmt.Errorf("simulated transport error")
+ })
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel() // already canceled before the call
+
+ start := time.Now()
+ _, err := s.Get(ctx, "bucket", "key")
+ elapsed := time.Since(start)
+
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("Get err = %v, want context.Canceled", err)
+ }
+ if elapsed > 500*time.Millisecond {
+ t.Fatalf("Get took %v on an already-canceled ctx; want < 500ms (no reconnect sleep)", elapsed)
+ }
+}
+
+// TestS3GetReturnsObjectData guards the happy path against the cancellation
+// change: a successful object read still returns the bytes unchanged.
+func TestS3GetReturnsObjectData(t *testing.T) {
+ s := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ return s3Response(200, "hello-bucket"), nil
+ })
+
+ got, err := s.Get(context.Background(), "bucket", "key")
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if string(got) != "hello-bucket" {
+ t.Fatalf("Get returned %q, want hello-bucket", got)
+ }
+}
+
+// TestS3GetSurfacesContextCancellationMidFlight covers the in-flight case: the
+// context is still live when GetObject is called, but is canceled while the
+// request is blocked. The post-error ctx check must surface context.Canceled
+// immediately instead of running the reconnect/backoff sleep.
+func TestS3GetSurfacesContextCancellationMidFlight(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ s := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ <-ctx.Done() // block until the context is canceled
+ return nil, context.Canceled
+ })
+ go func() {
+ time.Sleep(20 * time.Millisecond)
+ cancel()
+ }()
+
+ start := time.Now()
+ _, err := s.Get(ctx, "bucket", "key")
+ elapsed := time.Since(start)
+
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("Get err = %v, want context.Canceled", err)
+ }
+ if elapsed > 500*time.Millisecond {
+ t.Fatalf("Get took %v on mid-flight cancel; want < 500ms (no reconnect sleep)", elapsed)
+ }
+}