RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): pipeline cleanups (#21084)
xugangqiang14cf685b80b7
diff --git a/internal/ingestion/component/chunker/delim_compile_guard_test.go b/internal/ingestion/component/chunker/delim_compile_guard_test.gonew file mode 100644index 0000000000..3f51bcb0fd--- /dev/null+++ b/internal/ingestion/component/chunker/delim_compile_guard_test.go@@ -0,0 +1,124 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "testing"++ "ragflow/internal/parser/chunk"+)++// TestCompileDelimPatternContract pins the shared delimiter-compiler contract+// that both the main Delimiters list and the children_delimiters list go+// through (compileChildrenPattern was merged into compileDelimPattern because+// the two were byte-identical). The assertions follow+// chunk.CompileDelimiterPatternList: empty entries are inactive, backtick+// entries contribute their inner content, bare entries stay active, duplicates+// collapse, and the alternation is longest-first and regex-escaped.+func TestCompileDelimPatternContract(t *testing.T) {+ cases := []struct {+ name string+ delims []string+ want string // expected pattern string; "" means nil pattern+ }{+ {"nil list", nil, ""},+ {"empty list", []string{}, ""},+ {"only empty entry", []string{""}, ""},+ {"only empty backticks", []string{"``"}, ""},+ {"lone backtick is bare", []string{"`"}, "`"},+ {"single bare", []string{"a"}, "a"},+ {"backtick inner", []string{"`end`"}, "end"},+ {"bare duplicates collapse", []string{"a", "a"}, "a"},+ {"backtick and bare dedupe", []string{"`a`", "a"}, "a"},+ {"empty among actives", []string{"", "b", ""}, "b"},+ {"longest first", []string{"ab", "`abc`"}, `abc|ab`},+ {"regex metacharacters escaped", []string{"a.b"}, `a\.b`},+ {"unicode delimiters", []string{"。", ","}, `。|,`},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ got := compileDelimPattern(c.delims)+ if c.want == "" {+ if got != nil {+ t.Fatalf("compileDelimPattern(%q) = %q, want nil", c.delims, got.String())+ }+ return+ }+ if got == nil {+ t.Fatalf("compileDelimPattern(%q) = nil, want %q", c.delims, c.want)+ }+ if got.String() != c.want {+ t.Fatalf("compileDelimPattern(%q) = %q, want %q", c.delims, got.String(), c.want)+ }+ })+ }+}++// TestCompileDelimPatternMatchesLongestFirst guards the alternation order in+// action: on overlapping text the longest delimiter must win, otherwise a+// shorter prefix would silently shadow the longer delimiter.+func TestCompileDelimPatternMatchesLongestFirst(t *testing.T) {+ pattern := compileDelimPattern([]string{"ab", "`abc`"})+ if pattern == nil {+ t.Fatal("compileDelimPattern returned nil")+ }+ got := pattern.FindAllString("xxabcyy", -1)+ if len(got) != 1 || got[0] != "abc" {+ t.Fatalf("matches on \"xxabcyy\" = %v, want [abc] (longest first)", got)+ }+}++// TestCompileDelimPatternIsCaseSensitive: the compiler must not emit (?i); the+// chunker's delimiter matching is case-sensitive by contract.+func TestCompileDelimPatternIsCaseSensitive(t *testing.T) {+ pattern := compileDelimPattern([]string{"`end`"})+ if pattern == nil {+ t.Fatal("compileDelimPattern(`end`) returned nil")+ }+ if pattern.MatchString("END") {+ t.Error("pattern matched uppercase END; delimiter matching must be case-sensitive")+ }+ if !pattern.MatchString("the end") {+ t.Error("pattern did not match lowercase end")+ }+}++// TestCompileDelimPatternDelegatesToChunk pins that the wrapper stays a thin+// delegation to chunk.CompileDelimiterPatternList(..., keepBare=true): any+// divergence would silently fork children_delimiters behaviour from the main+// delimiter list.+func TestCompileDelimPatternDelegatesToChunk(t *testing.T) {+ inputs := [][]string{+ nil,+ {},+ {""},+ {"a"},+ {"`end`", "b"},+ {"。", ",", "`;`"},+ {"a.b", "c*d"},+ }+ for _, delims := range inputs {+ got := compileDelimPattern(delims)+ want := chunk.CompileDelimiterPatternList(delims, true)+ if (got == nil) != (want == nil) {+ t.Fatalf("compileDelimPattern(%q) nil-ness = %v, want %v", delims, got == nil, want == nil)+ }+ if got != nil && got.String() != want.String() {+ t.Fatalf("compileDelimPattern(%q) = %q, want %q", delims, got.String(), want.String())+ }+ }+}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 5c61decece..a67709148c 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -231,7 +231,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups}primaryPattern := compileDelimPattern(c.param.Delimiters)- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)if customDelimiter {units = splitGeneralUnits(units, primaryPattern, false)@@ -402,7 +402,7 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schemareturn emptyOutputs(), nil}primaryPattern := compileDelimPattern(c.param.Delimiters)- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)if !customDelimiter {attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)@@ -627,7 +627,7 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream screturn emptyOutputs(), nil}primaryPattern := compileDelimPattern(c.param.Delimiters)- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)if !customDelimiter {attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)@@ -969,7 +969,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream}chunks = append(chunks, cloneChunkDoc(unit))}- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)chunks = finalizeGeneralChunks(chunks, childrenPattern)if len(chunks) == 0 {return emptyOutputs(), nil@@ -1065,7 +1065,7 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schreturn emptyOutputs(), nil}primaryPattern := compileDelimPattern(c.param.Delimiters)- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)units = splitGeneralUnits(units, primaryPattern, !customDelimiter)attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)diff --git a/internal/ingestion/component/chunker/general_positions_slice_test.go b/internal/ingestion/component/chunker/general_positions_slice_test.goindex 489e0bc964..e1f1c6b87e 100644--- a/internal/ingestion/component/chunker/general_positions_slice_test.go+++ b/internal/ingestion/component/chunker/general_positions_slice_test.go@@ -53,7 +53,7 @@ func TestSplitGeneralChildrenSlicesPositionsAbut(t *testing.T) {PDFPositions: json.RawMessage(`[[1,10,20,0,100]]`),Positions: json.RawMessage(`[[1,10,20,0,100]]`),}- pattern := compileChildrenPattern([]string{";"})+ pattern := compileDelimPattern([]string{";"})got := splitGeneralChildren([]schema.ChunkDoc{chunk}, pattern)if len(got) != 2 {t.Fatalf("children pieces = %d, want 2", len(got))diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 4703f186b1..68cf9c0e4e 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -193,7 +193,7 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs}delimPattern := compileDelimPattern(c.param.Delimiters)- childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)switch upstream.OutputFormat {case schema.PayloadFormatMarkdown:@@ -1828,18 +1828,6 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp, keepreturn out}-// compileChildrenPattern is the children_delimiters version of-// compileDelimPattern. Returns nil when no delimiters exist.-// compileChildrenPattern builds the children-split regex from a-// `children_delimiters` list. Every non-empty entry is active (including bare-// ones), and backtick-wrapped entries contribute their inner content — see-// chunk.CompileDelimiterPatternList. Delegating keeps children splitting-// consistent with the main delimiter list (backtick stripping + rune-descending-// order) instead of re-implementing a divergent copy.-func compileChildrenPattern(delims []string) *regexp.Regexp {- return chunk.CompileDelimiterPatternList(delims, true)-}-func intValue(v *int) int {if v == nil {return 0diff --git a/internal/ingestion/component/document_storage.go b/internal/ingestion/component/document_storage.goindex 844acddcd4..cff4610c47 100644--- a/internal/ingestion/component/document_storage.go+++ b/internal/ingestion/component/document_storage.go@@ -49,25 +49,13 @@ func FetchBinary(ctx context.Context, bucket, path string) ([]byte, error) {if storageImpl == nil {return nil, fmt.Errorf("no storage backend registered")}-- type result struct {- data []byte- err error- }- done := make(chan result, 1)- go func() {- data, err := storageImpl.Get(ctx, bucket, path)- done <- result{data: data, err: err}- }()- select {- case <-ctx.Done():- return nil, ctx.Err()- case r := <-done:- if r.err != nil {- return nil, fmt.Errorf("storage.Get(%q, %q): %w", bucket, path, r.err)- }- return r.data, nil+ // The storage layer takes ctx and honors it, so a cancellation surfaces+ // through Get itself; no extra goroutine is needed to abort early.+ data, err := storageImpl.Get(ctx, bucket, path)+ if err != nil {+ return nil, fmt.Errorf("storage.Get(%q, %q): %w", bucket, path, err)}+ return data, nil}func resolveStorage() storage.Storage {diff --git a/internal/ingestion/component/document_storage_test.go b/internal/ingestion/component/document_storage_test.gonew file mode 100644index 0000000000..8277d82650--- /dev/null+++ b/internal/ingestion/component/document_storage_test.go@@ -0,0 +1,235 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "context"+ "errors"+ "fmt"+ "strings"+ "sync"+ "testing"+ "time"++ "ragflow/internal/storage"+)++// fetchSpyStorage overrides only Get; every other Storage method delegates to+// the embedded implementation (an in-memory one) so the spy stays small.+type fetchSpyStorage struct {+ storage.Storage+ get func(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error)+}++func (s *fetchSpyStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {+ return s.get(ctx, bucket, fnm, tenantID...)+}++// withFetchStorage installs s as the storage backend for one test.+func withFetchStorage(t *testing.T, s storage.Storage) {+ t.Helper()+ prev := SetStorageFactoryOverride+ SetStorageFactoryOverride = func() storage.Storage { return s }+ t.Cleanup(func() { SetStorageFactoryOverride = prev })+}++func TestFetchBinaryReturnsObjectAndForwardsContext(t *testing.T) {+ payload := []byte("pdf-bytes-\x00\x01\xff")+ type seen struct {+ bucket, fnm string+ ctx context.Context+ }+ var got seen+ withFetchStorage(t, &fetchSpyStorage{+ Storage: storage.NewMemoryStorage(),+ get: func(ctx context.Context, bucket, fnm string, _ ...string) ([]byte, error) {+ got = seen{bucket: bucket, fnm: fnm, ctx: ctx}+ return payload, nil+ },+ })++ ctx := context.Background()+ data, err := FetchBinary(ctx, "kb-1", "docs/a.pdf")+ if err != nil {+ t.Fatalf("FetchBinary: %v", err)+ }+ if string(data) != string(payload) {+ t.Fatalf("FetchBinary returned %q, want %q", data, payload)+ }+ if got.bucket != "kb-1" || got.fnm != "docs/a.pdf" {+ t.Fatalf("Get called with (%q, %q), want (kb-1, docs/a.pdf)", got.bucket, got.fnm)+ }+ // FetchBinary must hand the caller's ctx to the storage layer so a+ // cancellation reaches the in-flight download.+ if got.ctx != ctx {+ t.Error("FetchBinary did not forward the caller's ctx to storage.Get")+ }+}++func TestFetchBinaryWrapsStorageError(t *testing.T) {+ sentinel := errors.New("object missing")+ withFetchStorage(t, &fetchSpyStorage{+ Storage: storage.NewMemoryStorage(),+ get: func(_ context.Context, _, _ string, _ ...string) ([]byte, error) {+ return nil, sentinel+ },+ })++ _, err := FetchBinary(context.Background(), "kb-1", "docs/missing.pdf")+ if err == nil {+ t.Fatal("FetchBinary succeeded despite a storage error")+ }+ if !errors.Is(err, sentinel) {+ t.Fatalf("error %v does not wrap the storage error", err)+ }+ // The wrap must identify the object so operators can find it in logs.+ for _, want := range []string{`"kb-1"`, `"docs/missing.pdf"`} {+ if !strings.Contains(err.Error(), want) {+ t.Errorf("error %q does not mention %q", err.Error(), want)+ }+ }+}++func TestFetchBinaryAlreadyCanceledContext(t *testing.T) {+ called := false+ withFetchStorage(t, &fetchSpyStorage{+ Storage: storage.NewMemoryStorage(),+ get: func(ctx context.Context, _, _ string, _ ...string) ([]byte, error) {+ called = true+ return nil, ctx.Err()+ },+ })++ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ _, err := FetchBinary(ctx, "kb-1", "docs/a.pdf")+ if err == nil {+ t.Fatal("FetchBinary succeeded with a canceled ctx")+ }+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("error %v does not wrap context.Canceled", err)+ }+ if !called {+ t.Error("storage.Get was never reached; the cancellation must surface through it")+ }+}++// TestFetchBinaryCancelDuringDownload pins that a cancellation while the+// download is in flight surfaces promptly instead of waiting for the whole+// object. The spy models a storage layer that honors ctx, as the real backends+// do — which is why FetchBinary needs no goroutine of its own to abort early.+func TestFetchBinaryCancelDuringDownload(t *testing.T) {+ entered := make(chan struct{})+ withFetchStorage(t, &fetchSpyStorage{+ Storage: storage.NewMemoryStorage(),+ get: func(ctx context.Context, _, _ string, _ ...string) ([]byte, error) {+ close(entered)+ <-ctx.Done()+ return nil, ctx.Err()+ },+ })++ ctx, cancel := context.WithCancel(context.Background())+ go func() {+ <-entered+ cancel()+ }()++ start := time.Now()+ _, err := FetchBinary(ctx, "kb-1", "docs/slow.pdf")+ elapsed := time.Since(start)+ if err == nil || !errors.Is(err, context.Canceled) {+ t.Fatalf("FetchBinary returned %v, want a context.Canceled wrap", err)+ }+ if elapsed > 2*time.Second {+ t.Fatalf("FetchBinary took %v to observe the cancellation; it must not wait for the download", elapsed)+ }+}++func TestFetchBinaryLargePayload(t *testing.T) {+ payload := make([]byte, 4<<20) // 4 MiB+ for i := range payload {+ payload[i] = byte(i)+ }+ withFetchStorage(t, &fetchSpyStorage{+ Storage: storage.NewMemoryStorage(),+ get: func(_ context.Context, _, _ string, _ ...string) ([]byte, error) {+ out := make([]byte, len(payload))+ copy(out, payload)+ return out, nil+ },+ })++ data, err := FetchBinary(context.Background(), "kb-1", "docs/big.pdf")+ if err != nil {+ t.Fatalf("FetchBinary: %v", err)+ }+ if len(data) != len(payload) {+ t.Fatalf("FetchBinary returned %d bytes, want %d", len(data), len(payload))+ }+ for i := range data {+ if data[i] != payload[i] {+ t.Fatalf("payload differs at byte %d: got %d, want %d", i, data[i], payload[i])+ }+ }+}++func TestFetchBinaryConcurrentCalls(t *testing.T) {+ payload := []byte("shared-object")+ withFetchStorage(t, &fetchSpyStorage{+ Storage: storage.NewMemoryStorage(),+ get: func(_ context.Context, _, _ string, _ ...string) ([]byte, error) {+ return payload, nil+ },+ })++ const callers = 16+ var wg sync.WaitGroup+ errs := make([]error, callers)+ for i := 0; i < callers; i++ {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ data, err := FetchBinary(context.Background(), "kb-1", "docs/shared.pdf")+ if err == nil && string(data) != string(payload) {+ err = fmt.Errorf("got %q, want %q", data, payload)+ }+ errs[i] = err+ }(i)+ }+ wg.Wait()+ for i, err := range errs {+ if err != nil {+ t.Errorf("caller %d: %v", i, err)+ }+ }+}++func TestFetchBinaryWithoutStorageBackend(t *testing.T) {+ if storage.GetStorageFactory().GetStorage() != nil {+ t.Skip("a global storage backend is registered in this test binary; the nil-backend path cannot be observed")+ }+ withFetchStorage(t, nil)++ _, err := FetchBinary(context.Background(), "kb-1", "docs/a.pdf")+ if err == nil {+ t.Fatal("FetchBinary succeeded without any storage backend")+ }+ if !strings.Contains(err.Error(), "no storage backend registered") {+ t.Fatalf("error %q does not mention the missing storage backend", err.Error())+ }+}diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex 38a3b4f775..8bfa54ca07 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -691,14 +691,24 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {}}-// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family default-// output_format must be "json" (matching Python parser.py:212), not "text".+// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family output+// format must be "json" (matching Python parser.py:212), not "text".// With "text" the structured email fields (from/to/subject/attachments/...) are-// flattened into a blob and lost downstream.+// flattened into a blob and lost downstream. The value is owned by+// normalizeParserOutputFormats, so the assertion goes through the constructed+// component rather than the raw defaults.func TestDefaultEmailOutputFormatIsJSON(t *testing.T) {- got, _ := defaultSetups()["email"]["output_format"].(string)+ comp, err := NewParserComponent(nil)+ if err != nil {+ t.Fatalf("NewParserComponent: %v", err)+ }+ pc, ok := comp.(*ParserComponent)+ if !ok {+ t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)+ }+ got, _ := pc.setups["email"]["output_format"].(string)if got != "json" {- t.Fatalf("email default output_format = %q, want json", got)+ t.Fatalf("email output_format = %q, want json", got)}}diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 88c8fd0307..6282cad78d 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -299,58 +299,55 @@ func imageSetupRunsOCR(setup schema.ParserSetup) bool {}func defaultSetups() map[string]schema.ParserSetup {- return map[string]schema.ParserSetup{+ // Normalize up front so any caller (including tests that build a+ // ParserComponent directly from these defaults) gets the single JSON+ // output protocol. NewParserComponent re-normalizes idempotently; the+ // per-family literals were removed because normalizeParserOutputFormats+ // owns this key unconditionally.+ s := map[string]schema.ParserSetup{"pdf": {"parse_method": "deepdoc","flatten_media_to_text": false,"remove_toc": false,"remove_header_footer": false,"suffix": []string{"pdf"},- "output_format": "json",},"spreadsheet": {"parse_method": "deepdoc","flatten_media_to_text": false,"html4excel": false,- "output_format": "json","suffix": []string{"xls", "xlsx", "csv"},},"doc": {"remove_toc": false,"remove_header_footer": false,"suffix": []string{"doc"},- "output_format": "json",},"docx": {"flatten_media_to_text": false,"remove_toc": false,"remove_header_footer": false,"suffix": []string{"docx"},- "output_format": "json",},"markdown": {"flatten_media_to_text": false,"suffix": []string{"md", "markdown", "mdx"},"remove_toc": false,- "output_format": "json",},"text&code": {"suffix": []string{"txt", "py", "js", "java", "c", "cpp", "h", "php","go", "ts", "sh", "cs", "kt", "sql",},- "output_format": "json",},"html": {"suffix": []string{"htm", "html"},"remove_toc": false,"remove_header_footer": false,- "output_format": "json",},"slides": {- "parse_method": "deepdoc",- "suffix": []string{"pptx", "ppt"},- "output_format": "json",+ "parse_method": "deepdoc",+ "suffix": []string{"pptx", "ppt"},},"image": {// Default stays on the legacy parse_method shape. ocr_enabled is@@ -362,7 +359,6 @@ func defaultSetups() map[string]schema.ParserSetup {"llm_id": "","system_prompt": "","suffix": []string{"jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp"},- "output_format": "json",},"email": {"suffix": []string{"eml", "msg"},@@ -370,7 +366,6 @@ func defaultSetups() map[string]schema.ParserSetup {"from", "to", "cc", "bcc", "date", "subject","body", "attachments", "metadata",},- "output_format": "json",},"audio": {"suffix": []string{@@ -378,22 +373,20 @@ func defaultSetups() map[string]schema.ParserSetup {"aiff", "au", "midi", "wma", "realaudio", "vqf","oggvorbis", "ape",},- "output_format": "json",},"video": {- "suffix": []string{"mp4", "avi", "mkv"},- "output_format": "json",- "prompt": "",+ "suffix": []string{"mp4", "avi", "mkv"},+ "prompt": "",},"epub": {- "suffix": []string{"epub"},- "output_format": "json",+ "suffix": []string{"epub"},},"json": {- "suffix": []string{"json", "jsonl", "ldjson"},- "output_format": "json",+ "suffix": []string{"json", "jsonl", "ldjson"},},}+ normalizeParserOutputFormats(s)+ return s}// Inputs returns the static parameter metadata. The componentdiff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex bbad965b46..8cbfbf7e41 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -419,35 +419,6 @@ func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {}}-func TestDefaultSetups_DOCX_OutputFormatJSON(t *testing.T) {- setups := defaultSetups()- docx, ok := setups["docx"]- if !ok {- t.Fatal("defaultSetups: docx key missing")- }- got, ok := docx["output_format"].(string)- if !ok {- t.Fatal("defaultSetups: docx.output_format missing or not a string")- }- if got != "json" {- t.Errorf("docx.output_format = %q, want %q", got, "json")- }-}-func TestDefaultSetups_Audio_OutputFormatJSON(t *testing.T) {- setups := defaultSetups()- audio, ok := setups["audio"]- if !ok {- t.Fatal("defaultSetups: audio key missing")- }- got, ok := audio["output_format"].(string)- if !ok {- t.Fatal("defaultSetups: audio.output_format missing or not a string")- }- if got != "json" {- t.Errorf("audio.output_format = %q, want %q", got, "json")- }-}-func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) {setups := defaultSetups()got := &captureSetupConfigurer{}@@ -2191,6 +2162,9 @@ func TestPythonFamilyName_FileTypeConstants(t *testing.T) {// neither of which existed in defaultSetups.func TestConfigureParserFromSetups_VisualAural(t *testing.T) {setups := defaultSetups()+ // The component normalizes output_format onto every family before the+ // dispatch layer sees the setups, so mirror that here.+ normalizeParserOutputFormats(setups)t.Run("visual resolves to image setup", func(t *testing.T) {got := &captureSetupConfigurer{}diff --git a/internal/ingestion/component/parser_output_format_test.go b/internal/ingestion/component/parser_output_format_test.gonew file mode 100644index 0000000000..d891751d02--- /dev/null+++ b/internal/ingestion/component/parser_output_format_test.go@@ -0,0 +1,71 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "testing"+)++// TestDefaultSetupsNormalizedToJSON pins that the raw defaults already carry+// the single JSON output protocol. defaultSetups() must not ship a per-family+// output_format literal (normalizeParserOutputFormats owns that key), yet+// every family it returns must end up on JSON — otherwise a caller building a+// ParserComponent straight from defaultSetups() would silently get an+// un-normalized setup (the fragility flagged in the PR #21084 review).+func TestDefaultSetupsNormalizedToJSON(t *testing.T) {+ for family, setup := range defaultSetups() {+ if got := setup["output_format"]; got != "json" {+ t.Errorf("defaultSetups()[%q].output_format = %v, want json", family, got)+ }+ }+}++// TestParserOutputFormatAlwaysJSON pins the observable contract that survives+// the literal cleanup: however the component is constructed — defaults, an+// empty param map, or a legacy per-family override — every family ends up on+// the single JSON output protocol.+func TestParserOutputFormatAlwaysJSON(t *testing.T) {+ cases := []struct {+ name string+ params map[string]any+ }{+ {"defaults", nil},+ {"empty params", map[string]any{}},+ {"legacy markdown override", map[string]any{"pdf": map[string]any{"output_format": "markdown"}}},+ {"legacy text override", map[string]any{"text&code": map[string]any{"output_format": "text"}}},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ comp, err := NewParserComponent(c.params)+ if err != nil {+ t.Fatalf("NewParserComponent: %v", err)+ }+ pc, ok := comp.(*ParserComponent)+ if !ok {+ t.Fatalf("NewParserComponent returned %T, want *ParserComponent", comp)+ }+ if len(pc.setups) == 0 {+ t.Fatal("expected non-empty setups")+ }+ for family, setup := range pc.setups {+ if got := setup["output_format"]; got != "json" {+ t.Errorf("family %q output_format = %v, want json", family, got)+ }+ }+ })+ }+}diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 529235ac9c..94b26834cb 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -1,7 +1,6 @@package pipelineimport (- "fmt""reflect""strings"@@ -441,58 +440,3 @@ func IsChunkerComponent(componentID string) bool {lowerID := strings.ToLower(componentID)return strings.HasPrefix(lowerID, "generalchunker:") || strings.HasPrefix(lowerID, "tokenchunker:")}--// ResolveComponentParamsDefaults takes DSL JSON bytes and returns the-// component params defaults as an entity.JSONMap {cpnID: {param: value}}.-// This is a pure function — callers must load the DSL themselves.-func ResolveComponentParamsDefaults(dslJSON []byte) (entity.JSONMap, error) {- cp, err := ComponentParamsDefaults(dslJSON)- if err != nil {- return nil, err- }- out := make(entity.JSONMap, len(cp))- for k, v := range cp {- out[k] = v- }- return out, nil-}--// ResolveComponentParamsDefaultsFromIDs loads the DSL for the target pipeline-// (builtin via parserID or custom canvas via pipelineID) and returns the-// component params defaults. It is a convenience wrapper around-// LoadPipelineDSL + ResolveComponentParamsDefaults; use the two-step form-// when you already have the DSL bytes.-//-// Deprecated: prefer loading the DSL yourself and calling-// ResolveComponentParamsDefaults directly, to keep DAO dependencies-// out of the pipeline package.-func ResolveComponentParamsDefaultsFromIDs(parserID string, pipelineID *string) (entity.JSONMap, error) {- isCanvas := pipelineID != nil && strings.TrimSpace(*pipelineID) != ""- var dslJSON []byte- var err error- if isCanvas {- return nil, fmt.Errorf("ResolveComponentParamsDefaultsFromIDs: cannot load canvas DSL without DAO; " +- "load the DSL via service.LoadPipelineDSL first")- }- registry, regErr := DefaultRegistry()- if regErr != nil {- return nil, fmt.Errorf("builtin registry: %w", regErr)- }- if !registry.IsValid(parserID) {- return nil, fmt.Errorf("unknown builtin parser_id: %q", parserID)- }- dslStr, dslErr := LoadBuiltinDSL(parserID)- if dslErr != nil {- return nil, fmt.Errorf("load builtin DSL: %w", dslErr)- }- dslJSON = []byte(dslStr)- cp, err := ComponentParamsDefaults(dslJSON)- if err != nil {- return nil, err- }- out := make(entity.JSONMap, len(cp))- for k, v := range cp {- out[k] = v- }- return out, nil-}diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex b828df7bd8..4f2f6f4471 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -519,51 +519,51 @@ func TestBuildParserConfig_BuiltinExtractorKeepsTagFileID(t *testing.T) {}}-// --- ResolveComponentParamsDefaults ---+// --- ComponentParamsDefaults ----func TestResolveComponentParamsDefaults_Basic(t *testing.T) {+func TestComponentParamsDefaults_Basic(t *testing.T) {dslJSON := generalDSL(t)- result, err := ResolveComponentParamsDefaults(dslJSON)+ result, err := ComponentParamsDefaults(dslJSON)if err != nil {- t.Fatalf("ResolveComponentParamsDefaults: %v", err)+ t.Fatalf("ComponentParamsDefaults: %v", err)}if len(result) == 0 {t.Fatal("expected non-empty result")}// outputs should be stripped.- parser := result["Parser:HipSignsRhyme"].(map[string]any)+ parser := result["Parser:HipSignsRhyme"]if _, ok := parser["outputs"]; ok {t.Error("expected outputs to be stripped")}if _, ok := parser["pdf"]; !ok {t.Error("expected pdf to be present")}- chunker := result["Chunker:LegalReadersDecide"].(map[string]any)+ chunker := result["Chunker:LegalReadersDecide"]if chunker["chunk_size"] != float64(512) {t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"])}}-func TestResolveComponentParamsDefaults_InvalidJSON(t *testing.T) {- _, err := ResolveComponentParamsDefaults([]byte("not json"))+func TestComponentParamsDefaults_InvalidJSON(t *testing.T) {+ _, err := ComponentParamsDefaults([]byte("not json"))if err == nil {t.Error("expected error for invalid JSON")}}-func TestResolveComponentParamsDefaults_ResultIsMutable(t *testing.T) {- // Verify the returned map is a copy, not a reference to internal state.+func TestComponentParamsDefaults_ResultIsMutable(t *testing.T) {+ // Verify the returned map is a deep copy, not a reference to shared state.dslJSON := generalDSL(t)- result, err := ResolveComponentParamsDefaults(dslJSON)+ result, err := ComponentParamsDefaults(dslJSON)if err != nil {- t.Fatalf("ResolveComponentParamsDefaults: %v", err)+ t.Fatalf("ComponentParamsDefaults: %v", err)}// Mutate the result.- parser := result["Parser:HipSignsRhyme"].(map[string]any)+ parser := result["Parser:HipSignsRhyme"]delete(parser, "pdf")// Re-read: the second call should return a fresh copy unaffected by the mutation.- result2, _ := ResolveComponentParamsDefaults(dslJSON)- parser2 := result2["Parser:HipSignsRhyme"].(map[string]any)+ result2, _ := ComponentParamsDefaults(dslJSON)+ parser2 := result2["Parser:HipSignsRhyme"]if _, ok := parser2["pdf"]; !ok {t.Error("expected result to be independent copy (pdf preserved)")}diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.goindex 520625aa93..38c3dcef01 100644--- a/internal/ingestion/task/embedder.go+++ b/internal/ingestion/task/embedder.go@@ -25,6 +25,7 @@ import ("net""net/http""regexp"+ "slices""strconv""strings""sync"@@ -138,11 +139,11 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.Evar lastErr errorfor _, limit := range overLimitLadder(limiter.Limit(maxTokens)) {- candidate := trimAll(texts, limiter, maxTokens, limit)+ candidate, own := trimAll(texts, limiter.Counter(), limit)embeds, usage, err := e.embedWithRetry(ctx, candidate)if err == nil {- e.recordUsage(cal, calKey, limiter, candidate, usage)- return distributeTokenCount(embeds, usage, candidate, limiter), nil+ e.recordUsage(cal, calKey, own, usage)+ return distributeTokenCount(embeds, usage, own), nil}if !isOverLimitErr(err) {return nil, err@@ -153,7 +154,7 @@ func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.E// input's own count: a batch total is expected to exceed the window and// would imply a ratio below 1, i.e. teach us nothing.if cal != nil {- cal.ObserveOverLimit(calKey, ownTokenMax(candidate, limiter), maxTokens)+ cal.ObserveOverLimit(calKey, slices.Max(own), maxTokens)}common.Warn("embedding input over the model limit; shrinking and retrying",zap.String("model", derefString(e.model.ModelName)),@@ -208,38 +209,18 @@ func overLimitLadderToFloor(budget int) []int { return tokenizer.OverLimitLadder// input genuinely broken rather than merely long.const overLimitFloorTokens = tokenizer.OverLimitFloorTokens-// trimAll re-trims every text to `limit` tokens. Texts handed in by the caller-// are already trimmed to the full budget, and trimming a prefix again just makes-// it shorter, so this needs no access to the untrimmed input.-func trimAll(texts []string, limiter tokenizer.Limiter, maxTokens, limit int) []string {+// trimAll re-trims every text to `limit` tokens and reports each trimmed+// text's token count. Counting once here lets the distribute, calibration and+// over-limit paths reuse the numbers instead of re-counting the same batch.+func trimAll(texts []string, counter tokenizer.Counter, limit int) ([]string, []int) {out := make([]string, len(texts))+ own := make([]int, len(texts))for i, t := range texts {- out[i] = limiter.Counter().TrimToLimit(t, limit)+ trimmed := counter.TrimToLimit(t, limit)+ out[i] = trimmed+ own[i] = counter.Count(trimmed)}- return out-}--// ownTokenTotal estimates what the batch costs in our own counter, used for the-// usage-based calibration.-func ownTokenTotal(texts []string, limiter tokenizer.Limiter) int {- total := 0- for _, t := range texts {- total += limiter.Counter().Count(t)- }- return total-}--// ownTokenMax is the largest single input's cost in our own counter. The-// provider's window bounds each input individually, so this - not the batch-// total - is what an over-limit rejection tells us about.-func ownTokenMax(texts []string, limiter tokenizer.Limiter) int {- max := 0- for _, t := range texts {- if n := limiter.Counter().Count(t); n > max {- max = n- }- }- return max+ return out, own}// embedWithRetry performs one embedding call, retrying rate limits exactly as@@ -289,16 +270,14 @@ func (e *embedder) embedWithRetry(ctx context.Context, texts []string) ([]models}// distributeTokenCount attaches a token count to every input. Providers report-// usage per request, so the total is spread over the inputs in proportion to our-// own counts of exactly the texts we sent; the rounding remainder goes to the-// largest input so the sum matches the provider's number.-func distributeTokenCount(embeds []models.EmbeddingData, usageTotal int, texts []string, limiter tokenizer.Limiter) []componentpkg.EmbeddingResult {+// usage per request, so the total is spread over the inputs in proportion to+// our own counts of exactly the texts we sent (own, from trimAll); the rounding+// remainder goes to the largest input so the sum matches the provider's number.+func distributeTokenCount(embeds []models.EmbeddingData, usageTotal int, own []int) []componentpkg.EmbeddingResult {vecs := make([]componentpkg.EmbeddingResult, len(embeds))- own := make([]int, len(texts))total := 0- for i, t := range texts {- own[i] = limiter.Counter().Count(t)- total += own[i]+ for _, n := range own {+ total += n}if usageTotal <= 0 || total <= 0 {// No usage reported by the provider: fall back to our own count rather@@ -332,17 +311,20 @@ func distributeTokenCount(embeds []models.EmbeddingData, usageTotal int, texts [return vecs}-// recordUsage feeds the calibration: the provider's total for this batch against-// our own count of the same texts.-func (e *embedder) recordUsage(cal *tokenizer.Calibration, key string, limiter tokenizer.Limiter, texts []string, usageTotal int) {+// recordUsage feeds the calibration: the provider's total for this batch+// against our own count of the same texts (own, from trimAll).+func (e *embedder) recordUsage(cal *tokenizer.Calibration, key string, own []int, usageTotal int) {if cal == nil || usageTotal <= 0 {return}- own := ownTokenTotal(texts, limiter)- if own <= 0 {+ total := 0+ for _, n := range own {+ total += n+ }+ if total <= 0 {return}- cal.ObserveUsage(key, own, usageTotal)+ cal.ObserveUsage(key, total, usageTotal)}// encodeIsolating is the last resort: it embeds inputs one at a time so a single@@ -352,6 +334,7 @@ func (e *embedder) recordUsage(cal *tokenizer.Calibration, key string, limiter tfunc (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiter tokenizer.Limiter, maxTokens int) ([]componentpkg.EmbeddingResult, error) {out := make([]componentpkg.EmbeddingResult, len(texts))cal, calKey := limiter.Calibration()+ counter := limiter.Counter()var lastErr errorfor i, text := range texts {if err := ctx.Err(); err != nil {@@ -359,11 +342,13 @@ func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiter}embedded := falsefor _, limit := range overLimitLadderToFloor(limiter.Limit(maxTokens)) {- candidate := []string{limiter.Counter().TrimToLimit(text, limit)}+ trimmed := counter.TrimToLimit(text, limit)+ own := counter.Count(trimmed)+ candidate := []string{trimmed}embeds, usage, err := e.embedWithRetry(ctx, candidate)if err == nil {- e.recordUsage(cal, calKey, limiter, candidate, usage)- out[i] = distributeTokenCount(embeds, usage, candidate, limiter)[0]+ e.recordUsage(cal, calKey, []int{own}, usage)+ out[i] = distributeTokenCount(embeds, usage, []int{own})[0]embedded = truebreak}@@ -371,7 +356,7 @@ func (e *embedder) encodeIsolating(ctx context.Context, texts []string, limiterreturn nil, err}if cal != nil {- cal.ObserveOverLimit(calKey, limiter.Counter().Count(candidate[0]), maxTokens)+ cal.ObserveOverLimit(calKey, own, maxTokens)}lastErr = err}diff --git a/internal/ingestion/task/embedder_count_once_test.go b/internal/ingestion/task/embedder_count_once_test.gonew file mode 100644index 0000000000..734fd0c90a--- /dev/null+++ b/internal/ingestion/task/embedder_count_once_test.go@@ -0,0 +1,232 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "context"+ "sync"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity/models"+ "ragflow/internal/tokenizer"+)++// countingSpy wraps a real counter and records how often the embedder path+// counts and trims. The embedder must count each candidate text exactly once+// per ladder attempt (in trimAll); usage distribution, calibration and the+// over-limit observation reuse that number instead of re-running the+// tokenizer over the same texts.+type countingSpy struct {+ tokenizer.Counter+ mu sync.Mutex+ counts int+ trims int+}++func (c *countingSpy) Count(text string) int {+ c.mu.Lock()+ c.counts+++ c.mu.Unlock()+ return c.Counter.Count(text)+}++func (c *countingSpy) TrimToLimit(text string, limit int) string {+ c.mu.Lock()+ c.trims+++ c.mu.Unlock()+ return c.Counter.TrimToLimit(text, limit)+}++func (c *countingSpy) stats() (counts, trims int) {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.counts, c.trims+}++// installCountingSpy swaps the process-wide cl100k counter for a spy. The+// embedder under test declares no tokenizer of its own, so both the embedder+// and the model layer resolve cl100k; routing it through the spy makes every+// count observable. The original is restored on cleanup; if cl100k was never+// registered the spy stays installed, which is behaviorally identical because+// it delegates to the real counter.+func installCountingSpy(t *testing.T) *countingSpy {+ t.Helper()+ spy := &countingSpy{Counter: tokenizer.CountCL100K()}+ if orig, ok := tokenizer.CounterByID(tokenizer.CounterCL100K); ok {+ t.Cleanup(func() { tokenizer.RegisterCounter(tokenizer.CounterCL100K, orig) })+ }+ tokenizer.RegisterCounter(tokenizer.CounterCL100K, spy)+ return spy+}++// usageDriver reports a request-level usage like a real provider without+// counting anything itself, so the spy only observes the embedder's counting.+type usageDriver struct {+ stubDriver+ perText int+}++func (d *usageDriver) Embed(ctx context.Context, modelName *string, request models.EmbedRequest, apiConfig *models.APIConfig, embeddingConfig *models.EmbeddingConfig, usage *common.ModelUsage) ([]models.EmbeddingData, error) {+ if usage != nil {+ usage.InputTokens = d.perText * len(request.Texts)+ }+ return d.stubDriver.Embed(ctx, modelName, request, apiConfig, embeddingConfig, usage)+}++// TestEncodeSuccessPathCountsEachTextOnce is the headline count-once guard:+// on the success path the embedder may run the tokenizer once per text (the+// trim) and must count each trimmed text exactly once — the provider-usage+// distribution and the usage-based calibration reuse that count. The model+// layer's own trim of the same texts is out of scope here and not asserted.+func TestEncodeSuccessPathCountsEachTextOnce(t *testing.T) {+ spy := installCountingSpy(t)++ emb := newLimitEmbedder(&usageDriver{perText: 7}, 8192)+ texts := []string{"alpha beta", "gamma delta epsilon", "zeta"}+ vecs, err := emb.Encode(context.Background(), texts)+ if err != nil {+ t.Fatalf("Encode: %v", err)+ }+ if len(vecs) != len(texts) {+ t.Fatalf("got %d vectors, want %d", len(vecs), len(texts))+ }+ counts, _ := spy.stats()+ if counts != len(texts) {+ t.Errorf("Count called %d times, want %d (exactly one count per text on the success path)", counts, len(texts))+ }+}++// TestTrimAllCountsEachTextOnce pins the extraction itself: trimAll trims and+// counts every text exactly once and reports counts that match the trimmed+// texts the distribute/calibration paths will reuse.+func TestTrimAllCountsEachTextOnce(t *testing.T) {+ cases := []struct {+ name string+ texts []string+ limit int+ }{+ {"nil input", nil, 100},+ {"empty input", []string{}, 100},+ {"single text", []string{"alpha beta"}, 100},+ {"multiple texts", []string{"alpha beta", "gamma delta epsilon zeta", "eta"}, 100},+ {"empty string", []string{""}, 100},+ {"zero limit", []string{"alpha beta", "gamma"}, 0},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ spy := &countingSpy{Counter: tokenizer.CountCL100K()}+ out, own := trimAll(c.texts, spy, c.limit)++ counts, trims := spy.stats()+ if counts != len(c.texts) || trims != len(c.texts) {+ t.Fatalf("Count called %d times, TrimToLimit %d times, want %d each", counts, trims, len(c.texts))+ }+ if len(out) != len(c.texts) || len(own) != len(c.texts) {+ t.Fatalf("got (%d texts, %d counts), want %d each", len(out), len(own), len(c.texts))+ }+ for i := range c.texts {+ want := tokenizer.CountCL100K().Count(out[i])+ if own[i] != want {+ t.Errorf("own[%d] = %d, want the count of the trimmed text %d", i, own[i], want)+ }+ }+ })+ }+}++// TestRecordUsageObservesSummedOwnCount pins that the calibration observation+// uses the precomputed per-input counts: the provider total against their sum,+// with the documented early returns preserved. The assertion is deliberately+// tight (not just "> 1") so it catches a future change that stops summing or+// silently no-ops.+func TestRecordUsageObservesSummedOwnCount(t *testing.T) {+ // Use an isolated calibration so the assertion is not masked by the+ // process-wide DefaultCalibration's starting bound (1.05) nor by other+ // tests observing the same key.+ cal := tokenizer.NewCalibration(tokenizer.DefaultUncountedRatioUpper)+ const key = "spy-record-usage-key"+ emb := &embedder{}++ // own=[2,3,1] sums to 6 against usage 21 -> ratio 3.5. A weak "> 1"+ // assertion would also pass if recordUsage did nothing (default 1.05),+ // so pin the specific ratio.+ emb.recordUsage(cal, key, []int{2, 3, 1}, 21)+ if got := cal.RatioUpper(key); got < 3 || got > 4 {+ t.Fatalf("RatioUpper after observing own sum=6 vs usage=21 = %v, want ~3.5", got)+ }++ // Summation, not max/first element: the same total delivered as a single+ // element must produce the identical ratio. This fails if recordUsage ever+ // switches to max() or to the first element instead of the sum.+ const keySingle = "spy-record-usage-key-single"+ emb.recordUsage(cal, keySingle, []int{6}, 21)+ if got, want := cal.RatioUpper(keySingle), cal.RatioUpper(key); got != want {+ t.Fatalf("RatioUpper for summed [2,3,1] = %v but for [6] = %v; recordUsage must use the sum", want, got)+ }++ // Non-positive usage and all-zero own counts must not move the ratio.+ before := cal.RatioUpper(key)+ emb.recordUsage(cal, key, []int{2, 3, 1}, 0) // realCount <= 0 -> no-op+ emb.recordUsage(cal, key, []int{0, 0}, 21) // ownCount <= 0 -> no-op+ if after := cal.RatioUpper(key); after != before {+ t.Fatalf("RatioUpper moved from %v to %v on no-op observations", before, after)+ }+}++// TestEncodeTokenCountsFallBackToOwnCount pins that the count-once refactor+// preserves the accounting contract: when the provider reports no usage, each+// vector still carries our own count of the text that was actually sent.+func TestEncodeTokenCountsFallBackToOwnCount(t *testing.T) {+ installCountingSpy(t)++ emb := newLimitEmbedder(&stubDriver{}, 8192) // stubDriver reports no usage+ texts := []string{"alpha beta gamma", "delta epsilon"}+ vecs, err := emb.Encode(context.Background(), texts)+ if err != nil {+ t.Fatalf("Encode: %v", err)+ }+ for i, v := range vecs {+ want := tokenizer.CountCL100K().Count(texts[i])+ if v.TokenCount != want {+ t.Errorf("vector %d token count = %d, want our own count %d", i, v.TokenCount, want)+ }+ }+}++// TestEncodeTokenCountsSumToProviderUsage pins the distribution contract on+// the refactored path: the per-input counts still sum to the provider total.+func TestEncodeTokenCountsSumToProviderUsage(t *testing.T) {+ installCountingSpy(t)++ emb := newLimitEmbedder(&usageDriver{perText: 7}, 8192)+ texts := []string{"alpha beta", "gamma delta epsilon", "zeta"}+ vecs, err := emb.Encode(context.Background(), texts)+ if err != nil {+ t.Fatalf("Encode: %v", err)+ }+ sum := 0+ for _, v := range vecs {+ if v.TokenCount <= 0 {+ t.Errorf("vector token count = %d, want positive", v.TokenCount)+ }+ sum += v.TokenCount+ }+ if want := 7 * len(texts); sum != want {+ t.Errorf("token counts sum to %d, want the provider total %d", sum, want)+ }+}diff --git a/internal/ingestion/task/embedder_limit_test.go b/internal/ingestion/task/embedder_limit_test.goindex e26b572d37..878b724814 100644--- a/internal/ingestion/task/embedder_limit_test.go+++ b/internal/ingestion/task/embedder_limit_test.go@@ -147,8 +147,12 @@ func TestDistributeTokenCountSumsToProviderUsage(t *testing.T) {limiter := tokenizer.NewExactLimiter(tokenizer.CountCL100K())texts := []string{strings.Repeat("word ", 10), strings.Repeat("word ", 40), strings.Repeat("word ", 20)}embeds := make([]models.EmbeddingData, len(texts))+ own := make([]int, len(texts))+ for i, t := range texts {+ own[i] = limiter.Counter().Count(t)+ }- got := distributeTokenCount(embeds, 100, texts, limiter)+ got := distributeTokenCount(embeds, 100, own)sum := 0largest, largestIdx := -1, -1for i, v := range got {@@ -169,11 +173,14 @@ func TestDistributeTokenCountFallsBackToOwnCountWithoutUsage(t *testing.T) {limiter := tokenizer.NewExactLimiter(tokenizer.CountCL100K())texts := []string{"alpha beta", "gamma delta epsilon zeta"}embeds := make([]models.EmbeddingData, len(texts))- got := distributeTokenCount(embeds, 0, texts, limiter)+ own := make([]int, len(texts))+ for i, t := range texts {+ own[i] = limiter.Counter().Count(t)+ }+ got := distributeTokenCount(embeds, 0, own)for i := range got {- want := limiter.Counter().Count(texts[i])- if got[i].TokenCount != want {- t.Errorf("input %d token count = %d, want our own count %d", i, got[i].TokenCount, want)+ if got[i].TokenCount != own[i] {+ t.Errorf("input %d token count = %d, want our own count %d", i, got[i].TokenCount, own[i])}}}diff --git a/internal/ingestion/wire/wire.go b/internal/ingestion/wire/wire.goindex 004ef0f5ba..a9d2913f33 100644--- a/internal/ingestion/wire/wire.go+++ b/internal/ingestion/wire/wire.go@@ -24,11 +24,10 @@// fragile: a new entry that forgets the blank import sees// "unknown component" at run time.//-// This package centralises the import list. cmd entries should-// blank-import this package and call RegisterComponents once-// at startup. The function is a no-op at run time — its only-// purpose is to ensure the blank imports here are evaluated,-// which in turn triggers init() in each component package.+// This package centralises the import list: cmd entries+// blank-import this package, and the Go toolchain evaluates+// the blank imports at link time, triggering each component+// package's init().//// LOCATION NOTE: this lives under ingestion/ (not// ingestion/pipeline/) because pipeline.go imports the@@ -44,33 +43,5 @@ import (// Component registration: the blank imports trigger each// package's init() which calls runtime.DefaultRegistry.Register._ "ragflow/internal/ingestion/component" // File / Parser / Tokenizer / Extractor- _ "ragflow/internal/ingestion/component/chunker" // 4 chunker variants+ _ "ragflow/internal/ingestion/component/chunker" // chunker variants)--// RegisterComponents is the single bootstrap entry point that-// guarantees every ingestion component is registered before the-// pipeline runner resolves a name. It is a no-op at run time-// (the actual registration happens via the package-level init()-// triggered by the blank imports above); the function exists-// only so cmd entries have a deterministic symbol to call.-//-// Usage from a cmd entry:-//-// import _ "ragflow/internal/ingestion/wire"-//-// The blank import alone is sufficient — the Go toolchain will-// evaluate the blank imports at link time. The function is-// exported for callers that prefer a function-call style over a-// blank import:-//-// import "ragflow/internal/ingestion/wire"-// ...-// wire.RegisterComponents()-//-// Both styles are equivalent; the blank-import form is the-// convention in this repo today.-func RegisterComponents() {- // Intentional no-op. The init() functions in the blank-imported- // packages above have already registered the components by- // the time this function is callable.-}diff --git a/internal/ingestion/wire/wire_test.go b/internal/ingestion/wire/wire_test.gonew file mode 100644index 0000000000..bcac76530b--- /dev/null+++ b/internal/ingestion/wire/wire_test.go@@ -0,0 +1,51 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package wire++import (+ "testing"++ "ragflow/internal/agent/runtime"+)++// TestBlankImportRegistersIngestionComponents pins the registration contract+// this package exists for: blank-importing wire must make every ingestion+// component resolvable in the runtime registry. cmd entries rely on that+// import alone, so a removed or renamed blank import has to fail here rather+// than surface as "unknown component" at run time.+//+// The names are string literals on purpose: importing the component packages+// directly would register them via the test's own imports and hide a broken+// wire.go.+func TestBlankImportRegistersIngestionComponents(t *testing.T) {+ for _, name := range []string{+ "File", "Parser", "Tokenizer", "Extractor",+ "TokenChunker", "GeneralChunker", "PageChunker", "TableChunker",+ } {+ factory, category, _, ok := runtime.DefaultRegistry.Lookup(name)+ if !ok {+ t.Errorf("component %q is not registered; the blank imports in wire.go must trigger its init()", name)+ continue+ }+ if factory == nil {+ t.Errorf("component %q registered with a nil factory", name)+ }+ if category != runtime.CategoryIngestion {+ t.Errorf("component %q category = %q, want %q", name, category, runtime.CategoryIngestion)+ }+ }+}diff --git a/internal/storage/s3.go b/internal/storage/s3.goindex a67c20cea1..5313cdd175 100644--- a/internal/storage/s3.go+++ b/internal/storage/s3.go@@ -226,11 +226,21 @@ func (s *S3Storage) Get(ctx context.Context, bucket, fnm string, tenantID ...strbucket, fnm = s.resolveBucketAndPath(bucket, fnm)for i := 0; i < 2; i++ {+ // Honor cancellation up front so an already-canceled context returns+ // promptly instead of running the reconnect/backoff loop.+ if err := ctx.Err(); err != nil {+ return nil, err+ }result, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket),Key: aws.String(fnm),})if err != nil {+ // The request may have failed because ctx was canceled mid-flight;+ // surface that instead of sleeping and retrying with a dead context.+ if cerr := ctx.Err(); cerr != nil {+ return nil, cerr+ }common.Error("Failed to get object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))s.reconnect(ctx)time.Sleep(time.Second)@@ -244,6 +254,9 @@ func (s *S3Storage) Get(ctx context.Context, bucket, fnm string, tenantID ...strreturn err}()if readErr != nil {+ if cerr := ctx.Err(); cerr != nil {+ return nil, cerr+ }common.Error("Failed to read object data", readErr, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(readErr))s.reconnect(ctx)time.Sleep(time.Second)diff --git a/internal/storage/s3_cancel_test.go b/internal/storage/s3_cancel_test.gonew file mode 100644index 0000000000..c9f13fa66d--- /dev/null+++ b/internal/storage/s3_cancel_test.go@@ -0,0 +1,96 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package storage++import (+ "context"+ "errors"+ "fmt"+ "net/http"+ "testing"+ "time"+)++// TestS3GetSurfacesContextCancellation pins that Get does not swallow a+// canceled context behind its reconnect/backoff loop. On a canceled context+// it must return the context error promptly instead of sleeping for the+// backoff window and returning a generic "after retries" error.+//+// This is the root fix for the FetchBinary cancellation regression flagged in+// the review of PR #21084: the storage layer must honor ctx so cancellation+// surfaces through Get itself, for every backend including AWS S3.+func TestS3GetSurfacesContextCancellation(t *testing.T) {+ // Any transport error triggers the reconnect/backoff branch of Get.+ s := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {+ return nil, fmt.Errorf("simulated transport error")+ })++ ctx, cancel := context.WithCancel(context.Background())+ cancel() // already canceled before the call++ start := time.Now()+ _, err := s.Get(ctx, "bucket", "key")+ elapsed := time.Since(start)++ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Get err = %v, want context.Canceled", err)+ }+ if elapsed > 500*time.Millisecond {+ t.Fatalf("Get took %v on an already-canceled ctx; want < 500ms (no reconnect sleep)", elapsed)+ }+}++// TestS3GetReturnsObjectData guards the happy path against the cancellation+// change: a successful object read still returns the bytes unchanged.+func TestS3GetReturnsObjectData(t *testing.T) {+ s := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {+ return s3Response(200, "hello-bucket"), nil+ })++ got, err := s.Get(context.Background(), "bucket", "key")+ if err != nil {+ t.Fatalf("Get: %v", err)+ }+ if string(got) != "hello-bucket" {+ t.Fatalf("Get returned %q, want hello-bucket", got)+ }+}++// TestS3GetSurfacesContextCancellationMidFlight covers the in-flight case: the+// context is still live when GetObject is called, but is canceled while the+// request is blocked. The post-error ctx check must surface context.Canceled+// immediately instead of running the reconnect/backoff sleep.+func TestS3GetSurfacesContextCancellationMidFlight(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ s := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {+ <-ctx.Done() // block until the context is canceled+ return nil, context.Canceled+ })+ go func() {+ time.Sleep(20 * time.Millisecond)+ cancel()+ }()++ start := time.Now()+ _, err := s.Get(ctx, "bucket", "key")+ elapsed := time.Since(start)++ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Get err = %v, want context.Canceled", err)+ }+ if elapsed > 500*time.Millisecond {+ t.Fatalf("Get took %v on mid-flight cancel; want < 500ms (no reconnect sleep)", elapsed)+ }+}