Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): cleanup — stdlib reuse and shared component names (#21149)

xugangqiang1e0224d69d08
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 39614bc609..0db36befe9 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -65,7 +65,10 @@ import (
"ragflow/internal/utility"
)
-const componentNameExtractor = "Extractor"
+// componentNameExtractor is the canonical Extractor component name; the value
+// lives in the schema package so the pipeline layer can reference the same
+// identity.
+const componentNameExtractor = schema.ComponentNameExtractor
// defaultExtractorChatTimeout bounds one Extractor LLM call, including
// retries and backoff. Tag extraction has its own timeout.
@@ -1551,6 +1554,14 @@ func resolveExtractorChatConfig(ctx context.Context, db *gorm.DB, compositeLLMID
}
}
+ return newExtractorChatConfig(driver, modelName, apiConfig), nil
+}
+
+// newExtractorChatConfig builds the chat config from a resolved driver and its
+// API settings: the driver name lowercased as the llm_factory, with the
+// credentials dereferenced defensively (a nil apiConfig or nil fields mean
+// "not configured", not a panic).
+func newExtractorChatConfig(driver models.ModelDriver, modelName string, apiConfig *models.APIConfig) extractorChatConfig {
apiKey := ""
baseURL := ""
if apiConfig != nil {
@@ -1566,7 +1577,7 @@ func resolveExtractorChatConfig(ctx context.Context, db *gorm.DB, compositeLLMID
modelName: modelName,
apiKey: apiKey,
baseURL: baseURL,
- }, nil
+ }
}
// resolveExtractorChatDefaultConfig resolves the tenant's default chat
@@ -1588,22 +1599,7 @@ func resolveExtractorChatDefaultConfig(ctx context.Context, db *gorm.DB) extract
return extractorChatConfig{}
}
- apiKey := ""
- baseURL := ""
- if apiConfig != nil {
- if apiConfig.ApiKey != nil {
- apiKey = *apiConfig.ApiKey
- }
- if apiConfig.BaseURL != nil {
- baseURL = *apiConfig.BaseURL
- }
- }
- return extractorChatConfig{
- driver: strings.ToLower(driver.Name()),
- modelName: modelName,
- apiKey: apiKey,
- baseURL: baseURL,
- }
+ return newExtractorChatConfig(driver, modelName, apiConfig)
}
// isBareTenantModelID reports whether s is a 32-character hex string
diff --git a/internal/ingestion/component/extractor_chat_config_test.go b/internal/ingestion/component/extractor_chat_config_test.go
new file mode 100644
index 0000000000..cc2c30f9cd
--- /dev/null
+++ b/internal/ingestion/component/extractor_chat_config_test.go
@@ -0,0 +1,193 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/entity/models"
+)
+
+// namedDriver is a ModelDriver stub that only knows its name; the chat-config
+// resolvers touch nothing else on the driver.
+type namedDriver struct {
+ models.ModelDriver
+ name string
+}
+
+func (d namedDriver) Name() string { return d.name }
+
+// swapModelResolvers installs fake model resolvers for one test. Each argument
+// is optional; a nil argument leaves the corresponding production resolver in
+// place.
+func swapModelResolvers(t *testing.T, byConfig, byType func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType, ref string) (models.ModelDriver, string, *models.APIConfig, int, error)) {
+ t.Helper()
+ origConfig, origType := resolveModelConfig, resolveTenantModelByType
+ t.Cleanup(func() {
+ resolveModelConfig, resolveTenantModelByType = origConfig, origType
+ })
+ if byConfig != nil {
+ resolveModelConfig = byConfig
+ }
+ if byType != nil {
+ resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ return byType(ctx, db, tenantID, modelType, "")
+ }
+ }
+}
+
+func TestResolveExtractorChatConfig_CompositePath(t *testing.T) {
+ swapModelResolvers(t, func(_ context.Context, _ *gorm.DB, tenantID string, modelType entity.ModelType, ref string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ if tenantID != "tenant-1" {
+ t.Errorf("tenantID = %q, want tenant-1", tenantID)
+ }
+ if modelType != entity.ModelTypeChat {
+ t.Errorf("modelType = %v, want chat", modelType)
+ }
+ if ref != "gpt-4o@openai" {
+ t.Errorf("ref = %q, want the composite reference", ref)
+ }
+ return namedDriver{name: "OpenAI"}, "gpt-4o", &models.APIConfig{ApiKey: ptr("sk-secret"), BaseURL: ptr("https://api.example.com")}, 0, nil
+ }, nil)
+
+ ctx := extractorStateCtx(t, "tenant-1")
+ got, err := resolveExtractorChatConfig(ctx, nil, "gpt-4o@openai")
+ if err != nil {
+ t.Fatalf("resolveExtractorChatConfig: %v", err)
+ }
+ if got.driver != "openai" {
+ t.Errorf("driver = %q, want the lowercased factory name", got.driver)
+ }
+ if got.modelName != "gpt-4o" {
+ t.Errorf("modelName = %q, want gpt-4o", got.modelName)
+ }
+ if got.apiKey != "sk-secret" {
+ t.Errorf("apiKey = %q, want the configured key", got.apiKey)
+ }
+ if got.baseURL != "https://api.example.com" {
+ t.Errorf("baseURL = %q, want the configured base URL", got.baseURL)
+ }
+}
+
+func TestResolveExtractorChatConfig_MixedCaseDriverAndNilAPIConfig(t *testing.T) {
+ swapModelResolvers(t, func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, _ string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ return namedDriver{name: "AzureOpenAI"}, "gpt-4o", nil, 0, nil
+ }, nil)
+
+ got, err := resolveExtractorChatConfig(extractorStateCtx(t, "tenant-1"), nil, "gpt-4o@azure")
+ if err != nil {
+ t.Fatalf("resolveExtractorChatConfig: %v", err)
+ }
+ if got.driver != "azureopenai" {
+ t.Errorf("driver = %q, want azureopenai", got.driver)
+ }
+ if got.apiKey != "" || got.baseURL != "" {
+ t.Errorf("apiKey/baseURL = %q/%q, want empty for a nil api config", got.apiKey, got.baseURL)
+ }
+}
+
+func TestResolveExtractorChatConfig_ResolverErrorIsWrapped(t *testing.T) {
+ swapModelResolvers(t, func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, ref string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ return nil, "", nil, 0, errors.New("provider unreachable")
+ }, nil)
+
+ _, err := resolveExtractorChatConfig(extractorStateCtx(t, "tenant-1"), nil, "gpt-4o@openai")
+ if err == nil {
+ t.Fatal("expected an error when the resolver fails")
+ }
+ if !strings.Contains(err.Error(), "gpt-4o@openai") {
+ t.Errorf("error %q does not name the model reference", err.Error())
+ }
+}
+
+func TestResolveExtractorChatConfig_NoTenantOrStateYieldsEmptyConfig(t *testing.T) {
+ swapModelResolvers(t, func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, _ string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ t.Error("resolver must not run without a tenant_id")
+ return nil, "", nil, 0, nil
+ }, nil)
+
+ for name, ctx := range map[string]context.Context{
+ "empty tenant_id": extractorStateCtx(t, ""),
+ "no canvas state": t.Context(),
+ } {
+ t.Run(name, func(t *testing.T) {
+ got, err := resolveExtractorChatConfig(ctx, nil, "gpt-4o@openai")
+ if err != nil {
+ t.Fatalf("resolveExtractorChatConfig: %v", err)
+ }
+ if got != (extractorChatConfig{}) {
+ t.Fatalf("config = %+v, want the empty config", got)
+ }
+ })
+ }
+}
+
+func TestResolveExtractorChatDefaultConfig_TenantDefault(t *testing.T) {
+ swapModelResolvers(t, nil, func(_ context.Context, _ *gorm.DB, tenantID string, modelType entity.ModelType, _ string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ if tenantID != "tenant-1" || modelType != entity.ModelTypeChat {
+ t.Errorf("resolver args = (%q, %v)", tenantID, modelType)
+ }
+ return namedDriver{name: "SiliconFlow"}, "bge-m3", &models.APIConfig{ApiKey: ptr("sk-default"), BaseURL: ptr("https://api.siliconflow.cn")}, 0, nil
+ })
+
+ got := resolveExtractorChatDefaultConfig(extractorStateCtx(t, "tenant-1"), nil)
+ if got.driver != "siliconflow" {
+ t.Errorf("driver = %q, want siliconflow", got.driver)
+ }
+ if got.modelName != "bge-m3" {
+ t.Errorf("modelName = %q, want bge-m3", got.modelName)
+ }
+ if got.apiKey != "sk-default" || got.baseURL != "https://api.siliconflow.cn" {
+ t.Errorf("apiKey/baseURL = %q/%q, want the configured credentials", got.apiKey, got.baseURL)
+ }
+}
+
+func TestResolveExtractorChatDefaultConfig_FailuresYieldEmptyConfig(t *testing.T) {
+ t.Run("resolver error", func(t *testing.T) {
+ swapModelResolvers(t, nil, func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, _ string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ return nil, "", nil, 0, errors.New("no default model")
+ })
+ if got := resolveExtractorChatDefaultConfig(extractorStateCtx(t, "tenant-1"), nil); got != (extractorChatConfig{}) {
+ t.Fatalf("config = %+v, want the empty config on resolver error", got)
+ }
+ })
+
+ t.Run("nil driver", func(t *testing.T) {
+ swapModelResolvers(t, nil, func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, _ string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ return nil, "bge-m3", nil, 0, nil
+ })
+ if got := resolveExtractorChatDefaultConfig(extractorStateCtx(t, "tenant-1"), nil); got != (extractorChatConfig{}) {
+ t.Fatalf("config = %+v, want the empty config on a nil driver", got)
+ }
+ })
+
+ t.Run("no tenant", func(t *testing.T) {
+ swapModelResolvers(t, nil, func(_ context.Context, _ *gorm.DB, _ string, _ entity.ModelType, _ string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ t.Error("resolver must not run without a tenant_id")
+ return nil, "", nil, 0, nil
+ })
+ if got := resolveExtractorChatDefaultConfig(t.Context(), nil); got != (extractorChatConfig{}) {
+ t.Fatalf("config = %+v, want the empty config without canvas state", got)
+ }
+ })
+}
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 2a6c74b290..471a45bf4f 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -8,6 +8,8 @@ import (
"context"
"encoding/json"
"fmt"
+ "maps"
+ "slices"
"sort"
"strings"
@@ -46,7 +48,9 @@ var chunkerOutputs = map[string]string{
// (rag/flow/compiler/compiler.py registers component_name = "Compiler"), so a
// canvas saved by the Python frontend and Go's built-in ingestion templates
// both reference the node as "Compiler" and resolve to the same component.
-const componentNameCompiler = "Compiler"
+// The value itself lives in the schema package so the pipeline layer can
+// reference the same identity.
+const componentNameCompiler = schema.ComponentNameCompiler
// KnowledgeCompilerComponent is the runtime.Component surface. Param is set at
// construction from the DSL; per-call overrides flow through the inputs map.
@@ -804,12 +808,7 @@ func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]a
// LLM/embedding work begins.
// mapKeys returns the sorted keys of m, for diagnostics logging.
func mapKeys(m map[string]any) []string {
- keys := make([]string, 0, len(m))
- for k := range m {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- return keys
+ return slices.Sorted(maps.Keys(m))
}
func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, error) {
diff --git a/internal/ingestion/component/knowledge_compiler/mapkeys_test.go b/internal/ingestion/component/knowledge_compiler/mapkeys_test.go
new file mode 100644
index 0000000000..29b1222b5c
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/mapkeys_test.go
@@ -0,0 +1,79 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package knowledge_compiler
+
+import (
+ "fmt"
+ "slices"
+ "testing"
+)
+
+// TestMapKeysReturnsSortedKeys pins the diagnostics contract: mapKeys exists so
+// log lines are deterministic, so the output must be sorted and complete no
+// matter how the input map iterates.
+func TestMapKeysReturnsSortedKeys(t *testing.T) {
+ t.Run("unordered keys come back sorted", func(t *testing.T) {
+ got := mapKeys(map[string]any{"charlie": 1, "alpha": 2, "bravo": 3})
+ want := []string{"alpha", "bravo", "charlie"}
+ if !slices.Equal(got, want) {
+ t.Fatalf("mapKeys = %v, want %v", got, want)
+ }
+ })
+
+ t.Run("empty map yields empty slice", func(t *testing.T) {
+ got := mapKeys(map[string]any{})
+ if len(got) != 0 {
+ t.Fatalf("mapKeys(empty) = %v, want empty", got)
+ }
+ })
+
+ t.Run("nil map yields empty slice", func(t *testing.T) {
+ got := mapKeys(nil)
+ if len(got) != 0 {
+ t.Fatalf("mapKeys(nil) = %v, want empty", got)
+ }
+ })
+
+ t.Run("single key", func(t *testing.T) {
+ got := mapKeys(map[string]any{"only": 1})
+ if !slices.Equal(got, []string{"only"}) {
+ t.Fatalf("mapKeys = %v, want [only]", got)
+ }
+ })
+
+ t.Run("unicode keys sort by byte order like sort.Strings", func(t *testing.T) {
+ got := mapKeys(map[string]any{"中文": 1, "abc": 2, "é": 3, "ZZ": 4})
+ want := []string{"ZZ", "abc", "é", "中文"}
+ if !slices.Equal(got, want) {
+ t.Fatalf("mapKeys = %v, want %v", got, want)
+ }
+ })
+
+ t.Run("many keys stay sorted and complete", func(t *testing.T) {
+ m := make(map[string]any, 100)
+ for i := 99; i >= 0; i-- { // insert in reverse to fight map iteration order
+ m[fmt.Sprintf("key-%03d", i)] = i
+ }
+ got := mapKeys(m)
+ if len(got) != 100 {
+ t.Fatalf("mapKeys returned %d keys, want 100", len(got))
+ }
+ if !slices.IsSorted(got) {
+ t.Fatal("mapKeys output is not sorted")
+ }
+ })
+}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 7e00d23aaa..f2f1512a06 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -94,7 +94,9 @@ import (
"ragflow/internal/utility"
)
-const ComponentNameParser = "Parser"
+// ComponentNameParser is the canonical Parser component name; the value lives
+// in the schema package so the pipeline layer can reference the same identity.
+const ComponentNameParser = schema.ComponentNameParser
// pageFormFeed is the byte that text-page mode treats as a
// hard page boundary. Matches the ASCII form feed (\f, 0x0C) — the
diff --git a/internal/ingestion/component/schema/component_names.go b/internal/ingestion/component/schema/component_names.go
new file mode 100644
index 0000000000..631d78dd02
--- /dev/null
+++ b/internal/ingestion/component/schema/component_names.go
@@ -0,0 +1,31 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package schema
+
+// Canonical ingestion component names. These are the identities components
+// register under in runtime.DefaultRegistry and the values a DSL
+// component_name carries. Layers that must recognise a component by name
+// across the component/pipeline boundary (dispatch, param whitelists,
+// operator-facing failure messages) reference these constants instead of
+// repeating the literals; the schema package is the shared dependency both
+// sides already import. More names can join this set as other layers need
+// them.
+const (
+ ComponentNameParser = "Parser"
+ ComponentNameExtractor = "Extractor"
+ ComponentNameCompiler = "Compiler"
+)
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index fa11a31f02..9d6de83931 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -381,7 +381,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
language := globals.GlobalOrInput(ctx, inputs, "lang", "English")
- if contains(c.param.SearchMethod, "full_text") {
+ if slices.Contains(c.param.SearchMethod, "full_text") {
if err := tokenizeChunks(chunks, titleStem, language); err != nil {
return nil, err
}
@@ -1042,11 +1042,11 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
// This is a deliberate, go_intentional divergence. Do NOT "align" this to a
// path that injects a default embedding.
func shouldHaveEmbedding(searchMethods []string, kbID string) bool {
- return contains(searchMethods, "embedding") && kbID != ""
+ return slices.Contains(searchMethods, "embedding") && kbID != ""
}
func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []string, kbID string) error {
- needFullText := contains(searchMethods, "full_text")
+ needFullText := slices.Contains(searchMethods, "full_text")
needEmbedding := shouldHaveEmbedding(searchMethods, kbID)
if !needFullText && !needEmbedding {
return nil
@@ -1130,10 +1130,6 @@ func cloneInputs(m map[string]any) map[string]any {
return cp
}
-func contains(s []string, v string) bool {
- return slices.Contains(s, v)
-}
-
func intPtr(v int) *int { return &v }
// init registers Tokenizer under CategoryIngestion (plan §4
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index 9b1ff658ee..09e2d4638b 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/common"
kvrocks "ragflow/internal/engine/kvrocks"
"ragflow/internal/ingestion/component/globals"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/utility"
"go.uber.org/zap"
@@ -400,9 +401,9 @@ func formatComponentFailure(componentName string, err error) string {
}
if errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), context.DeadlineExceeded.Error()) {
switch componentName {
- case "Extractor":
+ case schema.ComponentNameExtractor:
return "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."
- case "Compiler":
+ case schema.ComponentNameCompiler:
return "Compiler: Document compilation timed out. Check that the configured model service is reachable and responsive, then retry."
}
}
diff --git a/internal/ingestion/pipeline/pipeline_failure_message_test.go b/internal/ingestion/pipeline/pipeline_failure_message_test.go
new file mode 100644
index 0000000000..2a37a61b69
--- /dev/null
+++ b/internal/ingestion/pipeline/pipeline_failure_message_test.go
@@ -0,0 +1,148 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package pipeline
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "os"
+ "strings"
+ "testing"
+)
+
+// TestFormatComponentFailure covers the operator-facing failure formatting:
+// per-component timeout guidance, prefix stripping, and the bare fallbacks.
+// These messages are what an operator reads when a document fails, so the
+// component identity in them must keep matching the registered names.
+func TestFormatComponentFailure(t *testing.T) {
+ t.Run("nil error reports the component only", func(t *testing.T) {
+ if got := formatComponentFailure("Extractor", nil); got != "Extractor Error" {
+ t.Fatalf("formatComponentFailure(nil) = %q, want %q", got, "Extractor Error")
+ }
+ })
+
+ t.Run("extractor timeout gets extraction guidance", func(t *testing.T) {
+ got := formatComponentFailure("Extractor", context.DeadlineExceeded)
+ want := "Extractor: Document extraction timed out while waiting for the configured LLM. Check that the model service is reachable, then retry."
+ if got != want {
+ t.Fatalf("formatComponentFailure = %q, want %q", got, want)
+ }
+ })
+
+ t.Run("compiler timeout gets compilation guidance", func(t *testing.T) {
+ got := formatComponentFailure("Compiler", context.DeadlineExceeded)
+ want := "Compiler: Document compilation timed out. Check that the configured model service is reachable and responsive, then retry."
+ if got != want {
+ t.Fatalf("formatComponentFailure = %q, want %q", got, want)
+ }
+ })
+
+ t.Run("other components fall through to the generic form", func(t *testing.T) {
+ got := formatComponentFailure("Tokenizer", context.DeadlineExceeded)
+ want := "Tokenizer: context deadline exceeded"
+ if got != want {
+ t.Fatalf("formatComponentFailure = %q, want %q", got, want)
+ }
+ })
+
+ t.Run("wrapped deadline is detected", func(t *testing.T) {
+ err := fmt.Errorf("component failed: %w", context.DeadlineExceeded)
+ got := formatComponentFailure("Extractor", err)
+ if !strings.Contains(got, "timed out") {
+ t.Fatalf("formatComponentFailure(wrapped) = %q, want the timeout guidance", got)
+ }
+ })
+
+ t.Run("timeout wording inside the message is detected", func(t *testing.T) {
+ err := errors.New("LLM call failed: CONTEXT DEADLINE EXCEEDED")
+ got := formatComponentFailure("Extractor", err)
+ if !strings.Contains(got, "timed out") {
+ t.Fatalf("formatComponentFailure(mixed case) = %q, want the timeout guidance", got)
+ }
+ })
+
+ t.Run("component prefix is stripped before reformatting", func(t *testing.T) {
+ got := formatComponentFailure("Extractor", errors.New("Extractor: model quota exceeded"))
+ if got != "Extractor: model quota exceeded" {
+ t.Fatalf("formatComponentFailure = %q, want the prefix not doubled", got)
+ }
+ })
+
+ t.Run("mixed case prefix is stripped", func(t *testing.T) {
+ got := formatComponentFailure("Extractor", errors.New("extractor: model quota exceeded"))
+ if got != "Extractor: model quota exceeded" {
+ t.Fatalf("formatComponentFailure = %q, want the case-insensitive prefix stripped", got)
+ }
+ })
+
+ t.Run("empty component name returns the bare message", func(t *testing.T) {
+ got := formatComponentFailure("", errors.New("boom"))
+ if got != "boom" {
+ t.Fatalf("formatComponentFailure(\"\") = %q, want %q", got, "boom")
+ }
+ })
+
+ t.Run("empty component name and nil error", func(t *testing.T) {
+ if got := formatComponentFailure("", nil); got != " Error" {
+ t.Fatalf("formatComponentFailure(\"\", nil) = %q, want %q", got, " Error")
+ }
+ })
+}
+
+// TestPipelineHasNoBareComponentNameLiterals guards the constant consolidation:
+// component identity strings in the pipeline package must come from the shared
+// schema constants, not inline literals, so a rename touches one place instead
+// of a grep. The scan is AST-based so comments (which may mention a name in
+// prose) are not flagged.
+func TestPipelineHasNoBareComponentNameLiterals(t *testing.T) {
+ banned := []string{
+ `"Extractor"`, `"extractor"`,
+ `"Compiler"`, `"compiler"`,
+ `"Parser"`, `"parser"`,
+ }
+ entries, err := os.ReadDir(".")
+ if err != nil {
+ t.Fatalf("read pipeline package dir: %v", err)
+ }
+ for _, entry := range entries {
+ name := entry.Name()
+ if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+ continue
+ }
+ fset := token.NewFileSet()
+ f, err := parser.ParseFile(fset, name, nil, 0)
+ if err != nil {
+ t.Fatalf("parse %s: %v", name, err)
+ }
+ ast.Inspect(f, func(n ast.Node) bool {
+ lit, ok := n.(*ast.BasicLit)
+ if !ok || lit.Kind != token.STRING {
+ return true
+ }
+ for _, want := range banned {
+ if lit.Value == want {
+ t.Errorf("%s contains the bare component-name literal %s; use the shared schema constant", name, want)
+ }
+ }
+ return true
+ })
+ }
+}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 94b26834cb..304019b75e 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -124,7 +124,7 @@ func extractJSONTags(v any) map[string]struct{} {
func isExtractorComponent(cpnID, componentName string) bool {
lowerName := strings.ToLower(componentName)
lowerID := strings.ToLower(cpnID)
- return lowerName == "extractor" ||
+ return lowerName == strings.ToLower(schema.ComponentNameExtractor) ||
strings.HasPrefix(lowerID, "extractor:") ||
strings.HasPrefix(lowerID, "extractor_")
}
@@ -148,7 +148,7 @@ func IsParserComponentID(cpnID string) bool {
// isParserComponent returns true if the component name or component ID
// indicates a Parser component.
func isParserComponent(cpnID, componentName string) bool {
- return strings.EqualFold(componentName, "parser") || IsParserComponentID(cpnID)
+ return strings.EqualFold(componentName, schema.ComponentNameParser) || IsParserComponentID(cpnID)
}
// NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries in