Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(tokenizer): replace the default fields instead of appending (#21799)

CCLUCKY4f539f62b349
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index fc7f0eaf8b..1c0bb2b162 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -297,17 +297,32 @@ func newTokenizerComponent(params map[string]any, resolver EmbedderResolver) (ru
}
}
if v, ok := params["fields"]; ok {
+ // Replace (not append) so fields = ["summary"] does not
+ // silently embed the default "text" as well. Blank names
+ // are dropped: an empty selection keeps the default
+ // instead of selecting no text at all.
+ var fields []string
+ add := func(s string) {
+ if s = strings.TrimSpace(s); s != "" {
+ fields = append(fields, s)
+ }
+ }
switch t := v.(type) {
case string:
- p.Fields = []string{t}
+ add(t)
case []any:
for _, x := range t {
if s, ok := x.(string); ok {
- p.Fields = append(p.Fields, s)
+ add(s)
}
}
case []string:
- p.Fields = append(p.Fields, t...)
+ for _, s := range t {
+ add(s)
+ }
+ }
+ if len(fields) > 0 {
+ p.Fields = fields
}
}
}
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 233e241753..ae766f85dc 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -383,6 +383,39 @@ func TestTokenizerComponent_NewTokenizerComponent_Defaults(t *testing.T) {
}
}
+// TestTokenizerComponent_NewTokenizerComponent_FieldsReplaceDefault
+// verifies that a caller-supplied fields value is the full set, not an
+// addition to the ["text"] default, and that an empty value keeps it.
+func TestTokenizerComponent_NewTokenizerComponent_FieldsReplaceDefault(t *testing.T) {
+ cases := []struct {
+ name string
+ fields any
+ want []string
+ }{
+ {name: "string", fields: "summary", want: []string{"summary"}},
+ {name: "string slice", fields: []string{"text"}, want: []string{"text"}},
+ {name: "any slice", fields: []any{"keywords", "summary"}, want: []string{"keywords", "summary"}},
+ {name: "empty string slice", fields: []string{}, want: []string{"text"}},
+ {name: "empty any slice", fields: []any{}, want: []string{"text"}},
+ {name: "blank string", fields: "", want: []string{"text"}},
+ {name: "blank entries dropped", fields: []any{"keywords", ""}, want: []string{"keywords"}},
+ {name: "whitespace string", fields: " ", want: []string{"text"}},
+ {name: "whitespace trimmed", fields: []string{" summary ", " "}, want: []string{"summary"}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c, err := NewTokenizerComponent(map[string]any{"fields": tc.fields})
+ if err != nil {
+ t.Fatalf("NewTokenizerComponent: %v", err)
+ }
+ got := c.(*TokenizerComponent).param.Fields
+ if !reflect.DeepEqual(got, tc.want) {
+ t.Fatalf("fields = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
+
// TestTokenizerComponent_NewTokenizerComponent_BadParam covers
// the param-validation branch (invalid search_method value).
func TestTokenizerComponent_NewTokenizerComponent_BadParam(t *testing.T) {