RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(tokenizer): replace the default fields instead of appending (#21799)
CCLUCKY4f539f62b349
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex fc7f0eaf8b..1c0bb2b162 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -297,17 +297,32 @@ func newTokenizerComponent(params map[string]any, resolver EmbedderResolver) (ru}}if v, ok := params["fields"]; ok {+ // Replace (not append) so fields = ["summary"] does not+ // silently embed the default "text" as well. Blank names+ // are dropped: an empty selection keeps the default+ // instead of selecting no text at all.+ var fields []string+ add := func(s string) {+ if s = strings.TrimSpace(s); s != "" {+ fields = append(fields, s)+ }+ }switch t := v.(type) {case string:- p.Fields = []string{t}+ add(t)case []any:for _, x := range t {if s, ok := x.(string); ok {- p.Fields = append(p.Fields, s)+ add(s)}}case []string:- p.Fields = append(p.Fields, t...)+ for _, s := range t {+ add(s)+ }+ }+ if len(fields) > 0 {+ p.Fields = fields}}}diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 233e241753..ae766f85dc 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -383,6 +383,39 @@ func TestTokenizerComponent_NewTokenizerComponent_Defaults(t *testing.T) {}}+// TestTokenizerComponent_NewTokenizerComponent_FieldsReplaceDefault+// verifies that a caller-supplied fields value is the full set, not an+// addition to the ["text"] default, and that an empty value keeps it.+func TestTokenizerComponent_NewTokenizerComponent_FieldsReplaceDefault(t *testing.T) {+ cases := []struct {+ name string+ fields any+ want []string+ }{+ {name: "string", fields: "summary", want: []string{"summary"}},+ {name: "string slice", fields: []string{"text"}, want: []string{"text"}},+ {name: "any slice", fields: []any{"keywords", "summary"}, want: []string{"keywords", "summary"}},+ {name: "empty string slice", fields: []string{}, want: []string{"text"}},+ {name: "empty any slice", fields: []any{}, want: []string{"text"}},+ {name: "blank string", fields: "", want: []string{"text"}},+ {name: "blank entries dropped", fields: []any{"keywords", ""}, want: []string{"keywords"}},+ {name: "whitespace string", fields: " ", want: []string{"text"}},+ {name: "whitespace trimmed", fields: []string{" summary ", " "}, want: []string{"summary"}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ c, err := NewTokenizerComponent(map[string]any{"fields": tc.fields})+ if err != nil {+ t.Fatalf("NewTokenizerComponent: %v", err)+ }+ got := c.(*TokenizerComponent).param.Fields+ if !reflect.DeepEqual(got, tc.want) {+ t.Fatalf("fields = %v, want %v", got, tc.want)+ }+ })+ }+}+// TestTokenizerComponent_NewTokenizerComponent_BadParam covers// the param-validation branch (invalid search_method value).func TestTokenizerComponent_NewTokenizerComponent_BadParam(t *testing.T) {