RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(meta): keep an empty auto meta filter unscoped instead of matching nothing (#20979)
Martin Civáň0f6f08ca017f
diff --git a/internal/agent/retrievalbridge/enhancer.go b/internal/agent/retrievalbridge/enhancer.goindex cb5dc2e7ac..4b685daae8 100644--- a/internal/agent/retrievalbridge/enhancer.go+++ b/internal/agent/retrievalbridge/enhancer.go@@ -71,7 +71,7 @@ func (e *Enhancer) FilterDocuments(if err != nil {return nil, err}- docIDs, noMatches := service.ApplyMetaDataFilter(+ docIDs := service.ApplyMetaDataFilter(ctx,filter,metadata,@@ -80,8 +80,14 @@ func (e *Enhancer) FilterDocuments(baseDocIDs,kbIDs,)- if noMatches {- return []string{service.NoMatchDocIDSentinel}, nil+ // nil means the metadata could not narrow the search -- an auto/semi_auto+ // filter that produced no conditions, or one refused because the value space+ // did not fit the model's context. The scope the caller already asked for+ // still applies; it is the only thing that does. Turning this into the+ // no-match sentinel would send "-999" down as the document scope and return+ // zero chunks, which is not what the filter said.+ if docIDs == nil {+ return baseDocIDs, nil}return docIDs, nil}diff --git a/internal/agent/retrievalbridge/enhancer_test.go b/internal/agent/retrievalbridge/enhancer_test.gonew file mode 100644index 0000000000..997e637f78--- /dev/null+++ b/internal/agent/retrievalbridge/enhancer_test.go@@ -0,0 +1,88 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// The document scope FilterDocuments returns is passed straight into+// nlp.RetrievalRequest.DocIDs, so it decides what the agent can find at all.+// Nothing downstream unpacks the "-999" no-match sentinel -- it reaches the doc+// store as a literal document id and matches no chunk. That is the intended+// answer for manual conditions the user wrote, and the wrong answer for an auto+// filter that could not narrow the search, which ApplyMetaDataFilter reports as+// nil, i.e. no metadata narrowing at all.++package retrievalbridge++import (+ "testing"++ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/service"+)++func newFilterChatModel() *modelModule.ChatModel {+ name := "fake-model"+ driver := modelModule.NewDummyModel(nil, modelModule.URLSuffix{})+ return &modelModule.ChatModel{ModelDriver: driver, ModelName: &name, APIConfig: &modelModule.APIConfig{}}+}++// An auto filter that yields no conditions -- because the question named no+// metadata, or because the value space did not fit the model's context and+// generation was refused -- must leave the caller's scope alone. Answering with+// the no-match sentinel turns "we could not narrow this" into zero recall,+// which is strictly worse than searching the caller's whole scope.+func TestFilterDocumentsKeepsBaseScopeWhenTheFilterCannotNarrow(t *testing.T) {+ base := []string{"doc-1", "doc-2"}++ got, err := NewEnhancer(nil, nil).FilterDocuments(+ t.Context(), map[string]any{"method": "auto"}, "who wrote it?", newFilterChatModel(), base, nil,+ )+ if err != nil {+ t.Fatalf("FilterDocuments: %v", err)+ }++ for _, id := range got {+ if id == service.NoMatchDocIDSentinel {+ t.Fatalf("got %v: a filter that produced no conditions must not scope retrieval to nothing", got)+ }+ }+ if len(got) != len(base) {+ t.Fatalf("got %v, want the caller's scope %v", got, base)+ }+ for i := range got {+ if got[i] != base[i] {+ t.Fatalf("got %v, want the caller's scope %v", got, base)+ }+ }+}++// Manual conditions are the user's own, so a manual filter that matches nothing+// still scopes retrieval to nothing. This is the one case the sentinel exists for.+func TestFilterDocumentsReturnsTheSentinelForManualNoMatch(t *testing.T) {+ filter := map[string]any{+ "method": "manual",+ "logic": "and",+ "manual": []any{map[string]any{"key": "author", "op": "=", "value": "nobody"}},+ }++ got, err := NewEnhancer(nil, nil).FilterDocuments(+ t.Context(), filter, "", nil, []string{"doc-1"}, nil,+ )+ if err != nil {+ t.Fatalf("FilterDocuments: %v", err)+ }+ if len(got) != 1 || got[0] != service.NoMatchDocIDSentinel {+ t.Fatalf("got %v, want [%s]", got, service.NoMatchDocIDSentinel)+ }+}diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex ac7c8a97a3..a5d8039826 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -288,8 +288,11 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.Retrievalcommon.Warn("Failed to get flatted metadata", zap.Error(err))} else {common.Info("metadata filter conditions", zap.Any("filter", filter))- filteredDocIDs, _ := service.ApplyMetaDataFilter(ctx, filter, flattedMeta, req.Question, chatModelForFilter, req.DocIDs, []string(req.Datasets))- docIDs = filteredDocIDs+ // nil means no metadata narrowing: search the caller's scope+ // unfiltered rather than scoping to nothing.+ if filteredDocIDs := service.ApplyMetaDataFilter(ctx, filter, flattedMeta, req.Question, chatModelForFilter, req.DocIDs, []string(req.Datasets)); filteredDocIDs != nil {+ docIDs = filteredDocIDs+ }common.Info("ApplyMetaDataFilter result", zap.Strings("docIDs", docIDs))}}diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 0f232ca005..0b2362a14d 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -93,7 +93,6 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchquestion := req.QuestiondatasetIDs := req.DatasetIDsmetadataFilter := req.MetadataFilter- hasMetadataCondition := req.MetadataCondition != nilif req.MetadataCondition != nil {manual := make([]interface{}, 0)if conditions, ok := req.MetadataCondition["conditions"].([]interface{}); ok {@@ -243,8 +242,11 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Searchcommon.Warn("Failed to get flatted metadata, using empty metadata for filter", zap.Error(err))flattedMeta = make(common.MetaData)}- filteredDocIDs, filterReturnedEmpty := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, documentIDs, datasetIDs)- docIDs = selectMetadataFilteredDocIDs(docIDs, filteredDocIDs, hasMetadataCondition, filterReturnedEmpty)+ // nil means the metadata filter produced no scope at all, so the+ // request keeps the document scope it came with.+ if filteredDocIDs := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, documentIDs, datasetIDs); filteredDocIDs != nil {+ docIDs = filteredDocIDs+ }}// Apply cross_languages and keyword extraction@@ -383,10 +385,3 @@ func knowledgeCompilationFilter(include *bool, scope string) map[string]interfac}return map[string]interface{}{"or": []map[string]interface{}{ordinary, {"scope_kwd": scope}}}}--func selectMetadataFilteredDocIDs(currentDocIDs, filteredDocIDs []string, hasMetadataCondition, filterReturnedEmpty bool) []string {- if hasMetadataCondition || !filterReturnedEmpty {- return filteredDocIDs- }- return currentDocIDs-}diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.goindex fa552ac432..d37cc00c0a 100644--- a/internal/service/dataset/search_test.go+++ b/internal/service/dataset/search_test.go@@ -164,35 +164,6 @@ func matchesKnowledgeCompilationFilter(filter, row map[string]interface{}) boolreturn true}-func TestSelectMetadataFilteredDocIDs(t *testing.T) {- tests := []struct {- name string- current []string- filtered []string- hasMetadataCondition bool- filterReturnedEmpty bool- want []string- }{- {name: "metadata filter narrows explicit document ids", current: []string{"doc-1"}, filtered: []string{"doc-1"}, want: []string{"doc-1"}},- {name: "metadata filter cannot widen explicit document ids", current: []string{"doc-1"}, filtered: []string{}, want: []string{}},- {name: "empty generated filter keeps explicit document ids", current: []string{"doc-1"}, filtered: nil, filterReturnedEmpty: true, want: []string{"doc-1"}},- {name: "metadata condition uses definitive empty result", current: []string{"doc-1"}, filtered: nil, hasMetadataCondition: true, filterReturnedEmpty: true, want: nil},- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := selectMetadataFilteredDocIDs(tt.current, tt.filtered, tt.hasMetadataCondition, tt.filterReturnedEmpty)- if len(got) != len(tt.want) {- t.Fatalf("got %v, want %v", got, tt.want)- }- for i := range got {- if got[i] != tt.want[i] {- t.Fatalf("got %v, want %v", got, tt.want)- }- }- })- }-}-func TestFormatDatasetSearchChunksCompilationVisibility(t *testing.T) {enabled, disabled := true, falsefor _, tt := range []struct {diff --git a/internal/service/metadata_filter.go b/internal/service/metadata_filter.goindex 6d6540bbef..5067913f41 100644--- a/internal/service/metadata_filter.go+++ b/internal/service/metadata_filter.go@@ -580,7 +580,8 @@ func MetadataConditionToDocIDs(metaData common.MetaData, metadataCondition map[sreturn strings.Join(filtered, ",")}-// ApplyMetaDataFilter applies metadata filtering rules and returns filtered doc_ids+// ApplyMetaDataFilter applies metadata filtering rules and returns the document+// scope retrieval must be constrained to.// Supports three modes:// - auto: generate filter conditions via LLM// - semi_auto: generate conditions using selected metadata keys only via LLM@@ -589,6 +590,21 @@ func MetadataConditionToDocIDs(metaData common.MetaData, metadataCondition map[s// When kbIDs is supplied, metadata filters are pushed down to the doc metadata// index (ES/Infinity) via FilterDocIdsByMetaPushdown instead of being evaluated// in-memory. The in-memory meta_filter path remains the fallback.+//+// The return value carries three distinct outcomes:+//+// - nil -- no metadata narrowing: the generated filter produced no+// conditions, or conditions that selected nothing. Callers keep their own+// document scope and search it unfiltered; a filter that fails to narrow is+// recoverable, one that narrows to the wrong documents is not.+// - []string{NoMatchDocIDSentinel} -- manual conditions that match no document.+// Only manual mode produces the sentinel, and it means zero results on+// purpose: the user asked for those exact conditions.+// - any other slice -- the documents the filter selected.+//+// nil must never be turned into the sentinel: that would answer "the metadata+// was unusable" with "no documents exist", which is the one outcome the user+// did not ask for.func ApplyMetaDataFilter(ctx context.Context,metaDataFilter map[string]interface{},@@ -598,9 +614,9 @@ func ApplyMetaDataFilter(baseDocIDs []string,kbIDs []string,manualValueResolver ...ManualValueResolver,-) ([]string, bool) {+) []string {if metaDataFilter == nil {- return baseDocIDs, false+ return baseDocIDs}method, _ := metaDataFilter["method"].(string)@@ -645,14 +661,16 @@ func ApplyMetaDataFilter(filters, err := GenMetaFilter(ctx, chatModel, metaData, question, nil)if err != nil {common.Warn("Failed to generate meta filter", zap.Error(err))- return baseDocIDs, false+ return baseDocIDs}filteredIDs := runMetadataFilter(filters.Conditions, filters.Logic)docIDs := constrainDocIDs(baseDocIDs, filteredIDs)if len(docIDs) == 0 {- return nil, true // Return nil to indicate auto filter returned empty+ // No conditions, or conditions that selected nothing: the model was+ // not able to narrow the search, so it stays unnarrowed.+ return nil}- return docIDs, false+ return docIDscase "semi_auto":selectedKeys := []string{}@@ -687,14 +705,14 @@ func ApplyMetaDataFilter(filters, err := GenMetaFilter(ctx, chatModel, filteredMeta, question, constraints)if err != nil {common.Warn("Failed to generate meta filter", zap.Error(err))- return baseDocIDs, false+ return baseDocIDs}filteredIDs := runMetadataFilter(filters.Conditions, filters.Logic)docIDs := constrainDocIDs(baseDocIDs, filteredIDs)if len(docIDs) == 0 {- return nil, true+ return nil}- return docIDs, false+ return docIDs}}@@ -705,7 +723,7 @@ func ApplyMetaDataFilter(logic = logicVal}if len(manualFilters) == 0 {- return baseDocIDs, false+ return baseDocIDs}// Apply manual_value_resolver callback if provided@@ -740,12 +758,13 @@ func ApplyMetaDataFilter(filteredIDs := runMetadataFilter(conditions, logic)docIDs := constrainDocIDs(baseDocIDs, filteredIDs)if len(manualFilters) > 0 && len(docIDs) == 0 {- return []string{NoMatchDocIDSentinel}, false+ // The user named these conditions, so no match is the answer.+ return []string{NoMatchDocIDSentinel}}- return docIDs, false+ return docIDs}- return baseDocIDs, false+ return baseDocIDs}// chatMetadataDocIDs is shared by standard retrieval and the agent's corpus tools.@@ -762,11 +781,7 @@ func (s *ChatPipelineService) chatMetadataDocIDs(ctx context.Context, chat *enticommon.Warn("loadMetaData failed; skipping meta_data_filter", zap.Error(err))return docIDs}- filtered, empty := ApplyMetaDataFilter(ctx, *chat.MetaDataFilter, meta, question, model, docIDs, kbIDs)- if empty {- return []string{NoMatchDocIDSentinel}- }- if filtered != nil {+ if filtered := ApplyMetaDataFilter(ctx, *chat.MetaDataFilter, meta, question, model, docIDs, kbIDs); filtered != nil {common.Debug("meta_data_filter applied", zap.Int("filtered_count", len(filtered)), zap.Int("pre_filter_count", len(docIDs)))return filtered}diff --git a/internal/service/metadata_filter_test.go b/internal/service/metadata_filter_test.goindex 626bcb7e4d..2f0811ec4b 100644--- a/internal/service/metadata_filter_test.go+++ b/internal/service/metadata_filter_test.go@@ -17,13 +17,16 @@package serviceimport (+ "context""encoding/json"+ "reflect""slices""sort""strings""testing""ragflow/internal/common"+ modelModule "ragflow/internal/entity/models")func TestRenderMetaFilterTemplate(t *testing.T) {@@ -147,9 +150,9 @@ func TestApplyMetaDataFilterManualNotInArray(t *testing.T) {t.Fatal(err)}// No dataset IDs means the request uses the in-memory filter.- got, empty := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, []string{"doc1", "doc3"}, nil)- if !slices.Equal(got, []string{"doc3"}) || empty {- t.Errorf("ApplyMetaDataFilter() = (%v, %v), want ([doc3], false)", got, empty)+ got := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, []string{"doc1", "doc3"}, nil)+ if !slices.Equal(got, []string{"doc3"}) {+ t.Errorf("ApplyMetaDataFilter() = %v, want [doc3]", got)}}@@ -586,3 +589,132 @@ func TestCompareValuesDirectly(t *testing.T) {}}}++// ApplyMetaDataFilter's outcomes must stay distinguishable: "the metadata could+// not narrow the search" (nil) is not the same answer as "no document matches"+// (["-999"]), because the returned slice is applied as a hard document scope.+func TestApplyMetaDataFilter_SeparatesNoNarrowingFromNoMatch(t *testing.T) {+ base := []string{"doc-1", "doc-2"}+ metas := common.MetaData{"author": {"Zhang San": {"doc-1"}, "Li Si": {"doc-3"}}}++ // The generator answering with no conditions is the common case: the+ // question names no metadata. That does not mean "no such document".+ for _, method := range []string{"auto", "semi_auto"} {+ t.Run(method+" without generated conditions returns no scope", func(t *testing.T) {+ chatModel, driver := newCapturingFilterModel(t)+ filter := map[string]interface{}{"method": method}+ if method == "semi_auto" {+ filter["semi_auto"] = []interface{}{"author"}+ }++ got := ApplyMetaDataFilter(t.Context(), filter, metas, "who wrote it?", chatModel, base, nil)++ if driver.calls != 1 {+ t.Fatalf("model calls: got %d, want 1", driver.calls)+ }+ if got != nil {+ t.Fatalf("got %v, want nil so the caller keeps its own document scope", got)+ }+ })+ }++ // A generated filter is applied as a hard scope, and an empty result cannot+ // tell "outside this scope" from a wrong or invented value, so it leaves+ // the caller's scope unnarrowed rather than matching nothing.+ for _, tc := range []struct {+ name string+ answer string+ want []string+ }{+ {+ name: "generated conditions selecting only documents outside the scope return no scope",+ answer: `{"conditions": [{"key": "author", "op": "=", "value": "Li Si"}], "logic": "and"}`,+ },+ {+ name: "generated conditions selecting nothing return no scope",+ answer: `{"conditions": [{"key": "author", "op": "=", "value": "Wang Wu"}], "logic": "and"}`,+ },+ {+ name: "generated conditions selecting documents in the scope return them",+ answer: `{"conditions": [{"key": "author", "op": "=", "value": "Zhang San"}], "logic": "and"}`,+ want: []string{"doc-1"},+ },+ } {+ t.Run("auto "+tc.name, func(t *testing.T) {+ chatModel, driver := newCapturingFilterModel(t)+ driver.answer = tc.answer++ got := ApplyMetaDataFilter(t.Context(), map[string]interface{}{"method": "auto"}, metas, "who wrote it?", chatModel, base, nil)++ if !reflect.DeepEqual(got, tc.want) {+ t.Fatalf("got %v, want %v", got, tc.want)+ }+ })+ }++ t.Run("manual conditions matching nothing return the sentinel", func(t *testing.T) {+ filter := map[string]interface{}{+ "method": "manual",+ "logic": "and",+ "manual": []interface{}{map[string]interface{}{"key": "author", "op": "=", "value": "nobody"}},+ }++ got := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, base, nil)++ if len(got) != 1 || got[0] != NoMatchDocIDSentinel {+ t.Fatalf("got %v, want [%s]: the user asked for these exact conditions", got, NoMatchDocIDSentinel)+ }+ })++ t.Run("manual conditions that match return their documents", func(t *testing.T) {+ filter := map[string]interface{}{+ "method": "manual",+ "logic": "and",+ "manual": []interface{}{map[string]interface{}{"key": "author", "op": "=", "value": "Zhang San"}},+ }++ got := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, base, nil)++ if len(got) != 1 || got[0] != "doc-1" {+ t.Fatalf("got %v, want [doc-1]", got)+ }+ })+}++// capturingFilterDriver records the system prompt gen_meta_filter sends and+// answers with answer, or an empty filter when it is unset, so a test can+// assert what the model was shown and choose what it generates.+type capturingFilterDriver struct {+ *modelModule.DummyModel+ calls int+ system string+ answer string+}++func (d *capturingFilterDriver) ChatWithMessages(+ _ context.Context,+ _ string,+ messages []modelModule.Message,+ _ *modelModule.APIConfig,+ _ *modelModule.ChatConfig,+ _ *common.ModelUsage,+) (*modelModule.ChatResponse, error) {+ d.calls+++ if len(messages) > 0 {+ if s, ok := messages[0].Content.(string); ok {+ d.system = s+ }+ }+ answer := d.answer+ if answer == "" {+ answer = `{"conditions": [], "logic": "and"}`+ }+ return &modelModule.ChatResponse{Answer: &answer}, nil+}++func newCapturingFilterModel(t *testing.T) (*modelModule.ChatModel, *capturingFilterDriver) {+ t.Helper()+ driver := &capturingFilterDriver{DummyModel: modelModule.NewDummyModel(nil, modelModule.URLSuffix{})}+ name := "fake-model"+ return &modelModule.ChatModel{ModelDriver: driver, ModelName: &name, APIConfig: &modelModule.APIConfig{}}, driver+}