Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(meta): keep an empty auto meta filter unscoped instead of matching nothing (#20979)

Martin Civáň0f6f08ca017f
diff --git a/internal/agent/retrievalbridge/enhancer.go b/internal/agent/retrievalbridge/enhancer.go
index cb5dc2e7ac..4b685daae8 100644
--- a/internal/agent/retrievalbridge/enhancer.go
+++ b/internal/agent/retrievalbridge/enhancer.go
@@ -71,7 +71,7 @@ func (e *Enhancer) FilterDocuments(
if err != nil {
return nil, err
}
- docIDs, noMatches := service.ApplyMetaDataFilter(
+ docIDs := service.ApplyMetaDataFilter(
ctx,
filter,
metadata,
@@ -80,8 +80,14 @@ func (e *Enhancer) FilterDocuments(
baseDocIDs,
kbIDs,
)
- if noMatches {
- return []string{service.NoMatchDocIDSentinel}, nil
+ // nil means the metadata could not narrow the search -- an auto/semi_auto
+ // filter that produced no conditions, or one refused because the value space
+ // did not fit the model's context. The scope the caller already asked for
+ // still applies; it is the only thing that does. Turning this into the
+ // no-match sentinel would send "-999" down as the document scope and return
+ // zero chunks, which is not what the filter said.
+ if docIDs == nil {
+ return baseDocIDs, nil
}
return docIDs, nil
}
diff --git a/internal/agent/retrievalbridge/enhancer_test.go b/internal/agent/retrievalbridge/enhancer_test.go
new file mode 100644
index 0000000000..997e637f78
--- /dev/null
+++ b/internal/agent/retrievalbridge/enhancer_test.go
@@ -0,0 +1,88 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// The document scope FilterDocuments returns is passed straight into
+// nlp.RetrievalRequest.DocIDs, so it decides what the agent can find at all.
+// Nothing downstream unpacks the "-999" no-match sentinel -- it reaches the doc
+// store as a literal document id and matches no chunk. That is the intended
+// answer for manual conditions the user wrote, and the wrong answer for an auto
+// filter that could not narrow the search, which ApplyMetaDataFilter reports as
+// nil, i.e. no metadata narrowing at all.
+
+package retrievalbridge
+
+import (
+ "testing"
+
+ modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/service"
+)
+
+func newFilterChatModel() *modelModule.ChatModel {
+ name := "fake-model"
+ driver := modelModule.NewDummyModel(nil, modelModule.URLSuffix{})
+ return &modelModule.ChatModel{ModelDriver: driver, ModelName: &name, APIConfig: &modelModule.APIConfig{}}
+}
+
+// An auto filter that yields no conditions -- because the question named no
+// metadata, or because the value space did not fit the model's context and
+// generation was refused -- must leave the caller's scope alone. Answering with
+// the no-match sentinel turns "we could not narrow this" into zero recall,
+// which is strictly worse than searching the caller's whole scope.
+func TestFilterDocumentsKeepsBaseScopeWhenTheFilterCannotNarrow(t *testing.T) {
+ base := []string{"doc-1", "doc-2"}
+
+ got, err := NewEnhancer(nil, nil).FilterDocuments(
+ t.Context(), map[string]any{"method": "auto"}, "who wrote it?", newFilterChatModel(), base, nil,
+ )
+ if err != nil {
+ t.Fatalf("FilterDocuments: %v", err)
+ }
+
+ for _, id := range got {
+ if id == service.NoMatchDocIDSentinel {
+ t.Fatalf("got %v: a filter that produced no conditions must not scope retrieval to nothing", got)
+ }
+ }
+ if len(got) != len(base) {
+ t.Fatalf("got %v, want the caller's scope %v", got, base)
+ }
+ for i := range got {
+ if got[i] != base[i] {
+ t.Fatalf("got %v, want the caller's scope %v", got, base)
+ }
+ }
+}
+
+// Manual conditions are the user's own, so a manual filter that matches nothing
+// still scopes retrieval to nothing. This is the one case the sentinel exists for.
+func TestFilterDocumentsReturnsTheSentinelForManualNoMatch(t *testing.T) {
+ filter := map[string]any{
+ "method": "manual",
+ "logic": "and",
+ "manual": []any{map[string]any{"key": "author", "op": "=", "value": "nobody"}},
+ }
+
+ got, err := NewEnhancer(nil, nil).FilterDocuments(
+ t.Context(), filter, "", nil, []string{"doc-1"}, nil,
+ )
+ if err != nil {
+ t.Fatalf("FilterDocuments: %v", err)
+ }
+ if len(got) != 1 || got[0] != service.NoMatchDocIDSentinel {
+ t.Fatalf("got %v, want [%s]", got, service.NoMatchDocIDSentinel)
+ }
+}
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index ac7c8a97a3..a5d8039826 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -288,8 +288,11 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.Retrieval
common.Warn("Failed to get flatted metadata", zap.Error(err))
} else {
common.Info("metadata filter conditions", zap.Any("filter", filter))
- filteredDocIDs, _ := service.ApplyMetaDataFilter(ctx, filter, flattedMeta, req.Question, chatModelForFilter, req.DocIDs, []string(req.Datasets))
- docIDs = filteredDocIDs
+ // nil means no metadata narrowing: search the caller's scope
+ // unfiltered rather than scoping to nothing.
+ if filteredDocIDs := service.ApplyMetaDataFilter(ctx, filter, flattedMeta, req.Question, chatModelForFilter, req.DocIDs, []string(req.Datasets)); filteredDocIDs != nil {
+ docIDs = filteredDocIDs
+ }
common.Info("ApplyMetaDataFilter result", zap.Strings("docIDs", docIDs))
}
}
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 0f232ca005..0b2362a14d 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -93,7 +93,6 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
question := req.Question
datasetIDs := req.DatasetIDs
metadataFilter := req.MetadataFilter
- hasMetadataCondition := req.MetadataCondition != nil
if req.MetadataCondition != nil {
manual := make([]interface{}, 0)
if conditions, ok := req.MetadataCondition["conditions"].([]interface{}); ok {
@@ -243,8 +242,11 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
common.Warn("Failed to get flatted metadata, using empty metadata for filter", zap.Error(err))
flattedMeta = make(common.MetaData)
}
- filteredDocIDs, filterReturnedEmpty := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, documentIDs, datasetIDs)
- docIDs = selectMetadataFilteredDocIDs(docIDs, filteredDocIDs, hasMetadataCondition, filterReturnedEmpty)
+ // nil means the metadata filter produced no scope at all, so the
+ // request keeps the document scope it came with.
+ if filteredDocIDs := service.ApplyMetaDataFilter(ctx, metadataFilter, flattedMeta, question, chatModelForFilter, documentIDs, datasetIDs); filteredDocIDs != nil {
+ docIDs = filteredDocIDs
+ }
}
// Apply cross_languages and keyword extraction
@@ -383,10 +385,3 @@ func knowledgeCompilationFilter(include *bool, scope string) map[string]interfac
}
return map[string]interface{}{"or": []map[string]interface{}{ordinary, {"scope_kwd": scope}}}
}
-
-func selectMetadataFilteredDocIDs(currentDocIDs, filteredDocIDs []string, hasMetadataCondition, filterReturnedEmpty bool) []string {
- if hasMetadataCondition || !filterReturnedEmpty {
- return filteredDocIDs
- }
- return currentDocIDs
-}
diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.go
index fa552ac432..d37cc00c0a 100644
--- a/internal/service/dataset/search_test.go
+++ b/internal/service/dataset/search_test.go
@@ -164,35 +164,6 @@ func matchesKnowledgeCompilationFilter(filter, row map[string]interface{}) bool
return true
}
-func TestSelectMetadataFilteredDocIDs(t *testing.T) {
- tests := []struct {
- name string
- current []string
- filtered []string
- hasMetadataCondition bool
- filterReturnedEmpty bool
- want []string
- }{
- {name: "metadata filter narrows explicit document ids", current: []string{"doc-1"}, filtered: []string{"doc-1"}, want: []string{"doc-1"}},
- {name: "metadata filter cannot widen explicit document ids", current: []string{"doc-1"}, filtered: []string{}, want: []string{}},
- {name: "empty generated filter keeps explicit document ids", current: []string{"doc-1"}, filtered: nil, filterReturnedEmpty: true, want: []string{"doc-1"}},
- {name: "metadata condition uses definitive empty result", current: []string{"doc-1"}, filtered: nil, hasMetadataCondition: true, filterReturnedEmpty: true, want: nil},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := selectMetadataFilteredDocIDs(tt.current, tt.filtered, tt.hasMetadataCondition, tt.filterReturnedEmpty)
- if len(got) != len(tt.want) {
- t.Fatalf("got %v, want %v", got, tt.want)
- }
- for i := range got {
- if got[i] != tt.want[i] {
- t.Fatalf("got %v, want %v", got, tt.want)
- }
- }
- })
- }
-}
-
func TestFormatDatasetSearchChunksCompilationVisibility(t *testing.T) {
enabled, disabled := true, false
for _, tt := range []struct {
diff --git a/internal/service/metadata_filter.go b/internal/service/metadata_filter.go
index 6d6540bbef..5067913f41 100644
--- a/internal/service/metadata_filter.go
+++ b/internal/service/metadata_filter.go
@@ -580,7 +580,8 @@ func MetadataConditionToDocIDs(metaData common.MetaData, metadataCondition map[s
return strings.Join(filtered, ",")
}
-// ApplyMetaDataFilter applies metadata filtering rules and returns filtered doc_ids
+// ApplyMetaDataFilter applies metadata filtering rules and returns the document
+// scope retrieval must be constrained to.
// Supports three modes:
// - auto: generate filter conditions via LLM
// - semi_auto: generate conditions using selected metadata keys only via LLM
@@ -589,6 +590,21 @@ func MetadataConditionToDocIDs(metaData common.MetaData, metadataCondition map[s
// When kbIDs is supplied, metadata filters are pushed down to the doc metadata
// index (ES/Infinity) via FilterDocIdsByMetaPushdown instead of being evaluated
// in-memory. The in-memory meta_filter path remains the fallback.
+//
+// The return value carries three distinct outcomes:
+//
+// - nil -- no metadata narrowing: the generated filter produced no
+// conditions, or conditions that selected nothing. Callers keep their own
+// document scope and search it unfiltered; a filter that fails to narrow is
+// recoverable, one that narrows to the wrong documents is not.
+// - []string{NoMatchDocIDSentinel} -- manual conditions that match no document.
+// Only manual mode produces the sentinel, and it means zero results on
+// purpose: the user asked for those exact conditions.
+// - any other slice -- the documents the filter selected.
+//
+// nil must never be turned into the sentinel: that would answer "the metadata
+// was unusable" with "no documents exist", which is the one outcome the user
+// did not ask for.
func ApplyMetaDataFilter(
ctx context.Context,
metaDataFilter map[string]interface{},
@@ -598,9 +614,9 @@ func ApplyMetaDataFilter(
baseDocIDs []string,
kbIDs []string,
manualValueResolver ...ManualValueResolver,
-) ([]string, bool) {
+) []string {
if metaDataFilter == nil {
- return baseDocIDs, false
+ return baseDocIDs
}
method, _ := metaDataFilter["method"].(string)
@@ -645,14 +661,16 @@ func ApplyMetaDataFilter(
filters, err := GenMetaFilter(ctx, chatModel, metaData, question, nil)
if err != nil {
common.Warn("Failed to generate meta filter", zap.Error(err))
- return baseDocIDs, false
+ return baseDocIDs
}
filteredIDs := runMetadataFilter(filters.Conditions, filters.Logic)
docIDs := constrainDocIDs(baseDocIDs, filteredIDs)
if len(docIDs) == 0 {
- return nil, true // Return nil to indicate auto filter returned empty
+ // No conditions, or conditions that selected nothing: the model was
+ // not able to narrow the search, so it stays unnarrowed.
+ return nil
}
- return docIDs, false
+ return docIDs
case "semi_auto":
selectedKeys := []string{}
@@ -687,14 +705,14 @@ func ApplyMetaDataFilter(
filters, err := GenMetaFilter(ctx, chatModel, filteredMeta, question, constraints)
if err != nil {
common.Warn("Failed to generate meta filter", zap.Error(err))
- return baseDocIDs, false
+ return baseDocIDs
}
filteredIDs := runMetadataFilter(filters.Conditions, filters.Logic)
docIDs := constrainDocIDs(baseDocIDs, filteredIDs)
if len(docIDs) == 0 {
- return nil, true
+ return nil
}
- return docIDs, false
+ return docIDs
}
}
@@ -705,7 +723,7 @@ func ApplyMetaDataFilter(
logic = logicVal
}
if len(manualFilters) == 0 {
- return baseDocIDs, false
+ return baseDocIDs
}
// Apply manual_value_resolver callback if provided
@@ -740,12 +758,13 @@ func ApplyMetaDataFilter(
filteredIDs := runMetadataFilter(conditions, logic)
docIDs := constrainDocIDs(baseDocIDs, filteredIDs)
if len(manualFilters) > 0 && len(docIDs) == 0 {
- return []string{NoMatchDocIDSentinel}, false
+ // The user named these conditions, so no match is the answer.
+ return []string{NoMatchDocIDSentinel}
}
- return docIDs, false
+ return docIDs
}
- return baseDocIDs, false
+ return baseDocIDs
}
// chatMetadataDocIDs is shared by standard retrieval and the agent's corpus tools.
@@ -762,11 +781,7 @@ func (s *ChatPipelineService) chatMetadataDocIDs(ctx context.Context, chat *enti
common.Warn("loadMetaData failed; skipping meta_data_filter", zap.Error(err))
return docIDs
}
- filtered, empty := ApplyMetaDataFilter(ctx, *chat.MetaDataFilter, meta, question, model, docIDs, kbIDs)
- if empty {
- return []string{NoMatchDocIDSentinel}
- }
- if filtered != nil {
+ if filtered := ApplyMetaDataFilter(ctx, *chat.MetaDataFilter, meta, question, model, docIDs, kbIDs); filtered != nil {
common.Debug("meta_data_filter applied", zap.Int("filtered_count", len(filtered)), zap.Int("pre_filter_count", len(docIDs)))
return filtered
}
diff --git a/internal/service/metadata_filter_test.go b/internal/service/metadata_filter_test.go
index 626bcb7e4d..2f0811ec4b 100644
--- a/internal/service/metadata_filter_test.go
+++ b/internal/service/metadata_filter_test.go
@@ -17,13 +17,16 @@
package service
import (
+ "context"
"encoding/json"
+ "reflect"
"slices"
"sort"
"strings"
"testing"
"ragflow/internal/common"
+ modelModule "ragflow/internal/entity/models"
)
func TestRenderMetaFilterTemplate(t *testing.T) {
@@ -147,9 +150,9 @@ func TestApplyMetaDataFilterManualNotInArray(t *testing.T) {
t.Fatal(err)
}
// No dataset IDs means the request uses the in-memory filter.
- got, empty := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, []string{"doc1", "doc3"}, nil)
- if !slices.Equal(got, []string{"doc3"}) || empty {
- t.Errorf("ApplyMetaDataFilter() = (%v, %v), want ([doc3], false)", got, empty)
+ got := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, []string{"doc1", "doc3"}, nil)
+ if !slices.Equal(got, []string{"doc3"}) {
+ t.Errorf("ApplyMetaDataFilter() = %v, want [doc3]", got)
}
}
@@ -586,3 +589,132 @@ func TestCompareValuesDirectly(t *testing.T) {
}
}
}
+
+// ApplyMetaDataFilter's outcomes must stay distinguishable: "the metadata could
+// not narrow the search" (nil) is not the same answer as "no document matches"
+// (["-999"]), because the returned slice is applied as a hard document scope.
+func TestApplyMetaDataFilter_SeparatesNoNarrowingFromNoMatch(t *testing.T) {
+ base := []string{"doc-1", "doc-2"}
+ metas := common.MetaData{"author": {"Zhang San": {"doc-1"}, "Li Si": {"doc-3"}}}
+
+ // The generator answering with no conditions is the common case: the
+ // question names no metadata. That does not mean "no such document".
+ for _, method := range []string{"auto", "semi_auto"} {
+ t.Run(method+" without generated conditions returns no scope", func(t *testing.T) {
+ chatModel, driver := newCapturingFilterModel(t)
+ filter := map[string]interface{}{"method": method}
+ if method == "semi_auto" {
+ filter["semi_auto"] = []interface{}{"author"}
+ }
+
+ got := ApplyMetaDataFilter(t.Context(), filter, metas, "who wrote it?", chatModel, base, nil)
+
+ if driver.calls != 1 {
+ t.Fatalf("model calls: got %d, want 1", driver.calls)
+ }
+ if got != nil {
+ t.Fatalf("got %v, want nil so the caller keeps its own document scope", got)
+ }
+ })
+ }
+
+ // A generated filter is applied as a hard scope, and an empty result cannot
+ // tell "outside this scope" from a wrong or invented value, so it leaves
+ // the caller's scope unnarrowed rather than matching nothing.
+ for _, tc := range []struct {
+ name string
+ answer string
+ want []string
+ }{
+ {
+ name: "generated conditions selecting only documents outside the scope return no scope",
+ answer: `{"conditions": [{"key": "author", "op": "=", "value": "Li Si"}], "logic": "and"}`,
+ },
+ {
+ name: "generated conditions selecting nothing return no scope",
+ answer: `{"conditions": [{"key": "author", "op": "=", "value": "Wang Wu"}], "logic": "and"}`,
+ },
+ {
+ name: "generated conditions selecting documents in the scope return them",
+ answer: `{"conditions": [{"key": "author", "op": "=", "value": "Zhang San"}], "logic": "and"}`,
+ want: []string{"doc-1"},
+ },
+ } {
+ t.Run("auto "+tc.name, func(t *testing.T) {
+ chatModel, driver := newCapturingFilterModel(t)
+ driver.answer = tc.answer
+
+ got := ApplyMetaDataFilter(t.Context(), map[string]interface{}{"method": "auto"}, metas, "who wrote it?", chatModel, base, nil)
+
+ if !reflect.DeepEqual(got, tc.want) {
+ t.Fatalf("got %v, want %v", got, tc.want)
+ }
+ })
+ }
+
+ t.Run("manual conditions matching nothing return the sentinel", func(t *testing.T) {
+ filter := map[string]interface{}{
+ "method": "manual",
+ "logic": "and",
+ "manual": []interface{}{map[string]interface{}{"key": "author", "op": "=", "value": "nobody"}},
+ }
+
+ got := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, base, nil)
+
+ if len(got) != 1 || got[0] != NoMatchDocIDSentinel {
+ t.Fatalf("got %v, want [%s]: the user asked for these exact conditions", got, NoMatchDocIDSentinel)
+ }
+ })
+
+ t.Run("manual conditions that match return their documents", func(t *testing.T) {
+ filter := map[string]interface{}{
+ "method": "manual",
+ "logic": "and",
+ "manual": []interface{}{map[string]interface{}{"key": "author", "op": "=", "value": "Zhang San"}},
+ }
+
+ got := ApplyMetaDataFilter(t.Context(), filter, metas, "", nil, base, nil)
+
+ if len(got) != 1 || got[0] != "doc-1" {
+ t.Fatalf("got %v, want [doc-1]", got)
+ }
+ })
+}
+
+// capturingFilterDriver records the system prompt gen_meta_filter sends and
+// answers with answer, or an empty filter when it is unset, so a test can
+// assert what the model was shown and choose what it generates.
+type capturingFilterDriver struct {
+ *modelModule.DummyModel
+ calls int
+ system string
+ answer string
+}
+
+func (d *capturingFilterDriver) ChatWithMessages(
+ _ context.Context,
+ _ string,
+ messages []modelModule.Message,
+ _ *modelModule.APIConfig,
+ _ *modelModule.ChatConfig,
+ _ *common.ModelUsage,
+) (*modelModule.ChatResponse, error) {
+ d.calls++
+ if len(messages) > 0 {
+ if s, ok := messages[0].Content.(string); ok {
+ d.system = s
+ }
+ }
+ answer := d.answer
+ if answer == "" {
+ answer = `{"conditions": [], "logic": "and"}`
+ }
+ return &modelModule.ChatResponse{Answer: &answer}, nil
+}
+
+func newCapturingFilterModel(t *testing.T) (*modelModule.ChatModel, *capturingFilterDriver) {
+ t.Helper()
+ driver := &capturingFilterDriver{DummyModel: modelModule.NewDummyModel(nil, modelModule.URLSuffix{})}
+ name := "fake-model"
+ return &modelModule.ChatModel{ModelDriver: driver, ModelName: &name, APIConfig: &modelModule.APIConfig{}}, driver
+}