Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(retrieval): preserve metadata filters during agentic thinking (#20707)

Sbaaoui Idriss2d34209f5d81
diff --git a/internal/agentic_rag/agent.go b/internal/agentic_rag/agent.go
index 76418be10a..332393313a 100644
--- a/internal/agentic_rag/agent.go
+++ b/internal/agentic_rag/agent.go
@@ -70,6 +70,9 @@ type Input struct {
// DatasetIDs is the conversation's bound dataset scope, decided by the
// session created in the UI. It is injected into the retrieval tools.
DatasetIDs []string
+ // DocIDs is the server-bound document scope. Nil leaves retrieval unrestricted;
+ // a non-nil empty slice restricts all corpus tools to no documents.
+ DocIDs []string
// Tools are the eino tools the agent may call. When empty, the tool set of
// the resolved template (TemplateID, else the config's default) is used.
// The web_search tool is NOT passed here: it is injected from the run's
@@ -292,6 +295,9 @@ func Run(ctx context.Context, in Input) (string, error) {
if in.Model == nil {
return "", errNilModel
}
+ if in.DocIDs != nil {
+ ctx = withDocumentScope(ctx, in.DocIDs)
+ }
tmpl, errT := resolveTemplateFor(in.TemplateID)
if errT != nil {
diff --git a/internal/agentic_rag/helper.go b/internal/agentic_rag/helper.go
index 1dac45a6bd..a79465fcba 100644
--- a/internal/agentic_rag/helper.go
+++ b/internal/agentic_rag/helper.go
@@ -114,6 +114,52 @@ func resolveDatasetScope(bound, requested []string) ([]string, error) {
return requested, nil
}
+type documentScopeContextKey struct{}
+
+// withDocumentScope binds the chat's allowed documents to every agent tool call.
+// A non-nil empty scope allows no documents; nil leaves document search unrestricted.
+func withDocumentScope(ctx context.Context, ids []string) context.Context {
+ return context.WithValue(ctx, documentScopeContextKey{}, ids)
+}
+
+func resolveDocumentScope(ctx context.Context, requested []string) ([]string, error) {
+ bound, ok := ctx.Value(documentScopeContextKey{}).([]string)
+ if !ok || bound == nil {
+ return requested, nil
+ }
+ if len(requested) == 0 {
+ return bound, nil
+ }
+ allowed := make(map[string]struct{}, len(bound))
+ for _, id := range bound {
+ allowed[id] = struct{}{}
+ }
+ for _, id := range requested {
+ if _, ok := allowed[id]; !ok {
+ return nil, fmt.Errorf("doc_id %q is outside the conversation's bound scope", id)
+ }
+ }
+ return requested, nil
+}
+
+func intersectDocumentScope(ctx context.Context, ids []string) []string {
+ bound, ok := ctx.Value(documentScopeContextKey{}).([]string)
+ if !ok || bound == nil {
+ return ids
+ }
+ allowed := make(map[string]struct{}, len(bound))
+ for _, id := range bound {
+ allowed[id] = struct{}{}
+ }
+ out := make([]string, 0, len(ids))
+ for _, id := range ids {
+ if _, ok := allowed[id]; ok {
+ out = append(out, id)
+ }
+ }
+ return out
+}
+
// clampFloat01 clamps a float into [0, 1], used for similarity weights the model
// may supply out of range.
func clampFloat01(v float64) float64 {
diff --git a/internal/agentic_rag/locate_search.go b/internal/agentic_rag/locate_search.go
index 71551697ef..bc039df3db 100644
--- a/internal/agentic_rag/locate_search.go
+++ b/internal/agentic_rag/locate_search.go
@@ -105,6 +105,13 @@ func runLocateSearch(ctx context.Context, spec locateSearchSpec) (string, error)
if len(docScope) > 10 {
docScope = docScope[:10]
}
+ docScope, err = resolveDocumentScope(ctx, docScope)
+ if err != nil {
+ return "", fmt.Errorf("%s: %w", spec.tool, err)
+ }
+ if docScope != nil && len(docScope) == 0 {
+ return formatLocateResultsXML(ctx, spec.tool, strings.Join(queries, " | "), nil), nil
+ }
svc := runtime.GetRetrievalService()
tenantID := spec.tenantID
diff --git a/internal/agentic_rag/tool_grep_chunks.go b/internal/agentic_rag/tool_grep_chunks.go
index 81d7b8af93..465c59dbfc 100644
--- a/internal/agentic_rag/tool_grep_chunks.go
+++ b/internal/agentic_rag/tool_grep_chunks.go
@@ -150,6 +150,13 @@ func (g *GrepChunksTool) invokableRun(ctx context.Context, argumentsInJSON strin
if err != nil {
return "", fmt.Errorf("grep_chunks: %w", err)
}
+ docScope, err := resolveDocumentScope(ctx, args.DocScope)
+ if err != nil {
+ return "", fmt.Errorf("grep_chunks: %w", err)
+ }
+ if docScope != nil && len(docScope) == 0 {
+ return formatGrepResults(ctx, query, nil, re), nil
+ }
if len(datasetIDs) == 0 {
// Bound scope is empty: short-circuit before touching the backend so the
// tool never reads outside the conversation's allowed datasets.
@@ -161,7 +168,7 @@ func (g *GrepChunksTool) invokableRun(ctx context.Context, argumentsInJSON strin
req := runtime.GrepRequest{
Pattern: query,
DatasetIDs: datasetIDs,
- DocScope: args.DocScope,
+ DocScope: docScope,
Sort: grepChunksSortFields, // order by doc_id, page_num_int, chunk_order_int
SelectFields: grepChunksSelectFields,
TenantID: tenantID,
diff --git a/internal/agentic_rag/tool_list_chunks.go b/internal/agentic_rag/tool_list_chunks.go
index 7dbcb7de1d..d446e13e03 100644
--- a/internal/agentic_rag/tool_list_chunks.go
+++ b/internal/agentic_rag/tool_list_chunks.go
@@ -153,6 +153,9 @@ func (l *ListChunksTool) invokableRun(ctx context.Context, argumentsInJSON strin
if docID == "" {
return "", fmt.Errorf("list_chunks: doc_id is required")
}
+ if _, err := resolveDocumentScope(ctx, []string{docID}); err != nil {
+ return "", fmt.Errorf("list_chunks: %w", err)
+ }
anchors := nonEmptyStrings(args.AnchorChunkIDs)
if len(anchors) == 0 {
return "", fmt.Errorf(
diff --git a/internal/agentic_rag/tool_search_bm25_chunks.go b/internal/agentic_rag/tool_search_bm25_chunks.go
index dfdf129d99..bc00c4c96a 100644
--- a/internal/agentic_rag/tool_search_bm25_chunks.go
+++ b/internal/agentic_rag/tool_search_bm25_chunks.go
@@ -152,12 +152,19 @@ func (k *SearchBm25ChunksTool) invokableRun(ctx context.Context, argumentsInJSON
if len(args.DocScope) > 10 {
args.DocScope = args.DocScope[:10]
}
+ docScope, err := resolveDocumentScope(ctx, args.DocScope)
+ if err != nil {
+ return "", fmt.Errorf("search_bm25_chunks: %w", err)
+ }
+ if docScope != nil && len(docScope) == 0 {
+ return formatLocateResultsXML(ctx, searchBm25ChunksToolName, strings.Join(queries, " | "), nil), nil
+ }
svc := runtime.GetBm25Service()
chunks, err := svc.SearchBm25(ctx, runtime.Bm25Request{
Queries: queries,
DatasetIDs: datasetIDs,
- DocScope: args.DocScope,
+ DocScope: docScope,
TopN: topN,
TenantID: k.tenantID,
})
diff --git a/internal/agentic_rag/tool_search_metadata.go b/internal/agentic_rag/tool_search_metadata.go
index 1bb539c7bf..bb0262b9cb 100644
--- a/internal/agentic_rag/tool_search_metadata.go
+++ b/internal/agentic_rag/tool_search_metadata.go
@@ -195,6 +195,7 @@ func (m *MetadataSearchTool) invokableRun(ctx context.Context, argumentsInJSON s
Logic: logic,
})
}
+ docIDs = intersectDocumentScope(ctx, docIDs)
if len(docIDs) == 0 {
return metadataSearchResult([]string{}, nil,
"No documents match the given metadata conditions. Loosen or change the filters, or use search_chunks / grep_chunks for unfiltered search.")
diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.go
index 20ad17694e..5507e8b961 100644
--- a/internal/service/agentic_pipeline.go
+++ b/internal/service/agentic_pipeline.go
@@ -113,6 +113,7 @@ func (s *ChatPipelineService) agenticRag(
emitResult(AsyncChatResult{Answer: fmt.Sprintf("**ERROR**: %s", chainErr.Error()), Final: true})
return
}
+ docIDs := s.chatMetadataDocIDs(runCtx, chat, lastUserQuestion(messages), chain[0], nil, chatDatasetIDs(chat))
// Apply the dialog's LLM settings with per-request overrides exactly
// like the regular AsyncChat path does, then let the template pin its
@@ -166,6 +167,7 @@ func (s *ChatPipelineService) agenticRag(
TemplateID: mode,
TenantID: chat.TenantID,
DatasetIDs: chatDatasetIDs(chat),
+ DocIDs: docIDs,
Stream: stream,
ToolCallCounts: toolCounts,
ToolCallErrors: toolErrors,
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 80e3cb4e59..5b2c9518fd 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -643,34 +643,7 @@ func (s *ChatPipelineService) AsyncChat(
// meta_data_filter — use LLM to map the question to metadata
// criteria, then filter docIDs to matching
// documents only.
- if chat.MetaDataFilter != nil && len(*chat.MetaDataFilter) > 0 && len(kbs) > 0 {
- kbIDs := kbIDStrings(kbs)
- if metaQ := questions[len(questions)-1]; metaQ != "" {
- var flattedMeta common.MetaData
- var mErr error
- if s.MetadataSvc != nil {
- flattedMeta, mErr = s.MetadataSvc.GetFlattedMetaByKBs(ctx, kbIDs)
- }
- if mErr == nil {
- if filtered, _ := ApplyMetaDataFilter(
- ctx,
- *chat.MetaDataFilter,
- flattedMeta,
- metaQ,
- chatModel,
- docIDs,
- kbIDs,
- ); filtered != nil {
- common.Debug("meta_data_filter applied",
- zap.Int("filtered_count", len(filtered)),
- zap.Int("pre_filter_count", len(docIDs)))
- docIDs = filtered
- }
- } else {
- common.Warn("loadMetaData failed; skipping meta_data_filter", zap.Error(mErr))
- }
- }
- }
+ docIDs = s.chatMetadataDocIDs(ctx, chat, questions[len(questions)-1], chatModel, docIDs, kbIDStrings(kbs))
// keyword — extract top-N keywords from the question via LLM and
// append them to the question text to boost lexical retrieval recall.
diff --git a/internal/service/metadata_filter.go b/internal/service/metadata_filter.go
index b3f37c17fe..42dbd78226 100644
--- a/internal/service/metadata_filter.go
+++ b/internal/service/metadata_filter.go
@@ -24,6 +24,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
+ "ragflow/internal/entity"
"regexp"
"strconv"
@@ -768,6 +769,31 @@ func ApplyMetaDataFilter(
return baseDocIDs, false
}
+// chatMetadataDocIDs is shared by standard retrieval and the agent's corpus tools.
+func (s *ChatPipelineService) chatMetadataDocIDs(ctx context.Context, chat *entity.Chat, question string, model *modelModule.ChatModel, docIDs, kbIDs []string) []string {
+ if chat.MetaDataFilter == nil || len(*chat.MetaDataFilter) == 0 || len(kbIDs) == 0 || question == "" {
+ return docIDs
+ }
+ var meta common.MetaData
+ var err error
+ if s.MetadataSvc != nil {
+ meta, err = s.MetadataSvc.GetFlattedMetaByKBs(ctx, kbIDs)
+ }
+ if err != nil {
+ common.Warn("loadMetaData failed; skipping meta_data_filter", zap.Error(err))
+ return docIDs
+ }
+ filtered, empty := ApplyMetaDataFilter(ctx, *chat.MetaDataFilter, meta, question, model, docIDs, kbIDs)
+ if empty {
+ return []string{NoMatchDocIDSentinel}
+ }
+ if filtered != nil {
+ common.Debug("meta_data_filter applied", zap.Int("filtered_count", len(filtered)), zap.Int("pre_filter_count", len(docIDs)))
+ return filtered
+ }
+ return docIDs
+}
+
func constrainDocIDs(baseDocIDs, filteredDocIDs []string) []string {
filteredDocIDs = common.Deduplicate(filteredDocIDs)
if len(baseDocIDs) == 0 {