RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(retrieval): preserve metadata filters during agentic thinking (#20707)
Sbaaoui Idriss2d34209f5d81
diff --git a/internal/agentic_rag/agent.go b/internal/agentic_rag/agent.goindex 76418be10a..332393313a 100644--- a/internal/agentic_rag/agent.go+++ b/internal/agentic_rag/agent.go@@ -70,6 +70,9 @@ type Input struct {// DatasetIDs is the conversation's bound dataset scope, decided by the// session created in the UI. It is injected into the retrieval tools.DatasetIDs []string+ // DocIDs is the server-bound document scope. Nil leaves retrieval unrestricted;+ // a non-nil empty slice restricts all corpus tools to no documents.+ DocIDs []string// Tools are the eino tools the agent may call. When empty, the tool set of// the resolved template (TemplateID, else the config's default) is used.// The web_search tool is NOT passed here: it is injected from the run's@@ -292,6 +295,9 @@ func Run(ctx context.Context, in Input) (string, error) {if in.Model == nil {return "", errNilModel}+ if in.DocIDs != nil {+ ctx = withDocumentScope(ctx, in.DocIDs)+ }tmpl, errT := resolveTemplateFor(in.TemplateID)if errT != nil {diff --git a/internal/agentic_rag/helper.go b/internal/agentic_rag/helper.goindex 1dac45a6bd..a79465fcba 100644--- a/internal/agentic_rag/helper.go+++ b/internal/agentic_rag/helper.go@@ -114,6 +114,52 @@ func resolveDatasetScope(bound, requested []string) ([]string, error) {return requested, nil}+type documentScopeContextKey struct{}++// withDocumentScope binds the chat's allowed documents to every agent tool call.+// A non-nil empty scope allows no documents; nil leaves document search unrestricted.+func withDocumentScope(ctx context.Context, ids []string) context.Context {+ return context.WithValue(ctx, documentScopeContextKey{}, ids)+}++func resolveDocumentScope(ctx context.Context, requested []string) ([]string, error) {+ bound, ok := ctx.Value(documentScopeContextKey{}).([]string)+ if !ok || bound == nil {+ return requested, nil+ }+ if len(requested) == 0 {+ return bound, nil+ }+ allowed := make(map[string]struct{}, len(bound))+ for _, id := range bound {+ allowed[id] = struct{}{}+ }+ for _, id := range requested {+ if _, ok := allowed[id]; !ok {+ return nil, fmt.Errorf("doc_id %q is outside the conversation's bound scope", id)+ }+ }+ return requested, nil+}++func intersectDocumentScope(ctx context.Context, ids []string) []string {+ bound, ok := ctx.Value(documentScopeContextKey{}).([]string)+ if !ok || bound == nil {+ return ids+ }+ allowed := make(map[string]struct{}, len(bound))+ for _, id := range bound {+ allowed[id] = struct{}{}+ }+ out := make([]string, 0, len(ids))+ for _, id := range ids {+ if _, ok := allowed[id]; ok {+ out = append(out, id)+ }+ }+ return out+}+// clampFloat01 clamps a float into [0, 1], used for similarity weights the model// may supply out of range.func clampFloat01(v float64) float64 {diff --git a/internal/agentic_rag/locate_search.go b/internal/agentic_rag/locate_search.goindex 71551697ef..bc039df3db 100644--- a/internal/agentic_rag/locate_search.go+++ b/internal/agentic_rag/locate_search.go@@ -105,6 +105,13 @@ func runLocateSearch(ctx context.Context, spec locateSearchSpec) (string, error)if len(docScope) > 10 {docScope = docScope[:10]}+ docScope, err = resolveDocumentScope(ctx, docScope)+ if err != nil {+ return "", fmt.Errorf("%s: %w", spec.tool, err)+ }+ if docScope != nil && len(docScope) == 0 {+ return formatLocateResultsXML(ctx, spec.tool, strings.Join(queries, " | "), nil), nil+ }svc := runtime.GetRetrievalService()tenantID := spec.tenantIDdiff --git a/internal/agentic_rag/tool_grep_chunks.go b/internal/agentic_rag/tool_grep_chunks.goindex 81d7b8af93..465c59dbfc 100644--- a/internal/agentic_rag/tool_grep_chunks.go+++ b/internal/agentic_rag/tool_grep_chunks.go@@ -150,6 +150,13 @@ func (g *GrepChunksTool) invokableRun(ctx context.Context, argumentsInJSON strinif err != nil {return "", fmt.Errorf("grep_chunks: %w", err)}+ docScope, err := resolveDocumentScope(ctx, args.DocScope)+ if err != nil {+ return "", fmt.Errorf("grep_chunks: %w", err)+ }+ if docScope != nil && len(docScope) == 0 {+ return formatGrepResults(ctx, query, nil, re), nil+ }if len(datasetIDs) == 0 {// Bound scope is empty: short-circuit before touching the backend so the// tool never reads outside the conversation's allowed datasets.@@ -161,7 +168,7 @@ func (g *GrepChunksTool) invokableRun(ctx context.Context, argumentsInJSON strinreq := runtime.GrepRequest{Pattern: query,DatasetIDs: datasetIDs,- DocScope: args.DocScope,+ DocScope: docScope,Sort: grepChunksSortFields, // order by doc_id, page_num_int, chunk_order_intSelectFields: grepChunksSelectFields,TenantID: tenantID,diff --git a/internal/agentic_rag/tool_list_chunks.go b/internal/agentic_rag/tool_list_chunks.goindex 7dbcb7de1d..d446e13e03 100644--- a/internal/agentic_rag/tool_list_chunks.go+++ b/internal/agentic_rag/tool_list_chunks.go@@ -153,6 +153,9 @@ func (l *ListChunksTool) invokableRun(ctx context.Context, argumentsInJSON strinif docID == "" {return "", fmt.Errorf("list_chunks: doc_id is required")}+ if _, err := resolveDocumentScope(ctx, []string{docID}); err != nil {+ return "", fmt.Errorf("list_chunks: %w", err)+ }anchors := nonEmptyStrings(args.AnchorChunkIDs)if len(anchors) == 0 {return "", fmt.Errorf(diff --git a/internal/agentic_rag/tool_search_bm25_chunks.go b/internal/agentic_rag/tool_search_bm25_chunks.goindex dfdf129d99..bc00c4c96a 100644--- a/internal/agentic_rag/tool_search_bm25_chunks.go+++ b/internal/agentic_rag/tool_search_bm25_chunks.go@@ -152,12 +152,19 @@ func (k *SearchBm25ChunksTool) invokableRun(ctx context.Context, argumentsInJSONif len(args.DocScope) > 10 {args.DocScope = args.DocScope[:10]}+ docScope, err := resolveDocumentScope(ctx, args.DocScope)+ if err != nil {+ return "", fmt.Errorf("search_bm25_chunks: %w", err)+ }+ if docScope != nil && len(docScope) == 0 {+ return formatLocateResultsXML(ctx, searchBm25ChunksToolName, strings.Join(queries, " | "), nil), nil+ }svc := runtime.GetBm25Service()chunks, err := svc.SearchBm25(ctx, runtime.Bm25Request{Queries: queries,DatasetIDs: datasetIDs,- DocScope: args.DocScope,+ DocScope: docScope,TopN: topN,TenantID: k.tenantID,})diff --git a/internal/agentic_rag/tool_search_metadata.go b/internal/agentic_rag/tool_search_metadata.goindex 1bb539c7bf..bb0262b9cb 100644--- a/internal/agentic_rag/tool_search_metadata.go+++ b/internal/agentic_rag/tool_search_metadata.go@@ -195,6 +195,7 @@ func (m *MetadataSearchTool) invokableRun(ctx context.Context, argumentsInJSON sLogic: logic,})}+ docIDs = intersectDocumentScope(ctx, docIDs)if len(docIDs) == 0 {return metadataSearchResult([]string{}, nil,"No documents match the given metadata conditions. Loosen or change the filters, or use search_chunks / grep_chunks for unfiltered search.")diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.goindex 20ad17694e..5507e8b961 100644--- a/internal/service/agentic_pipeline.go+++ b/internal/service/agentic_pipeline.go@@ -113,6 +113,7 @@ func (s *ChatPipelineService) agenticRag(emitResult(AsyncChatResult{Answer: fmt.Sprintf("**ERROR**: %s", chainErr.Error()), Final: true})return}+ docIDs := s.chatMetadataDocIDs(runCtx, chat, lastUserQuestion(messages), chain[0], nil, chatDatasetIDs(chat))// Apply the dialog's LLM settings with per-request overrides exactly// like the regular AsyncChat path does, then let the template pin its@@ -166,6 +167,7 @@ func (s *ChatPipelineService) agenticRag(TemplateID: mode,TenantID: chat.TenantID,DatasetIDs: chatDatasetIDs(chat),+ DocIDs: docIDs,Stream: stream,ToolCallCounts: toolCounts,ToolCallErrors: toolErrors,diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex 80e3cb4e59..5b2c9518fd 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -643,34 +643,7 @@ func (s *ChatPipelineService) AsyncChat(// meta_data_filter — use LLM to map the question to metadata// criteria, then filter docIDs to matching// documents only.- if chat.MetaDataFilter != nil && len(*chat.MetaDataFilter) > 0 && len(kbs) > 0 {- kbIDs := kbIDStrings(kbs)- if metaQ := questions[len(questions)-1]; metaQ != "" {- var flattedMeta common.MetaData- var mErr error- if s.MetadataSvc != nil {- flattedMeta, mErr = s.MetadataSvc.GetFlattedMetaByKBs(ctx, kbIDs)- }- if mErr == nil {- if filtered, _ := ApplyMetaDataFilter(- ctx,- *chat.MetaDataFilter,- flattedMeta,- metaQ,- chatModel,- docIDs,- kbIDs,- ); filtered != nil {- common.Debug("meta_data_filter applied",- zap.Int("filtered_count", len(filtered)),- zap.Int("pre_filter_count", len(docIDs)))- docIDs = filtered- }- } else {- common.Warn("loadMetaData failed; skipping meta_data_filter", zap.Error(mErr))- }- }- }+ docIDs = s.chatMetadataDocIDs(ctx, chat, questions[len(questions)-1], chatModel, docIDs, kbIDStrings(kbs))// keyword — extract top-N keywords from the question via LLM and// append them to the question text to boost lexical retrieval recall.diff --git a/internal/service/metadata_filter.go b/internal/service/metadata_filter.goindex b3f37c17fe..42dbd78226 100644--- a/internal/service/metadata_filter.go+++ b/internal/service/metadata_filter.go@@ -24,6 +24,7 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/engine"+ "ragflow/internal/entity""regexp""strconv"@@ -768,6 +769,31 @@ func ApplyMetaDataFilter(return baseDocIDs, false}+// chatMetadataDocIDs is shared by standard retrieval and the agent's corpus tools.+func (s *ChatPipelineService) chatMetadataDocIDs(ctx context.Context, chat *entity.Chat, question string, model *modelModule.ChatModel, docIDs, kbIDs []string) []string {+ if chat.MetaDataFilter == nil || len(*chat.MetaDataFilter) == 0 || len(kbIDs) == 0 || question == "" {+ return docIDs+ }+ var meta common.MetaData+ var err error+ if s.MetadataSvc != nil {+ meta, err = s.MetadataSvc.GetFlattedMetaByKBs(ctx, kbIDs)+ }+ if err != nil {+ common.Warn("loadMetaData failed; skipping meta_data_filter", zap.Error(err))+ return docIDs+ }+ filtered, empty := ApplyMetaDataFilter(ctx, *chat.MetaDataFilter, meta, question, model, docIDs, kbIDs)+ if empty {+ return []string{NoMatchDocIDSentinel}+ }+ if filtered != nil {+ common.Debug("meta_data_filter applied", zap.Int("filtered_count", len(filtered)), zap.Int("pre_filter_count", len(docIDs)))+ return filtered+ }+ return docIDs+}+func constrainDocIDs(baseDocIDs, filteredDocIDs []string) []string {filteredDocIDs = common.Deduplicate(filteredDocIDs)if len(baseDocIDs) == 0 {