RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
perf(retrieval): push metadata filters down to the metadata index (#18219)
Elias Shiehf44436a549bc
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.pyindex 8a7485b790..6ed59b24db 100644--- a/api/apps/restful_apis/chunk_api.py+++ b/api/apps/restful_apis/chunk_api.py@@ -57,7 +57,7 @@ from api.utils.reference_metadata_utils import (from common import settingsfrom common.constants import LLMType, ParserType, RetCode, TaskStatusfrom common.doc_store.doc_store_base import OrderByExpr-from common.metadata_utils import convert_conditions, meta_filter+from common.metadata_utils import convert_conditions, filter_doc_ids_by_metadatafrom common.misc_utils import thread_pool_execfrom common.string_utils import is_content_empty, remove_redundant_spacesfrom common.tag_feature_utils import validate_tag_features@@ -364,8 +364,12 @@ async def retrieval_test(tenant_id):if not doc_ids:metadata_condition = req.get("metadata_condition")if metadata_condition:- metas = DocMetadataService.get_flatted_meta_by_kbs(kb_ids)- doc_ids = meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and"))+ doc_ids = filter_doc_ids_by_metadata(+ kb_ids,+ convert_conditions(metadata_condition),+ metadata_condition.get("logic", "and"),+ lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),+ )if not doc_ids and metadata_condition.get("conditions"):return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})if metadata_condition and not doc_ids:diff --git a/common/metadata_utils.py b/common/metadata_utils.pyindex 3aa4c7c0e6..df762793fc 100644--- a/common/metadata_utils.py+++ b/common/metadata_utils.py@@ -207,20 +207,7 @@ async def apply_meta_data_filter(def _run_metadata_filter(conditions: list[dict], logic: str) -> list[str]:"""Run conditions through ES/Infinity push-down when possible, in-memory otherwise."""- if conditions and kb_ids:- try:- from api.db.services.doc_metadata_service import DocMetadataService-- doc_ids = DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)- logging.debug(f"Doc ids filtered by metadata: {doc_ids}")- if doc_ids is not None:- return doc_ids- except Exception as e:- logging.error(f"Metadata filter push down errored: {e}")-- # In-memory fallback- logging.debug("Metadata filter falls back to in-memory filter")- return meta_filter(_get_metas(), conditions, logic)+ return filter_doc_ids_by_metadata(kb_ids or [], conditions, logic, _get_metas)if method == "auto":filters: dict = await gen_meta_filter(chat_mdl, _get_metas(), question)@@ -268,7 +255,7 @@ def _try_meta_pushdown(conditions: list[dict],logic: str,) -> list[str] | None:- """Attempt the ES push-down path; return ``None`` to fall back in-memory.+ """Attempt metadata-index push-down; return ``None`` to fall back in memory.Lazy-imports ``DocMetadataService`` so this module stays usable inenvironments where the API/db layer hasn't been wired up (e.g. unit tests@@ -276,14 +263,34 @@ def _try_meta_pushdown("""try:from api.db.services.doc_metadata_service import DocMetadataService- except Exception as e:- logging.debug(f"[apply_meta_data_filter] push-down disabled, import failed: {e}")- return None- try:- return DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)- except Exception as e:- logging.warning(f"[apply_meta_data_filter] push-down errored, falling back: {e}")+ except ImportError as e:+ logging.debug(f"Metadata filter push-down disabled because the service import failed: {e}")return None+ return DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)+++def filter_doc_ids_by_metadata(+ kb_ids: list[str],+ conditions: list[dict],+ logic: str,+ metas_loader: Callable[[], dict],+) -> list[str]:+ """Filter document IDs through the metadata index with a lazy exact fallback."""+ doc_ids = _try_meta_pushdown(kb_ids, conditions, logic) if conditions and kb_ids else None+ if doc_ids is not None:+ logging.debug(+ "Metadata filter used push-down: kb_count=%d condition_count=%d matched_doc_count=%d",+ len(kb_ids),+ len(conditions),+ len(doc_ids),+ )+ return doc_ids+ logging.debug(+ "Metadata filter uses in-memory fallback: kb_count=%d condition_count=%d",+ len(kb_ids),+ len(conditions),+ )+ return meta_filter(metas_loader(), conditions, logic)def dedupe_list(values: list) -> list:diff --git a/test/unit_test/common/test_metadata_filter_routing.py b/test/unit_test/common/test_metadata_filter_routing.pynew file mode 100644index 0000000000..17b3628dd3--- /dev/null+++ b/test/unit_test/common/test_metadata_filter_routing.py@@ -0,0 +1,149 @@+import builtins+import logging+import sys+from types import SimpleNamespace++import pytest++from common import metadata_utils+++def test_supported_filter_uses_pushdown_without_loading_metadata(monkeypatch, caplog):+ conditions = [{"key": "department", "op": "=", "value": "engineering"}]++ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", lambda kb_ids, filters, logic: ["doc-1"])++ def fail_if_loaded():+ raise AssertionError("metadata must stay lazy when push-down succeeds")++ with caplog.at_level(logging.DEBUG):+ result = metadata_utils.filter_doc_ids_by_metadata(["kb-1"], conditions, "and", fail_if_loaded)++ assert result == ["doc-1"]+ assert "Metadata filter used push-down: kb_count=1 condition_count=1 matched_doc_count=1" in caplog.messages+ assert "doc-1" not in "\n".join(caplog.messages)+++def test_empty_pushdown_result_does_not_load_metadata(monkeypatch, caplog):+ conditions = [{"key": "department", "op": "=", "value": "legal"}]++ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", lambda kb_ids, filters, logic: [])++ def fail_if_loaded():+ raise AssertionError("an empty push-down result is definitive")++ with caplog.at_level(logging.DEBUG):+ result = metadata_utils.filter_doc_ids_by_metadata(["kb-1"], conditions, "and", fail_if_loaded)++ assert result == []+ assert "Metadata filter used push-down: kb_count=1 condition_count=1 matched_doc_count=0" in caplog.messages+++def test_unsupported_filter_preserves_multivalue_fallback_semantics(monkeypatch, caplog):+ conditions = [{"key": "tag", "op": "≠", "value": "alpha"}]+ metas = {"tag": {"alpha": ["doc-1"], "beta": ["doc-1", "doc-2"]}}+ load_count = 0++ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", lambda kb_ids, filters, logic: None)++ def load_metas():+ nonlocal load_count+ load_count += 1+ return metas++ with caplog.at_level(logging.DEBUG):+ result = metadata_utils.filter_doc_ids_by_metadata(["kb-1"], conditions, "and", load_metas)++ assert set(result) == {"doc-1", "doc-2"}+ assert load_count == 1+ assert "Metadata filter uses in-memory fallback: kb_count=1 condition_count=1" in caplog.messages+ assert "doc-1" not in "\n".join(caplog.messages)+++def test_filter_without_kb_scope_skips_pushdown(monkeypatch):+ conditions = [{"key": "department", "op": "=", "value": "engineering"}]++ def fail_if_called(*args):+ raise AssertionError("push-down requires a KB scope")++ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", fail_if_called)++ result = metadata_utils.filter_doc_ids_by_metadata(+ [],+ conditions,+ "and",+ lambda: {"department": {"engineering": ["doc-1"]}},+ )++ assert result == ["doc-1"]+++def test_pushdown_service_errors_are_not_masked(monkeypatch):+ class FailingDocMetadataService:+ @staticmethod+ def filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic):+ raise RuntimeError("unexpected service failure")++ real_import = builtins.__import__++ def import_with_failing_service(name, module_globals=None, module_locals=None, fromlist=(), level=0):+ if name == "api.db.services.doc_metadata_service":+ return SimpleNamespace(DocMetadataService=FailingDocMetadataService)+ return real_import(name, module_globals, module_locals, fromlist, level)++ monkeypatch.setattr(builtins, "__import__", import_with_failing_service)++ with pytest.raises(RuntimeError, match="unexpected service failure"):+ metadata_utils._try_meta_pushdown(+ ["kb-1"],+ [{"key": "department", "op": "=", "value": "engineering"}],+ "and",+ )+++def test_missing_pushdown_service_uses_fallback(monkeypatch):+ real_import = builtins.__import__++ def import_without_service(name, module_globals=None, module_locals=None, fromlist=(), level=0):+ if name == "api.db.services.doc_metadata_service":+ raise ImportError("service is unavailable")+ return real_import(name, module_globals, module_locals, fromlist, level)++ monkeypatch.setattr(builtins, "__import__", import_without_service)++ assert (+ metadata_utils._try_meta_pushdown(+ ["kb-1"],+ [{"key": "department", "op": "=", "value": "engineering"}],+ "and",+ )+ is None+ )+++async def test_apply_metadata_filter_uses_shared_router(monkeypatch):+ conditions = [{"key": "department", "op": "=", "value": "engineering"}]+ calls = []++ monkeypatch.setitem(+ sys.modules,+ "rag.prompts.generator",+ SimpleNamespace(gen_meta_filter=None),+ )++ def shared_router(kb_ids, routed_conditions, logic, metas_loader):+ calls.append((kb_ids, routed_conditions, logic, metas_loader))+ return ["doc-1"]++ monkeypatch.setattr(metadata_utils, "filter_doc_ids_by_metadata", shared_router)++ result = await metadata_utils.apply_meta_data_filter(+ {"method": "manual", "manual": conditions},+ kb_ids=["kb-1"],+ metas_loader=dict,+ )++ assert result == ["doc-1"]+ assert len(calls) == 1+ assert calls[0][:3] == (["kb-1"], conditions, "and")