Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

perf(retrieval): push metadata filters down to the metadata index (#18219)

Elias Shiehf44436a549bc
diff --git a/api/apps/restful_apis/chunk_api.py b/api/apps/restful_apis/chunk_api.py
index 8a7485b790..6ed59b24db 100644
--- a/api/apps/restful_apis/chunk_api.py
+++ b/api/apps/restful_apis/chunk_api.py
@@ -57,7 +57,7 @@ from api.utils.reference_metadata_utils import (
from common import settings
from common.constants import LLMType, ParserType, RetCode, TaskStatus
from common.doc_store.doc_store_base import OrderByExpr
-from common.metadata_utils import convert_conditions, meta_filter
+from common.metadata_utils import convert_conditions, filter_doc_ids_by_metadata
from common.misc_utils import thread_pool_exec
from common.string_utils import is_content_empty, remove_redundant_spaces
from common.tag_feature_utils import validate_tag_features
@@ -364,8 +364,12 @@ async def retrieval_test(tenant_id):
if not doc_ids:
metadata_condition = req.get("metadata_condition")
if metadata_condition:
- metas = DocMetadataService.get_flatted_meta_by_kbs(kb_ids)
- doc_ids = meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and"))
+ doc_ids = filter_doc_ids_by_metadata(
+ kb_ids,
+ convert_conditions(metadata_condition),
+ metadata_condition.get("logic", "and"),
+ lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),
+ )
if not doc_ids and metadata_condition.get("conditions"):
return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})
if metadata_condition and not doc_ids:
diff --git a/common/metadata_utils.py b/common/metadata_utils.py
index 3aa4c7c0e6..df762793fc 100644
--- a/common/metadata_utils.py
+++ b/common/metadata_utils.py
@@ -207,20 +207,7 @@ async def apply_meta_data_filter(
def _run_metadata_filter(conditions: list[dict], logic: str) -> list[str]:
"""Run conditions through ES/Infinity push-down when possible, in-memory otherwise."""
- if conditions and kb_ids:
- try:
- from api.db.services.doc_metadata_service import DocMetadataService
-
- doc_ids = DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)
- logging.debug(f"Doc ids filtered by metadata: {doc_ids}")
- if doc_ids is not None:
- return doc_ids
- except Exception as e:
- logging.error(f"Metadata filter push down errored: {e}")
-
- # In-memory fallback
- logging.debug("Metadata filter falls back to in-memory filter")
- return meta_filter(_get_metas(), conditions, logic)
+ return filter_doc_ids_by_metadata(kb_ids or [], conditions, logic, _get_metas)
if method == "auto":
filters: dict = await gen_meta_filter(chat_mdl, _get_metas(), question)
@@ -268,7 +255,7 @@ def _try_meta_pushdown(
conditions: list[dict],
logic: str,
) -> list[str] | None:
- """Attempt the ES push-down path; return ``None`` to fall back in-memory.
+ """Attempt metadata-index push-down; return ``None`` to fall back in memory.
Lazy-imports ``DocMetadataService`` so this module stays usable in
environments where the API/db layer hasn't been wired up (e.g. unit tests
@@ -276,14 +263,34 @@ def _try_meta_pushdown(
"""
try:
from api.db.services.doc_metadata_service import DocMetadataService
- except Exception as e:
- logging.debug(f"[apply_meta_data_filter] push-down disabled, import failed: {e}")
- return None
- try:
- return DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)
- except Exception as e:
- logging.warning(f"[apply_meta_data_filter] push-down errored, falling back: {e}")
+ except ImportError as e:
+ logging.debug(f"Metadata filter push-down disabled because the service import failed: {e}")
return None
+ return DocMetadataService.filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic)
+
+
+def filter_doc_ids_by_metadata(
+ kb_ids: list[str],
+ conditions: list[dict],
+ logic: str,
+ metas_loader: Callable[[], dict],
+) -> list[str]:
+ """Filter document IDs through the metadata index with a lazy exact fallback."""
+ doc_ids = _try_meta_pushdown(kb_ids, conditions, logic) if conditions and kb_ids else None
+ if doc_ids is not None:
+ logging.debug(
+ "Metadata filter used push-down: kb_count=%d condition_count=%d matched_doc_count=%d",
+ len(kb_ids),
+ len(conditions),
+ len(doc_ids),
+ )
+ return doc_ids
+ logging.debug(
+ "Metadata filter uses in-memory fallback: kb_count=%d condition_count=%d",
+ len(kb_ids),
+ len(conditions),
+ )
+ return meta_filter(metas_loader(), conditions, logic)
def dedupe_list(values: list) -> list:
diff --git a/test/unit_test/common/test_metadata_filter_routing.py b/test/unit_test/common/test_metadata_filter_routing.py
new file mode 100644
index 0000000000..17b3628dd3
--- /dev/null
+++ b/test/unit_test/common/test_metadata_filter_routing.py
@@ -0,0 +1,149 @@
+import builtins
+import logging
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+from common import metadata_utils
+
+
+def test_supported_filter_uses_pushdown_without_loading_metadata(monkeypatch, caplog):
+ conditions = [{"key": "department", "op": "=", "value": "engineering"}]
+
+ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", lambda kb_ids, filters, logic: ["doc-1"])
+
+ def fail_if_loaded():
+ raise AssertionError("metadata must stay lazy when push-down succeeds")
+
+ with caplog.at_level(logging.DEBUG):
+ result = metadata_utils.filter_doc_ids_by_metadata(["kb-1"], conditions, "and", fail_if_loaded)
+
+ assert result == ["doc-1"]
+ assert "Metadata filter used push-down: kb_count=1 condition_count=1 matched_doc_count=1" in caplog.messages
+ assert "doc-1" not in "\n".join(caplog.messages)
+
+
+def test_empty_pushdown_result_does_not_load_metadata(monkeypatch, caplog):
+ conditions = [{"key": "department", "op": "=", "value": "legal"}]
+
+ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", lambda kb_ids, filters, logic: [])
+
+ def fail_if_loaded():
+ raise AssertionError("an empty push-down result is definitive")
+
+ with caplog.at_level(logging.DEBUG):
+ result = metadata_utils.filter_doc_ids_by_metadata(["kb-1"], conditions, "and", fail_if_loaded)
+
+ assert result == []
+ assert "Metadata filter used push-down: kb_count=1 condition_count=1 matched_doc_count=0" in caplog.messages
+
+
+def test_unsupported_filter_preserves_multivalue_fallback_semantics(monkeypatch, caplog):
+ conditions = [{"key": "tag", "op": "≠", "value": "alpha"}]
+ metas = {"tag": {"alpha": ["doc-1"], "beta": ["doc-1", "doc-2"]}}
+ load_count = 0
+
+ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", lambda kb_ids, filters, logic: None)
+
+ def load_metas():
+ nonlocal load_count
+ load_count += 1
+ return metas
+
+ with caplog.at_level(logging.DEBUG):
+ result = metadata_utils.filter_doc_ids_by_metadata(["kb-1"], conditions, "and", load_metas)
+
+ assert set(result) == {"doc-1", "doc-2"}
+ assert load_count == 1
+ assert "Metadata filter uses in-memory fallback: kb_count=1 condition_count=1" in caplog.messages
+ assert "doc-1" not in "\n".join(caplog.messages)
+
+
+def test_filter_without_kb_scope_skips_pushdown(monkeypatch):
+ conditions = [{"key": "department", "op": "=", "value": "engineering"}]
+
+ def fail_if_called(*args):
+ raise AssertionError("push-down requires a KB scope")
+
+ monkeypatch.setattr(metadata_utils, "_try_meta_pushdown", fail_if_called)
+
+ result = metadata_utils.filter_doc_ids_by_metadata(
+ [],
+ conditions,
+ "and",
+ lambda: {"department": {"engineering": ["doc-1"]}},
+ )
+
+ assert result == ["doc-1"]
+
+
+def test_pushdown_service_errors_are_not_masked(monkeypatch):
+ class FailingDocMetadataService:
+ @staticmethod
+ def filter_doc_ids_by_meta_pushdown(kb_ids, conditions, logic):
+ raise RuntimeError("unexpected service failure")
+
+ real_import = builtins.__import__
+
+ def import_with_failing_service(name, module_globals=None, module_locals=None, fromlist=(), level=0):
+ if name == "api.db.services.doc_metadata_service":
+ return SimpleNamespace(DocMetadataService=FailingDocMetadataService)
+ return real_import(name, module_globals, module_locals, fromlist, level)
+
+ monkeypatch.setattr(builtins, "__import__", import_with_failing_service)
+
+ with pytest.raises(RuntimeError, match="unexpected service failure"):
+ metadata_utils._try_meta_pushdown(
+ ["kb-1"],
+ [{"key": "department", "op": "=", "value": "engineering"}],
+ "and",
+ )
+
+
+def test_missing_pushdown_service_uses_fallback(monkeypatch):
+ real_import = builtins.__import__
+
+ def import_without_service(name, module_globals=None, module_locals=None, fromlist=(), level=0):
+ if name == "api.db.services.doc_metadata_service":
+ raise ImportError("service is unavailable")
+ return real_import(name, module_globals, module_locals, fromlist, level)
+
+ monkeypatch.setattr(builtins, "__import__", import_without_service)
+
+ assert (
+ metadata_utils._try_meta_pushdown(
+ ["kb-1"],
+ [{"key": "department", "op": "=", "value": "engineering"}],
+ "and",
+ )
+ is None
+ )
+
+
+async def test_apply_metadata_filter_uses_shared_router(monkeypatch):
+ conditions = [{"key": "department", "op": "=", "value": "engineering"}]
+ calls = []
+
+ monkeypatch.setitem(
+ sys.modules,
+ "rag.prompts.generator",
+ SimpleNamespace(gen_meta_filter=None),
+ )
+
+ def shared_router(kb_ids, routed_conditions, logic, metas_loader):
+ calls.append((kb_ids, routed_conditions, logic, metas_loader))
+ return ["doc-1"]
+
+ monkeypatch.setattr(metadata_utils, "filter_doc_ids_by_metadata", shared_router)
+
+ result = await metadata_utils.apply_meta_data_filter(
+ {"method": "manual", "manual": conditions},
+ kb_ids=["kb-1"],
+ metas_loader=dict,
+ )
+
+ assert result == ["doc-1"]
+ assert len(calls) == 1
+ assert calls[0][:3] == (["kb-1"], conditions, "and")