RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(api): derive SQL retrieval tenant from dataset owner, not chat tenant (#18518)
ChendongPang03d0ad2ace42
diff --git a/api/db/services/dialog_service.py b/api/db/services/dialog_service.pyindex 9e0fc05d05..01e3663d04 100644--- a/api/db/services/dialog_service.py+++ b/api/db/services/dialog_service.py@@ -676,21 +676,38 @@ async def async_chat(dialog, messages, stream=True, **kwargs):logging.debug(f"field_map retrieved: {field_map}")# try to use sql if field mapping is good to goif field_map:- logging.debug("Use SQL to retrieval:{}".format(questions[-1]))- ans = await use_sql(questions[-1], field_map, dialog.tenant_id, chat_mdl, prompt_config.get("quote", True), dialog.kb_ids, doc_ids=scoped_doc_ids)- # For aggregate queries (COUNT, SUM, etc.), chunks may be empty but answer is still valid- if ans and (ans.get("reference", {}).get("chunks") or ans.get("answer")):- if include_reference_metadata and ans.get("reference", {}).get("chunks"):- if len(dialog.kb_ids) != 1 and any(not c.get("kb_id") for c in ans["reference"]["chunks"]):- logging.warning(- "Skipping some _enrich_chunks_with_document_metadata results because dialog.kb_ids has %d entries and use_sql returned chunks without kb_id.",- len(dialog.kb_ids),- )- _enrich_chunks_with_document_metadata(ans["reference"]["chunks"], metadata_fields)- yield ans- return+ # Derive the doc-store tenant/namespace from the referenced dataset's own+ # owner, not from dialog.tenant_id: a team-shared dataset may be owned by a+ # different tenant than the one who created this chat.+ sql_kbs = [kb for kb in kbs if kb.parser_config and kb.parser_config.get("field_map")]+ sql_tenant_ids = {kb.tenant_id for kb in sql_kbs}+ if len(sql_tenant_ids) > 1:+ # use_sql queries a single tenant's doc-store index per call, and+ # re-running it once per tenant is too slow to do inline (each call+ # round-trips an LLM to generate SQL). Skip SQL retrieval rather than+ # silently querying only one tenant's index and dropping the rest.+ logging.warning(+ "Skipping SQL retrieval: field-map datasets span multiple tenants (%s); falling back to vector search.",+ sql_tenant_ids,+ )else:- logging.debug("SQL failed or returned no results, falling back to vector search")+ sql_tenant_id = sql_kbs[0].tenant_id if sql_kbs else dialog.tenant_id+ sql_kb_ids = [kb.id for kb in sql_kbs] if sql_kbs else dialog.kb_ids+ logging.debug("Use SQL to retrieval:{}".format(questions[-1]))+ ans = await use_sql(questions[-1], field_map, sql_tenant_id, chat_mdl, prompt_config.get("quote", True), sql_kb_ids, doc_ids=scoped_doc_ids)+ # For aggregate queries (COUNT, SUM, etc.), chunks may be empty but answer is still valid+ if ans and (ans.get("reference", {}).get("chunks") or ans.get("answer")):+ if include_reference_metadata and ans.get("reference", {}).get("chunks"):+ if len(sql_kb_ids) != 1 and any(not c.get("kb_id") for c in ans["reference"]["chunks"]):+ logging.warning(+ "Skipping some _enrich_chunks_with_document_metadata results because sql_kb_ids has %d entries and use_sql returned chunks without kb_id.",+ len(sql_kb_ids),+ )+ _enrich_chunks_with_document_metadata(ans["reference"]["chunks"], metadata_fields)+ yield ans+ return+ else:+ logging.debug("SQL failed or returned no results, falling back to vector search")param_keys = [p["key"] for p in prompt_config.get("parameters", [])]if dialog.kb_ids and "knowledge" not in param_keys and "{knowledge}" in prompt_config.get("system", ""):diff --git a/test/unit_test/api/db/services/test_dialog_service_use_sql_source_columns.py b/test/unit_test/api/db/services/test_dialog_service_use_sql_source_columns.pyindex 7c4c602ffa..dc5f3bef4b 100644--- a/test/unit_test/api/db/services/test_dialog_service_use_sql_source_columns.py+++ b/test/unit_test/api/db/services/test_dialog_service_use_sql_source_columns.py@@ -350,3 +350,247 @@ def test_async_chat_uses_all_docs_when_no_doc_ids_selected(monkeypatch):assert retriever.calls[0]["kwargs"]["doc_ids"] is Noneassert "Chunk text from dataset." in chat_model.calls[0]["system_prompt"]assert result[0]["answer"] == "stub answer"+++def test_async_chat_sql_retrieval_uses_dataset_owner_tenant_not_dialog_tenant(monkeypatch):+ """Regression test for a team-shared dataset SQL-retrieval tenant mismatch.++ When a chat (owned by tenant "chat-creator-tenant") references a table+ dataset owned by a different tenant ("dataset-owner-tenant"), SQL+ retrieval must build the doc-store index/table name from the dataset+ owner's tenant, not from dialog.tenant_id -- otherwise it queries an+ index that doesn't exist (see GitHub issue #18514).+ """+ shared_kb = SimpleNamespace(+ id="kb-shared",+ tenant_id="dataset-owner-tenant",+ parser_config={"field_map": {"product": "Product Name"}},+ )+ chat_model = _StubChatModel(["unused"])+ dialog = SimpleNamespace(+ kb_ids=["kb-shared"],+ llm_id="chat-model",+ tenant_llm_id="",+ tenant_id="chat-creator-tenant",+ llm_setting={},+ similarity_threshold=0.1,+ vector_similarity_weight=0.2,+ top_n=8,+ top_k=32,+ meta_data_filter=None,+ prompt_config={+ "quote": True,+ "keyword": False,+ "tts": False,+ "empty_response": "",+ "system": "Use only this knowledge: {knowledge}",+ "parameters": [{"key": "knowledge", "optional": False}],+ "reasoning": False,+ "toc_enhance": False,+ "use_kg": False,+ },+ )++ captured_calls = []++ async def _fake_use_sql(_question, _field_map, tenant_id, _chat_mdl, _quote, kb_ids, doc_ids=None):+ captured_calls.append({"tenant_id": tenant_id, "kb_ids": kb_ids})+ return {"answer": "sql answer", "reference": {"chunks": [], "doc_aggs": []}, "prompt": ""}++ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _tid, _llm_id: ["chat"])+ monkeypatch.setattr(+ dialog_service,+ "resolve_model_config",+ lambda *_args, **_kwargs: {"llm_factory": "unit", "max_tokens": 4096, "model_type": "chat"},+ )+ monkeypatch.setattr(dialog_service.TenantLangfuseService, "filter_by_tenant", lambda **_kwargs: None)+ monkeypatch.setattr(+ dialog_service,+ "get_models",+ lambda _dialog, **_kwargs: ([shared_kb], object(), None, chat_model, None),+ )+ monkeypatch.setattr(dialog_service.KnowledgebaseService, "get_field_map", lambda _kb_ids: {"product": "Product Name"})+ monkeypatch.setattr(dialog_service, "use_sql", _fake_use_sql)++ async def _collect():+ items = []+ async for item in dialog_service.async_chat(dialog, [{"role": "user", "content": "How many products?"}], stream=False):+ items.append(item)+ return items++ result = asyncio.run(_collect())++ assert len(captured_calls) == 1+ assert captured_calls[0]["tenant_id"] == "dataset-owner-tenant"+ assert captured_calls[0]["kb_ids"] == ["kb-shared"]+ assert result[0]["answer"] == "sql answer"+++def test_async_chat_sql_retrieval_ignores_empty_field_map_datasets(monkeypatch):+ mapped_kb = SimpleNamespace(+ id="kb-mapped",+ tenant_id="mapped-tenant",+ parser_config={"field_map": {"product": "Product Name"}},+ )+ empty_map_kb = SimpleNamespace(+ id="kb-empty-map",+ tenant_id="other-tenant",+ parser_config={"field_map": {}},+ )+ chat_model = _StubChatModel(["unused"])+ dialog = SimpleNamespace(+ kb_ids=["kb-mapped", "kb-empty-map"],+ llm_id="chat-model",+ tenant_llm_id="",+ tenant_id="chat-creator-tenant",+ llm_setting={},+ similarity_threshold=0.1,+ vector_similarity_weight=0.2,+ top_n=8,+ top_k=32,+ meta_data_filter=None,+ prompt_config={+ "quote": True,+ "keyword": False,+ "tts": False,+ "empty_response": "",+ "system": "Use only this knowledge: {knowledge}",+ "parameters": [{"key": "knowledge", "optional": False}],+ "reasoning": False,+ "toc_enhance": False,+ "use_kg": False,+ },+ )++ captured_calls = []++ async def _fake_use_sql(_question, _field_map, tenant_id, _chat_mdl, _quote, kb_ids, doc_ids=None):+ captured_calls.append({"tenant_id": tenant_id, "kb_ids": kb_ids})+ return {"answer": "sql answer", "reference": {"chunks": [], "doc_aggs": []}, "prompt": ""}++ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _tid, _llm_id: ["chat"])+ monkeypatch.setattr(+ dialog_service,+ "resolve_model_config",+ lambda *_args, **_kwargs: {"llm_factory": "unit", "max_tokens": 4096, "model_type": "chat"},+ )+ monkeypatch.setattr(dialog_service.TenantLangfuseService, "filter_by_tenant", lambda **_kwargs: None)+ monkeypatch.setattr(+ dialog_service,+ "get_models",+ lambda _dialog, **_kwargs: ([mapped_kb, empty_map_kb], object(), None, chat_model, None),+ )+ monkeypatch.setattr(dialog_service.KnowledgebaseService, "get_field_map", lambda _kb_ids: {"product": "Product Name"})+ monkeypatch.setattr(dialog_service, "use_sql", _fake_use_sql)++ async def _collect():+ items = []+ async for item in dialog_service.async_chat(dialog, [{"role": "user", "content": "How many products?"}], stream=False):+ items.append(item)+ return items++ result = asyncio.run(_collect())++ assert captured_calls == [{"tenant_id": "mapped-tenant", "kb_ids": ["kb-mapped"]}]+ assert result[0]["answer"] == "sql answer"+++def test_async_chat_skips_sql_retrieval_for_mixed_tenant_field_map_datasets(monkeypatch):+ """Regression test: SQL retrieval must not silently pick one tenant's index+ when the field-map datasets referenced by a chat span multiple tenants.++ use_sql() queries a single tenant's doc-store index per call, and re-running+ it once per tenant inline is too slow (each call round-trips an LLM to+ generate SQL). So when the datasets span more than one tenant, SQL retrieval+ is skipped entirely and the flow falls back to vector search, instead of+ querying only one tenant's index and dropping the other tenants' data.+ """+ kb_a = SimpleNamespace(id="kb-a", tenant_id="tenant-a", parser_config={"field_map": {"product": "Product Name"}})+ kb_b = SimpleNamespace(id="kb-b", tenant_id="tenant-b", parser_config={"field_map": {"product": "Product Name"}})++ retriever = _StubAsyncRetriever(+ {+ "total": 1,+ "chunks": [+ {+ "chunk_id": "chunk-1",+ "content_ltks": "chunk text",+ "content_with_weight": "Chunk text from dataset.",+ "doc_id": "doc-1",+ "docnm_kwd": "doc.txt",+ "kb_id": "kb-a",+ "important_kwd": [],+ "positions": [],+ "vector": [0.1, 0.2],+ }+ ],+ "doc_aggs": [],+ }+ )+ chat_model = _StubChatModel(["stub answer"])+ dialog = SimpleNamespace(+ kb_ids=["kb-a", "kb-b"],+ llm_id="chat-model",+ tenant_llm_id="",+ tenant_id="tenant-a",+ llm_setting={},+ similarity_threshold=0.1,+ vector_similarity_weight=0.2,+ top_n=8,+ top_k=32,+ meta_data_filter=None,+ prompt_config={+ "quote": False,+ "keyword": False,+ "tts": False,+ "empty_response": "",+ "system": "Use only this knowledge: {knowledge}",+ "parameters": [{"key": "knowledge", "optional": False}],+ "reasoning": False,+ "toc_enhance": False,+ "use_kg": False,+ },+ )++ def _unexpected_use_sql(*_args, **_kwargs):+ raise AssertionError("use_sql must not be called for mixed-tenant field-map datasets")++ monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)+ monkeypatch.setattr(dialog_service, "resolve_model_type", lambda _tid, _llm_id: ["chat"])+ monkeypatch.setattr(+ dialog_service,+ "resolve_model_config",+ lambda *_args, **_kwargs: {"llm_factory": "unit", "max_tokens": 4096, "model_type": "chat"},+ )+ monkeypatch.setattr(dialog_service.TenantLangfuseService, "filter_by_tenant", lambda **_kwargs: None)+ monkeypatch.setattr(+ dialog_service,+ "get_models",+ lambda _dialog, **_kwargs: ([kb_a, kb_b], object(), None, chat_model, None),+ )+ monkeypatch.setattr(dialog_service.KnowledgebaseService, "get_field_map", lambda _kb_ids: {"product": "Product Name"})+ monkeypatch.setattr(dialog_service, "use_sql", _unexpected_use_sql)+ monkeypatch.setattr(dialog_service, "label_question", lambda _question, _kbs: None)+ monkeypatch.setattr(+ dialog_service,+ "kb_prompt",+ lambda kbinfos, _max_tokens: ["Chunk text from dataset."] if kbinfos["chunks"] else [],+ )+ monkeypatch.setattr(dialog_service, "message_fit_in", lambda msg, _max_tokens: (0, msg))++ async def _collect():+ items = []+ async for item in dialog_service.async_chat(dialog, [{"role": "user", "content": "How many products?"}], stream=False):+ items.append(item)+ return items++ result = asyncio.run(_collect())++ assert len(retriever.calls) == 1+ call_args = retriever.calls[0]["args"]+ assert set(call_args[2]) == {"tenant-a", "tenant-b"}+ assert call_args[3] == ["kb-a", "kb-b"]+ assert result[0]["answer"] == "stub answer"diff --git a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.pyindex 8a37ec4aea..a9b4c36ba6 100644--- a/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py+++ b/test/unit_test/api/db/services/test_gaussdb_dialog_sql.py@@ -666,7 +666,8 @@ def _run_configured_chat(kb_id = "abcdefabcdefabcdefabcdefabcdefab"if embd_mdl is None:embd_mdl = object()- get_field_map = Mock(return_value=field_map or {"amount": "number"})+ resolved_field_map = field_map if field_map is not None else {"amount": "number"}+ get_field_map = Mock(return_value=resolved_field_map)monkeypatch.setattr(dialog_service.settings, "retriever", retriever, raising=False)monkeypatch.setattr(dialog_service,@@ -684,7 +685,13 @@ def _run_configured_chat(monkeypatch.setattr(dialog_service,"get_models",- lambda *_args, **_kwargs: ([types.SimpleNamespace(tenant_id=tenant_id)], embd_mdl, None, chat, None),+ lambda *_args, **_kwargs: (+ [types.SimpleNamespace(id=kb_id, tenant_id=tenant_id, parser_config={"field_map": resolved_field_map})],+ embd_mdl,+ None,+ chat,+ None,+ ),)monkeypatch.setattr(dialog_service, "kb_prompt", lambda *_args, **_kwargs: ["fallback content"])monkeypatch.setattr(dialog_service, "message_fit_in", lambda messages, _limit: (0, messages))