RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(rag): key Infinity field_map by original column names, not pinyin (#18318)
Taranum Wasuabdabf883bc1
diff --git a/agent/component/llm.py b/agent/component/llm.pyindex 1bb44f1319..e3e56c0300 100644--- a/agent/component/llm.py+++ b/agent/component/llm.py@@ -25,7 +25,12 @@ from functools import partialfrom common.constants import LLMTypefrom api.db.services.dialog_service import _stream_with_think_deltafrom api.db.services.llm_service import LLMBundle-from api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_type+from api.db.joint_services.tenant_model_service import (+ get_model_config_from_provider_instance,+ get_model_type_by_name,+ resolve_model_config,+ resolve_model_type,+)from agent.component.base import ComponentBase, ComponentParamBasefrom common.connection_utils import timeoutfrom rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt, structured_output_prompt@@ -92,15 +97,10 @@ class LLM(ComponentBase):def __init__(self, canvas, component_id, param: ComponentParamBase):super().__init__(canvas, component_id, param)- try:- model_types = resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id)- model_type = "chat" if "chat" in model_types else model_types[0]- chat_model_config = resolve_model_config(self._canvas.get_tenant_id(), model_type, self._param.llm_id)- self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)- except Exception as e:- logging.warning(f"Fail to load LLM configuration for component. {e}")- self.chat_mdl = None-+ model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)+ model_type = "chat" if "chat" in model_types else model_types[0]+ chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)+ self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)self.imgs = []def get_input_form(self) -> dict[str, dict]:diff --git a/api/db/services/tenant_llm_service.py b/api/db/services/tenant_llm_service.pyindex efe47f1a6d..f5153a5a10 100644--- a/api/db/services/tenant_llm_service.py+++ b/api/db/services/tenant_llm_service.py@@ -556,12 +556,16 @@ class LLM4Tenant:Langfuse already exports spans from its own background processor andflushes at process exit, so releasing the reference is sufficient here."""- # Drop the Langfuse reference WITHOUT flushing/shutting down the shared- # client (see the docstring above for why this would deadlock).+ # Release the Langfuse client reference. ``Langfuse.flush()`` waits on+ # ``Queue.join()`` with no timeout, so we never call it here: the shared+ # ``LangfuseResourceManager`` flushes its queues at process exit, and+ # a per-task ``flush()`` would block the task executor indefinitely+ # if a consumer is wedged. ``self.langfuse = None`` drops our handle so+ # the next ``LLM4Tenant`` reuses the same shared client.self.langfuse = None# Release underlying model instance if it has a close method- if self.mdl and callable(getattr(self.mdl, "close", None)):+ if self.mdl and hasattr(self.mdl, "close") and callable(getattr(self.mdl, "close")):try:self.mdl.close()except Exception:diff --git a/rag/app/table.py b/rag/app/table.pyindex dc76316588..48a3ae3a0c 100644--- a/rag/app/table.py+++ b/rag/app/table.py@@ -548,7 +548,17 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,# field_map: only columns stored in chunk_data (metadata or both) — used for retrieval/SQLstored_indices = [i for i in range(len(clmns)) if column_roles.get(clmns[i], "both") in ("metadata", "both")]if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:- field_map = {py_clmns[i].lower(): str(clmns[i]).replace("_", " ") for i in stored_indices}+ # Regression for #18287: Infinity/OceanBase/SereneDB store+ # chunk_data keyed by the original column name, and the SQL+ # prompt examples reference those exact keys via+ # `json_extract_string(chunk_data, '$.FieldName')`. Apply+ # the underscore-to-space formatting only to the displayed+ # value; the key remains the raw column name verbatim so+ # that columns like `row_id` continue to map to `$.row_id`+ # in `chunk_data` instead of the unreachable `$.row id`.+ # GaussDB's pinyin-keyed contract is preserved by the+ # override block below.+ field_map = {str(clmns[i]): str(clmns[i]).replace("_", " ") for i in stored_indices}else:field_map = {clmns_map[i][0]: clmns_map[i][1] for i in stored_indices}if settings.DOC_ENGINE_GAUSSDB:diff --git a/rag/prompts/generator.py b/rag/prompts/generator.pyindex 7867481a2d..b25af285dc 100644--- a/rag/prompts/generator.py+++ b/rag/prompts/generator.py@@ -290,7 +290,11 @@ async def full_question(tenant_id=None, llm_id=None, messages=[], language=None,async def cross_languages(tenant_id, llm_id, query, languages=[]):from common.constants import LLMTypefrom api.db.services.llm_service import LLMBundle- from api.db.joint_services.tenant_model_service import resolve_model_config, get_tenant_default_model_by_type, resolve_model_type+ from api.db.joint_services.tenant_model_service import (+ get_tenant_default_model_by_type,+ resolve_model_config,+ resolve_model_type,+ )if llm_id and "vision" in resolve_model_type(tenant_id, llm_id):chat_model_config = resolve_model_config(tenant_id, LLMType.VISION, llm_id)diff --git a/test/unit_test/rag/app/test_table_chunk_column_roles.py b/test/unit_test/rag/app/test_table_chunk_column_roles.pyindex d9d3d2dc1e..d23fdcc3e5 100644--- a/test/unit_test/rag/app/test_table_chunk_column_roles.py+++ b/test/unit_test/rag/app/test_table_chunk_column_roles.py@@ -294,3 +294,81 @@ def test_chunk_updates_table_column_names(table_module, mock_update_kb: MagicMocdef test_chunk_count_matches_row_count(table_module, mock_update_kb: MagicMock):chunks = _run_chunk(table_module, {}, mock_update_kb)assert len(chunks) == 3+++# Regression for #18287: Infinity/OceanBase/SereneDB store chunk_data+# keyed by the original column name, and the SQL prompt examples+# reference those exact keys via `json_extract_string(chunk_data,+# '$.FieldName')`. Keys must therefore be the original column names+# — not the pinyin form (`py_clmns[i].lower()`), which silently+# broke non-ASCII columns because the LLM was told `xing_ming` but+# `chunk_data` was keyed by `姓名`.+def _enable_infinity(monkeypatch):+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", True)+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", False)+++def _enable_oceanbase(monkeypatch):+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", True)+++def test_chunk_infinity_field_map_uses_original_column_names(table_module, mock_update_kb: MagicMock, monkeypatch):+ _enable_infinity(monkeypatch)+ _run_chunk(table_module, {}, mock_update_kb)+ payload = mock_update_kb.call_args.args[1]+ field_map = payload["field_map"]+ # field_map keys must equal the original column names so they line up+ # with chunk_data storage keys used by the SQL prompt's+ # `json_extract_string(chunk_data, '$.FieldName')` examples.+ assert set(field_map.keys()) == {"row_id", "title", "content", "country", "category"}+ # No pinyin (e.g. "row_id_tks") should leak into field_map keys —+ # pinyin belongs in the type-suffixed ES path, not the Infinity+ # JSON-extract path.+ assert not any(k.endswith("_tks") for k in field_map.keys())+ # Keys must be the raw column name verbatim. Under the buggy+ # implementation `row_id` was being rewritten to `"row id"`, which+ # the SQL prompt's `$.row id` could not match against+ # `chunk_data["row_id"]`. Values, however, get underscore-to-space+ # formatting applied (so the LLM sees a human-readable label), but+ # the key must NOT be reformatted.+ assert "row_id" in field_map+ assert "row id" not in field_map+ assert field_map["row_id"] == "row id"+++def test_chunk_oceanbase_field_map_uses_original_column_names(table_module, mock_update_kb: MagicMock, monkeypatch):+ _enable_oceanbase(monkeypatch)+ _run_chunk(table_module, {}, mock_update_kb)+ payload = mock_update_kb.call_args.args[1]+ field_map = payload["field_map"]+ assert set(field_map.keys()) == {"row_id", "title", "content", "country", "category"}+ assert not any(k.endswith("_tks") for k in field_map.keys())+ assert "row_id" in field_map+ assert "row id" not in field_map+ assert field_map["row_id"] == "row id"+++def test_chunk_infinity_field_map_non_ascii_columns_are_keys(table_module, mock_update_kb: MagicMock, monkeypatch):+ """The original bug repros here: a non-ASCII column would be keyed by+ its pinyin form (`xing_ming`) under the old behavior, so the LLM+ is told a key that doesn't exist in `chunk_data`. After the fix the+ field_map key IS the original column name."""+ _enable_infinity(monkeypatch)+ test_csv = b"row_id,\xe5\xa7\x93\xe5\x90\x8d,\xe9\x95\xbf\xe5\xba\xa6\n1,Alice,42\n" # row_id, 姓名, 长度+ table_module.chunk(+ FILENAME,+ binary=test_csv,+ callback=_noop_callback,+ kb_id=KB_ID,+ parser_config={},+ lang="Chinese",+ )+ payload = mock_update_kb.call_args.args[1]+ field_map = payload["field_map"]+ # The original column names — including the non-ASCII ones — must+ # appear verbatim as keys so the SQL prompt's `$.姓名` matches the+ # JSON extraction in chunk_data.+ assert "\u59d3\u540d" in field_map # 姓名+ assert "\u957f\u5ea6" in field_map # 长度+ assert not any(k.endswith("_tks") for k in field_map.keys())