Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(rag): key Infinity field_map by original column names, not pinyin (#18318)

Taranum Wasuabdabf883bc1
diff --git a/agent/component/llm.py b/agent/component/llm.py
index 1bb44f1319..e3e56c0300 100644
--- a/agent/component/llm.py
+++ b/agent/component/llm.py
@@ -25,7 +25,12 @@ from functools import partial
from common.constants import LLMType
from api.db.services.dialog_service import _stream_with_think_delta
from api.db.services.llm_service import LLMBundle
-from api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_type
+from api.db.joint_services.tenant_model_service import (
+ get_model_config_from_provider_instance,
+ get_model_type_by_name,
+ resolve_model_config,
+ resolve_model_type,
+)
from agent.component.base import ComponentBase, ComponentParamBase
from common.connection_utils import timeout
from rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt, structured_output_prompt
@@ -92,15 +97,10 @@ class LLM(ComponentBase):
def __init__(self, canvas, component_id, param: ComponentParamBase):
super().__init__(canvas, component_id, param)
- try:
- model_types = resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id)
- model_type = "chat" if "chat" in model_types else model_types[0]
- chat_model_config = resolve_model_config(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
- self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)
- except Exception as e:
- logging.warning(f"Fail to load LLM configuration for component. {e}")
- self.chat_mdl = None
-
+ model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
+ model_type = "chat" if "chat" in model_types else model_types[0]
+ chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
+ self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)
self.imgs = []
def get_input_form(self) -> dict[str, dict]:
diff --git a/api/db/services/tenant_llm_service.py b/api/db/services/tenant_llm_service.py
index efe47f1a6d..f5153a5a10 100644
--- a/api/db/services/tenant_llm_service.py
+++ b/api/db/services/tenant_llm_service.py
@@ -556,12 +556,16 @@ class LLM4Tenant:
Langfuse already exports spans from its own background processor and
flushes at process exit, so releasing the reference is sufficient here.
"""
- # Drop the Langfuse reference WITHOUT flushing/shutting down the shared
- # client (see the docstring above for why this would deadlock).
+ # Release the Langfuse client reference. ``Langfuse.flush()`` waits on
+ # ``Queue.join()`` with no timeout, so we never call it here: the shared
+ # ``LangfuseResourceManager`` flushes its queues at process exit, and
+ # a per-task ``flush()`` would block the task executor indefinitely
+ # if a consumer is wedged. ``self.langfuse = None`` drops our handle so
+ # the next ``LLM4Tenant`` reuses the same shared client.
self.langfuse = None
# Release underlying model instance if it has a close method
- if self.mdl and callable(getattr(self.mdl, "close", None)):
+ if self.mdl and hasattr(self.mdl, "close") and callable(getattr(self.mdl, "close")):
try:
self.mdl.close()
except Exception:
diff --git a/rag/app/table.py b/rag/app/table.py
index dc76316588..48a3ae3a0c 100644
--- a/rag/app/table.py
+++ b/rag/app/table.py
@@ -548,7 +548,17 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
# field_map: only columns stored in chunk_data (metadata or both) — used for retrieval/SQL
stored_indices = [i for i in range(len(clmns)) if column_roles.get(clmns[i], "both") in ("metadata", "both")]
if settings.DOC_ENGINE_INFINITY or settings.DOC_ENGINE_OCEANBASE or settings.DOC_ENGINE_GAUSSDB or settings.DOC_ENGINE_SERENEDB:
- field_map = {py_clmns[i].lower(): str(clmns[i]).replace("_", " ") for i in stored_indices}
+ # Regression for #18287: Infinity/OceanBase/SereneDB store
+ # chunk_data keyed by the original column name, and the SQL
+ # prompt examples reference those exact keys via
+ # `json_extract_string(chunk_data, '$.FieldName')`. Apply
+ # the underscore-to-space formatting only to the displayed
+ # value; the key remains the raw column name verbatim so
+ # that columns like `row_id` continue to map to `$.row_id`
+ # in `chunk_data` instead of the unreachable `$.row id`.
+ # GaussDB's pinyin-keyed contract is preserved by the
+ # override block below.
+ field_map = {str(clmns[i]): str(clmns[i]).replace("_", " ") for i in stored_indices}
else:
field_map = {clmns_map[i][0]: clmns_map[i][1] for i in stored_indices}
if settings.DOC_ENGINE_GAUSSDB:
diff --git a/rag/prompts/generator.py b/rag/prompts/generator.py
index 7867481a2d..b25af285dc 100644
--- a/rag/prompts/generator.py
+++ b/rag/prompts/generator.py
@@ -290,7 +290,11 @@ async def full_question(tenant_id=None, llm_id=None, messages=[], language=None,
async def cross_languages(tenant_id, llm_id, query, languages=[]):
from common.constants import LLMType
from api.db.services.llm_service import LLMBundle
- from api.db.joint_services.tenant_model_service import resolve_model_config, get_tenant_default_model_by_type, resolve_model_type
+ from api.db.joint_services.tenant_model_service import (
+ get_tenant_default_model_by_type,
+ resolve_model_config,
+ resolve_model_type,
+ )
if llm_id and "vision" in resolve_model_type(tenant_id, llm_id):
chat_model_config = resolve_model_config(tenant_id, LLMType.VISION, llm_id)
diff --git a/test/unit_test/rag/app/test_table_chunk_column_roles.py b/test/unit_test/rag/app/test_table_chunk_column_roles.py
index d9d3d2dc1e..d23fdcc3e5 100644
--- a/test/unit_test/rag/app/test_table_chunk_column_roles.py
+++ b/test/unit_test/rag/app/test_table_chunk_column_roles.py
@@ -294,3 +294,81 @@ def test_chunk_updates_table_column_names(table_module, mock_update_kb: MagicMoc
def test_chunk_count_matches_row_count(table_module, mock_update_kb: MagicMock):
chunks = _run_chunk(table_module, {}, mock_update_kb)
assert len(chunks) == 3
+
+
+# Regression for #18287: Infinity/OceanBase/SereneDB store chunk_data
+# keyed by the original column name, and the SQL prompt examples
+# reference those exact keys via `json_extract_string(chunk_data,
+# '$.FieldName')`. Keys must therefore be the original column names
+# — not the pinyin form (`py_clmns[i].lower()`), which silently
+# broke non-ASCII columns because the LLM was told `xing_ming` but
+# `chunk_data` was keyed by `姓名`.
+def _enable_infinity(monkeypatch):
+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", True)
+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", False)
+
+
+def _enable_oceanbase(monkeypatch):
+ monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+ monkeypatch.setattr(settings, "DOC_ENGINE_OCEANBASE", True)
+
+
+def test_chunk_infinity_field_map_uses_original_column_names(table_module, mock_update_kb: MagicMock, monkeypatch):
+ _enable_infinity(monkeypatch)
+ _run_chunk(table_module, {}, mock_update_kb)
+ payload = mock_update_kb.call_args.args[1]
+ field_map = payload["field_map"]
+ # field_map keys must equal the original column names so they line up
+ # with chunk_data storage keys used by the SQL prompt's
+ # `json_extract_string(chunk_data, '$.FieldName')` examples.
+ assert set(field_map.keys()) == {"row_id", "title", "content", "country", "category"}
+ # No pinyin (e.g. "row_id_tks") should leak into field_map keys —
+ # pinyin belongs in the type-suffixed ES path, not the Infinity
+ # JSON-extract path.
+ assert not any(k.endswith("_tks") for k in field_map.keys())
+ # Keys must be the raw column name verbatim. Under the buggy
+ # implementation `row_id` was being rewritten to `"row id"`, which
+ # the SQL prompt's `$.row id` could not match against
+ # `chunk_data["row_id"]`. Values, however, get underscore-to-space
+ # formatting applied (so the LLM sees a human-readable label), but
+ # the key must NOT be reformatted.
+ assert "row_id" in field_map
+ assert "row id" not in field_map
+ assert field_map["row_id"] == "row id"
+
+
+def test_chunk_oceanbase_field_map_uses_original_column_names(table_module, mock_update_kb: MagicMock, monkeypatch):
+ _enable_oceanbase(monkeypatch)
+ _run_chunk(table_module, {}, mock_update_kb)
+ payload = mock_update_kb.call_args.args[1]
+ field_map = payload["field_map"]
+ assert set(field_map.keys()) == {"row_id", "title", "content", "country", "category"}
+ assert not any(k.endswith("_tks") for k in field_map.keys())
+ assert "row_id" in field_map
+ assert "row id" not in field_map
+ assert field_map["row_id"] == "row id"
+
+
+def test_chunk_infinity_field_map_non_ascii_columns_are_keys(table_module, mock_update_kb: MagicMock, monkeypatch):
+ """The original bug repros here: a non-ASCII column would be keyed by
+ its pinyin form (`xing_ming`) under the old behavior, so the LLM
+ is told a key that doesn't exist in `chunk_data`. After the fix the
+ field_map key IS the original column name."""
+ _enable_infinity(monkeypatch)
+ test_csv = b"row_id,\xe5\xa7\x93\xe5\x90\x8d,\xe9\x95\xbf\xe5\xba\xa6\n1,Alice,42\n" # row_id, 姓名, 长度
+ table_module.chunk(
+ FILENAME,
+ binary=test_csv,
+ callback=_noop_callback,
+ kb_id=KB_ID,
+ parser_config={},
+ lang="Chinese",
+ )
+ payload = mock_update_kb.call_args.args[1]
+ field_map = payload["field_map"]
+ # The original column names — including the non-ASCII ones — must
+ # appear verbatim as keys so the SQL prompt's `$.姓名` matches the
+ # JSON extraction in chunk_data.
+ assert "\u59d3\u540d" in field_map # 姓名
+ assert "\u957f\u5ea6" in field_map # 长度
+ assert not any(k.endswith("_tks") for k in field_map.keys())