RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix OceanBase chunk order field persistence (#16997)
zcxGGmu05c67ca854dc
diff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.pyindex 6020c2c312..ee4ea22259 100644--- a/rag/utils/ob_conn.py+++ b/rag/utils/ob_conn.py@@ -46,6 +46,7 @@ from rag.nlp import rag_tokenizerlogger = logging.getLogger("ragflow.ob_conn")column_order_id = Column("_order_id", Integer, nullable=True, comment="chunk order id for maintaining sequence")+column_chunk_order_int = Column("chunk_order_int", Integer, nullable=True, comment="chunk order index for retrieval sorting")column_group_id = Column("group_id", String(256), nullable=True, comment="group id for external retrieval")column_mom_id = Column("mom_id", String(256), nullable=True, comment="parent chunk id")column_chunk_data = Column("chunk_data", JSON, nullable=True, comment="table parser row data")@@ -97,6 +98,7 @@ column_definitions: list[Column] = [Column("metadata", JSON, nullable=True, comment="metadata for this chunk"),Column("extra", JSON, nullable=True, comment="extra information of non-general chunk"),column_order_id,+ column_chunk_order_int,column_group_id,column_mom_id,column_deleted_doc_id,@@ -137,6 +139,7 @@ FTS_COLUMNS_TKS: list[str] = [# Extra columns to add after table creation (for migration)EXTRA_COLUMNS: list[Column] = [column_order_id,+ column_chunk_order_int,column_group_id,column_mom_id,column_chunk_data,@@ -191,12 +194,20 @@ def get_default_value(column_name: str) -> Any:return 1elif column_name == "removed_kwd":return "N"- elif column_name == "_order_id":+ elif column_name in ["_order_id", "chunk_order_int"]:return 0else:return None+def _get_entity_value(column_name: str, value: Any) -> Any:+ if value is None:+ if column_name not in {"_order_id", "chunk_order_int"}:+ return None+ value = get_default_value(column_name)+ return get_column_value(column_name, value)++def get_metadata_filter_expression(metadata_filtering_conditions: dict) -> str:"""Convert metadata filtering conditions to MySQL JSON path expression.@@ -1210,19 +1221,20 @@ class OBConnection(OBConnectionBase):def _row_to_entity(self, data: Row, fields: list[str]) -> dict:entity = {}for i, field in enumerate(fields):- value = data[i]+ value = _get_entity_value(field, data[i])if value is None:continue- entity[field] = get_column_value(field, value)+ entity[field] = valuereturn entity@staticmethoddef _es_row_to_entity(data: dict) -> dict:entity = {}for k, v in data.items():- if v is None:+ value = _get_entity_value(k, v)+ if value is None:continue- entity[k] = get_column_value(k, v)+ entity[k] = valuereturn entity"""diff --git a/test/unit_test/rag/utils/test_ob_conn_chunk_order_int.py b/test/unit_test/rag/utils/test_ob_conn_chunk_order_int.pynew file mode 100644index 0000000000..786d06beec--- /dev/null+++ b/test/unit_test/rag/utils/test_ob_conn_chunk_order_int.py@@ -0,0 +1,35 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++from rag.utils.ob_conn import _get_entity_value, column_names, get_column_value, get_default_value+++class TestChunkOrderIntSchema:+ def test_chunk_order_int_is_a_first_class_oceanbase_column(self):+ assert "chunk_order_int" in column_names+ assert get_default_value("chunk_order_int") == 0+ assert get_column_value("chunk_order_int", "7") == 7++ def test_null_chunk_order_int_uses_default_entity_value(self):+ assert _get_entity_value("chunk_order_int", None) == 0++ def test_null_order_id_uses_default_entity_value(self):+ assert _get_entity_value("_order_id", None) == 0++ def test_null_non_order_columns_remain_none_on_read(self):+ assert _get_entity_value("available_int", None) is None+ assert _get_entity_value("removed_kwd", None) is None+ assert _get_entity_value("group_id", None) is Nonediff --git a/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py b/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.pyindex 4a6aedd1dd..ed9b59f77e 100644--- a/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py+++ b/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py@@ -67,7 +67,8 @@ RAGFLOW_COLUMNS = {"metadata": {"ob_type": "JSON", "nullable": True, "is_json": True},"extra": {"ob_type": "JSON", "nullable": True, "is_json": True},# New columns- "_order_id": {"ob_type": "Integer", "nullable": True},+ "_order_id": {"ob_type": "Integer", "nullable": True, "default": 0},+ "chunk_order_int": {"ob_type": "Integer", "nullable": True, "default": 0},"group_id": {"ob_type": "String(256)", "nullable": True},"mom_id": {"ob_type": "String(256)", "nullable": True},}diff --git a/tools/es-to-oceanbase-migration/tests/test_schema.py b/tools/es-to-oceanbase-migration/tests/test_schema.pyindex e856ae110f..4bd8499a9a 100644--- a/tools/es-to-oceanbase-migration/tests/test_schema.py+++ b/tools/es-to-oceanbase-migration/tests/test_schema.py@@ -138,6 +138,21 @@ class TestRAGFlowDataConverter:assert row["content_with_weight"] == "This is test content"assert row["available_int"] == 1+ def test_convert_document_defaults_missing_chunk_order(self):+ """Test missing chunk order defaults to the first chunk position."""+ converter = RAGFlowDataConverter()++ row = converter.convert_document(+ {+ "_id": "test-id-123",+ "_source": {+ "kb_id": "kb-001",+ },+ }+ )++ assert row["chunk_order_int"] == 0+def test_convert_with_vector(self):"""Test converting document with vector embedding."""converter = RAGFlowDataConverter()@@ -369,6 +384,9 @@ class TestConstants:assert RAGFLOW_COLUMNS["metadata"]["is_json"] is Trueassert RAGFLOW_COLUMNS["extra"]["is_json"] is True+ assert RAGFLOW_COLUMNS["_order_id"]["default"] == 0+ assert RAGFLOW_COLUMNS["chunk_order_int"]["default"] == 0+class TestRAGFlowSchemaConverterEdgeCases:"""Test edge cases for RAGFlowSchemaConverter."""