Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix OceanBase chunk order field persistence (#16997)

zcxGGmu05c67ca854dc
diff --git a/rag/utils/ob_conn.py b/rag/utils/ob_conn.py
index 6020c2c312..ee4ea22259 100644
--- a/rag/utils/ob_conn.py
+++ b/rag/utils/ob_conn.py
@@ -46,6 +46,7 @@ from rag.nlp import rag_tokenizer
logger = logging.getLogger("ragflow.ob_conn")
column_order_id = Column("_order_id", Integer, nullable=True, comment="chunk order id for maintaining sequence")
+column_chunk_order_int = Column("chunk_order_int", Integer, nullable=True, comment="chunk order index for retrieval sorting")
column_group_id = Column("group_id", String(256), nullable=True, comment="group id for external retrieval")
column_mom_id = Column("mom_id", String(256), nullable=True, comment="parent chunk id")
column_chunk_data = Column("chunk_data", JSON, nullable=True, comment="table parser row data")
@@ -97,6 +98,7 @@ column_definitions: list[Column] = [
Column("metadata", JSON, nullable=True, comment="metadata for this chunk"),
Column("extra", JSON, nullable=True, comment="extra information of non-general chunk"),
column_order_id,
+ column_chunk_order_int,
column_group_id,
column_mom_id,
column_deleted_doc_id,
@@ -137,6 +139,7 @@ FTS_COLUMNS_TKS: list[str] = [
# Extra columns to add after table creation (for migration)
EXTRA_COLUMNS: list[Column] = [
column_order_id,
+ column_chunk_order_int,
column_group_id,
column_mom_id,
column_chunk_data,
@@ -191,12 +194,20 @@ def get_default_value(column_name: str) -> Any:
return 1
elif column_name == "removed_kwd":
return "N"
- elif column_name == "_order_id":
+ elif column_name in ["_order_id", "chunk_order_int"]:
return 0
else:
return None
+def _get_entity_value(column_name: str, value: Any) -> Any:
+ if value is None:
+ if column_name not in {"_order_id", "chunk_order_int"}:
+ return None
+ value = get_default_value(column_name)
+ return get_column_value(column_name, value)
+
+
def get_metadata_filter_expression(metadata_filtering_conditions: dict) -> str:
"""
Convert metadata filtering conditions to MySQL JSON path expression.
@@ -1210,19 +1221,20 @@ class OBConnection(OBConnectionBase):
def _row_to_entity(self, data: Row, fields: list[str]) -> dict:
entity = {}
for i, field in enumerate(fields):
- value = data[i]
+ value = _get_entity_value(field, data[i])
if value is None:
continue
- entity[field] = get_column_value(field, value)
+ entity[field] = value
return entity
@staticmethod
def _es_row_to_entity(data: dict) -> dict:
entity = {}
for k, v in data.items():
- if v is None:
+ value = _get_entity_value(k, v)
+ if value is None:
continue
- entity[k] = get_column_value(k, v)
+ entity[k] = value
return entity
"""
diff --git a/test/unit_test/rag/utils/test_ob_conn_chunk_order_int.py b/test/unit_test/rag/utils/test_ob_conn_chunk_order_int.py
new file mode 100644
index 0000000000..786d06beec
--- /dev/null
+++ b/test/unit_test/rag/utils/test_ob_conn_chunk_order_int.py
@@ -0,0 +1,35 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from rag.utils.ob_conn import _get_entity_value, column_names, get_column_value, get_default_value
+
+
+class TestChunkOrderIntSchema:
+ def test_chunk_order_int_is_a_first_class_oceanbase_column(self):
+ assert "chunk_order_int" in column_names
+ assert get_default_value("chunk_order_int") == 0
+ assert get_column_value("chunk_order_int", "7") == 7
+
+ def test_null_chunk_order_int_uses_default_entity_value(self):
+ assert _get_entity_value("chunk_order_int", None) == 0
+
+ def test_null_order_id_uses_default_entity_value(self):
+ assert _get_entity_value("_order_id", None) == 0
+
+ def test_null_non_order_columns_remain_none_on_read(self):
+ assert _get_entity_value("available_int", None) is None
+ assert _get_entity_value("removed_kwd", None) is None
+ assert _get_entity_value("group_id", None) is None
diff --git a/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py b/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py
index 4a6aedd1dd..ed9b59f77e 100644
--- a/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py
+++ b/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py
@@ -67,7 +67,8 @@ RAGFLOW_COLUMNS = {
"metadata": {"ob_type": "JSON", "nullable": True, "is_json": True},
"extra": {"ob_type": "JSON", "nullable": True, "is_json": True},
# New columns
- "_order_id": {"ob_type": "Integer", "nullable": True},
+ "_order_id": {"ob_type": "Integer", "nullable": True, "default": 0},
+ "chunk_order_int": {"ob_type": "Integer", "nullable": True, "default": 0},
"group_id": {"ob_type": "String(256)", "nullable": True},
"mom_id": {"ob_type": "String(256)", "nullable": True},
}
diff --git a/tools/es-to-oceanbase-migration/tests/test_schema.py b/tools/es-to-oceanbase-migration/tests/test_schema.py
index e856ae110f..4bd8499a9a 100644
--- a/tools/es-to-oceanbase-migration/tests/test_schema.py
+++ b/tools/es-to-oceanbase-migration/tests/test_schema.py
@@ -138,6 +138,21 @@ class TestRAGFlowDataConverter:
assert row["content_with_weight"] == "This is test content"
assert row["available_int"] == 1
+ def test_convert_document_defaults_missing_chunk_order(self):
+ """Test missing chunk order defaults to the first chunk position."""
+ converter = RAGFlowDataConverter()
+
+ row = converter.convert_document(
+ {
+ "_id": "test-id-123",
+ "_source": {
+ "kb_id": "kb-001",
+ },
+ }
+ )
+
+ assert row["chunk_order_int"] == 0
+
def test_convert_with_vector(self):
"""Test converting document with vector embedding."""
converter = RAGFlowDataConverter()
@@ -369,6 +384,9 @@ class TestConstants:
assert RAGFLOW_COLUMNS["metadata"]["is_json"] is True
assert RAGFLOW_COLUMNS["extra"]["is_json"] is True
+ assert RAGFLOW_COLUMNS["_order_id"]["default"] == 0
+ assert RAGFLOW_COLUMNS["chunk_order_int"]["default"] == 0
+
class TestRAGFlowSchemaConverterEdgeCases:
"""Test edge cases for RAGFlowSchemaConverter."""