Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(es-to-oceanbase): preserve array values during JSON encoding (#20695)

lorenzozaneaa8185155af9
diff --git a/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py b/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py
index ed9b59f77e..4284002b0b 100644
--- a/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py
+++ b/tools/es-to-oceanbase-migration/src/es_ob_migration/schema.py
@@ -327,20 +327,7 @@ class RAGFlowDataConverter:
return json.dumps([value], ensure_ascii=False)
if isinstance(value, list):
- # Clean array values
- cleaned = []
- for item in value:
- if isinstance(item, str):
- # Clean special characters
- cleaned_str = item.strip()
- cleaned_str = cleaned_str.replace("\\", "\\\\")
- cleaned_str = cleaned_str.replace("\n", "\\n")
- cleaned_str = cleaned_str.replace("\r", "\\r")
- cleaned_str = cleaned_str.replace("\t", "\\t")
- cleaned.append(cleaned_str)
- else:
- cleaned.append(item)
- return json.dumps(cleaned, ensure_ascii=False)
+ return json.dumps(value, ensure_ascii=False)
# Single value - wrap in array
return json.dumps([value], ensure_ascii=False)
diff --git a/tools/es-to-oceanbase-migration/tests/test_schema.py b/tools/es-to-oceanbase-migration/tests/test_schema.py
index 4bd8499a9a..e9f37dee10 100644
--- a/tools/es-to-oceanbase-migration/tests/test_schema.py
+++ b/tools/es-to-oceanbase-migration/tests/test_schema.py
@@ -533,22 +533,21 @@ class TestRAGFlowDataConverterEdgeCases:
def test_convert_array_with_special_characters(self):
"""Test converting array with special characters."""
converter = RAGFlowDataConverter()
+ values = ["key\\with\\backslashes", "key\nwith\nnewlines", "key\rwith\rcarriage returns", "key\twith\ttabs", " padded "]
es_doc = {
"_id": "special_array",
"_source": {
"id": "special_array",
"kb_id": "kb_001",
- "important_kwd": ["key\nwith\nnewlines", "key\twith\ttabs"],
+ "important_kwd": values,
},
}
row = converter.convert_document(es_doc)
- # Should be JSON string with escaped characters
assert isinstance(row["important_kwd"], str)
- parsed = json.loads(row["important_kwd"])
- assert len(parsed) == 2
+ assert json.loads(row["important_kwd"]) == values
def test_convert_already_json_array(self):
"""Test converting already JSON-encoded array."""