Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): preserve CSV cell text in batch segment import (#42159)

CoralGarden52c6c891430c81
diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.py
index 89ee3d17d86..250788d0aab 100644
--- a/api/tasks/batch_create_segment_to_index_task.py
+++ b/api/tasks/batch_create_segment_to_index_task.py
@@ -112,7 +112,7 @@ def batch_create_segment_to_index_task(
file_path = f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}" # type: ignore
storage.download(upload_file_key, file_path)
- df = pd.read_csv(file_path)
+ df = pd.read_csv(file_path, dtype=str, keep_default_na=False)
content = []
for _, row in df.iterrows():
if document_config["doc_form"] == IndexStructureType.QA_INDEX:
diff --git a/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py b/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py
index 05827112d4a..580dd88382c 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_batch_create_segment_to_index_task.py
@@ -324,6 +324,49 @@ class TestBatchCreateSegmentToIndexTask:
cache_value = redis_client.get(cache_key)
assert cache_value == b"completed"
+ def test_batch_create_segment_to_index_task_preserves_csv_cell_text(
+ self, db_session_with_containers: Session, mock_external_service_dependencies
+ ):
+ """Test that numeric-looking and NA CSV cells remain text in QA segments."""
+ account, tenant = self._create_test_account_and_tenant(db_session_with_containers)
+ dataset = self._create_test_dataset(db_session_with_containers, account, tenant)
+ document = self._create_test_document(db_session_with_containers, account, tenant, dataset)
+ document.doc_form = IndexStructureType.QA_INDEX
+ db_session_with_containers.commit()
+ upload_file = self._create_test_upload_file(db_session_with_containers, account, tenant)
+
+ csv_content = "content,answer\n00123,NA\nNA,00456\n"
+ mock_storage = mock_external_service_dependencies["storage"]
+
+ def mock_download(key, file_path):
+ Path(file_path).write_text(csv_content, encoding="utf-8")
+
+ mock_storage.download.side_effect = mock_download
+
+ job_id = str(uuid.uuid4())
+ batch_create_segment_to_index_task(
+ job_id=job_id,
+ upload_file_id=upload_file.id,
+ dataset_id=dataset.id,
+ document_id=document.id,
+ tenant_id=tenant.id,
+ user_id=account.id,
+ )
+
+ segments = db_session_with_containers.scalars(
+ select(DocumentSegment).where(DocumentSegment.document_id == document.id).order_by(DocumentSegment.position)
+ ).all()
+
+ assert [(segment.content, segment.answer, segment.word_count) for segment in segments] == [
+ ("00123", "NA", 7),
+ ("NA", "00456", 7),
+ ]
+
+ from extensions.ext_redis import redis_client
+
+ cache_key = f"segment_batch_import_{job_id}"
+ assert redis_client.get(cache_key) == b"completed"
+
def test_batch_create_segment_to_index_task_dataset_not_found(
self, db_session_with_containers: Session, mock_external_service_dependencies
):