Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): preserve CSV cell text during knowledge import (#41922)

Sourav Rajvif3154e347e50
diff --git a/api/core/rag/extractor/csv_extractor.py b/api/core/rag/extractor/csv_extractor.py
index 778dbd40c68..b836e789c2d 100644
--- a/api/core/rag/extractor/csv_extractor.py
+++ b/api/core/rag/extractor/csv_extractor.py
@@ -31,7 +31,8 @@ class CSVExtractor(BaseExtractor):
self._encoding = encoding
self._autodetect_encoding = autodetect_encoding
self.source_column = source_column
- self.csv_args = csv_args or {}
+ # Preserve source text for indexing unless the caller requests type or NA conversion.
+ self.csv_args: dict[str, Any] = {"dtype": str, "keep_default_na": False, **(csv_args or {})}
@override
def extract(self) -> list[Document]:
diff --git a/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py
index f82ebb4d75d..6800ab9bb24 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py
@@ -22,6 +22,27 @@ class _ManagedStringIO(io.StringIO):
class TestCSVExtractor:
+ @pytest.mark.parametrize("value", ["00123", "1.00", "1e3", "NA", "NULL", "N/A", "", "hello"])
+ def test_extract_preserves_cell_text(self, tmp_path: Path, value: str) -> None:
+ file_path = tmp_path / "data.csv"
+ file_path.write_text(f"value,body\n{value},reference\n", encoding="utf-8")
+
+ docs = CSVExtractor(str(file_path), encoding="utf-8", source_column="value").extract()
+
+ assert len(docs) == 1
+ assert docs[0].page_content == f"value: {value};body: reference"
+ assert docs[0].metadata["source"] == value
+
+ def test_extract_honors_explicit_csv_args(self, tmp_path: Path) -> None:
+ file_path = tmp_path / "data.csv"
+ file_path.write_text("value;body\n00123;NA\n", encoding="utf-8")
+ csv_args = {"sep": ";", "dtype": {"value": int}, "keep_default_na": True}
+
+ docs = CSVExtractor(str(file_path), encoding="utf-8", csv_args=csv_args).extract()
+
+ assert docs[0].page_content == "value: 123.0;body: nan"
+ assert csv_args == {"sep": ";", "dtype": {"value": int}, "keep_default_na": True}
+
def test_extract_success_with_source_column(self, tmp_path: Path):
file_path = tmp_path / "data.csv"
file_path.write_text("id,body\nsource-1,hello\n", encoding="utf-8")