Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(rag/app/table): keep table cells that column type conversion cannot represent (#18460)

Omar Faig Orujlu7540a39596f5
diff --git a/rag/app/table.py b/rag/app/table.py
index 48a3ae3a0c..c32474d384 100644
--- a/rag/app/table.py
+++ b/rag/app/table.py
@@ -408,6 +408,7 @@ def column_data_type(arr):
type_priority = {"text": 0, "datetime": 1, "float": 2, "int": 3, "bool": 4}
counts = sorted(counts.items(), key=lambda x: (x[1] * -1, type_priority[x[0]]))
ty = counts[0][0]
+ conversion_failures = 0
for i in range(len(arr)):
if arr[i] is None:
continue
@@ -418,13 +419,21 @@ def column_data_type(arr):
arr[i] = None
continue
try:
- arr[i] = trans[ty](str(arr[i]))
- except Exception as e:
- arr[i] = None
- logging.warning(f"Column {i}: {e}")
+ converted = trans[ty](str(arr[i]))
+ except ValueError:
+ conversion_failures += 1
# Keep original value from openpyxl/pandas instead of dropping to None.
# This preserves cells (e.g. text in numeric columns) that would
# otherwise be silently discarded by forced column-level conversion.
+ continue
+ if converted is None:
+ continue
+ arr[i] = converted
+ if conversion_failures:
+ # Aggregate rather than log per-cell: individual cell values must not
+ # be written to application logs, and per-cell warnings would flood
+ # logs on large uploads.
+ logging.warning(f"column_data_type: kept {conversion_failures} cell(s) that could not convert to {ty}")
# if ty == "text":
# if len(arr) > 128 and uni / len(arr) < 0.1:
# ty = "keyword"
diff --git a/test/unit_test/rag/app/test_table_chunk_column_roles.py b/test/unit_test/rag/app/test_table_chunk_column_roles.py
index d23fdcc3e5..ed37a8df78 100644
--- a/test/unit_test/rag/app/test_table_chunk_column_roles.py
+++ b/test/unit_test/rag/app/test_table_chunk_column_roles.py
@@ -44,6 +44,11 @@ TEST_CSV = b"""row_id,title,content,country,category
TEST_DUPLICATE_COLUMNS_CSV = b"""name,name,name_2
Alice,Engineer,Team A
"""
+TEST_PARTIAL_NUMERIC_CSV = b"""row_id,amount,note
+1,100,first
+2,N/A,second
+3,300,third
+"""
FILENAME = "test.csv"
KB_ID = "test_kb_id"
@@ -372,3 +377,35 @@ def test_chunk_infinity_field_map_non_ascii_columns_are_keys(table_module, mock_
assert "\u59d3\u540d" in field_map # 姓名
assert "\u957f\u5ea6" in field_map # 长度
assert not any(k.endswith("_tks") for k in field_map.keys())
+
+
+ ("column", "expected_type"),
+ [
+ (["1", "2", "3", "N/A"], "int"),
+ (["1.5", "2.5", "3.5", "see note"], "float"),
+ (["yes", "no", "yes", "unknown"], "bool"),
+ (["2025-01-01", "2025-02-01", "ongoing"], "datetime"),
+ ],
+)
+def test_column_data_type_keeps_cells_it_cannot_convert(table_module, column, expected_type):
+ converted, ty = table_module.column_data_type(list(column))
+ assert ty == expected_type
+ assert converted[-1] == column[-1]
+ assert None not in converted
+
+
+def test_chunk_keeps_unconvertible_cell_in_content(table_module, mock_update_kb: MagicMock):
+ chunks = table_module.chunk(
+ FILENAME,
+ binary=TEST_PARTIAL_NUMERIC_CSV,
+ callback=_noop_callback,
+ kb_id=KB_ID,
+ parser_config={},
+ lang="Chinese",
+ )
+ assert len(chunks) == 3
+ assert chunks[0]["amount_long"] == 100
+ assert chunks[2]["amount_long"] == 300
+ assert chunks[1].get("amount_long") != "N/A"
+ assert "- amount: N/A" in chunks[1]["content_with_weight"]