RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(rag/app/table): keep table cells that column type conversion cannot represent (#18460)
Omar Faig Orujlu7540a39596f5
diff --git a/rag/app/table.py b/rag/app/table.pyindex 48a3ae3a0c..c32474d384 100644--- a/rag/app/table.py+++ b/rag/app/table.py@@ -408,6 +408,7 @@ def column_data_type(arr):type_priority = {"text": 0, "datetime": 1, "float": 2, "int": 3, "bool": 4}counts = sorted(counts.items(), key=lambda x: (x[1] * -1, type_priority[x[0]]))ty = counts[0][0]+ conversion_failures = 0for i in range(len(arr)):if arr[i] is None:continue@@ -418,13 +419,21 @@ def column_data_type(arr):arr[i] = Nonecontinuetry:- arr[i] = trans[ty](str(arr[i]))- except Exception as e:- arr[i] = None- logging.warning(f"Column {i}: {e}")+ converted = trans[ty](str(arr[i]))+ except ValueError:+ conversion_failures += 1# Keep original value from openpyxl/pandas instead of dropping to None.# This preserves cells (e.g. text in numeric columns) that would# otherwise be silently discarded by forced column-level conversion.+ continue+ if converted is None:+ continue+ arr[i] = converted+ if conversion_failures:+ # Aggregate rather than log per-cell: individual cell values must not+ # be written to application logs, and per-cell warnings would flood+ # logs on large uploads.+ logging.warning(f"column_data_type: kept {conversion_failures} cell(s) that could not convert to {ty}")# if ty == "text":# if len(arr) > 128 and uni / len(arr) < 0.1:# ty = "keyword"diff --git a/test/unit_test/rag/app/test_table_chunk_column_roles.py b/test/unit_test/rag/app/test_table_chunk_column_roles.pyindex d23fdcc3e5..ed37a8df78 100644--- a/test/unit_test/rag/app/test_table_chunk_column_roles.py+++ b/test/unit_test/rag/app/test_table_chunk_column_roles.py@@ -44,6 +44,11 @@ TEST_CSV = b"""row_id,title,content,country,categoryTEST_DUPLICATE_COLUMNS_CSV = b"""name,name,name_2Alice,Engineer,Team A"""+TEST_PARTIAL_NUMERIC_CSV = b"""row_id,amount,note+1,100,first+2,N/A,second+3,300,third+"""FILENAME = "test.csv"KB_ID = "test_kb_id"@@ -372,3 +377,35 @@ def test_chunk_infinity_field_map_non_ascii_columns_are_keys(table_module, mock_assert "\u59d3\u540d" in field_map # 姓名assert "\u957f\u5ea6" in field_map # 长度assert not any(k.endswith("_tks") for k in field_map.keys())+++ ("column", "expected_type"),+ [+ (["1", "2", "3", "N/A"], "int"),+ (["1.5", "2.5", "3.5", "see note"], "float"),+ (["yes", "no", "yes", "unknown"], "bool"),+ (["2025-01-01", "2025-02-01", "ongoing"], "datetime"),+ ],+)+def test_column_data_type_keeps_cells_it_cannot_convert(table_module, column, expected_type):+ converted, ty = table_module.column_data_type(list(column))+ assert ty == expected_type+ assert converted[-1] == column[-1]+ assert None not in converted+++def test_chunk_keeps_unconvertible_cell_in_content(table_module, mock_update_kb: MagicMock):+ chunks = table_module.chunk(+ FILENAME,+ binary=TEST_PARTIAL_NUMERIC_CSV,+ callback=_noop_callback,+ kb_id=KB_ID,+ parser_config={},+ lang="Chinese",+ )+ assert len(chunks) == 3+ assert chunks[0]["amount_long"] == 100+ assert chunks[2]["amount_long"] == 300+ assert chunks[1].get("amount_long") != "N/A"+ assert "- amount: N/A" in chunks[1]["content_with_weight"]