RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: recognize data type in parse method `table` (#17946)
Lynn2e0b82be3080
diff --git a/rag/app/table.py b/rag/app/table.pyindex b04a6bc36f..7c7cbd1431 100644--- a/rag/app/table.py+++ b/rag/app/table.py@@ -19,6 +19,7 @@ import csvimport ioimport loggingimport re+from datetime import datetimefrom io import BytesIOfrom xpinyin import Pinyinimport numpy as np@@ -364,16 +365,38 @@ def column_data_type(arr):for a in arr:if a is None:continue- if re.match(r"[+-]?[0-9]+$", str(a).replace("%%", "")) and not str(a).replace("%%", "").startswith("0"):+ if isinstance(a, float) and pd.isna(a):+ continue # skip NaN+ # Check native Python type first (from openpyxl via pandas DataFrame)+ if isinstance(a, bool):+ counts["bool"] += 1+ continue+ if isinstance(a, (int, np.integer)):+ if int(a) > 2**63 - 1:+ float_flag = True+ break+ counts["int"] += 1+ continue+ if isinstance(a, (float, np.floating)):+ counts["float"] += 1+ continue+ if isinstance(a, datetime):+ if pd.isna(a):+ continue # skip pd.NaT+ counts["datetime"] += 1+ continue+ # Fallback to string-based regex matching+ s = str(a)+ if re.match(r"[+-]?[0-9]+$", s.replace("%%", "")) and not s.replace("%%", "").startswith("0"):counts["int"] += 1- if int(str(a)) > 2**63 - 1:+ if int(s) > 2**63 - 1:float_flag = Truebreak- elif re.match(r"[+-]?[0-9.]{,19}$", str(a).replace("%%", "")) and not str(a).replace("%%", "").startswith("0"):+ elif re.match(r"[+-]?[0-9.]{,19}$", s.replace("%%", "")) and not s.replace("%%", "").startswith("0"):counts["float"] += 1- elif re.match(r"(true|yes|是|\*|✓|✔|☑|✅|√|false|no|否|⍻|×)$", str(a), flags=re.IGNORECASE):+ elif re.match(r"(true|yes|是|\*|✓|✔|☑|✅|√|false|no|否|⍻|×)$", s, flags=re.IGNORECASE):counts["bool"] += 1- elif trans_datatime(str(a)):+ elif trans_datatime(s):counts["datetime"] += 1else:counts["text"] += 1@@ -385,11 +408,20 @@ def column_data_type(arr):for i in range(len(arr)):if arr[i] is None:continue+ if isinstance(arr[i], float) and pd.isna(arr[i]):+ arr[i] = None+ continue+ if isinstance(arr[i], datetime) and pd.isna(arr[i]):+ arr[i] = None+ continuetry:arr[i] = trans[ty](str(arr[i]))except Exception as e:arr[i] = Nonelogging.warning(f"Column {i}: {e}")+ # Keep original value from openpyxl/pandas instead of dropping to None.+ # This preserves cells (e.g. text in numeric columns) that would+ # otherwise be silently discarded by forced column-level conversion.# if ty == "text":# if len(arr) > 128 and uni / len(arr) < 0.1:# ty = "keyword"@@ -578,7 +610,16 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,else:fld = clmns_map[j][0]if clmn_tys[j] != "text":- stored[fld] = row[col_name]+ val = row[col_name]+ # If a string value ended up in a non-text column,+ # it was preserved from a failed conversion in+ # column_data_type. Skip storing in the typed ES+ # field to avoid type mapping errors; the value is+ # already in text_fields for chunk content.+ if isinstance(val, str):+ pass+ else:+ stored[fld] = valelse:cell = row[col_name]stored[fld] = rag_tokenizer.tokenize(cell)