Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): keep spreadsheet cells that say N/A, None or NULL (#4452)
Lukasf420ecfa995b
diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.pyindex 645cf7f29..45c92f78d 100644--- a/docreader/parser/excel_parser.py+++ b/docreader/parser/excel_parser.py@@ -118,17 +118,16 @@ class ExcelParser(BaseParser):excel_sheet_name,xlsx_first_row_as_header=self.xlsx_first_row_as_header,)- # Remove rows where all values are NaN (completely empty rows)- df.dropna(how="all", inplace=True)-# Process each row in the DataFrame. The index is the 0-based- # sheet row (rows are read from row 1 and dropped rows keep+ # sheet row (rows are read from row 1 and skipped rows keep# their labels), so index + 1 is the row number users see.for row_index, row in df.iterrows():page_content = []- # Build key-value pairs for non-null values+ # Build key-value pairs for non-empty values. A blank cell is+ # read as "" (see _read_sheet_dataframe), so an empty row has+ # no pairs and is skipped below.for k, v in row.items():- if pd.notna(v) and not _is_image_function(v):+ if pd.notna(v) and v != "" and not _is_image_function(v):page_content.append(f"{k}: {v}")# Skip rows with no valid content@@ -180,7 +179,11 @@ def _read_sheet_dataframe(# Keep row 1 as data by default for both XLSX and legacy XLS. Users can# explicitly restore the historical behavior where row 1 supplies semantic# labels for every row.- df = excel_file.parse(sheet_name=sheet_name, header=None)+ # keep_default_na=False: by default pandas reads a cell whose text is+ # "N/A", "NA", "NULL", "None", "NaN" or "n/a" as missing, and its pair is+ # dropped, though the cell says "not applicable" or "none". Blank cells are+ # then read as "", which parse_into_text skips like a missing value.+ df = excel_file.parse(sheet_name=sheet_name, header=None, keep_default_na=False)if xlsx_first_row_as_header and len(df.index) >= 2:df.columns = _stable_header_labels(df.iloc[0].tolist())return df.iloc[1:].copy()diff --git a/docreader/tests/test_excel_parser.py b/docreader/tests/test_excel_parser.pyindex 4e8cca40a..a25225c29 100644--- a/docreader/tests/test_excel_parser.py+++ b/docreader/tests/test_excel_parser.py@@ -594,6 +594,67 @@ class ExcelParserTest(unittest.TestCase):self.assertEqual(chunks[0], "A: Name,B: Age")self.assertEqual(chunks[1], "A: Alice,B: 30")+ def test_cells_that_say_na_none_or_null_are_kept(self):+ # pandas reads these strings as missing values by default, which+ # dropped the pair although the cell has an answer in it.+ content = self._workbook_bytes(+ [+ ["Patient", "Allergies", "Region", "Result", "Note"],+ ["P1", "None", "NA", "N/A", "ok"],+ ["P2", "Penicillin", "EU", "NULL", None],+ ["P3", "n/a", "NaN", 42, "nan"],+ ]+ )++ document = ExcelParser(+ file_name="patients.xlsx",+ file_type="xlsx",+ xlsx_first_row_as_header=True,+ ).parse_into_text(content)++ self.assertEqual(+ [chunk.content.strip() for chunk in document.chunks],+ [+ "Patient: P1,Allergies: None,Region: NA,Result: N/A,Note: ok",+ "Patient: P2,Allergies: Penicillin,Region: EU,Result: NULL",+ "Patient: P3,Allergies: n/a,Region: NaN,Result: 42,Note: nan",+ ],+ )++ def test_a_header_cell_that_says_na_keeps_its_label(self):+ content = self._workbook_bytes([["Item", "NA", "EU"], ["pens", 3, 4]])++ document = ExcelParser(+ file_name="sales.xlsx",+ file_type="xlsx",+ xlsx_first_row_as_header=True,+ ).parse_into_text(content)++ self.assertEqual(document.chunks[0].content.strip(), "Item: pens,NA: 3,EU: 4")++ def test_blank_cells_and_blank_rows_are_still_skipped(self):+ content = self._workbook_bytes(+ [+ ["Item", "Count"],+ ["pens", None],+ [None, None],+ ["ink", 2],+ ]+ )++ document = ExcelParser(file_name="stock.xlsx", file_type="xlsx").parse_into_text(+ content+ )++ self.assertEqual(+ [chunk.content.strip() for chunk in document.chunks],+ ["A: Item,B: Count", "A: pens", "A: ink,B: 2"],+ )+ self.assertEqual(+ [block["locator"]["row_start"] for block in document.source_blocks],+ [1, 2, 4],+ )+def test_parse_phantom_shared_strings_workbook(self):document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())self.assertIn("hello", document.content)