Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): keep spreadsheet cells that say N/A, None or NULL (#4452)

Lukasf420ecfa995b
diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.py
index 645cf7f29..45c92f78d 100644
--- a/docreader/parser/excel_parser.py
+++ b/docreader/parser/excel_parser.py
@@ -118,17 +118,16 @@ class ExcelParser(BaseParser):
excel_sheet_name,
xlsx_first_row_as_header=self.xlsx_first_row_as_header,
)
- # Remove rows where all values are NaN (completely empty rows)
- df.dropna(how="all", inplace=True)
-
# Process each row in the DataFrame. The index is the 0-based
- # sheet row (rows are read from row 1 and dropped rows keep
+ # sheet row (rows are read from row 1 and skipped rows keep
# their labels), so index + 1 is the row number users see.
for row_index, row in df.iterrows():
page_content = []
- # Build key-value pairs for non-null values
+ # Build key-value pairs for non-empty values. A blank cell is
+ # read as "" (see _read_sheet_dataframe), so an empty row has
+ # no pairs and is skipped below.
for k, v in row.items():
- if pd.notna(v) and not _is_image_function(v):
+ if pd.notna(v) and v != "" and not _is_image_function(v):
page_content.append(f"{k}: {v}")
# Skip rows with no valid content
@@ -180,7 +179,11 @@ def _read_sheet_dataframe(
# Keep row 1 as data by default for both XLSX and legacy XLS. Users can
# explicitly restore the historical behavior where row 1 supplies semantic
# labels for every row.
- df = excel_file.parse(sheet_name=sheet_name, header=None)
+ # keep_default_na=False: by default pandas reads a cell whose text is
+ # "N/A", "NA", "NULL", "None", "NaN" or "n/a" as missing, and its pair is
+ # dropped, though the cell says "not applicable" or "none". Blank cells are
+ # then read as "", which parse_into_text skips like a missing value.
+ df = excel_file.parse(sheet_name=sheet_name, header=None, keep_default_na=False)
if xlsx_first_row_as_header and len(df.index) >= 2:
df.columns = _stable_header_labels(df.iloc[0].tolist())
return df.iloc[1:].copy()
diff --git a/docreader/tests/test_excel_parser.py b/docreader/tests/test_excel_parser.py
index 4e8cca40a..a25225c29 100644
--- a/docreader/tests/test_excel_parser.py
+++ b/docreader/tests/test_excel_parser.py
@@ -594,6 +594,67 @@ class ExcelParserTest(unittest.TestCase):
self.assertEqual(chunks[0], "A: Name,B: Age")
self.assertEqual(chunks[1], "A: Alice,B: 30")
+ def test_cells_that_say_na_none_or_null_are_kept(self):
+ # pandas reads these strings as missing values by default, which
+ # dropped the pair although the cell has an answer in it.
+ content = self._workbook_bytes(
+ [
+ ["Patient", "Allergies", "Region", "Result", "Note"],
+ ["P1", "None", "NA", "N/A", "ok"],
+ ["P2", "Penicillin", "EU", "NULL", None],
+ ["P3", "n/a", "NaN", 42, "nan"],
+ ]
+ )
+
+ document = ExcelParser(
+ file_name="patients.xlsx",
+ file_type="xlsx",
+ xlsx_first_row_as_header=True,
+ ).parse_into_text(content)
+
+ self.assertEqual(
+ [chunk.content.strip() for chunk in document.chunks],
+ [
+ "Patient: P1,Allergies: None,Region: NA,Result: N/A,Note: ok",
+ "Patient: P2,Allergies: Penicillin,Region: EU,Result: NULL",
+ "Patient: P3,Allergies: n/a,Region: NaN,Result: 42,Note: nan",
+ ],
+ )
+
+ def test_a_header_cell_that_says_na_keeps_its_label(self):
+ content = self._workbook_bytes([["Item", "NA", "EU"], ["pens", 3, 4]])
+
+ document = ExcelParser(
+ file_name="sales.xlsx",
+ file_type="xlsx",
+ xlsx_first_row_as_header=True,
+ ).parse_into_text(content)
+
+ self.assertEqual(document.chunks[0].content.strip(), "Item: pens,NA: 3,EU: 4")
+
+ def test_blank_cells_and_blank_rows_are_still_skipped(self):
+ content = self._workbook_bytes(
+ [
+ ["Item", "Count"],
+ ["pens", None],
+ [None, None],
+ ["ink", 2],
+ ]
+ )
+
+ document = ExcelParser(file_name="stock.xlsx", file_type="xlsx").parse_into_text(
+ content
+ )
+
+ self.assertEqual(
+ [chunk.content.strip() for chunk in document.chunks],
+ ["A: Item,B: Count", "A: pens", "A: ink,B: 2"],
+ )
+ self.assertEqual(
+ [block["locator"]["row_start"] for block in document.source_blocks],
+ [1, 2, 4],
+ )
+
def test_parse_phantom_shared_strings_workbook(self):
document = ExcelParser().parse_into_text(_xlsx_with_phantom_shared_strings())
self.assertIn("hello", document.content)