Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): preserve native Excel error cells (#3204)
mengxiaopengfaf54fdd463b
diff --git a/src/agentscope/rag/_parser/_excel.py b/src/agentscope/rag/_parser/_excel.pyindex 7e02149ad..0f385fc44 100644--- a/src/agentscope/rag/_parser/_excel.py+++ b/src/agentscope/rag/_parser/_excel.py@@ -76,6 +76,72 @@ def _extract_table_data(df: Any) -> list[list[str]]:return rows+def _restore_native_excel_errors(+ df: Any,+ excel_file: Any,+ sheet_name: str,+) -> None:+ """Restore native Excel error text that pandas converted to NaN.++ The openpyxl reader maps cells of Excel type ``e`` (``#DIV/0!``,+ ``#N/A``, and the other stored error values) to NaN before+ ``dtype=object`` and ``keep_default_na=False`` can preserve them.+ Those values are copied back from the workbook pandas already+ opened, and only into cells that are still missing. Blanks, text,+ and numbers are left as pandas read them. Formulas are not+ evaluated, and ``.xls`` workbooks are left unchanged.++ Args:+ df (`pandas.DataFrame`):+ Sheet data read with ``header=None``.+ excel_file (`pandas.ExcelFile`):+ The open Excel file whose ``book`` holds the worksheet.+ sheet_name (`str`):+ Name of the sheet ``df`` was read from.+ """+ import pandas as pd++ if df.empty or not df.isna().any().any():+ return++ # ``sheetnames`` is openpyxl-only. xlrd books (``.xls``) are skipped+ # rather than guessing which NaNs are errors.+ book = getattr(excel_file, "book", None)+ if book is None:+ return+ sheetnames = getattr(book, "sheetnames", None)+ if sheetnames is None or sheet_name not in sheetnames:+ return++ worksheet = book[sheet_name]+ if not hasattr(worksheet, "iter_rows"):+ return++ # Local import: openpyxl is an optional RAG dependency.+ from openpyxl.cell.cell import TYPE_ERROR++ n_rows, n_cols = df.shape+ for row in worksheet.iter_rows():+ for cell in row:+ # Empty padding cells use data type "n", not TYPE_ERROR.+ if getattr(cell, "data_type", None) != TYPE_ERROR:+ continue+ value = getattr(cell, "value", None)+ if not isinstance(value, str) or not value:+ continue+ row_idx = getattr(cell, "row", 0) - 1+ col_idx = getattr(cell, "column", 0) - 1+ if (+ row_idx < 0+ or col_idx < 0+ or row_idx >= n_rows+ or col_idx >= n_cols+ ):+ continue+ if pd.isna(df.iat[row_idx, col_idx]):+ df.iat[row_idx, col_idx] = value++def _extract_images_from_worksheet(worksheet: Any,filename: str,@@ -325,6 +391,18 @@ class ExcelParser(ParserBase):# Keep the first row as cell data: pandas column labels would rename# duplicate headers and replace blank headers with "Unnamed: ...".if not df.empty:+ # Pandas turns native Excel error cells into NaN. Copy the+ # stored error text back before those NaNs are rendered as+ # blanks. Other missing cells stay empty.+ try:+ _restore_native_excel_errors(df, excel_file, sheet_name)+ except Exception as e:+ logger.warning(+ "Failed to restore native Excel errors in sheet '%s': "+ "%s",+ sheet_name,+ e,+ )table_data = _extract_table_data(df)if self.table_format == "markdown":diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.pyindex 314242896..b1ff96fd6 100644--- a/tests/rag_parser_test.py+++ b/tests/rag_parser_test.py@@ -1602,6 +1602,139 @@ class ExcelParserTest(IsolatedAsyncioTestCase):],)+ async def test_native_error_cells_preserved(self) -> None:+ """Stored Excel error values stay visible in both renderers.++ pandas' openpyxl reader turns type ``e`` cells into NaN. The+ parser must copy those stored values back, while blanks, text,+ numbers, and unevaluated formulas stay as they were.+ """+ from openpyxl import Workbook++ workbook = Workbook()+ sheet = workbook.active+ sheet.title = "Errors"+ sheet.append(["Case", "Value", "Note"])+ sheet.append(["#NULL!", "text-after", 1])+ sheet.append(["native_div0", "#DIV/0!", None])+ sheet.append(["blank", None, "still-blank"])+ sheet.append(["text_na", "NA", "00123"])+ sheet.append(["integer", 42, None])+ sheet.append(["literal", "#GETTING_DATA", "N/A"])+ sheet.append(["formula", "=1/0", None])+ sheet.append(["native_na", "#N/A", "#VALUE!"])+ self.assertEqual(sheet["A2"].data_type, "e")+ self.assertEqual(sheet["A2"].value, "#NULL!")+ self.assertEqual(sheet["B3"].data_type, "e")+ self.assertEqual(sheet["B3"].value, "#DIV/0!")+ self.assertEqual(sheet["B9"].data_type, "e")+ self.assertEqual(sheet["C9"].data_type, "e")+ self.assertEqual(sheet["C9"].value, "#VALUE!")+ self.assertEqual(sheet["B7"].data_type, "s")+ self.assertEqual(sheet["B8"].data_type, "f")+ self.assertIsNone(sheet["B4"].value)+ buffer = io.BytesIO()+ workbook.save(buffer)+ workbook.close()+ payload = buffer.getvalue()++ rows = [+ ["Case", "Value", "Note"],+ ["#NULL!", "text-after", "1"],+ ["native_div0", "#DIV/0!", ""],+ ["blank", "", "still-blank"],+ ["text_na", "NA", "00123"],+ ["integer", "42", ""],+ ["literal", "#GETTING_DATA", "N/A"],+ ["formula", "", ""],+ ["native_na", "#N/A", "#VALUE!"],+ ]+ markdown = (+ "| Case | Value | Note |\n"+ "| --- | --- | --- |\n"+ "| #NULL! | text-after | 1 |\n"+ "| native_div0 | #DIV/0! | |\n"+ "| blank | | still-blank |\n"+ "| text_na | NA | 00123 |\n"+ "| integer | 42 | |\n"+ "| literal | #GETTING_DATA | N/A |\n"+ "| formula | | |\n"+ "| native_na | #N/A | #VALUE! |\n"+ )+ markdown_coords = (+ "| [A1] Case | [B1] Value | [C1] Note |\n"+ "| --- | --- | --- |\n"+ "| [A2] #NULL! | [B2] text-after | [C2] 1 |\n"+ "| [A3] native_div0 | [B3] #DIV/0! | [C3] |\n"+ "| [A4] blank | [B4] | [C4] still-blank |\n"+ "| [A5] text_na | [B5] NA | [C5] 00123 |\n"+ "| [A6] integer | [B6] 42 | [C6] |\n"+ "| [A7] literal | [B7] #GETTING_DATA | [C7] N/A |\n"+ "| [A8] formula | [B8] | [C8] |\n"+ "| [A9] native_na | [B9] #N/A | [C9] #VALUE! |\n"+ )++ for table_format in ("markdown", "json"):+ for coordinates in (False, True):+ with self.subTest(+ table_format=table_format,+ coordinates=coordinates,+ ):+ parser = ExcelParser(+ table_format=table_format,+ include_cell_coordinates=coordinates,+ include_sheet_names=False,+ )+ sections = await parser.parse(payload, "errors.xlsx")+ self.assertEqual(len(sections), 1)+ text = sections[0].content.text+ if table_format == "markdown":+ expected = markdown_coords if coordinates else markdown+ elif coordinates:+ body = "\n".join(+ json.dumps(+ {+ f"{'ABC'[col]}{row + 1}": cell+ for col, cell in enumerate(values)+ },+ ensure_ascii=False,+ )+ for row, values in enumerate(rows)+ )+ expected = (+ "<system-info>A table loaded as a JSON "+ f"array:</system-info>\n{body}"+ )+ else:+ body = "\n".join(+ json.dumps(values, ensure_ascii=False)+ for values in rows+ )+ expected = (+ "<system-info>A table loaded as a JSON "+ f"array:</system-info>\n{body}"+ )+ self.assertEqual(+ [section.model_dump() for section in sections],+ [+ {+ "content": {+ "type": "text",+ "text": expected,+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "errors.xlsx",+ "metadata": {},+ },+ ],+ )+ self.assertIn("#DIV/0!", text)+ self.assertIn("#N/A", text)+ self.assertIn("#VALUE!", text)+ self.assertIn("#NULL!", text)+async def test_multi_sheet_merged_by_default(self) -> None:"""``separate_sheet=False`` (default) merges sheets into oneSection."""