Skip to content

Build and run agents you can see, understand and trust.

fix(rag): preserve duplicate and blank Excel headers (#2948)

little-spoon779af237357c6f
diff --git a/src/agentscope/rag/_parser/_excel.py b/src/agentscope/rag/_parser/_excel.py
index 2c133a9e0..7e02149ad 100644
--- a/src/agentscope/rag/_parser/_excel.py
+++ b/src/agentscope/rag/_parser/_excel.py
@@ -46,7 +46,7 @@ def _get_excel_column_name(col_index: int) -> str:
def _extract_table_data(df: Any) -> list[list[str]]:
- """Extract table data from a pandas DataFrame.
+ """Extract table data from a DataFrame read with ``header=None``.
NaN values are converted to empty strings, and Windows-style line
breaks (``\\r\\n``) are normalised to ``\\n``.
@@ -62,8 +62,7 @@ def _extract_table_data(df: Any) -> list[list[str]]:
"""
import pandas as pd
- header = [str(col).strip() for col in df.columns]
- rows: list[list[str]] = [header]
+ rows: list[list[str]] = []
for _, row in df.iterrows():
cells: list[str] = []
for val in row:
@@ -315,6 +314,7 @@ class ExcelParser(ParserBase):
# Keep cell text as-is instead of letting pandas infer types or NAs
df = excel_file.parse(
sheet_name=sheet_name,
+ header=None,
dtype=object,
keep_default_na=False,
)
@@ -322,8 +322,9 @@ class ExcelParser(ParserBase):
logger.warning("Failed to parse sheet '%s': %s", sheet_name, e)
return sheet_sections
- # A header-only sheet is "empty" to pandas but still has columns
- if len(df.columns) > 0:
+ # Keep the first row as cell data: pandas column labels would rename
+ # duplicate headers and replace blank headers with "Unnamed: ...".
+ if not df.empty:
table_data = _extract_table_data(df)
if self.table_format == "markdown":
diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.py
index 572647914..212d5ed6f 100644
--- a/tests/rag_parser_test.py
+++ b/tests/rag_parser_test.py
@@ -1110,6 +1110,59 @@ class PPTParserTest(IsolatedAsyncioTestCase):
class ExcelParserTest(IsolatedAsyncioTestCase):
"""Behavioural coverage for :class:`ExcelParser`."""
+ async def test_duplicate_and_blank_headers(self) -> None:
+ """Headers remain cell values, without pandas-generated labels."""
+ from openpyxl import Workbook
+
+ workbook = Workbook()
+ workbook.active.append(["Name", "Name", None])
+ workbook.active.append(["a", "b", "c"])
+ buffer = io.BytesIO()
+ workbook.save(buffer)
+ workbook.close()
+
+ for table_format in ("markdown", "json"):
+ for coordinates in (False, True):
+ with self.subTest(
+ table_format=table_format,
+ coordinates=coordinates,
+ ):
+ parser = ExcelParser(
+ table_format=table_format,
+ include_cell_coordinates=coordinates,
+ include_sheet_names=False,
+ )
+ sections = await parser.parse(
+ buffer.getvalue(),
+ "headers.xlsx",
+ )
+ self.assertEqual(len(sections), 1)
+ text = sections[0].content.text
+ if table_format == "markdown":
+ expected = (
+ "| [A1] Name | [B1] Name | [C1] |\n"
+ "| --- | --- | --- |\n"
+ "| [A2] a | [B2] b | [C2] c |\n"
+ if coordinates
+ else "| Name | Name | |\n"
+ "| --- | --- | --- |\n"
+ "| a | b | c |\n"
+ )
+ self.assertEqual(text, expected)
+ else:
+ rows = [
+ json.loads(line) for line in text.splitlines()[1:]
+ ]
+ self.assertEqual(
+ rows,
+ [
+ {"A1": "Name", "B1": "Name", "C1": ""},
+ {"A2": "a", "B2": "b", "C2": "c"},
+ ]
+ if coordinates
+ else [["Name", "Name", ""], ["a", "b", "c"]],
+ )
+
async def test_invalid_input_errors(self) -> None:
"""Missing paths and invalid workbooks use documented errors."""
parser = ExcelParser()