Skip to content

Build and run agents you can see, understand and trust.

fix(rag): drop a leading byte order mark in TextParser (#3745)

Jingbo Chai64eea899e4b3
diff --git a/src/agentscope/rag/_parser/_text.py b/src/agentscope/rag/_parser/_text.py
index 6fd41161a..18d89db96 100644
--- a/src/agentscope/rag/_parser/_text.py
+++ b/src/agentscope/rag/_parser/_text.py
@@ -66,6 +66,34 @@ class TextParser(ParserBase):
"""
self.encoding = encoding
+ def _decode(self, raw: bytes, filename: str) -> str:
+ """Decode file bytes, dropping a leading byte order mark.
+
+ The mark is an encoding signature rather than content; files
+ such as Excel's "CSV UTF-8" exports always start with one.
+
+ Args:
+ raw (`bytes`):
+ The file bytes.
+ filename (`str`):
+ The source filename, used in the error message.
+
+ Returns:
+ `str`:
+ The decoded text without a leading ``U+FEFF``.
+
+ Raises:
+ `ValueError`: If the bytes cannot be decoded with the
+ configured encoding.
+ """
+ try:
+ text = raw.decode(self.encoding)
+ except UnicodeDecodeError as e:
+ raise ValueError(
+ f"Failed to decode {filename!r} as {self.encoding!r}: {e}",
+ ) from e
+ return text.removeprefix("\ufeff")
+
async def parse(
self,
file: bytes | str,
@@ -97,24 +125,11 @@ class TextParser(ParserBase):
if isinstance(file, str):
if os.path.isfile(file):
with open(file, "rb") as fp:
- raw = fp.read()
- try:
- text = raw.decode(self.encoding)
- except UnicodeDecodeError as e:
- raise ValueError(
- f"Failed to decode {filename!r} as "
- f"{self.encoding!r}: {e}",
- ) from e
+ text = self._decode(fp.read(), filename)
else:
text = file
else:
- try:
- text = file.decode(self.encoding)
- except UnicodeDecodeError as e:
- raise ValueError(
- f"Failed to decode {filename!r} as "
- f"{self.encoding!r}: {e}",
- ) from e
+ text = self._decode(file, filename)
return [
Section(
diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.py
index 5aff9d447..e76b9675d 100644
--- a/tests/rag_parser_test.py
+++ b/tests/rag_parser_test.py
@@ -639,6 +639,27 @@ class TextParserTest(IsolatedAsyncioTestCase):
with self.assertRaises(ValueError):
await parser.parse(b"\xff\xfe", "bad.txt")
+ async def test_leading_byte_order_mark_is_dropped(self) -> None:
+ """A UTF-8 byte order mark is not part of the decoded text."""
+ import tempfile
+
+ parser = TextParser()
+ for raw, text in (
+ (b"\xef\xbb\xbfname,age\n", "name,age\n"),
+ (b"\xef\xbb\xbf", ""),
+ (b"name,age\n", "name,age\n"),
+ ):
+ with self.subTest(raw=raw):
+ sections = await parser.parse(raw, "x.csv")
+ self.assertEqual(sections[0].content.text, text)
+
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "x.csv")
+ with open(path, "wb") as f:
+ f.write(b"\xef\xbb\xbfname,age\n")
+ sections = await parser.parse(path, "x.csv")
+ self.assertEqual(sections[0].content.text, "name,age\n")
+
class PDFParserTest(IsolatedAsyncioTestCase):
"""Behavioural coverage for :class:`PDFParser`."""