Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): drop a leading byte order mark in TextParser (#3745)
Jingbo Chai64eea899e4b3
diff --git a/src/agentscope/rag/_parser/_text.py b/src/agentscope/rag/_parser/_text.pyindex 6fd41161a..18d89db96 100644--- a/src/agentscope/rag/_parser/_text.py+++ b/src/agentscope/rag/_parser/_text.py@@ -66,6 +66,34 @@ class TextParser(ParserBase):"""self.encoding = encoding+ def _decode(self, raw: bytes, filename: str) -> str:+ """Decode file bytes, dropping a leading byte order mark.++ The mark is an encoding signature rather than content; files+ such as Excel's "CSV UTF-8" exports always start with one.++ Args:+ raw (`bytes`):+ The file bytes.+ filename (`str`):+ The source filename, used in the error message.++ Returns:+ `str`:+ The decoded text without a leading ``U+FEFF``.++ Raises:+ `ValueError`: If the bytes cannot be decoded with the+ configured encoding.+ """+ try:+ text = raw.decode(self.encoding)+ except UnicodeDecodeError as e:+ raise ValueError(+ f"Failed to decode {filename!r} as {self.encoding!r}: {e}",+ ) from e+ return text.removeprefix("\ufeff")+async def parse(self,file: bytes | str,@@ -97,24 +125,11 @@ class TextParser(ParserBase):if isinstance(file, str):if os.path.isfile(file):with open(file, "rb") as fp:- raw = fp.read()- try:- text = raw.decode(self.encoding)- except UnicodeDecodeError as e:- raise ValueError(- f"Failed to decode {filename!r} as "- f"{self.encoding!r}: {e}",- ) from e+ text = self._decode(fp.read(), filename)else:text = fileelse:- try:- text = file.decode(self.encoding)- except UnicodeDecodeError as e:- raise ValueError(- f"Failed to decode {filename!r} as "- f"{self.encoding!r}: {e}",- ) from e+ text = self._decode(file, filename)return [Section(diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.pyindex 5aff9d447..e76b9675d 100644--- a/tests/rag_parser_test.py+++ b/tests/rag_parser_test.py@@ -639,6 +639,27 @@ class TextParserTest(IsolatedAsyncioTestCase):with self.assertRaises(ValueError):await parser.parse(b"\xff\xfe", "bad.txt")+ async def test_leading_byte_order_mark_is_dropped(self) -> None:+ """A UTF-8 byte order mark is not part of the decoded text."""+ import tempfile++ parser = TextParser()+ for raw, text in (+ (b"\xef\xbb\xbfname,age\n", "name,age\n"),+ (b"\xef\xbb\xbf", ""),+ (b"name,age\n", "name,age\n"),+ ):+ with self.subTest(raw=raw):+ sections = await parser.parse(raw, "x.csv")+ self.assertEqual(sections[0].content.text, text)++ with tempfile.TemporaryDirectory() as tmp:+ path = os.path.join(tmp, "x.csv")+ with open(path, "wb") as f:+ f.write(b"\xef\xbb\xbfname,age\n")+ sections = await parser.parse(path, "x.csv")+ self.assertEqual(sections[0].content.text, "name,age\n")+class PDFParserTest(IsolatedAsyncioTestCase):"""Behavioural coverage for :class:`PDFParser`."""