Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): stop a non-ASCII base64 payload from failing the whole parse
VinhChuae185dd1a940
diff --git a/docreader/tests/test_endecode.py b/docreader/tests/test_endecode.pynew file mode 100644index 000000000..dba95bc34--- /dev/null+++ b/docreader/tests/test_endecode.py@@ -0,0 +1,60 @@+import base64+import binascii+import re+import unittest++from docreader.utils.endecode import encode_image++# One valid PNG signature, the shortest payload that proves a decode happened.+PNG_SIGNATURE = bytes.fromhex("89504e470d0a1a0a")+VALID_B64 = base64.b64encode(PNG_SIGNATURE).decode()+++class TestEncodeImage(unittest.TestCase):+ def test_valid_base64_round_trips(self):+ self.assertEqual(encode_image(VALID_B64), PNG_SIGNATURE)++ def test_ignore_swallows_bad_padding(self):+ """A truncated payload raises binascii.Error, which 'ignore' absorbs."""+ self.assertEqual(encode_image(VALID_B64[:-3], errors="ignore"), b"")++ def test_ignore_swallows_non_ascii(self):+ """Non-ASCII input is a decoding error too, so 'ignore' must absorb it.++ base64.b64decode rejects a non-ASCII str with a plain ValueError rather+ than binascii.Error, so catching only binascii.Error lets it escape a+ call that asked not to raise.+ """+ self.assertEqual(encode_image(VALID_B64 + ' "图片"', errors="ignore"), b"")++ def test_strict_still_raises_on_non_ascii(self):+ with self.assertRaises(ValueError):+ encode_image(VALID_B64 + ' "图片"')++ def test_strict_still_raises_on_bad_padding(self):+ with self.assertRaises(binascii.Error):+ encode_image(VALID_B64[:-3])+++class TestMarkdownBase64Reachability(unittest.TestCase):+ """The payload group of the markdown data-URI pattern is not ASCII-only.++ Pinned here rather than in the parser tests because importing+ docreader.parser pulls in the whole parser dependency set; the pattern+ itself is what decides whether encode_image can ever see non-ASCII.+ """++ # docreader/parser/markdown_parser.py+ B64_PATTERN = re.compile(r"!\[(.*?)\]\(data:image/([^;]+);base64,([^\)]+)\)")++ def test_markdown_title_lands_in_the_payload_group(self):+ markdown = '' % VALID_B64+ match = self.B64_PATTERN.search(markdown)+ self.assertIsNotNone(match)+ self.assertIn("图片", match.group(3))+ # What the parser then hands to encode_image(errors="ignore").+ self.assertEqual(encode_image(match.group(3), errors="ignore"), b"")+++if __name__ == "__main__":+ unittest.main()diff --git a/docreader/utils/endecode.py b/docreader/utils/endecode.pyindex a38f3559a..d7f675a86 100644--- a/docreader/utils/endecode.py+++ b/docreader/utils/endecode.py@@ -9,7 +9,6 @@ with a focus on image and text data conversion:"""import base64-import binasciiimport ioimport loggingfrom typing import List, Union@@ -84,7 +83,7 @@ def encode_image(image: str, errors="strict") -> bytes:Args:image: Base64 encoded string representation of an imageerrors: Error handling scheme for decoding errors:- - 'strict' (default): Raise binascii.Error on decoding errors+ - 'strict' (default): Raise on decoding errors- 'ignore': Return empty bytes on decoding errors- Any other name registered with codecs.register_error@@ -92,7 +91,10 @@ def encode_image(image: str, errors="strict") -> bytes:bytes: Decoded image bytes, or empty bytes if errors='ignore' and decoding failsRaises:- binascii.Error: If decoding fails and errors='strict'+ ValueError: If decoding fails and errors='strict'. base64.b64decode+ signals bad padding or a bad alphabet with binascii.Error, and a+ non-ASCII str with a plain ValueError; binascii.Error subclasses+ ValueError, so both arrive as one.Example:>>> base64_str = "iVBORw0KGgoAAAANSUhEUgAAAAUA..."@@ -103,8 +105,10 @@ def encode_image(image: str, errors="strict") -> bytes:try:# Attempt to decode the base64 string to bytesimage_bytes = base64.b64decode(image)- except binascii.Error as e:- # Handle decoding errors based on the errors parameter+ except ValueError as e:+ # Catching binascii.Error alone would miss the non-ASCII case: a str+ # that is not pure ASCII fails before the decoder runs and surfaces as+ # a plain ValueError, which errors="ignore" is supposed to absorb.if errors == "ignore":return b""else: