Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): stop a non-ASCII base64 payload from failing the whole parse

VinhChuae185dd1a940
diff --git a/docreader/tests/test_endecode.py b/docreader/tests/test_endecode.py
new file mode 100644
index 000000000..dba95bc34
--- /dev/null
+++ b/docreader/tests/test_endecode.py
@@ -0,0 +1,60 @@
+import base64
+import binascii
+import re
+import unittest
+
+from docreader.utils.endecode import encode_image
+
+# One valid PNG signature, the shortest payload that proves a decode happened.
+PNG_SIGNATURE = bytes.fromhex("89504e470d0a1a0a")
+VALID_B64 = base64.b64encode(PNG_SIGNATURE).decode()
+
+
+class TestEncodeImage(unittest.TestCase):
+ def test_valid_base64_round_trips(self):
+ self.assertEqual(encode_image(VALID_B64), PNG_SIGNATURE)
+
+ def test_ignore_swallows_bad_padding(self):
+ """A truncated payload raises binascii.Error, which 'ignore' absorbs."""
+ self.assertEqual(encode_image(VALID_B64[:-3], errors="ignore"), b"")
+
+ def test_ignore_swallows_non_ascii(self):
+ """Non-ASCII input is a decoding error too, so 'ignore' must absorb it.
+
+ base64.b64decode rejects a non-ASCII str with a plain ValueError rather
+ than binascii.Error, so catching only binascii.Error lets it escape a
+ call that asked not to raise.
+ """
+ self.assertEqual(encode_image(VALID_B64 + ' "图片"', errors="ignore"), b"")
+
+ def test_strict_still_raises_on_non_ascii(self):
+ with self.assertRaises(ValueError):
+ encode_image(VALID_B64 + ' "图片"')
+
+ def test_strict_still_raises_on_bad_padding(self):
+ with self.assertRaises(binascii.Error):
+ encode_image(VALID_B64[:-3])
+
+
+class TestMarkdownBase64Reachability(unittest.TestCase):
+ """The payload group of the markdown data-URI pattern is not ASCII-only.
+
+ Pinned here rather than in the parser tests because importing
+ docreader.parser pulls in the whole parser dependency set; the pattern
+ itself is what decides whether encode_image can ever see non-ASCII.
+ """
+
+ # docreader/parser/markdown_parser.py
+ B64_PATTERN = re.compile(r"!\[(.*?)\]\(data:image/([^;]+);base64,([^\)]+)\)")
+
+ def test_markdown_title_lands_in_the_payload_group(self):
+ markdown = '![x](data:image/png;base64,%s "图片")' % VALID_B64
+ match = self.B64_PATTERN.search(markdown)
+ self.assertIsNotNone(match)
+ self.assertIn("图片", match.group(3))
+ # What the parser then hands to encode_image(errors="ignore").
+ self.assertEqual(encode_image(match.group(3), errors="ignore"), b"")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docreader/utils/endecode.py b/docreader/utils/endecode.py
index a38f3559a..d7f675a86 100644
--- a/docreader/utils/endecode.py
+++ b/docreader/utils/endecode.py
@@ -9,7 +9,6 @@ with a focus on image and text data conversion:
"""
import base64
-import binascii
import io
import logging
from typing import List, Union
@@ -84,7 +83,7 @@ def encode_image(image: str, errors="strict") -> bytes:
Args:
image: Base64 encoded string representation of an image
errors: Error handling scheme for decoding errors:
- - 'strict' (default): Raise binascii.Error on decoding errors
+ - 'strict' (default): Raise on decoding errors
- 'ignore': Return empty bytes on decoding errors
- Any other name registered with codecs.register_error
@@ -92,7 +91,10 @@ def encode_image(image: str, errors="strict") -> bytes:
bytes: Decoded image bytes, or empty bytes if errors='ignore' and decoding fails
Raises:
- binascii.Error: If decoding fails and errors='strict'
+ ValueError: If decoding fails and errors='strict'. base64.b64decode
+ signals bad padding or a bad alphabet with binascii.Error, and a
+ non-ASCII str with a plain ValueError; binascii.Error subclasses
+ ValueError, so both arrive as one.
Example:
>>> base64_str = "iVBORw0KGgoAAAANSUhEUgAAAAUA..."
@@ -103,8 +105,10 @@ def encode_image(image: str, errors="strict") -> bytes:
try:
# Attempt to decode the base64 string to bytes
image_bytes = base64.b64decode(image)
- except binascii.Error as e:
- # Handle decoding errors based on the errors parameter
+ except ValueError as e:
+ # Catching binascii.Error alone would miss the non-ASCII case: a str
+ # that is not pure ASCII fails before the decoder runs and surfaces as
+ # a plain ValueError, which errors="ignore" is supposed to absorb.
if errors == "ignore":
return b""
else: