Python tool for converting files and office documents to Markdown.
fix(ipynb): strip the UTF-8 BOM so a notebook is not emitted as raw JSON (#2425)
kevin2e71e117eab8
diff --git a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py b/packages/markitdown/src/markitdown/converters/_ipynb_converter.pyindex 9152218e..6bc1c167 100644--- a/packages/markitdown/src/markitdown/converters/_ipynb_converter.py+++ b/packages/markitdown/src/markitdown/converters/_ipynb_converter.py@@ -54,6 +54,11 @@ class IpynbConverter(DocumentConverter):# Parse and convert the notebookencoding = stream_info.charset or "utf-8"notebook_content = file_stream.read().decode(encoding=encoding)++ # Editors and Windows tooling prepend a UTF-8 BOM, which json rejects+ # outright; strip it so the notebook is read as a notebook.+ notebook_content = notebook_content.lstrip("\ufeff")+return self._convert(json.loads(notebook_content))def _convert(self, notebook_content: dict) -> DocumentConverterResult:diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex b84ebd54..f1bde1e0 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -1412,6 +1412,72 @@ def test_ipynb_heading_title_preserves_leading_hash() -> None:assert result.title == "#hashtag campaign results"+_UTF8_BOM = b"\xef\xbb\xbf"++_BOM_NOTEBOOK = {+ "nbformat": 4,+ "nbformat_minor": 5,+ "metadata": {},+ "cells": [+ {"cell_type": "markdown", "source": ["# Quarterly notes\n"], "metadata": {}},+ {"cell_type": "code", "source": ["print('hello')\n"], "metadata": {}},+ ],+}+++def test_ipynb_with_a_utf8_bom_is_still_converted() -> None:+ """A leading BOM must not stop the notebook from being parsed."""+ from markitdown.converters._ipynb_converter import IpynbConverter++ data = _UTF8_BOM + json.dumps(_BOM_NOTEBOOK).encode("utf-8")++ result = IpynbConverter().convert(+ io.BytesIO(data), StreamInfo(extension=".ipynb", charset="utf-8")+ )++ assert "# Quarterly notes" in result.markdown+ assert "```python\nprint('hello')" in result.markdown+ assert result.title == "Quarterly notes"+++def test_ipynb_with_a_utf8_bom_is_not_emitted_as_raw_json() -> None:+ """The whole stack must not fall through to the plain-text converter."""+ data = _UTF8_BOM + json.dumps(_BOM_NOTEBOOK).encode("utf-8")++ result = MarkItDown().convert_stream(+ io.BytesIO(data), stream_info=StreamInfo(extension=".ipynb")+ )++ assert result.markdown.startswith("# Quarterly notes")+ assert "nbformat" not in result.markdown+++def test_ipynb_without_a_bom_is_unchanged() -> None:+ """The case that already worked must produce exactly the same markdown."""+ from markitdown.converters._ipynb_converter import IpynbConverter++ data = json.dumps(_BOM_NOTEBOOK).encode("utf-8")++ result = IpynbConverter().convert(+ io.BytesIO(data), StreamInfo(extension=".ipynb", charset="utf-8")+ )++ assert result.markdown == "# Quarterly notes\n\n\n```python\nprint('hello')\n\n```"+++def test_ipynb_utf8_sig_charset_still_works() -> None:+ """A charset that already consumes the BOM must not be double-stripped."""+ from markitdown.converters._ipynb_converter import IpynbConverter++ data = _UTF8_BOM + json.dumps(_BOM_NOTEBOOK).encode("utf-8")++ result = IpynbConverter().convert(+ io.BytesIO(data), StreamInfo(extension=".ipynb", charset="utf-8-sig")+ )++ assert "# Quarterly notes" in result.markdown++def test_ipynb_accepts_non_ascii() -> None:"""IpynbConverter.accepts() must not raise on non-ASCII binary content."""from markitdown.converters._ipynb_converter import IpynbConverter