Python tool for converting files and office documents to Markdown.
fix(youtube): fall back to HTML when no video content is extracted (#2469)
afourney5640da71452d
diff --git a/packages/markitdown/src/markitdown/converters/_youtube_converter.py b/packages/markitdown/src/markitdown/converters/_youtube_converter.pyindex c3779743..70e8457b 100644--- a/packages/markitdown/src/markitdown/converters/_youtube_converter.py+++ b/packages/markitdown/src/markitdown/converters/_youtube_converter.py@@ -7,6 +7,7 @@ from urllib.parse import parse_qs, urlparse, unquotefrom .._base_converter import DocumentConverter, DocumentConverterResultfrom .._stream_info import StreamInfo+from ._html_converter import HtmlConverter# Optional YouTube transcription supporttry:@@ -35,7 +36,7 @@ ACCEPTED_FILE_EXTENSIONS = [class YouTubeConverter(DocumentConverter):- """Handle YouTube specially, focusing on the video title, description, and transcript."""+ """Extract YouTube metadata and transcripts, or fall back to HTML."""def _get_video_id(self, url: str) -> Union[str, None]:"""Extract a YouTube video ID from supported URL formats."""@@ -92,6 +93,7 @@ class YouTubeConverter(DocumentConverter):**kwargs: Any, # Options to pass to the converter) -> DocumentConverterResult:# Parse the stream+ start_position = file_stream.tell()encoding = "utf-8" if stream_info.charset is None else stream_info.charsetsoup = bs4.BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)@@ -134,7 +136,7 @@ class YouTubeConverter(DocumentConverter):pass# Start preparing the page- webpage_text = "# YouTube\n"+ webpage_text = ""title = self._get(metadata, ["title", "og:title", "name"]) or ""@@ -203,8 +205,12 @@ class YouTubeConverter(DocumentConverter):if transcript_text:webpage_text += f"\n### Transcript\n{transcript_text}\n"+ if not webpage_text:+ file_stream.seek(start_position)+ return HtmlConverter().convert(file_stream, stream_info, **kwargs)+return DocumentConverterResult(- markdown=webpage_text,+ markdown="# YouTube\n" + webpage_text,title=title,)diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex bc031246..18690d18 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -1339,7 +1339,7 @@ def test_pptx_chart_with_title_text_frame() -> None:def test_youtube_converter_missing_title_metadata() -> None:- """Test that YouTubeConverter converts streams with and without title metadata without raising AssertionError."""+ """Missing titles fall back to HTML when no video content is extracted."""from unittest.mock import patchfrom markitdown.converters._youtube_converter import YouTubeConverter@@ -1358,8 +1358,8 @@ def test_youtube_converter_missing_title_metadata() -> None:html_content_no_title = b"<html><head></head><body>Video Content</body></html>"stream_no_title = io.BytesIO(html_content_no_title)result_no_title = converter.convert(stream_no_title, stream_info)- assert result_no_title.title == ""- assert "# YouTube" in result_no_title.markdown+ assert result_no_title.title is None+ assert result_no_title.markdown == "Video Content"# Case 2: Stream with an empty <title> taghtml_content_empty_title = (@@ -1367,15 +1367,15 @@ def test_youtube_converter_missing_title_metadata() -> None:)stream_empty_title = io.BytesIO(html_content_empty_title)result_empty_title = converter.convert(stream_empty_title, stream_info)- assert result_empty_title.title == ""- assert "# YouTube" in result_empty_title.markdown+ assert result_empty_title.title is None+ assert result_empty_title.markdown == "Video Content"# Case 3: Stream whose title is only available from the <title> taghtml_content_title_tag = b"<html><head><title>Fallback Title</title></head><body>Video Content</body></html>"stream_title_tag = io.BytesIO(html_content_title_tag)result_title_tag = converter.convert(stream_title_tag, stream_info)assert result_title_tag.title == "Fallback Title"- assert "# YouTube" in result_title_tag.markdown+ assert result_title_tag.markdown == "# YouTube\n\n## Fallback Title\n"def test_zip_duplicate_filenames_preserve_each_entry() -> None:diff --git a/packages/markitdown/tests/test_youtube_converter.py b/packages/markitdown/tests/test_youtube_converter.pynew file mode 100644index 00000000..4293f620--- /dev/null+++ b/packages/markitdown/tests/test_youtube_converter.py@@ -0,0 +1,168 @@+import io+from types import SimpleNamespace+from unittest.mock import MagicMock++import pytest++from markitdown import MarkItDown, StreamInfo+from markitdown.converters import HtmlConverter, YouTubeConverter+import markitdown.converters._youtube_converter as youtube_module+++def transcript_api(monkeypatch: pytest.MonkeyPatch) -> MagicMock:+ api = MagicMock()+ api.return_value.list.return_value = []+ api.return_value.fetch.return_value = []+ monkeypatch.setattr(youtube_module, "YouTubeTranscriptApi", api, raising=False)+ monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", False)+ return api++[email protected]("head", ["", "<title></title>"])[email protected]("use_dispatcher", [False, True])[email protected]("transcript_capable", [False, True])+def test_youtube_empty_extraction_preserves_html(+ head: str,+ use_dispatcher: bool,+ transcript_capable: bool,+ transcript_api: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", transcript_capable+ )+ html = (+ f"<html><head>{head}"+ "<script>do_not_render_script()</script>"+ "<style>.do_not_render_style { color: red; }</style></head><body>"+ "<h1>Saved transcript</h1>"+ "<table><tr><th>Time</th><th>Text</th></tr>"+ "<tr><td>00:01</td><td>caf\u00e9</td></tr></table></body></html>"+ ).encode("windows-1252")+ prefix = b"<p>Ignored stream prefix</p>"+ stream = io.BytesIO(prefix + html)+ stream.seek(len(prefix))+ stream_info = StreamInfo(+ mimetype="text/html",+ extension=".html",+ charset="windows-1252",+ url="https://www.youtube.com/watch?v=12345",+ )++ if use_dispatcher:+ result = MarkItDown().convert_stream(+ stream, stream_info=stream_info, heading_style="ATX_CLOSED"+ )+ else:+ result = YouTubeConverter().convert(+ stream, stream_info, heading_style="ATX_CLOSED"+ )++ expected = HtmlConverter().convert(+ io.BytesIO(html), stream_info, heading_style="ATX_CLOSED"+ )+ assert result.markdown == expected.markdown+ assert result.title == expected.title+ assert "# Saved transcript #" in result.markdown+ assert "| 00:01 | caf\u00e9 |" in result.markdown+ assert "Ignored stream prefix" not in result.markdown+ assert "do_not_render" not in result.markdown+ if transcript_capable:+ transcript_api.return_value.fetch.assert_called_once()+ else:+ transcript_api.assert_not_called()+++ ("head", "title", "content"),+ [+ ("<title>Video title</title>", "Video title", "\n## Video title\n"),+ (+ '<meta property="og:title" content="Video title">',+ "Video title",+ "\n## Video title\n",+ ),+ (+ '<meta itemprop="name" content="Video title">',+ "Video title",+ "\n## Video title\n",+ ),+ (+ '<meta name="description" content="Video description">',+ "",+ "\n### Description\nVideo description\n",+ ),+ (+ '<meta property="og:description" content="Video description">',+ "",+ "\n### Description\nVideo description\n",+ ),+ (+ '<script>var ytInitialData = {"attributedDescriptionBodyText": '+ '{"content": "Video description"}};</script>',+ "",+ "\n### Description\nVideo description\n",+ ),+ (+ '<meta itemprop="interactionCount" content="42">'+ '<meta name="keywords" content="example">'+ '<meta itemprop="duration" content="PT1M">',+ "",+ "\n### Video Metadata\n"+ "- **Views:** 42\n- **Keywords:** example\n- **Runtime:** PT1M\n\n",+ ),+ ],+)+def test_youtube_preserves_metadata_output(+ head: str, title: str, content: str, transcript_api: MagicMock+) -> None:+ html = f"<html><head>{head}</head><body>Unused HTML body</body></html>"+ result = YouTubeConverter().convert(+ io.BytesIO(html.encode("utf-8")),+ StreamInfo(+ mimetype="text/html",+ charset="utf-8",+ url="https://www.youtube.com/watch?v=12345",+ ),+ )++ assert result.title == title+ assert result.markdown == "# YouTube\n" + content+ transcript_api.assert_not_called()++[email protected]("title", ["", "Video title"])+def test_youtube_preserves_library_transcript(+ title: str,+ transcript_api: MagicMock,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", True)+ transcript_api.return_value.list.return_value = [+ SimpleNamespace(language_code="fr")+ ]+ transcript_api.return_value.fetch.return_value = [+ SimpleNamespace(text="Bonjour"),+ SimpleNamespace(text="tout le monde"),+ ]+ head = f"<title>{title}</title>" if title else ""+ html = f"<html><head>{head}</head><body>Unused HTML body</body></html>"++ result = YouTubeConverter().convert(+ io.BytesIO(html.encode("utf-8")),+ StreamInfo(+ mimetype="text/html",+ charset="utf-8",+ url="https://www.youtube.com/watch?v=12345",+ ),+ youtube_transcript_languages=["fr"],+ )++ title_section = f"\n## {title}\n" if title else ""+ assert result.title == title+ assert result.markdown == (+ f"# YouTube\n{title_section}\n### Transcript\nBonjour tout le monde\n"+ )+ transcript_api.return_value.list.assert_called_once_with("12345")+ transcript_api.return_value.fetch.assert_called_once_with("12345", languages=["fr"])