Skip to content

Python tool for converting files and office documents to Markdown.

fix(youtube): fall back to HTML when no video content is extracted (#2469)

afourney5640da71452d
diff --git a/packages/markitdown/src/markitdown/converters/_youtube_converter.py b/packages/markitdown/src/markitdown/converters/_youtube_converter.py
index c3779743..70e8457b 100644
--- a/packages/markitdown/src/markitdown/converters/_youtube_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_youtube_converter.py
@@ -7,6 +7,7 @@ from urllib.parse import parse_qs, urlparse, unquote
from .._base_converter import DocumentConverter, DocumentConverterResult
from .._stream_info import StreamInfo
+from ._html_converter import HtmlConverter
# Optional YouTube transcription support
try:
@@ -35,7 +36,7 @@ ACCEPTED_FILE_EXTENSIONS = [
class YouTubeConverter(DocumentConverter):
- """Handle YouTube specially, focusing on the video title, description, and transcript."""
+ """Extract YouTube metadata and transcripts, or fall back to HTML."""
def _get_video_id(self, url: str) -> Union[str, None]:
"""Extract a YouTube video ID from supported URL formats."""
@@ -92,6 +93,7 @@ class YouTubeConverter(DocumentConverter):
**kwargs: Any, # Options to pass to the converter
) -> DocumentConverterResult:
# Parse the stream
+ start_position = file_stream.tell()
encoding = "utf-8" if stream_info.charset is None else stream_info.charset
soup = bs4.BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)
@@ -134,7 +136,7 @@ class YouTubeConverter(DocumentConverter):
pass
# Start preparing the page
- webpage_text = "# YouTube\n"
+ webpage_text = ""
title = self._get(metadata, ["title", "og:title", "name"]) or ""
@@ -203,8 +205,12 @@ class YouTubeConverter(DocumentConverter):
if transcript_text:
webpage_text += f"\n### Transcript\n{transcript_text}\n"
+ if not webpage_text:
+ file_stream.seek(start_position)
+ return HtmlConverter().convert(file_stream, stream_info, **kwargs)
+
return DocumentConverterResult(
- markdown=webpage_text,
+ markdown="# YouTube\n" + webpage_text,
title=title,
)
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index bc031246..18690d18 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -1339,7 +1339,7 @@ def test_pptx_chart_with_title_text_frame() -> None:
def test_youtube_converter_missing_title_metadata() -> None:
- """Test that YouTubeConverter converts streams with and without title metadata without raising AssertionError."""
+ """Missing titles fall back to HTML when no video content is extracted."""
from unittest.mock import patch
from markitdown.converters._youtube_converter import YouTubeConverter
@@ -1358,8 +1358,8 @@ def test_youtube_converter_missing_title_metadata() -> None:
html_content_no_title = b"<html><head></head><body>Video Content</body></html>"
stream_no_title = io.BytesIO(html_content_no_title)
result_no_title = converter.convert(stream_no_title, stream_info)
- assert result_no_title.title == ""
- assert "# YouTube" in result_no_title.markdown
+ assert result_no_title.title is None
+ assert result_no_title.markdown == "Video Content"
# Case 2: Stream with an empty <title> tag
html_content_empty_title = (
@@ -1367,15 +1367,15 @@ def test_youtube_converter_missing_title_metadata() -> None:
)
stream_empty_title = io.BytesIO(html_content_empty_title)
result_empty_title = converter.convert(stream_empty_title, stream_info)
- assert result_empty_title.title == ""
- assert "# YouTube" in result_empty_title.markdown
+ assert result_empty_title.title is None
+ assert result_empty_title.markdown == "Video Content"
# Case 3: Stream whose title is only available from the <title> tag
html_content_title_tag = b"<html><head><title>Fallback Title</title></head><body>Video Content</body></html>"
stream_title_tag = io.BytesIO(html_content_title_tag)
result_title_tag = converter.convert(stream_title_tag, stream_info)
assert result_title_tag.title == "Fallback Title"
- assert "# YouTube" in result_title_tag.markdown
+ assert result_title_tag.markdown == "# YouTube\n\n## Fallback Title\n"
def test_zip_duplicate_filenames_preserve_each_entry() -> None:
diff --git a/packages/markitdown/tests/test_youtube_converter.py b/packages/markitdown/tests/test_youtube_converter.py
new file mode 100644
index 00000000..4293f620
--- /dev/null
+++ b/packages/markitdown/tests/test_youtube_converter.py
@@ -0,0 +1,168 @@
+import io
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+from markitdown.converters import HtmlConverter, YouTubeConverter
+import markitdown.converters._youtube_converter as youtube_module
+
+
+def transcript_api(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ api = MagicMock()
+ api.return_value.list.return_value = []
+ api.return_value.fetch.return_value = []
+ monkeypatch.setattr(youtube_module, "YouTubeTranscriptApi", api, raising=False)
+ monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", False)
+ return api
+
+
[email protected]("head", ["", "<title></title>"])
[email protected]("use_dispatcher", [False, True])
[email protected]("transcript_capable", [False, True])
+def test_youtube_empty_extraction_preserves_html(
+ head: str,
+ use_dispatcher: bool,
+ transcript_capable: bool,
+ transcript_api: MagicMock,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", transcript_capable
+ )
+ html = (
+ f"<html><head>{head}"
+ "<script>do_not_render_script()</script>"
+ "<style>.do_not_render_style { color: red; }</style></head><body>"
+ "<h1>Saved transcript</h1>"
+ "<table><tr><th>Time</th><th>Text</th></tr>"
+ "<tr><td>00:01</td><td>caf\u00e9</td></tr></table></body></html>"
+ ).encode("windows-1252")
+ prefix = b"<p>Ignored stream prefix</p>"
+ stream = io.BytesIO(prefix + html)
+ stream.seek(len(prefix))
+ stream_info = StreamInfo(
+ mimetype="text/html",
+ extension=".html",
+ charset="windows-1252",
+ url="https://www.youtube.com/watch?v=12345",
+ )
+
+ if use_dispatcher:
+ result = MarkItDown().convert_stream(
+ stream, stream_info=stream_info, heading_style="ATX_CLOSED"
+ )
+ else:
+ result = YouTubeConverter().convert(
+ stream, stream_info, heading_style="ATX_CLOSED"
+ )
+
+ expected = HtmlConverter().convert(
+ io.BytesIO(html), stream_info, heading_style="ATX_CLOSED"
+ )
+ assert result.markdown == expected.markdown
+ assert result.title == expected.title
+ assert "# Saved transcript #" in result.markdown
+ assert "| 00:01 | caf\u00e9 |" in result.markdown
+ assert "Ignored stream prefix" not in result.markdown
+ assert "do_not_render" not in result.markdown
+ if transcript_capable:
+ transcript_api.return_value.fetch.assert_called_once()
+ else:
+ transcript_api.assert_not_called()
+
+
+ ("head", "title", "content"),
+ [
+ ("<title>Video title</title>", "Video title", "\n## Video title\n"),
+ (
+ '<meta property="og:title" content="Video title">',
+ "Video title",
+ "\n## Video title\n",
+ ),
+ (
+ '<meta itemprop="name" content="Video title">',
+ "Video title",
+ "\n## Video title\n",
+ ),
+ (
+ '<meta name="description" content="Video description">',
+ "",
+ "\n### Description\nVideo description\n",
+ ),
+ (
+ '<meta property="og:description" content="Video description">',
+ "",
+ "\n### Description\nVideo description\n",
+ ),
+ (
+ '<script>var ytInitialData = {"attributedDescriptionBodyText": '
+ '{"content": "Video description"}};</script>',
+ "",
+ "\n### Description\nVideo description\n",
+ ),
+ (
+ '<meta itemprop="interactionCount" content="42">'
+ '<meta name="keywords" content="example">'
+ '<meta itemprop="duration" content="PT1M">',
+ "",
+ "\n### Video Metadata\n"
+ "- **Views:** 42\n- **Keywords:** example\n- **Runtime:** PT1M\n\n",
+ ),
+ ],
+)
+def test_youtube_preserves_metadata_output(
+ head: str, title: str, content: str, transcript_api: MagicMock
+) -> None:
+ html = f"<html><head>{head}</head><body>Unused HTML body</body></html>"
+ result = YouTubeConverter().convert(
+ io.BytesIO(html.encode("utf-8")),
+ StreamInfo(
+ mimetype="text/html",
+ charset="utf-8",
+ url="https://www.youtube.com/watch?v=12345",
+ ),
+ )
+
+ assert result.title == title
+ assert result.markdown == "# YouTube\n" + content
+ transcript_api.assert_not_called()
+
+
[email protected]("title", ["", "Video title"])
+def test_youtube_preserves_library_transcript(
+ title: str,
+ transcript_api: MagicMock,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", True)
+ transcript_api.return_value.list.return_value = [
+ SimpleNamespace(language_code="fr")
+ ]
+ transcript_api.return_value.fetch.return_value = [
+ SimpleNamespace(text="Bonjour"),
+ SimpleNamespace(text="tout le monde"),
+ ]
+ head = f"<title>{title}</title>" if title else ""
+ html = f"<html><head>{head}</head><body>Unused HTML body</body></html>"
+
+ result = YouTubeConverter().convert(
+ io.BytesIO(html.encode("utf-8")),
+ StreamInfo(
+ mimetype="text/html",
+ charset="utf-8",
+ url="https://www.youtube.com/watch?v=12345",
+ ),
+ youtube_transcript_languages=["fr"],
+ )
+
+ title_section = f"\n## {title}\n" if title else ""
+ assert result.title == title
+ assert result.markdown == (
+ f"# YouTube\n{title_section}\n### Transcript\nBonjour tout le monde\n"
+ )
+ transcript_api.return_value.list.assert_called_once_with("12345")
+ transcript_api.return_value.fetch.assert_called_once_with("12345", languages=["fr"])