Python tool for converting files and office documents to Markdown.
Support short YouTube URLs in converter (#1882)
Mukunda Rao Katta7de6ce4943b9
diff --git a/packages/markitdown/src/markitdown/converters/_youtube_converter.py b/packages/markitdown/src/markitdown/converters/_youtube_converter.pyindex 9ca814c6..c3779743 100644--- a/packages/markitdown/src/markitdown/converters/_youtube_converter.py+++ b/packages/markitdown/src/markitdown/converters/_youtube_converter.py@@ -37,6 +37,24 @@ ACCEPTED_FILE_EXTENSIONS = [class YouTubeConverter(DocumentConverter):"""Handle YouTube specially, focusing on the video title, description, and transcript."""+ def _get_video_id(self, url: str) -> Union[str, None]:+ """Extract a YouTube video ID from supported URL formats."""+ parsed_url = urlparse(url)+ hostname = parsed_url.netloc.lower()+ path_parts = [part for part in parsed_url.path.split("/") if part]++ if hostname in {"youtu.be", "www.youtu.be"}:+ return path_parts[0] if path_parts else None++ if hostname in {"youtube.com", "www.youtube.com", "m.youtube.com"}:+ if path_parts[:1] == ["watch"]:+ params = parse_qs(parsed_url.query)+ return params.get("v", [None])[0]+ if path_parts[:1] in (["shorts"], ["embed"]):+ return path_parts[1] if len(path_parts) > 1 else None++ return None+def accepts(self,file_stream: BinaryIO,@@ -53,7 +71,7 @@ class YouTubeConverter(DocumentConverter):url = unquote(url)url = url.replace(r"\?", "?").replace(r"\=", "=")- if not url.startswith("https://www.youtube.com/watch?"):+ if not self._get_video_id(url):# Not a YouTube URLreturn False@@ -146,10 +164,8 @@ class YouTubeConverter(DocumentConverter):if IS_YOUTUBE_TRANSCRIPT_CAPABLE:ytt_api = YouTubeTranscriptApi()transcript_text = ""- parsed_url = urlparse(stream_info.url) # type: ignore- params = parse_qs(parsed_url.query) # type: ignore- if "v" in params and params["v"][0]:- video_id = str(params["v"][0])+ video_id = self._get_video_id(stream_info.url or "")+ if video_id:transcript_list = ytt_api.list(video_id)languages = ["en"]for transcript in transcript_list:diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex 2fa52f71..1effa4cd 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -23,6 +23,7 @@ from markitdown import (FileConversionException,StreamInfo,)+from markitdown.converters import YouTubeConverter# This file contains module tests that are not directly tested by the FileTestVectors.# This includes things like helper functions and runtime conversion options@@ -214,6 +215,38 @@ def test_stream_info_operations() -> None:assert updated_stream_info.url == "url.1"+ ("url", "video_id"),+ [+ ("https://www.youtube.com/watch?v=V2qZ_lgxTzg", "V2qZ_lgxTzg"),+ ("https://youtu.be/V2qZ_lgxTzg", "V2qZ_lgxTzg"),+ ("https://www.youtube.com/shorts/V2qZ_lgxTzg", "V2qZ_lgxTzg"),+ ("https://www.youtube.com/embed/V2qZ_lgxTzg", "V2qZ_lgxTzg"),+ ],+)+def test_youtube_converter_extracts_supported_video_ids(+ url: str, video_id: str+) -> None:+ converter = YouTubeConverter()+ assert converter._get_video_id(url) == video_id+++ "url",+ [+ "https://www.youtube.com/watch?v=V2qZ_lgxTzg",+ "https://youtu.be/V2qZ_lgxTzg",+ "https://www.youtube.com/shorts/V2qZ_lgxTzg",+ ],+)+def test_youtube_converter_accepts_supported_url_formats(url: str) -> None:+ converter = YouTubeConverter()+ assert converter.accepts(+ io.BytesIO(b"<html></html>"),+ StreamInfo(url=url, extension=".html"),+ )++def test_data_uris() -> None:# Test basic parsing of data URIsdata_uri = "data:text/plain;base64,SGVsbG8sIFdvcmxkIQ=="