A feature-rich command-line audio/video downloader
Allow extractors to designate formats/subtitles for impersonation (#13778)
bashonly32809eb2da92
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.pyindex 9c9ee64a8c..68074a5626 100644--- a/yt_dlp/YoutubeDL.py+++ b/yt_dlp/YoutubeDL.py@@ -52,7 +52,7 @@ from .networking.exceptions import (SSLError,network_exceptions,)-from .networking.impersonate import ImpersonateRequestHandler+from .networking.impersonate import ImpersonateRequestHandler, ImpersonateTargetfrom .plugins import directories as plugin_directories, load_all_pluginsfrom .postprocessor import (EmbedThumbnailPP,@@ -3231,6 +3231,16 @@ class YoutubeDL:}else:params = self.params++ impersonate = info.pop('impersonate', None)+ # Do not override --impersonate with extractor-specified impersonation+ if params.get('impersonate') is None:+ available_target, requested_targets = self._parse_impersonate_targets(impersonate)+ if available_target:+ info['impersonate'] = available_target+ elif requested_targets:+ self.report_warning(self._unavailable_targets_message(requested_targets), only_once=True)+fd = get_suitable_downloader(info, params, to_stdout=(name == '-'))(self, params)if not test:for ph in self._progress_hooks:@@ -4183,6 +4193,31 @@ class YoutubeDL:for rh in self._request_director.handlers.values()if isinstance(rh, ImpersonateRequestHandler))+ def _parse_impersonate_targets(self, impersonate):+ if impersonate in (True, ''):+ impersonate = ImpersonateTarget()++ requested_targets = [+ t if isinstance(t, ImpersonateTarget) else ImpersonateTarget.from_str(t)+ for t in variadic(impersonate)+ ] if impersonate else []++ available_target = next(filter(self._impersonate_target_available, requested_targets), None)++ return available_target, requested_targets++ @staticmethod+ def _unavailable_targets_message(requested_targets, note=None, is_error=False):+ note = note or 'The extractor specified to use impersonation for this download'+ specific_targets = ', '.join(filter(None, map(str, requested_targets)))+ message = (+ 'no impersonate target is available' if not specific_targets+ else f'none of these impersonate targets are available: {specific_targets}')+ return (+ f'{note}, but {message}. {"See" if is_error else "If you encounter errors, then see"}'+ f' https://github.com/yt-dlp/yt-dlp#impersonation '+ f'for information on installing the required dependencies')+def urlopen(self, req):""" Start an HTTP download """if isinstance(req, str):diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.pyindex 9c34bd289a..17458b9b94 100644--- a/yt_dlp/downloader/__init__.py+++ b/yt_dlp/downloader/__init__.py@@ -99,7 +99,7 @@ def _get_suitable_downloader(info_dict, protocol, params, default):if external_downloader is None:if info_dict['to_stdout'] and FFmpegFD.can_merge_formats(info_dict, params):return FFmpegFD- elif external_downloader.lower() != 'native':+ elif external_downloader.lower() != 'native' and info_dict.get('impersonate') is None:ed = get_external_downloader(external_downloader)if ed.can_download(info_dict, external_downloader):return eddiff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.pyindex 90bfcaf552..073860f6f9 100644--- a/yt_dlp/downloader/http.py+++ b/yt_dlp/downloader/http.py@@ -27,6 +27,9 @@ class HttpFD(FileDownloader):def real_download(self, filename, info_dict):url = info_dict['url']request_data = info_dict.get('request_data', None)+ request_extensions = {}+ if info_dict.get('impersonate') is not None:+ request_extensions['impersonate'] = info_dict['impersonate']class DownloadContext(dict):__getattr__ = dict.get@@ -109,7 +112,7 @@ class HttpFD(FileDownloader):if try_call(lambda: range_end >= ctx.content_len):range_end = ctx.content_len - 1- request = Request(url, request_data, headers)+ request = Request(url, request_data, headers, extensions=request_extensions)has_range = range_start is not Noneif has_range:request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}'diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.pyindex d601e17514..8a914abf0b 100644--- a/yt_dlp/extractor/common.py+++ b/yt_dlp/extractor/common.py@@ -38,7 +38,6 @@ from ..networking.exceptions import (TransportError,network_exceptions,)-from ..networking.impersonate import ImpersonateTargetfrom ..utils import (IDENTITY,JSON_LD_RE,@@ -259,6 +258,11 @@ class InfoExtractor:* key The key (as hex) used to decrypt fragments.If `key` is given, any key URI will be ignored* iv The IV (as hex) used to decrypt fragments+ * impersonate Impersonate target(s). Can be any of the following entities:+ * an instance of yt_dlp.networking.impersonate.ImpersonateTarget+ * a string in the format of CLIENT[:OS]+ * a list or a tuple of CLIENT[:OS] strings or ImpersonateTarget instances+ * a boolean value; True means any impersonate target is sufficient* downloader_options A dictionary of downloader options(For internal use only)* http_chunk_size Chunk size for HTTP downloads@@ -336,6 +340,7 @@ class InfoExtractor:* "name": Name or description of the subtitles* "http_headers": A dictionary of additional HTTP headersto add to the request.+ * "impersonate": Impersonate target(s); same as the "formats" field"ext" will be calculated from URL if missingautomatic_captions: Like 'subtitles'; contains automatically generatedcaptions instead of normal subtitles@@ -884,26 +889,17 @@ class InfoExtractor:extensions = {}- if impersonate in (True, ''):- impersonate = ImpersonateTarget()- requested_targets = [- t if isinstance(t, ImpersonateTarget) else ImpersonateTarget.from_str(t)- for t in variadic(impersonate)- ] if impersonate else []-- available_target = next(filter(self._downloader._impersonate_target_available, requested_targets), None)+ available_target, requested_targets = self._downloader._parse_impersonate_targets(impersonate)if available_target:extensions['impersonate'] = available_targetelif requested_targets:- message = 'The extractor is attempting impersonation, but '- message += (- 'no impersonate target is available' if not str(impersonate)- else f'none of these impersonate targets are available: "{", ".join(map(str, requested_targets))}"')- info_msg = ('see https://github.com/yt-dlp/yt-dlp#impersonation '- 'for information on installing the required dependencies')+ msg = 'The extractor is attempting impersonation'if require_impersonation:- raise ExtractorError(f'{message}; {info_msg}', expected=True)- self.report_warning(f'{message}; if you encounter errors, then {info_msg}', only_once=True)+ raise ExtractorError(+ self._downloader._unavailable_targets_message(requested_targets, note=msg, is_error=True),+ expected=True)+ self.report_warning(+ self._downloader._unavailable_targets_message(requested_targets, note=msg), only_once=True)try:return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query, extensions))