A feature-rich command-line audio/video downloader
[ie/rtve] Rework extractors (#10388)
Florentin Le Moalf07ee91c7192
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.pyindex 9ae1bb36bd..047af92820 100644--- a/yt_dlp/extractor/_extractors.py+++ b/yt_dlp/extractor/_extractors.py@@ -1783,7 +1783,6 @@ from .rtvcplay import (from .rtve import (RTVEALaCartaIE,RTVEAudioIE,- RTVEInfantilIE,RTVELiveIE,RTVETelevisionIE,)diff --git a/yt_dlp/extractor/rtve.py b/yt_dlp/extractor/rtve.pyindex 7e0b666ab3..2812d93059 100644--- a/yt_dlp/extractor/rtve.py+++ b/yt_dlp/extractor/rtve.py@@ -1,35 +1,142 @@import base64import ioimport struct+import urllib.parsefrom .common import InfoExtractorfrom ..utils import (ExtractorError,+ clean_html,determine_ext,float_or_none,+ make_archive_id,+ parse_iso8601,qualities,- remove_end,- remove_start,- try_get,+ url_or_none,)+from ..utils.traversal import subs_list_to_dict, traverse_obj-class RTVEALaCartaIE(InfoExtractor):+class RTVEBaseIE(InfoExtractor):+ # Reimplementation of https://js2.rtve.es/pages/app-player/3.5.1/js/pf_video.js+ @staticmethod+ def _decrypt_url(png):+ encrypted_data = io.BytesIO(base64.b64decode(png)[8:])+ while True:+ length_data = encrypted_data.read(4)+ length = struct.unpack('!I', length_data)[0]+ chunk_type = encrypted_data.read(4)+ if chunk_type == b'IEND':+ break+ data = encrypted_data.read(length)+ if chunk_type == b'tEXt':+ data = bytes(filter(None, data))+ alphabet_data, _, url_data = data.partition(b'#')+ quality_str, _, url_data = url_data.rpartition(b'%%')+ quality_str = quality_str.decode() or ''+ alphabet = RTVEBaseIE._get_alphabet(alphabet_data)+ url = RTVEBaseIE._get_url(alphabet, url_data)+ yield quality_str, url+ encrypted_data.read(4) # CRC++ @staticmethod+ def _get_url(alphabet, url_data):+ url = ''+ f = 0+ e = 3+ b = 1+ for char in url_data.decode('iso-8859-1'):+ if f == 0:+ l = int(char) * 10+ f = 1+ else:+ if e == 0:+ l += int(char)+ url += alphabet[l]+ e = (b + 3) % 4+ f = 0+ b += 1+ else:+ e -= 1+ return url++ @staticmethod+ def _get_alphabet(alphabet_data):+ alphabet = []+ e = 0+ d = 0+ for char in alphabet_data.decode('iso-8859-1'):+ if d == 0:+ alphabet.append(char)+ d = e = (e + 1) % 4+ else:+ d -= 1+ return alphabet++ def _extract_png_formats_and_subtitles(self, video_id, media_type='videos'):+ formats, subtitles = [], {}+ q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])+ for manager in ('rtveplayw', 'default'):+ png = self._download_webpage(+ f'http://www.rtve.es/ztnr/movil/thumbnail/{manager}/{media_type}/{video_id}.png',+ video_id, 'Downloading url information', query={'q': 'v2'}, fatal=False)+ if not png:+ continue++ for quality, video_url in self._decrypt_url(png):+ ext = determine_ext(video_url)+ if ext == 'm3u8':+ fmts, subs = self._extract_m3u8_formats_and_subtitles(+ video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)+ formats.extend(fmts)+ self._merge_subtitles(subs, target=subtitles)+ elif ext == 'mpd':+ fmts, subs = self._extract_mpd_formats_and_subtitles(+ video_url, video_id, 'dash', fatal=False)+ formats.extend(fmts)+ self._merge_subtitles(subs, target=subtitles)+ else:+ formats.append({+ 'format_id': quality,+ 'quality': q(quality),+ 'url': video_url,+ })+ return formats, subtitles++ def _parse_metadata(self, metadata):+ return traverse_obj(metadata, {+ 'title': ('title', {str.strip}),+ 'alt_title': ('alt', {str.strip}),+ 'description': ('description', {clean_html}),+ 'timestamp': ('dateOfEmission', {parse_iso8601(delimiter=' ')}),+ 'release_timestamp': ('publicationDate', {parse_iso8601(delimiter=' ')}),+ 'modified_timestamp': ('modificationDate', {parse_iso8601(delimiter=' ')}),+ 'thumbnail': (('thumbnail', 'image', 'imageSEO'), {url_or_none}, any),+ 'duration': ('duration', {float_or_none(scale=1000)}),+ 'is_live': ('live', {bool}),+ 'series': (('programTitle', ('programInfo', 'title')), {clean_html}, any),+ })+++class RTVEALaCartaIE(RTVEBaseIE):IE_NAME = 'rtve.es:alacarta'- IE_DESC = 'RTVE a la carta'- _VALID_URL = r'https?://(?:www\.)?rtve\.es/(m/)?(alacarta/videos|filmoteca)/[^/]+/[^/]+/(?P<id>\d+)'+ IE_DESC = 'RTVE a la carta and Play'+ _VALID_URL = [+ r'https?://(?:www\.)?rtve\.es/(?:m/)?(?:(?:alacarta|play)/videos|filmoteca)/(?!directo)(?:[^/?#]+/){2}(?P<id>\d+)',+ r'https?://(?:www\.)?rtve\.es/infantil/serie/[^/?#]+/video/[^/?#]+/(?P<id>\d+)',+ ]_TESTS = [{- 'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',- 'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',+ 'url': 'http://www.rtve.es/alacarta/videos/la-aventura-del-saber/aventuraentornosilla/3088905/',+ 'md5': 'a964547824359a5753aef09d79fe984b','info_dict': {- 'id': '2491869',+ 'id': '3088905','ext': 'mp4',- 'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',- 'duration': 5024.566,- 'series': 'Balonmano',+ 'title': 'En torno a la silla',+ 'duration': 1216.981,+ 'series': 'La aventura del Saber',+ 'thumbnail': 'https://img2.rtve.es/v/aventuraentornosilla_3088905.png',},- 'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],}, {'note': 'Live stream','url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',@@ -38,140 +145,88 @@ class RTVEALaCartaIE(InfoExtractor):'ext': 'mp4','title': 're:^24H LIVE [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$','is_live': True,+ 'live_status': 'is_live',+ 'thumbnail': r're:https://img2\.rtve\.es/v/.*\.png',},'params': {'skip_download': 'live stream',},}, {'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',- 'md5': 'd850f3c8731ea53952ebab489cf81cbf',+ 'md5': 'f3cf0d1902d008c48c793e736706c174','info_dict': {'id': '4236788','ext': 'mp4',- 'title': 'Servir y proteger - Capítulo 104',- 'duration': 3222.0,+ 'title': 'Episodio 104',+ 'duration': 3222.8,+ 'thumbnail': r're:https://img2\.rtve\.es/v/.*\.png',+ 'series': 'Servir y proteger',},- 'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],}, {'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve','only_matching': True,}, {'url': 'http://www.rtve.es/filmoteca/no-do/not-1-introduccion-primer-noticiario-espanol/1465256/','only_matching': True,+ }, {+ 'url': 'https://www.rtve.es/play/videos/saber-vivir/07-07-24/16177116/',+ 'md5': 'a5b24fcdfa3ff5cb7908aba53d22d4b6',+ 'info_dict': {+ 'id': '16177116',+ 'ext': 'mp4',+ 'title': 'Saber vivir - 07/07/24',+ 'thumbnail': r're:https://img2\.rtve\.es/v/.*\.png',+ 'duration': 2162.68,+ 'series': 'Saber vivir',+ },+ }, {+ 'url': 'https://www.rtve.es/infantil/serie/agus-lui-churros-crafts/video/gusano/7048976/',+ 'info_dict': {+ 'id': '7048976',+ 'ext': 'mp4',+ 'title': 'Gusano',+ 'thumbnail': r're:https://img2\.rtve\.es/v/.*\.png',+ 'duration': 292.86,+ 'series': 'Agus & Lui: Churros y Crafts',+ '_old_archive_ids': ['rtveinfantil 7048976'],+ },}]- def _real_initialize(self):- user_agent_b64 = base64.b64encode(self.get_param('http_headers')['User-Agent'].encode()).decode('utf-8')- self._manager = self._download_json(- 'http://www.rtve.es/odin/loki/' + user_agent_b64,- None, 'Fetching manager info')['manager']-- @staticmethod- def _decrypt_url(png):- encrypted_data = io.BytesIO(base64.b64decode(png)[8:])- while True:- length = struct.unpack('!I', encrypted_data.read(4))[0]- chunk_type = encrypted_data.read(4)- if chunk_type == b'IEND':- break- data = encrypted_data.read(length)- if chunk_type == b'tEXt':- alphabet_data, text = data.split(b'\0')- quality, url_data = text.split(b'%%')- alphabet = []- e = 0- d = 0- for l in alphabet_data.decode('iso-8859-1'):- if d == 0:- alphabet.append(l)- d = e = (e + 1) % 4- else:- d -= 1- url = ''- f = 0- e = 3- b = 1- for letter in url_data.decode('iso-8859-1'):- if f == 0:- l = int(letter) * 10- f = 1- else:- if e == 0:- l += int(letter)- url += alphabet[l]- e = (b + 3) % 4- f = 0- b += 1- else:- e -= 1-- yield quality.decode(), url- encrypted_data.read(4) # CRC-- def _extract_png_formats(self, video_id):- png = self._download_webpage(- f'http://www.rtve.es/ztnr/movil/thumbnail/{self._manager}/videos/{video_id}.png',- video_id, 'Downloading url information', query={'q': 'v2'})- q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])- formats = []- for quality, video_url in self._decrypt_url(png):- ext = determine_ext(video_url)- if ext == 'm3u8':- formats.extend(self._extract_m3u8_formats(- video_url, video_id, 'mp4', 'm3u8_native',- m3u8_id='hls', fatal=False))- elif ext == 'mpd':- formats.extend(self._extract_mpd_formats(- video_url, video_id, 'dash', fatal=False))- else:- formats.append({- 'format_id': quality,- 'quality': q(quality),- 'url': video_url,- })- return formats+ def _get_subtitles(self, video_id):+ subtitle_data = self._download_json(+ f'https://api2.rtve.es/api/videos/{video_id}/subtitulos.json', video_id,+ 'Downloading subtitles info')+ return traverse_obj(subtitle_data, ('page', 'items', ..., {+ 'id': ('lang', {str}),+ 'url': ('src', {url_or_none}),+ }, all, {subs_list_to_dict(lang='es')}))def _real_extract(self, url):video_id = self._match_id(url)- info = self._download_json(+ metadata = self._download_json(f'http://www.rtve.es/api/videos/{video_id}/config/alacarta_videos.json',video_id)['page']['items'][0]- if info['state'] == 'DESPU':+ if metadata['state'] == 'DESPU':raise ExtractorError('The video is no longer available', expected=True)- title = info['title'].strip()- formats = self._extract_png_formats(video_id)+ formats, subtitles = self._extract_png_formats_and_subtitles(video_id)- subtitles = None- sbt_file = info.get('sbtFile')- if sbt_file:- subtitles = self.extract_subtitles(video_id, sbt_file)+ self._merge_subtitles(self.extract_subtitles(video_id), target=subtitles)- is_live = info.get('live') is True+ is_infantil = urllib.parse.urlparse(url).path.startswith('/infantil/')return {'id': video_id,- 'title': title,'formats': formats,- 'thumbnail': info.get('image'),'subtitles': subtitles,- 'duration': float_or_none(info.get('duration'), 1000),- 'is_live': is_live,- 'series': info.get('programTitle'),+ **self._parse_metadata(metadata),+ '_old_archive_ids': [make_archive_id('rtveinfantil', video_id)] if is_infantil else None,}- def _get_subtitles(self, video_id, sub_file):- subs = self._download_json(- sub_file + '.json', video_id,- 'Downloading subtitles info')['page']['items']- return dict(- (s['lang'], [{'ext': 'vtt', 'url': s['src']}])- for s in subs)--class RTVEAudioIE(RTVEALaCartaIE): # XXX: Do not subclass from concrete IE+class RTVEAudioIE(RTVEBaseIE):IE_NAME = 'rtve.es:audio'IE_DESC = 'RTVE audio'- _VALID_URL = r'https?://(?:www\.)?rtve\.es/(alacarta|play)/audios/[^/]+/[^/]+/(?P<id>[0-9]+)'+ _VALID_URL = r'https?://(?:www\.)?rtve\.es/(alacarta|play)/audios/(?:[^/?#]+/){2}(?P<id>\d+)'_TESTS = [{'url': 'https://www.rtve.es/alacarta/audios/a-hombros-de-gigantes/palabra-ingeniero-codigos-informaticos-27-04-21/5889192/',@@ -180,9 +235,11 @@ class RTVEAudioIE(RTVEALaCartaIE): # XXX: Do not subclass from concrete IE'id': '5889192','ext': 'mp3','title': 'Códigos informáticos',- 'thumbnail': r're:https?://.+/1598856591583.jpg',+ 'alt_title': 'Códigos informáticos - Escuchar ahora','duration': 349.440,'series': 'A hombros de gigantes',+ 'description': 'md5:72b0d7c1ca20fd327bdfff7ac0171afb',+ 'thumbnail': 'https://img2.rtve.es/a/palabra-ingeniero-codigos-informaticos-270421_5889192.png',},}, {'url': 'https://www.rtve.es/play/audios/en-radio-3/ignatius-farray/5791165/',@@ -191,9 +248,11 @@ class RTVEAudioIE(RTVEALaCartaIE): # XXX: Do not subclass from concrete IE'id': '5791165','ext': 'mp3','title': 'Ignatius Farray',+ 'alt_title': 'En Radio 3 - Ignatius Farray - 13/02/21 - escuchar ahora','thumbnail': r're:https?://.+/1613243011863.jpg','duration': 3559.559,'series': 'En Radio 3',+ 'description': 'md5:124aa60b461e0b1724a380bad3bc4040',},}, {'url': 'https://www.rtve.es/play/audios/frankenstein-o-el-moderno-prometeo/capitulo-26-ultimo-muerte-victor-juan-jose-plans-mary-shelley/6082623/',@@ -202,126 +261,101 @@ class RTVEAudioIE(RTVEALaCartaIE): # XXX: Do not subclass from concrete IE'id': '6082623','ext': 'mp3','title': 'Capítulo 26 y último: La muerte de Victor',+ 'alt_title': 'Frankenstein o el moderno Prometeo - Capítulo 26 y último: La muerte de Victor','thumbnail': r're:https?://.+/1632147445707.jpg','duration': 3174.086,'series': 'Frankenstein o el moderno Prometeo',+ 'description': 'md5:4ee6fcb82ebe2e46d267e1d1c1a8f7b5',},}]- def _extract_png_formats(self, audio_id):- """- This function retrieves media related png thumbnail which obfuscate- valuable information about the media. This information is decrypted- via base class _decrypt_url function providing media quality and- media url- """- png = self._download_webpage(- f'http://www.rtve.es/ztnr/movil/thumbnail/{self._manager}/audios/{audio_id}.png',- audio_id, 'Downloading url information', query={'q': 'v2'})- q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])- formats = []- for quality, audio_url in self._decrypt_url(png):- ext = determine_ext(audio_url)- if ext == 'm3u8':- formats.extend(self._extract_m3u8_formats(- audio_url, audio_id, 'mp4', 'm3u8_native',- m3u8_id='hls', fatal=False))- elif ext == 'mpd':- formats.extend(self._extract_mpd_formats(- audio_url, audio_id, 'dash', fatal=False))- else:- formats.append({- 'format_id': quality,- 'quality': q(quality),- 'url': audio_url,- })- return formats-def _real_extract(self, url):audio_id = self._match_id(url)- info = self._download_json(- f'https://www.rtve.es/api/audios/{audio_id}.json',- audio_id)['page']['items'][0]+ metadata = self._download_json(+ f'https://www.rtve.es/api/audios/{audio_id}.json', audio_id)['page']['items'][0]++ formats, subtitles = self._extract_png_formats_and_subtitles(audio_id, media_type='audios')return {'id': audio_id,- 'title': info['title'].strip(),- 'thumbnail': info.get('thumbnail'),- 'duration': float_or_none(info.get('duration'), 1000),- 'series': try_get(info, lambda x: x['programInfo']['title']),- 'formats': self._extract_png_formats(audio_id),+ 'formats': formats,+ 'subtitles': subtitles,+ **self._parse_metadata(metadata),}-class RTVEInfantilIE(RTVEALaCartaIE): # XXX: Do not subclass from concrete IE- IE_NAME = 'rtve.es:infantil'- IE_DESC = 'RTVE infantil'- _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/[^/]+/video/[^/]+/(?P<id>[0-9]+)/'-- _TESTS = [{- 'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',- 'md5': '5747454717aedf9f9fdf212d1bcfc48d',- 'info_dict': {- 'id': '3040283',- 'ext': 'mp4',- 'title': 'Maneras de vivir',- 'thumbnail': r're:https?://.+/1426182947956\.JPG',- 'duration': 357.958,- },- 'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],- }]---class RTVELiveIE(RTVEALaCartaIE): # XXX: Do not subclass from concrete IE+class RTVELiveIE(RTVEBaseIE):IE_NAME = 'rtve.es:live'IE_DESC = 'RTVE.es live streams'- _VALID_URL = r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'+ _VALID_URL = [+ r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)',+ r'https?://(?:www\.)?rtve\.es/play/videos/directo/[^/?#]+/(?P<id>[a-zA-Z0-9-]+)',+ ]_TESTS = [{'url': 'http://www.rtve.es/directo/la-1/','info_dict': {'id': 'la-1','ext': 'mp4',- 'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',+ 'live_status': 'is_live',+ 'title': str,+ 'description': str,+ 'thumbnail': r're:https://img\d\.rtve\.es/resources/thumbslive/\d+\.jpg',+ 'timestamp': int,+ 'upload_date': str,},- 'params': {- 'skip_download': 'live stream',+ 'params': {'skip_download': 'live stream'},+ }, {+ 'url': 'https://www.rtve.es/play/videos/directo/deportes/tdp/',+ 'info_dict': {+ 'id': 'tdp',+ 'ext': 'mp4',+ 'live_status': 'is_live',+ 'title': str,+ 'description': str,+ 'thumbnail': r're:https://img2\d\.rtve\.es/resources/thumbslive/\d+\.jpg',+ 'timestamp': int,+ 'upload_date': str,},+ 'params': {'skip_download': 'live stream'},+ }, {+ 'url': 'http://www.rtve.es/play/videos/directo/canales-lineales/la-1/',+ 'only_matching': True,}]def _real_extract(self, url):- mobj = self._match_valid_url(url)- video_id = mobj.group('id')-+ video_id = self._match_id(url)webpage = self._download_webpage(url, video_id)- title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')- title = remove_start(title, 'Estoy viendo ')- vidplayer_id = self._search_regex(- (r'playerId=player([0-9]+)',- r'class=["\'].*?\blive_mod\b.*?["\'][^>]+data-assetid=["\'](\d+)',- r'data-id=["\'](\d+)'),- webpage, 'internal video ID')+ data_setup = self._search_json(+ r'<div[^>]+class="[^"]*videoPlayer[^"]*"[^>]*data-setup=\'',+ webpage, 'data_setup', video_id)++ formats, subtitles = self._extract_png_formats_and_subtitles(data_setup['idAsset'])return {'id': video_id,- 'title': title,- 'formats': self._extract_png_formats(vidplayer_id),+ **self._search_json_ld(webpage, video_id, fatal=False),+ 'title': self._html_extract_title(webpage),+ 'formats': formats,+ 'subtitles': subtitles,'is_live': True,}class RTVETelevisionIE(InfoExtractor):IE_NAME = 'rtve.es:television'- _VALID_URL = r'https?://(?:www\.)?rtve\.es/television/[^/]+/[^/]+/(?P<id>\d+).shtml'+ _VALID_URL = r'https?://(?:www\.)?rtve\.es/television/[^/?#]+/[^/?#]+/(?P<id>\d+).shtml'_TEST = {- 'url': 'http://www.rtve.es/television/20160628/revolucion-del-movil/1364141.shtml',+ 'url': 'https://www.rtve.es/television/20091103/video-inedito-del-8o-programa/299020.shtml','info_dict': {- 'id': '3069778',+ 'id': '572515','ext': 'mp4',- 'title': 'Documentos TV - La revolución del móvil',- 'duration': 3496.948,+ 'title': 'Clase inédita',+ 'duration': 335.817,+ 'thumbnail': r're:https://img2\.rtve\.es/v/.*\.png',+ 'series': 'El coro de la cárcel',},'params': {'skip_download': True,@@ -332,11 +366,8 @@ class RTVETelevisionIE(InfoExtractor):page_id = self._match_id(url)webpage = self._download_webpage(url, page_id)- alacarta_url = self._search_regex(- r'data-location="alacarta_videos"[^<]+url":"(http://www\.rtve\.es/alacarta.+?)&',- webpage, 'alacarta url', default=None)- if alacarta_url is None:- raise ExtractorError(- 'The webpage doesn\'t contain any video', expected=True)+ play_url = self._html_search_meta('contentUrl', webpage)+ if play_url is None:+ raise ExtractorError('The webpage doesn\'t contain any video', expected=True)- return self.url_result(alacarta_url, ie=RTVEALaCartaIE.ie_key())+ return self.url_result(play_url, ie=RTVEALaCartaIE.ie_key())