Skip to content

A feature-rich command-line audio/video downloader

[ie/go] Fix provider-locked content extraction (#13131)

Max2e5bf002dad1
diff --git a/yt_dlp/extractor/go.py b/yt_dlp/extractor/go.py
index 83c1979db8..4e138a828f 100644
--- a/yt_dlp/extractor/go.py
+++ b/yt_dlp/extractor/go.py
@@ -7,161 +7,157 @@ from ..utils import (
int_or_none,
join_nonempty,
parse_age_limit,
- remove_end,
- remove_start,
- traverse_obj,
- try_get,
unified_timestamp,
urlencode_postdata,
)
+from ..utils.traversal import traverse_obj
class GoIE(AdobePassIE):
_SITE_INFO = {
'abc': {
'brand': '001',
- 'requestor_id': 'ABC',
+ 'requestor_id': 'dtci',
+ 'provider_id': 'ABC',
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI4OTcwMjlkYS0yYjM1LTQyOWUtYWQ0NS02ZjZiZjVkZTdhOTUiLCJuYmYiOjE2MjAxNzM5NjksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNjIwMTczOTY5fQ.SC69DVJWSL8sIe-vVUrP6xS_kzHKqwz9PdKYexs_y-f7Vin6mM-7S-W1TE_-K55O0pyf-TL4xYgvm6LIye8CckG-nZfVwNPV4huduov0jmIcxCQFeUwkHULG2IaA44wfBVUBdaHgkhPweZ2amjycO_IXtez-gBXOLbE3B7Gx9j_5ISCFtyVUblThKfoGyQv6KT6t8Vpmc4ZSKCCQp74KWFFypydb9ucego1taW_nQD06Cdf4yByLd6NaTBceMcIKbug9b9gxFm3XBgJ5q3z7KGo1Kr6XalAV5j4m-fQ91wczlTilX8FM4AljMupyRM9mA_aEADILQ4hS79q4SM0w6w',
},
'freeform': {
'brand': '002',
'requestor_id': 'ABCFamily',
- },
- 'watchdisneychannel': {
- 'brand': '004',
- 'resource_id': 'Disney',
- },
- 'watchdisneyjunior': {
- 'brand': '008',
- 'resource_id': 'DisneyJunior',
- },
- 'watchdisneyxd': {
- 'brand': '009',
- 'resource_id': 'DisneyXD',
+ 'provider_id': 'ABCFamily',
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZWM2MGYyNC0xYzRjLTQ1NzQtYjc0Zi03ZmM4N2E5YWMzMzgiLCJuYmYiOjE1ODc2NjU5MjMsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTg3NjY1OTIzfQ.flCn3dhvmvPnWmV0JV8Fm0YFyj07yPez9-n1GFEwVIm_S2wQVWbWyJhqsAyLZVFrhOMZYTqmPS3OHxGwTwXkEYn6PD7o_vIVG3oqi-Xn1m5jRt_Gazw5qEtpat6VE7bvKGSD3ZhcidOrsCk8NcYyq75u61NHDvSl81pcedJjVRVUpsqrEwmo0aVbA0C8PX3ri0mEbGvkMKvHn8E60xp-PSE-VK8SDT0plwPu_TwUszkZ6-_I8_2xcv_WBqcXFkAVg7Q-iNJXgQvmNsrpcrYuLvi6hEH4ZLtoDcXU6MhwTQAJTiHSo8x9aHX1_qFP09CzlNOFQbC2ZEJdP9SvA53SLQ',
},
'disneynow': {
- 'brand': '011',
+ 'brand': '011', # also: '004', '008', '009'
+ 'requestor_id': 'DisneyChannels',
+ 'provider_id': 'DisneyChannels',
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI1MzAzNTRiOS04NDNiLTRkNjAtYTQ3ZS0yNzk1MzlkOTIyNTciLCJuYmYiOjE1NTg5ODc0NDksImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTU4OTg3NDQ5fQ.Jud6YS6-J2h0h6po0oMheDym0qRTJQGj4kzacrz4DFuEwhcBkkykW6pF5pKuAUJy9HCZ40oDAHe2KcTlDJjCZF5tDaUEfdihakZ9cC_rG7MU-QoRne8qaB_dPDKwGuk-ZyWD8eV3zwTJmbGo8hDxYTEU81YNCxwhyc_BPDr5TYiubbmpP3_pTnXmSpuL58isJ2peSKWlX9BacuXtBY25c_QnPFKk-_EETm7IHkTpDazde1QfHWGu4s4yJpKGk8RVVujVG6h6ELlL-ZeYLilBm7iS7h1TYG1u7fJhyZRL7isaom6NvAzsvN3ngss1fLwt8decP8wzdFHrbYTdTjW8qw',
'resource_id': 'Disney',
},
- 'fxnow.fxnetworks': {
- 'brand': '025',
+ 'fxnetworks': {
+ 'brand': '025', # also: '020'
+ 'requestor_id': 'dtci',
+ 'provider_id': 'fx', # also 'fxx', 'fxm'
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIzYWRhYWZiNC02OTAxLTRlYzktOTdmNy1lYWZkZTJkODJkN2EiLCJuYmYiOjE1NjIwMjQwNzYsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTYyMDI0MDc2fQ.dhKMpZK50AObbZYrMiYPSfWtzXHUaeMP3jrIY4Cgfvh0GaEgk0Mns_zp78jypFeZgRtPVleQMQDNq2YEloRLcAGqP1aa6WVDglnK77ZWUm4IKai14Rwf3A6YBhSRoO2_lMmUGkuTf6gZY-kMIPqBYKqzTQiQl4HbniPFodIzFRiuI9QJVrkoyTGrJL4oqiX08PoFI3Z-TOti1Heu3EbFC-GveQHhlinYrzU7rbiAqLEz7FImtfBDsnXX1Y3uJDLYM3Bq4Oh0nrzTv1Fd62wNsCNErHHIbELidh1zZF0ujvt7ReuZUwAitm0UhEJ7OxNOUbEQWtae6pVNscvdvTFMpg',
+ },
+ 'nationalgeographic': {
+ 'brand': '026', # also '023'
'requestor_id': 'dtci',
+ 'provider_id': 'ngc', # also 'ngw'
+ 'software_statement': 'eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMzE4YTM1Ni05Mjc4LTQ4NjEtYTFmNi1jMTIzMzg1ZWMzYzMiLCJuYmYiOjE1NjIwMjM4MjgsImlzcyI6ImF1dGguYWRvYmUuY29tIiwiaWF0IjoxNTYyMDIzODI4fQ.Le-2OzF9-jrhJ7ZfWtLWk5iSHGVZoxeU1w0_fO--Heli0OwRZsRq2slSmx-oZTzxuWmAgDEiBkWSDcDK6sM25DrCLsdsJa3MBuZ-slBRtH8aq3HpNoqqLkU-vg6gRUEKMtwBUtwCu_9aKUCayYtndWv4b1DjVQeSrteOW5NNudWVYleAe0kxeNJQHo5If9SCzDudKVJktFUjhNks4QPOC_uONPkRRlL9D0fNvtOY-LRFckfcHhf5z9l1iZjeukV0YhdKnuw1wyiaWrQXBUDiBfbkCRd2DM-KnelqPxfiXCaTjGKDURRBO3pz33ebge3IFXSiU5vl4qHQ8xvunzGpFw',
},
}
- _VALID_URL = r'''(?x)
- https?://
- (?P<sub_domain>
- (?:{}\.)?go|fxnow\.fxnetworks|
- (?:www\.)?(?:abc|freeform|disneynow)
- )\.com/
- (?:
- (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)|
- (?:[^/]+/)*(?P<display_id>[^/?\#]+)
- )
- '''.format(r'\.|'.join(list(_SITE_INFO.keys())))
+ _URL_PATH_RE = r'(?:video|episode|movies-and-specials)/(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
+ _VALID_URL = [
+ fr'https?://(?:www\.)?(?P<site>abc)\.com/{_URL_PATH_RE}',
+ fr'https?://(?:www\.)?(?P<site>freeform)\.com/{_URL_PATH_RE}',
+ fr'https?://(?:www\.)?(?P<site>disneynow)\.com/{_URL_PATH_RE}',
+ fr'https?://fxnow\.(?P<site>fxnetworks)\.com/{_URL_PATH_RE}',
+ fr'https?://(?:www\.)?(?P<site>nationalgeographic)\.com/tv/{_URL_PATH_RE}',
+ ]
_TESTS = [{
- 'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
+ 'url': 'https://abc.com/episode/4192c0e6-26e5-47a8-817b-ce8272b9e440/playlist/PL551127435',
'info_dict': {
- 'id': 'VDKA3807643',
+ 'id': 'VDKA10805898',
'ext': 'mp4',
- 'title': 'The Traitor in the White House',
- 'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
- },
- 'params': {
- # m3u8 download
- 'skip_download': True,
+ 'title': 'Switch the Flip',
+ 'description': 'To help get Brian’s life in order, Stewie and Brian swap bodies using a machine that Stewie invents.',
+ 'age_limit': 14,
+ 'duration': 1297,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'Family Guy',
+ 'season': 'Season 16',
+ 'season_number': 16,
+ 'episode': 'Episode 17',
+ 'episode_number': 17,
+ 'timestamp': 1746082800.0,
+ 'upload_date': '20250501',
},
- 'skip': 'This content is no longer available.',
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'This video requires AdobePass MSO credentials',
}, {
- 'url': 'https://disneynow.com/shows/big-hero-6-the-series',
+ 'url': 'https://disneynow.com/episode/21029660-ba06-4406-adb0-a9a78f6e265e/playlist/PL553044961',
'info_dict': {
- 'title': 'Doraemon',
- 'id': 'SH55574025',
+ 'id': 'VDKA39546942',
+ 'ext': 'mp4',
+ 'title': 'Zero Friends Again',
+ 'description': 'Relationships fray under the pressures of a difficult journey.',
+ 'age_limit': 0,
+ 'duration': 1721,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'Star Wars: Skeleton Crew',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'episode': 'Episode 6',
+ 'episode_number': 6,
+ 'timestamp': 1746946800.0,
+ 'upload_date': '20250511',
},
- 'playlist_mincount': 51,
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'This video requires AdobePass MSO credentials',
}, {
- 'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
+ 'url': 'https://fxnow.fxnetworks.com/episode/09f4fa6f-c293-469e-aebe-32c9ca5842a7/playlist/PL554408064',
'info_dict': {
- 'id': 'VDKA3609139',
- 'title': 'This Guilty Blood',
- 'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
- 'age_limit': 14,
+ 'id': 'VDKA38112033',
+ 'ext': 'mp4',
+ 'title': 'The Return of Jerry',
+ 'description': 'The vampires’ long-lost fifth roommate returns. Written by Paul Simms; directed by Kyle Newacheck.',
+ 'age_limit': 17,
+ 'duration': 1493,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'What We Do in the Shadows',
+ 'season': 'Season 6',
+ 'season_number': 6,
'episode': 'Episode 1',
- 'upload_date': '20170102',
- 'season': 'Season 2',
- 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abcf/Shadowhunters/video/201/ae5f75608d86bf88aa4f9f4aa76ab1b7/579x325-Q100_ae5f75608d86bf88aa4f9f4aa76ab1b7.jpg',
- 'duration': 2544,
- 'season_number': 2,
- 'series': 'Shadowhunters',
'episode_number': 1,
- 'timestamp': 1483387200,
- 'ext': 'mp4',
- },
- 'params': {
- 'geo_bypass_ip_block': '3.244.239.0/24',
- # m3u8 download
- 'skip_download': True,
+ 'timestamp': 1729573200.0,
+ 'upload_date': '20241022',
},
+ 'params': {'skip_download': 'm3u8'},
+ 'skip': 'This video requires AdobePass MSO credentials',
}, {
- 'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/12-the-knock',
+ 'url': 'https://www.freeform.com/episode/bda0eaf7-761a-4838-aa44-96f794000844/playlist/PL553044961',
'info_dict': {
- 'id': 'VDKA26050359',
- 'title': 'The Knock',
- 'description': 'md5:0c2947e3ada4c31f28296db7db14aa64',
- 'age_limit': 14,
+ 'id': 'VDKA39007340',
'ext': 'mp4',
- 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abc/TheRookie/video/412/daf830d06e83b11eaf5c0a299d993ae3/1556x876-Q75_daf830d06e83b11eaf5c0a299d993ae3.jpg',
- 'episode': 'Episode 12',
- 'season_number': 4,
- 'season': 'Season 4',
- 'timestamp': 1642975200,
- 'episode_number': 12,
- 'upload_date': '20220123',
- 'series': 'The Rookie',
- 'duration': 2572,
- },
- 'params': {
- 'geo_bypass_ip_block': '3.244.239.0/24',
- # m3u8 download
- 'skip_download': True,
+ 'title': 'Angel\'s Landing',
+ 'description': 'md5:91bf084e785c968fab16734df7313446',
+ 'age_limit': 14,
+ 'duration': 2523,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'How I Escaped My Cult',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'episode': 'Episode 2',
+ 'episode_number': 2,
+ 'timestamp': 1740038400.0,
+ 'upload_date': '20250220',
},
+ 'params': {'skip_download': 'm3u8'},
}, {
- 'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
+ 'url': 'https://www.nationalgeographic.com/tv/episode/ca694661-1186-41ae-8089-82f64d69b16d/playlist/PL554408064',
'info_dict': {
- 'id': 'VDKA12782841',
- 'title': 'First Look: Better Things - Season 2',
- 'description': 'md5:fa73584a95761c605d9d54904e35b407',
+ 'id': 'VDKA39492078',
'ext': 'mp4',
- 'age_limit': 14,
- 'upload_date': '20170825',
- 'duration': 161,
- 'series': 'Better Things',
- 'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/fx/BetterThings/video/12782841/b6b05e58264121cc2c98811318e6d507/1556x876-Q75_b6b05e58264121cc2c98811318e6d507.jpg',
- 'timestamp': 1503661074,
- },
- 'params': {
- 'geo_bypass_ip_block': '3.244.239.0/24',
- # m3u8 download
- 'skip_download': True,
+ 'title': 'Heart of the Emperors',
+ 'description': 'md5:4fc50a2878f030bb3a7eac9124dca677',
+ 'age_limit': 0,
+ 'duration': 2775,
+ 'thumbnail': r're:https?://.+/.+\.jpg',
+ 'series': 'Secrets of the Penguins',
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'episode': 'Episode 1',
+ 'episode_number': 1,
+ 'timestamp': 1745204400.0,
+ 'upload_date': '20250421',
},
+ 'params': {'skip_download': 'm3u8'},
}, {
- 'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
- 'only_matching': True,
- }, {
- 'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
+ 'url': 'https://www.freeform.com/movies-and-specials/c38281fc-9f8f-47c7-8220-22394f9df2e1',
'only_matching': True,
}, {
- # brand 004
- 'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
- 'only_matching': True,
- }, {
- # brand 008
- 'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
- 'only_matching': True,
- }, {
- 'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
- 'only_matching': True,
- }, {
- 'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner',
+ 'url': 'https://abc.com/video/219a454a-172c-41bf-878a-d169e6bc0bdc/playlist/PL5523098420',
'only_matching': True,
}]
@@ -171,58 +167,29 @@ class GoIE(AdobePassIE):
f'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/{brand}/001/-1/{show_id}/-1/{video_id}/-1/-1.json',
display_id)['video']
+ def _extract_global_var(self, name, webpage, video_id):
+ return self._search_json(
+ fr'window\[["\']{re.escape(name)}["\']\]\s*=',
+ webpage, f'{name.strip("_")} JSON', video_id)
+
def _real_extract(self, url):
- mobj = self._match_valid_url(url)
- sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.')
- video_id, display_id = mobj.group('id', 'display_id')
- site_info = self._SITE_INFO.get(sub_domain, {})
- brand = site_info.get('brand')
- if not video_id or not site_info:
- webpage = self._download_webpage(url, display_id or video_id)
- data = self._parse_json(
- self._search_regex(
- r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage,
- 'data', default='{}'),
- display_id or video_id, fatal=False)
- # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot
- layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict)
- video_id = None
- if layout:
- video_id = try_get(
- layout,
- (lambda x: x['videoid'], lambda x: x['video']['id']),
- str)
- if not video_id:
- video_id = self._search_regex(
- (
- # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
- # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
- r'data-video-id=["\']*(VDKA\w+)',
- # page.analytics.videoIdCode
- r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)',
- # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
- r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)',
- ), webpage, 'video id', default=video_id)
- if not site_info:
- brand = self._search_regex(
- (r'data-brand=\s*["\']\s*(\d+)',
- r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
- default='004')
- site_info = next(
- si for _, si in self._SITE_INFO.items()
- if si.get('brand') == brand)
- if not video_id:
- # show extraction works for Disney, DisneyJunior and DisneyXD
- # ABC and Freeform has different layout
- show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
- videos = self._extract_videos(brand, show_id=show_id)
- show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
- entries = []
- for video in videos:
- entries.append(self.url_result(
- video['url'], 'Go', video.get('id'), video.get('title')))
- entries.reverse()
- return self.playlist_result(entries, show_id, show_title)
+ site, display_id = self._match_valid_url(url).group('site', 'id')
+ webpage = self._download_webpage(url, display_id)
+ config = self._extract_global_var('__CONFIG__', webpage, display_id)
+ data = self._extract_global_var(config['globalVar'], webpage, display_id)
+ video_id = traverse_obj(data, (
+ 'page', 'content', 'video', 'layout', (('video', 'id'), 'videoid'), {str}, any))
+ if not video_id:
+ video_id = self._search_regex([
+ # data-track-video_id="VDKA39492078"
+ # data-track-video_id_code="vdka39492078"
+ # data-video-id="'VDKA3609139'"
+ r'data-(?:track-)?video[_-]id(?:_code)?=["\']*((?:vdka|VDKA)\d+)',
+ # page.analytics.videoIdCode
+ r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\d+)'], webpage, 'video ID')
+
+ site_info = self._SITE_INFO[site]
+ brand = site_info['brand']
video_data = self._extract_videos(brand, video_id)[0]
video_id = video_data['id']
title = video_data['title']
@@ -238,26 +205,31 @@ class GoIE(AdobePassIE):
if ext == 'm3u8':
video_type = video_data.get('type')
data = {
- 'video_id': video_data['id'],
+ 'video_id': video_id,
'video_type': video_type,
'brand': brand,
'device': '001',
+ 'app_name': 'webplayer-abc',
}
if video_data.get('accesslevel') == '1':
- requestor_id = site_info.get('requestor_id', 'DisneyChannels')
+ provider_id = site_info['provider_id']
+ software_statement = traverse_obj(data, ('app', 'config', (
+ ('features', 'auth', 'softwareStatement'),
+ ('tvAuth', 'SOFTWARE_STATEMENTS', 'PRODUCTION'),
+ ), {str}, any)) or site_info['software_statement']
resource = site_info.get('resource_id') or self._get_mvpd_resource(
- requestor_id, title, video_id, None)
+ provider_id, title, video_id, None)
auth = self._extract_mvpd_auth(
- url, video_id, requestor_id, resource)
+ url, video_id, site_info['requestor_id'], resource, software_statement)
data.update({
'token': auth,
'token_type': 'ap',
- 'adobe_requestor_id': requestor_id,
+ 'adobe_requestor_id': provider_id,
})
else:
self._initialize_geo_bypass({'countries': ['US']})
entitlement = self._download_json(
- 'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
+ 'https://prod.gatekeeper.us-abc.symphony.edgedatg.go.com/vp2/ws-secure/entitlement/2020/playmanifest_secure.json',
video_id, data=urlencode_postdata(data))
errors = entitlement.get('errors', {}).get('errors', [])
if errors:
@@ -267,7 +239,7 @@ class GoIE(AdobePassIE):
error['message'], countries=['US'])
error_message = ', '.join([error['message'] for error in errors])
raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True)
- asset_url += '?' + entitlement['uplynkData']['sessionKey']
+ asset_url += '?' + entitlement['entitlement']['uplynkData']['sessionKey']
fmts, subs = self._extract_m3u8_formats_and_subtitles(
asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False)
formats.extend(fmts)