Skip to content

A feature-rich command-line audio/video downloader

[ie/omnyfm] Add extractors (#15942)

doe1080fa383a9efa02
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index ac6c10e71d..94c4e5fabc 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -1314,6 +1314,11 @@ from .oftv import (
)
from .oktoberfesttv import OktoberfestTVIE
from .olympics import OlympicsReplayIE
+from .omnyfm import (
+ OmnyfmIE,
+ OmnyfmPlaylistIE,
+ OmnyfmShowIE,
+)
from .on24 import On24IE
from .ondemandkorea import (
OnDemandKoreaIE,
diff --git a/yt_dlp/extractor/omnyfm.py b/yt_dlp/extractor/omnyfm.py
new file mode 100644
index 0000000000..caf4b7f798
--- /dev/null
+++ b/yt_dlp/extractor/omnyfm.py
@@ -0,0 +1,272 @@
+import functools
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+ OnDemandPagedList,
+ clean_html,
+ filter_dict,
+ float_or_none,
+ int_or_none,
+ parse_duration,
+ parse_iso8601,
+ parse_qs,
+ update_url,
+ url_or_none,
+)
+from ..utils.traversal import (
+ require,
+ traverse_obj,
+ trim_str,
+)
+
+
+class OmnyfmIE(InfoExtractor):
+ _VALID_URL = r'https?://omny\.fm/shows/(?P<uploader_id>[\w-]+)/(?P<id>(?!playlists(?:[/?#"\']|$))[\w-]+)(?:/embed)?(?=[?#"\']|$)'
+ _EMBED_REGEX = [rf'<iframe[^>]+\bsrc\s*=\s*(["\'])(?P<url>{_VALID_URL}[^"\']*)\1']
+ _TESTS = [{
+ 'url': 'https://omny.fm/shows/sleep-hub/cannabinoids-and-sleep',
+ 'md5': 'e45ec0ce43da757a0be6ca117ec01bdc',
+ 'info_dict': {
+ 'id': 'cannabinoids-and-sleep',
+ 'ext': 'mp3',
+ 'title': 'Cannabinoids and Sleep',
+ 'categories': 'count:1',
+ 'chapters': [
+ {'start_time': 0, 'title': 'Introduction'},
+ {'start_time': 138, 'title': 'Theme: Cannabinoids and Sleep'},
+ {'start_time': 1487, 'title': 'Clinical Tip'},
+ {'start_time': 1635, 'title': 'Pick of the Month'},
+ {'start_time': 1795, 'title': 'What\'s Coming Up?'},
+ ],
+ 'description': 'md5:c0fd2d29f3148382d344cfbd012fb00d',
+ 'duration': 1840.274,
+ 'episode': 'Episode 48',
+ 'episode_number': 48,
+ 'modified_date': r're:\d{8}',
+ 'modified_timestamp': int,
+ 'tags': 'count:6',
+ 'thumbnail': r're:https?://www\.omnycontent\.com/.+',
+ 'timestamp': 1574013600,
+ 'upload_date': '20191117',
+ 'uploader': 'Sleep Talk',
+ 'uploader_id': 'sleep-hub',
+ },
+ }, {
+ 'url': 'https://omny.fm/shows/the-origin-of-things/a-song-of-hope/embed',
+ 'md5': 'd7600ef33e3f139ff1bb8946f3651b15',
+ 'info_dict': {
+ 'id': 'a-song-of-hope',
+ 'ext': 'mp3',
+ 'title': 'A song of hope',
+ 'categories': 'count:3',
+ 'description': 'md5:f8e710765c341a48cfda8dacd428f56d',
+ 'duration': 478.955,
+ 'episode': 'Episode 17',
+ 'episode_number': 17,
+ 'modified_date': r're:\d{8}',
+ 'modified_timestamp': int,
+ 'season': 'Season 3',
+ 'season_number': 3,
+ 'tags': 'count:27',
+ 'thumbnail': r're:https?://www\.omnycontent\.com/.+',
+ 'timestamp': 1679445000,
+ 'upload_date': '20230322',
+ 'uploader': 'The Origin Of Things',
+ 'uploader_id': 'the-origin-of-things',
+ },
+ }]
+ _WEBPAGE_TESTS = [{
+ 'url': 'https://www.asahi.com/special/podcasts/item/?itemid=311a5f48-ad71-4548-b1f2-af5e00747fbc',
+ 'md5': '4c788bf03323734524a7c0f98d9956ed',
+ 'info_dict': {
+ 'id': 'sdgs-271',
+ 'ext': 'mp3',
+ 'title': '「どこかのだれかの人生のにおいがする」 SDGsを音声番組で身近に #271',
+ 'categories': 'count:5',
+ 'description': 'md5:fa086ecce764d81c51648a82d0fe4850',
+ 'duration': 1870.524,
+ 'episode': 'Episode 271',
+ 'episode_number': 271,
+ 'modified_date': r're:\d{8}',
+ 'modified_timestamp': int,
+ 'season': 'Season 1',
+ 'season_number': 1,
+ 'tags': 'count:4',
+ 'thumbnail': r're:https?://www\.omnycontent\.com/.+',
+ 'timestamp': 1670266800,
+ 'upload_date': '20221205',
+ 'uploader': '朝日新聞ポッドキャスト',
+ 'uploader_id': 'asahi',
+ 'webpage_url': 'https://omny.fm/shows/asahi/sdgs-271',
+ },
+ }]
+
+ def _real_extract(self, url):
+ uploader_id, audio_id = self._match_valid_url(url).group('uploader_id', 'id')
+ webpage = self._download_webpage(url, audio_id)
+ nextjs_data = self._search_nextjs_data(webpage, audio_id)
+ clip = traverse_obj(nextjs_data, ('props', 'pageProps', 'clip', {dict}))
+
+ return {
+ 'id': audio_id,
+ 'section_start': traverse_obj(url, ({parse_qs}, 't', -1, {parse_duration})),
+ 'uploader_id': uploader_id,
+ 'vcodec': 'none',
+ **traverse_obj(clip, {
+ 'title': ('Title', {clean_html}, filter),
+ 'chapters': ('Chapters', lambda _, v: parse_duration(v['Position']) is not None, {
+ 'title': ('Name', {clean_html}, filter),
+ 'start_time': ('Position', {parse_duration}),
+ }),
+ 'description': ('Description', {clean_html}, filter),
+ 'duration': ('DurationSeconds', {float_or_none}),
+ 'episode_number': ('Episode', {int_or_none}),
+ 'filesize': ('PublishedAudioSizeInBytes', {int_or_none}),
+ 'modified_timestamp': ('ModifiedAtUtc', {parse_iso8601}),
+ 'season_number': ('Season', {int_or_none}),
+ 'tags': ('Tags', ..., {clean_html}, filter, all, filter),
+ 'thumbnail': ('ImageUrl', {update_url(query=None)}),
+ 'timestamp': ('PublishedUtc', {parse_iso8601}),
+ 'url': ('AudioUrl', {url_or_none}, {require('audio URL')}),
+ 'webpage_url': ('PublishedUrl', {url_or_none}),
+ }),
+ **traverse_obj(clip, ('Program', {
+ 'categories': ('Categories', ..., {clean_html}, filter, all, filter),
+ 'uploader': ('Name', {clean_html}, filter),
+ })),
+ }
+
+
+class OmnyfmPlaylistBaseIE(InfoExtractor):
+ _API_BASE = 'https://api.omny.fm'
+ _BASE_URL = 'https://omny.fm/shows'
+ _PAGE_SIZE = 100
+
+ def _yield_clips(self, clips, uploader_id):
+ for audio_id in traverse_obj(clips, (
+ 'Clips', ..., 'Slug', {str},
+ )):
+ yield self.url_result(
+ f'{self._BASE_URL}/{uploader_id}/{audio_id}', OmnyfmIE)
+
+
+class OmnyfmPlaylistIE(OmnyfmPlaylistBaseIE):
+ _VALID_URL = r'https?://omny\.fm/shows/(?P<uploader_id>[\w-]+)/playlists(?:/(?P<id>[\w-]+))?(?:/embed)?/?(?=[?#"\']|$)'
+ _EMBED_REGEX = [fr'<iframe[^>]+\bsrc=(["\'])(?P<url>{_VALID_URL}[^"\']*)\1']
+ _TESTS = [{
+ 'url': 'https://omny.fm/shows/sleep-hub/playlists/sleep-talk',
+ 'info_dict': {
+ 'id': 'sleep-talk',
+ 'title': 'Sleep Talk - Talking all things sleep',
+ 'description': 'md5:c1d7e5bf32100a432307d2d32c4ab74a',
+ 'thumbnail': r're:https?://www\.omnycontent\.com/.+',
+ },
+ 'playlist_mincount': 79,
+ }, {
+ 'url': 'https://omny.fm/shows/bayfm-program03/playlists',
+ 'info_dict': {
+ 'id': 'bayfm-program03',
+ },
+ 'playlist_count': 4,
+ }]
+ _WEBPAGE_TESTS = [{
+ 'url': 'https://www.asahi.com/articles/ASP763WDKP4JDIFI002.html',
+ 'info_dict': {
+ 'id': 'podcast',
+ 'title': 'ニュースの現場から',
+ 'description': 'md5:ed1f78462ebed09258ca31b1da5ff640',
+ 'thumbnail': r're:https?://www\.omnycontent\.com/.+',
+ 'webpage_url': 'https://omny.fm/shows/asahi/playlists/podcast',
+ },
+ 'playlist_mincount': 2517,
+ }]
+
+ def _entries(self, uploader_id, playlist_id):
+ clip_id = None
+
+ for page in itertools.count(1):
+ clips = self._download_json(
+ f'{self._API_BASE}/programs/{uploader_id}/playlists/{playlist_id}/clips',
+ playlist_id, f'Downloading page {page}', query=filter_dict({
+ 'clipId': clip_id,
+ 'direction': 'AfterExclusive',
+ 'pageSize': self._PAGE_SIZE,
+ }))
+
+ yield from self._yield_clips(clips, uploader_id)
+
+ if not clips.get('NextClipsAvailable'):
+ break
+
+ clip_id = traverse_obj(clips, ('Clips', -1, 'Id', {str}))
+ if not clip_id:
+ break
+
+ def _real_extract(self, url):
+ uploader_id, playlist_id = self._match_valid_url(url).group('uploader_id', 'id')
+ webpage = self._download_webpage(url, playlist_id or uploader_id)
+ nextjs_data = self._search_nextjs_data(webpage, playlist_id or uploader_id)
+ page_props = traverse_obj(nextjs_data, ('props', 'pageProps', {dict}))
+
+ if not playlist_id:
+ entries = [self.url_result(
+ f'{self._BASE_URL}/{uploader_id}/playlists/{playlist_id}', OmnyfmPlaylistIE,
+ ) for playlist_id in traverse_obj(page_props, (
+ 'playlistsWithClips', ..., 'playlist', 'Slug', {str},
+ ))]
+
+ return self.playlist_result(entries, uploader_id)
+
+ return self.playlist_result(
+ self._entries(uploader_id, playlist_id), playlist_id,
+ **traverse_obj(page_props, ('playlist', {
+ 'title': ('Title', {clean_html}, filter),
+ 'description': ('Description', {clean_html}, filter),
+ 'thumbnail': ('ArtworkUrl', {update_url(query=None)}),
+ 'webpage_url': ('EmbedUrl', {url_or_none}, {trim_str(end='/embed')}),
+ })))
+
+
+class OmnyfmShowIE(OmnyfmPlaylistBaseIE):
+ _VALID_URL = r'https?://omny\.fm/shows/(?P<id>[\w-]+)/?(?:[?#]|$)'
+ _TESTS = [{
+ 'url': 'https://omny.fm/shows/the-origin-of-things',
+ 'info_dict': {
+ 'id': 'the-origin-of-things',
+ 'title': 'The Origin Of Things',
+ 'description': 'md5:52b7fba08201d050639c78ea88cc782e',
+ 'thumbnail': r're:https?://www\.omnycontent\.com/.+',
+ },
+ 'playlist_mincount': 75,
+ }]
+
+ def _fetch_page(self, uploader_id, organization_id, program_id, page):
+ clips = self._download_json(
+ f'{self._API_BASE}/orgs/{organization_id}/programs/{program_id}/clips',
+ uploader_id, f'Downloading page {page + 1}', query={
+ 'cursor': page,
+ 'pageSize': self._PAGE_SIZE,
+ })
+
+ yield from self._yield_clips(clips, uploader_id)
+
+ def _real_extract(self, url):
+ uploader_id = self._match_id(url)
+ webpage = self._download_webpage(url, uploader_id)
+ nextjs_data = self._search_nextjs_data(webpage, uploader_id)
+ program = traverse_obj(nextjs_data, ('props', 'pageProps', 'program', {dict}))
+
+ organization_id = traverse_obj(program, (
+ 'OrganizationId', {str}, {require('organization ID')}))
+ program_id = traverse_obj(program, ('Id', {str}, {require('program ID')}))
+ entries = OnDemandPagedList(
+ functools.partial(self._fetch_page, uploader_id, organization_id, program_id), self._PAGE_SIZE)
+
+ return self.playlist_result(
+ entries, uploader_id,
+ **traverse_obj(program, {
+ 'title': ('Name', {clean_html}, filter),
+ 'description': ('Description', {clean_html}, filter),
+ 'thumbnail': ('ArtworkUrl', {update_url(query=None)}),
+ }))