A feature-rich command-line audio/video downloader
[ie/zan] Add extractor (#16086)
doe1080ad9a6f25f693
diff --git a/README.md b/README.mdindex a3af033142..a2017443e8 100644--- a/README.md+++ b/README.md@@ -1976,6 +1976,9 @@ The following extractors use this feature:* `client`: Client to extract video data from. The currently available clients are `android`, `ios`, `macos` and `web`. Only one client can be used. The `macos` client is used by default, but the `web` client is used when logged-in. The `web` client only works with account cookies or login credentials. The `android` and `ios` clients only work with previously cached OAuth tokens* `original_format_policy`: Policy for when to try extracting original formats. One of `always`, `never`, or `auto`. The default `auto` policy tries to avoid exceeding the web client's API rate-limit by only making an extra request when Vimeo publicizes the video's downloadability+#### zan+* `split_angles`: Split multi-angle streams into separate angle formats. Forces re-encoding of the video stream during download, and requires ffmpeg. Either `true` or `false` (default)+**Note**: These options may be changed/removed in the future without concern for backward compatibility<!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.pyindex 5dba1d9e54..c72f042fc3 100644--- a/yt_dlp/extractor/_extractors.py+++ b/yt_dlp/extractor/_extractors.py@@ -2394,6 +2394,7 @@ from .zaiko import (ZaikoETicketIE,ZaikoIE,)+from .zan import ZanIEfrom .zapiks import ZapiksIEfrom .zattoo import (BBVTVIE,diff --git a/yt_dlp/extractor/zan.py b/yt_dlp/extractor/zan.pynew file mode 100644index 0000000000..15ece33e07--- /dev/null+++ b/yt_dlp/extractor/zan.py@@ -0,0 +1,238 @@+import datetime as dt+import itertools+import math++from .common import InfoExtractor+from ..utils import (+ ExtractorError,+ clean_html,+ extract_attributes,+ float_or_none,+ int_or_none,+ merge_dicts,+ parse_iso8601,+ parse_m3u8_attributes,+ parse_resolution,+ urlencode_postdata,+ urljoin,+)+from ..utils.traversal import find_element, traverse_obj+++class ZanIE(InfoExtractor):+ IE_NAME = 'zan'+ IE_DESC = 'Z-aN'++ _BASE_URL = 'https://www.zan-live.com'+ _GEO_BYPASS = False+ _GEO_COUNTRIES = ['JP']+ _VALID_URL = r'https?://(www\.)?zan-live\.com/[^/?#]+/live/play/\d+/(?P<id>\d+)'+ _TESTS = [{+ 'url': 'https://www.zan-live.com/en/live/play/1797/663',+ 'info_dict': {+ 'id': '663',+ 'ext': 'mp4',+ 'title': 'The sample video page',+ 'alt_title': 'こちらはサンプル動画の再生テストページとなります。',+ 'description': 'md5:12ba331396215fe345b9362c56f3da86',+ 'release_date': '20220228',+ 'release_timestamp': 1646060400,+ 'thumbnail': r're:https?://storage\.zan-live\.com/image/.+\.(?:jpe?g|png)',+ },+ }, {+ 'url': 'https://www.zan-live.com/ja/live/play/6910/4268',+ 'info_dict': {+ 'id': '4268',+ 'ext': 'mp4',+ 'title': '縁結びのゆかり様1stイベント 結び、結ばれ、桃源郷',+ 'alt_title': '縁結びのゆかり様1stイベント 結び、結ばれ、桃源郷 昼公演[アーカイブ]',+ 'description': 'md5:f53156a49ce5b45265d07966679ab494',+ 'release_date': '20260614',+ 'release_timestamp': 1781419500,+ 'thumbnail': r're:https?://storage\.zan-live\.com/image/.+\.(?:jpe?g|png)',+ },+ }]++ @staticmethod+ def _fixup_m3u8_formats(formats, m3u8_doc, m3u8_url):+ for stream_inf, media_url in itertools.pairwise(+ map(str.strip, m3u8_doc.splitlines()),+ ):+ if not stream_inf.startswith('#EXT-X-STREAM-INF:') or not media_url:+ continue++ res = traverse_obj(stream_inf, (+ {parse_m3u8_attributes}, 'DISPLAY-NAME', {parse_resolution}))+ if not res:+ continue++ format_url = urljoin(m3u8_url, media_url)+ for fmt in traverse_obj(formats, (+ lambda _, v: v.get('url') == format_url,+ )):+ fmt.update(merge_dicts(fmt, res))++ @staticmethod+ def _get_multiangle_layout(ma_type, ma_number):+ # Z-aN packs all angles into one video frame+ # Square divisions use an n x n grid+ # 6 divisions use a 3 x 3 grid where angle 1 covers a 2 x 2 block+ # https://static.zan-live.com/static/js/live/shaka_multiangle.js++ divisions = int_or_none(ma_type.partition('_')[0]) or 0+ if divisions <= 0:+ return None, None++ unit = math.isqrt(divisions)+ if unit ** 2 == divisions:+ areas = [(i % unit, i // unit, 1, 1) for i in range(divisions)]+ elif divisions == 6:+ unit = 3+ areas = [+ (0, 0, 2, 2),+ *[(2, i, 1, 1) for i in range(2)],+ *[(i, 2, 1, 1) for i in range(3)],+ ]+ else:+ return None, None++ if not (ma_number and 1 <= ma_number <= len(areas)):+ ma_number = len(areas)++ return unit, areas[:ma_number]++ @staticmethod+ def _multiangle_crop(x, y, w, h, unit, margin_y):+ margin_x = margin_y * 9 / 16++ return (+ f'crop='+ f'trunc((iw*{w}/{unit}-{margin_x * 2:g})/2)*2:'+ f'trunc((ih*{h}/{unit}-{margin_y * 2:g})/2)*2:'+ f'trunc(iw*{x}/{unit}+{margin_x:g}):'+ f'trunc(ih*{y}/{unit}+{margin_y:g})'+ )++ def _multiangle_formats(self, formats, ma_type, ma_number, ma_margin):+ if self._configuration_arg('split_angles', ['false'])[0] == 'false':+ self.to_screen(+ 'Multi-angle formats are available. Use --extractor-args '+ '"zan:split_angles=true" to extract separate angle formats')+ return formats++ unit, areas = self._get_multiangle_layout(ma_type, ma_number)+ if areas is None:+ self.report_warning(f'Unsupported multiangle type: {ma_type}')+ return formats++ angle_formats = []+ for fmt in formats:+ if fmt.get('vcodec') == 'none':+ continue++ height = traverse_obj(fmt, ('height', {int_or_none}))+ for i, (x, y, w, h) in enumerate(areas, 1):+ angle_formats.append({+ **fmt,+ 'downloader_options': {+ 'ffmpeg_args_out': [+ '-vf', self._multiangle_crop(x, y, w, h, unit, ma_margin),+ '-c:v', 'libx264',+ '-c:a', 'copy',+ ],+ },+ 'format_id': f'{fmt["format_id"]}-angle{i}',+ 'height': int_or_none(+ (height * h / unit - ma_margin * 2) // 2 * 2) if height else None,+ 'protocol': 'm3u8',+ 'source_preference': -i,+ })++ return angle_formats++ def _real_extract(self, url):+ video_id = self._match_id(url)+ webpage = self._download_webpage(url, video_id)+ if error_msg := traverse_obj(webpage, (+ {find_element(cls='p-common_message__headline--error')}, {clean_html}, filter,+ )):+ self.raise_geo_restricted(error_msg, countries=self._GEO_COUNTRIES)++ csrf_token = self._html_search_meta('csrf-token', webpage, default=None)+ pct = self._html_search_meta('vod-pct', webpage, default=None)+ token = self._html_search_meta('live-player-token', webpage, default=None)+ if not all((csrf_token, pct, token)):+ self.raise_login_required()++ status = self._download_json(+ f'{self._BASE_URL}/api/live/{video_id}/getLiveStatus', video_id, headers={+ 'X-Csrf-Token': csrf_token,+ }, data=urlencode_postdata({+ 'pct': pct,+ 'token': token,+ }))+ if not traverse_obj(status, ('isSuccess', {bool})):+ raise ExtractorError('Unexpected error')++ result = traverse_obj(status, ('result', {dict}))+ for key, required, error_message in (+ ('isFinished', False, 'This video is no longer available'),+ ('canPlay', True, 'Ticket has expired'),+ ):+ if traverse_obj(result, (key, {bool})) is not required:+ raise ExtractorError(error_message, expected=True)++ is_live = not traverse_obj(result, ('isVod', {bool}))+ release_timestamp = parse_iso8601(self._html_search_meta('open-live-date', webpage))+ srv_time = traverse_obj(status, ('srvTime', {int_or_none}), default=0)++ if is_live and release_timestamp and srv_time < release_timestamp:+ start_time = dt.datetime.fromtimestamp(+ release_timestamp, dt.timezone.utc,+ ).astimezone().strftime('%Y-%m-%d %H:%M:%S %Z')+ self.raise_no_formats(+ f'This livestream is scheduled to start at {start_time}', expected=True)++ return {+ 'id': video_id,+ 'live_status': 'is_upcoming',+ 'release_timestamp': release_timestamp,+ }++ m3u8_url = self._html_search_meta('live-url', webpage, fatal=True)+ m3u8_doc, urlh = self._download_webpage_handle(+ m3u8_url, video_id, note='Downloading m3u8 information')+ m3u8_url = urlh.url++ formats, _ = self._parse_m3u8_formats_and_subtitles(m3u8_doc, m3u8_url, 'mp4')+ self._fixup_m3u8_formats(formats, m3u8_doc, m3u8_url)++ if ma_type := self._html_search_meta('multiangle-type', webpage, default=None):+ ma_number = int_or_none(self._html_search_meta(+ 'multiangle-number', webpage, default=None))+ ma_margin = float_or_none(self._html_search_meta(+ 'multiangle-margin', webpage, default=None), default=0)+ formats = self._multiangle_formats(formats, ma_type, ma_number, ma_margin)++ detail_url = traverse_obj(webpage, (+ {find_element(cls='linkTxt')},+ {find_element(cls='d-flex align-items-center', html=True)},+ {extract_attributes}, 'href', {urljoin(f'{self._BASE_URL}/')}))+ detail = self._download_webpage(detail_url, video_id, fatal=False) or ''++ return {+ 'id': video_id,+ 'alt_title': traverse_obj(webpage, (+ {self._og_search_title}, {clean_html}, filter)),+ 'formats': formats,+ 'is_live': is_live,+ 'release_timestamp': release_timestamp,+ 'thumbnail': self._og_search_thumbnail(detail),+ **traverse_obj(detail, {+ 'title': ({self._og_search_title}, {clean_html}, filter),+ 'description': ((+ {find_element(cls='p-eventinfo__detail')},+ {find_element(cls='groupDetail')},+ ), {clean_html}, filter, any),+ }),+ }