Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: do not skip initialization segments (HLS) (#1977)
NTFSvolume10c1fa02df28
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.pyindex d7eb63826..046bfbdcc 100644--- a/cyberdrop_dl/downloader/hls.py+++ b/cyberdrop_dl/downloader/hls.py@@ -15,7 +15,7 @@ if TYPE_CHECKING:from collections.abc import Awaitable, Callable, Generator, Iterable, Sequencefrom pathlib import Path- from m3u8.model import Segment+ from m3u8.model import InitializationSection, Segmentfrom cyberdrop_dl.utils.m3u8 import M3U8, Rendition@@ -39,18 +39,16 @@ class Streams(NamedTuple):class HLSSegment(NamedTuple):idx: int- part: strname: strurl: AbsoluteHttpURL-def _parse_segments(segments: Sequence[Segment]) -> Generator[HLSSegment]:+def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Generator[HLSSegment]:padding = max(5, len(str(len(segments))))for index, segment in enumerate(segments, 1):assert segment.uriyield HLSSegment(idx=index - 1,- part=segment.title,name=f"{index:0{padding}d}{constants.TempExt.HLS}",url=parse_url(segment.absolute_uri),)@@ -82,6 +80,14 @@ def _create_media_segments(yield seg_media_item+def _segments(m3u8: M3U8) -> list[Segment | InitializationSection]:+ segments = m3u8.segment_map + m3u8.segments+ if not segments:+ msg = f"{m3u8.media_type} m3u8 manifest ({m3u8.base_uri}) has no valid segments"+ raise DownloadError(HTTPStatus.NO_CONTENT, msg)+ return segments++async def _download_m3u8(m3u8: M3U8,temp_dir: Path,@@ -90,41 +96,44 @@ async def _download_m3u8(sem: asyncio.BoundedSemaphore,) -> Path:assert m3u8.media_type- if not m3u8.segments:- raise DownloadError(- HTTPStatus.NO_CONTENT,- f"{m3u8.media_type} m3u8 manifest ({m3u8.base_uri}) has no valid segments",- )-+ segments = _segments(m3u8)output = _prepare_output_path(m3u8, media_item.path)if await aio.is_file(output):return output- async def download_segment(seg_media_item: MediaItem) -> SegmentDownloadResult:+ async def download(seg_media_item: MediaItem) -> SegmentDownloadResult:return SegmentDownloadResult(seg_media_item, await download_fn(seg_media_item))- segments = _create_media_segments(+ m_segments = _create_media_segments(media_item,- _parse_segments(m3u8.segments),+ _parse_segments(segments),download_folder=temp_dir / m3u8.media_type,)- logger.debug(- f"Starting HLS download ({m3u8.media_type}, {len(m3u8.segments):,} segments) for {media_item.real_url}"- )+ logger.debug(f"Starting HLS download ({m3u8.media_type}, {len(segments):,} segments) for {media_item.real_url}")+ results = await _download_segments(m_segments, m3u8.total_segments, download, sem)+ await _merge_segments(tuple(result.item.path for result in results), output, m3u8.media_type)+ return output+++async def _download_segments(+ segments: Iterable[MediaItem],+ count: int,+ download: Callable[[MediaItem], Awaitable[SegmentDownloadResult]],+ sem: asyncio.BoundedSemaphore,+) -> list[SegmentDownloadResult]:results = await aio.map(- download_segment,+ download,segments,task_limit=sem,)n_successful = sum(1 for result in results if result.downloaded)- if n_successful != len(m3u8.segments):- msg = f"Download of some segments failed. Successful: {n_successful:,}/{len(m3u8.segments):,} "- raise DownloadError("HLS Seg Error", msg, media_item)+ if n_successful != count:+ msg = f"Download of some segments failed. Successful: {n_successful:,}/{count:,} "+ raise DownloadError("HLS Seg Error", msg)- await _merge_segments(tuple(result.item.path for result in results), output, m3u8.media_type)- return output+ return resultsasync def _merge_segments(seg_paths: Sequence[Path], output: Path, media_type: str) -> None:diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex 18dec7c13..2bafecdc3 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -248,7 +248,7 @@ class Downloader:with self.manager.scrape_mapper.tui.downloads.download_hls(media_item.filename,media_item.domain,- segments=sum(len(m.segments) for m in rendition if m is not None),+ segments=sum(m.total_segments for m in rendition if m is not None),url=media_item.url,):await self._hls_download(media_item, rendition)diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pyindex 0525eec59..3246359b2 100644--- a/cyberdrop_dl/utils/m3u8.py+++ b/cyberdrop_dl/utils/m3u8.py@@ -155,6 +155,10 @@ class M3U8(_M3U8):self.media_type: Literal["video", "audio", "subtitle"] | None = media_typesuper().__init__(content, base_uri=str(base_uri) if base_uri else None)+ @property+ def total_segments(self) -> int:+ return len(self.segment_map) + len(self.segments)+def __repr__(self) -> str:return (f"{type(self)}(media_type={self.media_type!r}, base_uri={self.base_uri!r}, is_variant={self.is_variant!r})"diff --git a/tests/test_hls.py b/tests/test_hls.pyindex 819207e35..4278f2422 100644--- a/tests/test_hls.py+++ b/tests/test_hls.py@@ -44,7 +44,6 @@ def test_parse_segments() -> None:first = result[0]assert type(first) is hls.HLSSegmentassert first.idx == 0- assert first.part is Noneassert first.url == AbsoluteHttpURL("https://example.com/m3u8/test")assert first.name == "00001.cdl_hls"last = result[-1]@@ -64,7 +63,6 @@ def test_create_media_segments() -> None:)segment = hls.HLSSegment(idx=22,- part="image.jpg",name="00023.cdl_hls",url=AbsoluteHttpURL("https://example.com/m3u8/test/segments001.ts"),)@@ -92,3 +90,30 @@ def test_prepare_output_path(tmp_path: Path) -> None:output = tmp_path / "download" / "m3u8.txt"file = hls._prepare_output_path(m3u8, output)assert file == tmp_path / "download" / "m3u8.subtitle.vtt"+++def test_init_segments_should_be_include() -> None:+ content = """+ #EXTM3U+ #EXT-X-VERSION:6+ #EXT-X-PLAYLIST-TYPE:VOD+ #EXT-X-MEDIA-SEQUENCE:0+ #EXT-X-TARGETDURATION:5+ #EXT-X-MAP:URI="../../ac555a6ea0431c298d53d486a2cc1059/video/1080/init.mp4"+ #EXT-X-INDEPENDENT-SEGMENTS+ #EXTINF:4.00000,+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_1.mp4+ #EXTINF:4.00000,+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_2.mp4+ #EXTINF:4.00000,+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_3.mp4+ #EXTINF:4.00000,+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_4.mp4+ #EXTINF:2,+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_260.mp4+ #EXT-X-ENDLIST+ """+ m3u8 = M3U8(content)+ assert len(m3u8.segment_map) == 1+ assert len(m3u8.segments) == 5+ assert len(hls._segments(m3u8)) == 6