Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: do not skip initialization segments (HLS) (#1977)

NTFSvolume10c1fa02df28
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.py
index d7eb63826..046bfbdcc 100644
--- a/cyberdrop_dl/downloader/hls.py
+++ b/cyberdrop_dl/downloader/hls.py
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence
from pathlib import Path
- from m3u8.model import Segment
+ from m3u8.model import InitializationSection, Segment
from cyberdrop_dl.utils.m3u8 import M3U8, Rendition
@@ -39,18 +39,16 @@ class Streams(NamedTuple):
class HLSSegment(NamedTuple):
idx: int
- part: str
name: str
url: AbsoluteHttpURL
-def _parse_segments(segments: Sequence[Segment]) -> Generator[HLSSegment]:
+def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Generator[HLSSegment]:
padding = max(5, len(str(len(segments))))
for index, segment in enumerate(segments, 1):
assert segment.uri
yield HLSSegment(
idx=index - 1,
- part=segment.title,
name=f"{index:0{padding}d}{constants.TempExt.HLS}",
url=parse_url(segment.absolute_uri),
)
@@ -82,6 +80,14 @@ def _create_media_segments(
yield seg_media_item
+def _segments(m3u8: M3U8) -> list[Segment | InitializationSection]:
+ segments = m3u8.segment_map + m3u8.segments
+ if not segments:
+ msg = f"{m3u8.media_type} m3u8 manifest ({m3u8.base_uri}) has no valid segments"
+ raise DownloadError(HTTPStatus.NO_CONTENT, msg)
+ return segments
+
+
async def _download_m3u8(
m3u8: M3U8,
temp_dir: Path,
@@ -90,41 +96,44 @@ async def _download_m3u8(
sem: asyncio.BoundedSemaphore,
) -> Path:
assert m3u8.media_type
- if not m3u8.segments:
- raise DownloadError(
- HTTPStatus.NO_CONTENT,
- f"{m3u8.media_type} m3u8 manifest ({m3u8.base_uri}) has no valid segments",
- )
-
+ segments = _segments(m3u8)
output = _prepare_output_path(m3u8, media_item.path)
if await aio.is_file(output):
return output
- async def download_segment(seg_media_item: MediaItem) -> SegmentDownloadResult:
+ async def download(seg_media_item: MediaItem) -> SegmentDownloadResult:
return SegmentDownloadResult(seg_media_item, await download_fn(seg_media_item))
- segments = _create_media_segments(
+ m_segments = _create_media_segments(
media_item,
- _parse_segments(m3u8.segments),
+ _parse_segments(segments),
download_folder=temp_dir / m3u8.media_type,
)
- logger.debug(
- f"Starting HLS download ({m3u8.media_type}, {len(m3u8.segments):,} segments) for {media_item.real_url}"
- )
+ logger.debug(f"Starting HLS download ({m3u8.media_type}, {len(segments):,} segments) for {media_item.real_url}")
+ results = await _download_segments(m_segments, m3u8.total_segments, download, sem)
+ await _merge_segments(tuple(result.item.path for result in results), output, m3u8.media_type)
+ return output
+
+
+async def _download_segments(
+ segments: Iterable[MediaItem],
+ count: int,
+ download: Callable[[MediaItem], Awaitable[SegmentDownloadResult]],
+ sem: asyncio.BoundedSemaphore,
+) -> list[SegmentDownloadResult]:
results = await aio.map(
- download_segment,
+ download,
segments,
task_limit=sem,
)
n_successful = sum(1 for result in results if result.downloaded)
- if n_successful != len(m3u8.segments):
- msg = f"Download of some segments failed. Successful: {n_successful:,}/{len(m3u8.segments):,} "
- raise DownloadError("HLS Seg Error", msg, media_item)
+ if n_successful != count:
+ msg = f"Download of some segments failed. Successful: {n_successful:,}/{count:,} "
+ raise DownloadError("HLS Seg Error", msg)
- await _merge_segments(tuple(result.item.path for result in results), output, m3u8.media_type)
- return output
+ return results
async def _merge_segments(seg_paths: Sequence[Path], output: Path, media_type: str) -> None:
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index 18dec7c13..2bafecdc3 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -248,7 +248,7 @@ class Downloader:
with self.manager.scrape_mapper.tui.downloads.download_hls(
media_item.filename,
media_item.domain,
- segments=sum(len(m.segments) for m in rendition if m is not None),
+ segments=sum(m.total_segments for m in rendition if m is not None),
url=media_item.url,
):
await self._hls_download(media_item, rendition)
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index 0525eec59..3246359b2 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -155,6 +155,10 @@ class M3U8(_M3U8):
self.media_type: Literal["video", "audio", "subtitle"] | None = media_type
super().__init__(content, base_uri=str(base_uri) if base_uri else None)
+ @property
+ def total_segments(self) -> int:
+ return len(self.segment_map) + len(self.segments)
+
def __repr__(self) -> str:
return (
f"{type(self)}(media_type={self.media_type!r}, base_uri={self.base_uri!r}, is_variant={self.is_variant!r})"
diff --git a/tests/test_hls.py b/tests/test_hls.py
index 819207e35..4278f2422 100644
--- a/tests/test_hls.py
+++ b/tests/test_hls.py
@@ -44,7 +44,6 @@ def test_parse_segments() -> None:
first = result[0]
assert type(first) is hls.HLSSegment
assert first.idx == 0
- assert first.part is None
assert first.url == AbsoluteHttpURL("https://example.com/m3u8/test")
assert first.name == "00001.cdl_hls"
last = result[-1]
@@ -64,7 +63,6 @@ def test_create_media_segments() -> None:
)
segment = hls.HLSSegment(
idx=22,
- part="image.jpg",
name="00023.cdl_hls",
url=AbsoluteHttpURL("https://example.com/m3u8/test/segments001.ts"),
)
@@ -92,3 +90,30 @@ def test_prepare_output_path(tmp_path: Path) -> None:
output = tmp_path / "download" / "m3u8.txt"
file = hls._prepare_output_path(m3u8, output)
assert file == tmp_path / "download" / "m3u8.subtitle.vtt"
+
+
+def test_init_segments_should_be_include() -> None:
+ content = """
+ #EXTM3U
+ #EXT-X-VERSION:6
+ #EXT-X-PLAYLIST-TYPE:VOD
+ #EXT-X-MEDIA-SEQUENCE:0
+ #EXT-X-TARGETDURATION:5
+ #EXT-X-MAP:URI="../../ac555a6ea0431c298d53d486a2cc1059/video/1080/init.mp4"
+ #EXT-X-INDEPENDENT-SEGMENTS
+ #EXTINF:4.00000,
+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_1.mp4
+ #EXTINF:4.00000,
+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_2.mp4
+ #EXTINF:4.00000,
+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_3.mp4
+ #EXTINF:4.00000,
+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_4.mp4
+ #EXTINF:2,
+ ../../ac555a6ea0431c298d53d486a2cc1059/video/1080/seg_260.mp4
+ #EXT-X-ENDLIST
+ """
+ m3u8 = M3U8(content)
+ assert len(m3u8.segment_map) == 1
+ assert len(m3u8.segments) == 5
+ assert len(hls._segments(m3u8)) == 6