Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: m3u8 substitles merging (#174)
NTFSvolume4c879525aaaa
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.pyindex f9ba5a04c..b97c13f4e 100644--- a/cyberdrop_dl/crawlers/_hls.py+++ b/cyberdrop_dl/crawlers/_hls.py@@ -43,7 +43,7 @@ class HLSParser(ABC):rendition = m3u8.select_best_rendition(m3u8_obj, only=only, exclude=exclude)return await self._resolve_rendition(rendition, headers)m3u8_obj.media_type = "video"- return m3u8.Rendition(m3u8_obj), None+ return m3u8.Rendition(m3u8_obj, None, None), Noneasync def _resolve_rendition(self,@@ -51,19 +51,24 @@ class HLSParser(ABC):/,headers: Mapping[str, str] | None = None,):- video, *audio_and_subs = await asyncio.gather(++ async def resolve(url: AbsoluteHttpURL | None, media_type: Literal["video", "audio", "subtitle"]):+ if not url:+ return+ return await self._request_m3u8(url, headers, media_type)++ video, audio, subs = await asyncio.gather(*(- self._request_m3u8(url, headers, name)+ resolve(url, name)for name, url in zip(("video", "audio", "subtitle"),rendition.urls,strict=True,)- if url))-- return m3u8.Rendition(video, *audio_and_subs), rendition+ assert video+ return m3u8.Rendition(video, audio, subs), renditionasync def _request_m3u8(self,diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 5f810300f..9fd9ac703 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -194,10 +194,14 @@ class Downloader:download_folder = temp_dir / m3u8.media_typen_segmets = len(m3u8.segments)+ real_ext = parse_url(m3u8.segments[0].absolute_uri).suffixif n_segmets > 1:- suffix = f".{m3u8.media_type}.ts"+ if m3u8.media_type == "subtitle":+ suffix = f".{m3u8.media_type}{real_ext}"+ else:+ suffix = f".{m3u8.media_type}.ts"else:- suffix = media_item.path.suffix + parse_url(m3u8.segments[0].absolute_uri).suffix+ suffix = media_item.path.suffix + real_extoutput = media_item.path.with_suffix(suffix)if await aio.is_file(output):@@ -214,9 +218,12 @@ class Downloader:seg_paths = [result.item.path for result in tasks_results]if n_segmets > 1:- ffmpeg_result = await ffmpeg.concat(seg_paths, output)- if not ffmpeg_result.success:- raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)+ if m3u8.media_type == "subtitle":+ await ffmpeg.merge_subs(seg_paths, output)+ else:+ ffmpeg_result = await ffmpeg.concat(seg_paths, output, same_folder=False)+ if not ffmpeg_result.success:+ raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)else:_ = await asyncio.to_thread(seg_paths[0].rename, output)return outputdiff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.pyindex 5972a21b5..8a0e261a1 100644--- a/cyberdrop_dl/ffmpeg.py+++ b/cyberdrop_dl/ffmpeg.py@@ -148,11 +148,26 @@ async def _try_delete(file: Path) -> None:async def _delete_files(files: Iterable[Path], *, same_folder: bool) -> None:if same_folder:folder = next(iter(files)).parent+ logger.debug("Deleting all files inside '%s'", folder)await asyncio.to_thread(shutil.rmtree, folder, ignore_errors=True)else:_ = await asyncio.gather(*map(_try_delete, files))+async def merge_subs(files: Iterable[Path], output: Path) -> None:+ logger.debug("Merging subs to '%s'", output)+ await asyncio.to_thread(_raw_concat, files, output)+++def _raw_concat(files: Iterable[Path], output: Path) -> None:+ with output.open("wb") as out:+ for file in files:+ with file.open("rb") as fp_in:+ out.write(fp_in.read())++ file.unlink()++@overloadasync def probe(input: Path, /) -> FFprobeResult: ...diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pyindex 3f074442b..383ef6d52 100644--- a/cyberdrop_dl/utils/m3u8.py+++ b/cyberdrop_dl/utils/m3u8.py@@ -1,6 +1,6 @@from __future__ import annotations-from dataclasses import dataclass+import dataclassesfrom datetime import timedeltafrom enum import StrEnumfrom functools import cached_property@@ -14,7 +14,7 @@ from cyberdrop_dl.mediaprops import Codecs, Resolutionfrom cyberdrop_dl.utils import parse_urlif TYPE_CHECKING:- from collections.abc import Generator, Iterable+ from collections.abc import Generator, Iterable, Iteratorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -66,13 +66,17 @@ class MediaURLs(NamedTuple):subtitle: AbsoluteHttpURL | None-class Rendition(NamedTuple):[email protected](slots=True)+class Rendition:video: M3U8- audio: M3U8 | None = None- subtitle: M3U8 | None = None+ audio: M3U8 | None+ subtitle: M3U8 | None+ def __iter__(self) -> Iterator[M3U8 | None]:+ return iter(dataclasses.astuple(self))-@dataclass(frozen=True, slots=True, order=True)+[email protected](frozen=True, slots=True, order=True)class StreamInfo:"""Exactly the same as m3u8.model.StreamInfo but as a dataclass, to support sorting rendition groups with the same resolution but different bitrates (bandwidth)"""@@ -95,7 +99,7 @@ class StreamInfo:__str__ = m3u8.model.StreamInfo.__str__-@dataclass(frozen=True, slots=True, order=True)[email protected](frozen=True, slots=True, order=True)class RenditionDetails:resolution: Resolutioncodecs: Codecsdiff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.pyindex 7d28f3385..c7f6e2e1b 100644--- a/tests/test_ffmpeg.py+++ b/tests/test_ffmpeg.py@@ -1,4 +1,5 @@import datetime+from pathlib import Pathimport pytest@@ -65,3 +66,31 @@ def test_parse_duration(input: str, hours: float, minutes: float, seconds: floatdef test_parse_null_duration(input: str) -> None:output = ffmpeg._parse_duration(input)assert output is None+++class TestMergeSubs:+ def test_normal_merge(self, tmp_path: Path) -> None:+ srcs = [tmp_path / f"sub{i}.srt" for i in range(1, 4)]+ for p, content in zip(srcs, (b"AAA\n", b"BBB\n", b"CCC\n"), strict=True):+ p.write_bytes(content)++ out = tmp_path / "merged.srt"++ ffmpeg._raw_concat(srcs, out)++ assert out.read_bytes() == b"AAA\nBBB\nCCC\n"+ for p in srcs:+ assert not p.exists()++ def test_empty_input(self, tmp_path: Path) -> None:+ out = tmp_path / "empty.srt"+ ffmpeg._raw_concat([], out)+ assert out.read_bytes() == b""++ def test_single_file(self, tmp_path: Path) -> None:+ src = tmp_path / "subtitles.srt"+ src.write_bytes(b" test \n")+ out = tmp_path / "out.srt"+ ffmpeg._raw_concat([src], out)+ assert out.read_bytes() == b" test \n"+ assert not src.exists()