Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: m3u8 substitles merging (#174)

NTFSvolume4c879525aaaa
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.py
index f9ba5a04c..b97c13f4e 100644
--- a/cyberdrop_dl/crawlers/_hls.py
+++ b/cyberdrop_dl/crawlers/_hls.py
@@ -43,7 +43,7 @@ class HLSParser(ABC):
rendition = m3u8.select_best_rendition(m3u8_obj, only=only, exclude=exclude)
return await self._resolve_rendition(rendition, headers)
m3u8_obj.media_type = "video"
- return m3u8.Rendition(m3u8_obj), None
+ return m3u8.Rendition(m3u8_obj, None, None), None
async def _resolve_rendition(
self,
@@ -51,19 +51,24 @@ class HLSParser(ABC):
/,
headers: Mapping[str, str] | None = None,
):
- video, *audio_and_subs = await asyncio.gather(
+
+ async def resolve(url: AbsoluteHttpURL | None, media_type: Literal["video", "audio", "subtitle"]):
+ if not url:
+ return
+ return await self._request_m3u8(url, headers, media_type)
+
+ video, audio, subs = await asyncio.gather(
*(
- self._request_m3u8(url, headers, name)
+ resolve(url, name)
for name, url in zip(
("video", "audio", "subtitle"),
rendition.urls,
strict=True,
)
- if url
)
)
-
- return m3u8.Rendition(video, *audio_and_subs), rendition
+ assert video
+ return m3u8.Rendition(video, audio, subs), rendition
async def _request_m3u8(
self,
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 5f810300f..9fd9ac703 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -194,10 +194,14 @@ class Downloader:
download_folder = temp_dir / m3u8.media_type
n_segmets = len(m3u8.segments)
+ real_ext = parse_url(m3u8.segments[0].absolute_uri).suffix
if n_segmets > 1:
- suffix = f".{m3u8.media_type}.ts"
+ if m3u8.media_type == "subtitle":
+ suffix = f".{m3u8.media_type}{real_ext}"
+ else:
+ suffix = f".{m3u8.media_type}.ts"
else:
- suffix = media_item.path.suffix + parse_url(m3u8.segments[0].absolute_uri).suffix
+ suffix = media_item.path.suffix + real_ext
output = media_item.path.with_suffix(suffix)
if await aio.is_file(output):
@@ -214,9 +218,12 @@ class Downloader:
seg_paths = [result.item.path for result in tasks_results]
if n_segmets > 1:
- ffmpeg_result = await ffmpeg.concat(seg_paths, output)
- if not ffmpeg_result.success:
- raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)
+ if m3u8.media_type == "subtitle":
+ await ffmpeg.merge_subs(seg_paths, output)
+ else:
+ ffmpeg_result = await ffmpeg.concat(seg_paths, output, same_folder=False)
+ if not ffmpeg_result.success:
+ raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)
else:
_ = await asyncio.to_thread(seg_paths[0].rename, output)
return output
diff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.py
index 5972a21b5..8a0e261a1 100644
--- a/cyberdrop_dl/ffmpeg.py
+++ b/cyberdrop_dl/ffmpeg.py
@@ -148,11 +148,26 @@ async def _try_delete(file: Path) -> None:
async def _delete_files(files: Iterable[Path], *, same_folder: bool) -> None:
if same_folder:
folder = next(iter(files)).parent
+ logger.debug("Deleting all files inside '%s'", folder)
await asyncio.to_thread(shutil.rmtree, folder, ignore_errors=True)
else:
_ = await asyncio.gather(*map(_try_delete, files))
+async def merge_subs(files: Iterable[Path], output: Path) -> None:
+ logger.debug("Merging subs to '%s'", output)
+ await asyncio.to_thread(_raw_concat, files, output)
+
+
+def _raw_concat(files: Iterable[Path], output: Path) -> None:
+ with output.open("wb") as out:
+ for file in files:
+ with file.open("rb") as fp_in:
+ out.write(fp_in.read())
+
+ file.unlink()
+
+
@overload
async def probe(input: Path, /) -> FFprobeResult: ...
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index 3f074442b..383ef6d52 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from dataclasses import dataclass
+import dataclasses
from datetime import timedelta
from enum import StrEnum
from functools import cached_property
@@ -14,7 +14,7 @@ from cyberdrop_dl.mediaprops import Codecs, Resolution
from cyberdrop_dl.utils import parse_url
if TYPE_CHECKING:
- from collections.abc import Generator, Iterable
+ from collections.abc import Generator, Iterable, Iterator
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -66,13 +66,17 @@ class MediaURLs(NamedTuple):
subtitle: AbsoluteHttpURL | None
-class Rendition(NamedTuple):
[email protected](slots=True)
+class Rendition:
video: M3U8
- audio: M3U8 | None = None
- subtitle: M3U8 | None = None
+ audio: M3U8 | None
+ subtitle: M3U8 | None
+ def __iter__(self) -> Iterator[M3U8 | None]:
+ return iter(dataclasses.astuple(self))
-@dataclass(frozen=True, slots=True, order=True)
+
[email protected](frozen=True, slots=True, order=True)
class StreamInfo:
"""Exactly the same as m3u8.model.StreamInfo but as a dataclass, to support sorting rendition groups with the same resolution but different bitrates (bandwidth)"""
@@ -95,7 +99,7 @@ class StreamInfo:
__str__ = m3u8.model.StreamInfo.__str__
-@dataclass(frozen=True, slots=True, order=True)
[email protected](frozen=True, slots=True, order=True)
class RenditionDetails:
resolution: Resolution
codecs: Codecs
diff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.py
index 7d28f3385..c7f6e2e1b 100644
--- a/tests/test_ffmpeg.py
+++ b/tests/test_ffmpeg.py
@@ -1,4 +1,5 @@
import datetime
+from pathlib import Path
import pytest
@@ -65,3 +66,31 @@ def test_parse_duration(input: str, hours: float, minutes: float, seconds: float
def test_parse_null_duration(input: str) -> None:
output = ffmpeg._parse_duration(input)
assert output is None
+
+
+class TestMergeSubs:
+ def test_normal_merge(self, tmp_path: Path) -> None:
+ srcs = [tmp_path / f"sub{i}.srt" for i in range(1, 4)]
+ for p, content in zip(srcs, (b"AAA\n", b"BBB\n", b"CCC\n"), strict=True):
+ p.write_bytes(content)
+
+ out = tmp_path / "merged.srt"
+
+ ffmpeg._raw_concat(srcs, out)
+
+ assert out.read_bytes() == b"AAA\nBBB\nCCC\n"
+ for p in srcs:
+ assert not p.exists()
+
+ def test_empty_input(self, tmp_path: Path) -> None:
+ out = tmp_path / "empty.srt"
+ ffmpeg._raw_concat([], out)
+ assert out.read_bytes() == b""
+
+ def test_single_file(self, tmp_path: Path) -> None:
+ src = tmp_path / "subtitles.srt"
+ src.write_bytes(b" test \n")
+ out = tmp_path / "out.srt"
+ ffmpeg._raw_concat([src], out)
+ assert out.read_bytes() == b" test \n"
+ assert not src.exists()