Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add `ffprobe` checks and fallback to builtin binary (#1154)

NTFSvolume9fba56aff4b7
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 15c2aaedb..f6065b31b 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -23,6 +23,7 @@ from cyberdrop_dl.exceptions import (
InvalidContentTypeError,
RestrictedFiletypeError,
)
+from cyberdrop_dl.utils import ffmpeg
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none, parse_url
@@ -150,8 +151,11 @@ class Downloader:
@error_handling_wrapper
async def download_hls(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> None:
await self.client.mark_incomplete(media_item, self.domain)
- if not self.manager.ffmpeg.is_available:
- raise DownloadError("FFmpeg Error", "FFmpeg is required for HLS downloads but is not available", media_item)
+ try:
+ ffmpeg.check_is_available()
+ except RuntimeError as e:
+ msg = f"{e} - ffmpeg and ffprobe are required for HLS downloads"
+ raise DownloadError("FFmpeg Error", msg, media_item) from None
media_item.complete_file = media_item.download_folder / media_item.filename
self.update_queued_files()
@@ -162,7 +166,7 @@ class Downloader:
await asyncio.to_thread(video.rename, media_item.complete_file)
else:
parts = [part for part in (video, audio, subtitles) if part]
- ffmpeg_result = await self.manager.ffmpeg.merge(*parts, output_file=media_item.complete_file)
+ ffmpeg_result = await ffmpeg.merge(parts, media_item.complete_file)
if not ffmpeg_result.success:
raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)
@@ -192,7 +196,7 @@ class Downloader:
seg_paths = [item.complete_file for item in items if item.complete_file]
output = media_item.complete_file.with_suffix(f".{media_type}.ts")
- ffmpeg_result = await self.manager.ffmpeg.concat(*seg_paths, output_file=output)
+ ffmpeg_result = await ffmpeg.concat(seg_paths, output)
if not ffmpeg_result.success:
raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)
results.append(output)
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index cff809143..d3a182743 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -22,8 +22,8 @@ from cyberdrop_dl.managers.progress_manager import ProgressManager
from cyberdrop_dl.managers.realdebrid_manager import RealDebridManager
from cyberdrop_dl.managers.storage_manager import StorageManager
from cyberdrop_dl.ui.textual import TextualUI
+from cyberdrop_dl.utils import ffmpeg
from cyberdrop_dl.utils.args import ParsedArgs, parse_args
-from cyberdrop_dl.utils.ffmpeg import FFmpeg, get_ffmpeg_version
from cyberdrop_dl.utils.logger import QueuedLogger, log
from cyberdrop_dl.utils.transfer import transfer_v5_db_to_v6
from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information
@@ -58,7 +58,6 @@ class Manager:
self.progress_manager: ProgressManager = field(init=False)
self.live_manager: LiveManager = field(init=False)
self.textual_log_queue: queue.Queue = field(init=False)
- self.ffmpeg: FFmpeg = field(init=False)
self._textual_ui: TextualUI = field(init=False)
self._loaded_args_config: bool = False
@@ -170,9 +169,6 @@ class Manager:
if not isinstance(self.real_debrid_manager, RealDebridManager):
self.real_debrid_manager = RealDebridManager(self)
- if not isinstance(self.ffmpeg, FFmpeg):
- self.ffmpeg = FFmpeg(self)
-
await self.async_db_hash_startup()
constants.MAX_NAME_LENGTHS["FILE"] = self.config_manager.global_settings_data.general.max_file_name_length
@@ -229,19 +225,23 @@ class Manager:
cli_only_args = self.parsed_args.cli_only_args.model_dump_json(indent=4)
system_info = get_system_information()
- log("Starting Cyberdrop-DL Process")
- log(f"Running Version: {__version__}")
- log(f"System Info:{system_info}")
- log(f"Using Config: {self.config_manager.loaded_config}")
- log(f"Using Config File: {self.config_manager.settings}")
- log(f"Using Input File: {self.path_manager.input_file}")
- log(f"Using Download Folder: {self.path_manager.download_folder}")
- log(f"Using Database File: {self.path_manager.history_db}")
- log(f"Using CLI only options: {cli_only_args}")
- log(f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}")
- log(f"Using Settings: \n{config_settings}")
- log(f"Using Global Settings: \n{global_settings}")
- log(f"Using FFmpeg version: {get_ffmpeg_version()}")
+ args_info = (
+ "Starting Cyberdrop-DL Process",
+ f"Running Version: {__version__}",
+ f"System Info:{system_info}",
+ f"Using Config: {self.config_manager.loaded_config}",
+ f"Using Config File: {self.config_manager.settings}",
+ f"Using Input File: {self.path_manager.input_file}",
+ f"Using Download Folder: {self.path_manager.download_folder}",
+ f"Using Database File: {self.path_manager.history_db}",
+ f"Using CLI only options: {cli_only_args}",
+ f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}",
+ f"Using Settings: \n{config_settings}",
+ f"Using Global Settings: \n{global_settings}",
+ f"Using ffmpeg version: {ffmpeg.get_ffmpeg_version()}",
+ f"Using ffprobe version: {ffmpeg.get_ffprobe_version()}",
+ )
+ log("\n".join(args_info))
async def async_db_close(self) -> None:
"Partial shutdown for managers used for hash directory scanner"
diff --git a/cyberdrop_dl/utils/ffmpeg.py b/cyberdrop_dl/utils/ffmpeg.py
index 556057b3d..6bf9d6c1f 100644
--- a/cyberdrop_dl/utils/ffmpeg.py
+++ b/cyberdrop_dl/utils/ffmpeg.py
@@ -1,19 +1,21 @@
from __future__ import annotations
import asyncio
+import itertools
import json
import shutil
import subprocess
-from dataclasses import asdict, dataclass, field
-from datetime import datetime, timedelta
+from dataclasses import asdict, dataclass
+from datetime import timedelta
from fractions import Fraction
from functools import lru_cache
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Required, Self, TypedDict, overload
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Required, Self, TypeAlias, TypedDict, overload
import aiofiles
import aiofiles.os
-from multidict import CIMultiDict
+from multidict import CIMultiDict, CIMultiDictProxy
+from videoprops import which_ffprobe as _builtin_ffprobe
from yarl import URL
from cyberdrop_dl.utils.logger import log_debug
@@ -22,100 +24,120 @@ from cyberdrop_dl.utils.utilities import get_valid_dict, is_absolute_http_url
if TYPE_CHECKING:
from collections.abc import Generator, Mapping, Sequence
- from cyberdrop_dl.managers.manager import Manager
+ _CMD: TypeAlias = Sequence[str | Path]
-FFPROBE_CALL_PREFIX = "ffprobe", "-hide_banner", "-loglevel", "error", "-show_streams", "-print_format", "json"
-FFMPEG_CALL_PREFIX = "ffmpeg", "-y", "-loglevel", "error"
-FFMPEG_FIXUP_INPUT_ARGS = "-map", "0", "-ignore_unknown", "-c", "copy", "-f", "mp4", "-movflags", "+faststart"
-FFMPEG_FIXUP_AUDIO_FILTER_ARGS = "-bsf:a", "aac_adtstoasc"
+CODEC_COPY = "-c", "copy"
MERGE_INPUT_ARGS = "-map", "0"
CONCAT_INPUT_ARGS = "-f", "concat", "-safe", "0", "-i"
-CODEC_COPY = "-c", "copy"
+FFMPEG_CALL_PREFIX = "ffmpeg", "-y", "-loglevel", "error"
+FFMPEG_FIXUP_INPUT_ARGS = "-map", "0", "-ignore_unknown", *CODEC_COPY, "-f", "mp4", "-movflags", "+faststart"
+FFMPEG_FIXUP_AUDIO_DTS_FILTER_ARGS = "-bsf:a", "aac_adtstoasc"
+FFPROBE_CALL_PREFIX = "ffprobe", "-hide_banner", "-loglevel", "error", "-show_streams", "-print_format", "json"
+_FFPROBE_AVAILABLE = False
+_EMPTY_FFPROBE_OUTPUT: FFprobeOutput = {"streams": []}
-class FFmpeg:
- def __init__(self, manager: Manager) -> None:
- self.manager = manager
- self.cache_folder = self.manager.path_manager.cache_folder.resolve()
- self.version = get_ffmpeg_version()
- self.is_available = bool(self.version)
-
- async def concat(self, *input_files: Path, output_file: Path, same_folder: bool = True) -> SubProcessResult:
- if not self.is_available:
- raise RuntimeError("ffmpeg is not available")
- now = datetime.now().strftime("%Y%m%d_%H%M%S")
- concat_file_name = f"{now} - {output_file.name[:40]}.ffmpeg_input.txt"
- concat_file_path = self.cache_folder / concat_file_name
- await _create_concat_input_file(*input_files, file_path=concat_file_path)
- result = await _concat(concat_file_path, output_file)
- if result.success:
- if same_folder:
- folder = input_files[0].parent
- await asyncio.to_thread(shutil.rmtree, folder, True)
- else:
- await async_delete_files(concat_file_path, *input_files)
- return result
+def check_is_available() -> None:
+ if not get_ffmpeg_version():
+ raise RuntimeError("ffmpeg is not available")
+ if not get_ffprobe_version():
+ raise RuntimeError("ffprobe is not available")
- async def merge(self, *input_files: Path, output_file: Path) -> SubProcessResult:
- if not self.is_available:
- raise RuntimeError("ffmpeg is not available")
- result = await _merge(*input_files, output_file=output_file)
- if result.success:
- await async_delete_files(*input_files)
- return result
+@lru_cache
+def which_ffmpeg() -> str | None:
+ return shutil.which("ffmpeg")
- @overload
- @staticmethod
- async def probe(input: Path, /) -> FFprobeResult: ...
- @overload
- @staticmethod
- async def probe(input: URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult: ...
+@lru_cache
+def which_ffprobe() -> str | None:
+ global _FFPROBE_AVAILABLE
+ try:
+ bin_path = shutil.which("ffprobe") or (_builtin_ffprobe() + "[CDL builtin]")
+ _FFPROBE_AVAILABLE = True
+ return bin_path
+ except RuntimeError:
+ return
- @staticmethod
- async def probe(input: Path | URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult:
- if isinstance(input, URL):
- assert is_absolute_http_url(input)
- elif isinstance(input, Path):
- assert input.is_absolute()
- assert not headers
+def get_ffmpeg_version() -> str | None:
+ if bin_path := which_ffmpeg():
+ return _get_bin_version(bin_path)
+
+def get_ffprobe_version() -> str | None:
+ if bin_path := which_ffprobe():
+ return _get_bin_version(bin_path)
+
+
+async def concat(input_files: Sequence[Path], output_file: Path, *, same_folder: bool = True) -> SubProcessResult:
+ concat_file_path = output_file.with_suffix(output_file.suffix + ".ffmpeg_concat.txt")
+ await _create_concat_input_file(input_files, output_file=concat_file_path)
+ result = await _concat(concat_file_path, output_file)
+ if result.success:
+ if same_folder:
+ folder = input_files[0].parent
+ await asyncio.to_thread(shutil.rmtree, folder, ignore_errors=True)
else:
- raise ValueError("Can only probe a Path or a yarl.URL")
+ await _async_delete_files(input_files)
- command = *FFPROBE_CALL_PREFIX, str(input)
- if headers:
- add_headers = []
- for name, value in headers.items():
- add_headers.extend(["-headers", f"{name}: {value}"])
+ await asyncio.to_thread(concat_file_path.unlink)
+ return result
+
+
+async def merge(input_files: Sequence[Path], output_file: Path) -> SubProcessResult:
+ result = await _merge(input_files, output_file=output_file)
+ if result.success:
+ await _async_delete_files(input_files)
+ return result
+
+
+@overload
+async def probe(input: Path, /) -> FFprobeResult: ...
+
+
+@overload
+async def probe(input: URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult: ...
- command = *command, *add_headers
- result = await _run_command(command)
- default: FFprobeOutput = {"streams": []}
- output = json.loads(result.stdout) if result.success else default
- return FFprobeResult(output)
+async def probe(input: Path | URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult:
+ assert _FFPROBE_AVAILABLE
+ if isinstance(input, URL):
+ assert is_absolute_http_url(input)
-async def async_delete_files(*files: Path) -> None:
+ elif isinstance(input, Path):
+ assert input.is_absolute()
+ assert not headers
+
+ else:
+ raise ValueError("Can only probe a Path or a yarl.URL")
+
+ command = *FFPROBE_CALL_PREFIX, str(input)
+ if headers:
+ headers_cmd = itertools.chain.from_iterable(("-headers", f"{name}: {value}") for name, value in headers.items())
+ command = *command, *headers_cmd
+ result = await _run_command(command)
+ output = json.loads(result.stdout) if result.success else _EMPTY_FFPROBE_OUTPUT
+ return FFprobeResult.from_output(output)
+
+
+async def _async_delete_files(files: Sequence[Path]) -> None:
await asyncio.gather(*[aiofiles.os.unlink(file) for file in files])
-async def _create_concat_input_file(*input: Path, file_path: Path) -> None:
- # input paths MUST be absolute!!
- async with aiofiles.open(file_path, "w", encoding="utf8") as f:
- for file in input:
- await f.write(f"file '{file}'\n")
+async def _create_concat_input_file(input_files: Sequence[Path], output_file: Path) -> None:
+ """Input paths MUST be absolute!!."""
+ async with aiofiles.open(output_file, "w", encoding="utf8") as f:
+ await f.writelines(f"file '{file}'\n" for file in input_files)
async def _fixup_concatenated_video_file(input_file: Path, output_file: Path) -> SubProcessResult:
- command = *FFMPEG_CALL_PREFIX, "-i", str(input_file), *FFMPEG_FIXUP_INPUT_ARGS
- probe_result = await FFmpeg.probe(input_file)
- if probe_result and probe_result.audio.codec_name == "aac":
- command += FFMPEG_FIXUP_AUDIO_FILTER_ARGS
- command = *command, str(output_file)
+ command = *FFMPEG_CALL_PREFIX, "-i", input_file, *FFMPEG_FIXUP_INPUT_ARGS
+ probe_result = await probe(input_file)
+ if probe_result and (audio := probe_result.audio) and audio.codec == "aac":
+ command += FFMPEG_FIXUP_AUDIO_DTS_FILTER_ARGS
+ command = *command, output_file
result = await _run_command(command)
if result.success:
await asyncio.to_thread(input_file.unlink)
@@ -123,33 +145,32 @@ async def _fixup_concatenated_video_file(input_file: Path, output_file: Path) ->
async def _concat(concat_input_file: Path, output_file: Path) -> SubProcessResult:
- concatenated_file_name = output_file.with_suffix(".concat" + output_file.suffix)
- command = *FFMPEG_CALL_PREFIX, *CONCAT_INPUT_ARGS, str(concat_input_file), *CODEC_COPY, str(concatenated_file_name)
+ concatenated_file = output_file.with_suffix(".concat" + output_file.suffix)
+ command = *FFMPEG_CALL_PREFIX, *CONCAT_INPUT_ARGS, concat_input_file, *CODEC_COPY, concatenated_file
result = await _run_command(command)
if not result.success:
return result
- return await _fixup_concatenated_video_file(concatenated_file_name, output_file)
+ return await _fixup_concatenated_video_file(concatenated_file, output_file)
-async def _merge(*input_files: Path, output_file: Path) -> SubProcessResult:
- input_args = sum([("-i", str(path)) for path in input_files], ())
- command = *FFMPEG_CALL_PREFIX, *input_args, *MERGE_INPUT_ARGS, *CODEC_COPY, str(output_file)
+async def _merge(input_files: Sequence[Path], output_file: Path) -> SubProcessResult:
+ input_args = itertools.chain.from_iterable(("-i", path) for path in input_files)
+ command = *FFMPEG_CALL_PREFIX, *input_args, *MERGE_INPUT_ARGS, *CODEC_COPY, output_file
return await _run_command(command)
@lru_cache
-def get_ffmpeg_version() -> str | None:
+def _get_bin_version(bin_path: str) -> str | None:
try:
- ffmpeg_path = shutil.which("ffmpeg")
- if not ffmpeg_path:
- return None
- cmd = [ffmpeg_path, "-version"]
- p = subprocess.run(cmd, timeout=5, check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+ cmd = bin_path, "-version"
+ p = subprocess.run(
+ cmd, timeout=5, check=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
+ )
stdout = p.stdout.decode("utf-8", errors="ignore")
except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired, OSError, ValueError):
- return None
+ return
else:
- return stdout.split("version", 1)[-1].split("Copyright")[0].strip()
+ return stdout.partition("version")[-1].partition("Copyright")[0].strip()
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ FFprobe ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -165,7 +186,7 @@ class Duration(NamedTuple):
@staticmethod
def parse(duration: float | str) -> float:
if isinstance(duration, float | int):
- return duration
+ return float(duration)
try:
return float(duration)
except (ValueError, TypeError):
@@ -195,7 +216,7 @@ class FFprobeOutput(TypedDict, total=False):
streams: Required[list[StreamDict]]
-class Tags(CIMultiDict[Any]): ...
+class Tags(CIMultiDictProxy[Any]): ...
class FPS(float):
@@ -203,7 +224,7 @@ class FPS(float):
return str(int(self)) if self.is_integer() else f"{self:.2f}"
-@dataclass(frozen=True, kw_only=True)
+@dataclass(frozen=True, slots=True, kw_only=True)
class Stream:
index: int
codec_name: str
@@ -223,33 +244,33 @@ class Stream:
@classmethod
def validate(cls, stream_info: Mapping[str, Any]) -> dict[str, Any]:
info = get_valid_dict(cls, stream_info)
- tags = Tags(stream_info.get("tags", {}))
+ tags = Tags(CIMultiDict(stream_info.get("tags", {})))
duration: float | str | None = stream_info.get("duration") or tags.get("duration")
bitrate = int(stream_info.get("bitrate") or stream_info.get("bit_rate") or 0) or None
duration = Duration.parse(duration) if duration else None
return info | {"tags": tags, "duration": duration, "bitrate": bitrate}
@classmethod
- def new(cls, stream_info: Mapping[str, Any]) -> Self:
+ def from_dict(cls, stream_info: Mapping[str, Any]) -> Self:
return cls(**cls.validate(stream_info))
def as_dict(self) -> dict[str, Any]:
return asdict(self) | {"tags": dict(self.tags)}
-@dataclass(frozen=True, kw_only=True)
+@dataclass(frozen=True, slots=True, kw_only=True)
class AudioStream(Stream):
sample_rate: int | None
codec_type: Literal["audio"] = "audio"
@classmethod
def validate(cls, stream_info: StreamDict) -> dict[str, Any]:
+ defaults = super(AudioStream, cls).validate(stream_info)
sample_rate = int(float(stream_info.get("sample_rate", 0))) or None
- defaults = super().validate(stream_info)
return defaults | {"sample_rate": sample_rate}
-@dataclass(frozen=True, kw_only=True)
+@dataclass(frozen=True, slots=True, kw_only=True)
class VideoStream(Stream):
width: int | None
height: int | None
@@ -268,24 +289,25 @@ class VideoStream(Stream):
if (avg_fps := stream_info.get("avg_frame_rate")) and str(avg_fps) not in {"0/0", "0", "0.0"}:
fps: FPS | None = FPS(Fraction(avg_fps))
- defaults = super().validate(stream_info)
+ defaults = super(VideoStream, cls).validate(stream_info)
return defaults | {"width": width, "height": height, "fps": fps, "resolution": resolution}
-@dataclass
+@dataclass(frozen=True, slots=True)
class FFprobeResult:
ffprobe_output: FFprobeOutput
- streams: tuple[Stream, ...] = field(init=False)
+ streams: tuple[Stream, ...]
- def __post_init__(self) -> None:
+ @staticmethod
+ def from_output(ffprobe_output: FFprobeOutput) -> FFprobeResult:
streams: list[Stream] = []
- for stream in self.ffprobe_output.get("streams", []):
+ for stream in ffprobe_output.get("streams", []):
if stream["codec_type"] == "video":
- streams.append(VideoStream.new(stream))
+ streams.append(VideoStream.from_dict(stream))
elif stream["codec_type"] == "audio":
- streams.append(AudioStream.new(stream))
+ streams.append(AudioStream.from_dict(stream))
- self.streams = tuple(streams)
+ return FFprobeResult(ffprobe_output, tuple(streams))
def video_streams(self) -> Generator[VideoStream]:
for stream in self.streams:
@@ -298,14 +320,14 @@ class FFprobeResult:
yield stream
@property
- def audio(self) -> AudioStream:
+ def audio(self) -> AudioStream | None:
"""First audio stream"""
- return next(self.audio_streams())
+ return next(self.audio_streams(), None)
@property
- def video(self) -> VideoStream:
+ def video(self) -> VideoStream | None:
"""First video stream"""
- return next(self.video_streams())
+ return next(self.video_streams(), None)
def __bool__(self) -> bool:
return bool(self.ffprobe_output.get("streams"))
@@ -319,21 +341,28 @@ class SubProcessResult(NamedTuple):
stdout: str
stderr: str
success: bool
- command: tuple
+ command: _CMD
- def as_dict(self) -> dict:
- joined_command = " ".join(self.command)
+ def as_jsonable_dict(self) -> dict[str, Any]:
+ joined_command = " ".join(map(str, self.command))
return self._asdict() | {"command": joined_command}
-async def _run_command(command: Sequence[str]) -> SubProcessResult:
- joined_command = " ".join(command)
- log_debug(f"Running command: {joined_command}")
- process = await asyncio.create_subprocess_exec(*command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+async def _run_command(command: _CMD) -> SubProcessResult:
+ assert not isinstance(command, str)
+ bin_path, cmd = command[0], command[1:]
+ if bin_path == "ffmpeg":
+ bin_path = which_ffmpeg()
+ elif bin_path == "ffprobe":
+ bin_path = which_ffprobe()
+ assert bin_path
+ command_ = bin_path, *cmd
+ log_debug(f"Running command: {command_}")
+ process = await asyncio.create_subprocess_exec(*command_, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = await process.communicate()
return_code = process.returncode
stdout_str = stdout.decode("utf-8", errors="ignore")
stderr_str = stderr.decode("utf-8", errors="ignore")
- results = SubProcessResult(return_code, stdout_str, stderr_str, return_code == 0, tuple(command))
- log_debug(results.as_dict())
+ results = SubProcessResult(return_code, stdout_str, stderr_str, return_code == 0, command_)
+ log_debug(results.as_jsonable_dict())
return results