Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add `ffprobe` checks and fallback to builtin binary (#1154)
NTFSvolume9fba56aff4b7
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 15c2aaedb..f6065b31b 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -23,6 +23,7 @@ from cyberdrop_dl.exceptions import (InvalidContentTypeError,RestrictedFiletypeError,)+from cyberdrop_dl.utils import ffmpegfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none, parse_url@@ -150,8 +151,11 @@ class Downloader:@error_handling_wrapperasync def download_hls(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> None:await self.client.mark_incomplete(media_item, self.domain)- if not self.manager.ffmpeg.is_available:- raise DownloadError("FFmpeg Error", "FFmpeg is required for HLS downloads but is not available", media_item)+ try:+ ffmpeg.check_is_available()+ except RuntimeError as e:+ msg = f"{e} - ffmpeg and ffprobe are required for HLS downloads"+ raise DownloadError("FFmpeg Error", msg, media_item) from Nonemedia_item.complete_file = media_item.download_folder / media_item.filenameself.update_queued_files()@@ -162,7 +166,7 @@ class Downloader:await asyncio.to_thread(video.rename, media_item.complete_file)else:parts = [part for part in (video, audio, subtitles) if part]- ffmpeg_result = await self.manager.ffmpeg.merge(*parts, output_file=media_item.complete_file)+ ffmpeg_result = await ffmpeg.merge(parts, media_item.complete_file)if not ffmpeg_result.success:raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)@@ -192,7 +196,7 @@ class Downloader:seg_paths = [item.complete_file for item in items if item.complete_file]output = media_item.complete_file.with_suffix(f".{media_type}.ts")- ffmpeg_result = await self.manager.ffmpeg.concat(*seg_paths, output_file=output)+ ffmpeg_result = await ffmpeg.concat(seg_paths, output)if not ffmpeg_result.success:raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)results.append(output)diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex cff809143..d3a182743 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -22,8 +22,8 @@ from cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.managers.realdebrid_manager import RealDebridManagerfrom cyberdrop_dl.managers.storage_manager import StorageManagerfrom cyberdrop_dl.ui.textual import TextualUI+from cyberdrop_dl.utils import ffmpegfrom cyberdrop_dl.utils.args import ParsedArgs, parse_args-from cyberdrop_dl.utils.ffmpeg import FFmpeg, get_ffmpeg_versionfrom cyberdrop_dl.utils.logger import QueuedLogger, logfrom cyberdrop_dl.utils.transfer import transfer_v5_db_to_v6from cyberdrop_dl.utils.utilities import close_if_defined, get_system_information@@ -58,7 +58,6 @@ class Manager:self.progress_manager: ProgressManager = field(init=False)self.live_manager: LiveManager = field(init=False)self.textual_log_queue: queue.Queue = field(init=False)- self.ffmpeg: FFmpeg = field(init=False)self._textual_ui: TextualUI = field(init=False)self._loaded_args_config: bool = False@@ -170,9 +169,6 @@ class Manager:if not isinstance(self.real_debrid_manager, RealDebridManager):self.real_debrid_manager = RealDebridManager(self)- if not isinstance(self.ffmpeg, FFmpeg):- self.ffmpeg = FFmpeg(self)-await self.async_db_hash_startup()constants.MAX_NAME_LENGTHS["FILE"] = self.config_manager.global_settings_data.general.max_file_name_length@@ -229,19 +225,23 @@ class Manager:cli_only_args = self.parsed_args.cli_only_args.model_dump_json(indent=4)system_info = get_system_information()- log("Starting Cyberdrop-DL Process")- log(f"Running Version: {__version__}")- log(f"System Info:{system_info}")- log(f"Using Config: {self.config_manager.loaded_config}")- log(f"Using Config File: {self.config_manager.settings}")- log(f"Using Input File: {self.path_manager.input_file}")- log(f"Using Download Folder: {self.path_manager.download_folder}")- log(f"Using Database File: {self.path_manager.history_db}")- log(f"Using CLI only options: {cli_only_args}")- log(f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}")- log(f"Using Settings: \n{config_settings}")- log(f"Using Global Settings: \n{global_settings}")- log(f"Using FFmpeg version: {get_ffmpeg_version()}")+ args_info = (+ "Starting Cyberdrop-DL Process",+ f"Running Version: {__version__}",+ f"System Info:{system_info}",+ f"Using Config: {self.config_manager.loaded_config}",+ f"Using Config File: {self.config_manager.settings}",+ f"Using Input File: {self.path_manager.input_file}",+ f"Using Download Folder: {self.path_manager.download_folder}",+ f"Using Database File: {self.path_manager.history_db}",+ f"Using CLI only options: {cli_only_args}",+ f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}",+ f"Using Settings: \n{config_settings}",+ f"Using Global Settings: \n{global_settings}",+ f"Using ffmpeg version: {ffmpeg.get_ffmpeg_version()}",+ f"Using ffprobe version: {ffmpeg.get_ffprobe_version()}",+ )+ log("\n".join(args_info))async def async_db_close(self) -> None:"Partial shutdown for managers used for hash directory scanner"diff --git a/cyberdrop_dl/utils/ffmpeg.py b/cyberdrop_dl/utils/ffmpeg.pyindex 556057b3d..6bf9d6c1f 100644--- a/cyberdrop_dl/utils/ffmpeg.py+++ b/cyberdrop_dl/utils/ffmpeg.py@@ -1,19 +1,21 @@from __future__ import annotationsimport asyncio+import itertoolsimport jsonimport shutilimport subprocess-from dataclasses import asdict, dataclass, field-from datetime import datetime, timedelta+from dataclasses import asdict, dataclass+from datetime import timedeltafrom fractions import Fractionfrom functools import lru_cachefrom pathlib import Path-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Required, Self, TypedDict, overload+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Required, Self, TypeAlias, TypedDict, overloadimport aiofilesimport aiofiles.os-from multidict import CIMultiDict+from multidict import CIMultiDict, CIMultiDictProxy+from videoprops import which_ffprobe as _builtin_ffprobefrom yarl import URLfrom cyberdrop_dl.utils.logger import log_debug@@ -22,100 +24,120 @@ from cyberdrop_dl.utils.utilities import get_valid_dict, is_absolute_http_urlif TYPE_CHECKING:from collections.abc import Generator, Mapping, Sequence- from cyberdrop_dl.managers.manager import Manager+ _CMD: TypeAlias = Sequence[str | Path]-FFPROBE_CALL_PREFIX = "ffprobe", "-hide_banner", "-loglevel", "error", "-show_streams", "-print_format", "json"-FFMPEG_CALL_PREFIX = "ffmpeg", "-y", "-loglevel", "error"-FFMPEG_FIXUP_INPUT_ARGS = "-map", "0", "-ignore_unknown", "-c", "copy", "-f", "mp4", "-movflags", "+faststart"-FFMPEG_FIXUP_AUDIO_FILTER_ARGS = "-bsf:a", "aac_adtstoasc"+CODEC_COPY = "-c", "copy"MERGE_INPUT_ARGS = "-map", "0"CONCAT_INPUT_ARGS = "-f", "concat", "-safe", "0", "-i"-CODEC_COPY = "-c", "copy"+FFMPEG_CALL_PREFIX = "ffmpeg", "-y", "-loglevel", "error"+FFMPEG_FIXUP_INPUT_ARGS = "-map", "0", "-ignore_unknown", *CODEC_COPY, "-f", "mp4", "-movflags", "+faststart"+FFMPEG_FIXUP_AUDIO_DTS_FILTER_ARGS = "-bsf:a", "aac_adtstoasc"+FFPROBE_CALL_PREFIX = "ffprobe", "-hide_banner", "-loglevel", "error", "-show_streams", "-print_format", "json"+_FFPROBE_AVAILABLE = False+_EMPTY_FFPROBE_OUTPUT: FFprobeOutput = {"streams": []}-class FFmpeg:- def __init__(self, manager: Manager) -> None:- self.manager = manager- self.cache_folder = self.manager.path_manager.cache_folder.resolve()- self.version = get_ffmpeg_version()- self.is_available = bool(self.version)-- async def concat(self, *input_files: Path, output_file: Path, same_folder: bool = True) -> SubProcessResult:- if not self.is_available:- raise RuntimeError("ffmpeg is not available")- now = datetime.now().strftime("%Y%m%d_%H%M%S")- concat_file_name = f"{now} - {output_file.name[:40]}.ffmpeg_input.txt"- concat_file_path = self.cache_folder / concat_file_name- await _create_concat_input_file(*input_files, file_path=concat_file_path)- result = await _concat(concat_file_path, output_file)- if result.success:- if same_folder:- folder = input_files[0].parent- await asyncio.to_thread(shutil.rmtree, folder, True)- else:- await async_delete_files(concat_file_path, *input_files)- return result+def check_is_available() -> None:+ if not get_ffmpeg_version():+ raise RuntimeError("ffmpeg is not available")+ if not get_ffprobe_version():+ raise RuntimeError("ffprobe is not available")- async def merge(self, *input_files: Path, output_file: Path) -> SubProcessResult:- if not self.is_available:- raise RuntimeError("ffmpeg is not available")- result = await _merge(*input_files, output_file=output_file)- if result.success:- await async_delete_files(*input_files)- return result+@lru_cache+def which_ffmpeg() -> str | None:+ return shutil.which("ffmpeg")- @overload- @staticmethod- async def probe(input: Path, /) -> FFprobeResult: ...- @overload- @staticmethod- async def probe(input: URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult: ...+@lru_cache+def which_ffprobe() -> str | None:+ global _FFPROBE_AVAILABLE+ try:+ bin_path = shutil.which("ffprobe") or (_builtin_ffprobe() + "[CDL builtin]")+ _FFPROBE_AVAILABLE = True+ return bin_path+ except RuntimeError:+ return- @staticmethod- async def probe(input: Path | URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult:- if isinstance(input, URL):- assert is_absolute_http_url(input)- elif isinstance(input, Path):- assert input.is_absolute()- assert not headers+def get_ffmpeg_version() -> str | None:+ if bin_path := which_ffmpeg():+ return _get_bin_version(bin_path)++def get_ffprobe_version() -> str | None:+ if bin_path := which_ffprobe():+ return _get_bin_version(bin_path)+++async def concat(input_files: Sequence[Path], output_file: Path, *, same_folder: bool = True) -> SubProcessResult:+ concat_file_path = output_file.with_suffix(output_file.suffix + ".ffmpeg_concat.txt")+ await _create_concat_input_file(input_files, output_file=concat_file_path)+ result = await _concat(concat_file_path, output_file)+ if result.success:+ if same_folder:+ folder = input_files[0].parent+ await asyncio.to_thread(shutil.rmtree, folder, ignore_errors=True)else:- raise ValueError("Can only probe a Path or a yarl.URL")+ await _async_delete_files(input_files)- command = *FFPROBE_CALL_PREFIX, str(input)- if headers:- add_headers = []- for name, value in headers.items():- add_headers.extend(["-headers", f"{name}: {value}"])+ await asyncio.to_thread(concat_file_path.unlink)+ return result+++async def merge(input_files: Sequence[Path], output_file: Path) -> SubProcessResult:+ result = await _merge(input_files, output_file=output_file)+ if result.success:+ await _async_delete_files(input_files)+ return result+++@overload+async def probe(input: Path, /) -> FFprobeResult: ...+++@overload+async def probe(input: URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult: ...- command = *command, *add_headers- result = await _run_command(command)- default: FFprobeOutput = {"streams": []}- output = json.loads(result.stdout) if result.success else default- return FFprobeResult(output)+async def probe(input: Path | URL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult:+ assert _FFPROBE_AVAILABLE+ if isinstance(input, URL):+ assert is_absolute_http_url(input)-async def async_delete_files(*files: Path) -> None:+ elif isinstance(input, Path):+ assert input.is_absolute()+ assert not headers++ else:+ raise ValueError("Can only probe a Path or a yarl.URL")++ command = *FFPROBE_CALL_PREFIX, str(input)+ if headers:+ headers_cmd = itertools.chain.from_iterable(("-headers", f"{name}: {value}") for name, value in headers.items())+ command = *command, *headers_cmd+ result = await _run_command(command)+ output = json.loads(result.stdout) if result.success else _EMPTY_FFPROBE_OUTPUT+ return FFprobeResult.from_output(output)+++async def _async_delete_files(files: Sequence[Path]) -> None:await asyncio.gather(*[aiofiles.os.unlink(file) for file in files])-async def _create_concat_input_file(*input: Path, file_path: Path) -> None:- # input paths MUST be absolute!!- async with aiofiles.open(file_path, "w", encoding="utf8") as f:- for file in input:- await f.write(f"file '{file}'\n")+async def _create_concat_input_file(input_files: Sequence[Path], output_file: Path) -> None:+ """Input paths MUST be absolute!!."""+ async with aiofiles.open(output_file, "w", encoding="utf8") as f:+ await f.writelines(f"file '{file}'\n" for file in input_files)async def _fixup_concatenated_video_file(input_file: Path, output_file: Path) -> SubProcessResult:- command = *FFMPEG_CALL_PREFIX, "-i", str(input_file), *FFMPEG_FIXUP_INPUT_ARGS- probe_result = await FFmpeg.probe(input_file)- if probe_result and probe_result.audio.codec_name == "aac":- command += FFMPEG_FIXUP_AUDIO_FILTER_ARGS- command = *command, str(output_file)+ command = *FFMPEG_CALL_PREFIX, "-i", input_file, *FFMPEG_FIXUP_INPUT_ARGS+ probe_result = await probe(input_file)+ if probe_result and (audio := probe_result.audio) and audio.codec == "aac":+ command += FFMPEG_FIXUP_AUDIO_DTS_FILTER_ARGS+ command = *command, output_fileresult = await _run_command(command)if result.success:await asyncio.to_thread(input_file.unlink)@@ -123,33 +145,32 @@ async def _fixup_concatenated_video_file(input_file: Path, output_file: Path) ->async def _concat(concat_input_file: Path, output_file: Path) -> SubProcessResult:- concatenated_file_name = output_file.with_suffix(".concat" + output_file.suffix)- command = *FFMPEG_CALL_PREFIX, *CONCAT_INPUT_ARGS, str(concat_input_file), *CODEC_COPY, str(concatenated_file_name)+ concatenated_file = output_file.with_suffix(".concat" + output_file.suffix)+ command = *FFMPEG_CALL_PREFIX, *CONCAT_INPUT_ARGS, concat_input_file, *CODEC_COPY, concatenated_fileresult = await _run_command(command)if not result.success:return result- return await _fixup_concatenated_video_file(concatenated_file_name, output_file)+ return await _fixup_concatenated_video_file(concatenated_file, output_file)-async def _merge(*input_files: Path, output_file: Path) -> SubProcessResult:- input_args = sum([("-i", str(path)) for path in input_files], ())- command = *FFMPEG_CALL_PREFIX, *input_args, *MERGE_INPUT_ARGS, *CODEC_COPY, str(output_file)+async def _merge(input_files: Sequence[Path], output_file: Path) -> SubProcessResult:+ input_args = itertools.chain.from_iterable(("-i", path) for path in input_files)+ command = *FFMPEG_CALL_PREFIX, *input_args, *MERGE_INPUT_ARGS, *CODEC_COPY, output_filereturn await _run_command(command)@lru_cache-def get_ffmpeg_version() -> str | None:+def _get_bin_version(bin_path: str) -> str | None:try:- ffmpeg_path = shutil.which("ffmpeg")- if not ffmpeg_path:- return None- cmd = [ffmpeg_path, "-version"]- p = subprocess.run(cmd, timeout=5, check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)+ cmd = bin_path, "-version"+ p = subprocess.run(+ cmd, timeout=5, check=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL+ )stdout = p.stdout.decode("utf-8", errors="ignore")except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired, OSError, ValueError):- return None+ returnelse:- return stdout.split("version", 1)[-1].split("Copyright")[0].strip()+ return stdout.partition("version")[-1].partition("Copyright")[0].strip()# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ FFprobe ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@@ -165,7 +186,7 @@ class Duration(NamedTuple):@staticmethoddef parse(duration: float | str) -> float:if isinstance(duration, float | int):- return duration+ return float(duration)try:return float(duration)except (ValueError, TypeError):@@ -195,7 +216,7 @@ class FFprobeOutput(TypedDict, total=False):streams: Required[list[StreamDict]]-class Tags(CIMultiDict[Any]): ...+class Tags(CIMultiDictProxy[Any]): ...class FPS(float):@@ -203,7 +224,7 @@ class FPS(float):return str(int(self)) if self.is_integer() else f"{self:.2f}"-@dataclass(frozen=True, kw_only=True)+@dataclass(frozen=True, slots=True, kw_only=True)class Stream:index: intcodec_name: str@@ -223,33 +244,33 @@ class Stream:@classmethoddef validate(cls, stream_info: Mapping[str, Any]) -> dict[str, Any]:info = get_valid_dict(cls, stream_info)- tags = Tags(stream_info.get("tags", {}))+ tags = Tags(CIMultiDict(stream_info.get("tags", {})))duration: float | str | None = stream_info.get("duration") or tags.get("duration")bitrate = int(stream_info.get("bitrate") or stream_info.get("bit_rate") or 0) or Noneduration = Duration.parse(duration) if duration else Nonereturn info | {"tags": tags, "duration": duration, "bitrate": bitrate}@classmethod- def new(cls, stream_info: Mapping[str, Any]) -> Self:+ def from_dict(cls, stream_info: Mapping[str, Any]) -> Self:return cls(**cls.validate(stream_info))def as_dict(self) -> dict[str, Any]:return asdict(self) | {"tags": dict(self.tags)}-@dataclass(frozen=True, kw_only=True)+@dataclass(frozen=True, slots=True, kw_only=True)class AudioStream(Stream):sample_rate: int | Nonecodec_type: Literal["audio"] = "audio"@classmethoddef validate(cls, stream_info: StreamDict) -> dict[str, Any]:+ defaults = super(AudioStream, cls).validate(stream_info)sample_rate = int(float(stream_info.get("sample_rate", 0))) or None- defaults = super().validate(stream_info)return defaults | {"sample_rate": sample_rate}-@dataclass(frozen=True, kw_only=True)+@dataclass(frozen=True, slots=True, kw_only=True)class VideoStream(Stream):width: int | Noneheight: int | None@@ -268,24 +289,25 @@ class VideoStream(Stream):if (avg_fps := stream_info.get("avg_frame_rate")) and str(avg_fps) not in {"0/0", "0", "0.0"}:fps: FPS | None = FPS(Fraction(avg_fps))- defaults = super().validate(stream_info)+ defaults = super(VideoStream, cls).validate(stream_info)return defaults | {"width": width, "height": height, "fps": fps, "resolution": resolution}-@dataclass+@dataclass(frozen=True, slots=True)class FFprobeResult:ffprobe_output: FFprobeOutput- streams: tuple[Stream, ...] = field(init=False)+ streams: tuple[Stream, ...]- def __post_init__(self) -> None:+ @staticmethod+ def from_output(ffprobe_output: FFprobeOutput) -> FFprobeResult:streams: list[Stream] = []- for stream in self.ffprobe_output.get("streams", []):+ for stream in ffprobe_output.get("streams", []):if stream["codec_type"] == "video":- streams.append(VideoStream.new(stream))+ streams.append(VideoStream.from_dict(stream))elif stream["codec_type"] == "audio":- streams.append(AudioStream.new(stream))+ streams.append(AudioStream.from_dict(stream))- self.streams = tuple(streams)+ return FFprobeResult(ffprobe_output, tuple(streams))def video_streams(self) -> Generator[VideoStream]:for stream in self.streams:@@ -298,14 +320,14 @@ class FFprobeResult:yield stream@property- def audio(self) -> AudioStream:+ def audio(self) -> AudioStream | None:"""First audio stream"""- return next(self.audio_streams())+ return next(self.audio_streams(), None)@property- def video(self) -> VideoStream:+ def video(self) -> VideoStream | None:"""First video stream"""- return next(self.video_streams())+ return next(self.video_streams(), None)def __bool__(self) -> bool:return bool(self.ffprobe_output.get("streams"))@@ -319,21 +341,28 @@ class SubProcessResult(NamedTuple):stdout: strstderr: strsuccess: bool- command: tuple+ command: _CMD- def as_dict(self) -> dict:- joined_command = " ".join(self.command)+ def as_jsonable_dict(self) -> dict[str, Any]:+ joined_command = " ".join(map(str, self.command))return self._asdict() | {"command": joined_command}-async def _run_command(command: Sequence[str]) -> SubProcessResult:- joined_command = " ".join(command)- log_debug(f"Running command: {joined_command}")- process = await asyncio.create_subprocess_exec(*command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)+async def _run_command(command: _CMD) -> SubProcessResult:+ assert not isinstance(command, str)+ bin_path, cmd = command[0], command[1:]+ if bin_path == "ffmpeg":+ bin_path = which_ffmpeg()+ elif bin_path == "ffprobe":+ bin_path = which_ffprobe()+ assert bin_path+ command_ = bin_path, *cmd+ log_debug(f"Running command: {command_}")+ process = await asyncio.create_subprocess_exec(*command_, stdout=subprocess.PIPE, stderr=subprocess.PIPE)stdout, stderr = await process.communicate()return_code = process.returncodestdout_str = stdout.decode("utf-8", errors="ignore")stderr_str = stderr.decode("utf-8", errors="ignore")- results = SubProcessResult(return_code, stdout_str, stderr_str, return_code == 0, tuple(command))- log_debug(results.as_dict())+ results = SubProcessResult(return_code, stdout_str, stderr_str, return_code == 0, command_)+ log_debug(results.as_jsonable_dict())return results