Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: use std loggging for ffmpeg (#30)
NTFSvolume8ba2b27af79b
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 9911b335a..501619041 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVarfrom aiohttp import ClientConnectorError, ClientError, ClientResponseError-from cyberdrop_dl import constants+from cyberdrop_dl import constants, ffmpegfrom cyberdrop_dl.data_structures.url_objects import HlsSegment, MediaItemfrom cyberdrop_dl.exceptions import (DownloadError,@@ -26,7 +26,7 @@ from cyberdrop_dl.exceptions import (SkipDownloadError,TooManyCrawlerErrors,)-from cyberdrop_dl.utils import aio, ffmpeg+from cyberdrop_dl.utils import aiofrom cyberdrop_dl.utils.logger import log, log_debugfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, parse_url@@ -229,13 +229,16 @@ class Downloader:async def _download_rendition_group(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> tuple[Path, Path | None, Path | None]:++ temp_dir = media_item.complete_file.with_suffix(constants.TempExt.HLS)+async def download(m3u8: M3U8):assert m3u8.media_typeif not m3u8.segments:raise DownloadError(204, f"{m3u8.media_type} m3u8 manifest ({m3u8.base_uri}) has no valid segments")- download_folder = media_item.complete_file.with_suffix(constants.TempExt.HLS) / m3u8.media_type- coros = self._prepare_hls_downloads(media_item, m3u8, download_folder)+ download_folder = temp_dir / m3u8.media_type+n_segmets = len(m3u8.segments)if n_segmets > 1:suffix = f".{m3u8.media_type}.ts"@@ -247,7 +250,10 @@ class Downloader:return outputbatch_size = _VIDEO_HLS_BATCH_SIZE if m3u8.media_type == "video" else _AUDIO_HLS_BATCH_SIZE- tasks_results = await aio.gather(coros, batch_size=batch_size)++ tasks_results = await aio.gather(+ self._prepare_hls_downloads(media_item, m3u8, download_folder), batch_size=batch_size+ )n_successful = sum(1 for result in tasks_results if result.downloaded)if n_successful != n_segmets:@@ -261,7 +267,7 @@ class Downloader:if not ffmpeg_result.success:raise DownloadError("FFmpeg Concat Error", ffmpeg_result.stderr, media_item)else:- await asyncio.to_thread(seg_paths[0].rename, output)+ _ = await asyncio.to_thread(seg_paths[0].rename, output)return outputaudio = subtitles = None@@ -279,6 +285,10 @@ class Downloader:if m3u8_group.audio:audio = await download(m3u8_group.audio)video = await download(m3u8_group.video)+ try:+ await asyncio.to_thread(temp_dir.rmdir)+ except OSError:+ passreturn video, audio, subtitlesdef _prepare_hls_downloads(diff --git a/cyberdrop_dl/utils/ffmpeg.py b/cyberdrop_dl/ffmpeg.pysimilarity index 56%rename from cyberdrop_dl/utils/ffmpeg.pyrename to cyberdrop_dl/ffmpeg.pyindex f7c149b4f..65a461523 100644--- a/cyberdrop_dl/utils/ffmpeg.py+++ b/cyberdrop_dl/ffmpeg.py@@ -1,30 +1,30 @@from __future__ import annotationsimport asyncio+import dataclassesimport functoolsimport itertoolsimport json+import loggingimport shutilimport subprocess-from dataclasses import asdict, dataclassfrom fractions import Fractionfrom pathlib import Path-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Required, Self, TypeAlias, TypedDict, overload+from typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias, TypedDict, overload-import aiofiles-import aiofiles.osfrom multidict import CIMultiDict, CIMultiDictProxy-from yarl import URL-from cyberdrop_dl.utils.logger import log_debug-from cyberdrop_dl.utils.utilities import get_valid_dict, is_absolute_http_url+from cyberdrop_dl.utils.utilities import get_valid_dictif TYPE_CHECKING:- from collections.abc import Generator, Mapping, Sequence+ from collections.abc import Generator, Iterable, Iterator, Mappingfrom cyberdrop_dl.data_structures import AbsoluteHttpURL- _CMD: TypeAlias = Sequence[str | Path]+ _CMD: TypeAlias = Iterable[str | Path]+++logger = logging.getLogger(__name__)class Args:@@ -46,7 +46,6 @@ _FFPROBE_CALL_PREFIX = ("-print_format","json",)-_EMPTY_FFPROBE_OUTPUT: FFprobeOutput = {"streams": []}def check_is_available() -> None:@@ -55,7 +54,7 @@ def check_is_available() -> None:_check_ffprobe()-def _check_ffprobe():+def _check_ffprobe() -> None:if not get_ffprobe_version():raise RuntimeError("ffprobe is not available")@@ -82,102 +81,117 @@ def get_ffprobe_version() -> str | None:return _get_bin_version(bin_path)-async def concat(input_files: Sequence[Path], output_file: Path, *, same_folder: bool = True) -> SubProcessResult:- concat_file_path = output_file.with_suffix(output_file.suffix + ".ffmpeg_concat.txt")- await _create_concat_input_file(input_files, output_file=concat_file_path)- result = await _concat(concat_file_path, output_file)+async def merge(input_files: Iterable[Path], output_file: Path) -> SubProcessResult:+ result = await _merge(input_files, output_file)if result.success:- if same_folder:- folder = input_files[0].parent- await asyncio.to_thread(shutil.rmtree, folder, ignore_errors=True)- else:- await _async_delete_files(input_files)-- await asyncio.to_thread(concat_file_path.unlink)+ await _delete_files(input_files, same_folder=False)return result-async def merge(input_files: Sequence[Path], output_file: Path) -> SubProcessResult:- result = await _merge(input_files, output_file=output_file)- if result.success:- await _async_delete_files(input_files)- return result---@overload-async def probe(input: Path, /) -> FFprobeResult: ...---@overload-async def probe(input: AbsoluteHttpURL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult: ...+async def _merge(input_files: Iterable[Path], output_file: Path) -> SubProcessResult:+ inputs = itertools.chain.from_iterable(("-i", path) for path in input_files)+ command = *_FFMPEG_CALL_PREFIX, *inputs, *Args.MAP_ALL_STREAMS, *Args.CODEC_COPY, output_file+ return await _run_command(command)-async def probe(input: Path | AbsoluteHttpURL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult:- _check_ffprobe()- if isinstance(input, URL):- assert is_absolute_http_url(input)+async def concat(input_files: Iterable[Path], output_file: Path, *, same_folder: bool = True) -> SubProcessResult:+ concat_file = output_file.with_suffix(output_file.suffix + ".ffmpeg_concat.txt")+ await _create_concat_file(input_files, output_file=concat_file)+ try:+ result = await _concat(concat_file, output_file)+ if result.success:+ await _delete_files(input_files, same_folder=same_folder)+ finally:+ await _try_delete(concat_file)- elif isinstance(input, Path):- assert input.is_absolute()- assert not headers+ return result- else:- raise ValueError("Can only probe a Path or a yarl.URL")- command = *_FFPROBE_CALL_PREFIX, str(input)- if headers:- headers_cmd = itertools.chain.from_iterable(("-headers", f"{name}: {value}") for name, value in headers.items())- command = *command, *headers_cmd+async def _concat(input: Path, output: Path) -> SubProcessResult:+ concatenated_file = output.with_suffix(".concat" + output.suffix)+ command = *_FFMPEG_CALL_PREFIX, *Args.CONCAT, input, *Args.CODEC_COPY, concatenated_fileresult = await _run_command(command)- output = json.loads(result.stdout) if result.success else _EMPTY_FFPROBE_OUTPUT- return FFprobeResult.from_output(output)+ if not result.success:+ return result+ return await _fixup_concat_video(concatenated_file, output)-async def _async_delete_files(files: Sequence[Path]) -> None:- await asyncio.gather(*[aiofiles.os.unlink(file) for file in files])+async def _create_concat_file(input_files: Iterable[Path], output_file: Path) -> None:+ # Input paths MUST be absolute!!.+ def write() -> None:+ with output_file.open("w", encoding="utf8") as f:+ f.writelines(f"file '{file}'\n" for file in input_files)-async def _create_concat_input_file(input_files: Sequence[Path], output_file: Path) -> None:- """Input paths MUST be absolute!!."""- async with aiofiles.open(output_file, "w", encoding="utf8") as f:- await f.writelines(f"file '{file}'\n" for file in input_files)+ return await asyncio.to_thread(write)-async def _fixup_concatenated_video_file(input_file: Path, output_file: Path) -> SubProcessResult:+async def _fixup_concat_video(input_file: Path, output_file: Path) -> SubProcessResult:command = *_FFMPEG_CALL_PREFIX, "-i", input_file, *Args.FIXUP_MP4probe_result = await probe(input_file)if probe_result and (audio := probe_result.audio) and audio.codec == "aac":- command += Args.FIXUP_AUDIO_DTS_FILTER+ command = *command, *Args.FIXUP_AUDIO_DTS_FILTERcommand = *command, output_fileresult = await _run_command(command)if result.success:- await asyncio.to_thread(input_file.unlink)+ await _try_delete(input_file)return result-async def _concat(concat_input_file: Path, output_file: Path) -> SubProcessResult:- concatenated_file = output_file.with_suffix(".concat" + output_file.suffix)- command = *_FFMPEG_CALL_PREFIX, *Args.CONCAT, concat_input_file, *Args.CODEC_COPY, concatenated_file- result = await _run_command(command)- if not result.success:- return result- return await _fixup_concatenated_video_file(concatenated_file, output_file)+async def _try_delete(file: Path) -> None:+ try:+ await asyncio.to_thread(file.unlink, missing_ok=True)+ except OSError as e:+ logger.warning(f"Unable to delete '{file}' {e}")-async def _merge(input_files: Sequence[Path], output_file: Path) -> SubProcessResult:- inputs = itertools.chain.from_iterable(("-i", path) for path in input_files)- command = *_FFMPEG_CALL_PREFIX, *inputs, *Args.MAP_ALL_STREAMS, *Args.CODEC_COPY, output_file- return await _run_command(command)+async def _delete_files(files: Iterable[Path], *, same_folder: bool) -> None:+ if same_folder:+ folder = next(iter(files)).parent+ await asyncio.to_thread(shutil.rmtree, folder, ignore_errors=True)+ else:+ _ = await asyncio.gather(*map(_try_delete, files))+++@overload+async def probe(input: Path, /) -> FFprobeResult: ...+++@overload+async def probe(input: AbsoluteHttpURL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult: ...+++async def probe(input: Path | AbsoluteHttpURL, /, *, headers: Mapping[str, str] | None = None) -> FFprobeResult:+ _check_ffprobe()++ if isinstance(input, Path):+ assert input.is_absolute()+ assert not headers++ command = *_FFPROBE_CALL_PREFIX, str(input)+ if headers:+ command = (+ *command,+ *itertools.chain.from_iterable(("-headers", f"{name}: {value}") for name, value in headers.items()),+ )+ result = await _run_command(command)+ if not result.success:+ return _EMPTY_FFPROBE_RESULT+ return FFprobeResult.from_output(json.loads(result.stdout))def _get_bin_version(bin_path: str) -> str | None:try:- cmd = bin_path, "-version"- p = subprocess.run(- cmd, timeout=5, check=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL- )- stdout = p.stdout.decode("utf-8", errors="ignore")- except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired, OSError, ValueError):+ stdout = subprocess.run(+ (bin_path, "-version"),+ timeout=5,+ check=True,+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ ).stdout.decode("utf-8", errors="ignore")++ except Exception:returnelse:return stdout.partition("version")[-1].partition("Copyright")[0].strip()@@ -191,15 +205,16 @@ def _parse_duration(duration: str | float | None) -> TruncatedFloat | None:return Noneif isinstance(duration, (float, int)):- seconds = duration+ seconds: float | int = durationelse:try:- *rest, seconds = duration.strip().split(":")+ *rest, seconds_str = duration.strip().split(":")- seconds = float(seconds)+ seconds = float(seconds_str)for idx, value in enumerate(reversed(rest), 1):seconds += int(value) * 60**idx+except Exception:return None@@ -207,13 +222,13 @@ def _parse_duration(duration: str | float | None) -> TruncatedFloat | None:return TruncatedFloat(seconds)-class StreamDict(TypedDict, total=False):- index: Required[int]- codec_type: Required[Literal["video", "audio", "subtitle"]]+class StreamDict(TypedDict):+ index: int+ codec_type: Literal["video", "audio", "subtitle"]-class FFprobeOutput(TypedDict, total=False):- streams: Required[list[StreamDict]]+class FFprobeOutput(TypedDict):+ streams: list[StreamDict]class Tags(CIMultiDictProxy[Any]): ...@@ -224,7 +239,7 @@ class TruncatedFloat(float):return str(int(self)) if self.is_integer() else f"{self:.2f}"-@dataclass(frozen=True, slots=True, kw_only=True)[email protected](slots=True, kw_only=True)class Stream:index: intcodec: str@@ -248,14 +263,11 @@ class Stream:def from_dict(cls, stream_info: StreamDict) -> Self:return cls(**cls.validate(stream_info))- def as_jsonable_dict(self) -> dict[str, Any]:- return asdict(self) | {"tags": dict(self.tags)}--@dataclass(frozen=True, slots=True, kw_only=True)[email protected](slots=True, kw_only=True)class AudioStream(Stream):sample_rate: int | None- codec_type: Literal["audio"] = "audio"+ codec_type: Literal["audio"] = "audio" # pyright: ignore[reportIncompatibleVariableOverride]@classmethoddef validate(cls, stream_info: StreamDict) -> dict[str, Any]:@@ -264,13 +276,13 @@ class AudioStream(Stream):return defaults | {"sample_rate": sample_rate}-@dataclass(frozen=True, slots=True, kw_only=True)[email protected](slots=True, kw_only=True)class VideoStream(Stream):width: int | Noneheight: int | Nonefps: TruncatedFloat | Noneresolution: str | None- codec_type: Literal["video"] = "video"+ codec_type: Literal["video"] = "video" # pyright: ignore[reportIncompatibleVariableOverride]@classmethoddef validate(cls, stream_info: StreamDict) -> dict[str, Any]:@@ -280,14 +292,14 @@ class VideoStream(Stream):if width and height:resolution: str | None = f"{width}x{height}"- if (avg_fps := stream_info.get("avg_frame_rate")) and str(avg_fps) not in {"0/0", "0", "0.0"}:+ if (avg_fps := stream_info.get("avg_frame_rate")) and str(avg_fps) not in ("0/0", "0", "0.0"):fps: TruncatedFloat | None = TruncatedFloat(Fraction(avg_fps))defaults = super(VideoStream, cls).validate(stream_info)return defaults | {"width": width, "height": height, "fps": fps, "resolution": resolution}-@dataclass(frozen=True, slots=True, kw_only=True)[email protected](slots=True)class Format:size: int | Nonebitrate: int | None@@ -306,20 +318,38 @@ class Format:)-@dataclass(frozen=True, slots=True)[email protected](slots=True)class FFprobeResult:ffprobe_output: FFprobeOutput- streams: tuple[Stream, ...]+ streams: tuple[VideoStream | AudioStream, ...]format: Format+ audio: AudioStream | None = dataclasses.field(init=False)+ """First audio stream"""+ video: VideoStream | None = dataclasses.field(init=False)+ """First video stream"""++ def __post_init__(self) -> None:+ self.audio = next(self.audio_streams(), None)+ self.video = next(self.video_streams(), None)++ def __bool__(self) -> bool:+ return bool(self.streams)++ def __iter__(self) -> Iterator[Stream]:+ return iter(self.streams)+@staticmethoddef from_output(ffprobe_output: FFprobeOutput) -> FFprobeResult:def streams():- for stream in ffprobe_output.get("streams", []):- if stream["codec_type"] == "video":- yield VideoStream.from_dict(stream)- elif stream["codec_type"] == "audio":- yield AudioStream.from_dict(stream)+ for stream in ffprobe_output.get("streams", ()):+ match stream["codec_type"]:+ case "video":+ yield VideoStream.from_dict(stream)+ case "audio":+ yield AudioStream.from_dict(stream)+ case _:+ passreturn FFprobeResult(ffprobe_output,@@ -329,58 +359,54 @@ class FFprobeResult:def video_streams(self) -> Generator[VideoStream]:for stream in self.streams:- if isinstance(stream, VideoStream):+ if stream.codec_type == "video":yield streamdef audio_streams(self) -> Generator[AudioStream]:for stream in self.streams:- if isinstance(stream, AudioStream):+ if stream.codec_type == "audio":yield stream- @property- def audio(self) -> AudioStream | None:- """First audio stream"""- return next(self.audio_streams(), None)-- @property- def video(self) -> VideoStream | None:- """First video stream"""- return next(self.video_streams(), None)-- def __bool__(self) -> bool:- return bool(self.ffprobe_output.get("streams"))+_EMPTY_FFPROBE_RESULT: FFprobeResult = FFprobeResult.from_output({"streams": []})# ~~~~~~~~~~~~~~~~~~~~~~ Subprocess ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-class SubProcessResult(NamedTuple):[email protected](slots=True)+class SubProcessResult:return_code: int | Nonestdout: strstderr: strsuccess: boolcommand: _CMD- def as_jsonable_dict(self) -> dict[str, Any]:+ def __json__(self) -> dict[str, Any]:joined_command = " ".join(map(str, self.command))- return self._asdict() | {"command": joined_command}+ return dataclasses.asdict(self) | {"command": joined_command}++ def __str__(self) -> str:+ return str(self.__json__())async def _run_command(command: _CMD) -> SubProcessResult:assert not isinstance(command, str)- bin_path, cmd = command[0], command[1:]- if bin_path == "ffmpeg":- bin_path = which_ffmpeg()- elif bin_path == "ffprobe":- bin_path = which_ffprobe()- assert bin_path- command_ = bin_path, *cmd- log_debug(f"Running command: {command_}")- process = await asyncio.create_subprocess_exec(*command_, stdout=subprocess.PIPE, stderr=subprocess.PIPE)+ program, *cmd = command+ if program == "ffmpeg":+ program = which_ffmpeg()+ elif program == "ffprobe":+ program = which_ffprobe()++ assert program+ logger.debug(f"Running command: {program, *cmd}")+ process = await asyncio.create_subprocess_exec(program, *cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)stdout, stderr = await process.communicate()- return_code = process.returncode- stdout_str = stdout.decode("utf-8", errors="ignore")- stderr_str = stderr.decode("utf-8", errors="ignore")- results = SubProcessResult(return_code, stdout_str, stderr_str, return_code == 0, command_)- log_debug(results.as_jsonable_dict())- return results+ result = SubProcessResult(+ process.returncode,+ stdout=stdout.decode("utf-8", errors="ignore"),+ stderr=stderr.decode("utf-8", errors="ignore"),+ success=process.returncode == 0,+ command=cmd,+ )+ logger.debug(result)+ return resultdiff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 7bf752250..912dc08c4 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -21,10 +21,10 @@ from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.clients.scraper_client import ScraperClientfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, ScrapeError, TooManyCrawlerErrors+from cyberdrop_dl.ffmpeg import probefrom cyberdrop_dl.ui.prompts.user_prompts import get_cookies_from_browsersfrom cyberdrop_dl.utils.aio import WeakAsyncLocksfrom cyberdrop_dl.utils.cookie_management import read_netscape_files-from cyberdrop_dl.utils.ffmpeg import probefrom cyberdrop_dl.utils.logger import log, log_debug, log_spacer_VALID_EXTENSIONS = (diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex ca87a508c..918d159f8 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, NamedTuple, TypeVarfrom pydantic import BaseModel-from cyberdrop_dl import __version__, constants+from cyberdrop_dl import __version__, constants, ffmpegfrom cyberdrop_dl.cli import ParsedArgs, parse_argsfrom cyberdrop_dl.database import Databasefrom cyberdrop_dl.database.transfer import transfer_v5_db_to_v6@@ -21,7 +21,6 @@ from cyberdrop_dl.managers.log_manager import LogManagerfrom cyberdrop_dl.managers.path_manager import PathManagerfrom cyberdrop_dl.managers.progress_manager import ProgressManagerfrom cyberdrop_dl.managers.storage_manager import StorageManager-from cyberdrop_dl.utils import ffmpegfrom cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, logfrom cyberdrop_dl.utils.utilities import close_if_defined, get_system_informationdiff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex f817f44f8..60b350b9f 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -11,8 +11,8 @@ import imagesizefrom cyberdrop_dl import constantsfrom cyberdrop_dl.constants import FILE_FORMATS+from cyberdrop_dl.ffmpeg import probefrom cyberdrop_dl.utils import strings-from cyberdrop_dl.utils.ffmpeg import probefrom cyberdrop_dl.utils.logger import log, log_with_colorfrom cyberdrop_dl.utils.utilities import purge_dir_treediff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.pyindex 659052375..0e64b6982 100644--- a/tests/test_ffmpeg.py+++ b/tests/test_ffmpeg.py@@ -2,8 +2,8 @@ import datetimeimport pytest+from cyberdrop_dl import ffmpegfrom cyberdrop_dl.data_structures import AbsoluteHttpURL-from cyberdrop_dl.utils import ffmpegFFPROBE_IS_INSTALLED = bool(ffmpeg.get_ffprobe_version())