Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: switch to id based logging (ffmpeg) (#1753)
NTFSvolumed56554d4ea22
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 82f7af4c3..eead4e684 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -143,14 +143,12 @@ class Downloader:if media_item.url.path in self.processed_items and not self._ignore_history:return- try:- ffmpeg.check_is_available()- except RuntimeError:+ if not ffmpeg.is_installed():msg = "ffmpeg is not installed. (Required for HLS downloads)"if os.name == "nt":msg += ". Get it from: https://www.gyan.dev/ffmpeg/builds/"- raise DownloadError("FFmpeg Error", msg, media_item) from None+ raise DownloadError("FFmpeg Error", msg, media_item)async with self._download_context(media_item):await self._start_hls_download(media_item, m3u8_group)diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex 70f0f0523..dc4725acc 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -17,15 +17,15 @@ def _env(name: str, *, censor: bool = False) -> str | None:RUNNING_IN_TERMUX = bool(os.getenv("TERMUX_VERSION") or os.getenv("TERMUX_MAIN_PACKAGE_FORMAT") or "com.termux" in os.getenv("$PREFIX", ""))-PORTRAIT_MODE = bool(RUNNING_IN_TERMUX or _env("PORTRAIT_MODE"))+PORTRAIT_MODE = bool(_env("PORTRAIT_MODE") or RUNNING_IN_TERMUX)DEBUG_LOG_FOLDER = _env("DEBUG_LOG_FOLDER")MAX_CRAWLER_ERRORS = int(_env("MAX_CRAWLER_ERRORS") or 10)DEBUG_MODE = bool(- DEBUG_LOG_FOLDER- or _env("DEBUG_MODE")+ _env("DEBUG_MODE")+ or DEBUG_LOG_FOLDERor os.getenv("PYCHARM_HOSTED")or os.getenv("TERM_PROGRAM") in ("vscode", "zed"))diff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.pyindex 07a2b877c..7b50f27b8 100644--- a/cyberdrop_dl/ffmpeg.py+++ b/cyberdrop_dl/ffmpeg.py@@ -8,6 +8,7 @@ import jsonimport loggingimport shutilimport subprocess+import uuidfrom fractions import Fractionfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias, TypedDict, overload@@ -17,7 +18,7 @@ from multidict import CIMultiDict, CIMultiDictProxyfrom cyberdrop_dl.utils import DictDataclassif TYPE_CHECKING:- from collections.abc import Generator, Iterable, Iterator, Mapping+ from collections.abc import Generator, Iterable, Iterator, Mapping, Sequencefrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -48,7 +49,15 @@ _FFPROBE_CALL_PREFIX = ()-def check_is_available() -> None:+def is_installed() -> bool:+ try:+ _check()+ return True+ except RuntimeError:+ return False+++def _check() -> None:if not version():raise RuntimeError("ffmpeg is not installed")_check_ffprobe()@@ -388,12 +397,13 @@ class SubProcessResult:return_code: int | Nonestdout: strstderr: str- success: bool- command: _CMD++ @property+ def success(self) -> bool:+ return self.return_code == 0def __json__(self) -> dict[str, Any]:- joined_command = " ".join(map(str, self.command))- me = dataclasses.asdict(self) | {"command": joined_command}+ me = dataclasses.asdict(self)try:stdout = json.loads(self.stdout)except json.JSONDecodeError:@@ -406,24 +416,26 @@ class SubProcessResult:return str(self.__json__())-async def _run_command(command: _CMD) -> SubProcessResult:+async def _run_command(command: Sequence[str | Path]) -> SubProcessResult:assert not isinstance(command, str)program, *cmd = command+if program == "ffmpeg":- program = which_ffmpeg()+ bin = which_ffmpeg()elif program == "ffprobe":- program = which_ffprobe()+ bin = which_ffprobe()+ else:+ raise ValueError(f"Unexpected program in command {command}")- assert program- logger.debug(f"Running command: {program, *cmd}")- process = await asyncio.create_subprocess_exec(program, *cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)+ assert bin+ process_id = str(uuid.uuid4())+ logger.debug("Running %s subprocess [id=%s]\n:%s", program, process_id, {"command": [bin, *map(str, cmd)]})+ process = await asyncio.create_subprocess_exec(bin, *cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)stdout, stderr = await process.communicate()result = SubProcessResult(- process.returncode,stdout=stdout.decode("utf-8", errors="ignore"),stderr=stderr.decode("utf-8", errors="ignore"),- success=process.returncode == 0,- command=cmd,+ return_code=process.returncode,)- logger.debug(result)+ logger.debug("%s subprocess [id=%s] output:\n%s", program, process_id, result)return resultdiff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.pyindex 2b0adea02..e0aecae3c 100644--- a/cyberdrop_dl/managers/manager.py+++ b/cyberdrop_dl/managers/manager.py@@ -125,13 +125,27 @@ class Manager:"CLI options": self.cli_args.model_dump(mode="json"),"Auth": auth,"Settings": config_settings.model_dump(mode="json"),- "Global Settings": self.config.global_settings.model_dump(mode="json"),+ "Global settings": self.config.global_settings.model_dump(mode="json"),"Enviroment": env.ALL_VARS,"Enviroment resolved": env.ALL_VARS_RESOLVED,"argv": tuple(sys.argv[1:]),})+ if ffmpeg.is_installed():+ logger.debug(+ {+ "ffmpeg": {+ "binary": ffmpeg.which_ffmpeg(),+ "version": ffmpeg.version(),+ },+ "ffprobe": {+ "binary": ffmpeg.which_ffprobe(),+ "version": ffmpeg.ffprobe_version(),+ },+ }+ )+try:db_size = self.appdata.db_file.stat().st_sizeexcept FileNotFoundError:@@ -139,11 +153,7 @@ class Manager:logger.debug("Database size: %s", ByteSize(db_size).human_readable(decimal=True))- if ffmpeg.version():- logger.debug("ffmpeg version: %s", ffmpeg.version())- logger.debug("ffprobe version: %s", ffmpeg.ffprobe_version())-- else:+ if not ffmpeg.is_installed():msg = "ffmpeg is not installed. HLS downloads will fail"if os.name == "nt":msg += ". Get it from: https://www.gyan.dev/ffmpeg/builds/"diff --git a/cyberdrop_dl/sorter.py b/cyberdrop_dl/sorter.pyindex 26857fcd7..caaf547c5 100644--- a/cyberdrop_dl/sorter.py+++ b/cyberdrop_dl/sorter.py@@ -199,7 +199,7 @@ class Sorter:)dest = await asyncio.to_thread(_move_file, file, dest, self.incrementer_format)if dest:- logger.warning("Moved '{}' to '{}'", file, dest)+ logger.debug("Moved '{}' to '{}'", file, dest)else:self.tui.stats.errors += 1return bool(dest)