Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: switch to id based logging (ffmpeg) (#1753)

NTFSvolumed56554d4ea22
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 82f7af4c3..eead4e684 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -143,14 +143,12 @@ class Downloader:
if media_item.url.path in self.processed_items and not self._ignore_history:
return
- try:
- ffmpeg.check_is_available()
- except RuntimeError:
+ if not ffmpeg.is_installed():
msg = "ffmpeg is not installed. (Required for HLS downloads)"
if os.name == "nt":
msg += ". Get it from: https://www.gyan.dev/ffmpeg/builds/"
- raise DownloadError("FFmpeg Error", msg, media_item) from None
+ raise DownloadError("FFmpeg Error", msg, media_item)
async with self._download_context(media_item):
await self._start_hls_download(media_item, m3u8_group)
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index 70f0f0523..dc4725acc 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -17,15 +17,15 @@ def _env(name: str, *, censor: bool = False) -> str | None:
RUNNING_IN_TERMUX = bool(
os.getenv("TERMUX_VERSION") or os.getenv("TERMUX_MAIN_PACKAGE_FORMAT") or "com.termux" in os.getenv("$PREFIX", "")
)
-PORTRAIT_MODE = bool(RUNNING_IN_TERMUX or _env("PORTRAIT_MODE"))
+PORTRAIT_MODE = bool(_env("PORTRAIT_MODE") or RUNNING_IN_TERMUX)
DEBUG_LOG_FOLDER = _env("DEBUG_LOG_FOLDER")
MAX_CRAWLER_ERRORS = int(_env("MAX_CRAWLER_ERRORS") or 10)
DEBUG_MODE = bool(
- DEBUG_LOG_FOLDER
- or _env("DEBUG_MODE")
+ _env("DEBUG_MODE")
+ or DEBUG_LOG_FOLDER
or os.getenv("PYCHARM_HOSTED")
or os.getenv("TERM_PROGRAM") in ("vscode", "zed")
)
diff --git a/cyberdrop_dl/ffmpeg.py b/cyberdrop_dl/ffmpeg.py
index 07a2b877c..7b50f27b8 100644
--- a/cyberdrop_dl/ffmpeg.py
+++ b/cyberdrop_dl/ffmpeg.py
@@ -8,6 +8,7 @@ import json
import logging
import shutil
import subprocess
+import uuid
from fractions import Fraction
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Self, TypeAlias, TypedDict, overload
@@ -17,7 +18,7 @@ from multidict import CIMultiDict, CIMultiDictProxy
from cyberdrop_dl.utils import DictDataclass
if TYPE_CHECKING:
- from collections.abc import Generator, Iterable, Iterator, Mapping
+ from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -48,7 +49,15 @@ _FFPROBE_CALL_PREFIX = (
)
-def check_is_available() -> None:
+def is_installed() -> bool:
+ try:
+ _check()
+ return True
+ except RuntimeError:
+ return False
+
+
+def _check() -> None:
if not version():
raise RuntimeError("ffmpeg is not installed")
_check_ffprobe()
@@ -388,12 +397,13 @@ class SubProcessResult:
return_code: int | None
stdout: str
stderr: str
- success: bool
- command: _CMD
+
+ @property
+ def success(self) -> bool:
+ return self.return_code == 0
def __json__(self) -> dict[str, Any]:
- joined_command = " ".join(map(str, self.command))
- me = dataclasses.asdict(self) | {"command": joined_command}
+ me = dataclasses.asdict(self)
try:
stdout = json.loads(self.stdout)
except json.JSONDecodeError:
@@ -406,24 +416,26 @@ class SubProcessResult:
return str(self.__json__())
-async def _run_command(command: _CMD) -> SubProcessResult:
+async def _run_command(command: Sequence[str | Path]) -> SubProcessResult:
assert not isinstance(command, str)
program, *cmd = command
+
if program == "ffmpeg":
- program = which_ffmpeg()
+ bin = which_ffmpeg()
elif program == "ffprobe":
- program = which_ffprobe()
+ bin = which_ffprobe()
+ else:
+ raise ValueError(f"Unexpected program in command {command}")
- assert program
- logger.debug(f"Running command: {program, *cmd}")
- process = await asyncio.create_subprocess_exec(program, *cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ assert bin
+ process_id = str(uuid.uuid4())
+ logger.debug("Running %s subprocess [id=%s]\n:%s", program, process_id, {"command": [bin, *map(str, cmd)]})
+ process = await asyncio.create_subprocess_exec(bin, *cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = await process.communicate()
result = SubProcessResult(
- process.returncode,
stdout=stdout.decode("utf-8", errors="ignore"),
stderr=stderr.decode("utf-8", errors="ignore"),
- success=process.returncode == 0,
- command=cmd,
+ return_code=process.returncode,
)
- logger.debug(result)
+ logger.debug("%s subprocess [id=%s] output:\n%s", program, process_id, result)
return result
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 2b0adea02..e0aecae3c 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -125,13 +125,27 @@ class Manager:
"CLI options": self.cli_args.model_dump(mode="json"),
"Auth": auth,
"Settings": config_settings.model_dump(mode="json"),
- "Global Settings": self.config.global_settings.model_dump(mode="json"),
+ "Global settings": self.config.global_settings.model_dump(mode="json"),
"Enviroment": env.ALL_VARS,
"Enviroment resolved": env.ALL_VARS_RESOLVED,
"argv": tuple(sys.argv[1:]),
}
)
+ if ffmpeg.is_installed():
+ logger.debug(
+ {
+ "ffmpeg": {
+ "binary": ffmpeg.which_ffmpeg(),
+ "version": ffmpeg.version(),
+ },
+ "ffprobe": {
+ "binary": ffmpeg.which_ffprobe(),
+ "version": ffmpeg.ffprobe_version(),
+ },
+ }
+ )
+
try:
db_size = self.appdata.db_file.stat().st_size
except FileNotFoundError:
@@ -139,11 +153,7 @@ class Manager:
logger.debug("Database size: %s", ByteSize(db_size).human_readable(decimal=True))
- if ffmpeg.version():
- logger.debug("ffmpeg version: %s", ffmpeg.version())
- logger.debug("ffprobe version: %s", ffmpeg.ffprobe_version())
-
- else:
+ if not ffmpeg.is_installed():
msg = "ffmpeg is not installed. HLS downloads will fail"
if os.name == "nt":
msg += ". Get it from: https://www.gyan.dev/ffmpeg/builds/"
diff --git a/cyberdrop_dl/sorter.py b/cyberdrop_dl/sorter.py
index 26857fcd7..caaf547c5 100644
--- a/cyberdrop_dl/sorter.py
+++ b/cyberdrop_dl/sorter.py
@@ -199,7 +199,7 @@ class Sorter:
)
dest = await asyncio.to_thread(_move_file, file, dest, self.incrementer_format)
if dest:
- logger.warning("Moved '{}' to '{}'", file, dest)
+ logger.debug("Moved '{}' to '{}'", file, dest)
else:
self.tui.stats.errors += 1
return bool(dest)