Skip to content

Bulk asynchronous downloader for multiple file hosts

8.0 KB
Raw Blame
from __future__ import annotations

import contextlib
import datetime as dt
import logging
import os
import sys
import time
from typing import TYPE_CHECKING, Any, Self, final

from cyberdrop_dl import __version__, aio, constants, dependencies, env, ffmpeg, stats
from cyberdrop_dl.cache import cache_context
from cyberdrop_dl.clients.downloads import DownloadClient
from cyberdrop_dl.clients.http import HTTPClient
from cyberdrop_dl.config import Config
from cyberdrop_dl.config.appdata import AppData
from cyberdrop_dl.database import Database
from cyberdrop_dl.dedupe import Czkawka
from cyberdrop_dl.hasher import Hasher
from cyberdrop_dl.logs import LoggerAdapter, capture_logs, log_spacer
from cyberdrop_dl.models.validators import bytesize_to_str
from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLED
from cyberdrop_dl.signature import dataclass_repr
from cyberdrop_dl.sorter import Sorter
from cyberdrop_dl.url_objects import FinishedDownload
from cyberdrop_dl.utils import enter_context, get_system_information

if TYPE_CHECKING:
    from collections.abc import Generator
    from pathlib import Path

    from cyberdrop_dl.commands import CLIarguments
    from cyberdrop_dl.scrape_mapper import ScrapeMapper, ScrapeStats
    from cyberdrop_dl.url_objects import MediaItem


logger = LoggerAdapter(logging.getLogger(__name__), extra={"cdl_no_truncate": True})


@final
@dataclass_repr("cli_args", "_config", "http_client", "download_client")
class Manager:
    def __init__(
        self,
        cli_args: CLIarguments | None = None,
        appdata: AppData | None = None,
        config: Config | None = None,
    ) -> None:
        from cyberdrop_dl.commands import CLIarguments

        self.cache: dict[str, Any] = {}
        self._appdata: AppData | None = appdata
        self.cli_args: CLIarguments = cli_args or CLIarguments()
        self._config: Config | None = config

        self._completed: list[FinishedDownload] = []
        self._hasher: Hasher | None = None

        self.http_client = HTTPClient(self.config)

        self.download_client: DownloadClient = DownloadClient(self)
        self.scrape_mapper: ScrapeMapper
        self.database: Database
        self.deduper: Czkawka
        self.sorter: Sorter
        self.exit_stack: contextlib.ExitStack = contextlib.ExitStack()

    @property
    def hasher(self) -> Hasher:
        if self._hasher is None:
            self._hasher = self.exit_stack.enter_context(Hasher.create(self.config, self.database))
        return self._hasher

    @property
    def appdata(self) -> AppData:
        if self._appdata is None:
            self._appdata = AppData.default()
        return self._appdata

    @property
    def config(self) -> Config:
        if self._config is None:
            self._config = Config.from_file(self.appdata.config_file)
        return self._config

    def __resolve_paths(self) -> None:
        self.appdata.mkdirs()
        self.config.resolve_paths()

    @contextlib.contextmanager
    def __call__(self) -> Generator[Self]:
        self.__resolve_paths()
        self.database = Database(self.appdata.db_file, self.config.ignore_history)
        self.deduper = Czkawka.from_manager(self)
        self.sorter = Sorter.from_config(self.config)
        with (
            self.exit_stack,
            cache_context(self.appdata.cache_file, self.cache),
            enter_context(REFRESH_RATE, self.config.ui.refresh_rate),
            enter_context(TUI_DISABLED, self.config.ui.mode.is_disabled),
        ):
            try:
                yield self
            finally:
                del self.deduper
                del self.sorter

    async def post_download(self, item: MediaItem) -> None:
        if item.is_segment or not item.downloaded:
            return

        if self.config.hashing.mode is constants.HashMode.IN_PLACE:
            await self.hasher.hash_item(item)

        self._completed.append(FinishedDownload.from_item(item))

    @property
    def completed_downloads(self) -> list[FinishedDownload]:
        return self._completed

    def log_config_settings(self) -> None:
        logger.info(f"Running cyberdrop-dl v{__version__}")
        _log_enviroment()
        logger.debug({"CLI options": self.cli_args.__json__()})
        _log_config(self.config)
        _log_ffmpeg()
        _log_database(self.appdata.db_file)
        _log_dependencies()

    def print_stats(self, stats: ScrapeStats) -> str:
        if not self.config.ui.show_stats:
            return ""

        log_spacer()

        with capture_logs() as stream:
            self.__print_stats(stats)

        return stream.getvalue()

    def __print_stats(self, scrape_stats: ScrapeStats) -> None:

        scrape_time = time.monotonic() - scrape_stats.start_time
        startup_time = scrape_stats.start_time - constants.START_TIME

        def format_td(seconds: float) -> str:
            seconds, millis = divmod(seconds, 1)
            formatted = str(dt.timedelta(seconds=seconds))
            if millis:
                formatted += f"{str(millis)[1:5]}"
            return formatted

        logger.info("Run Stats:", extra={"color": "cyan"})
        logger.info(f"  Config file: {self.config.source}")
        logger.info(f"  URLs source: {scrape_stats.source}")
        logger.info(f"  URLs: {scrape_stats.count:,}")
        logger.info(f"  URL groups: {len(scrape_stats.unique_groups):,}")
        logger.info(f"  Logs folder: {self.config.logs.effective_log_folder}")
        logger.info(
            f"  Total runtime: {format_td(scrape_time + startup_time)} (startup: {startup_time:.03f}s, scrape: {format_td(scrape_time)})",
        )
        logger.info("  Total downloaded data: %s", bytesize_to_str(self.scrape_mapper.tui.downloads.bytes_downloaded))

        if scrape_stats.domain_stats:
            log_spacer()
            logger.info("URLs by domain (includes children):", extra={"color": "cyan"})

            def lines() -> Generator[str]:
                for domain, count in sorted(scrape_stats.domain_stats.items()):
                    yield f" - {domain}: {count:,}"

            logger.info("\n".join(lines()))

        stats.print(self.scrape_mapper.tui.files.stats)
        stats.print(self.scrape_mapper.tui.scrape_errors)
        stats.print(self.hasher.stats)
        stats.print(self.deduper.stats)
        stats.print(self.sorter.stats)
        stats.print_errors(
            tuple(self.scrape_mapper.tui.scrape_errors),
            tuple(self.scrape_mapper.tui.download_errors),
        )

    async def get_cookie_files(self) -> list[Path]:
        path = self.config.cookies
        if not path:
            return []

        if await aio.is_file(path):
            return [path]

        if await aio.is_dir(path):
            return [f async for f in aio.glob(path, "*.txt")]

        return []


def _log_dependencies() -> None:
    if not env.DEBUG_MODE:
        return
    logger.debug({"dependencies": dict(dependencies())})


def _log_database(path: Path) -> None:
    try:
        db_size = path.stat().st_size
    except FileNotFoundError:
        db_size = 0

    logger.debug("Database file: %s (%s)", path, bytesize_to_str(db_size))


def _log_enviroment() -> None:
    logger.debug(
        {
            "System": get_system_information(),
            "Enviroment": env.ALL_VARS,
            "Enviroment resolved": env.ALL_VARS_RESOLVED,
            "argv": sys.argv[1:],
        }
    )


def _log_config(config: Config) -> None:
    logger.debug("Config file: %s", config.source)
    logger.debug("Auth: \n%s", config.auth.censored_dump())
    logger.debug(config.model_dump_json(exclude={"auth"}, indent=2))


def _log_ffmpeg() -> None:
    if not ffmpeg.is_installed():
        msg = "ffmpeg is not installed. HLS downloads will fail"
        if os.name == "nt":
            msg += ". Get it from: https://www.gyan.dev/ffmpeg/builds/"

        logger.warning(msg)
        return

    logger.debug(
        {
            "ffmpeg": {"binary": ffmpeg.which_ffmpeg(), "version": ffmpeg.version()},
            "ffprobe": {"binary": ffmpeg.which_ffprobe(), "version": ffmpeg.ffprobe_version()},
        }
    )