Skip to content

Bulk asynchronous downloader for multiple file hosts

16.1 KB
Raw Blame
from __future__ import annotations

import base64
import contextlib
import copy
import dataclasses
import datetime as dt
import logging
from enum import IntEnum
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Self, final, overload

import yarl
from typing_extensions import TypeIs

from cyberdrop_dl import signature
from cyberdrop_dl.exceptions import MaxChildrenError
from cyberdrop_dl.filepath import sanitize_folder

if TYPE_CHECKING:
    from collections.abc import Awaitable, Callable, Generator, Iterable, Mapping, Sequence

    @final
    class AbsoluteHttpURL(yarl.URL):
        @signature.copy(yarl.URL.__new__)
        def __new__(cls) -> Self: ...

        @signature.copy(yarl.URL.__truediv__)
        def __truediv__(self, name: str) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.__mod__)
        def __mod__(self, query: object) -> AbsoluteHttpURL: ...

        @property
        def host(self) -> str: ...  # pyright: ignore[reportIncompatibleVariableOverride]

        @property
        def scheme(self) -> Literal["http", "https"]: ...  # pyright: ignore[reportIncompatibleVariableOverride]

        @property
        def absolute(self) -> Literal[True]: ...  # pyright: ignore[reportIncompatibleVariableOverride]

        @property
        def parent(self) -> AbsoluteHttpURL: ...  # pyright: ignore[reportIncompatibleVariableOverride]

        @signature.copy(yarl.URL.with_path)
        def with_path(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.with_host)
        def with_host(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.origin)
        def origin(self) -> AbsoluteHttpURL: ...

        @overload
        def with_query(self, query: yarl.Query) -> AbsoluteHttpURL: ...

        @overload
        def with_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.with_query)
        def with_query(self) -> AbsoluteHttpURL: ...

        @overload
        def extend_query(self, query: yarl.Query) -> AbsoluteHttpURL: ...

        @overload
        def extend_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.extend_query)
        def extend_query(self) -> AbsoluteHttpURL: ...

        @overload
        def update_query(self, query: yarl.Query) -> AbsoluteHttpURL: ...

        @overload
        def update_query(self, **kwargs: yarl.QueryVariable) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.update_query)
        def update_query(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.without_query_params)
        def without_query_params(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.with_fragment)
        def with_fragment(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.with_name)
        def with_name(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.with_suffix)
        def with_suffix(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.join)
        def join(self) -> AbsoluteHttpURL: ...

        @signature.copy(yarl.URL.joinpath)
        def joinpath(self) -> AbsoluteHttpURL: ...

    class _FakePath(Path): ...

else:
    AbsoluteHttpURL = yarl.URL

    def _FakePath():  # noqa: N802
        return None


def is_absolute_http_url(url: yarl.URL) -> TypeIs[AbsoluteHttpURL]:
    return url.absolute and url.scheme in {"http", "https"}


class ScrapeItemType(IntEnum):
    FORUM = 0
    FORUM_POST = 1
    PROFILE = 2
    ALBUM = 3


logger = logging.getLogger(__name__)


@dataclasses.dataclass(slots=True, frozen=True)
class ByteRange:
    start: int
    end: int | None = None

    def __add__(self, offset: int) -> Self:
        start = self.start + offset
        if self.end and self.end >= start:
            raise ValueError(f"Unsatisfiable BYTE RANGE {offset = } current_range = {self}")

        return type(self)(start, self.end)

    def __str__(self) -> str:
        return f"bytes={self.start}-{self.end - 1 if self.end else ''}"


@dataclasses.dataclass(slots=True, frozen=True)
class MuxVideo:
    video: AbsoluteHttpURL
    audio: AbsoluteHttpURL

    def __json__(self) -> dict[str, Any]:
        return {"video": str(self.video), "audio": str(self.audio)}


@final
@dataclasses.dataclass(slots=True, kw_only=True, frozen=True)
class FinishedDownload:
    # Simplified version of media item w no metadata, bounded callables and some attributes removed, to reduce memory usage
    url: AbsoluteHttpURL
    referer: AbsoluteHttpURL
    original_filename: str
    ext: str
    size: int
    xxhash: str | None
    path: Path
    id: tuple[str, ...]

    def __hash__(self) -> int:
        return hash(self.id)

    @property
    def is_segment(self) -> Literal[False]:
        return False

    @classmethod
    def from_item(cls, item: MediaItem) -> Self:
        assert item.size is not None
        assert item.is_segment is False
        return cls(
            url=item.url,
            referer=item.referer,
            original_filename=item.original_filename,
            ext=item.ext,
            size=item.size,
            xxhash=item.xxhash,
            path=item.path,
            id=item.id,
        )


@final
@signature.dataclass_repr("url", "id", "debrid_url", "uploaded_at_date")
@dataclasses.dataclass(slots=True, kw_only=True, repr=False)
class MediaItem:
    url: AbsoluteHttpURL
    domain: str
    referer: AbsoluteHttpURL
    download_folder: Path
    db_path: str
    filename: str

    original_filename: str = ""
    download_filename: str | None = None
    ext: str = ""
    size: int | None = None
    debrid_url: Callable[[], Awaitable[AbsoluteHttpURL]] | AbsoluteHttpURL | None = None
    duration: float | None = None
    is_segment: bool = False
    album_id: str | None = None
    uploaded_at: int | None = None
    xxhash: str | None = None
    thumbnail: AbsoluteHttpURL | None = None

    parents: tuple[AbsoluteHttpURL, ...] = dataclasses.field(default_factory=tuple)
    attempts: int = dataclasses.field(init=False, default=0)
    partial_file: Path = dataclasses.field(init=False, default=_FakePath())
    path: Path = dataclasses.field(init=False, default=_FakePath())
    downloaded: bool = dataclasses.field(init=False, default=False)

    metadata: object = dataclasses.field(init=False, default_factory=dict)

    uploaded_at_date: dt.datetime | None = dataclasses.field(init=False, default=None)
    extra_info: dict[str, Any] = dataclasses.field(init=False, default_factory=dict)

    id: tuple[str, ...] = dataclasses.field(init=False)
    base64_id: str = dataclasses.field(init=False)
    headers: dict[str, str] = dataclasses.field(init=False, default_factory=dict)
    json_check: Callable[..., None] | None = None

    def __post_init__(self) -> None:
        self.ext = self.ext or Path(self.filename).suffix
        self.id = self.domain, self.db_path
        self.original_filename = self.original_filename or self.filename
        if self.url.scheme == "metadata":
            self.db_path = ""
            self.id = *self.id, "metadata"

        if self.uploaded_at:
            assert type(self.uploaded_at) is int, f"Invalid {self.uploaded_at =!r} from {self.referer}"
            self.uploaded_at_date = dt.datetime.fromtimestamp(self.uploaded_at, tz=dt.UTC)

        self.base64_id = base64.urlsafe_b64encode("".join(self.id).encode()).decode().rstrip("=")

    def __hash__(self) -> int:
        return hash(self.id)

    @property
    def real_url(self) -> AbsoluteHttpURL:
        if self.debrid_url and not callable(self.debrid_url):
            return self.debrid_url
        return self.url

    async def resolve(self) -> AbsoluteHttpURL:
        if callable(self.debrid_url):
            self.debrid_url = await self.debrid_url()

        return self.debrid_url or self.url

    def __iter__(self) -> Generator[tuple[str, Any]]:
        for field in dataclasses.fields(self):
            if field.name in ("is_segment", "json_check"):
                continue
            yield field.name, getattr(self, field.name)

    def serialize(self) -> dict[str, Any]:
        me = dict(self)
        if callable(self.debrid_url):
            me["debrid_url"] = None
        if self.xxhash:
            me["xxhash"] = f"xxh128:{self.xxhash}"
        return me

    def as_segment(self, filename: str, url: AbsoluteHttpURL) -> MediaItem:
        me = MediaItem(
            url=url,
            domain=self.domain,
            download_folder=self.download_folder,
            filename=filename,
            db_path=self.db_path,
            referer=self.url,
            album_id=self.album_id,
            ext=self.ext,
            parents=self.parents,
            uploaded_at=self.uploaded_at,
            is_segment=True,
        )
        me.headers = self.headers.copy()
        me.extra_info = self.extra_info.copy()
        return me


@dataclasses.dataclass(slots=True, frozen=True)
class RetryInfo:
    domain: str
    url_path: str
    download_path: Path
    download_filename: str
    referer: AbsoluteHttpURL


@final
@signature.dataclass_repr("url", "folders", "uploaded_at")
@dataclasses.dataclass(kw_only=True, slots=True, repr=False)
class ScrapeItem:
    url: AbsoluteHttpURL
    part_of_album: bool = False
    album_id: str | None = None
    download_folder: Path = Path("downloads")

    folders: list[str] = dataclasses.field(default_factory=list)
    parents: list[AbsoluteHttpURL] = dataclasses.field(default_factory=list)
    parent_threads: set[AbsoluteHttpURL] = dataclasses.field(default_factory=set)
    max_children: Mapping[ScrapeItemType, int] | None = None
    password: str | None = None
    referer: AbsoluteHttpURL | None = None

    _children_count: int = 0
    _children_limit: int = 0
    _type: ScrapeItemType | None = None
    _uploaded_at: int | None = None
    retry_info: RetryInfo | None = None
    markers: list[Any] = dataclasses.field(default_factory=list)

    @classmethod
    def from_url(cls, url: yarl.URL | str) -> Self:
        url = AbsoluteHttpURL(url)
        assert is_absolute_http_url(url)
        get = url.query.get
        try:
            referer = _url_from_query(get("referer"))
        except ValueError:
            logger.exception(
                "Unable to parse query 'referer' value (%s) from url %s , ignoring...", get("referer"), url
            )
            referer = None

        return cls(url=url, password=get("password"), referer=referer)

    @property
    def uploaded_at(self) -> int | None:
        return self._uploaded_at

    @uploaded_at.setter
    def uploaded_at(self, value: float | None) -> None:  # pyright: ignore[reportPropertyTypeMismatch]
        self._uploaded_at = None if value is None else int(value)

    @property
    def upload_date(self) -> dt.datetime | None:
        if self._uploaded_at:
            return dt.datetime.fromtimestamp(self._uploaded_at, tz=dt.UTC)

    @upload_date.setter
    def upload_date(self, date: dt.datetime | None) -> None:
        if date:
            self.uploaded_at = date.timestamp()

    @property
    def type(self) -> ScrapeItemType | None:
        return self._type

    @type.setter  # noqa: A003
    def type(self, item_type: ScrapeItemType | None) -> None:
        self._type = item_type
        self._children_count = self._children_limit = 0
        if item_type is None or not self.max_children:
            return

        try:
            self._children_limit = self.max_children[item_type]
        except LookupError:
            pass

    @property
    @contextlib.contextmanager
    def track_changes(self) -> Generator[Self]:
        old_url = self.url
        try:
            yield self
        finally:
            if old_url != self.url:
                logger.info(f"URL transformation applied: \n  {old_url = !s}\n  new_url = {self.url}")

    def append_folders(self, *folders: str) -> None:
        for folder in folders:
            self._append_folder(folder)

    def _append_folder(self, folder: str, /) -> None:
        if not folder:
            return

        folder = sanitize_folder(folder)
        if _has_domain(folder) and (last_domain := _extract_last_domain(self.folders)):
            folder = _remove_domain_if_duplicate(folder, last_domain)

        self.folders.append(folder)

    def add_children(self, number: int = 1) -> None:
        self._children_count += number
        if self._children_limit and self._children_count >= self._children_limit:
            raise MaxChildrenError(origin=self)

    def reset(self, *, reset_parents: bool = False, reset_parent_title: bool = False) -> None:
        """Resets `album_id`, `type` and `posible_datetime` back to `None`

        Reset `part_of_album` back to `False`
        """
        self.album_id = self.uploaded_at = None
        self.type = None
        self.part_of_album = False
        if reset_parents:
            self.parents = []
            self.parent_threads = set()
        if reset_parent_title:
            self.folders.clear()

    def setup_as(self, title: str, item_type: ScrapeItemType, /, *, album_id: str | None = None) -> None:
        self.part_of_album = True
        if album_id:
            self.album_id = album_id
        if self.type != item_type:
            self.type = item_type
        self.append_folders(title)

    def create_new(
        self,
        url: AbsoluteHttpURL,
        *,
        part_of_album: bool = False,
        album_id: str | None = None,
        add_parent: AbsoluteHttpURL | bool | None = None,
    ) -> Self:
        """Creates a scrape item."""

        scrape_item = self.copy()
        assert is_absolute_http_url(url)

        if add_parent:
            new_parent = self.url if add_parent is True else add_parent
            assert is_absolute_http_url(new_parent)
            scrape_item.parents.append(new_parent)

        scrape_item.url = url
        scrape_item.part_of_album = part_of_album or scrape_item.part_of_album
        scrape_item.album_id = album_id or scrape_item.album_id
        return scrape_item

    def create_child(self, url: AbsoluteHttpURL) -> Self:
        return self.create_new(url, part_of_album=True, add_parent=True)

    def setup_as_album(self: ScrapeItem, title: str, *, album_id: str | None = None) -> None:
        return self.setup_as(title, ScrapeItemType.ALBUM, album_id=album_id)

    def setup_as_profile(self: ScrapeItem, title: str, *, album_id: str | None = None) -> None:
        return self.setup_as(title, ScrapeItemType.PROFILE, album_id=album_id)

    def setup_as_forum(self: ScrapeItem, title: str, *, album_id: str | None = None) -> None:
        return self.setup_as(title, ScrapeItemType.FORUM, album_id=album_id)

    def setup_as_post(self: ScrapeItem, title: str, *, album_id: str | None = None) -> None:
        return self.setup_as(title, ScrapeItemType.FORUM_POST, album_id=album_id)

    def create_children(self, urls: Iterable[AbsoluteHttpURL]) -> Generator[Self]:
        for url in urls:
            yield self.create_child(url)
            self.add_children()

    @property
    def is_loose_file(self) -> bool:
        return not self.folders or not self.part_of_album

    @property
    def path(self) -> Path:
        return Path(*self.folders)

    def copy(self) -> Self:
        """Returns a deep copy of this scrape_item"""
        me = copy.deepcopy(self)
        me.markers = self.markers.copy()
        return me

    def get_referer(self) -> AbsoluteHttpURL | None:
        if self.referer:
            return self.referer
        if self.parents:
            return self.parents[-1]


def _has_domain(folder: str) -> bool:
    return folder.endswith(")") and " (" in folder


def _remove_domain_if_duplicate(folder: str, last_domain: str) -> str:
    og_folder, _, current_domain = folder.rpartition(" (")
    if last_domain == current_domain[:-1]:
        return og_folder
    return folder


def _extract_last_domain(folders: Sequence[str]) -> str | None:
    for folder in reversed(folders):
        if folder.endswith(")"):
            try:
                return folder[folder.rindex("(") + 1 : -1]
            except IndexError:
                pass

    return None


def _url_from_query(query_url: str | None) -> AbsoluteHttpURL | None:
    if query_url:
        url = AbsoluteHttpURL(query_url)
        if not is_absolute_http_url(url):
            raise ValueError("URL needs to be a valid HTTP URL")
        return url