Skip to content

Bulk asynchronous downloader for multiple file hosts

6.5 KB
Raw Blame
from __future__ import annotations

import asyncio
import dataclasses
import html
from typing import TYPE_CHECKING, NamedTuple, cast, overload

from bs4 import BeautifulSoup
from bs4.filter import SoupStrainer

from cyberdrop_dl.exceptions import ScrapeError

if TYPE_CHECKING:
    from collections.abc import Generator

    from bs4.element import Tag

    from cyberdrop_dl.constants import HttpMethod


def _choose_html_parser() -> str:
    try:
        __import__("lxml")
    except ImportError:
        return "html.parser"
    else:
        return "lxml"


HTML_PARSER = _choose_html_parser()


class SelectorError(ScrapeError):
    def __init__(self, message: str | None = None) -> None:
        super().__init__(422, message)


class CssAttributeSelector(NamedTuple):
    element: str
    attribute: str = ""

    def __call__(self, soup: Tag) -> str:
        return select(soup, self.element, self.attribute)

    def text(self, tag: Tag) -> str:
        return select_text(tag, self.element)


@dataclasses.dataclass(slots=True)
class HTMLForm:
    method: HttpMethod
    action: str
    inputs: dict[str, str | None]


def _select_one(tag: Tag, selector: str) -> Tag:
    """Same as `tag.select_one` but asserts the result is not `None`"""
    result = tag.select_one(selector)
    if result is None:
        raise SelectorError(f"{selector} tag not found")
    return result


def select_text(tag: Tag, selector: str, *, strip: bool = True, decompose: str | None = None) -> str:
    """Same as `tag.select_one.get_text(strip=strip)` but asserts the result is not `None`"""
    inner_tag = select(tag, selector)
    if decompose:
        for trash in inner_tag.select(decompose):
            trash.decompose()
    return text(inner_tag, strip=strip)


def select_tag_text(html: str, tag_name: str) -> str:
    return select_text(soup(html, parse_only=tag_name), tag_name)


def attr_or_none(tag: Tag, *attributes: str) -> str | None:
    """Same as `tag.get(attribute)` but asserts the result is a single str"""
    for attribute in attributes:
        if result := _attr_or_none(tag, attribute):
            return result


def _attr_or_none(tag: Tag, attribute: str) -> str | None:
    attribute_ = attribute
    if attribute_ == "srcset":
        if (srcset := tag.get(attribute_)) and type(srcset) is str:
            return best_from_srcset(srcset)
        attribute_ = "src"

    value = tag.get("data-src") or tag.get(attribute_) if attribute_ == "src" else tag.get(attribute_)
    if value is None:
        return None
    if type(value) is not str:
        raise SelectorError(f"Expected a single value for {attribute = !r}, got multiple")
    return value


def attr(tag: Tag, *attributes: str) -> str:
    """Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""
    if result := attr_or_none(tag, *attributes):
        return result
    raise SelectorError(f"{attributes = } not found")


def text(tag: Tag, *, strip: bool = True) -> str:
    return tag.get_text(strip=strip)


@overload
def select(tag: Tag, selector: str) -> Tag: ...


@overload
def select(tag: Tag, selector: str, attribute: str) -> str: ...


def select(tag: Tag, selector: str, attribute: str | None = None) -> Tag | str:
    inner_tag = _select_one(tag, selector)
    if not attribute:
        return inner_tag
    return attr(inner_tag, attribute)


@overload
def iselect(tag: Tag, selector: str) -> Generator[Tag]: ...


@overload
def iselect(tag: Tag, selector: str, attribute: str) -> Generator[str]: ...


def iselect(tag: Tag, selector: str, attribute: str | None = None) -> Generator[Tag] | Generator[str]:
    """Same as `tag.select(selector)`, but it returns a generator instead of a list."""
    tags = tag.css.iselect(selector)
    if not attribute:
        yield from tags

    else:
        for inner_tag in tags:
            if attr := attr_or_none(inner_tag, attribute):
                yield attr


def iselect_text(soup: Tag, /, selector: str, contains: tuple[str, ...] | str = ()) -> Generator[str]:
    if isinstance(contains, str):
        contains = (contains,)

    for tag in iselect(soup, selector):
        content = text(tag)
        for key in contains:
            if key not in content:
                continue
        yield content


def parse_srcset(srcset: str) -> Generator[tuple[str, str]]:
    for option in srcset.split(","):
        match option.strip().rsplit(maxsplit=1):
            case [url, decriptor]:
                yield decriptor, url
            case [url]:
                yield "1x", url
            case _:
                continue


def best_from_srcset(srcset: str) -> str:
    options = dict(parse_srcset(srcset))
    best = max(options, key=lambda decriptor: float(decriptor.rstrip("wx")))
    return options[best]


def decompose(tag: Tag, selector: str) -> None:
    for inner_tag in tag.select(selector):
        inner_tag.decompose()


def rstrip_domain(title: str, domain: str) -> str:
    assert domain
    second_level_domain = domain.rsplit(".", 1)[0].casefold().removeprefix("www.")

    def sanitize(string: str, char: str) -> str:
        front, _, tail = string.rpartition(char)
        if front and second_level_domain in tail.casefold():
            return front.strip()
        return string

    for char in sorted(("|", " - "), key=title.rfind, reverse=True):
        title = sanitize(title, char)

    return title


def page_title(soup: Tag, domain: str | None = None) -> str:
    title = select_text(soup, "title")
    if domain:
        return rstrip_domain(title, domain)
    return title


def parse_form(form: Tag, /) -> HTMLForm:
    inputs: dict[str, str | None] = {}
    for elem in iselect(form, "input"):
        name = attr_or_none(elem, "name") or attr(elem, "id")
        inputs[name] = attr_or_none(elem, "value")

    method = attr(form, "method").upper()
    action = attr(form, "action")
    return HTMLForm(cast("HttpMethod", method), action, inputs)


def where(*selectors: str) -> str:
    assert len(selectors) >= 2, ":where only makes sense combining 2 or more selectors"
    return ":where(" + ", ".join(selectors) + ")"


def soup(content: str, parse_only: tuple[str, ...] | str | None = None) -> BeautifulSoup:
    return BeautifulSoup(content, HTML_PARSER, parse_only=SoupStrainer(parse_only) if parse_only is not None else None)


async def asoup(content: str, parse_only: tuple[str, ...] | str | None = None) -> BeautifulSoup:
    return await asyncio.to_thread(soup, content, parse_only)


unescape = html.unescape
iframes = CssAttributeSelector("iframe", "src")
images = CssAttributeSelector("img", "srcset")
links = CssAttributeSelector(":any-link", "href")