Skip to content

Bulk asynchronous downloader for multiple file hosts

5.7 KB
Raw Blame
from __future__ import annotations

import contextlib
import dataclasses
import logging
import re
from enum import StrEnum
from pathlib import Path
from typing import TYPE_CHECKING, Any

from cyberdrop_dl import aio
from cyberdrop_dl.url_objects import AbsoluteHttpURL, RetryInfo, ScrapeItem
from cyberdrop_dl.utils.dataclass import deserialize

if TYPE_CHECKING:
    import datetime as dt
    from collections.abc import AsyncGenerator, Generator, Iterable, Mapping

    import aiosqlite

logger = logging.getLogger(__name__)

_FETCH_MANY_SIZE = 1000
_REGEX_LINKS = re.compile(r"(?:http.*?)(?=($|\n|\r\n|\r|\s|\"|\[/URL]|']\[|]\[|\[/img]))")

type URLsSource = Path | Iterable[AbsoluteHttpURL | Path]


class RetryQuery(StrEnum):
    FAILED = """
    SELECT domain, url_path, referer, download_path, download_filename FROM media
    WHERE
      completed = 0 AND created_at BETWEEN ? AND ?
    ORDER BY
      created_at DESC;
    """

    ALL = """
    SELECT domain, url_path, referer, download_path, download_filename FROM media
    WHERE
      created_at BETWEEN ? AND ?
    ORDER BY
      created_at DESC;
    """


class RetrySource(StrEnum):
    FAILED = "retry failed"
    ALL = "retry all"


@dataclasses.dataclass(slots=True, frozen=True)
class RetryScrapeSource:
    source: RetrySource
    after: dt.date
    before: dt.date


async def load_items_from_path(path: Path) -> AsyncGenerator[ScrapeItem]:
    gen_fn = load_items_from_file if await aio.is_file(path) else load_items_from_folder
    async with contextlib.aclosing(gen_fn(path)) as items:
        async for item in items:
            yield item


async def load_items_from_folder(path: Path) -> AsyncGenerator[ScrapeItem]:
    files = sorted(
        [f async for f in aio.glob(path, "*.txt") if not f.name.startswith(".")], key=lambda x: str(x).casefold()
    )
    for file in files:
        async with contextlib.aclosing(_parse_input_file_groups(file)) as lines:
            async for group_name, urls in lines:
                for url in urls:
                    item = ScrapeItem.from_url(url)
                    item.append_folders(file.stem)
                    item.part_of_album = True
                    if group_name:
                        item.append_folders(group_name)
                    yield item


async def load_items_from_file(file: Path) -> AsyncGenerator[ScrapeItem]:
    async with contextlib.aclosing(_parse_input_file_groups(file)) as lines:
        async for group_name, urls in lines:
            for url in urls:
                item = ScrapeItem.from_url(url)
                if group_name:
                    item.append_folders(group_name)
                    item.part_of_album = True
                yield item


async def _parse_input_file_groups(input_file: Path) -> AsyncGenerator[tuple[str, list[AbsoluteHttpURL]]]:
    if not await aio.is_file(input_file):
        yield ("", [])
        return

    block_quote = False
    ignored_urls = 0
    current_group_name = ""
    async with aio.open(input_file, encoding="utf8") as f:
        async for line in f:
            if line.startswith(("---", "===")):  # New group begins here
                current_group_name = line.replace("---", "").replace("===", "").strip()

            if current_group_name:
                yield (current_group_name, list(_regex_links(line)))
                continue

            if line == "#\n":  # A block quote begins or ends here
                block_quote = not block_quote
                ignored_urls = 0

            if not block_quote:
                yield ("", list(_regex_links(line)))
            else:
                ignored_urls += sum(1 for _ in _regex_links(line))

    if block_quote and ignored_urls:
        logger.warning(
            "Block comment in %s was never closed: %s URL(s) after its opening '#' line were ignored. "
            "Add a line with only '#' where the comment should end",
            input_file,
            ignored_urls,
        )


async def load_items_from_iterable(items: Iterable[AbsoluteHttpURL | Path]) -> AsyncGenerator[ScrapeItem]:
    paths: list[Path] = []
    for url in items:
        if isinstance(url, Path):
            paths.append(url)
            continue
        yield ScrapeItem.from_url(url)

    if not paths:
        return

    for path in sorted(paths, key=lambda x: str(x).casefold()):
        async with contextlib.aclosing(load_items_from_file(path)) as p_items:
            async for item in p_items:
                yield item


def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:
    """Regex grab the links from the URLs.txt file.

    This allows code blocks or full paragraphs to be copy and pasted into the URLs.txt.
    """

    line = line.strip()
    if line.startswith("#"):
        return

    for url in (url.group() for url in re.finditer(_REGEX_LINKS, line)):
        try:
            yield AbsoluteHttpURL(url, encoded="%" in url)
        except Exception as e:  # noqa: BLE001
            logger.error(f"Unable to parse URL from input file: {url} {e!r}")


async def load_items_from_db(
    db_conn: aiosqlite.Connection,
    query: RetryQuery,
    *,
    after: dt.date,
    before: dt.date,
) -> AsyncGenerator[ScrapeItem]:
    cursor = await db_conn.execute(query, (after.isoformat(), before.isoformat()))
    while rows := await cursor.fetchmany(_FETCH_MANY_SIZE):
        for row in rows:
            yield _create_item_from_row(dict(row))


def _create_item_from_row(row: Mapping[str, Any]) -> ScrapeItem:
    referer: str = row["referer"]
    url = AbsoluteHttpURL(referer, encoded="%" in referer)
    item = ScrapeItem.from_url(url)
    item.part_of_album = True
    item.retry_info = deserialize(RetryInfo, dict(row), referer=url, download_path=Path(row["download_path"]))
    return item