Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from __future__ import annotations
import asyncio
import contextlib
import dataclasses
import logging
import signal
import sys
import time
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Self, override
from cyberdrop_dl import aio, env, filepath, storage
from cyberdrop_dl.constants import BLOCKED_DOMAINS
from cyberdrop_dl.crawlers import ALLOW_NO_EXT, create_crawlers
from cyberdrop_dl.csv_logs import CSVLogsManager
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.logs import log_spacer
from cyberdrop_dl.models.validators import bytesize_to_str
from cyberdrop_dl.progress.scraping import ScrapingUI
from cyberdrop_dl.scrape_source import (
RetryQuery,
RetryScrapeSource,
URLsSource,
load_items_from_db,
load_items_from_iterable,
load_items_from_path,
)
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem, ScrapeItemType
from cyberdrop_dl.utils import remove_trailing_slash
from cyberdrop_dl.utils._url import URLDeduper, matches_any_host
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Generator, Iterator
from cyberdrop_dl.clients.jd.client import JDownloader
from cyberdrop_dl.config import Config
from cyberdrop_dl.config.crawlers import GenericCrawlers
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler
from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
from cyberdrop_dl.manager import Manager
logger = logging.getLogger(__name__)
@dataclasses.dataclass(slots=True, eq=False)
class CrawlerFactory:
manager: Manager
task_mngr: aio.TaskManager
tui: ScrapingUI
_instances: dict[type[Crawler], Crawler] = dataclasses.field(repr=False, default_factory=dict)
@override
def __repr__(self) -> str:
return f"<{type(self).__name__}(instances={len(self._instances):,})>"
def __getitem__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:
instance = self.get(obj)
if instance is None:
raise KeyError(obj)
return instance
def __call__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:
instance = self.get(obj)
if instance is None:
instance = self._instances[obj] = obj(self.manager, self.task_mngr, self.tui)
return instance
def __contains__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> bool:
return obj in self._instances
def get[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT | None:
return self._instances.get(obj) # pyright: ignore[reportReturnType]
def __iter__(self) -> Iterator[Crawler]:
return iter(self._instances.values())
def clear(self) -> None:
self._instances.clear()
@dataclasses.dataclass(slots=True)
class ScrapeStats:
source: Path | str
count: int = dataclasses.field(init=False, default=0)
groups: list[str] = dataclasses.field(init=False, default_factory=list)
url_count: dict[str, int] = dataclasses.field(init=False, default_factory=dict)
start_time: float = dataclasses.field(init=False, default_factory=time.monotonic)
@property
def unique_groups(self) -> list[str]:
return list(dict.fromkeys(self.groups))
@property
def domain_stats(self) -> dict[str, int]:
return dict(sorted(self.url_count.items(), key=lambda x: x[1]))
def update(self, item: ScrapeItem) -> None:
self.count += 1
if item.folders:
self.groups.append("/".join(item.folders))
@dataclasses.dataclass(slots=True)
class ScrapeMapper:
"""This class maps links to their respective handlers, or JDownloader if they are unsupported."""
manager: Manager
crawlers: dict[str, type[Crawler]] = dataclasses.field(init=False, default_factory=dict)
task_mngr: aio.TaskManager = dataclasses.field(init=False, default_factory=aio.TaskManager)
tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)
logs: CSVLogsManager = dataclasses.field(init=False)
_direct_http: DirectHttpFileCrawler = dataclasses.field(init=False)
_jdownloader: JDownloader = dataclasses.field(init=False)
_real_debrid: RealDebridCrawler = dataclasses.field(init=False)
_seen_urls: URLDeduper = dataclasses.field(init=False, default_factory=URLDeduper, repr=False)
_factory: CrawlerFactory = dataclasses.field(init=False)
_ready: bool = dataclasses.field(init=False, default=False)
_shutting_down: bool = dataclasses.field(init=False, default=False)
def shutdown(self) -> None:
"Shutdown the scrape queue and setup handling of KeyboardInterrupt"
self._shutting_down = True
def __repr__(self) -> str:
fields = (
f"seen_url={len(self._seen_urls):,}",
f"crawlers={len(self.crawlers):,}",
f"tasks_mngr={self.task_mngr!r}",
f"factory={self._factory!r}",
)
return f"<{type(self).__name__}>({', '.join(fields)})"
def _scrape_queue(self) -> int:
return sum(crawler.waiting_items for crawler in self._factory)
def _download_queue(self) -> int:
total = sum(crawler.downloader.capacity.waiting for crawler in self._factory)
self.tui.files.stats.queued = total
return total
def __post_init__(self) -> None:
from cyberdrop_dl.clients.jd.client import JDownloader
from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler
from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
self._direct_http = DirectHttpFileCrawler(self.manager, self.task_mngr, self.tui)
self._jdownloader = JDownloader.from_config(self.manager.config)
self._real_debrid = RealDebridCrawler(self.manager, self.task_mngr, self.tui)
self._factory = CrawlerFactory(self.manager, self.task_mngr, self.tui)
self.logs = CSVLogsManager.from_config(self.manager.config, self.task_mngr.logs)
self.tui.scrape.get_queue = self._scrape_queue
self.tui.downloads.get_queue = self._download_queue
async def _init_crawlers(self) -> None:
crawlers = get_crawlers_mapping()
self.crawlers.update(crawlers)
n_generics = 0
for crawler in _create_generic_crawlers(self.manager.config.crawlers.generic):
n_generics += 1
register_crawler(self.crawlers, crawler, from_user=True)
msg = f"Loaded {len(crawlers) + n_generics:,} crawlers ({len(crawlers):,} concrete, {n_generics:,} generic)"
logger.debug(msg)
_disable_crawlers_by_config(self.crawlers, *self.manager.config.crawlers.disabled)
await self._register_peertube()
async def _register_peertube(self) -> None:
# User may have disabled peertube
if "peertube" not in self.crawlers:
return
if "pytest" in sys.modules:
return
from cyberdrop_dl.crawlers._peertube import PeerTubeCrawler
for url in self.manager.config.crawlers.generic.peertube:
if other := _best_match(self.crawlers, url.host):
msg = GENERIC_MAP_ERROR.format(url, PeerTubeCrawler.NAME, other.NAME)
logger.error(msg)
continue
self.crawlers[url.host] = PeerTubeCrawler
peertube = self._factory(PeerTubeCrawler)
for host in await peertube.get_instances():
crawler = self.crawlers.setdefault(host, PeerTubeCrawler)
if crawler.DOMAIN == PeerTubeCrawler.DOMAIN:
continue
logger.warning("Found PeerTube site '%s' mapped to a non PeerTube crawler: %s", host, crawler.INFO)
@contextlib.asynccontextmanager
async def __call__(self) -> AsyncGenerator[Self]:
from cyberdrop_dl.downloader.hls import CONCURRENT_SEGMENTS
assert not self.task_mngr.scrape.done.is_set()
self.logs.delete_old_logs()
config = self.manager.config
_ = CONCURRENT_SEGMENTS.set(config.downloads.concurrent_segments)
_ = ALLOW_NO_EXT.set(config.filters.allow_files_with_no_extension)
filepath.setup(config.max_file_name_length, config.max_folder_name_length, config.restrict_path)
if config.ui.portrait:
env.FORCE_PORTRAIT_MODE = True
config.download_folder.mkdir(parents=True, exist_ok=True)
if config.sort.enabled:
config.sort.output_folder.mkdir(parents=True, exist_ok=True)
logger.debug("Using %s as chunk size", bytesize_to_str(self.manager.download_client.chunk_size))
await self.manager.http_client.load_cookie_files(await self.manager.get_cookie_files())
self.tui.mode = self.manager.config.ui.mode
if config.network.dump_responses:
self.manager.http_client.request_done_callback = self.logs.write_response
## IMPORTANT: Order of each context matters!
with self.__cancel_context():
async with (
self.manager.http_client,
storage.monitor(config.min_free_space),
self.task_mngr.logs,
self.task_mngr.downloads,
self.task_mngr.scrape,
):
self.manager.scrape_mapper = self
yield self
@contextlib.contextmanager
def __cancel_context(self) -> Generator[None]:
cancelled: bool = False
with self.tui(), self._patch_kb_interrupt_sig():
try:
yield
except asyncio.CancelledError:
# This is a KeyboardInterrupt cause we never cancel tasks
if not self._shutting_down:
raise
cancelled = True
if cancelled:
logger.warning("Scraping aborted ('Ctrl + C' pressed)")
@contextlib.contextmanager
def _patch_kb_interrupt_sig(self) -> Generator[None]:
asyncio_runner_sig_handler = signal.getsignal(signal.SIGINT)
new_signal: Callable[..., Any] | None = None
if callable(asyncio_runner_sig_handler):
def override(*args, **kwargs):
self.tui.status.shutdown()
return asyncio_runner_sig_handler(*args, **kwargs)
try:
signal.signal(signal.SIGINT, override)
except ValueError:
pass
else:
new_signal = override
try:
yield
finally:
if new_signal is not None and signal.getsignal(signal.SIGINT) is new_signal:
signal.signal(signal.SIGINT, asyncio_runner_sig_handler)
async def __async_init__(self) -> None:
if self._ready:
return
await self._init_crawlers()
try:
await self._jdownloader.connect(self.manager.http_client)
except Exception:
logger.exception("Failed to connect to jDownloader")
await self._real_debrid.__async_init__()
await self._direct_http.__async_post_init__()
self._ready = True
async def _wait_until_scrape_is_done(self, stats: ScrapeStats) -> None:
await self.task_mngr.scrape.done.wait()
self.tui.hide_scrape_panel()
for crawler in self._factory:
if count := len(crawler.scraped_urls):
stats.url_count[crawler.DOMAIN] = stats.url_count.setdefault(crawler.DOMAIN, 0) + count
# Let GC destroy all crawlers and just keep a reference to the downloaders for the UI
dl_caps = tuple(crawler.downloader.capacity for crawler in self._factory)
def download_queue() -> int:
total = sum(cap.waiting for cap in dl_caps)
self.tui.files.stats.queued = total
return total
self.tui.downloads.get_queue = download_queue
self._factory.clear()
self._seen_urls.clear()
for crawler in self._direct_http, self._real_debrid:
crawler.scraped_urls.clear()
async def run(self, src: URLsSource | RetryScrapeSource | None = None) -> ScrapeStats:
if self._shutting_down:
raise RuntimeError("Scraper is already shutting down")
await self.__async_init__()
if src is None:
return ScrapeStats("")
stats, get_items = _parse_source(src, self.manager)
async def dispatch() -> None:
async with contextlib.aclosing(get_items) as items:
self.task_mngr.downloads.create_task(self._wait_until_scrape_is_done(stats))
max_children = _build_max_children_map(self.manager.config)
async for item in items:
item.max_children = max_children
item.download_folder = self.manager.config.download_folder
if self._should_scrape(item):
stats.update(item)
self.task_mngr.scrape.create_task(self._send_to_crawler(item))
if not stats.count:
logger.warning("No valid links found")
self.task_mngr.scrape.create_task(dispatch())
return stats
async def send_to_crawler(self, scrape_item: ScrapeItem) -> None:
if self._should_scrape(scrape_item):
await self._send_to_crawler(scrape_item)
async def _send_to_crawler(self, scrape_item: ScrapeItem) -> None:
if cls := _best_match(self.crawlers, scrape_item.url.host):
crawler = self._factory(cls)
await crawler.__async_init__()
if crawler.__url_config__.trim:
scrape_item.url = remove_trailing_slash(scrape_item.url)
self.task_mngr.scrape.create_task(crawler.run(scrape_item), name=str(scrape_item.url))
return
if not self._real_debrid.disabled and self._real_debrid.api.is_supported(scrape_item.url):
logger.info(f"Using RealDebrid for unsupported URL: {scrape_item.url}")
self.task_mngr.scrape.create_task(self._real_debrid.run(scrape_item), name=str(scrape_item.url))
return
try:
await self._direct_http.fetch(scrape_item)
except (NoExtensionError, ValueError):
pass
else:
return
if self._jdownloader.is_enabled_for(scrape_item.url):
success = await self._send_to_jdownloader(scrape_item)
self.tui.scrape_errors.add_unsupported(sent_to_jdownloader=success)
return
logger.warning(f"Unsupported URL: {scrape_item.url}")
self.logs.write_unsupported(scrape_item.url, scrape_item.parents[0] if scrape_item.parents else None)
self.tui.scrape_errors.add_unsupported()
async def _send_to_jdownloader(self, scrape_item: ScrapeItem) -> bool:
logger.info(f"Sending unsupported URL to JDownloader: {scrape_item.url}")
try:
await self._jdownloader.send(scrape_item.url, scrape_item.path.as_posix(), scrape_item.path)
except JDownloaderError as e:
logger.error(f"Failed to send {scrape_item.url} to JDownloader\n{e.message}")
origin = scrape_item.parents[0] if scrape_item.parents else None
self.logs.write_unsupported(scrape_item.url, origin)
return False
else:
return True
def _should_scrape(self, scrape_item: ScrapeItem) -> bool:
if scrape_item.url in self._seen_urls:
return False
if _skip_by_config(scrape_item.url, self.manager.config):
self.tui.files.stats.skipped += 1
return False
return True
def get_crawlers_mapping() -> dict[str, type[Crawler]]:
from cyberdrop_dl.crawlers import Registry
crawlers_map: dict[str, type[Crawler]] = {}
for crawler in sorted(Registry.get_crawlers(), key=lambda c: c.NAME):
register_crawler(crawlers_map, crawler)
copy = crawlers_map.copy()
crawlers_map.clear()
crawlers_map.update(sorted(copy.items()))
return crawlers_map
GENERIC_MAP_ERROR = (
"Unable to assign {} to generic crawler {}. "
"URL conflicts with URL format of builtin crawler {}. "
"URL will be ignored"
)
def register_crawler(
crawlers_map: dict[str, type[Crawler]],
crawler: type[Crawler],
*,
from_user: bool | Literal["raise"] = False,
) -> None:
for domain in crawler.INFO.scrape_mapper_keys:
other = crawlers_map.get(domain)
if from_user:
if not other and (match := _best_match(crawlers_map, crawler.PRIMARY_URL.host)):
other = match
if other:
msg = GENERIC_MAP_ERROR.format(crawler.PRIMARY_URL, crawler.NAME, other.NAME)
if from_user == "raise":
raise ValueError(msg)
logger.error(msg)
continue
logger.info("Successfully mapped %s to crawler %s", crawler.PRIMARY_URL, crawler._generic_base_)
elif other:
if domain in crawlers_map:
logger.warning("%s from %s already registered by %s", domain, crawler.NAME, other)
crawlers_map[domain] = crawler
def _create_generic_crawlers(generics_config: GenericCrawlers) -> Generator[type[Crawler]]:
from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
if generics_config.chevereto:
yield from create_crawlers(generics_config.chevereto, CheveretoCrawler)
if generics_config.wordpress_html:
from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler
yield from create_crawlers(generics_config.wordpress_html, WordPressHTMLCrawler)
if generics_config.wordpress_media:
from cyberdrop_dl.crawlers.wordpress import WordPressMediaCrawler
yield from create_crawlers(generics_config.wordpress_media, WordPressMediaCrawler)
if generics_config.discourse:
from cyberdrop_dl.crawlers.discourse import DiscourseCrawler
yield from create_crawlers(generics_config.discourse, DiscourseCrawler)
if generics_config.kvs:
from cyberdrop_dl.crawlers._kvs import GenericKVSCrawler
yield from create_crawlers(generics_config.kvs, GenericKVSCrawler)
if generics_config.video:
from cyberdrop_dl.crawlers._video import GenericVideoCrawler
yield from create_crawlers(generics_config.video, GenericVideoCrawler)
if generics_config.patreon:
from cyberdrop_dl.crawlers.patreon import PatreonGenericCrawler
yield from create_crawlers(generics_config.patreon, PatreonGenericCrawler)
def _disable_crawlers_by_config(current_crawlers: dict[str, type[Crawler]], *crawlers_to_disable: str) -> None:
if not crawlers_to_disable:
return
crawlers_to_disable = tuple(sorted({name.casefold() for name in crawlers_to_disable}))
new_crawlers_mapping = {
domain: crawler
for domain, crawler in current_crawlers.items()
if crawler.INFO.site.casefold() not in crawlers_to_disable
}
disabled_crawlers = set(current_crawlers.values()) - set(new_crawlers_mapping.values())
if len(disabled_crawlers) != len(crawlers_to_disable):
msg = (
f"{len(crawlers_to_disable)} Crawler names where provided to disable"
f", but only {len(disabled_crawlers)} {'is' if len(disabled_crawlers) == 1 else 'are'} a valid crawler's name."
)
logger.warning(msg)
if disabled_crawlers:
current_crawlers.clear()
current_crawlers.update(new_crawlers_mapping)
crawlers_info = "\n".join(
str({info.site: info.supported_domains}) for info in sorted(c.INFO for c in disabled_crawlers)
)
logger.info(f"Crawlers disabled by config: \n{crawlers_info}")
log_spacer()
def _best_match[T: Crawler | type[Crawler]](crawlers: dict[str, T], domain: str) -> T | None:
if found := crawlers.get(domain):
return found
matches = (host for host, cls in crawlers.items() if host in domain and cls.check_host_match(domain))
try:
best_match = max(matches, key=len)
except (ValueError, TypeError):
return None
else:
crawlers[domain] = found = crawlers[best_match]
return found
def _build_max_children_map(config: Config) -> dict[ScrapeItemType, int]:
max_children = config.max_children
return {
ScrapeItemType.FORUM: max_children.forum,
ScrapeItemType.FORUM_POST: max_children.forum_post,
ScrapeItemType.PROFILE: max_children.profile,
ScrapeItemType.ALBUM: max_children.album,
}
def _parse_source(
src: RetryScrapeSource | URLsSource, manager: Manager
) -> tuple[ScrapeStats, AsyncGenerator[ScrapeItem]]:
match src:
case RetryScrapeSource():
source = src.source.value
query = RetryQuery[src.source.name]
items = load_items_from_db(
manager.database.conn,
query,
after=src.after,
before=src.before,
)
case Path():
source = src
items = load_items_from_path(src)
case _:
source = "CLI args"
items = load_items_from_iterable(src)
return ScrapeStats(source), items
def _skip_by_config(url: AbsoluteHttpURL, config: Config) -> bool:
if matches_any_host(url, BLOCKED_DOMAINS):
logger.info("Skipping %s as it is a blocked domain", url)
return True
hosts = config.filters.skip_hosts
if hosts and matches_any_host(url, hosts):
logger.info("Skipping %s by skip_hosts config", url)
return True
hosts = config.filters.only_hosts
if hosts and not matches_any_host(url, hosts):
logger.info("Skipping %s by only_hosts config", url)
return True
return False