Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: delay crawlers import as must as possible
NTFSvolume61948c2ca096
diff --git a/.zed/debug.json b/.zed/debug.jsonindex b3129e2e2..469d2ef3f 100644--- a/.zed/debug.json+++ b/.zed/debug.json@@ -7,7 +7,7 @@"justMyCode": false,"purpose": ["debug-test"],"args": [- "--download",+ "download","--ignore-history","--dump-json","--min-free-space",diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex c6cd06cb0..81cc0ceaa 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -1,17 +1,26 @@from __future__ import annotations+import importlib+import pkgutilimport re-from typing import TYPE_CHECKING+import weakref+from contextvars import ContextVar+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, is_absolute_http_urlfrom cyberdrop_dl.utils import remove_trailing_slashif TYPE_CHECKING:- from collections.abc import Iterable+ from collections.abc import Generator, Iterable+ from types import ModuleTypefrom cyberdrop_dl.crawlers.crawler import Crawler+ALLOW_NO_EXT: ContextVar[bool] = ContextVar("ALLOW_NO_EXT", default=False)+SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)++def create_crawlers[CrawlerT: Crawler](urls: Iterable[str] | Iterable[AbsoluteHttpURL], base_crawler: type[CrawlerT]) -> set[type[CrawlerT]]:@@ -43,3 +52,65 @@ def _make_crawler_name(input_string: str) -> str:if cap_name[0].isdigit():cap_name = "_" + cap_namereturn f"{cap_name}Crawler"+++class Registry:+ abc: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()+ concrete: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()+ generic: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()+ names: ClassVar[set[str]] = set()+ # generics are concrete crawlers that are not bound to any specific site+ # They can be mapped to a site by just subclassing and setting a PRIMARY URL. ex: Chevereto++ _loaded: bool = False++ @classmethod+ def import_all(cls) -> None:+ if cls._loaded:+ return++ assert __package__+ module = importlib.import_module(__package__)+ errors = tuple(cls._import_from(module))+ if errors:+ error = RuntimeError("cyberdrop-dl installation is corrupted")+ error.add_note("A complete uninstall and reinstall should fix crawler import errors")+ raise BaseExceptionGroup("", (*errors, error))++ cls._loaded = True++ @classmethod+ def _import_from(cls, module: ModuleType) -> Generator[ImportError]:+ """Import every module (and sub-package) inside *pkg_name*."""+ for sub_module_info in pkgutil.iter_modules(module.__path__, module.__name__ + "."):+ try:+ sub_module = cls._import_module(sub_module_info.name)+ except ImportError as e:+ yield e+ else:+ if sub_module_info.ispkg:+ yield from cls._import_from(sub_module)++ @classmethod+ def _import_module(cls, name: str, /) -> ModuleType:+ try:+ return importlib.import_module(name)+ except ImportError as e:+ msg = f"Could not import crawlers from module '{name}' [{e.msg}]"+ raise ImportError(msg).with_traceback(e.__traceback__) from None++ @classmethod+ def get_crawlers(+ cls,+ *,+ concrete: bool = True,+ generic: bool = False,+ abc: bool = False,+ ) -> Generator[type[Crawler]]:+ cls.import_all()+ if concrete:+ yield from cls.concrete+ if generic:+ yield from cls.generic+ if abc:+ yield from cls.abcdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex fa5f69523..307c1f4c7 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -4,11 +4,8 @@ import asyncioimport contextlibimport dataclassesimport functools-import importlibimport logging-import pkgutilimport re-import weakreffrom abc import ABC, abstractmethodfrom collections import Counterfrom contextvars import ContextVar@@ -19,6 +16,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, Sefrom cyberdrop_dl import aio, env, signaturefrom cyberdrop_dl.clients.http import JSON_CHECK, HTTPClient, HTTPMixin, RequestContextfrom cyberdrop_dl.constants import CDL_USER_AGENT+from cyberdrop_dl.crawlers import ALLOW_NO_EXT, SKIP_DOWNLOAD, Registryfrom cyberdrop_dl.crawlers._hls import HLSMixinfrom cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError@@ -41,7 +39,6 @@ if TYPE_CHECKING:import datetimeimport http.cookiesfrom collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterable, Mapping- from types import ModuleTypeimport yarlfrom bs4 import BeautifulSoup, Tag@@ -61,11 +58,8 @@ type SupportedPaths = dict[str, OneOrTuple[str]]type SupportedDomains = OneOrTuple[str]type RateLimit = tuple[float, float]-SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)-ALLOW_NO_EXT: ContextVar[bool] = ContextVar("ALLOW_NO_EXT", default=False)-ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("ORIGIN")-+_ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("ORIGIN")_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"@@ -125,52 +119,6 @@ class SiteCookies:return tuple(self.raw.keys())-class Registry:- abc: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()- concrete: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()- generic: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()- names: ClassVar[set[str]] = set()- # generics are concrete crawlers that are not bound to any specific site- # They can be mapped to a site by just subclassing and setting a PRIMARY URL. ex: Chevereto-- _loaded: bool = False-- @classmethod- def import_all(cls) -> None:- if cls._loaded:- return-- assert __package__- module = importlib.import_module(__package__)- errors = tuple(cls._import_from(module))- if errors:- error = RuntimeError("cyberdrop-dl installation is corrupted")- error.add_note("A complete uninstall and reinstall should fix crawler import errors")- raise BaseExceptionGroup("", (*errors, error))-- cls._loaded = True-- @classmethod- def _import_from(cls, module: ModuleType) -> Generator[ImportError]:- """Import every module (and sub-package) inside *pkg_name*."""- for sub_module_info in pkgutil.iter_modules(module.__path__, module.__name__ + "."):- try:- sub_module = cls._import_module(sub_module_info.name)- except ImportError as e:- yield e- else:- if sub_module_info.ispkg:- yield from cls._import_from(sub_module)-- @classmethod- def _import_module(cls, name: str, /) -> ModuleType:- try:- return importlib.import_module(name)- except ImportError as e:- msg = f"Could not import crawlers from module '{name}' [{e.msg}]"- raise ImportError(msg).with_traceback(e.__traceback__) from None--class _CrawlerLogger(logging.LoggerAdapter[logging.Logger]):def __init__(self, crawler_name: str) -> None:self._crawler_name: str = crawler_name@@ -404,7 +352,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):@final@propertydef origin(self) -> AbsoluteHttpURL:- return ORIGIN.get()+ return _ORIGIN.get()@propertydef separate_posts(self) -> bool:@@ -469,7 +417,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):def new_task_id(self, url: AbsoluteHttpURL):"""Creates a new task_id (shows the URL in the UI and logs)"""self.log.info(f"Scraping {url}")- _ = ORIGIN.set(url.origin())+ _ = _ORIGIN.set(url.origin())return self.scrape_mapper.tui.scrape.new(url)@final@@ -961,7 +909,7 @@ class API(HTTPMixin, ABC):@final@propertydef origin(self) -> AbsoluteHttpURL:- return ORIGIN.get()+ return _ORIGIN.get()def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex 5400a2509..baed63383 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -11,16 +11,8 @@ from typing import TYPE_CHECKING, Any, Literal, Selffrom pydantic.types import ByteSizefrom cyberdrop_dl import aio, env, filepath, storage-from cyberdrop_dl.clients.jdownloader import JDownloaderfrom cyberdrop_dl.constants import BlockedDomains-from cyberdrop_dl.crawlers import create_crawlers-from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler-from cyberdrop_dl.crawlers.crawler import ALLOW_NO_EXT-from cyberdrop_dl.crawlers.discourse import DiscourseCrawler-from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler-from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler-from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediaCrawler-from cyberdrop_dl.downloader.hls import CONCURRENT_SEGMENTS+from cyberdrop_dl.crawlers import ALLOW_NO_EXT, create_crawlersfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionErrorfrom cyberdrop_dl.logs import log_spacerfrom cyberdrop_dl.progress.scraping import ScrapingUI@@ -31,8 +23,11 @@ if TYPE_CHECKING:from collections.abc import AsyncGenerator, Coroutine, Generator, Iterable, Iterator, Sequencefrom pathlib import Path+ from cyberdrop_dl.clients.jdownloader import JDownloaderfrom cyberdrop_dl.config.crawlers import GenericCrawlersfrom cyberdrop_dl.crawlers.crawler import Crawler+ from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler+ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawlerfrom cyberdrop_dl.manager import Manager@@ -48,8 +43,8 @@ def _filter_by_domain(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bo@dataclasses.dataclass(slots=True, eq=False)class CrawlerFactory:- manager: Manager- _instances: dict[type[Crawler], Crawler] = dataclasses.field(default_factory=dict)+ manager: Manager = dataclasses.field(repr=False)+ _instances: dict[type[Crawler], Crawler] = dataclasses.field(repr=False, default_factory=dict)def __getitem__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:instance = self.get(obj)@@ -90,9 +85,9 @@ class ScrapeStats:@dataclasses.dataclass(slots=True)-class TaskGroups:- scrape: asyncio.TaskGroup- downloads: asyncio.TaskGroup+class TaskGroups[T: asyncio.TaskGroup]:+ scrape: T+ downloads: T@dataclasses.dataclass(slots=True)@@ -105,8 +100,9 @@ class ScrapeMapper:_direct_http: DirectHttpFileCrawler = dataclasses.field(init=False)_jdownloader: JDownloader = dataclasses.field(init=False)_real_debrid: RealDebridCrawler = dataclasses.field(init=False)- _task_groups: TaskGroups = dataclasses.field(- init=False, default_factory=lambda: TaskGroups(asyncio.TaskGroup(), asyncio.TaskGroup())+ _task_groups: TaskGroups[asyncio.TaskGroup] = dataclasses.field(+ init=False,+ default_factory=lambda: TaskGroups(asyncio.TaskGroup(), asyncio.TaskGroup()),)_seen_urls: set[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=set)_crawlers_disabled_at_runtime: set[str] = dataclasses.field(init=False, default_factory=set)@@ -114,6 +110,9 @@ class ScrapeMapper:tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)_done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)+ def __repr__(self) -> str:+ return f"<{type(self).__name__}(seen_url={len(self._seen_urls):,}, done={self._done!r})>"+def _scrape_queue(self) -> int:return sum(crawler.waiting_items for crawler in self._factory)@@ -123,6 +122,10 @@ class ScrapeMapper:return totaldef __post_init__(self) -> None:+ from cyberdrop_dl.clients.jdownloader import JDownloader+ from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler+ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler+self._direct_http = DirectHttpFileCrawler(self.manager)self._jdownloader = JDownloader.from_config(self.manager.config)self._real_debrid = RealDebridCrawler(self.manager)@@ -142,16 +145,23 @@ class ScrapeMapper:_ = self._task_groups.downloads.create_task(coro)def _init_crawlers(self) -> None:+ crawlers = get_crawlers_mapping()+ self.crawlers.update(crawlers)- self.crawlers.update(get_crawlers_mapping())-+ n_generics = 0for crawler in _create_generic_crawlers(self.manager.config.crawlers.generic):+ n_generics += 1register_crawler(self.crawlers, crawler, from_user=True)+ msg = f"Loaded {len(crawlers) + n_generics:,} crawlers ({len(crawlers):,} concrete, {n_generics:,} generic)"+ logger.debug(msg)+_disable_crawlers_by_config(self.crawlers, *self.manager.config.crawlers.disabled)@contextlib.asynccontextmanagerasync def __call__(self) -> AsyncGenerator[Self]:+ from cyberdrop_dl.downloader.hls import CONCURRENT_SEGMENTS+assert not self._done.is_set()config = self.manager.config_ = filepath.MAX_FILE_LEN.set(config.max_file_name_length)@@ -390,16 +400,12 @@ def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:logger.error(f"Unable to parse URL from input file: {link} {e:!r}")-def get_crawlers_mapping(*, include_generics: bool = False) -> dict[str, type[Crawler]]:- from cyberdrop_dl.crawlers.crawler import Registry-- Registry.import_all()+def get_crawlers_mapping() -> dict[str, type[Crawler]]:+ from cyberdrop_dl.crawlers import Registrycrawlers_map: dict[str, type[Crawler]] = {}- crawlers = Registry.concrete | Registry.generic if include_generics else Registry.concrete-- for crawler in sorted(crawlers, key=lambda c: c.NAME):+ for crawler in sorted(Registry.get_crawlers(), key=lambda c: c.NAME):register_crawler(crawlers_map, crawler)copy = crawlers_map.copy()@@ -440,15 +446,25 @@ def register_crawler(def _create_generic_crawlers(generics_config: GenericCrawlers) -> Generator[type[Crawler]]:+ from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler++ if generics_config.chevereto:+ yield from create_crawlers(generics_config.chevereto, CheveretoCrawler)++ if generics_config.wordpress_html:+ from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler++ yield from create_crawlers(generics_config.wordpress_html, WordPressHTMLCrawler)++ if generics_config.wordpress_media:+ from cyberdrop_dl.crawlers.wordpress import WordPressMediaCrawler++ yield from create_crawlers(generics_config.wordpress_media, WordPressMediaCrawler)++ if generics_config.discourse:+ from cyberdrop_dl.crawlers.discourse import DiscourseCrawler- for domains, cls in (- (generics_config.wordpress_html, WordPressHTMLCrawler),- (generics_config.wordpress_media, WordPressMediaCrawler),- (generics_config.discourse, DiscourseCrawler),- (generics_config.chevereto, CheveretoCrawler),- ):- if domains:- yield from create_crawlers(domains, cls)+ yield from create_crawlers(generics_config.discourse, DiscourseCrawler)def _disable_crawlers_by_config(current_crawlers: dict[str, type[Crawler]], *crawlers_to_disable: str) -> None:diff --git a/cyberdrop_dl/supported_sites.py b/cyberdrop_dl/supported_sites.pyindex 1251220e5..259288914 100644--- a/cyberdrop_dl/supported_sites.py+++ b/cyberdrop_dl/supported_sites.py@@ -12,14 +12,11 @@ if TYPE_CHECKING:from cyberdrop_dl.crawlers.crawler import CrawlerInfo-def _gen_crawlers_info() -> Generator[CrawlerInfo]:- from cyberdrop_dl.crawlers.crawler import Registry+def _gen_crawlers_info() -> list[CrawlerInfo]:+ from cyberdrop_dl.crawlers import Registry- Registry.import_all()-- crawlers = Registry.generic | Registry.concrete- infos = (crawler.INFO for crawler in crawlers)- yield from sorted(infos, key=lambda x: x.site.casefold())+ infos = (crawler.INFO for crawler in Registry.get_crawlers(generic=True))+ return sorted(infos, key=lambda x: x.site.casefold())def as_rich_table() -> Table:diff --git a/scripts/tools/find_subclasses.py b/scripts/tools/find_subclasses.pyindex db4ab104d..68d8bf500 100644--- a/scripts/tools/find_subclasses.py+++ b/scripts/tools/find_subclasses.py@@ -15,11 +15,10 @@ if TYPE_CHECKING:def find_subclasses_of(domain: str):- from cyberdrop_dl.crawlers.crawler import Registry+ from cyberdrop_dl.crawlers import Registry- Registry.import_all()+ crawlers = tuple(Registry.get_crawlers(generic=True, abc=True))- crawlers = tuple(Registry.abc | Registry.generic | Registry.concrete)if domain.endswith("Crawler"):target = next(c for c in crawlers if c.__name__ == domain)else:diff --git a/scripts/tools/make_test_cases.py b/scripts/tools/make_test_cases.pyindex 36be92c83..93ecdac8e 100644--- a/scripts/tools/make_test_cases.py+++ b/scripts/tools/make_test_cases.py@@ -8,7 +8,7 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom cyberdrop_dl.__main__ import run_cdl-from cyberdrop_dl.crawlers.crawler import SKIP_DOWNLOAD+from cyberdrop_dl.crawlers import SKIP_DOWNLOADif TYPE_CHECKING:from collections.abc import Generatordiff --git a/scripts/tools/ping.py b/scripts/tools/ping.pyindex eebd20eba..c558f42c1 100644--- a/scripts/tools/ping.py+++ b/scripts/tools/ping.py@@ -6,7 +6,7 @@ from asyncio import subprocessfrom typing import TYPE_CHECKINGfrom cyberdrop_dl import aio-from cyberdrop_dl.crawlers.crawler import Registry+from cyberdrop_dl.crawlers import Registryfrom cyberdrop_dl.logs import setup_console_loggingif TYPE_CHECKING:@@ -48,9 +48,7 @@ async def try_ping(url: AbsoluteHttpURL) -> bool:async def main() -> None:- Registry.import_all()-- urls = tuple(crawler.PRIMARY_URL for crawler in Registry.concrete)+ urls = tuple(crawler.PRIMARY_URL for crawler in Registry.get_crawlers())logger.info(f"Pinning {len(urls):,} hosts")results = await aio.map(try_ping, urls, task_limit=30)success = sum(1 for _ in filter(None, results))diff --git a/scripts/tools/update_docs.py b/scripts/tools/update_docs.pyindex aaced79b2..79875ce5a 100644--- a/scripts/tools/update_docs.py+++ b/scripts/tools/update_docs.py@@ -61,10 +61,9 @@ def _update_file(file: Path, new_content: str, *, marker: str) -> None:def _get_custom_ua_crawlers() -> list[str]:- from cyberdrop_dl.crawlers.crawler import Registry+ from cyberdrop_dl.crawlers import Registry- Registry.import_all()- return sorted(c.FOLDER_DOMAIN for c in Registry.concrete if c._DEFAULT_UA and CDL_USER_AGENT in c._DEFAULT_UA)+ return sorted(c.FOLDER_DOMAIN for c in Registry.get_crawlers() if c._DEFAULT_UA and CDL_USER_AGENT in c._DEFAULT_UA)def update_supported_sites() -> None:diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex 1ff7c4b44..4e6dd1df7 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -222,7 +222,7 @@ async def test_direct_http_crawler(running_manager: Manager, url: str, filename:def test_invalid_crawler_modules_should_raise_import_error() -> None:- from cyberdrop_dl.crawlers.crawler import Registry+ from cyberdrop_dl.crawlers import Registrywith pytest.raises(ImportError, match="Could not import crawlers from module"):Registry._import_module("cyberdrop_dl.crawler.fake_crawler_12345")@@ -231,7 +231,8 @@ def test_invalid_crawler_modules_should_raise_import_error() -> None:def test_public_methods_have_error_handling_wrapper() -> None:import inspect- from cyberdrop_dl.crawlers.crawler import Crawler, Registry+ from cyberdrop_dl.crawlers import Registry+ from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.utils._errors import is_error_wrappeddef returns_none(func: Callable[..., Any]) -> bool:@@ -254,10 +255,8 @@ def test_public_methods_have_error_handling_wrapper() -> None:if name not in base_methods and not is_error_wrapped(method) and returns_none(method))- Registry.import_all()-errors: list[Exception] = []- for crawler in sorted(Registry.concrete | Registry.generic, key=lambda x: x.__name__):+ for crawler in sorted(Registry.get_crawlers(generic=True), key=lambda x: x.__name__):unwrapped_methods = sorted(unsafe_public_methods(crawler))if unwrapped_methods:errors.append(ValueError(crawler.__name__, unwrapped_methods))