Skip to content

Bulk asynchronous downloader for multiple file hosts

perf: delay crawlers import as must as possible

NTFSvolume61948c2ca096
diff --git a/.zed/debug.json b/.zed/debug.json
index b3129e2e2..469d2ef3f 100644
--- a/.zed/debug.json
+++ b/.zed/debug.json
@@ -7,7 +7,7 @@
"justMyCode": false,
"purpose": ["debug-test"],
"args": [
- "--download",
+ "download",
"--ignore-history",
"--dump-json",
"--min-free-space",
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index c6cd06cb0..81cc0ceaa 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -1,17 +1,26 @@
from __future__ import annotations
+import importlib
+import pkgutil
import re
-from typing import TYPE_CHECKING
+import weakref
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.url_objects import AbsoluteHttpURL, is_absolute_http_url
from cyberdrop_dl.utils import remove_trailing_slash
if TYPE_CHECKING:
- from collections.abc import Iterable
+ from collections.abc import Generator, Iterable
+ from types import ModuleType
from cyberdrop_dl.crawlers.crawler import Crawler
+ALLOW_NO_EXT: ContextVar[bool] = ContextVar("ALLOW_NO_EXT", default=False)
+SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)
+
+
def create_crawlers[CrawlerT: Crawler](
urls: Iterable[str] | Iterable[AbsoluteHttpURL], base_crawler: type[CrawlerT]
) -> set[type[CrawlerT]]:
@@ -43,3 +52,65 @@ def _make_crawler_name(input_string: str) -> str:
if cap_name[0].isdigit():
cap_name = "_" + cap_name
return f"{cap_name}Crawler"
+
+
+class Registry:
+ abc: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()
+ concrete: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()
+ generic: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()
+ names: ClassVar[set[str]] = set()
+ # generics are concrete crawlers that are not bound to any specific site
+ # They can be mapped to a site by just subclassing and setting a PRIMARY URL. ex: Chevereto
+
+ _loaded: bool = False
+
+ @classmethod
+ def import_all(cls) -> None:
+ if cls._loaded:
+ return
+
+ assert __package__
+ module = importlib.import_module(__package__)
+ errors = tuple(cls._import_from(module))
+ if errors:
+ error = RuntimeError("cyberdrop-dl installation is corrupted")
+ error.add_note("A complete uninstall and reinstall should fix crawler import errors")
+ raise BaseExceptionGroup("", (*errors, error))
+
+ cls._loaded = True
+
+ @classmethod
+ def _import_from(cls, module: ModuleType) -> Generator[ImportError]:
+ """Import every module (and sub-package) inside *pkg_name*."""
+ for sub_module_info in pkgutil.iter_modules(module.__path__, module.__name__ + "."):
+ try:
+ sub_module = cls._import_module(sub_module_info.name)
+ except ImportError as e:
+ yield e
+ else:
+ if sub_module_info.ispkg:
+ yield from cls._import_from(sub_module)
+
+ @classmethod
+ def _import_module(cls, name: str, /) -> ModuleType:
+ try:
+ return importlib.import_module(name)
+ except ImportError as e:
+ msg = f"Could not import crawlers from module '{name}' [{e.msg}]"
+ raise ImportError(msg).with_traceback(e.__traceback__) from None
+
+ @classmethod
+ def get_crawlers(
+ cls,
+ *,
+ concrete: bool = True,
+ generic: bool = False,
+ abc: bool = False,
+ ) -> Generator[type[Crawler]]:
+ cls.import_all()
+ if concrete:
+ yield from cls.concrete
+ if generic:
+ yield from cls.generic
+ if abc:
+ yield from cls.abc
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index fa5f69523..307c1f4c7 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -4,11 +4,8 @@ import asyncio
import contextlib
import dataclasses
import functools
-import importlib
import logging
-import pkgutil
import re
-import weakref
from abc import ABC, abstractmethod
from collections import Counter
from contextvars import ContextVar
@@ -19,6 +16,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, Final, Literal, Se
from cyberdrop_dl import aio, env, signature
from cyberdrop_dl.clients.http import JSON_CHECK, HTTPClient, HTTPMixin, RequestContext
from cyberdrop_dl.constants import CDL_USER_AGENT
+from cyberdrop_dl.crawlers import ALLOW_NO_EXT, SKIP_DOWNLOAD, Registry
from cyberdrop_dl.crawlers._hls import HLSMixin
from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
@@ -41,7 +39,6 @@ if TYPE_CHECKING:
import datetime
import http.cookies
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine, Generator, Iterable, Mapping
- from types import ModuleType
import yarl
from bs4 import BeautifulSoup, Tag
@@ -61,11 +58,8 @@ type SupportedPaths = dict[str, OneOrTuple[str]]
type SupportedDomains = OneOrTuple[str]
type RateLimit = tuple[float, float]
-SKIP_DOWNLOAD: ContextVar[bool] = ContextVar("SKIP_DOWNLOAD", default=False)
-ALLOW_NO_EXT: ContextVar[bool] = ContextVar("ALLOW_NO_EXT", default=False)
-ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("ORIGIN")
-
+_ORIGIN: ContextVar[AbsoluteHttpURL] = ContextVar("ORIGIN")
_HASH_PREFIXES = "md5:", "sha1:", "sha256:", "xxh128:"
@@ -125,52 +119,6 @@ class SiteCookies:
return tuple(self.raw.keys())
-class Registry:
- abc: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()
- concrete: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()
- generic: weakref.WeakSet[type[Crawler]] = weakref.WeakSet()
- names: ClassVar[set[str]] = set()
- # generics are concrete crawlers that are not bound to any specific site
- # They can be mapped to a site by just subclassing and setting a PRIMARY URL. ex: Chevereto
-
- _loaded: bool = False
-
- @classmethod
- def import_all(cls) -> None:
- if cls._loaded:
- return
-
- assert __package__
- module = importlib.import_module(__package__)
- errors = tuple(cls._import_from(module))
- if errors:
- error = RuntimeError("cyberdrop-dl installation is corrupted")
- error.add_note("A complete uninstall and reinstall should fix crawler import errors")
- raise BaseExceptionGroup("", (*errors, error))
-
- cls._loaded = True
-
- @classmethod
- def _import_from(cls, module: ModuleType) -> Generator[ImportError]:
- """Import every module (and sub-package) inside *pkg_name*."""
- for sub_module_info in pkgutil.iter_modules(module.__path__, module.__name__ + "."):
- try:
- sub_module = cls._import_module(sub_module_info.name)
- except ImportError as e:
- yield e
- else:
- if sub_module_info.ispkg:
- yield from cls._import_from(sub_module)
-
- @classmethod
- def _import_module(cls, name: str, /) -> ModuleType:
- try:
- return importlib.import_module(name)
- except ImportError as e:
- msg = f"Could not import crawlers from module '{name}' [{e.msg}]"
- raise ImportError(msg).with_traceback(e.__traceback__) from None
-
-
class _CrawlerLogger(logging.LoggerAdapter[logging.Logger]):
def __init__(self, crawler_name: str) -> None:
self._crawler_name: str = crawler_name
@@ -404,7 +352,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
@final
@property
def origin(self) -> AbsoluteHttpURL:
- return ORIGIN.get()
+ return _ORIGIN.get()
@property
def separate_posts(self) -> bool:
@@ -469,7 +417,7 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
def new_task_id(self, url: AbsoluteHttpURL):
"""Creates a new task_id (shows the URL in the UI and logs)"""
self.log.info(f"Scraping {url}")
- _ = ORIGIN.set(url.origin())
+ _ = _ORIGIN.set(url.origin())
return self.scrape_mapper.tui.scrape.new(url)
@final
@@ -961,7 +909,7 @@ class API(HTTPMixin, ABC):
@final
@property
def origin(self) -> AbsoluteHttpURL:
- return ORIGIN.get()
+ return _ORIGIN.get()
def _make_scrape_mapper_keys(cls: type[Crawler] | Crawler) -> tuple[str, ...]:
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index 5400a2509..baed63383 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -11,16 +11,8 @@ from typing import TYPE_CHECKING, Any, Literal, Self
from pydantic.types import ByteSize
from cyberdrop_dl import aio, env, filepath, storage
-from cyberdrop_dl.clients.jdownloader import JDownloader
from cyberdrop_dl.constants import BlockedDomains
-from cyberdrop_dl.crawlers import create_crawlers
-from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
-from cyberdrop_dl.crawlers.crawler import ALLOW_NO_EXT
-from cyberdrop_dl.crawlers.discourse import DiscourseCrawler
-from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler
-from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
-from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediaCrawler
-from cyberdrop_dl.downloader.hls import CONCURRENT_SEGMENTS
+from cyberdrop_dl.crawlers import ALLOW_NO_EXT, create_crawlers
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
from cyberdrop_dl.logs import log_spacer
from cyberdrop_dl.progress.scraping import ScrapingUI
@@ -31,8 +23,11 @@ if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Coroutine, Generator, Iterable, Iterator, Sequence
from pathlib import Path
+ from cyberdrop_dl.clients.jdownloader import JDownloader
from cyberdrop_dl.config.crawlers import GenericCrawlers
from cyberdrop_dl.crawlers.crawler import Crawler
+ from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler
+ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
from cyberdrop_dl.manager import Manager
@@ -48,8 +43,8 @@ def _filter_by_domain(scrape_item: ScrapeItem, domain_list: Sequence[str]) -> bo
@dataclasses.dataclass(slots=True, eq=False)
class CrawlerFactory:
- manager: Manager
- _instances: dict[type[Crawler], Crawler] = dataclasses.field(default_factory=dict)
+ manager: Manager = dataclasses.field(repr=False)
+ _instances: dict[type[Crawler], Crawler] = dataclasses.field(repr=False, default_factory=dict)
def __getitem__[CrawlerT: Crawler](self, obj: type[CrawlerT]) -> CrawlerT:
instance = self.get(obj)
@@ -90,9 +85,9 @@ class ScrapeStats:
@dataclasses.dataclass(slots=True)
-class TaskGroups:
- scrape: asyncio.TaskGroup
- downloads: asyncio.TaskGroup
+class TaskGroups[T: asyncio.TaskGroup]:
+ scrape: T
+ downloads: T
@dataclasses.dataclass(slots=True)
@@ -105,8 +100,9 @@ class ScrapeMapper:
_direct_http: DirectHttpFileCrawler = dataclasses.field(init=False)
_jdownloader: JDownloader = dataclasses.field(init=False)
_real_debrid: RealDebridCrawler = dataclasses.field(init=False)
- _task_groups: TaskGroups = dataclasses.field(
- init=False, default_factory=lambda: TaskGroups(asyncio.TaskGroup(), asyncio.TaskGroup())
+ _task_groups: TaskGroups[asyncio.TaskGroup] = dataclasses.field(
+ init=False,
+ default_factory=lambda: TaskGroups(asyncio.TaskGroup(), asyncio.TaskGroup()),
)
_seen_urls: set[AbsoluteHttpURL] = dataclasses.field(init=False, default_factory=set)
_crawlers_disabled_at_runtime: set[str] = dataclasses.field(init=False, default_factory=set)
@@ -114,6 +110,9 @@ class ScrapeMapper:
tui: ScrapingUI = dataclasses.field(init=False, default_factory=ScrapingUI)
_done: asyncio.Event = dataclasses.field(init=False, default_factory=asyncio.Event)
+ def __repr__(self) -> str:
+ return f"<{type(self).__name__}(seen_url={len(self._seen_urls):,}, done={self._done!r})>"
+
def _scrape_queue(self) -> int:
return sum(crawler.waiting_items for crawler in self._factory)
@@ -123,6 +122,10 @@ class ScrapeMapper:
return total
def __post_init__(self) -> None:
+ from cyberdrop_dl.clients.jdownloader import JDownloader
+ from cyberdrop_dl.crawlers.http_direct import DirectHttpFileCrawler
+ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
+
self._direct_http = DirectHttpFileCrawler(self.manager)
self._jdownloader = JDownloader.from_config(self.manager.config)
self._real_debrid = RealDebridCrawler(self.manager)
@@ -142,16 +145,23 @@ class ScrapeMapper:
_ = self._task_groups.downloads.create_task(coro)
def _init_crawlers(self) -> None:
+ crawlers = get_crawlers_mapping()
+ self.crawlers.update(crawlers)
- self.crawlers.update(get_crawlers_mapping())
-
+ n_generics = 0
for crawler in _create_generic_crawlers(self.manager.config.crawlers.generic):
+ n_generics += 1
register_crawler(self.crawlers, crawler, from_user=True)
+ msg = f"Loaded {len(crawlers) + n_generics:,} crawlers ({len(crawlers):,} concrete, {n_generics:,} generic)"
+ logger.debug(msg)
+
_disable_crawlers_by_config(self.crawlers, *self.manager.config.crawlers.disabled)
@contextlib.asynccontextmanager
async def __call__(self) -> AsyncGenerator[Self]:
+ from cyberdrop_dl.downloader.hls import CONCURRENT_SEGMENTS
+
assert not self._done.is_set()
config = self.manager.config
_ = filepath.MAX_FILE_LEN.set(config.max_file_name_length)
@@ -390,16 +400,12 @@ def _regex_links(line: str) -> Generator[AbsoluteHttpURL]:
logger.error(f"Unable to parse URL from input file: {link} {e:!r}")
-def get_crawlers_mapping(*, include_generics: bool = False) -> dict[str, type[Crawler]]:
- from cyberdrop_dl.crawlers.crawler import Registry
-
- Registry.import_all()
+def get_crawlers_mapping() -> dict[str, type[Crawler]]:
+ from cyberdrop_dl.crawlers import Registry
crawlers_map: dict[str, type[Crawler]] = {}
- crawlers = Registry.concrete | Registry.generic if include_generics else Registry.concrete
-
- for crawler in sorted(crawlers, key=lambda c: c.NAME):
+ for crawler in sorted(Registry.get_crawlers(), key=lambda c: c.NAME):
register_crawler(crawlers_map, crawler)
copy = crawlers_map.copy()
@@ -440,15 +446,25 @@ def register_crawler(
def _create_generic_crawlers(generics_config: GenericCrawlers) -> Generator[type[Crawler]]:
+ from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
+
+ if generics_config.chevereto:
+ yield from create_crawlers(generics_config.chevereto, CheveretoCrawler)
+
+ if generics_config.wordpress_html:
+ from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler
+
+ yield from create_crawlers(generics_config.wordpress_html, WordPressHTMLCrawler)
+
+ if generics_config.wordpress_media:
+ from cyberdrop_dl.crawlers.wordpress import WordPressMediaCrawler
+
+ yield from create_crawlers(generics_config.wordpress_media, WordPressMediaCrawler)
+
+ if generics_config.discourse:
+ from cyberdrop_dl.crawlers.discourse import DiscourseCrawler
- for domains, cls in (
- (generics_config.wordpress_html, WordPressHTMLCrawler),
- (generics_config.wordpress_media, WordPressMediaCrawler),
- (generics_config.discourse, DiscourseCrawler),
- (generics_config.chevereto, CheveretoCrawler),
- ):
- if domains:
- yield from create_crawlers(domains, cls)
+ yield from create_crawlers(generics_config.discourse, DiscourseCrawler)
def _disable_crawlers_by_config(current_crawlers: dict[str, type[Crawler]], *crawlers_to_disable: str) -> None:
diff --git a/cyberdrop_dl/supported_sites.py b/cyberdrop_dl/supported_sites.py
index 1251220e5..259288914 100644
--- a/cyberdrop_dl/supported_sites.py
+++ b/cyberdrop_dl/supported_sites.py
@@ -12,14 +12,11 @@ if TYPE_CHECKING:
from cyberdrop_dl.crawlers.crawler import CrawlerInfo
-def _gen_crawlers_info() -> Generator[CrawlerInfo]:
- from cyberdrop_dl.crawlers.crawler import Registry
+def _gen_crawlers_info() -> list[CrawlerInfo]:
+ from cyberdrop_dl.crawlers import Registry
- Registry.import_all()
-
- crawlers = Registry.generic | Registry.concrete
- infos = (crawler.INFO for crawler in crawlers)
- yield from sorted(infos, key=lambda x: x.site.casefold())
+ infos = (crawler.INFO for crawler in Registry.get_crawlers(generic=True))
+ return sorted(infos, key=lambda x: x.site.casefold())
def as_rich_table() -> Table:
diff --git a/scripts/tools/find_subclasses.py b/scripts/tools/find_subclasses.py
index db4ab104d..68d8bf500 100644
--- a/scripts/tools/find_subclasses.py
+++ b/scripts/tools/find_subclasses.py
@@ -15,11 +15,10 @@ if TYPE_CHECKING:
def find_subclasses_of(domain: str):
- from cyberdrop_dl.crawlers.crawler import Registry
+ from cyberdrop_dl.crawlers import Registry
- Registry.import_all()
+ crawlers = tuple(Registry.get_crawlers(generic=True, abc=True))
- crawlers = tuple(Registry.abc | Registry.generic | Registry.concrete)
if domain.endswith("Crawler"):
target = next(c for c in crawlers if c.__name__ == domain)
else:
diff --git a/scripts/tools/make_test_cases.py b/scripts/tools/make_test_cases.py
index 36be92c83..93ecdac8e 100644
--- a/scripts/tools/make_test_cases.py
+++ b/scripts/tools/make_test_cases.py
@@ -8,7 +8,7 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any
from cyberdrop_dl.__main__ import run_cdl
-from cyberdrop_dl.crawlers.crawler import SKIP_DOWNLOAD
+from cyberdrop_dl.crawlers import SKIP_DOWNLOAD
if TYPE_CHECKING:
from collections.abc import Generator
diff --git a/scripts/tools/ping.py b/scripts/tools/ping.py
index eebd20eba..c558f42c1 100644
--- a/scripts/tools/ping.py
+++ b/scripts/tools/ping.py
@@ -6,7 +6,7 @@ from asyncio import subprocess
from typing import TYPE_CHECKING
from cyberdrop_dl import aio
-from cyberdrop_dl.crawlers.crawler import Registry
+from cyberdrop_dl.crawlers import Registry
from cyberdrop_dl.logs import setup_console_logging
if TYPE_CHECKING:
@@ -48,9 +48,7 @@ async def try_ping(url: AbsoluteHttpURL) -> bool:
async def main() -> None:
- Registry.import_all()
-
- urls = tuple(crawler.PRIMARY_URL for crawler in Registry.concrete)
+ urls = tuple(crawler.PRIMARY_URL for crawler in Registry.get_crawlers())
logger.info(f"Pinning {len(urls):,} hosts")
results = await aio.map(try_ping, urls, task_limit=30)
success = sum(1 for _ in filter(None, results))
diff --git a/scripts/tools/update_docs.py b/scripts/tools/update_docs.py
index aaced79b2..79875ce5a 100644
--- a/scripts/tools/update_docs.py
+++ b/scripts/tools/update_docs.py
@@ -61,10 +61,9 @@ def _update_file(file: Path, new_content: str, *, marker: str) -> None:
def _get_custom_ua_crawlers() -> list[str]:
- from cyberdrop_dl.crawlers.crawler import Registry
+ from cyberdrop_dl.crawlers import Registry
- Registry.import_all()
- return sorted(c.FOLDER_DOMAIN for c in Registry.concrete if c._DEFAULT_UA and CDL_USER_AGENT in c._DEFAULT_UA)
+ return sorted(c.FOLDER_DOMAIN for c in Registry.get_crawlers() if c._DEFAULT_UA and CDL_USER_AGENT in c._DEFAULT_UA)
def update_supported_sites() -> None:
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index 1ff7c4b44..4e6dd1df7 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -222,7 +222,7 @@ async def test_direct_http_crawler(running_manager: Manager, url: str, filename:
def test_invalid_crawler_modules_should_raise_import_error() -> None:
- from cyberdrop_dl.crawlers.crawler import Registry
+ from cyberdrop_dl.crawlers import Registry
with pytest.raises(ImportError, match="Could not import crawlers from module"):
Registry._import_module("cyberdrop_dl.crawler.fake_crawler_12345")
@@ -231,7 +231,8 @@ def test_invalid_crawler_modules_should_raise_import_error() -> None:
def test_public_methods_have_error_handling_wrapper() -> None:
import inspect
- from cyberdrop_dl.crawlers.crawler import Crawler, Registry
+ from cyberdrop_dl.crawlers import Registry
+ from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.utils._errors import is_error_wrapped
def returns_none(func: Callable[..., Any]) -> bool:
@@ -254,10 +255,8 @@ def test_public_methods_have_error_handling_wrapper() -> None:
if name not in base_methods and not is_error_wrapped(method) and returns_none(method)
)
- Registry.import_all()
-
errors: list[Exception] = []
- for crawler in sorted(Registry.concrete | Registry.generic, key=lambda x: x.__name__):
+ for crawler in sorted(Registry.get_crawlers(generic=True), key=lambda x: x.__name__):
unwrapped_methods = sorted(unsafe_public_methods(crawler))
if unwrapped_methods:
errors.append(ValueError(crawler.__name__, unwrapped_methods))