Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make crawler info JSON serializable (#2045)
NTFSvolume6d3ba4e5a37e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d938d4d12..95c516d09 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,11 +27,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Added- Twitter/x.com support+- `show` command now accepts a `--json` flag to output site information as json### Changed- Removed twitter.com and x.com from blocked domains-- Update primary URL to pixhost.cc (Pixhost)+- Updated primary URL to pixhost.cc (Pixhost)- Always include album id in folder name when a gallery's title is "Untitled Gallery" (Pixhost)### Removeddiff --git a/cyberdrop_dl/cli.py b/cyberdrop_dl/cli.pyindex 759feece6..b8bbfffac 100644--- a/cyberdrop_dl/cli.py+++ b/cyberdrop_dl/cli.py@@ -63,12 +63,15 @@ def main_menu(@app.command-def show() -> None:+def show(+ *,+ json: Annotated[bool, Parameter(help="Output full site information as a JSON array")] = False,+) -> None:"""Show a list of all supported sites"""from cyberdrop_dl.commands import supported_sites- table = supported_sites.as_rich_table()- app.console.print(table)+ content = supported_sites.as_json() if json else supported_sites.as_rich_table()+ app.console.print(content, soft_wrap=json)@app.command@@ -76,7 +79,7 @@ def report() -> None:"""Generate and display information about the system"""from cyberdrop_dl.commands.report import generate_report- app.console.print(generate_report())+ app.console.print(generate_report(), soft_wrap=True)for cmd in download, database_app, cleanup_app, config_app, cache_app, retry_app:diff --git a/cyberdrop_dl/commands/config.py b/cyberdrop_dl/commands/config.pyindex 9350d424d..abe415ecc 100644--- a/cyberdrop_dl/commands/config.py+++ b/cyberdrop_dl/commands/config.py@@ -35,4 +35,4 @@ def edit(*, cli: CLIarguments | None = None) -> None:@app.command()def new() -> None:"Create a new config with default options and print it"- app.console.print(Config().dump_yaml())+ app.console.print(Config().dump_yaml(), soft_wrap=True)diff --git a/cyberdrop_dl/commands/supported_sites.py b/cyberdrop_dl/commands/supported_sites.pyindex 259288914..ead1d5b17 100644--- a/cyberdrop_dl/commands/supported_sites.py+++ b/cyberdrop_dl/commands/supported_sites.py@@ -1,25 +1,38 @@from __future__ import annotationsfrom textwrap import dedent-from typing import TYPE_CHECKING--from rich.table import Table-from rich.text import Text+from typing import TYPE_CHECKING, Anyif TYPE_CHECKING:from collections.abc import Generator, Iterable+ from rich.json import JSON+ from rich.table import Table+from cyberdrop_dl.crawlers.crawler import CrawlerInfodef _gen_crawlers_info() -> list[CrawlerInfo]:from cyberdrop_dl.crawlers import Registry- infos = (crawler.INFO for crawler in Registry.get_crawlers(generic=True))- return sorted(infos, key=lambda x: x.site.casefold())+ return sorted(crawler.INFO for crawler in Registry.get_crawlers(generic=True))+++def _as_json() -> dict[str, dict[str, Any]]:+ return {info.site: info.__json__() for info in _gen_crawlers_info()}+++def as_json() -> JSON:++ from rich.json import JSON++ return JSON.from_data(_as_json())def as_rich_table() -> Table:+ from rich.table import Table+ from rich.text import Text+table = Table(title=Text("cyberdrop-dl supported sites", style="green"),show_lines=True,@@ -31,7 +44,7 @@ def as_rich_table() -> Table:for crawler_info in _gen_crawlers_info():table.add_row(crawler_info.site,- str(crawler_info.primary_url).rstrip("/"),+ str(crawler_info.primary_url),"\n".join(crawler_info.supported_domains),)diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 915dfd3fb..aad6ee7c5 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -164,7 +164,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):"""THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"- ATTACHMENT_URL_PARTS: ClassVar[Sequence[str]] = "attachments", "data", "uploads"+ ATTACHMENT_URL_PARTS: ClassVar[tuple[str, ...]] = "attachments", "data", "uploads"ATTACHMENT_HOSTS: ClassVar[Sequence[str]] = ()SUPPORTS_THREAD_RECURSION: ClassVar[bool] = FalseLOGIN_USER_COOKIE_NAME: ClassVar[str] = ""diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex aa74c167f..9168d0401 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -26,7 +26,8 @@ from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.models.validators import stringsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, is_absolute_http_urlfrom cyberdrop_dl.utils import css, dates, enter_context, is_blob_or_svg, m3u8, parse_url, unique-from cyberdrop_dl.utils.dataclass import ConfigDataclass, frozen+from cyberdrop_dl.utils._url import remove_trailing_slash+from cyberdrop_dl.utils.dataclass import ConfigDataclass, DictDataclass, frozenfrom cyberdrop_dl.utils.errors import error_handling_contextif TYPE_CHECKING:@@ -102,12 +103,20 @@ _DB_PATH_BUILDERS: MappingProxyType[str, URLHasher] = MappingProxyType(class CrawlerInfo:site: strprimary_url: AbsoluteHttpURL+ scrape_mapper_keys: tuple[str, ...]supported_domains: tuple[str, ...]- supported_paths: SupportedPaths+ supported_paths: dict[str, tuple[str, ...]]++ __iter__ = DictDataclass.__iter__@classmethod- def generic(cls, name: str, paths: SupportedPaths) -> Self:- return cls(name, "::GENERIC CRAWLER::", (), paths) # pyright: ignore[reportArgumentType]+ def abstract(cls, name: str, site: str, paths: Mapping[str, tuple[str, ...]]) -> Self:+ return cls(site, f"::{name} CRAWLER::", (), (), paths) # pyright: ignore[reportArgumentType]++ def __json__(self) -> dict[str, Any]:+ me = dict(self)+ me["primary_url"] = str(self.primary_url)+ return me@frozen(kw_only=False)@@ -267,36 +276,41 @@ class Crawler(HTTPMixin, HLSMixin, ABC):_check_init_overrides(cls)cls.NAME: str = cls.__name__.removesuffix("Crawler")cls.IS_GENERIC: bool = is_generic- cls.SUPPORTED_PATHS = _sort_supported_paths(cls.SUPPORTED_PATHS) # pyright: ignore[reportConstantRedefinition]+ paths = _sort_supported_paths(cls.SUPPORTED_PATHS)+ cls.SUPPORTED_PATHS = paths # pyright: ignore[reportConstantRedefinition, reportAttributeAccessIssue]cls.IS_ABC: bool = is_abcadd_to_registry = bool(not is_debug or (is_debug and env.ENABLE_DEBUG_CRAWLERS))if cls.IS_GENERIC:- cls.SCRAPE_MAPPER_KEYS = ()- cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.NAME, cls.SUPPORTED_PATHS)+ cls.INFO: CrawlerInfo = CrawlerInfo.abstract("GENERIC", cls.NAME, paths)if add_to_registry:Registry.generic.add(cls)returnif is_abc:+ cls.INFO = CrawlerInfo.abstract("ABC", cls.NAME, paths) # pyright: ignore[reportConstantRedefinition]if add_to_registry:Registry.abc.add(cls)returnif cls.NAME != "RealDebrid":Crawler._assert_fields_overrides(cls, "PRIMARY_URL", "DOMAIN", "SUPPORTED_PATHS")+ cls.PRIMARY_URL = remove_trailing_slash(cls.PRIMARY_URL) # pyright: ignore[reportConstantRedefinition]cls.REPLACE_OLD_DOMAINS_REGEX: str | None = "|".join(cls.OLD_DOMAINS) if cls.OLD_DOMAINS else None_prepare_supported_domains(cls)_validate_supported_paths(cls)- cls.SCRAPE_MAPPER_KEYS: tuple[str, ...] = _make_scrape_mapper_keys(cls) # pyright: ignore[reportConstantRedefinition]+cls.FOLDER_DOMAIN = cls.FOLDER_DOMAIN or cls.DOMAIN.capitalize() # pyright: ignore[reportConstantRedefinition]++ scrape_keys = _make_scrape_mapper_keys(cls)cls.INFO = CrawlerInfo( # pyright: ignore[reportConstantRedefinition]site=cls.FOLDER_DOMAIN,primary_url=cls.PRIMARY_URL,- supported_domains=_make_wiki_supported_domains(cls.SCRAPE_MAPPER_KEYS),- supported_paths=cls.SUPPORTED_PATHS,+ supported_domains=_make_wiki_supported_domains(scrape_keys),+ scrape_mapper_keys=scrape_keys,+ supported_paths=paths,)if add_to_registry:Registry.concrete.add(cls)@@ -991,11 +1005,11 @@ def _make_wiki_supported_domains(scrape_mapper_keys: tuple[str, ...]) -> tuple[sreturn tuple(sorted(generalize(domain) for domain in scrape_mapper_keys))-def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, OneOrTuple[str]]:- def try_sort(value: OneOrTuple[str]) -> OneOrTuple[str]:+def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, tuple[str, ...]]:+ def try_sort(value: OneOrTuple[str]) -> tuple[str, ...]:if isinstance(value, tuple):return tuple(sorted(value))- return value+ return (value,)path_pairs = ((key, try_sort(value)) for key, value in supported_paths.items())return dict(sorted(path_pairs, key=lambda x: x[0].casefold()))diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.pyindex 80ba4ad36..7d326933b 100644--- a/cyberdrop_dl/crawlers/wordpress/__init__.py+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py@@ -59,18 +59,6 @@ class WordPressBaseCrawler(Crawler, is_abc=True):"Tag": "/tag/<tag_slug>","Post": "/<post_slug>/","All Posts": "/posts/",- "Date Range": (- "...?before=<date>",- "...?after=<date>",- "...?before=<date&after=<date>",- ),- "**NOTE**": """-- For `Date Range`, <date> must be a valid iso 8601 date, ex: `2022-12-06`.-- `Date Range` can be combined with `Category`, `Tag` and `All Posts`.- ex: To only download categories from a date range: ,- `/category/<category_slug>?before=<date>`""",}DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"WP_USE_REGEX: ClassVar[bool] = Truediff --git a/cyberdrop_dl/crawlers/xenforo/allporncomix.py b/cyberdrop_dl/crawlers/xenforo/allporncomix.pyindex 55c12f429..480b832cc 100644--- a/cyberdrop_dl/crawlers/xenforo/allporncomix.py+++ b/cyberdrop_dl/crawlers/xenforo/allporncomix.py@@ -12,4 +12,4 @@ class AllPornComixCrawler(XenforoCrawler):DOMAIN: ClassVar[str] = "allporncomix"FOLDER_DOMAIN: ClassVar[str] = "AllPornComix"login_required = False- IGNORE_EMBEDED_IMAGES_SRC: ClassVar = False+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = Falsediff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex 28f312233..ef808c0e5 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -23,7 +23,7 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import Iterable, Sequence+ from collections.abc import Sequencefrom cyberdrop_dl.crawlers.crawler import SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -53,17 +53,13 @@ DEFAULT_XF_SELECTORS = MessageBoardSelectors()-def _escape(strings: Iterable[str]) -> str:- return r"\|".join(strings)--class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):- ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"+ ATTACHMENT_URL_PARTS: ClassVar[tuple[str, ...]] = "attachments", "data", "uploads"THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Attachments": f"/({_escape(ATTACHMENT_URL_PARTS)})/...",+ "Attachments": f"/{'|'.join(ATTACHMENT_URL_PARTS)}/...","Threads": (- f"/({_escape(THREAD_PART_NAMES)})/<thread_name_and_id>",+ f"/{'|'.join(THREAD_PART_NAMES)}/<thread_name_and_id>","/posts/<post_id>","/goto/<post_id>",),diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.pyindex a82b751fb..2774841ff 100644--- a/cyberdrop_dl/crawlers/xvideos.py+++ b/cyberdrop_dl/crawlers/xvideos.py@@ -4,10 +4,10 @@ import asyncioimport itertoolsimport jsonfrom collections import defaultdict-from typing import TYPE_CHECKING, Any, ClassVar+from typing import TYPE_CHECKING, Any, ClassVar, finalfrom cyberdrop_dl import aio-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css, extr_text@@ -27,13 +27,10 @@ _EXTENDED_ACCOUNTS = tuple()-def _escape(strings: list[str]) -> str:- return r"\|".join(strings)---_ACCOUNT_PATHS = (f"/{_escape(sorted(_EXTENDED_ACCOUNTS))}/<name>", "/<channel_name>")+_ACCOUNT_PATHS = (f"/{'|'.join(sorted(_EXTENDED_ACCOUNTS))}/<name>", "/<channel_name>")+@finalclass Selectors:ACCOUNT_INFO_JS = "script:-soup-contains('\"id_user\":')"HLS_VIDEO_JS = "script:-soup-contains('setVideoHLS(')"@@ -44,7 +41,7 @@ class Selectors:class XVideosCrawler(Crawler):- SUPPORTED_DOMAINS = (+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ("xvideos.com","xvideos.es","xvideos-india.com",@@ -55,7 +52,7 @@ class XVideosCrawler(Crawler):"Video": ("/video<id>/<title>","/video.<encoded_id>/<title>",- f"/{_escape(sorted(_EXTENDED_ACCOUNTS))}#quickies/({_escape(['a', 'h', 'v'])})/<video_id>",+ f"/{'|'.join(sorted(_EXTENDED_ACCOUNTS))}#quickies/({'|'.join(['a', 'h', 'v'])})/<video_id>",),"Account": _ACCOUNT_PATHS,"Account Videos": tuple(f"{path}#_tabVideos" for path in _ACCOUNT_PATHS),diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.pyindex d5c7c464b..0e9e767bc 100644--- a/cyberdrop_dl/scrape_mapper.py+++ b/cyberdrop_dl/scrape_mapper.py@@ -304,7 +304,7 @@ def register_crawler(from_user: bool | Literal["raise"] = False,) -> None:- for domain in crawler.SCRAPE_MAPPER_KEYS:+ for domain in crawler.INFO.scrape_mapper_keys:other = crawlers_map.get(domain)if from_user:if not other and (match := _best_match(crawlers_map, crawler.PRIMARY_URL.host)):