Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make crawler info JSON serializable (#2045)

NTFSvolume6d3ba4e5a37e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d938d4d12..95c516d09 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,11 +27,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Twitter/x.com support
+- `show` command now accepts a `--json` flag to output site information as json
### Changed
- Removed twitter.com and x.com from blocked domains
-- Update primary URL to pixhost.cc (Pixhost)
+- Updated primary URL to pixhost.cc (Pixhost)
- Always include album id in folder name when a gallery's title is "Untitled Gallery" (Pixhost)
### Removed
diff --git a/cyberdrop_dl/cli.py b/cyberdrop_dl/cli.py
index 759feece6..b8bbfffac 100644
--- a/cyberdrop_dl/cli.py
+++ b/cyberdrop_dl/cli.py
@@ -63,12 +63,15 @@ def main_menu(
@app.command
-def show() -> None:
+def show(
+ *,
+ json: Annotated[bool, Parameter(help="Output full site information as a JSON array")] = False,
+) -> None:
"""Show a list of all supported sites"""
from cyberdrop_dl.commands import supported_sites
- table = supported_sites.as_rich_table()
- app.console.print(table)
+ content = supported_sites.as_json() if json else supported_sites.as_rich_table()
+ app.console.print(content, soft_wrap=json)
@app.command
@@ -76,7 +79,7 @@ def report() -> None:
"""Generate and display information about the system"""
from cyberdrop_dl.commands.report import generate_report
- app.console.print(generate_report())
+ app.console.print(generate_report(), soft_wrap=True)
for cmd in download, database_app, cleanup_app, config_app, cache_app, retry_app:
diff --git a/cyberdrop_dl/commands/config.py b/cyberdrop_dl/commands/config.py
index 9350d424d..abe415ecc 100644
--- a/cyberdrop_dl/commands/config.py
+++ b/cyberdrop_dl/commands/config.py
@@ -35,4 +35,4 @@ def edit(*, cli: CLIarguments | None = None) -> None:
@app.command()
def new() -> None:
"Create a new config with default options and print it"
- app.console.print(Config().dump_yaml())
+ app.console.print(Config().dump_yaml(), soft_wrap=True)
diff --git a/cyberdrop_dl/commands/supported_sites.py b/cyberdrop_dl/commands/supported_sites.py
index 259288914..ead1d5b17 100644
--- a/cyberdrop_dl/commands/supported_sites.py
+++ b/cyberdrop_dl/commands/supported_sites.py
@@ -1,25 +1,38 @@
from __future__ import annotations
from textwrap import dedent
-from typing import TYPE_CHECKING
-
-from rich.table import Table
-from rich.text import Text
+from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
+ from rich.json import JSON
+ from rich.table import Table
+
from cyberdrop_dl.crawlers.crawler import CrawlerInfo
def _gen_crawlers_info() -> list[CrawlerInfo]:
from cyberdrop_dl.crawlers import Registry
- infos = (crawler.INFO for crawler in Registry.get_crawlers(generic=True))
- return sorted(infos, key=lambda x: x.site.casefold())
+ return sorted(crawler.INFO for crawler in Registry.get_crawlers(generic=True))
+
+
+def _as_json() -> dict[str, dict[str, Any]]:
+ return {info.site: info.__json__() for info in _gen_crawlers_info()}
+
+
+def as_json() -> JSON:
+
+ from rich.json import JSON
+
+ return JSON.from_data(_as_json())
def as_rich_table() -> Table:
+ from rich.table import Table
+ from rich.text import Text
+
table = Table(
title=Text("cyberdrop-dl supported sites", style="green"),
show_lines=True,
@@ -31,7 +44,7 @@ def as_rich_table() -> Table:
for crawler_info in _gen_crawlers_info():
table.add_row(
crawler_info.site,
- str(crawler_info.primary_url).rstrip("/"),
+ str(crawler_info.primary_url),
"\n".join(crawler_info.supported_domains),
)
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 915dfd3fb..aad6ee7c5 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -164,7 +164,7 @@ class MessageBoardCrawler(Crawler, is_abc=True):
"""
THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"
- ATTACHMENT_URL_PARTS: ClassVar[Sequence[str]] = "attachments", "data", "uploads"
+ ATTACHMENT_URL_PARTS: ClassVar[tuple[str, ...]] = "attachments", "data", "uploads"
ATTACHMENT_HOSTS: ClassVar[Sequence[str]] = ()
SUPPORTS_THREAD_RECURSION: ClassVar[bool] = False
LOGIN_USER_COOKIE_NAME: ClassVar[str] = ""
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index aa74c167f..9168d0401 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -26,7 +26,8 @@ from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.models.validators import strings
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem, is_absolute_http_url
from cyberdrop_dl.utils import css, dates, enter_context, is_blob_or_svg, m3u8, parse_url, unique
-from cyberdrop_dl.utils.dataclass import ConfigDataclass, frozen
+from cyberdrop_dl.utils._url import remove_trailing_slash
+from cyberdrop_dl.utils.dataclass import ConfigDataclass, DictDataclass, frozen
from cyberdrop_dl.utils.errors import error_handling_context
if TYPE_CHECKING:
@@ -102,12 +103,20 @@ _DB_PATH_BUILDERS: MappingProxyType[str, URLHasher] = MappingProxyType(
class CrawlerInfo:
site: str
primary_url: AbsoluteHttpURL
+ scrape_mapper_keys: tuple[str, ...]
supported_domains: tuple[str, ...]
- supported_paths: SupportedPaths
+ supported_paths: dict[str, tuple[str, ...]]
+
+ __iter__ = DictDataclass.__iter__
@classmethod
- def generic(cls, name: str, paths: SupportedPaths) -> Self:
- return cls(name, "::GENERIC CRAWLER::", (), paths) # pyright: ignore[reportArgumentType]
+ def abstract(cls, name: str, site: str, paths: Mapping[str, tuple[str, ...]]) -> Self:
+ return cls(site, f"::{name} CRAWLER::", (), (), paths) # pyright: ignore[reportArgumentType]
+
+ def __json__(self) -> dict[str, Any]:
+ me = dict(self)
+ me["primary_url"] = str(self.primary_url)
+ return me
@frozen(kw_only=False)
@@ -267,36 +276,41 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
_check_init_overrides(cls)
cls.NAME: str = cls.__name__.removesuffix("Crawler")
cls.IS_GENERIC: bool = is_generic
- cls.SUPPORTED_PATHS = _sort_supported_paths(cls.SUPPORTED_PATHS) # pyright: ignore[reportConstantRedefinition]
+ paths = _sort_supported_paths(cls.SUPPORTED_PATHS)
+ cls.SUPPORTED_PATHS = paths # pyright: ignore[reportConstantRedefinition, reportAttributeAccessIssue]
cls.IS_ABC: bool = is_abc
add_to_registry = bool(not is_debug or (is_debug and env.ENABLE_DEBUG_CRAWLERS))
if cls.IS_GENERIC:
- cls.SCRAPE_MAPPER_KEYS = ()
- cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.NAME, cls.SUPPORTED_PATHS)
+ cls.INFO: CrawlerInfo = CrawlerInfo.abstract("GENERIC", cls.NAME, paths)
if add_to_registry:
Registry.generic.add(cls)
return
if is_abc:
+ cls.INFO = CrawlerInfo.abstract("ABC", cls.NAME, paths) # pyright: ignore[reportConstantRedefinition]
if add_to_registry:
Registry.abc.add(cls)
return
if cls.NAME != "RealDebrid":
Crawler._assert_fields_overrides(cls, "PRIMARY_URL", "DOMAIN", "SUPPORTED_PATHS")
+ cls.PRIMARY_URL = remove_trailing_slash(cls.PRIMARY_URL) # pyright: ignore[reportConstantRedefinition]
cls.REPLACE_OLD_DOMAINS_REGEX: str | None = "|".join(cls.OLD_DOMAINS) if cls.OLD_DOMAINS else None
_prepare_supported_domains(cls)
_validate_supported_paths(cls)
- cls.SCRAPE_MAPPER_KEYS: tuple[str, ...] = _make_scrape_mapper_keys(cls) # pyright: ignore[reportConstantRedefinition]
+
cls.FOLDER_DOMAIN = cls.FOLDER_DOMAIN or cls.DOMAIN.capitalize() # pyright: ignore[reportConstantRedefinition]
+
+ scrape_keys = _make_scrape_mapper_keys(cls)
cls.INFO = CrawlerInfo( # pyright: ignore[reportConstantRedefinition]
site=cls.FOLDER_DOMAIN,
primary_url=cls.PRIMARY_URL,
- supported_domains=_make_wiki_supported_domains(cls.SCRAPE_MAPPER_KEYS),
- supported_paths=cls.SUPPORTED_PATHS,
+ supported_domains=_make_wiki_supported_domains(scrape_keys),
+ scrape_mapper_keys=scrape_keys,
+ supported_paths=paths,
)
if add_to_registry:
Registry.concrete.add(cls)
@@ -991,11 +1005,11 @@ def _make_wiki_supported_domains(scrape_mapper_keys: tuple[str, ...]) -> tuple[s
return tuple(sorted(generalize(domain) for domain in scrape_mapper_keys))
-def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, OneOrTuple[str]]:
- def try_sort(value: OneOrTuple[str]) -> OneOrTuple[str]:
+def _sort_supported_paths(supported_paths: SupportedPaths) -> dict[str, tuple[str, ...]]:
+ def try_sort(value: OneOrTuple[str]) -> tuple[str, ...]:
if isinstance(value, tuple):
return tuple(sorted(value))
- return value
+ return (value,)
path_pairs = ((key, try_sort(value)) for key, value in supported_paths.items())
return dict(sorted(path_pairs, key=lambda x: x[0].casefold()))
diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.py
index 80ba4ad36..7d326933b 100644
--- a/cyberdrop_dl/crawlers/wordpress/__init__.py
+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py
@@ -59,18 +59,6 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
"Tag": "/tag/<tag_slug>",
"Post": "/<post_slug>/",
"All Posts": "/posts/",
- "Date Range": (
- "...?before=<date>",
- "...?after=<date>",
- "...?before=<date&after=<date>",
- ),
- "**NOTE**": """
-
- For `Date Range`, <date> must be a valid iso 8601 date, ex: `2022-12-06`.
-
- `Date Range` can be combined with `Category`, `Tag` and `All Posts`.
- ex: To only download categories from a date range: ,
- `/category/<category_slug>?before=<date>`""",
}
DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"
WP_USE_REGEX: ClassVar[bool] = True
diff --git a/cyberdrop_dl/crawlers/xenforo/allporncomix.py b/cyberdrop_dl/crawlers/xenforo/allporncomix.py
index 55c12f429..480b832cc 100644
--- a/cyberdrop_dl/crawlers/xenforo/allporncomix.py
+++ b/cyberdrop_dl/crawlers/xenforo/allporncomix.py
@@ -12,4 +12,4 @@ class AllPornComixCrawler(XenforoCrawler):
DOMAIN: ClassVar[str] = "allporncomix"
FOLDER_DOMAIN: ClassVar[str] = "AllPornComix"
login_required = False
- IGNORE_EMBEDED_IMAGES_SRC: ClassVar = False
+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = False
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index 28f312233..ef808c0e5 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -23,7 +23,7 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import Iterable, Sequence
+ from collections.abc import Sequence
from cyberdrop_dl.crawlers.crawler import SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -53,17 +53,13 @@ DEFAULT_XF_SELECTORS = MessageBoardSelectors(
)
-def _escape(strings: Iterable[str]) -> str:
- return r"\|".join(strings)
-
-
class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):
- ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"
+ ATTACHMENT_URL_PARTS: ClassVar[tuple[str, ...]] = "attachments", "data", "uploads"
THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Attachments": f"/({_escape(ATTACHMENT_URL_PARTS)})/...",
+ "Attachments": f"/{'|'.join(ATTACHMENT_URL_PARTS)}/...",
"Threads": (
- f"/({_escape(THREAD_PART_NAMES)})/<thread_name_and_id>",
+ f"/{'|'.join(THREAD_PART_NAMES)}/<thread_name_and_id>",
"/posts/<post_id>",
"/goto/<post_id>",
),
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index a82b751fb..2774841ff 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -4,10 +4,10 @@ import asyncio
import itertools
import json
from collections import defaultdict
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar, final
from cyberdrop_dl import aio
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css, extr_text
@@ -27,13 +27,10 @@ _EXTENDED_ACCOUNTS = tuple(
)
-def _escape(strings: list[str]) -> str:
- return r"\|".join(strings)
-
-
-_ACCOUNT_PATHS = (f"/{_escape(sorted(_EXTENDED_ACCOUNTS))}/<name>", "/<channel_name>")
+_ACCOUNT_PATHS = (f"/{'|'.join(sorted(_EXTENDED_ACCOUNTS))}/<name>", "/<channel_name>")
+@final
class Selectors:
ACCOUNT_INFO_JS = "script:-soup-contains('\"id_user\":')"
HLS_VIDEO_JS = "script:-soup-contains('setVideoHLS(')"
@@ -44,7 +41,7 @@ class Selectors:
class XVideosCrawler(Crawler):
- SUPPORTED_DOMAINS = (
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (
"xvideos.com",
"xvideos.es",
"xvideos-india.com",
@@ -55,7 +52,7 @@ class XVideosCrawler(Crawler):
"Video": (
"/video<id>/<title>",
"/video.<encoded_id>/<title>",
- f"/{_escape(sorted(_EXTENDED_ACCOUNTS))}#quickies/({_escape(['a', 'h', 'v'])})/<video_id>",
+ f"/{'|'.join(sorted(_EXTENDED_ACCOUNTS))}#quickies/({'|'.join(['a', 'h', 'v'])})/<video_id>",
),
"Account": _ACCOUNT_PATHS,
"Account Videos": tuple(f"{path}#_tabVideos" for path in _ACCOUNT_PATHS),
diff --git a/cyberdrop_dl/scrape_mapper.py b/cyberdrop_dl/scrape_mapper.py
index d5c7c464b..0e9e767bc 100644
--- a/cyberdrop_dl/scrape_mapper.py
+++ b/cyberdrop_dl/scrape_mapper.py
@@ -304,7 +304,7 @@ def register_crawler(
from_user: bool | Literal["raise"] = False,
) -> None:
- for domain in crawler.SCRAPE_MAPPER_KEYS:
+ for domain in crawler.INFO.scrape_mapper_keys:
other = crawlers_map.get(domain)
if from_user:
if not other and (match := _best_match(crawlers_map, crawler.PRIMARY_URL.host)):