Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add support for any wordpress site (#1076)

NTFSvolumecc5a58ba4412
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py
index c3c70df9e..797886155 100755
--- a/cyberdrop_dl/config/config_model.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -51,6 +51,7 @@ class DownloadOptions(BaseModel):
valid_keys = ("default", "title", "id", "number", "date")
try:
value.format(**dict.fromkeys(valid_keys, "TEST"))
+ return value
except KeyError as e:
msg = f"'{e.args[0]}' is not a valid key for this option. Valid keys: {valid_keys}"
raise ValueError(msg) from None
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 48024e41d..82fd80b42 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -6,7 +6,6 @@ from cyberdrop_dl import env
from .acidimg import AcidImgCrawler
from .archivebate import ArchiveBateCrawler
from .ashemaletube import AShemaleTubeCrawler
-from .bestprettygirl import BestPrettyGirlCrawler
from .box_dot_com import BoxDotComCrawler
from .bunkrr import BunkrrCrawler
from .bunkrr_albums_io import BunkrAlbumsIOCrawler
@@ -91,6 +90,7 @@ from .twitter_images import TwimgCrawler
from .twpornstars import TwPornstarsCrawler
from .vipr_dot_im import ViprImCrawler
from .wetransfer import WeTransferCrawler
+from .wordpress import WP_CRAWLERS
from .xbunkr import XBunkrCrawler
from .xenforo import XF_CRAWLERS, SimpCityCrawler
from .xhamster import XhamsterCrawler
@@ -100,11 +100,11 @@ from .youjizz import YouJizzCrawler
ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler")}
ALL_CRAWLERS = ALL_CRAWLERS - {Crawler}
-ALL_CRAWLERS.update(XF_CRAWLERS)
+ALL_CRAWLERS.update(XF_CRAWLERS, WP_CRAWLERS)
DEBUG_CRAWLERS = {SimpCityCrawler, BunkrAlbumsIOCrawler, MegaNzCrawler, FikFapCrawler}
-CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERS
-
if env.ENABLE_DEBUG_CRAWLERS == "d396ab8c85fcb1fecd22c8d9b58acf944a44e6d35014e9dd39e42c9a64091eda":
- CRAWLERS.update(DEBUG_CRAWLERS)
+ CRAWLERS = ALL_CRAWLERS
+else:
+ CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERS
__all__ = ["ALL_CRAWLERS", "CRAWLERS", "DEBUG_CRAWLERS", "Crawler"]
diff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.py
deleted file mode 100644
index 13b3c0080..000000000
--- a/cyberdrop_dl/crawlers/bestprettygirl.py
+++ /dev/null
@@ -1,105 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, ClassVar
-
-from aiolimiter import AsyncLimiter
-
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.utils import css
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
-
-if TYPE_CHECKING:
- from bs4 import BeautifulSoup
-
- from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-
-
-class Selectors:
- IMAGES = "div.elementor-widget-theme-post-content div.elementor-widget-container img"
- VIDEO_IFRAME = "div.elementor-widget-theme-post-content div.elementor-widget-container iframe"
- TITLE = "meta[property='og:title']"
- DATE = "meta[property='article:published_time']"
- COLLECTION_ITEM = "article.elementor-post.post a"
-
-
-_SELECTORS = Selectors()
-
-COLLECTION_PARTS = "category", "tag", "date"
-
-PRIMARY_URL = AbsoluteHttpURL("https://bestprettygirl.com/")
-
-
-class BestPrettyGirlCrawler(Crawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Collection": COLLECTION_PARTS,
- "Gallery": "/<gallery_name>/",
- }
- DOMAIN: ClassVar[str] = "bestprettygirl.com"
- FOLDER_DOMAIN: ClassVar[str] = "BestPrettyGirl"
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
- NEXT_PAGE_SELECTOR = "a.page-numbers.next"
-
- def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(4, 1)
-
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- is_date = len(scrape_item.url.parts) > 3
- if any(p in scrape_item.url.parts for p in COLLECTION_PARTS) or is_date:
- return await self.collection(scrape_item)
- await self.gallery(scrape_item)
-
- @error_handling_wrapper
- async def collection(self, scrape_item: ScrapeItem) -> None:
- collection_type = title = ""
- async for soup in self.web_pager(scrape_item.url):
- if not collection_type:
- title_tag = soup.select_one(_SELECTORS.TITLE)
- if not title_tag:
- raise ScrapeError(422)
-
- for part in COLLECTION_PARTS:
- if part in scrape_item.url.parts:
- collection_type = part
- break
-
- if not collection_type:
- collection_type = "date"
- date_parts = scrape_item.url.parts[1:4]
- title = "-".join(date_parts)
-
- else:
- title: str = title_tag.get_text(strip=True)
- title = title.removeprefix("Tag:").removeprefix("Category:").strip()
- title = self.create_title(f"{title} [{collection_type}]")
-
- scrape_item.setup_as_album(title)
-
- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.COLLECTION_ITEM):
- self.manager.task_group.create_task(self.run(new_scrape_item))
-
- @error_handling_wrapper
- async def gallery(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
-
- og_title: str = css.select_one_get_attr(soup, _SELECTORS.TITLE, "content")
- date_str: str = css.select_one_get_attr(soup, _SELECTORS.DATE, "content")
- title = self.create_title(og_title)
- scrape_item.setup_as_album(title)
- scrape_item.possible_datetime = self.parse_date(date_str)
-
- trash_split = "-0000-"
- trash_len: int = 0
- for _, link in self.iter_tags(soup, _SELECTORS.IMAGES, "src"):
- if not trash_len:
- trash_len = link.name.find(trash_split) + len(trash_split)
- filename, ext = self.get_filename_and_ext(link.name)
- custom_filename = link.name[trash_len:]
- custom_filename, _ = self.get_filename_and_ext(custom_filename)
- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
-
- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_IFRAME, "data-src"):
- new_scrape_item.url = new_scrape_item.url.with_host("vidply.com")
- self.handle_external_links(new_scrape_item)
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 444d480f4..e5e062f4b 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -1,11 +1,11 @@
from __future__ import annotations
import asyncio
+import datetime
import re
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import dataclass, field
-from datetime import datetime
from functools import partial, wraps
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, TypeAlias, TypeVar, final
@@ -163,6 +163,8 @@ class Crawler(ABC):
if path_name != "Direct links":
assert paths, f"{cls.__name__} has not paths for {path_name}"
+ if path_name.startswith("*"): # note
+ return
if isinstance(paths, str):
paths = (paths,)
for path in paths:
@@ -363,21 +365,25 @@ class Crawler(ABC):
break
return title
+ @property
+ def separate_posts(self) -> bool:
+ return self.manager.config.download_options.separate_posts
+
def create_separate_post_title(
self,
title: str | None = None,
id: str | None = None,
- date: datetime | int | None = None,
+ date: datetime.datetime | datetime.date | int | None = None,
/,
) -> str:
- if not self.manager.config.download_options.separate_posts:
+ if not self.separate_posts:
return ""
title_format = self.manager.config.download_options.separate_posts_format
if title_format.strip().casefold() == "{default}":
title_format = self.DEFAULT_POST_TITLE_FORMAT
if isinstance(date, int):
- date = datetime.fromtimestamp(date)
- if isinstance(date, datetime):
+ date = datetime.datetime.fromtimestamp(date)
+ if isinstance(date, datetime.datetime | datetime.date):
date_str = date.isoformat()
else:
date_str: str | None = date
@@ -510,14 +516,23 @@ class Crawler(ABC):
if parsed_date := self._parse_date(date_or_datetime, format):
return to_timestamp(parsed_date)
- def _parse_date(self, date_or_datetime: str, format: str | None = None, /) -> datetime | None:
+ def parse_iso_date(self, date_or_datetime: str, /) -> TimeStamp | None:
+ if parsed_date := self._parse_date(date_or_datetime, None, iso=True):
+ return to_timestamp(parsed_date)
+
+ def _parse_date(
+ self, date_or_datetime: str, format: str | None = None, /, *, iso: bool = False
+ ) -> datetime.datetime | None:
+ assert not (iso and format)
msg = f"Date parsing for {self.DOMAIN} seems to be broken. Please report this as a bug at {NEW_ISSUE_URL}"
if not date_or_datetime:
log(f"{msg}: Unable to extract date from soup", 30)
return None
try:
- if format:
- parsed_date = datetime.strptime(date_or_datetime, format)
+ if iso:
+ parsed_date = datetime.datetime.fromisoformat(date_or_datetime)
+ elif format:
+ parsed_date = datetime.datetime.strptime(date_or_datetime, format)
else:
parsed_date = parse_date(date_or_datetime)
except (ValueError, TypeError) as e:
@@ -629,3 +644,33 @@ def make_custom_filename(stem: str, ext: str, extra_info: list[str], only_trunca
truncated_stem = truncate_str(stem, truncate_len)
return f"{truncated_stem}{ext}", invalid_extra_info_chars
+
+
+class Site(NamedTuple):
+ PRIMARY_URL: AbsoluteHttpURL
+ DOMAIN: str
+ SUPPORTED_DOMAINS: SupportedDomains = ()
+ FOLDER_DOMAIN: str = ""
+
+
+_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)
+
+
+def _create_subclass(url_string: str, base_class: type[_CrawlerT]) -> type[_CrawlerT]:
+ primary_url = AbsoluteHttpURL(url_string)
+ domain = primary_url.host.removeprefix("www.")
+ class_name = _make_crawler_name(domain)
+ class_attributes = Site(primary_url, domain)._asdict()
+ return type(class_name, (base_class,), class_attributes) # type: ignore[reportReturnType]
+
+
+def _make_crawler_name(input_string: str) -> str:
+ clean_string = re.sub(r"[^a-zA-Z0-9]+", " ", input_string).strip()
+ cap_name = clean_string.title().replace(" ", "")
+ assert cap_name and cap_name.isalnum(), (
+ f"Can not generate a valid class name from {input_string}. Needs to be defined as a concrete class"
+ )
+ if cap_name[0].isdigit():
+ cap_name = "_" + cap_name
+
+ return f"{cap_name}Crawler"
diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.py
new file mode 100644
index 000000000..565a55812
--- /dev/null
+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py
@@ -0,0 +1,14 @@
+"""All wordpress sites fruntion exactly the same. We can create subclasses dynamically by their URL"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .bestprettygirl import BestPrettyGirlCrawler
+
+if TYPE_CHECKING:
+ from ._wordpress import WordPressBaseCrawler
+
+WP_CRAWLERS: set[type[WordPressBaseCrawler]] = {BestPrettyGirlCrawler}
+WP_CRAWLERS_MAP = {c.__name__: c for c in WP_CRAWLERS}
+__all__ = list(WP_CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]
diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
new file mode 100644
index 000000000..7f440751f
--- /dev/null
+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
@@ -0,0 +1,355 @@
+"""General crawler for all wordpress sites
+
+Reference: https://developer.wordpress.org/rest-api/reference/#rest-api-developer-endpoint-reference
+"""
+
+from __future__ import annotations
+
+import datetime
+import itertools
+import re
+from abc import abstractmethod
+from typing import TYPE_CHECKING, ClassVar, TypeVar, final
+
+from aiolimiter import AsyncLimiter
+from bs4 import BeautifulSoup
+from pydantic import BaseModel
+
+from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.data_structures.url_objects import QueryDatetimeRange
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.utils import css, open_graph
+from cyberdrop_dl.utils.dates import to_timestamp
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, no_error, unique
+
+from .models import HTML, Category, CategorySequence, ColletionType, Post, PostSequence, Tag, TagSequence
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterable, Iterable
+
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+
+_T = TypeVar("_T")
+_ModelT = TypeVar("_ModelT", bound=BaseModel)
+_POST_PER_REQUEST = 100
+_HTTP_URL_REGEX = re.compile(
+ r"https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z]{2,12}\b([-a-zA-Z0-9@:%_+.~#?&/=]*)"
+) # Same as Xenforo
+_EXT_REGEX = re.compile(r"\d{2,}x\d{2,}(\.\w+)?")
+
+
+Selector = css.CssAttributeSelector
+
+
+class Selectors:
+ POST_CONTENT: ClassVar = Selector("#content, .entry-content [id*='post-']")
+ POST_ID: ClassVar = Selector("[id*='post-']", "id")
+ IMG: ClassVar = Selector("img[class*='wp-image']", "srcset")
+ POST_LINK_FROM_PAGE = Selector(".post a", "href")
+ NEXT_PAGE = Selector("a.page-numbers.next", "href")
+
+
+_SELECTORS = Selectors()
+_POST_ID_REGEX = re.compile(r"(?:postid-|post-)(\d+)")
+
+
+class WordPressBaseCrawler(Crawler, is_abc=True):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Category": "/category/<category_slug>",
+ "Tag": "/tag/<tag_slug>",
+ "Post": "/<post_slug>/",
+ "All Posts": "/posts/",
+ "Date Range": (
+ "...?before=<date>",
+ "...?after=<date>",
+ "...?before=<date&after=<date>",
+ ),
+ "**NOTE**": """
+
+ For `Date Range`, <date> must be a valid iso 8601 date, ex: `2022-12-06`.
+
+ `Date Range` can be combined with `Category`, `Tag` and `All Posts`.
+ ex: To only download categories from a date range: ,
+ `/category/<category_slug>?before=<date>`""",
+ }
+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"
+ WP_USE_REGEX: ClassVar = True
+ SUPPORTS_THREAD_RECURSION: ClassVar = False
+
+ def __init_subclass__(cls, **kwargs) -> None:
+ super().__init_subclass__(**kwargs)
+ assert cls.fetch is WordPressBaseCrawler.fetch
+ assert cls.fetch_with_date_range is WordPressBaseCrawler.fetch_with_date_range
+
+ def __post_init__(self) -> None:
+ self.request_limiter = AsyncLimiter(3, 1)
+
+ @final
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ date_range = QueryDatetimeRange.from_url(scrape_item.url)
+ scrape_item.url = scrape_item.url.with_query(None)
+ if date_range:
+ self.log(f"Scraping {scrape_item.url} with date range: {date_range.as_query()}")
+ return await self.fetch_with_date_range(scrape_item, date_range)
+
+ @property
+ @final
+ def separate_posts(self) -> bool:
+ # For wordpress we should always create a separate folder. Each post is an individual page
+ return True
+
+ @staticmethod
+ @final
+ def is_attachment(url: AbsoluteHttpURL) -> bool:
+ return "wp-content" in url.parts and bool(url.suffix)
+
+ @final
+ async def fetch_with_date_range(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["posts"]:
+ return await self.all_posts(scrape_item, date_range)
+ case [ColletionType.CATEGORY.value, _]:
+ return await self.category_or_tag(scrape_item, ColletionType.CATEGORY, date_range)
+ case [ColletionType.TAG.value, _]:
+ return await self.category_or_tag(scrape_item, ColletionType.TAG, date_range)
+ case ["wp-json", *_]:
+ raise ValueError
+
+ if _ := _match_date_from_path(scrape_item.url):
+ # TODO: Handle this
+ raise ValueError
+ return await self.post(scrape_item)
+
+ @abstractmethod
+ async def category_or_tag(
+ self, scrape_item: ScrapeItem, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None
+ ) -> None: ...
+
+ @abstractmethod
+ async def post(self, scrape_item: ScrapeItem) -> None: ...
+
+ @abstractmethod
+ async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None: ...
+
+ @final
+ async def filter_post(
+ self, scrape_item: ScrapeItem, post: Post, date_range: QueryDatetimeRange | None = None
+ ) -> None:
+ if date_range and not date_range.is_in_range(post.date_gmt):
+ self.log(f"Skipping post {post.link} as it is out of date range. Post date: {[post.date_gmt]}")
+ return
+ new_scrape_item = scrape_item.create_child(self.parse_url(post.link))
+ await self.handle_post(new_scrape_item, post)
+ scrape_item.add_children()
+
+ @final
+ async def handle_post(self, scrape_item: ScrapeItem, post: Post, *, is_single_post: bool = False) -> None:
+ post_id = str(post.id)
+ title = self.create_separate_post_title(post.title, post_id, post.date_gmt.date())
+ if is_single_post:
+ title = self.create_title(title)
+ scrape_item.setup_as_album(title, album_id=post_id)
+ scrape_item.possible_datetime = to_timestamp(post.date_gmt)
+ if post.thumbnail:
+ await self.direct_file(scrape_item, self.parse_url(post.thumbnail))
+ return await self.handle_post_content(scrape_item, post)
+
+ @final
+ async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:
+ if self.is_attachment(link):
+ return await self.direct_file(scrape_item, link)
+ if self.PRIMARY_URL.host == link.host:
+ return
+ new_scrape_item = scrape_item.create_new(link)
+ new_scrape_item.type = None
+ new_scrape_item.reset_childen()
+ self.handle_external_links(new_scrape_item)
+ scrape_item.add_children()
+
+ async def handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
+ for link in self.iter_parse_url(_iter_links(post.content, self.WP_USE_REGEX)):
+ await self.handle_link(scrape_item, link)
+
+ def parse_url(self, link: str) -> AbsoluteHttpURL:
+ # TODO: handle more domains and move it to the base crawler
+ link = _get_original_quality_link(link)
+ url = super().parse_url(link)
+ if url.host == "ouo.io" and (redirect_url := url.query.get("s")):
+ return super().parse_url(redirect_url)
+ return url
+
+ @final
+ def iter_parse_url(self, iterable: Iterable[str]) -> Iterable[AbsoluteHttpURL]:
+ for link_str in unique(iterable):
+ try:
+ yield self.parse_url(link_str)
+ except Exception:
+ continue
+
+
+class WordPressAPICrawler(WordPressBaseCrawler, is_abc=True):
+ WP_CATEGORIES_ENDPOINT: ClassVar = "/wp-json/wp/v2/categories"
+ WP_TAGS_ENDPOINT: ClassVar = "/wp-json/wp/v2/tags"
+ WP_POSTS_ENDPOINT: ClassVar = "/wp-json/wp/v2/posts"
+
+ def __init_subclass__(cls, is_abc: bool = False, **kwargs) -> None:
+ super().__init_subclass__(is_abc=is_abc, **kwargs)
+ if is_abc:
+ return
+ cls.WP_API_CATEGORIES_URL = cls.PRIMARY_URL / cls.WP_CATEGORIES_ENDPOINT.removeprefix("/")
+ cls.WP_API_TAGS_URL = cls.PRIMARY_URL / cls.WP_TAGS_ENDPOINT.removeprefix("/")
+ cls.WP_API_POSTS_URL = cls.PRIMARY_URL / cls.WP_POSTS_ENDPOINT.removeprefix("/")
+
+ @error_handling_wrapper
+ async def category_or_tag(
+ self, scrape_item: ScrapeItem, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None
+ ) -> None:
+ if colletion_type is ColletionType.CATEGORY:
+ model, api_url = CategorySequence, self.WP_API_CATEGORIES_URL.with_query(slug=scrape_item.url.name)
+ else:
+ model, api_url = TagSequence, self.WP_API_TAGS_URL.with_query(slug=scrape_item.url.name)
+ collections = await self.__make_request(model, api_url)
+ if not collections:
+ raise ScrapeError(404)
+ await self.__handle_collection(scrape_item, collections[0], date_range)
+
+ @error_handling_wrapper
+ async def post(self, scrape_item: ScrapeItem) -> None:
+ posts = await self.__make_request(PostSequence, self.WP_API_POSTS_URL.with_query(slug=scrape_item.url.name))
+ if not posts:
+ raise ScrapeError(404)
+ return await self.handle_post(scrape_item, posts[0], is_single_post=True)
+
+ @error_handling_wrapper
+ async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
+ api_url = self.WP_API_POSTS_URL
+ if date_range:
+ api_url = api_url.update_query(date_range.as_query())
+ scrape_item.setup_as_profile(self.create_title("Posts"))
+ async for post in self.post_pager(api_url):
+ await self.filter_post(scrape_item, post)
+
+ async def __make_request(self, model_cls: type[_ModelT], api_url: AbsoluteHttpURL) -> _ModelT:
+ async with self.request_limiter:
+ json_text = await self.client.get_text(self.DOMAIN, api_url)
+ return model_cls.model_validate_json(json_text)
+
+ async def __handle_collection(
+ self, scrape_item: ScrapeItem, collection: Category | Tag, date_range: QueryDatetimeRange | None = None
+ ) -> None:
+ title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")
+ scrape_item.setup_as_profile(title)
+ assert collection.id
+ api_url = self.WP_API_POSTS_URL.with_query({collection._type: collection.id})
+ if date_range:
+ api_url = api_url.update_query(date_range.as_query())
+ async for post in self.post_pager(api_url):
+ await self.filter_post(scrape_item, post)
+
+ async def post_pager(self, url: AbsoluteHttpURL, init_page: int | None = None) -> AsyncIterable[Post]:
+ had_at_least_1_post = False
+ for page in itertools.count(init_page or 1):
+ n_post = 0
+ api_url = url.update_query(per_page=_POST_PER_REQUEST, page=page)
+ for post in await self.__make_request(PostSequence, api_url):
+ n_post += 1
+ had_at_least_1_post = True
+ yield post
+ if n_post < _POST_PER_REQUEST:
+ break
+ if not had_at_least_1_post:
+ raise ScrapeError(404)
+
+
+class WordPressSoupCrawler(WordPressBaseCrawler, is_abc=True):
+ async def __make_request(self, api_url: AbsoluteHttpURL) -> BeautifulSoup:
+ async with self.request_limiter:
+ return await self.client.get_soup(self.DOMAIN, api_url)
+
+ async def category_or_tag(
+ self, scrape_item: ScrapeItem, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None
+ ) -> None:
+ if colletion_type is ColletionType.CATEGORY:
+ collection = Category(slug=scrape_item.url.name, link=str(scrape_item.url))
+ else:
+ collection = Tag(slug=scrape_item.url.name, link=str(scrape_item.url))
+ await self.__handle_collection(scrape_item, collection, date_range)
+
+ async def __handle_collection(
+ self, scrape_item: ScrapeItem, collection: Category | Tag, date_range: QueryDatetimeRange | None = None
+ ) -> None:
+ title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")
+ scrape_item.setup_as_profile(title)
+ await self.post_url_pager(scrape_item, date_range)
+
+ async def post(self, scrape_item: ScrapeItem) -> None:
+ soup = await self.__make_request(scrape_item.url)
+ post = self.parse_post(scrape_item, soup)
+ return await self.handle_post(scrape_item, post, is_single_post=True)
+
+ def parse_post(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> Post:
+ data = {
+ "post_id": get_post_id(soup),
+ "slug": scrape_item.url.name,
+ "title": open_graph.get_title(soup),
+ "date_gmt": open_graph.get("published_time", soup),
+ "link": str(scrape_item.url),
+ "content": _SELECTORS.POST_CONTENT(soup),
+ }
+ return Post.model_validate(data)
+
+ async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
+ scrape_item.setup_as_profile(self.create_title("Posts"))
+ await self.post_url_pager(scrape_item, date_range)
+
+ async def post_url_pager(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:
+ async for soup in self.web_pager(scrape_item.url):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.POST_LINK_FROM_PAGE.element):
+ if (
+ date_range
+ and (date_from_path := _match_date_from_path(new_scrape_item.url))
+ and not date_range.is_in_range(date_from_path)
+ ):
+ self.log(
+ f"Skipping post {new_scrape_item.url} as it is out of date range. Post date: {{date_from_path}}"
+ )
+ continue
+ self.manager.task_group.create_task(self.run(new_scrape_item))
+
+
+@no_error
+def _match_date_from_path(url: AbsoluteHttpURL) -> datetime.datetime | None:
+ match url.parts[1:]:
+ case [year, month, day]:
+ return datetime.datetime(int(year), int(month), int(day))
+ case _:
+ return
+
+
+def _get_original_quality_link(link: str) -> str:
+ stem, _, tail = link.rpartition("-")
+ if match := re.search(_EXT_REGEX, tail):
+ return f"{stem}.{match.group()}"
+ return link
+
+
+def _iter_links(html: HTML, use_regex: bool) -> itertools.chain[str]:
+ soup = BeautifulSoup(html, "html.parser")
+ images = css.iget(soup, *css.images)
+ iframes = css.iget(soup, *css.iframes)
+ if use_regex:
+ regex = (match.group() for match in re.finditer(_HTTP_URL_REGEX, html))
+ return itertools.chain(images, iframes, regex)
+ return itertools.chain(images, iframes)
+
+
+@no_error
+def get_post_id(soup: BeautifulSoup) -> int:
+ id_text = _SELECTORS.POST_ID(soup)
+ if match := _POST_ID_REGEX.search(id_text):
+ post_id = match.group(1)
+ for trash in ("post", "id", "-"):
+ post_id = post_id.removeprefix(trash)
+ return int(post_id)
+ raise ValueError
diff --git a/cyberdrop_dl/crawlers/wordpress/bestprettygirl.py b/cyberdrop_dl/crawlers/wordpress/bestprettygirl.py
new file mode 100644
index 000000000..662b51859
--- /dev/null
+++ b/cyberdrop_dl/crawlers/wordpress/bestprettygirl.py
@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from typing import ClassVar
+
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+
+from ._wordpress import WordPressAPICrawler
+
+
+class BestPrettyGirlCrawler(WordPressAPICrawler):
+ DOMAIN: ClassVar[str] = "bestprettygirl.com"
+ FOLDER_DOMAIN: ClassVar[str] = "BestPrettyGirl"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bestprettygirl.com/")
diff --git a/cyberdrop_dl/crawlers/wordpress/models.py b/cyberdrop_dl/crawlers/wordpress/models.py
new file mode 100644
index 000000000..11f614d8e
--- /dev/null
+++ b/cyberdrop_dl/crawlers/wordpress/models.py
@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Annotated, Literal, NewType, TypeVar
+
+from bs4 import BeautifulSoup
+from pydantic import AfterValidator, AliasPath, BaseModel, Field
+
+from cyberdrop_dl.compat import StrEnum
+from cyberdrop_dl.models.base_models import SequenceModel
+
+_ModelT = TypeVar("_ModelT", bound=BaseModel)
+
+
+def make_soup(string: str) -> BeautifulSoup:
+ return BeautifulSoup(string, "html.parser")
+
+
+def unescape_html(string: str) -> str:
+ return make_soup(string).get_text(strip=True)
+
+
+def add_utc_tz(parsed_date: datetime) -> datetime:
+ return parsed_date.replace(tzinfo=UTC)
+
+
+TitleFromHTML = Annotated[str, AfterValidator(unescape_html)]
+AwareDatetimeUTC = Annotated[datetime, AfterValidator(add_utc_tz)]
+
+
+class ColletionType(StrEnum):
+ TAG = "tag"
+ CATEGORY = "category"
+
+
+HTML = NewType("HTML", str)
+
+
+class WordPressModel(BaseModel):
+ id: int | None = None
+ slug: str
+ link: str
+
+
+class Post(WordPressModel):
+ title: TitleFromHTML = Field(validation_alias=AliasPath("title", "rendered"))
+ content: HTML = Field(validation_alias=AliasPath("content", "rendered"))
+ thumbnail: str | None = Field(default=None, validation_alias=AliasPath("acf", "fifu_image_url"))
+ date_gmt: AwareDatetimeUTC
+
+
+class Collection(WordPressModel):
+ description: str = ""
+ taxonomy: str
+
+
+class Category(Collection):
+ taxonomy: Literal["category"] = "category"
+ _type: ColletionType = ColletionType.CATEGORY
+
+
+class Tag(Category):
+ taxonomy: Literal["post_tag"] = "post_tag"
+ _type: ColletionType = ColletionType.TAG
+
+
+class PostSequence(SequenceModel[Post]): ...
+
+
+class TagSequence(SequenceModel[Tag]): ...
+
+
+class CategorySequence(SequenceModel[Category]): ...
+
+
+class PostExtraData(Post):
+ # Not used at the moment
+ date: datetime
+ modified: datetime
+ modified_gmt: AwareDatetimeUTC
+ status: Literal["publish", "future", "draft", "pending", "private"]
+ type: str
+ categories: list[int] = []
+ tags: list[int] = []
+ format: Literal["standard", "aside", "chat", "gallery", "link", "image", "quote", "status", "video", "audio"]
+
+
+class CollectionExtraData(Collection):
+ # Not used at the moment
+ count: int
+ parent: int
diff --git a/cyberdrop_dl/crawlers/xenforo/__init__.py b/cyberdrop_dl/crawlers/xenforo/__init__.py
index 1216fb82f..b8715e05a 100644
--- a/cyberdrop_dl/crawlers/xenforo/__init__.py
+++ b/cyberdrop_dl/crawlers/xenforo/__init__.py
@@ -23,5 +23,5 @@ XF_CRAWLERS: set[type[XenforoCrawler]] = {
XBunkerCrawler,
}
-CRAWLERS_MAP = {c.__name__: c for c in XF_CRAWLERS}
-__all__ = list(CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]
+XF_CRAWLERS_MAP = {c.__name__: c for c in XF_CRAWLERS}
+__all__ = list(XF_CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 0035d8845..8f4057e20 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -6,7 +6,7 @@ from dataclasses import InitVar, dataclass, field
from enum import IntEnum
from functools import partialmethod
from pathlib import Path
-from typing import TYPE_CHECKING, Literal, NamedTuple, ParamSpec, Self, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, ParamSpec, Self, TypeVar, overload
import yarl
@@ -17,6 +17,7 @@ R = TypeVar("R")
T = TypeVar("T")
if TYPE_CHECKING:
+ import datetime
import functools
import inspect
from collections.abc import Callable
@@ -269,13 +270,13 @@ class ScrapeItem:
def create_new(
self,
- url: yarl.URL,
+ url: AbsoluteHttpURL,
*,
new_title_part: str = "",
part_of_album: bool = False,
album_id: str | None = None,
possible_datetime: int | None = None,
- add_parent: yarl.URL | bool | None = None,
+ add_parent: AbsoluteHttpURL | bool | None = None,
) -> ScrapeItem:
"""Creates a scrape item."""
from cyberdrop_dl.utils.utilities import is_absolute_http_url
@@ -284,7 +285,7 @@ class ScrapeItem:
assert is_absolute_http_url(url)
scrape_item.url = url
if add_parent:
- new_parent = add_parent if isinstance(add_parent, yarl.URL) else self.url
+ new_parent = add_parent if isinstance(add_parent, AbsoluteHttpURL) else self.url
assert is_absolute_http_url(new_parent)
scrape_item.parents.append(new_parent)
if new_title_part:
@@ -300,14 +301,43 @@ class ScrapeItem:
setup_as_forum = partialmethod(setup_as, type=FORUM)
setup_as_post = partialmethod(setup_as, type=FORUM_POST)
- def origin(self) -> yarl.URL | None:
+ def origin(self) -> AbsoluteHttpURL | None:
if self.parents:
return self.parents[0]
- def parent(self) -> yarl.URL | None:
+ def parent(self) -> AbsoluteHttpURL | None:
if self.parents:
return self.parents[-1]
def copy(self) -> Self:
"""Returns a deep copy of this scrape_item"""
return copy.deepcopy(self)
+
+
+class QueryDatetimeRange(NamedTuple):
+ before: datetime.datetime | None = None
+ after: datetime.datetime | None = None
+
+ @staticmethod
+ def from_url(url: AbsoluteHttpURL) -> QueryDatetimeRange | None:
+ self = QueryDatetimeRange(_date_from_query_param(url, "before"), _date_from_query_param(url, "after"))
+ if self == (None, None):
+ return None
+ if (self.before and self.after) and (self.before <= self.after):
+ raise ValueError
+ return self
+
+ def is_in_range(self, other: datetime.datetime) -> bool:
+ if (self.before and other >= self.before) or (self.after and other <= self.after):
+ return False
+ return True
+
+ def as_query(self) -> dict[str, Any]:
+ return {name: value.isoformat() for name, value in self._asdict().items() if value}
+
+
+def _date_from_query_param(url: AbsoluteHttpURL, query_param: str) -> datetime.datetime | None:
+ from cyberdrop_dl.utils.dates import parse_aware_iso_datetime
+
+ if value := url.query.get(query_param):
+ return parse_aware_iso_datetime(value)
diff --git a/cyberdrop_dl/models/base_models.py b/cyberdrop_dl/models/base_models.py
index d50fca9f0..f067f8d65 100755
--- a/cyberdrop_dl/models/base_models.py
+++ b/cyberdrop_dl/models/base_models.py
@@ -1,14 +1,26 @@
"""Pydantic models"""
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping, Sequence
from pathlib import Path
+from typing import TypeVar
import yarl
-from pydantic import AnyUrl, BaseModel, ConfigDict, Secret, SerializationInfo, model_serializer, model_validator
+from pydantic import (
+ AnyUrl,
+ BaseModel,
+ ConfigDict,
+ RootModel,
+ Secret,
+ SerializationInfo,
+ model_serializer,
+ model_validator,
+)
from cyberdrop_dl.models.types import HttpURL
from cyberdrop_dl.models.validators import to_apprise_url_dict
+_ModelT = TypeVar("_ModelT", bound=BaseModel)
+
class AliasModel(BaseModel):
model_config = ConfigDict(populate_by_name=True)
@@ -51,3 +63,17 @@ class PathAliasModel(AliasModel):
setattr(self, name, self.replace_config_and_resolve(value, config_name))
elif isinstance(value, PathAliasModel):
value.resolve_paths(config_name)
+
+
+class SequenceModel(RootModel[list[_ModelT]], Sequence[_ModelT]):
+ def __len__(self) -> int:
+ return len(self.root)
+
+ def __iter__(self) -> Iterator[_ModelT]:
+ yield from self.root
+
+ def __getitem__(self, index: int) -> _ModelT:
+ return self.root[index]
+
+ def __bool__(self) -> bool:
+ return bool(len(self))
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 2ed1342a4..e368b4675 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -8,7 +8,7 @@ import bs4.css
from cyberdrop_dl.exceptions import ScrapeError
if TYPE_CHECKING:
- from collections.abc import Callable, Generator
+ from collections.abc import Callable, Generator, Iterable
from bs4 import Tag
@@ -59,10 +59,16 @@ def select_one_get_text(tag: Tag, selector: str, strip: bool = True, *, decompos
# get_attr_no_error should be get_attr_or_none. `or_none` implies no error
def get_attr_or_none(tag: Tag, attribute: str) -> str | None:
"""Same as `tag.get(attribute)` but asserts the result is a single str"""
- if attribute == "src":
- value = tag.get("data-src") or tag.get(attribute)
+ attribute_ = attribute
+ if attribute_ == "srcset" and (srcset := tag.get(attribute_)):
+ if isinstance(srcset, str):
+ return _parse_srcset(srcset)
+ attribute_ = "src"
+
+ if attribute_ == "src":
+ value = tag.get("data-src") or tag.get(attribute_)
else:
- value = tag.get(attribute)
+ value = tag.get(attribute_)
if isinstance(value, list):
raise SelectorError(f"Expected a single value for {attribute = !r}, got multiple")
return value
@@ -99,3 +105,17 @@ def select_one_get_attr_or_none(tag: Tag, selector: str, attribute: str) -> str
def iselect(tag: Tag, selector: str) -> Generator[Tag]:
"""Same as `tag.select(selector)`, but it returns a generator instead of a list."""
yield from bs4.css.CSS(tag).iselect(selector)
+
+
+def _parse_srcset(srcset: str) -> str:
+ # The best src is the last one (usually)
+ return [src.split(" ")[0] for src in srcset.split(", ")][-1]
+
+
+def iget(tag: Tag, selector: str, attribute: str) -> Iterable[str]:
+ for inner_tag in iselect(tag, selector):
+ yield get_attr(inner_tag, attribute)
+
+
+iframes = CssAttributeSelector("iframe", "src")
+images = CssAttributeSelector("img", "srcset")
diff --git a/cyberdrop_dl/utils/dates.py b/cyberdrop_dl/utils/dates.py
index 3c148d628..7943294fb 100644
--- a/cyberdrop_dl/utils/dates.py
+++ b/cyberdrop_dl/utils/dates.py
@@ -142,6 +142,16 @@ def get_parser(parser_kind: ParserKind | None = None, date_order: DateOrder | No
return DateParser(parser_kind, date_order)
+def parse_aware_iso_datetime(value: str) -> datetime.datetime | None:
+ try:
+ parsed_date = datetime.datetime.fromisoformat(value)
+ if parsed_date.tzinfo is None:
+ parsed_date.replace(tzinfo=datetime.UTC)
+ return parsed_date
+ except Exception:
+ return
+
+
if __name__ == "__main__":
print(parse_date("today at noon")) # noqa: T201
print(parse_date("today")) # noqa: T201
diff --git a/cyberdrop_dl/utils/open_graph.py b/cyberdrop_dl/utils/open_graph.py
new file mode 100644
index 000000000..1f73c3bbb
--- /dev/null
+++ b/cyberdrop_dl/utils/open_graph.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ import bs4
+
+
[email protected](frozen=True, slots=True)
+class OpenGraph:
+ """Open Graph properties. Each attribute corresponds to an OG property."""
+
+ required_attrs = ("title", "type", "image", "url", "description")
+
+ audio: str | None = None
+ description: str | None = None
+ determiner: str | None = None
+ image: str | None = None
+ image: str | None = None
+ locale: str | None = None
+ published_time: str | None = None
+ site_name: str | None = None
+ title: str | None = None
+ type: str | None = None
+ url: str | None = None
+ video: str | None = None
+
+ def is_valid(self) -> bool:
+ return all(getattr(self, attr, False) for attr in self.required_attrs)
+
+
+_og_fields = dataclasses.fields(OpenGraph)
+
+
+def parse(soup: bs4.BeautifulSoup) -> OpenGraph:
+ """Extracts Open Graph properties (og properties) from soup."""
+ props: dict[str, str | None] = {}
+ for meta in soup.select('meta[property^="og:"][content]'):
+ property_name = meta["property"].replace("og:", "") # type: ignore
+ if property_name in _og_fields:
+ props[property_name] = meta["content"] or None # type: ignore
+
+ if not props.get("title") and (title := soup.select_one("title, h1")):
+ props["title"] = title.get_text(strip=True) or None
+
+ return OpenGraph(**props)
+
+
+def get(name: str, /, soup: bs4.BeautifulSoup) -> str | None:
+ if meta := soup.select_one(f'meta[property^="og:{name}"][content]'):
+ return meta["content"] or None # type: ignore
+
+
+def get_title(soup: bs4.BeautifulSoup) -> str | None:
+ return get("title", soup)
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 80255e0cb..cc498627a 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -38,7 +38,7 @@ from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.logger import log, log_debug, log_spacer, log_with_color
if TYPE_CHECKING:
- from collections.abc import Callable, Coroutine, Mapping
+ from collections.abc import Callable, Coroutine, Iterable, Mapping
from curl_cffi.requests.models import Response as CurlResponse
from rich.text import Text
@@ -618,6 +618,46 @@ def is_blob_or_svg(link: str) -> bool:
return any(link.startswith(x) for x in _BLOB_OR_SVG)
+def no_error(func: Callable[P, R]) -> Callable[P, R | None]:
+ def call(*args, **kwargs) -> R | None:
+ try:
+ return func(*args, **kwargs)
+ except Exception:
+ return
+
+ return call
+
+
+def unique(iterable: Iterable[T], *, hashable: bool = True) -> Iterable[T]:
+ """Yields unique values from iterable, keeping original order"""
+ if hashable:
+ seen: set[T] | list[T] = set()
+ add: Callable[[T], None] = seen.add
+ else:
+ seen = []
+ add = seen.append
+
+ for value in iterable:
+ if value not in seen:
+ add(value)
+ yield value
+
+
+def get_valid_kwargs(func: Callable[..., Any], kwargs: Mapping[str, T], accept_kwargs: bool = True) -> Mapping[str, T]:
+ """Get the subset of ``kwargs`` that are valid params for ``func`` and their values are not `None`
+
+ If func takes **kwargs, returns everything"""
+ params = inspect.signature(func).parameters
+ if accept_kwargs and any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
+ return kwargs
+
+ return {k: v for k, v in kwargs.items() if k in params.keys() and v is not None}
+
+
+def call_w_valid_kwargs(cls: Callable[..., R], kwargs: Mapping[str, Any]) -> R:
+ return cls(**get_valid_kwargs(cls, kwargs))
+
+
log_cyan = partial(log_with_color, style="cyan", level=20)
log_yellow = partial(log_with_color, style="yellow", level=20)
log_green = partial(log_with_color, style="green", level=20)