Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add support for any wordpress site (#1076)
NTFSvolumecc5a58ba4412
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.pyindex c3c70df9e..797886155 100755--- a/cyberdrop_dl/config/config_model.py+++ b/cyberdrop_dl/config/config_model.py@@ -51,6 +51,7 @@ class DownloadOptions(BaseModel):valid_keys = ("default", "title", "id", "number", "date")try:value.format(**dict.fromkeys(valid_keys, "TEST"))+ return valueexcept KeyError as e:msg = f"'{e.args[0]}' is not a valid key for this option. Valid keys: {valid_keys}"raise ValueError(msg) from Nonediff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 48024e41d..82fd80b42 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -6,7 +6,6 @@ from cyberdrop_dl import envfrom .acidimg import AcidImgCrawlerfrom .archivebate import ArchiveBateCrawlerfrom .ashemaletube import AShemaleTubeCrawler-from .bestprettygirl import BestPrettyGirlCrawlerfrom .box_dot_com import BoxDotComCrawlerfrom .bunkrr import BunkrrCrawlerfrom .bunkrr_albums_io import BunkrAlbumsIOCrawler@@ -91,6 +90,7 @@ from .twitter_images import TwimgCrawlerfrom .twpornstars import TwPornstarsCrawlerfrom .vipr_dot_im import ViprImCrawlerfrom .wetransfer import WeTransferCrawler+from .wordpress import WP_CRAWLERSfrom .xbunkr import XBunkrCrawlerfrom .xenforo import XF_CRAWLERS, SimpCityCrawlerfrom .xhamster import XhamsterCrawler@@ -100,11 +100,11 @@ from .youjizz import YouJizzCrawlerALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler")}ALL_CRAWLERS = ALL_CRAWLERS - {Crawler}-ALL_CRAWLERS.update(XF_CRAWLERS)+ALL_CRAWLERS.update(XF_CRAWLERS, WP_CRAWLERS)DEBUG_CRAWLERS = {SimpCityCrawler, BunkrAlbumsIOCrawler, MegaNzCrawler, FikFapCrawler}-CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERS-if env.ENABLE_DEBUG_CRAWLERS == "d396ab8c85fcb1fecd22c8d9b58acf944a44e6d35014e9dd39e42c9a64091eda":- CRAWLERS.update(DEBUG_CRAWLERS)+ CRAWLERS = ALL_CRAWLERS+else:+ CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERS__all__ = ["ALL_CRAWLERS", "CRAWLERS", "DEBUG_CRAWLERS", "Crawler"]diff --git a/cyberdrop_dl/crawlers/bestprettygirl.py b/cyberdrop_dl/crawlers/bestprettygirl.pydeleted file mode 100644index 13b3c0080..000000000--- a/cyberdrop_dl/crawlers/bestprettygirl.py+++ /dev/null@@ -1,105 +0,0 @@-from __future__ import annotations--from typing import TYPE_CHECKING, ClassVar--from aiolimiter import AsyncLimiter--from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import ScrapeError-from cyberdrop_dl.utils import css-from cyberdrop_dl.utils.utilities import error_handling_wrapper--if TYPE_CHECKING:- from bs4 import BeautifulSoup-- from cyberdrop_dl.data_structures.url_objects import ScrapeItem---class Selectors:- IMAGES = "div.elementor-widget-theme-post-content div.elementor-widget-container img"- VIDEO_IFRAME = "div.elementor-widget-theme-post-content div.elementor-widget-container iframe"- TITLE = "meta[property='og:title']"- DATE = "meta[property='article:published_time']"- COLLECTION_ITEM = "article.elementor-post.post a"---_SELECTORS = Selectors()--COLLECTION_PARTS = "category", "tag", "date"--PRIMARY_URL = AbsoluteHttpURL("https://bestprettygirl.com/")---class BestPrettyGirlCrawler(Crawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Collection": COLLECTION_PARTS,- "Gallery": "/<gallery_name>/",- }- DOMAIN: ClassVar[str] = "bestprettygirl.com"- FOLDER_DOMAIN: ClassVar[str] = "BestPrettyGirl"- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL- NEXT_PAGE_SELECTOR = "a.page-numbers.next"-- def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(4, 1)-- async def fetch(self, scrape_item: ScrapeItem) -> None:- is_date = len(scrape_item.url.parts) > 3- if any(p in scrape_item.url.parts for p in COLLECTION_PARTS) or is_date:- return await self.collection(scrape_item)- await self.gallery(scrape_item)-- @error_handling_wrapper- async def collection(self, scrape_item: ScrapeItem) -> None:- collection_type = title = ""- async for soup in self.web_pager(scrape_item.url):- if not collection_type:- title_tag = soup.select_one(_SELECTORS.TITLE)- if not title_tag:- raise ScrapeError(422)-- for part in COLLECTION_PARTS:- if part in scrape_item.url.parts:- collection_type = part- break-- if not collection_type:- collection_type = "date"- date_parts = scrape_item.url.parts[1:4]- title = "-".join(date_parts)-- else:- title: str = title_tag.get_text(strip=True)- title = title.removeprefix("Tag:").removeprefix("Category:").strip()- title = self.create_title(f"{title} [{collection_type}]")-- scrape_item.setup_as_album(title)-- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.COLLECTION_ITEM):- self.manager.task_group.create_task(self.run(new_scrape_item))-- @error_handling_wrapper- async def gallery(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)-- og_title: str = css.select_one_get_attr(soup, _SELECTORS.TITLE, "content")- date_str: str = css.select_one_get_attr(soup, _SELECTORS.DATE, "content")- title = self.create_title(og_title)- scrape_item.setup_as_album(title)- scrape_item.possible_datetime = self.parse_date(date_str)-- trash_split = "-0000-"- trash_len: int = 0- for _, link in self.iter_tags(soup, _SELECTORS.IMAGES, "src"):- if not trash_len:- trash_len = link.name.find(trash_split) + len(trash_split)- filename, ext = self.get_filename_and_ext(link.name)- custom_filename = link.name[trash_len:]- custom_filename, _ = self.get_filename_and_ext(custom_filename)- await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)-- for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.VIDEO_IFRAME, "data-src"):- new_scrape_item.url = new_scrape_item.url.with_host("vidply.com")- self.handle_external_links(new_scrape_item)diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 444d480f4..e5e062f4b 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -1,11 +1,11 @@from __future__ import annotationsimport asyncio+import datetimeimport refrom abc import ABC, abstractmethodfrom collections.abc import Mappingfrom dataclasses import dataclass, field-from datetime import datetimefrom functools import partial, wrapsfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, TypeAlias, TypeVar, final@@ -163,6 +163,8 @@ class Crawler(ABC):if path_name != "Direct links":assert paths, f"{cls.__name__} has not paths for {path_name}"+ if path_name.startswith("*"): # note+ returnif isinstance(paths, str):paths = (paths,)for path in paths:@@ -363,21 +365,25 @@ class Crawler(ABC):breakreturn title+ @property+ def separate_posts(self) -> bool:+ return self.manager.config.download_options.separate_posts+def create_separate_post_title(self,title: str | None = None,id: str | None = None,- date: datetime | int | None = None,+ date: datetime.datetime | datetime.date | int | None = None,/,) -> str:- if not self.manager.config.download_options.separate_posts:+ if not self.separate_posts:return ""title_format = self.manager.config.download_options.separate_posts_formatif title_format.strip().casefold() == "{default}":title_format = self.DEFAULT_POST_TITLE_FORMATif isinstance(date, int):- date = datetime.fromtimestamp(date)- if isinstance(date, datetime):+ date = datetime.datetime.fromtimestamp(date)+ if isinstance(date, datetime.datetime | datetime.date):date_str = date.isoformat()else:date_str: str | None = date@@ -510,14 +516,23 @@ class Crawler(ABC):if parsed_date := self._parse_date(date_or_datetime, format):return to_timestamp(parsed_date)- def _parse_date(self, date_or_datetime: str, format: str | None = None, /) -> datetime | None:+ def parse_iso_date(self, date_or_datetime: str, /) -> TimeStamp | None:+ if parsed_date := self._parse_date(date_or_datetime, None, iso=True):+ return to_timestamp(parsed_date)++ def _parse_date(+ self, date_or_datetime: str, format: str | None = None, /, *, iso: bool = False+ ) -> datetime.datetime | None:+ assert not (iso and format)msg = f"Date parsing for {self.DOMAIN} seems to be broken. Please report this as a bug at {NEW_ISSUE_URL}"if not date_or_datetime:log(f"{msg}: Unable to extract date from soup", 30)return Nonetry:- if format:- parsed_date = datetime.strptime(date_or_datetime, format)+ if iso:+ parsed_date = datetime.datetime.fromisoformat(date_or_datetime)+ elif format:+ parsed_date = datetime.datetime.strptime(date_or_datetime, format)else:parsed_date = parse_date(date_or_datetime)except (ValueError, TypeError) as e:@@ -629,3 +644,33 @@ def make_custom_filename(stem: str, ext: str, extra_info: list[str], only_truncatruncated_stem = truncate_str(stem, truncate_len)return f"{truncated_stem}{ext}", invalid_extra_info_chars+++class Site(NamedTuple):+ PRIMARY_URL: AbsoluteHttpURL+ DOMAIN: str+ SUPPORTED_DOMAINS: SupportedDomains = ()+ FOLDER_DOMAIN: str = ""+++_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)+++def _create_subclass(url_string: str, base_class: type[_CrawlerT]) -> type[_CrawlerT]:+ primary_url = AbsoluteHttpURL(url_string)+ domain = primary_url.host.removeprefix("www.")+ class_name = _make_crawler_name(domain)+ class_attributes = Site(primary_url, domain)._asdict()+ return type(class_name, (base_class,), class_attributes) # type: ignore[reportReturnType]+++def _make_crawler_name(input_string: str) -> str:+ clean_string = re.sub(r"[^a-zA-Z0-9]+", " ", input_string).strip()+ cap_name = clean_string.title().replace(" ", "")+ assert cap_name and cap_name.isalnum(), (+ f"Can not generate a valid class name from {input_string}. Needs to be defined as a concrete class"+ )+ if cap_name[0].isdigit():+ cap_name = "_" + cap_name++ return f"{cap_name}Crawler"diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.pynew file mode 100644index 000000000..565a55812--- /dev/null+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py@@ -0,0 +1,14 @@+"""All wordpress sites fruntion exactly the same. We can create subclasses dynamically by their URL"""++from __future__ import annotations++from typing import TYPE_CHECKING++from .bestprettygirl import BestPrettyGirlCrawler++if TYPE_CHECKING:+ from ._wordpress import WordPressBaseCrawler++WP_CRAWLERS: set[type[WordPressBaseCrawler]] = {BestPrettyGirlCrawler}+WP_CRAWLERS_MAP = {c.__name__: c for c in WP_CRAWLERS}+__all__ = list(WP_CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.pynew file mode 100644index 000000000..7f440751f--- /dev/null+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py@@ -0,0 +1,355 @@+"""General crawler for all wordpress sites++Reference: https://developer.wordpress.org/rest-api/reference/#rest-api-developer-endpoint-reference+"""++from __future__ import annotations++import datetime+import itertools+import re+from abc import abstractmethod+from typing import TYPE_CHECKING, ClassVar, TypeVar, final++from aiolimiter import AsyncLimiter+from bs4 import BeautifulSoup+from pydantic import BaseModel++from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.data_structures.url_objects import QueryDatetimeRange+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.utils import css, open_graph+from cyberdrop_dl.utils.dates import to_timestamp+from cyberdrop_dl.utils.utilities import error_handling_wrapper, no_error, unique++from .models import HTML, Category, CategorySequence, ColletionType, Post, PostSequence, Tag, TagSequence++if TYPE_CHECKING:+ from collections.abc import AsyncIterable, Iterable++ from cyberdrop_dl.crawlers.crawler import SupportedPaths+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem++_T = TypeVar("_T")+_ModelT = TypeVar("_ModelT", bound=BaseModel)+_POST_PER_REQUEST = 100+_HTTP_URL_REGEX = re.compile(+ r"https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z]{2,12}\b([-a-zA-Z0-9@:%_+.~#?&/=]*)"+) # Same as Xenforo+_EXT_REGEX = re.compile(r"\d{2,}x\d{2,}(\.\w+)?")+++Selector = css.CssAttributeSelector+++class Selectors:+ POST_CONTENT: ClassVar = Selector("#content, .entry-content [id*='post-']")+ POST_ID: ClassVar = Selector("[id*='post-']", "id")+ IMG: ClassVar = Selector("img[class*='wp-image']", "srcset")+ POST_LINK_FROM_PAGE = Selector(".post a", "href")+ NEXT_PAGE = Selector("a.page-numbers.next", "href")+++_SELECTORS = Selectors()+_POST_ID_REGEX = re.compile(r"(?:postid-|post-)(\d+)")+++class WordPressBaseCrawler(Crawler, is_abc=True):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Category": "/category/<category_slug>",+ "Tag": "/tag/<tag_slug>",+ "Post": "/<post_slug>/",+ "All Posts": "/posts/",+ "Date Range": (+ "...?before=<date>",+ "...?after=<date>",+ "...?before=<date&after=<date>",+ ),+ "**NOTE**": """++ For `Date Range`, <date> must be a valid iso 8601 date, ex: `2022-12-06`.++ `Date Range` can be combined with `Category`, `Tag` and `All Posts`.+ ex: To only download categories from a date range: ,+ `/category/<category_slug>?before=<date>`""",+ }+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date} - {id} - {title}"+ WP_USE_REGEX: ClassVar = True+ SUPPORTS_THREAD_RECURSION: ClassVar = False++ def __init_subclass__(cls, **kwargs) -> None:+ super().__init_subclass__(**kwargs)+ assert cls.fetch is WordPressBaseCrawler.fetch+ assert cls.fetch_with_date_range is WordPressBaseCrawler.fetch_with_date_range++ def __post_init__(self) -> None:+ self.request_limiter = AsyncLimiter(3, 1)++ @final+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ date_range = QueryDatetimeRange.from_url(scrape_item.url)+ scrape_item.url = scrape_item.url.with_query(None)+ if date_range:+ self.log(f"Scraping {scrape_item.url} with date range: {date_range.as_query()}")+ return await self.fetch_with_date_range(scrape_item, date_range)++ @property+ @final+ def separate_posts(self) -> bool:+ # For wordpress we should always create a separate folder. Each post is an individual page+ return True++ @staticmethod+ @final+ def is_attachment(url: AbsoluteHttpURL) -> bool:+ return "wp-content" in url.parts and bool(url.suffix)++ @final+ async def fetch_with_date_range(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None) -> None:+ match scrape_item.url.parts[1:]:+ case ["posts"]:+ return await self.all_posts(scrape_item, date_range)+ case [ColletionType.CATEGORY.value, _]:+ return await self.category_or_tag(scrape_item, ColletionType.CATEGORY, date_range)+ case [ColletionType.TAG.value, _]:+ return await self.category_or_tag(scrape_item, ColletionType.TAG, date_range)+ case ["wp-json", *_]:+ raise ValueError++ if _ := _match_date_from_path(scrape_item.url):+ # TODO: Handle this+ raise ValueError+ return await self.post(scrape_item)++ @abstractmethod+ async def category_or_tag(+ self, scrape_item: ScrapeItem, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None+ ) -> None: ...++ @abstractmethod+ async def post(self, scrape_item: ScrapeItem) -> None: ...++ @abstractmethod+ async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None: ...++ @final+ async def filter_post(+ self, scrape_item: ScrapeItem, post: Post, date_range: QueryDatetimeRange | None = None+ ) -> None:+ if date_range and not date_range.is_in_range(post.date_gmt):+ self.log(f"Skipping post {post.link} as it is out of date range. Post date: {[post.date_gmt]}")+ return+ new_scrape_item = scrape_item.create_child(self.parse_url(post.link))+ await self.handle_post(new_scrape_item, post)+ scrape_item.add_children()++ @final+ async def handle_post(self, scrape_item: ScrapeItem, post: Post, *, is_single_post: bool = False) -> None:+ post_id = str(post.id)+ title = self.create_separate_post_title(post.title, post_id, post.date_gmt.date())+ if is_single_post:+ title = self.create_title(title)+ scrape_item.setup_as_album(title, album_id=post_id)+ scrape_item.possible_datetime = to_timestamp(post.date_gmt)+ if post.thumbnail:+ await self.direct_file(scrape_item, self.parse_url(post.thumbnail))+ return await self.handle_post_content(scrape_item, post)++ @final+ async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:+ if self.is_attachment(link):+ return await self.direct_file(scrape_item, link)+ if self.PRIMARY_URL.host == link.host:+ return+ new_scrape_item = scrape_item.create_new(link)+ new_scrape_item.type = None+ new_scrape_item.reset_childen()+ self.handle_external_links(new_scrape_item)+ scrape_item.add_children()++ async def handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:+ for link in self.iter_parse_url(_iter_links(post.content, self.WP_USE_REGEX)):+ await self.handle_link(scrape_item, link)++ def parse_url(self, link: str) -> AbsoluteHttpURL:+ # TODO: handle more domains and move it to the base crawler+ link = _get_original_quality_link(link)+ url = super().parse_url(link)+ if url.host == "ouo.io" and (redirect_url := url.query.get("s")):+ return super().parse_url(redirect_url)+ return url++ @final+ def iter_parse_url(self, iterable: Iterable[str]) -> Iterable[AbsoluteHttpURL]:+ for link_str in unique(iterable):+ try:+ yield self.parse_url(link_str)+ except Exception:+ continue+++class WordPressAPICrawler(WordPressBaseCrawler, is_abc=True):+ WP_CATEGORIES_ENDPOINT: ClassVar = "/wp-json/wp/v2/categories"+ WP_TAGS_ENDPOINT: ClassVar = "/wp-json/wp/v2/tags"+ WP_POSTS_ENDPOINT: ClassVar = "/wp-json/wp/v2/posts"++ def __init_subclass__(cls, is_abc: bool = False, **kwargs) -> None:+ super().__init_subclass__(is_abc=is_abc, **kwargs)+ if is_abc:+ return+ cls.WP_API_CATEGORIES_URL = cls.PRIMARY_URL / cls.WP_CATEGORIES_ENDPOINT.removeprefix("/")+ cls.WP_API_TAGS_URL = cls.PRIMARY_URL / cls.WP_TAGS_ENDPOINT.removeprefix("/")+ cls.WP_API_POSTS_URL = cls.PRIMARY_URL / cls.WP_POSTS_ENDPOINT.removeprefix("/")++ @error_handling_wrapper+ async def category_or_tag(+ self, scrape_item: ScrapeItem, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None+ ) -> None:+ if colletion_type is ColletionType.CATEGORY:+ model, api_url = CategorySequence, self.WP_API_CATEGORIES_URL.with_query(slug=scrape_item.url.name)+ else:+ model, api_url = TagSequence, self.WP_API_TAGS_URL.with_query(slug=scrape_item.url.name)+ collections = await self.__make_request(model, api_url)+ if not collections:+ raise ScrapeError(404)+ await self.__handle_collection(scrape_item, collections[0], date_range)++ @error_handling_wrapper+ async def post(self, scrape_item: ScrapeItem) -> None:+ posts = await self.__make_request(PostSequence, self.WP_API_POSTS_URL.with_query(slug=scrape_item.url.name))+ if not posts:+ raise ScrapeError(404)+ return await self.handle_post(scrape_item, posts[0], is_single_post=True)++ @error_handling_wrapper+ async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:+ api_url = self.WP_API_POSTS_URL+ if date_range:+ api_url = api_url.update_query(date_range.as_query())+ scrape_item.setup_as_profile(self.create_title("Posts"))+ async for post in self.post_pager(api_url):+ await self.filter_post(scrape_item, post)++ async def __make_request(self, model_cls: type[_ModelT], api_url: AbsoluteHttpURL) -> _ModelT:+ async with self.request_limiter:+ json_text = await self.client.get_text(self.DOMAIN, api_url)+ return model_cls.model_validate_json(json_text)++ async def __handle_collection(+ self, scrape_item: ScrapeItem, collection: Category | Tag, date_range: QueryDatetimeRange | None = None+ ) -> None:+ title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")+ scrape_item.setup_as_profile(title)+ assert collection.id+ api_url = self.WP_API_POSTS_URL.with_query({collection._type: collection.id})+ if date_range:+ api_url = api_url.update_query(date_range.as_query())+ async for post in self.post_pager(api_url):+ await self.filter_post(scrape_item, post)++ async def post_pager(self, url: AbsoluteHttpURL, init_page: int | None = None) -> AsyncIterable[Post]:+ had_at_least_1_post = False+ for page in itertools.count(init_page or 1):+ n_post = 0+ api_url = url.update_query(per_page=_POST_PER_REQUEST, page=page)+ for post in await self.__make_request(PostSequence, api_url):+ n_post += 1+ had_at_least_1_post = True+ yield post+ if n_post < _POST_PER_REQUEST:+ break+ if not had_at_least_1_post:+ raise ScrapeError(404)+++class WordPressSoupCrawler(WordPressBaseCrawler, is_abc=True):+ async def __make_request(self, api_url: AbsoluteHttpURL) -> BeautifulSoup:+ async with self.request_limiter:+ return await self.client.get_soup(self.DOMAIN, api_url)++ async def category_or_tag(+ self, scrape_item: ScrapeItem, colletion_type: ColletionType, date_range: QueryDatetimeRange | None = None+ ) -> None:+ if colletion_type is ColletionType.CATEGORY:+ collection = Category(slug=scrape_item.url.name, link=str(scrape_item.url))+ else:+ collection = Tag(slug=scrape_item.url.name, link=str(scrape_item.url))+ await self.__handle_collection(scrape_item, collection, date_range)++ async def __handle_collection(+ self, scrape_item: ScrapeItem, collection: Category | Tag, date_range: QueryDatetimeRange | None = None+ ) -> None:+ title = self.create_title(f"{collection.description or collection.slug} [{collection._type}]")+ scrape_item.setup_as_profile(title)+ await self.post_url_pager(scrape_item, date_range)++ async def post(self, scrape_item: ScrapeItem) -> None:+ soup = await self.__make_request(scrape_item.url)+ post = self.parse_post(scrape_item, soup)+ return await self.handle_post(scrape_item, post, is_single_post=True)++ def parse_post(self, scrape_item: ScrapeItem, soup: BeautifulSoup) -> Post:+ data = {+ "post_id": get_post_id(soup),+ "slug": scrape_item.url.name,+ "title": open_graph.get_title(soup),+ "date_gmt": open_graph.get("published_time", soup),+ "link": str(scrape_item.url),+ "content": _SELECTORS.POST_CONTENT(soup),+ }+ return Post.model_validate(data)++ async def all_posts(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:+ scrape_item.setup_as_profile(self.create_title("Posts"))+ await self.post_url_pager(scrape_item, date_range)++ async def post_url_pager(self, scrape_item: ScrapeItem, date_range: QueryDatetimeRange | None = None) -> None:+ async for soup in self.web_pager(scrape_item.url):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.POST_LINK_FROM_PAGE.element):+ if (+ date_range+ and (date_from_path := _match_date_from_path(new_scrape_item.url))+ and not date_range.is_in_range(date_from_path)+ ):+ self.log(+ f"Skipping post {new_scrape_item.url} as it is out of date range. Post date: {{date_from_path}}"+ )+ continue+ self.manager.task_group.create_task(self.run(new_scrape_item))+++@no_error+def _match_date_from_path(url: AbsoluteHttpURL) -> datetime.datetime | None:+ match url.parts[1:]:+ case [year, month, day]:+ return datetime.datetime(int(year), int(month), int(day))+ case _:+ return+++def _get_original_quality_link(link: str) -> str:+ stem, _, tail = link.rpartition("-")+ if match := re.search(_EXT_REGEX, tail):+ return f"{stem}.{match.group()}"+ return link+++def _iter_links(html: HTML, use_regex: bool) -> itertools.chain[str]:+ soup = BeautifulSoup(html, "html.parser")+ images = css.iget(soup, *css.images)+ iframes = css.iget(soup, *css.iframes)+ if use_regex:+ regex = (match.group() for match in re.finditer(_HTTP_URL_REGEX, html))+ return itertools.chain(images, iframes, regex)+ return itertools.chain(images, iframes)+++@no_error+def get_post_id(soup: BeautifulSoup) -> int:+ id_text = _SELECTORS.POST_ID(soup)+ if match := _POST_ID_REGEX.search(id_text):+ post_id = match.group(1)+ for trash in ("post", "id", "-"):+ post_id = post_id.removeprefix(trash)+ return int(post_id)+ raise ValueErrordiff --git a/cyberdrop_dl/crawlers/wordpress/bestprettygirl.py b/cyberdrop_dl/crawlers/wordpress/bestprettygirl.pynew file mode 100644index 000000000..662b51859--- /dev/null+++ b/cyberdrop_dl/crawlers/wordpress/bestprettygirl.py@@ -0,0 +1,13 @@+from __future__ import annotations++from typing import ClassVar++from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL++from ._wordpress import WordPressAPICrawler+++class BestPrettyGirlCrawler(WordPressAPICrawler):+ DOMAIN: ClassVar[str] = "bestprettygirl.com"+ FOLDER_DOMAIN: ClassVar[str] = "BestPrettyGirl"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bestprettygirl.com/")diff --git a/cyberdrop_dl/crawlers/wordpress/models.py b/cyberdrop_dl/crawlers/wordpress/models.pynew file mode 100644index 000000000..11f614d8e--- /dev/null+++ b/cyberdrop_dl/crawlers/wordpress/models.py@@ -0,0 +1,91 @@+from __future__ import annotations++from datetime import UTC, datetime+from typing import Annotated, Literal, NewType, TypeVar++from bs4 import BeautifulSoup+from pydantic import AfterValidator, AliasPath, BaseModel, Field++from cyberdrop_dl.compat import StrEnum+from cyberdrop_dl.models.base_models import SequenceModel++_ModelT = TypeVar("_ModelT", bound=BaseModel)+++def make_soup(string: str) -> BeautifulSoup:+ return BeautifulSoup(string, "html.parser")+++def unescape_html(string: str) -> str:+ return make_soup(string).get_text(strip=True)+++def add_utc_tz(parsed_date: datetime) -> datetime:+ return parsed_date.replace(tzinfo=UTC)+++TitleFromHTML = Annotated[str, AfterValidator(unescape_html)]+AwareDatetimeUTC = Annotated[datetime, AfterValidator(add_utc_tz)]+++class ColletionType(StrEnum):+ TAG = "tag"+ CATEGORY = "category"+++HTML = NewType("HTML", str)+++class WordPressModel(BaseModel):+ id: int | None = None+ slug: str+ link: str+++class Post(WordPressModel):+ title: TitleFromHTML = Field(validation_alias=AliasPath("title", "rendered"))+ content: HTML = Field(validation_alias=AliasPath("content", "rendered"))+ thumbnail: str | None = Field(default=None, validation_alias=AliasPath("acf", "fifu_image_url"))+ date_gmt: AwareDatetimeUTC+++class Collection(WordPressModel):+ description: str = ""+ taxonomy: str+++class Category(Collection):+ taxonomy: Literal["category"] = "category"+ _type: ColletionType = ColletionType.CATEGORY+++class Tag(Category):+ taxonomy: Literal["post_tag"] = "post_tag"+ _type: ColletionType = ColletionType.TAG+++class PostSequence(SequenceModel[Post]): ...+++class TagSequence(SequenceModel[Tag]): ...+++class CategorySequence(SequenceModel[Category]): ...+++class PostExtraData(Post):+ # Not used at the moment+ date: datetime+ modified: datetime+ modified_gmt: AwareDatetimeUTC+ status: Literal["publish", "future", "draft", "pending", "private"]+ type: str+ categories: list[int] = []+ tags: list[int] = []+ format: Literal["standard", "aside", "chat", "gallery", "link", "image", "quote", "status", "video", "audio"]+++class CollectionExtraData(Collection):+ # Not used at the moment+ count: int+ parent: intdiff --git a/cyberdrop_dl/crawlers/xenforo/__init__.py b/cyberdrop_dl/crawlers/xenforo/__init__.pyindex 1216fb82f..b8715e05a 100644--- a/cyberdrop_dl/crawlers/xenforo/__init__.py+++ b/cyberdrop_dl/crawlers/xenforo/__init__.py@@ -23,5 +23,5 @@ XF_CRAWLERS: set[type[XenforoCrawler]] = {XBunkerCrawler,}-CRAWLERS_MAP = {c.__name__: c for c in XF_CRAWLERS}-__all__ = list(CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]+XF_CRAWLERS_MAP = {c.__name__: c for c in XF_CRAWLERS}+__all__ = list(XF_CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 0035d8845..8f4057e20 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -6,7 +6,7 @@ from dataclasses import InitVar, dataclass, fieldfrom enum import IntEnumfrom functools import partialmethodfrom pathlib import Path-from typing import TYPE_CHECKING, Literal, NamedTuple, ParamSpec, Self, TypeVar, overload+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, ParamSpec, Self, TypeVar, overloadimport yarl@@ -17,6 +17,7 @@ R = TypeVar("R")T = TypeVar("T")if TYPE_CHECKING:+ import datetimeimport functoolsimport inspectfrom collections.abc import Callable@@ -269,13 +270,13 @@ class ScrapeItem:def create_new(self,- url: yarl.URL,+ url: AbsoluteHttpURL,*,new_title_part: str = "",part_of_album: bool = False,album_id: str | None = None,possible_datetime: int | None = None,- add_parent: yarl.URL | bool | None = None,+ add_parent: AbsoluteHttpURL | bool | None = None,) -> ScrapeItem:"""Creates a scrape item."""from cyberdrop_dl.utils.utilities import is_absolute_http_url@@ -284,7 +285,7 @@ class ScrapeItem:assert is_absolute_http_url(url)scrape_item.url = urlif add_parent:- new_parent = add_parent if isinstance(add_parent, yarl.URL) else self.url+ new_parent = add_parent if isinstance(add_parent, AbsoluteHttpURL) else self.urlassert is_absolute_http_url(new_parent)scrape_item.parents.append(new_parent)if new_title_part:@@ -300,14 +301,43 @@ class ScrapeItem:setup_as_forum = partialmethod(setup_as, type=FORUM)setup_as_post = partialmethod(setup_as, type=FORUM_POST)- def origin(self) -> yarl.URL | None:+ def origin(self) -> AbsoluteHttpURL | None:if self.parents:return self.parents[0]- def parent(self) -> yarl.URL | None:+ def parent(self) -> AbsoluteHttpURL | None:if self.parents:return self.parents[-1]def copy(self) -> Self:"""Returns a deep copy of this scrape_item"""return copy.deepcopy(self)+++class QueryDatetimeRange(NamedTuple):+ before: datetime.datetime | None = None+ after: datetime.datetime | None = None++ @staticmethod+ def from_url(url: AbsoluteHttpURL) -> QueryDatetimeRange | None:+ self = QueryDatetimeRange(_date_from_query_param(url, "before"), _date_from_query_param(url, "after"))+ if self == (None, None):+ return None+ if (self.before and self.after) and (self.before <= self.after):+ raise ValueError+ return self++ def is_in_range(self, other: datetime.datetime) -> bool:+ if (self.before and other >= self.before) or (self.after and other <= self.after):+ return False+ return True++ def as_query(self) -> dict[str, Any]:+ return {name: value.isoformat() for name, value in self._asdict().items() if value}+++def _date_from_query_param(url: AbsoluteHttpURL, query_param: str) -> datetime.datetime | None:+ from cyberdrop_dl.utils.dates import parse_aware_iso_datetime++ if value := url.query.get(query_param):+ return parse_aware_iso_datetime(value)diff --git a/cyberdrop_dl/models/base_models.py b/cyberdrop_dl/models/base_models.pyindex d50fca9f0..f067f8d65 100755--- a/cyberdrop_dl/models/base_models.py+++ b/cyberdrop_dl/models/base_models.py@@ -1,14 +1,26 @@"""Pydantic models"""-from collections.abc import Mapping+from collections.abc import Iterator, Mapping, Sequencefrom pathlib import Path+from typing import TypeVarimport yarl-from pydantic import AnyUrl, BaseModel, ConfigDict, Secret, SerializationInfo, model_serializer, model_validator+from pydantic import (+ AnyUrl,+ BaseModel,+ ConfigDict,+ RootModel,+ Secret,+ SerializationInfo,+ model_serializer,+ model_validator,+)from cyberdrop_dl.models.types import HttpURLfrom cyberdrop_dl.models.validators import to_apprise_url_dict+_ModelT = TypeVar("_ModelT", bound=BaseModel)+class AliasModel(BaseModel):model_config = ConfigDict(populate_by_name=True)@@ -51,3 +63,17 @@ class PathAliasModel(AliasModel):setattr(self, name, self.replace_config_and_resolve(value, config_name))elif isinstance(value, PathAliasModel):value.resolve_paths(config_name)+++class SequenceModel(RootModel[list[_ModelT]], Sequence[_ModelT]):+ def __len__(self) -> int:+ return len(self.root)++ def __iter__(self) -> Iterator[_ModelT]:+ yield from self.root++ def __getitem__(self, index: int) -> _ModelT:+ return self.root[index]++ def __bool__(self) -> bool:+ return bool(len(self))diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 2ed1342a4..e368b4675 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -8,7 +8,7 @@ import bs4.cssfrom cyberdrop_dl.exceptions import ScrapeErrorif TYPE_CHECKING:- from collections.abc import Callable, Generator+ from collections.abc import Callable, Generator, Iterablefrom bs4 import Tag@@ -59,10 +59,16 @@ def select_one_get_text(tag: Tag, selector: str, strip: bool = True, *, decompos# get_attr_no_error should be get_attr_or_none. `or_none` implies no errordef get_attr_or_none(tag: Tag, attribute: str) -> str | None:"""Same as `tag.get(attribute)` but asserts the result is a single str"""- if attribute == "src":- value = tag.get("data-src") or tag.get(attribute)+ attribute_ = attribute+ if attribute_ == "srcset" and (srcset := tag.get(attribute_)):+ if isinstance(srcset, str):+ return _parse_srcset(srcset)+ attribute_ = "src"++ if attribute_ == "src":+ value = tag.get("data-src") or tag.get(attribute_)else:- value = tag.get(attribute)+ value = tag.get(attribute_)if isinstance(value, list):raise SelectorError(f"Expected a single value for {attribute = !r}, got multiple")return value@@ -99,3 +105,17 @@ def select_one_get_attr_or_none(tag: Tag, selector: str, attribute: str) -> strdef iselect(tag: Tag, selector: str) -> Generator[Tag]:"""Same as `tag.select(selector)`, but it returns a generator instead of a list."""yield from bs4.css.CSS(tag).iselect(selector)+++def _parse_srcset(srcset: str) -> str:+ # The best src is the last one (usually)+ return [src.split(" ")[0] for src in srcset.split(", ")][-1]+++def iget(tag: Tag, selector: str, attribute: str) -> Iterable[str]:+ for inner_tag in iselect(tag, selector):+ yield get_attr(inner_tag, attribute)+++iframes = CssAttributeSelector("iframe", "src")+images = CssAttributeSelector("img", "srcset")diff --git a/cyberdrop_dl/utils/dates.py b/cyberdrop_dl/utils/dates.pyindex 3c148d628..7943294fb 100644--- a/cyberdrop_dl/utils/dates.py+++ b/cyberdrop_dl/utils/dates.py@@ -142,6 +142,16 @@ def get_parser(parser_kind: ParserKind | None = None, date_order: DateOrder | Noreturn DateParser(parser_kind, date_order)+def parse_aware_iso_datetime(value: str) -> datetime.datetime | None:+ try:+ parsed_date = datetime.datetime.fromisoformat(value)+ if parsed_date.tzinfo is None:+ parsed_date.replace(tzinfo=datetime.UTC)+ return parsed_date+ except Exception:+ return++if __name__ == "__main__":print(parse_date("today at noon")) # noqa: T201print(parse_date("today")) # noqa: T201diff --git a/cyberdrop_dl/utils/open_graph.py b/cyberdrop_dl/utils/open_graph.pynew file mode 100644index 000000000..1f73c3bbb--- /dev/null+++ b/cyberdrop_dl/utils/open_graph.py@@ -0,0 +1,56 @@+from __future__ import annotations++import dataclasses+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ import bs4++[email protected](frozen=True, slots=True)+class OpenGraph:+ """Open Graph properties. Each attribute corresponds to an OG property."""++ required_attrs = ("title", "type", "image", "url", "description")++ audio: str | None = None+ description: str | None = None+ determiner: str | None = None+ image: str | None = None+ image: str | None = None+ locale: str | None = None+ published_time: str | None = None+ site_name: str | None = None+ title: str | None = None+ type: str | None = None+ url: str | None = None+ video: str | None = None++ def is_valid(self) -> bool:+ return all(getattr(self, attr, False) for attr in self.required_attrs)+++_og_fields = dataclasses.fields(OpenGraph)+++def parse(soup: bs4.BeautifulSoup) -> OpenGraph:+ """Extracts Open Graph properties (og properties) from soup."""+ props: dict[str, str | None] = {}+ for meta in soup.select('meta[property^="og:"][content]'):+ property_name = meta["property"].replace("og:", "") # type: ignore+ if property_name in _og_fields:+ props[property_name] = meta["content"] or None # type: ignore++ if not props.get("title") and (title := soup.select_one("title, h1")):+ props["title"] = title.get_text(strip=True) or None++ return OpenGraph(**props)+++def get(name: str, /, soup: bs4.BeautifulSoup) -> str | None:+ if meta := soup.select_one(f'meta[property^="og:{name}"][content]'):+ return meta["content"] or None # type: ignore+++def get_title(soup: bs4.BeautifulSoup) -> str | None:+ return get("title", soup)diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 80255e0cb..cc498627a 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -38,7 +38,7 @@ from cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.logger import log, log_debug, log_spacer, log_with_colorif TYPE_CHECKING:- from collections.abc import Callable, Coroutine, Mapping+ from collections.abc import Callable, Coroutine, Iterable, Mappingfrom curl_cffi.requests.models import Response as CurlResponsefrom rich.text import Text@@ -618,6 +618,46 @@ def is_blob_or_svg(link: str) -> bool:return any(link.startswith(x) for x in _BLOB_OR_SVG)+def no_error(func: Callable[P, R]) -> Callable[P, R | None]:+ def call(*args, **kwargs) -> R | None:+ try:+ return func(*args, **kwargs)+ except Exception:+ return++ return call+++def unique(iterable: Iterable[T], *, hashable: bool = True) -> Iterable[T]:+ """Yields unique values from iterable, keeping original order"""+ if hashable:+ seen: set[T] | list[T] = set()+ add: Callable[[T], None] = seen.add+ else:+ seen = []+ add = seen.append++ for value in iterable:+ if value not in seen:+ add(value)+ yield value+++def get_valid_kwargs(func: Callable[..., Any], kwargs: Mapping[str, T], accept_kwargs: bool = True) -> Mapping[str, T]:+ """Get the subset of ``kwargs`` that are valid params for ``func`` and their values are not `None`++ If func takes **kwargs, returns everything"""+ params = inspect.signature(func).parameters+ if accept_kwargs and any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):+ return kwargs++ return {k: v for k, v in kwargs.items() if k in params.keys() and v is not None}+++def call_w_valid_kwargs(cls: Callable[..., R], kwargs: Mapping[str, Any]) -> R:+ return cls(**get_valid_kwargs(cls, kwargs))++log_cyan = partial(log_with_color, style="cyan", level=20)log_yellow = partial(log_with_color, style="yellow", level=20)log_green = partial(log_with_color, style="green", level=20)