Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: xenforo pass 2 (#1095)
NTFSvolume35167e73b019
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex fd4a8a2f7..95b632c28 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -47,6 +47,7 @@ from .imgur import ImgurCrawlerfrom .imx_to import ImxToCrawlerfrom .incestflix import IncestflixCrawlerfrom .influencer_bitches import InfluencerBitchesCrawler+from .invision import INVISION_CRAWLERSfrom .jpg5 import JPG5Crawlerfrom .kemono import KemonoCrawlerfrom .luscious import LusciousCrawler@@ -90,10 +91,10 @@ from .tokyomotion import TokioMotionCrawlerfrom .toonily import ToonilyCrawlerfrom .twitter_images import TwimgCrawlerfrom .twpornstars import TwPornstarsCrawler-from .vbulletin import ViperGirlsCrawler # TODO: import all vBulletin crawlers as a set+from .vbulletin import VBULLETIN_CRAWLERSfrom .vipr_dot_im import ViprImCrawlerfrom .wetransfer import WeTransferCrawler-from .wordpress import WP_CRAWLERS, WP_GENERIC_CRAWLERS+from .wordpress import WP_CRAWLERS, WordPressHTMLCrawler, WordPressMediaCrawlerfrom .xbunkr import XBunkrCrawlerfrom .xenforo import XF_CRAWLERS, SimpCityCrawlerfrom .xhamster import XhamsterCrawler@@ -101,8 +102,8 @@ from .xxxbunker import XXXBunkerCrawlerfrom .yandex_disk import YandexDiskCrawlerfrom .youjizz import YouJizzCrawler-FORUM_CRAWLERS = DISCOURSE_CRAWLERS.union(XF_CRAWLERS)-GENERIC_CRAWLERS: set[type[Crawler]] = WP_GENERIC_CRAWLERS.union({DiscourseCrawler})+FORUM_CRAWLERS = XF_CRAWLERS.union(INVISION_CRAWLERS, DISCOURSE_CRAWLERS, VBULLETIN_CRAWLERS)+GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler}ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler}diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 7fb70bd93..5db029c99 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -1,49 +1,314 @@+"""Base crawlers to scrape any message board / forum++If the message board has a public API, inherit from MessageBoard (ex: Discourse)++If the message board needs to scrape the actual HTML of page, Inherit for HTMLMessageBoard++"""+# ruff : noqa: RUF009+from __future__ import annotations+import dataclasses+import datetime+import refrom abc import abstractmethod-from typing import TYPE_CHECKING, ClassVar, final+from collections import defaultdict+from typing import TYPE_CHECKING, ClassVar, Protocol, final++from bs4 import BeautifulSoup, Tag+from cyberdrop_dl.constants import HTTP_REGEX_LINKSfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError+from cyberdrop_dl.utils import css+from cyberdrop_dl.utils.dates import TimeStamp, to_timestamp+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, is_absolute_http_url, is_blob_or_svgif TYPE_CHECKING:+ from collections.abc import AsyncGenerator, Iterable, Sequence++ from aiohttp_client_cache.response import AnyResponse+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem+LINK_TRASH_MAPPING = {".th.": ".", ".md.": ".", "ifr": "watch"}++Selector = css.CssAttributeSelector++[email protected](frozen=True, slots=True)+class PostSelectors:+ article: str # the entire html of the post (comments, attachments, user avatar, signature, etc...)+ content: str # text, links and images (NO attachments)+ id: Selector+ attachments: Selector+ article_trash: Sequence[str] = ("signature", "footer")+ content_trash: Sequence[str] = ("blockquote", "fauxBlockLink")++ # Most sites should only need to overwrite the attributes above+ date: Selector = Selector("time", "datetime")+ embeds: Selector = Selector("iframe", "src")+ images: Selector = Selector("img.bbImage", "src")+ a_tag_w_image: Selector = Selector("a:has(img.bbImage)[href]", "href")+ lazy_load_embeds: Selector = Selector('[class*=iframe][onclick*="loadMedia(this, \'//"]', "onclick")+ links: Selector = Selector("a:not(:has(img))", "href")+ videos: Selector = Selector("video source", "src")+-class ForumCrawler(Crawler, is_abc=True):[email protected](frozen=True, slots=True)+class MessageBoardSelectors:+ posts: PostSelectors+ confirmation_button: Selector+ next_page: Selector+ last_page: Selector+ current_page: Selector+ title: Selector+ title_trash: Sequence[str] = ("span",)++[email protected](frozen=True, slots=True, order=True)+class ForumPost:+ id: int+ date: datetime.datetime | None+ article: Tag = dataclasses.field(compare=False)+ content: Tag = dataclasses.field(compare=False)++ @staticmethod+ def new(article: Tag, selectors: PostSelectors) -> ForumPost:+ for trash in selectors.article_trash:+ css.decompose(article, trash)+ content = css.select_one(article, selectors.content)+ for trash in selectors.content_trash:+ css.decompose(article, trash)+ try:+ date = datetime.datetime.fromisoformat(css.select_one_get_attr(article, *selectors.date))+ except Exception:+ date = None++ id_str = css.get_attr(article, selectors.id.attribute)+ post_id = int(id_str.rsplit("-", 1)[-1])+ return ForumPost(post_id, date, article, content)++ @property+ def timestamp(self) -> TimeStamp | None:+ if self.date:+ return to_timestamp(self.date)+++class ForumPostProtocol(Protocol):+ # Concrete classes may define their own custom `ForumPost` class (ex: a Pydantic Model from an API response)+ # Those classes need to satisfy this Protocol to make sure they work with all of `MessageBoard` methods+ # This is just identify type errors.+ # Subclass implementation does not need to conform to this if they override the necessary methods+ @property+ def id(self) -> int: ...+ @property+ def date(self) -> datetime.datetime | None: ...+ @property+ def article(self) -> Tag: ...+ @property+ def content(self) -> Tag: ...+ @property+ def timestamp(self) -> TimeStamp | None: ...++[email protected](frozen=True, slots=True, order=True)+class Thread:+ id: int+ name: str+ page: int+ post_id: int | None+ url: AbsoluteHttpURL+++class ThreadProtocol(Protocol):+ # Concrete classes may define their own custom `Thread` class (ex: discourse defines `Topic` from an API Response)+ # Those classes need to satisfy this Protocol to make sure they work with all of `MessageBoard` methods+ # This is just identify type errors.+ # Subclass implementation does not need to conform to this if they override the necessary methods+ @property+ def id(self) -> int: ...+ @property+ def name(self) -> str: ...+ @property+ def page(self) -> int: ...+ @property+ def post_id(self) -> int | None: ...+ @property+ def url(self) -> AbsoluteHttpURL: ...+++# quotes_2 : embed fauxBlockLink+class MessageBoardCrawler(Crawler, is_abc=True):+ """Base crawler for every MessageBoard.++ A Message board should have: forums, threads (also known as topics) and posts.++ Concrete classes MUST:+ - implement `parse_thread`+ - implement `make_post_url`+ - implement `thread`+ - implement `post`++ Concrete classes SHOULD define `ATTACHMENT_HOSTS` if internal images of the site are stored on servers with a different domain++ NOTE: Always use this crawler as base, even if the message board logic does not match perfectly.++ In those cases, override `fetch`,`fetch_thread`, `parse_url` or any other non final method as needed++ This crawler is NOT meant to scrape image boards (like 4chan)+ """++ THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"+ ATTACHMENT_URL_PARTS: ClassVar[Sequence[str]] = "attachments", "data", "uploads"+ ATTACHMENT_HOSTS: ClassVar[Sequence[str]] = ()SUPPORTS_THREAD_RECURSION: ClassVar[bool] = False+ LOGIN_USER_COOKIE_NAME: ClassVar[str] = ""++ # True: Login is mandatory. If login fails, the crawler will be disabled+ # False: Login is optional, but CDL will try to log in anyway. ex: Forums where only some threads require auth+ # None: Completely skip login check and request. Always try to scrape as is the user is logged in+ # TODO: move login logic to the base crawler+ login_required: ClassVar[bool | None] = None++ @classmethod+ @abstractmethod+ def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> ThreadProtocol: ...+ @abstractmethod+ async def post(self, scrape_item: ScrapeItem, post: ForumPostProtocol) -> None: ...++ @abstractmethod+ async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None: ...++ async def forum(self, scrape_item: ScrapeItem) -> None:+ # Subclasses can define custom logic for this method.+ # They would need to also override `fetch` since the default fetch does not take this method into account+ raise NotImplementedError++ async def resolve_confirmation_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ # Not every forum has confirmation link so overriding this method is optional+ # Implementation of this method MUST return `None` instead of raising an error+ raise NotImplementedError++ async def async_startup(self) -> None:+ await self.login()++ @final+ async def login(self) -> None:+ if self.login_required is None:+ return++ if not self.logged_in:+ login_url = self.PRIMARY_URL / "login"+ await self._login(login_url)++ @final@propertydef scrape_single_forum_post(self) -> bool:return self.manager.config_manager.settings_data.download_options.scrape_single_forum_post+ @final@propertydef max_thread_depth(self) -> int:return self.manager.config_manager.settings_data.download_options.maximum_thread_depth- @classmethod- @abstractmethod- def is_attachment(cls, url: AbsoluteHttpURL) -> bool: ...+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ if not self.logged_in and self.login_required is True:+ return+ scrape_item.url = self.parse_url(str(scrape_item.url))+ if self.is_attachment(scrape_item.url):+ return await self.handle_internal_link(scrape_item)+ if is_confirmation_link(scrape_item.url):+ return await self.follow_confirmation_link(scrape_item)- @abstractmethod- async def forum(self, scrape_item: ScrapeItem) -> None: ...+ await self.fetch_thread(scrape_item)- @abstractmethod- async def thread(self, scrape_item: ScrapeItem) -> None: ...+ async def fetch_thread(self, scrape_item: ScrapeItem) -> None:+ thread_part_index = len(self.PRIMARY_URL.parts)+ match scrape_item.url.parts[thread_part_index:]:+ case [thread_part, thread_name_and_id, *_] if thread_part in self.THREAD_PART_NAMES:+ self.check_thread_recursion(scrape_item)+ thread = self.parse_thread(scrape_item.url, thread_name_and_id)+ return await self.thread(scrape_item, thread)+ case ["goto" | "posts", _, *_]:+ self.check_thread_recursion(scrape_item)+ return await self.follow_redirect_w_get(scrape_item)+ case _:+ raise ValueError- @abstractmethod- async def post(self, scrape_item: ScrapeItem) -> None: ...+ def is_attachment(self, link: AbsoluteHttpURL | str) -> bool:+ if not link:+ return False+ if isinstance(link, str):+ link = self.parse_url(link)+ by_parts = len(link.parts) > 2 and any(p in link.parts for p in self.ATTACHMENT_URL_PARTS)+ by_host = any(host in link.host for host in self.ATTACHMENT_HOSTS)+ return by_parts or by_host++ # TODO: Move this to base crawler+ @final+ @error_handling_wrapper+ async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:+ async with self.request_limiter:+ response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)+ assert is_absolute_http_url(response.url)+ scrape_item.url = response.url+ self.manager.task_group.create_task(self.run(scrape_item))++ # TODO: Move this to base crawler+ @final+ @error_handling_wrapper+ async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:+ head = await self.client.get_head(self.DOMAIN, scrape_item.url)+ redirect = head.get("location")+ if not redirect:+ raise ScrapeError(422)+ scrape_item.url = self.parse_url(redirect)+ self.manager.task_group.create_task(self.run(scrape_item))++ @final+ async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:+ url = await self.resolve_confirmation_link(scrape_item.url)+ if url: # If there was an error, this will be None+ scrape_item.url = url+ # This could end up back in here if the URL goes to another thread+ return self.handle_external_links(scrape_item)++ @final+ def check_thread_recursion(self, scrape_item: ScrapeItem) -> None:+ if self.stop_thread_recursion(scrape_item):+ parents = f"{len(scrape_item.parent_threads)} parent thread(s)"+ msg = (+ f"Skipping nested thread URL with {parents}:"+ f"URL: {scrape_item.url}\n"+ f"Parent: {scrape_item.parent}\n"+ f"Origin: {scrape_item.origin}\n"+ )+ raise MaxChildrenError(msg)++ @final+ def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:+ if n_parents := len(scrape_item.parent_threads):+ if n_parents > self.max_thread_depth:+ return True++ return self.SUPPORTS_THREAD_RECURSION and bool(self.max_thread_depth)++ return False+ @final@error_handling_wrapperasync def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:if link == self.PRIMARY_URL:returnif self.is_attachment(link):- return await self.handle_internal_link(scrape_item.create_new(link))- if self.PRIMARY_URL.host in link.host and self.stop_thread_recursion(scrape_item):- msg = f"Skipping nested thread URL {scrape_item.url} found on {scrape_item.origin}"- return self.log(msg)+ return await self.handle_internal_link(scrape_item, link)+ if self.PRIMARY_URL.host == link.host:+ self.manager.task_group.create_task(self.run(scrape_item.create_child(link)))+ returnnew_scrape_item = scrape_item.create_child(link)new_scrape_item.type = Nonenew_scrape_item.reset_childen()@@ -51,24 +316,12 @@ class ForumCrawler(Crawler, is_abc=True):scrape_item.add_children()@error_handling_wrapper- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:- # This logic is for discourse- # if len(scrape_item.url.parts) < 5 and not scrape_item.url.suffix:- # return- filename, ext = self.get_filename_and_ext(scrape_item.url.name)+ async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:+ link = link or scrape_item.url+ filename, ext = self.get_filename_and_ext(link.name)scrape_item.add_to_parent_title("Attachments")scrape_item.part_of_album = True- await self.handle_file(scrape_item.url, scrape_item, filename, ext)-- @final- def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:- if (- not self.SUPPORTS_THREAD_RECURSION- or not self.max_thread_depth- or (len(scrape_item.parent_threads) > self.max_thread_depth)- ):- return True- return False+ await self.handle_file(link, scrape_item, filename, ext)@finalasync def write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:@@ -76,5 +329,388 @@ class ForumCrawler(Crawler, is_abc=True):returnawait self.manager.log_manager.write_last_post_log(last_post_url)+ # TODO: Move this to the base crawler+ # TODO: Define an unified workflow for crawlers to perform and check login+ @final+ @error_handling_wrapper+ async def _login(self, login_url: AbsoluteHttpURL) -> None:+ host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)+ session_cookie = host_cookies.get(self.LOGIN_USER_COOKIE_NAME)+ session_cookie = session_cookie.value if session_cookie else None+ msg = f"No cookies found for {self.FOLDER_DOMAIN}"+ if not session_cookie and self.login_required:+ raise LoginError(message=msg)++ _, self.logged_in = await self.check_login_with_request(login_url)+ if self.logged_in:+ return+ if session_cookie:+ msg = f"Cookies for {self.FOLDER_DOMAIN} are not valid."+ if self.login_required:+ raise LoginError(message=msg)++ msg += " Scraping without an account"+ self.log(msg, 30)++ async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:+ text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)+ return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))+++class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):+ """Base crawler that knows how to scrape the html of every MessageBoard.++ Threads of the MessageBoard MUST be paginated.++ Concrete classes MUST:+ - define: `SELECTORS`, `POST_URL_PART_NAME` and `PAGE_URL_PART_NAME`++ This crawler delegates images to other crawlers by default+ Concrete classes MAY handle images themselfs if they know how to. This will improve performance by reducing the number of requests++ To handle images, concrete classes need to:+ - override `IGNORE_EMBEDED_IMAGES_SRC` to `False`+ - override `is_thumbnail`+ - override `thumbnail_to_img`++ Concrete classes SHOULD define `ATTACHMENT_HOSTS` if internal images of the site are stored on servers with a different domain+ """++ IGNORE_EMBEDED_IMAGES_SRC = True+ SELECTORS: ClassVar[MessageBoardSelectors]+ POST_URL_PART_NAME: ClassVar[str]+ PAGE_URL_PART_NAME: ClassVar[str]++ def __init_subclass__(cls, is_abc: bool = False, **kwargs) -> None:+ super().__init_subclass__(is_abc=is_abc, **kwargs)+ if is_abc:+ return+ REQUIRED_FIELDS = "SELECTORS", "POST_URL_PART_NAME", "PAGE_URL_PART_NAME"+ for field_name in REQUIRED_FIELDS:+ assert getattr(cls, field_name, None), f"Subclass {cls.__name__} must override: {field_name}"++ def __post_init__(self) -> None:+ self.scraped_threads = set()++ @final+ async def async_startup(self) -> None:+ await super().async_startup()+ self.register_cache_filter(self.PRIMARY_URL, self.check_is_not_last_page)++ async def check_is_not_last_page(self, response: AnyResponse) -> bool:+ return await check_is_not_last_page(response, self.SELECTORS)++ @classmethod+ def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool:+ return False++ @classmethod+ def thumbnail_to_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ return None++ @classmethod+ def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> ThreadProtocol:+ return parse_thread(url, thread_name_and_id, cls.PAGE_URL_PART_NAME, cls.POST_URL_PART_NAME)++ @classmethod+ def make_post_url(cls, thread: ThreadProtocol, post_id: int) -> AbsoluteHttpURL:+ return thread.url / f"{cls.POST_URL_PART_NAME}-{post_id}"++ @error_handling_wrapper+ async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:+ scrape_item.setup_as_forum("")+ if thread.url in self.scraped_threads:+ return++ scrape_item.parent_threads.add(thread.url)+ if self.scrape_single_forum_post and not thread.post_id:+ msg = "`--scrape-single-forum-post` is `True`, but the provided URL has no post id"+ raise ScrapeError("User Error", msg)++ self.scraped_threads.add(thread.url)+ await self.process_thread(scrape_item, thread)++ async def process_thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:+ title: str = ""+ last_post_url = thread.url+ async for soup in self.thread_pager(scrape_item):+ if not title:+ try:+ title = self.create_title(get_post_title(soup, self.SELECTORS), thread_id=thread.id)+ except ScrapeError as e:+ self.log_debug("Got an unprocessable soup", 40, exc_info=e)+ raise+ scrape_item.add_to_parent_title(title)++ continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)+ if not continue_scraping:+ break++ await self.write_last_forum_post(thread.url, last_post_url)++ def process_thread_page(+ self, scrape_item: ScrapeItem, thread: ThreadProtocol, soup: BeautifulSoup+ ) -> tuple[bool, AbsoluteHttpURL]:+ continue_scraping = False+ post_url = thread.url+ for article in soup.select(self.SELECTORS.posts.article):+ current_post = ForumPost.new(article, self.SELECTORS.posts)+ continue_scraping, scrape_this_post = check_post_id(+ thread.post_id, current_post.id, self.scrape_single_forum_post+ )+ if scrape_this_post:+ post_url = self.make_post_url(thread, current_post.id)+ new_scrape_item = scrape_item.create_new(+ thread.url,+ possible_datetime=current_post.timestamp,+ add_parent=post_url,+ )+ self.manager.task_group.create_task(self.post(new_scrape_item, current_post))+ try:+ scrape_item.add_children()+ except MaxChildrenError:+ break++ if not continue_scraping:+ break+ return continue_scraping, post_url++ @error_handling_wrapper+ async def post(self, scrape_item: ScrapeItem, post: ForumPostProtocol) -> None:+ scrape_item.setup_as_post("")+ post_title = self.create_separate_post_title(None, str(post.id), post.date)+ scrape_item.add_to_parent_title(post_title)++ seen: set[str] = set()+ duplicates: set[str] = set()+ stats: dict[str, int] = defaultdict(int)+ for scraper in (self._attachments, self._images, self._videos, self._external_links):+ for link in scraper(post):+ duplicates.add(link) if link in seen else seen.add(link)+ stats[scraper.__name__.removeprefix("_")] += 1+ await self.process_child(scrape_item, link)++ for scraper in (self._embeds, self._lazy_load_embeds):+ for link in scraper(post):+ duplicates.add(link) if link in seen else seen.add(link)+ stats[scraper.__name__.removeprefix("_")] += 1+ await self.process_child(scrape_item, link, embeds=True)++ if seen:+ self.log(f"[{self.FOLDER_DOMAIN}] post #{post.id} stats = {dict(stats)}")+ if duplicates:+ self.log_bug_report(f"Found duplicate links. Selectors are too generic: {duplicates}")++ def _external_links(self, post: ForumPostProtocol) -> Iterable[str]:+ selector = self.SELECTORS.posts.links+ links = css.iselect(post.content, selector.element)+ valid_links = (link for link in links if not self.is_username_or_attachment(link))+ return iter_links(valid_links, selector.attribute)++ def _images(self, post: ForumPostProtocol) -> Iterable[str]:+ if self.IGNORE_EMBEDED_IMAGES_SRC:+ selector = self.SELECTORS.posts.a_tag_w_image+ else:+ selector = self.SELECTORS.posts.images+ images = css.iselect(post.content, selector.element)+ return iter_links(images, selector.attribute)++ def _videos(self, post: ForumPostProtocol) -> Iterable[str]:+ selector = self.SELECTORS.posts.videos+ videos = css.iselect(post.content, selector.element)+ return iter_links(videos, selector.attribute)++ def _attachments(self, post: ForumPostProtocol) -> Iterable[str]:+ selector = self.SELECTORS.posts.attachments+ attachments = css.iselect(post.article, selector.element)+ return iter_links(attachments, selector.attribute)++ def _embeds(self, post: ForumPostProtocol) -> Iterable[str]:+ selector = self.SELECTORS.posts.embeds+ embeds = css.iselect(post.content, selector.element)+ return iter_links(embeds, selector.attribute)++ def _lazy_load_embeds(self, post: ForumPostProtocol) -> Iterable[str]:+ selector = self.SELECTORS.posts.lazy_load_embeds+ for lazy_media in css.iselect(post.content, selector.element):+ yield get_text_between(css.get_attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")++ async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:+ async for soup in self._web_pager(scrape_item.url, self.get_next_page):+ yield soup++ def get_next_page(self, soup: BeautifulSoup) -> str | None:+ return css.select_one_get_attr_or_none(soup, *self.SELECTORS.next_page)++ @final+ @error_handling_wrapper+ async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:+ link_str_ = pre_process_child(link_str, embeds)+ if not link_str_:+ return+ link = await self.get_absolute_link(link_str_)+ if not link:+ return+ if self.is_thumbnail(link):+ link = self.thumbnail_to_img(link)+ if not link:+ return self.log(f"Skipping thumbnail: {link}")+ await self.handle_link(scrape_item, link)++ async def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ if isinstance(link, str):+ absolute_link = self.parse_url(clean_link_str(link))+ else:+ absolute_link = link+ if is_confirmation_link(absolute_link):+ return await self.resolve_confirmation_link(absolute_link)+ return absolute_link++ @error_handling_wrapper+ async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ async with self.request_limiter:+ soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)+ selector = self.SELECTORS.confirmation_button+ confirm_button = soup.select_one(selector.element)+ if not confirm_button:+ return+ link_str: str = css.get_attr(confirm_button, selector.attribute)+ link_str = link_str.split('" class="link link--internal', 1)[0]+ new_link = self.parse_url(link_str)+ return await self.get_absolute_link(new_link)++ async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:+ link = link or scrape_item.url+ slug = link.name or link.parent.name+ if slug.isdigit():+ return await self.follow_redirect_w_head(scrape_item.create_new(link))++ await super().handle_internal_link(scrape_item, link)++ def is_username_or_attachment(self, link_obj: Tag) -> bool:+ if link_obj.select_one(".username"):+ return True+ try:+ if link_str := css.get_attr(link_obj, self.SELECTORS.posts.links.element):+ return self.is_attachment(link_str)+ except Exception:+ pass+ return False+++def iter_links(links: Iterable[Tag], attribute: str) -> Iterable[str]:+ for link_tag in links:+ try:+ yield css.get_attr(link_tag, attribute)+ except Exception:+ continue+++def parse_thread(+ url: AbsoluteHttpURL, thread_name_and_id: str, page_part_name: str, post_part_name: str+) -> ThreadProtocol:+ name_index = url.parts.index(thread_name_and_id)+ name, id_ = parse_thread_name_and_id(thread_name_and_id)+ page, post_id = get_thread_page_and_post(url, name_index, page_part_name, post_part_name)+ canonical_url = get_thread_canonical_url(url, name_index)+ return Thread(id_, name, page, post_id, canonical_url)+++def parse_thread_name_and_id(thread_name_and_id: str) -> tuple[str, int]:+ try:+ name, id_str = thread_name_and_id.rsplit(".", 1)+ except ValueError:+ id_str, name = thread_name_and_id.split("-", 1)+ return name, int(id_str)+++def get_thread_canonical_url(url: AbsoluteHttpURL, thread_name_index: int) -> AbsoluteHttpURL:+ new_parts = url.parts[1 : thread_name_index + 1]+ new_path = "/".join(new_parts)+ return url.with_path(new_path)+++def get_thread_page_and_post(+ url: AbsoluteHttpURL, thread_name_index: int, page_name: str, post_name: str+) -> tuple[int, int | None]:+ extra_parts = url.parts[thread_name_index + 1 :]+ if url.fragment:+ extra_parts = *extra_parts, url.fragment++ def find_number(search_value: str) -> int | None:+ for sec in extra_parts:+ if search_value in sec:+ return int(sec.rsplit(search_value, 1)[-1].replace("-", "").strip())++ post_id = find_number(post_name)+ page_number = find_number(page_name) or 1+ return page_number, post_id+++async def check_is_not_last_page(response: AnyResponse, selectors: MessageBoardSelectors) -> bool:+ soup = BeautifulSoup(await response.text(), "html.parser")+ return not is_last_page(soup, selectors)+++def is_last_page(soup: BeautifulSoup, selectors: MessageBoardSelectors) -> bool:+ try:+ last_page = css.select_one_get_attr(soup, *selectors.last_page)+ current_page = css.select_one_get_attr(soup, *selectors.current_page)+ except (AttributeError, IndexError, css.SelectorError):+ return True+ return current_page == last_page+++def get_post_title(soup: BeautifulSoup, selectors: MessageBoardSelectors) -> str:+ try:+ title_block = css.select_one(soup, selectors.title.element)+ for trash in selectors.title_trash:+ css.decompose(title_block, trash)+ except (AttributeError, AssertionError, css.SelectorError) as e:+ raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e++ if title := " ".join(css.get_text(title_block).split()):+ return title+ raise ScrapeError(422)+++def extract_embed_url(embed_str: str) -> str:+ embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")+ if match := re.search(HTTP_REGEX_LINKS, embed_str):+ return match.group(0).replace("www.", "")+ return embed_str+++def clean_link_str(link: str) -> str:+ for old, new in LINK_TRASH_MAPPING.items():+ link = link.replace(old, new)+ return link+++def is_confirmation_link(link: AbsoluteHttpURL) -> bool:+ return "masked" in link.parts or "link-confirmation" in link.path+++def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]:+ """Checks if the program should scrape the current post.++ Returns (continue_scraping, scrape_this_post)"""+ if init_post_id:+ if init_post_id > current_post_id:+ return (True, False)+ elif init_post_id == current_post_id:+ return (not scrape_single_forum_post, True)+ else:+ return (not scrape_single_forum_post, not scrape_single_forum_post)++ assert not scrape_single_forum_post # We should have raised an exception earlier+ return True, True+++def pre_process_child(link_str: str, embeds: bool = False) -> str | None:+ assert isinstance(link_str, str)+ if embeds:+ link_str = extract_embed_url(link_str)-MessageBoardCrawler = ForumCrawler+ if link_str and not is_blob_or_svg(link_str):+ return link_strdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 8bfb124d3..760cbd943 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -8,7 +8,7 @@ from collections.abc import Iterable, Mappingfrom dataclasses import dataclass, fieldfrom functools import partial, wrapsfrom pathlib import Path-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, TypeAlias, TypeVar, final+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NoReturn, ParamSpec, TypeAlias, TypeVar, finalimport yarlfrom aiolimiter import AsyncLimiter@@ -18,6 +18,7 @@ from cyberdrop_dl import constantsfrom cyberdrop_dl.constants import NEW_ISSUE_URLfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloader+from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionErrorfrom cyberdrop_dl.scraper import filtersfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.database.tables.history_table import get_db_path@@ -99,6 +100,13 @@ def create_task_id(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Clog(f"Scrape Failed: {UNKNOWN_URL_PATH_MSG}: {scrape_item.url}", 40)self.manager.progress_manager.scrape_stats_progress.add_failure(UNKNOWN_URL_PATH_MSG)await self.manager.log_manager.write_scrape_error_log(scrape_item.url, UNKNOWN_URL_PATH_MSG)+ except MaxChildrenError:++ @error_handling_wrapper+ async def raise_e(self, scrape_item: ScrapeItem) -> NoReturn:+ raise++ await raise_e(self, scrape_item)finally:self.scraping_progress.remove_task(task_id)@@ -207,8 +215,10 @@ class Crawler(ABC):await self.manager.states.RUNNING.wait()self.waiting_items += 1scrape_prefix = "Scraping"- if self.DOMAIN == "generic":+ if self.IS_FALLBACK_GENERIC:scrape_prefix += " (unsupported domain)"+ else:+ scrape_prefix += f" [{self.FOLDER_DOMAIN}]"async with self._semaphore:self.waiting_items -= 1@@ -246,6 +256,10 @@ class Crawler(ABC):media_item = MediaItem(url, scrape_item, download_folder, filename, original_filename, debrid_link, ext=ext)await self.handle_media_item(media_item, m3u8_media)+ def log_bug_report(self, msg: str, level: int = 30) -> None:+ msg += f". Please file a bug report at {NEW_ISSUE_URL}"+ log(msg, level)+async def handle_media_item(self, media_item: MediaItem, m3u8_media: M3U8Media | None = None) -> None:await self.manager.states.RUNNING.wait()if media_item.datetime and not isinstance(media_item.datetime, int):@@ -323,20 +337,19 @@ class Crawler(ABC):# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~- def get_filename_and_ext(self, filename: str, *args, assume_ext: str | None = None, **kwargs):+ def get_filename_and_ext(+ self, filename: str, forum: bool = False, assume_ext: str | None = None+ ) -> tuple[str, str]:"""Wrapper around `utils.get_filename_and_ext`.-- If `ignore_options.exclude_files_with_no_extension` is `True`, `assume_ext` is not None and the file has no extension,- The value of `assume_ext` will be used as `ext`-- In any other case, it will just call `utils.get_filename_and_ext`+ Calls it as is.+ If that fails, appedns `assume_ext` and tries again, but only if the user had exclude_files_with_no_extension = `False`"""- filename_as_path = Path(filename)- if assume_ext and self.allow_no_extension and not filename_as_path.suffix:- filename_as_path = filename_as_path.with_suffix(assume_ext)- new_filename, ext = get_filename_and_ext(str(filename_as_path), *args, *kwargs)- return Path(new_filename).stem, ext- return get_filename_and_ext(filename, *args, *kwargs)+ try:+ return get_filename_and_ext(filename, forum)+ except NoExtensionError:+ if assume_ext and self.allow_no_extension:+ return get_filename_and_ext(filename + assume_ext, forum)+ raisedef check_album_results(self, url: URL, album_results: dict[str, Any]) -> bool:"""Checks whether an album has completed given its domain and album id."""diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.pyindex c2ae5fd55..7a0681abc 100644--- a/cyberdrop_dl/crawlers/discourse/_discourse.py+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py@@ -89,6 +89,9 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):async def forum(self, scrape_item: ScrapeItem) -> None:raise NotImplementedError+ def parse_thread(self, *args) -> None:+ raise NotImplementedError+async def thread(self, scrape_item: ScrapeItem, topic: Topic) -> None:await self.topic(scrape_item, topic)@@ -124,7 +127,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):returnasync def post(self, scrape_item: ScrapeItem, post: AvailablePost) -> None:- title = self.create_separate_post_title(post.title, str(post.id), None)+ title = self.create_separate_post_title(post.title, str(post.id), post.created_at)scrape_item.setup_as_post(title)for link in self.extract_links(post):await self.handle_link(scrape_item, link)diff --git a/cyberdrop_dl/crawlers/invision/__init__.py b/cyberdrop_dl/crawlers/invision/__init__.pynew file mode 100644index 000000000..3bbf0ee06--- /dev/null+++ b/cyberdrop_dl/crawlers/invision/__init__.py@@ -0,0 +1,8 @@+from ._invision import InvisionCrawler+from .bellazon import BellazonCrawler++INVISION_CRAWLERS: set[type[InvisionCrawler]] = {BellazonCrawler}+++INVISION_CRAWLERS_MAP = {c.__name__: c for c in INVISION_CRAWLERS}+__all__ = list(INVISION_CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]diff --git a/cyberdrop_dl/crawlers/invision/_invision.py b/cyberdrop_dl/crawlers/invision/_invision.pynew file mode 100644index 000000000..2c6433c4d--- /dev/null+++ b/cyberdrop_dl/crawlers/invision/_invision.py@@ -0,0 +1,52 @@+"""Generic crawler for any Invision forum++A Invision site has a tag with a `ipsCopyright` class and the text: "Powered By Invision"+"""++from __future__ import annotations++import dataclasses+from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers._forum import HTMLMessageBoardCrawler+from cyberdrop_dl.crawlers.xenforo.xenforo import DEFAULT_XF_POST_SELECTORS, DEFAULT_XF_SELECTORS, XenforoCrawler+from cyberdrop_dl.utils import css++if TYPE_CHECKING:+ from cyberdrop_dl.crawlers._forum import MessageBoardSelectors+ from cyberdrop_dl.crawlers.crawler import SupportedPaths+++Selector = css.CssAttributeSelector+++DEFAULT_INVISION_POST_SELECTORS = dataclasses.replace(+ DEFAULT_XF_POST_SELECTORS,+ article="div[class*=ipsComment_content]",+ content="div.cPost_contentWrap",+ article_trash=(".message-signature", ".message-footer"),+ content_trash=("blockquote", "fauxBlockLink"),+ id=Selector("data-commentid", "data-commentid"),+ attachments=Selector("a[class*=ipsAttachLink]", "href"),+ images=Selector("a[class*=ipsAttachLink_image]", "href"),+ videos=Selector("video.ipsEmbeddedVideo source", "src"),+)+++DEFAULT_INVISION_SELECTORS = dataclasses.replace(+ DEFAULT_XF_SELECTORS,+ posts=DEFAULT_INVISION_POST_SELECTORS,+ title=Selector("span.ipsType_break.ipsContained span"),+ next_page=Selector("li.ipsPagination_next a[href]", "href"),+)+++class InvisionCrawler(HTMLMessageBoardCrawler, is_abc=True):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = XenforoCrawler.SUPPORTED_PATHS | {"**NOTE**": "base crawler: Invision"} # type: ignore+ SUPPORTS_THREAD_RECURSION: ClassVar[bool] = True+ SELECTORS: ClassVar[MessageBoardSelectors] = DEFAULT_INVISION_SELECTORS+ POST_URL_PART_NAME: ClassVar[str] = "comment"+ PAGE_URL_PART_NAME: ClassVar[str] = "page"+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = False+ LOGIN_USER_COOKIE_NAME: ClassVar[str] = "UNKNOWN" # TODO: Find out cookie name+ ATTACHMENT_HOSTS = ()diff --git a/cyberdrop_dl/crawlers/invision/bellazon.py b/cyberdrop_dl/crawlers/invision/bellazon.pynew file mode 100644index 000000000..127198373--- /dev/null+++ b/cyberdrop_dl/crawlers/invision/bellazon.py@@ -0,0 +1,14 @@+from __future__ import annotations++from typing import ClassVar++from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL++from ._invision import InvisionCrawler+++class BellazonCrawler(InvisionCrawler):+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.bellazon.com/main/")+ DOMAIN: ClassVar[str] = "bellazon"+ FOLDER_DOMAIN: ClassVar[str] = "Bellazon"+ login_required = Falsediff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex aba54c3e8..d364f9a1a 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -46,7 +46,7 @@ class JPG5Crawler(CheveretoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg5.su")def __post_init__(self) -> None:- self.request_limiter = AsyncLimiter(1, 5)+ self.request_limiter = AsyncLimiter(1, 1)async def fetch(self, scrape_item: ScrapeItem) -> None:scrape_item.url = fix_host(scrape_item.url)diff --git a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.pyindex 5ff7e8bcf..26bc06d9e 100644--- a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py+++ b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py@@ -5,12 +5,14 @@ import itertoolsfrom typing import TYPE_CHECKING, ClassVarfrom xml.etree import ElementTree-from cyberdrop_dl.crawlers.xenforo.xenforo import Thread, XenforoCrawler+from cyberdrop_dl.crawlers.xenforo.xenforo import XenforoCrawlerfrom cyberdrop_dl.exceptions import MaxChildrenError, ScrapeErrorif TYPE_CHECKING:+ import datetimefrom collections.abc import Iterable+ from cyberdrop_dl.crawlers._forum import Threadfrom cyberdrop_dl.crawlers.crawler import SupportedPathsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem@@ -22,6 +24,7 @@ class Post:id: inttitle: strxml: ElementTree.Element[str]+ date: datetime.datetime | None = None@staticmethoddef new(element: ElementTree.Element[str]) -> Post:@@ -65,7 +68,7 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801async def async_startup(self) -> None:if not self.logged_in:login_url = self.PRIMARY_URL / "login.php"- await self.login_setup(login_url)+ await self._login(login_url)self.register_cache_filter(self.PRIMARY_URL, lambda _: True)diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.pyindex 1e1697c25..5b1bc0159 100644--- a/cyberdrop_dl/crawlers/wordpress/__init__.py+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py@@ -12,4 +12,4 @@ if TYPE_CHECKING:WP_CRAWLERS: set[type[WordPressBaseCrawler]] = {BestPrettyGirlCrawler}WP_CRAWLERS_MAP = {c.__name__: c for c in WP_CRAWLERS}-WP_GENERIC_CRAWLERS = {WordPressMediaCrawler, WordPressHTMLCrawler}+__all__ = [*WP_CRAWLERS_MAP.keys(), "WordPressMediaCrawler", "WordPressHTMLCrawler"] # type: ignore[reportUnsupportedDunderAll]diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.pyindex f90cd0af8..21336e108 100644--- a/cyberdrop_dl/crawlers/wordpress/_wordpress.py+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py@@ -171,7 +171,8 @@ class WordPressBaseCrawler(Crawler, is_abc=True):async def handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:for link in self.iter_parse_url(_iter_links(post.content, self.WP_USE_REGEX)):- await self.handle_link(scrape_item, link)+ if link:+ await self.handle_link(scrape_item, link)def parse_url(self, link: str) -> AbsoluteHttpURL:# TODO: handle more domains and move it to the base crawler@@ -334,7 +335,7 @@ def _get_original_quality_link(link: str) -> str:return link-def _iter_links(html: HTML, use_regex: bool) -> itertools.chain[str]:+def _iter_links(html: HTML, use_regex: bool) -> Iterable[str]:soup = BeautifulSoup(html, "html.parser")images = css.iget(soup, *css.images)iframes = css.iget(soup, *css.iframes)diff --git a/cyberdrop_dl/crawlers/xenforo/__init__.py b/cyberdrop_dl/crawlers/xenforo/__init__.pyindex b8715e05a..f69a18668 100644--- a/cyberdrop_dl/crawlers/xenforo/__init__.py+++ b/cyberdrop_dl/crawlers/xenforo/__init__.py@@ -1,5 +1,4 @@from .allporncomix import AllPornComixCrawler-from .bellazon import BellazonCrawlerfrom .celebforum import CelebForumCrawlerfrom .f95zone import F95ZoneCrawlerfrom .leakedmodels import LeakedModelsCrawler@@ -12,7 +11,6 @@ from .xenforo import XenforoCrawlerXF_CRAWLERS: set[type[XenforoCrawler]] = {AllPornComixCrawler,- BellazonCrawler,CelebForumCrawler,F95ZoneCrawler,LeakedModelsCrawler,diff --git a/cyberdrop_dl/crawlers/xenforo/allporncomix.py b/cyberdrop_dl/crawlers/xenforo/allporncomix.pyindex 71bb4c0db..6012cc22c 100644--- a/cyberdrop_dl/crawlers/xenforo/allporncomix.py+++ b/cyberdrop_dl/crawlers/xenforo/allporncomix.py@@ -12,3 +12,4 @@ class AllPornComixCrawler(XenforoCrawler):DOMAIN: ClassVar[str] = "allporncomix"FOLDER_DOMAIN: ClassVar[str] = "AllPornComix"login_required = False+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar = Falsediff --git a/cyberdrop_dl/crawlers/xenforo/bellazon.py b/cyberdrop_dl/crawlers/xenforo/bellazon.pydeleted file mode 100644index 037ac7fab..000000000--- a/cyberdrop_dl/crawlers/xenforo/bellazon.py+++ /dev/null@@ -1,33 +0,0 @@-from __future__ import annotations--from typing import ClassVar--from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL--from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors--_post_selectors = PostSelectors(- article="div[class*=ipsComment_content]",- attachments=Selector("a[class*=ipsAttachLink]", "href"),- content=Selector("div.cPost_contentWrap"),- id=Selector("data-commentid", "data-commentid"),- images=Selector("a[class*=ipsAttachLink_image]", "href"),- videos=Selector("video.ipsEmbeddedVideo source", "src"),-)---# TODO: Bellazon uses Invision, not Xenforo. The selectors are completely different.-# See: https://github.com/jbsparrow/CyberDropDownloader/pull/1079#issuecomment-2982845352-class BellazonCrawler(XenforoCrawler):- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.bellazon.com/main/")- DOMAIN: ClassVar[str] = "bellazon"- FOLDER_DOMAIN: ClassVar[str] = "Bellazon"- login_required = False-- XF_THREAD_URL_PART = "topic"- XF_POST_URL_PART_NAME = "comment-"- XF_SELECTORS = XenforoSelectors(- posts=_post_selectors,- title=Selector("span.ipsType_break.ipsContained span"),- next_page=Selector("li.ipsPagination_next a[href]", "href"),- )diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.pyindex fde678e24..f7b47b374 100644--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py@@ -11,11 +11,13 @@ class CelebForumCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://celebforum.to")DOMAIN: ClassVar[str] = "celebforum"FOLDER_DOMAIN: ClassVar[str] = "CelebForum"+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar = True # images src is always a thumbnail- def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:- if link.host == self.PRIMARY_URL.host:+ @classmethod+ def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool:+ if link.host == cls.PRIMARY_URL.host:if all(part in link.parts for part in ["data", "attachments"]): # Thumbnails- return None+ return Trueif all(part in link.parts for part in ["data", "assets"]): # Placeholder content for insufficient rank- return None- return link+ return True+ return Falsediff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.pyindex 21a00abfa..df4caf1fe 100644--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py@@ -1,16 +1,12 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils.utilities import error_handling_wrapperfrom .xenforo import XenforoCrawler-if TYPE_CHECKING:- from yarl import URL--_confirmation_data = ({"xhr": "1", "download": "1"},)@@ -21,14 +17,22 @@ class F95ZoneCrawler(XenforoCrawler):@error_handling_wrapperasync def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:- """Override to handle protected link confirmation."""async with self.request_limiter:json_resp = await self.client.post_data(self.DOMAIN, link, data=_confirmation_data)if json_resp["status"] == "ok":return self.parse_url(json_resp["msg"])- def filter_link(self, link: AbsoluteHttpURL) -> URL:- if "thumb" in link.parts:- return link.with_path(link.path.replace("/thumb/", ""))- return link+ @classmethod+ def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool:+ return "thumb" in link.parts++ @classmethod+ def thumbnail_to_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ return url.with_path(url.path.replace("/thumb/", ""))++ def parse_url(self, link: str) -> AbsoluteHttpURL:+ url = super().parse_url(link)+ if self.is_thumbnail(url):+ return self.thumbnail_to_img(url)+ return urldiff --git a/cyberdrop_dl/crawlers/xenforo/simpcity.py b/cyberdrop_dl/crawlers/xenforo/simpcity.pyindex 2ccb879d7..0b21d0ae4 100644--- a/cyberdrop_dl/crawlers/xenforo/simpcity.py+++ b/cyberdrop_dl/crawlers/xenforo/simpcity.py@@ -2,6 +2,8 @@ from __future__ import annotationsfrom typing import ClassVar+from aiolimiter import AsyncLimiter+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom .xenforo import XenforoCrawler@@ -11,5 +13,10 @@ class SimpCityCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://simpcity.su")DOMAIN: ClassVar[str] = "simpcity"FOLDER_DOMAIN: ClassVar[str] = "SimpCity"- XF_USER_COOKIE_NAME = "dontlikebots_user"+ LOGIN_USER_COOKIE_NAME = "dontlikebots_user"login_required = False+ IGNORE_EMBEDED_IMAGES_SRC = False++ def __post_init__(self) -> None:+ super().__post_init__()+ self.request_limiter = AsyncLimiter(1, 10)diff --git a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.pyindex fb6d28e0d..4fe5f8c07 100644--- a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py+++ b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py@@ -11,3 +11,4 @@ class SocialMediaGirlsCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://forums.socialmediagirls.com")DOMAIN: ClassVar[str] = "socialmediagirls"FOLDER_DOMAIN: ClassVar[str] = "SocialMediaGirls"+ IGNORE_EMBEDED_IMAGES_SRC = Falsediff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.pyindex 61c7e342e..b34d59bf5 100644--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py@@ -16,22 +16,22 @@ class TitsInTopsCrawler(XenforoCrawler):DOMAIN: ClassVar[str] = "titsintops"FOLDER_DOMAIN: ClassVar[str] = "TitsInTops"- def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL:- return AbsoluteHttpURL(- str(link)- .replace("index.php%3F", "index.php/")- .replace("index.php?", "index.php/")- .replace("index.php/goto", "index.php?goto")- )-- def pre_filter_link(self, link: str) -> str:- return link.replace("index.php?", "index.php/").replace("index.php%3F", "index.php/")-- def is_image_username_or_attachment(self, link_obj: Tag) -> bool:+ def parse_url(self, link: str) -> AbsoluteHttpURL:+ return super().parse_url(fix_link(link))++ def is_username_or_attachment(self, link_obj: Tag) -> bool:text = css.get_text(link_obj)if "view attachment" in text.lower():return Truetitle = css.get_attr_no_error(link_obj, "title")if title and "permanent link" in title.lower():return True- return super().is_image_username_or_attachment(link_obj)+ return super().is_username_or_attachment(link_obj)+++def fix_link(link: str) -> str:+ return (+ link.replace("index.php%3F", "index.php/")+ .replace("index.php?", "index.php/")+ .replace("index.php?goto", "index.php/goto")+ )diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex ba203eb3a..2dfebdf95 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -1,409 +1,96 @@-# ruff : noqa: RUF009+"""Generic crawler for any Xenforo++A Xenforo site has these attributes attached to the main html tag of the site:+id="XF" This identifies the site as a Xenforo site+data-cookie-prefix="ogaddgmetaprof_" The full cookies name will be `ogaddgmetaprof_user`+data-xf="2.3" Version number+++Xenforo sites have a REST API but the APi is private only. Admins of the site need to grand access user by user+"""from __future__ import annotationsimport asyncio-import dataclasses-import datetime-import refrom typing import TYPE_CHECKING, ClassVar-from bs4 import BeautifulSoup, Tag+from bs4 import BeautifulSoup-from cyberdrop_dl.constants import HTTP_REGEX_LINKS-from cyberdrop_dl.crawlers._forum import ForumCrawler+from cyberdrop_dl.crawlers._forum import HTMLMessageBoardCrawler, MessageBoardSelectors, PostSelectorsfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError+from cyberdrop_dl.exceptions import LoginError, ScrapeErrorfrom cyberdrop_dl.utils import css-from cyberdrop_dl.utils.dates import to_timestamp-from cyberdrop_dl.utils.utilities import (- error_handling_wrapper,- get_text_between,- is_absolute_http_url,- is_blob_or_svg,-)+from cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import AsyncGenerator, Iterable-- from aiohttp_client_cache.response import AnyResponse+ from collections.abc import Iterable, Sequencefrom cyberdrop_dl.crawlers.crawler import SupportedPaths- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem-+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-LINK_TRASH_MAPPING = {- ".th.": ".",- ".md.": ".",- "ifr": "watch",-}Selector = css.CssAttributeSelector-LAST_PAGE_SELECTOR = Selector("li.pageNav-page a:last-of-type", "href")-CURRENT_PAGE_SELECTOR = Selector("li.pageNav-page.pageNav-page--current a", "href")--[email protected](frozen=True, slots=True)-class PostSelectors:- article: str = "article.message[id*=post]" # the entire html of the post (comments, attachments, user avatar, signature, etc...)-- attachments: Selector = Selector(".message-attachments a[href]", "href")- content: Selector = Selector(".message-userContent") # text, links and images (NO attachments)- date: Selector = Selector("time", "datetime")- embeds: Selector = Selector("iframe", "src")- id: Selector = Selector(".u-concealed a[href]", "href") # TODO: This needs a better selector- images: Selector = Selector("img.bbImage", "src")- lazy_load_embeds: Selector = Selector('[class*=iframe][onclick*="loadMedia(this, \'//"]', "onclick")- links: Selector = Selector(":any-link", "href")- videos: Selector = Selector("video source", "src")--[email protected](frozen=True, slots=True)-class XenforoSelectors:- confirmation_button: Selector = Selector("a[class*=button--cta][href]", "href")- next_page: Selector = Selector("a[class*=pageNav-jump--next][href]", "href")- posts: PostSelectors = PostSelectors()- quotes: Selector = Selector("blockquote")- title_trash: Selector = Selector("span")- title: Selector = Selector("h1[class*=p-title-value]")- last_page: Selector = LAST_PAGE_SELECTOR- current_page: Selector = CURRENT_PAGE_SELECTOR--[email protected](frozen=True, slots=True, order=True)-class ForumPost:- id: int- date: datetime.datetime- article: Tag = dataclasses.field(compare=False)- content: Tag = dataclasses.field(compare=False)-- @staticmethod- def new(article: Tag, selectors: PostSelectors, post_name: str = "post") -> ForumPost:- content = css.select_one(article, selectors.content.element)- date = datetime.datetime.fromisoformat(css.select_one_get_attr(article, *selectors.date))- if selectors.id.element == selectors.id.attribute:- post_id = int(css.get_attr(article, selectors.id.element))- else:- id_str = css.select_one_get_attr(article, *selectors.id)- post_id = int(id_str.rsplit(post_name, 1)[-1].replace("-", ""))- return ForumPost(post_id, date, article, content)-[email protected](frozen=True, slots=True)-class Thread:- id: int- name: str- page: int- post_id: int | None- url: AbsoluteHttpURL+DEFAULT_XF_POST_SELECTORS = PostSelectors(+ article="article.message[id*=post]",+ content=".message-userContent",+ article_trash=(".message-signature", ".message-footer"),+ content_trash=("blockquote", "fauxBlockLink"),+ id=Selector("article.message[id*=post]", "id"),+ attachments=Selector(".message-attachments a[href]", "href"),+)-DEFAULT_XF_SELECTORS = XenforoSelectors()-KNOWN_THREAD_PART_NAMES = "thread", "topic", "tema"-KNOWN_THREAD_PART_NAMES = ",".join(f"{part},{part}s" for part in KNOWN_THREAD_PART_NAMES).split(",")+DEFAULT_XF_SELECTORS = MessageBoardSelectors(+ posts=DEFAULT_XF_POST_SELECTORS,+ confirmation_button=Selector("a[class*=button--cta][href]", "href"),+ next_page=Selector("a[class*=pageNav-jump--next][href]", "href"),+ title_trash=("span",),+ title=Selector("h1[class*=p-title-value]"),+ last_page=Selector("li.pageNav-page a:last-of-type", "href"),+ current_page=Selector("li.pageNav-page.pageNav-page--current a", "href"),+)def _escape(strings: Iterable[str]) -> str:return r"\|".join(strings)-class XenforoCrawler(ForumCrawler, is_abc=True):- XF_ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"+class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):+ ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"+ THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Attachments": f"/({_escape(XF_ATTACHMENT_URL_PARTS)})/...",+ "Attachments": f"/({_escape(ATTACHMENT_URL_PARTS)})/...","Threads": (- f"/({_escape(KNOWN_THREAD_PART_NAMES)})/<thread_name_and_id>",+ f"/({_escape(THREAD_PART_NAMES)})/<thread_name_and_id>","/posts/<post_id>","/goto/<post_id>",),"**NOTE**": "base crawler: Xenforo",}SUPPORTS_THREAD_RECURSION: ClassVar[bool] = True- XF_SELECTORS = DEFAULT_XF_SELECTORS- XF_POST_URL_PART_NAME = "post-"- XF_PAGE_URL_PART_NAME = "page-"- XF_THREAD_URL_PART = "threads"- XF_USER_COOKIE_NAME = "xf_user"- XF_ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"+ SELECTORS: ClassVar[MessageBoardSelectors] = DEFAULT_XF_SELECTORS+ POST_URL_PART_NAME: ClassVar[str] = "post"+ PAGE_URL_PART_NAME: ClassVar[str] = "page"+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = True+ LOGIN_USER_COOKIE_NAME: ClassVar[str] = "xf_user"+ # Attachments hosts should technically be defined on each specific Crawler, but they do no harm here+ ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"login_required = True- def __post_init__(self) -> None:- self.scraped_threads = set()-- async def async_startup(self) -> None:- if not self.logged_in:- login_url = self.PRIMARY_URL / "login"- await self.login_setup(login_url)-- self.register_cache_filter(self.PRIMARY_URL, check_is_not_last_page)-def get_filename_and_ext(self, filename: str) -> tuple[str, str]:+ # The `forum` keyword is misleading now. It only works for Xenforo sites, not every forum+ # TODO: Change `forum` parameter to `xenforo`return super().get_filename_and_ext(filename, forum=True)- async def fetch(self, scrape_item: ScrapeItem) -> None:- if not self.logged_in and self.login_required:- return- url = self.pre_filter_link(str(scrape_item.url))- scrape_item.url = self.parse_url(url)- if self.is_attachment(scrape_item.url):- return await self.handle_internal_link(scrape_item)- if is_confirmation_link(scrape_item.url):- return await self.follow_confirmation_link(scrape_item)-- await self.fetch_thread(scrape_item)-- async def fetch_thread(self, scrape_item: ScrapeItem) -> None:- thread_part_index = len(self.PRIMARY_URL.parts)- match scrape_item.url.parts[thread_part_index:]:- case [thread_part, thread_name_and_id, *_] if thread_part in KNOWN_THREAD_PART_NAMES:- return await self.thread(scrape_item, thread_name_and_id)- case ["goto" | "posts", _, *_]:- return await self.follow_redirect_w_get(scrape_item)-- raise ValueError-- async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:- url = await self.resolve_confirmation_link(scrape_item.url)- if url: # If there was an error, this will be None- scrape_item.url = url- # This could end up back in here if the URL goes to another thread- return self.handle_external_links(scrape_item)-- @error_handling_wrapper- async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:- async with self.request_limiter:- response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)- assert is_absolute_http_url(response.url)- scrape_item.url = response.url- self.manager.task_group.create_task(self.run(scrape_item))-@error_handling_wrapper- async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:- head = await self.client.get_head(self.DOMAIN, scrape_item.url)- redirect = head.get("location")- if not redirect:- raise ScrapeError(422)- scrape_item.url = self.parse_url(redirect)- self.manager.task_group.create_task(self.run(scrape_item))-- async def forum(self, scrape_item: ScrapeItem) -> None:- raise NotImplementedError-- @error_handling_wrapper- async def thread(self, scrape_item: ScrapeItem, thread_name_and_id: str) -> None:- scrape_item.setup_as_forum("")- thread = parse_thread(- scrape_item.url, thread_name_and_id, self.XF_PAGE_URL_PART_NAME, self.XF_POST_URL_PART_NAME- )- if thread.url in self.scraped_threads:- return-- scrape_item.parent_threads.add(thread.url)- if self.scrape_single_forum_post and not thread.post_id:- msg = "`--scrape-single-forum-post` is `True`, but the provided URL has no post id"- raise ScrapeError("User Error", msg)-- self.scraped_threads.add(thread.url)- await self.process_thread(scrape_item, thread)-- async def process_thread(self, scrape_item: ScrapeItem, thread: Thread) -> None:- title: str = ""- last_post_url = thread.url- async for soup in self.thread_pager(scrape_item):- if not title:- try:- title = self.create_title(get_post_title(soup, self.XF_SELECTORS), thread_id=thread.id)- except ScrapeError as e:- self.log_debug("Got an unprocessable soup", 40, exc_info=e)- raise- scrape_item.add_to_parent_title(title)-- continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)- if not continue_scraping:- break-- await self.write_last_forum_post(thread.url, last_post_url)-- def process_thread_page(- self, scrape_item: ScrapeItem, thread: Thread, soup: BeautifulSoup- ) -> tuple[bool, AbsoluteHttpURL]:- continue_scraping = False- post_url = thread.url- for article in soup.select(self.XF_SELECTORS.posts.article):- current_post = ForumPost.new(article, self.XF_SELECTORS.posts, self.XF_POST_URL_PART_NAME)- continue_scraping, scrape_this_post = check_post_id(- thread.post_id, current_post.id, self.scrape_single_forum_post- )- if scrape_this_post:- post_url = thread.url / f"{self.XF_POST_URL_PART_NAME}{current_post.id}"- new_scrape_item = scrape_item.create_new(- thread.url,- possible_datetime=to_timestamp(current_post.date),- add_parent=post_url,- )- self.manager.task_group.create_task(self.post(new_scrape_item, current_post))- try:- scrape_item.add_children()- except MaxChildrenError:- break-- if not continue_scraping:- break- return continue_scraping, post_url-- @error_handling_wrapper- async def post(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- scrape_item.setup_as_post("")- post_title = self.create_separate_post_title(None, str(post.id), post.date)- scrape_item.add_to_parent_title(post_title)- for scraper in (- self._attachments,- self._embeds,- self._images,- self._links,- self._videos,- self._lazy_load_embeds,- ):- await scraper(scrape_item, post)-- async def _links(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = self.XF_SELECTORS.posts.links- links = css.iselect(post.content, selector.element)- valid_links = (link for link in links if not self.is_image_username_or_attachment(link))- await self.process_children(scrape_item, valid_links, selector.attribute)-- async def _images(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = self.XF_SELECTORS.posts.images- images = css.iselect(post.content, selector.element)- await self.process_children(scrape_item, images, selector.attribute)-- async def _videos(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = self.XF_SELECTORS.posts.videos- videos = css.iselect(post.content, selector.element)- await self.process_children(scrape_item, videos, selector.attribute)-- async def _embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = self.XF_SELECTORS.posts.embeds- embeds = css.iselect(post.content, selector.element)- await self.process_children(scrape_item, embeds, selector.attribute, embeds=True)-- async def _attachments(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = self.XF_SELECTORS.posts.attachments- attachments = css.iselect(post.article, selector.element)- await self.process_children(scrape_item, attachments, selector.attribute)-- async def _lazy_load_embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = self.XF_SELECTORS.posts.lazy_load_embeds- for lazy_media in css.iselect(post.content, selector.element):- link_str = get_text_between(css.get_attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")- await self.process_child(scrape_item, link_str, embeds=True)-- async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:- async for soup in self._web_pager(scrape_item.url, self.get_next_page):- yield soup-- def get_next_page(self, soup: BeautifulSoup) -> str | None:- if next_page := css.select_one_get_attr_or_none(soup, *self.XF_SELECTORS.next_page):- return self.pre_filter_link(next_page)-- @error_handling_wrapper- async def process_children(- self, scrape_item: ScrapeItem, links: Iterable[Tag], attribute: str, *, embeds: bool = False- ) -> None:- for link_tag in links:- if link_str := css.get_attr_no_error(link_tag, attribute):- await self.process_child(scrape_item, link_str, embeds=embeds)-- @error_handling_wrapper- async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:- link_str_ = pre_process_child(link_str, embeds)- if not link_str_:- return- link = await self.get_absolute_link(link_str_)- link = self.filter_link(link)- if not link:- return- await self.handle_link(scrape_item, link)-- def is_attachment(self, link: AbsoluteHttpURL | str) -> bool:- if not link:- return False- if isinstance(link, str):- link = self.parse_url(link)- by_parts = len(link.parts) > 2 and any(p in link.parts for p in self.XF_ATTACHMENT_URL_PARTS)- by_host = any(host in link.host for host in self.XF_ATTACHMENT_HOSTS)- return by_parts or by_host-- async def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpURL | None:- if isinstance(link, str):- absolute_link = self.parse_url(clean_link_str(link))- else:- absolute_link = link- if is_confirmation_link(absolute_link):- return await self.resolve_confirmation_link(absolute_link)- return absolute_link-- @error_handling_wrapper- async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)- selector = self.XF_SELECTORS.confirmation_button- confirm_button = soup.select_one(selector.element)- if not confirm_button:- return- link_str: str = css.get_attr(confirm_button, selector.attribute)- link_str = link_str.split('" class="link link--internal', 1)[0]- new_link = self.parse_url(link_str)- return await self.get_absolute_link(new_link)-- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:- """Handles internal links."""- slug = scrape_item.url.name or scrape_item.url.parent.name- if slug.isdigit():- return await self.follow_redirect_w_head(scrape_item)-- return await super().handle_internal_link(scrape_item)-- def is_image_username_or_attachment(self, link_obj: Tag) -> bool:- if link_obj.select_one("img"):- return True- if link_obj.select_one(".username"):- return True- if link_str := css.get_attr_no_error(link_obj, self.XF_SELECTORS.posts.links.element):- return self.is_attachment(link_str)- return False-- def pre_filter_link(self, link: str) -> str:- return link-- def filter_link(self, link: AbsoluteHttpURL | None) -> AbsoluteHttpURL | None:- return link-- @error_handling_wrapper- async def login_setup(self, login_url: AbsoluteHttpURL) -> None:- host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)- session_cookie = host_cookies.get(self.XF_USER_COOKIE_NAME)- session_cookie = session_cookie.value if session_cookie else None- msg = f"No cookies found for {self.FOLDER_DOMAIN}"- if not session_cookie and self.login_required:- raise LoginError(message=msg)-- _, self.logged_in = await self.check_login_with_request(login_url)-- if self.logged_in:- return- if session_cookie:- msg = f"Cookies for {self.FOLDER_DOMAIN} are not valid."- if self.login_required:- raise LoginError(message=msg)+ async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:+ """Logic to login as a Xenforo user- msg += " Scraping without an account"- self.log(msg, 30)+ This was deprecated in v6.5.0 but the code itself it useful for debuggig without cookie extraction.+ Login functionality may come back in a future version..."""- @error_handling_wrapper- async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:- """Logs in to a forum."""manual_login = username and passwordmissing_credentials = not (manual_login or session_cookie)if missing_credentials:@@ -411,7 +98,7 @@ class XenforoCrawler(ForumCrawler, is_abc=True):raise LoginError(message=msg)if session_cookie:- cookies = {"xf_user": session_cookie}+ cookies = {self.LOGIN_USER_COOKIE_NAME: session_cookie}self.update_cookies(cookies)credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}@@ -424,7 +111,7 @@ class XenforoCrawler(ForumCrawler, is_abc=True):retries: int,wait_time: int | None = None,) -> None:- # Try from cookies+ # Check first if we have cookies and they are validtext, logged_in = await self.check_login_with_request(login_url)if logged_in:self.logged_in = True@@ -448,91 +135,6 @@ class XenforoCrawler(ForumCrawler, is_abc=True):msg = f"Failed to login on {self.FOLDER_DOMAIN} after {retries} attempts"raise LoginError(message=msg)- async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:- text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)- return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))---def parse_thread(url: AbsoluteHttpURL, thread_name_and_id: str, page_part_name: str, post_part_name: str) -> Thread:- name_index = url.parts.index(thread_name_and_id)- name, id_ = parse_thread_name_and_id(thread_name_and_id)- page, post_id = get_thread_page_and_post(url, name_index, page_part_name, post_part_name)- canonical_url = get_thread_canonical_url(url, name_index)- return Thread(id_, name, page, post_id, canonical_url)---def parse_thread_name_and_id(thread_name_and_id: str) -> tuple[str, int]:- try:- name, id_str = thread_name_and_id.rsplit(".", 1)- except ValueError:- id_str, name = thread_name_and_id.split("-", 1)- return name, int(id_str)---def get_thread_canonical_url(url: AbsoluteHttpURL, thread_name_index: int) -> AbsoluteHttpURL:- new_parts = url.parts[1 : thread_name_index + 1]- new_path = "/".join(new_parts)- return url.with_path(new_path)---def get_thread_page_and_post(- url: AbsoluteHttpURL, thread_name_index: int, page_name: str, post_name: str-) -> tuple[int, int | None]:- extra_parts = url.parts[thread_name_index + 1 :]- if url.fragment:- extra_parts = *extra_parts, url.fragment-- def find_number(search_value: str) -> int | None:- for sec in extra_parts:- if search_value in sec:- return int(sec.replace(search_value, "").replace("-", "").strip())-- post_id = find_number(post_name)- page_number = find_number(page_name) or 1- return page_number, post_id---async def check_is_not_last_page(response: AnyResponse) -> bool:- soup = BeautifulSoup(await response.text(), "html.parser")- return not is_last_page(soup)---def is_last_page(soup: BeautifulSoup) -> bool:- try:- last_page = css.select_one_get_attr(soup, *LAST_PAGE_SELECTOR)- current_page = css.select_one_get_attr(soup, *CURRENT_PAGE_SELECTOR)- except (AttributeError, IndexError, css.SelectorError):- return True- return current_page == last_page---def get_post_title(soup: BeautifulSoup, selectors: XenforoSelectors) -> str:- try:- title_block = css.select_one(soup, selectors.title.element)- trash = title_block.select(selectors.title_trash.element)- for element in trash:- element.decompose()- except (AttributeError, AssertionError, css.SelectorError) as e:- raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e-- if title := " ".join(css.get_text(title_block).split()):- return title- raise ScrapeError(422)---def extract_embed_url(embed_str: str) -> str:- embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")- if match := re.search(HTTP_REGEX_LINKS, embed_str):- return match.group(0).replace("www.", "")- return embed_str---# TODO: move to parse_url-def clean_link_str(link: str) -> str:- for old, new in LINK_TRASH_MAPPING.items():- link = link.replace(old, new)- return link-def parse_login_form(resp_text: str) -> dict[str, str]:soup = BeautifulSoup(resp_text, "html.parser")@@ -545,32 +147,3 @@ def parse_login_form(resp_text: str) -> dict[str, str]:if data:return dataraise ScrapeError(422)---def is_confirmation_link(link: AbsoluteHttpURL) -> bool:- return "masked" in link.parts or "link-confirmation" in link.path---def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]:- """Checks if the program should scrape the current post.-- Returns (continue_scraping, scrape_this_post)"""- if init_post_id:- if init_post_id > current_post_id:- return (True, False)- elif init_post_id == current_post_id:- return (not scrape_single_forum_post, True)- else:- return (not scrape_single_forum_post, not scrape_single_forum_post)-- assert not scrape_single_forum_post # We should have raised an exception early- return True, True---def pre_process_child(link_str: str, embeds: bool = False) -> str | None:- assert isinstance(link_str, str)- if embeds:- link_str = extract_embed_url(link_str)-- if link_str and not is_blob_or_svg(link_str):- return link_strdiff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex 4d63fbeb3..d6c857d6c 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -176,7 +176,7 @@ class Video(XHamsterItem):mp4_sources: dict[str, Any] = Field({}, validation_alias=AliasPath("sources", "mp4"))def get_formats(self) -> Generator[Format]:- yield Format(0, 0, self.mp4_file)+ yield Format(0, "", self.mp4_file)for resolution, details in self.mp4_sources.items():height = int(resolution.removesuffix("p"))link: str = details["link"] if isinstance(details, dict) else detailsdiff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 2f2ce7d68..f7e93dcec 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -112,9 +112,15 @@ def _parse_srcset(srcset: str) -> str:return [src.split(" ")[0] for src in srcset.split(", ")][-1]-def iget(tag: Tag, selector: str, attribute: str) -> Generator[str | None]:+def iget(tag: Tag, selector: str, attribute: str) -> Generator[str]:for inner_tag in iselect(tag, selector):- yield get_attr_or_none(inner_tag, attribute)+ if link := get_attr_or_none(inner_tag, attribute):+ yield link+++def decompose(tag: Tag, selector: str) -> None:+ for inner_tag in tag.select(selector):+ inner_tag.decompose()iframes = CssAttributeSelector("iframe", "src")diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.pyindex de34d54dc..315628705 100644--- a/cyberdrop_dl/utils/logger.py+++ b/cyberdrop_dl/utils/logger.py@@ -5,7 +5,7 @@ import loggingimport queuefrom logging.handlers import QueueHandler, QueueListenerfrom pathlib import Path-from typing import IO, TYPE_CHECKING+from typing import IO, TYPE_CHECKING, Anyfrom rich._log_render import LogRenderfrom rich.console import Console, Group@@ -215,7 +215,7 @@ class RedactedConsole(Console):return _redact_message(output)-def process_log_msg(message: dict | Exception | str) -> str:+def process_log_msg(message: dict[str, Any] | Exception | str) -> str:if isinstance(message, dict):return json.dumps(message, indent=4, ensure_ascii=False)return str(message)@@ -227,14 +227,14 @@ def create_rich_log_msg(msg: str, level: int = 10) -> Text:return rich_level + indent_string(msg)-def log(message: dict | Exception | str, level: int = 10, **kwargs) -> None:+def log(message: dict[str, Any] | Exception | str, level: int = 10, prefix: str = "", **kwargs) -> None:"""Simple logging function."""- msg = process_log_msg(message)+ msg = prefix + process_log_msg(message)logger.log(level, msg, **kwargs)log_debug(msg, level, **kwargs)-def log_debug(message: dict | Exception | str, level: int = 10, **kwargs) -> None:+def log_debug(message: dict[str, Any] | Exception | str, level: int = 10, **kwargs) -> None:"""Simple logging function."""if env.DEBUG_VAR:msg = process_log_msg(message)diff --git a/tests/crawlers/test_vbulletin.py b/tests/crawlers/test_vbulletin.pyindex 7a8fa7eb1..78f4e8915 100644--- a/tests/crawlers/test_vbulletin.py+++ b/tests/crawlers/test_vbulletin.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport pytest-from cyberdrop_dl.crawlers.xenforo import xenforo+from cyberdrop_dl.crawlers import _forum, vbulletinfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL@@ -35,6 +35,6 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL)def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str, int, int], canonical_url: str) -> None:url_, canonical_url_ = AbsoluteHttpURL(url), AbsoluteHttpURL(canonical_url)- result_ = xenforo.Thread(*result, canonical_url_)- parsed = xenforo.parse_thread(url_, thread_name_and_id, "page", "post")+ result_ = _forum.Thread(*result, canonical_url_)+ parsed = vbulletin.vBulletinCrawler.parse_thread(url_, thread_name_and_id)assert result_ == parseddiff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.pyindex 70e7f2296..aba68d0ad 100644--- a/tests/crawlers/test_xenforo.py+++ b/tests/crawlers/test_xenforo.py@@ -6,6 +6,7 @@ from unittest import mockimport pytestfrom bs4 import BeautifulSoup+from cyberdrop_dl.crawlers import _forumfrom cyberdrop_dl.crawlers import xenforo as crawlersfrom cyberdrop_dl.crawlers.xenforo import xenforofrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem@@ -47,11 +48,11 @@ def _post(message_attachments: str = "",id: int = 12345,crawler: xenforo.XenforoCrawler | None = None,-) -> xenforo.ForumPost:+) -> _forum.ForumPost:crawler = crawler or TEST_CRAWLERhtml = _html(POST_TEMPLATE.format(id=id, message_body=message_body, message_attachments=message_attachments))- soup = BeautifulSoup(html, "html.parser")- return xenforo.ForumPost.new(soup, crawler.XF_SELECTORS.posts)+ article = BeautifulSoup(html, "html.parser").select("article")[0]+ return _forum.ForumPost.new(article, crawler.SELECTORS.posts)def _item_call(value: Any) -> mock._Call:@@ -199,8 +200,8 @@ async def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch))def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str, int, int], canonical_url: str) -> None:url_, canonical_url_ = AbsoluteHttpURL(url), AbsoluteHttpURL(canonical_url)- result_ = xenforo.Thread(*result, canonical_url_)- parsed = xenforo.parse_thread(url_, thread_name_and_id, "page", "post")+ result_ = _forum.Thread(*result, canonical_url_)+ parsed = TEST_CRAWLER.parse_thread(url_, thread_name_and_id)assert result_ == parsed@@ -238,7 +239,7 @@ def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str,],)def test_clean_link_url(link: str, out: str) -> None:- assert xenforo.clean_link_str(link) == out+ assert _forum.clean_link_str(link) == outdef test_parse_login_form_success() -> None:@@ -357,7 +358,7 @@ def test_parse_login_form_no_input_form_should_fail() -> None:],)def test_extract_embed_url(input_string: str, expected_output: str) -> None:- assert xenforo.extract_embed_url(input_string) == expected_output+ assert _forum.extract_embed_url(input_string) == expected_output@pytest.mark.xfail # regex can not handle URLs with commans in it (kvs)@@ -375,40 +376,40 @@ def test_extract_embed_url(input_string: str, expected_output: str) -> None:],)def test_extract_embed_url_complex_path(input_string: str, expected_output: str) -> None:- assert xenforo.extract_embed_url(input_string) == expected_output+ assert _forum.extract_embed_url(input_string) == expected_outputdef test_extract_embed_url_should_extract_only_one_url() -> None:input_str = r"first_url \/\/first.com/path second_url \/\/www.second.net/another"expected_output = "https://first.com/path"- assert xenforo.extract_embed_url(input_str) == expected_output+ assert _forum.extract_embed_url(input_str) == expected_outputdef test_extract_embed_url_with_escaped_backslashes_and_double_slashes() -> None:input_str = r"some_path\\to\\file\/\/imagepond.net\/clip.mov"expected_output = "https://imagepond.net/clip.mov"- assert xenforo.extract_embed_url(input_str) == expected_output+ assert _forum.extract_embed_url(input_str) == expected_outputdef test_extract_embed_url_absolute_url_no_scheme() -> None:input_str = r"\/\/jpg5.su"expected_output = r"https://jpg5.su"- assert xenforo.extract_embed_url(input_str) == expected_output+ assert _forum.extract_embed_url(input_str) == expected_outputdef test_extract_embed_url_should_not_modify_absolute_urls() -> None:input_str = r"https://celebforum.to/maps"expected_output = "https://celebforum.to/maps"- assert xenforo.extract_embed_url(input_str) == expected_output+ assert _forum.extract_embed_url(input_str) == expected_outputdef test_extract_embed_url_string_ends_with_url() -> None:input_str = r"some text https://www.reddit.com/"expected_output = "https://reddit.com/"- assert xenforo.extract_embed_url(input_str) == expected_output+ assert _forum.extract_embed_url(input_str) == expected_output-async def test_lazy_load_embeds() -> None:+def test_lazy_load_embeds() -> None:post = _post("""<div class="generic2wide-iframe-div" onclick="loadMedia(this, '//redgifs.com/ifr/downrightcluelesswirm');"><span data-s9e-mediaembed="redgifs">@@ -417,19 +418,35 @@ async def test_lazy_load_embeds() -> None:</span></span></div>""")- expected_output = "//redgifs.com/ifr/downrightcluelesswirm"- with _amock() as mocked:- await TEST_CRAWLER._lazy_load_embeds(scrape_item, post)- mocked.assert_called_once_with(scrape_item, expected_output, embeds=True)+ result = list(TEST_CRAWLER._lazy_load_embeds(post))+ assert len(result) == 1+ assert "//redgifs.com/ifr/downrightcluelesswirm" == result[0]@pytest.mark.parametrize(("cls", "post_content", "expected_result"),[+ (+ crawlers.AllPornComixCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>""",+ ["https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"],+ ),(crawlers.SimpCityCrawler,"""- <div class="bbWrapper"><a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener"><imgsrc="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"@@ -461,21 +478,229 @@ async def test_lazy_load_embeds() -> None:"https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg","https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",],- )+ ),+ (+ crawlers.SimpCityCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ ),+ (+ crawlers.SimpCityCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ ),+ (+ crawlers.SimpCityCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ ),+ (+ crawlers.SimpCityCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ ),+ (+ crawlers.SimpCityCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ ),+ (+ crawlers.SimpCityCrawler,+ """+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ ),],)-async def test_post_images(cls: type[xenforo.XenforoCrawler], post_content: str, expected_result: list[str]) -> None:+def test_post_images(cls: type[xenforo.XenforoCrawler], post_content: str, expected_result: list[str]) -> None:crawler = crawler_instances[cls]post = _post(post_content, crawler=crawler)- with _amock(crawler=crawler) as mocked:- await crawler._images(scrape_item, post)- count, expected_count = mocked.call_count, len(expected_result)- assert count == expected_count, f"Found {count} links, expected {expected_count} links"- for result, expected in zip(mocked.call_args_list, expected_result, strict=True):- assert result.args[1] == expected+ results = list(crawler._images(post))+ count, expected_count = len(results), len(expected_result)+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"+ assert results == expected_result-async def test_embeds_can_extract_google_drive_links() -> None:+def test_embeds_can_extract_google_drive_links() -> None:# https://github.com/jbsparrow/CyberDropDownloader/issues/775crawler = crawler_instances[crawlers.SimpCityCrawler]content = """@@ -492,14 +717,13 @@ async def test_embeds_can_extract_google_drive_links() -> None:"""post = _post(content, crawler=crawler)expected_result = "//drive.google.com/file/d/1gfDjCbNXgJafY6ILQIrgbnuptSCFbM0J/preview"- with _amock(crawler=crawler) as mocked:- await crawler._embeds(scrape_item, post)- mocked.assert_called_once()- assert mocked.call_args.args[1] == expected_result+ results = list(crawler._embeds(post))+ assert len(results) == 1+ assert results[0] == expected_result-async def test_post_smg_extract_attachments() -> None:- # https://agithub.com/jbsparrow/CyberDropDownloader/issues/1070+def test_post_smg_extract_attachments() -> None:+ # https://github.com/jbsparrow/CyberDropDownloader/issues/1070attachments = """<h4 class="block-textHeader">Attachments</h4><ul class="attachmentList">@@ -564,12 +788,28 @@ async def test_post_smg_extract_attachments() -> None:"https://smgmedia2.socialmediagirls.com/forum/2022/04/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg","https://smgmedia2.socialmediagirls.com/forum/2022/04/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png",]- with _amock(crawler=crawler) as mocked:- await crawler._attachments(scrape_item, post)- count, expected_count = mocked.call_count, len(expected_result)- assert count == expected_count, f"Found {count} links, expected {expected_count} links"- for result, expected in zip(mocked.call_args_list, expected_result, strict=True):- assert result.args[1] == expected++ result = list(crawler._attachments(post))+ count, expected_count = len(result), len(expected_result)+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"+ assert result == expected_result+++def test_post_celebforum_should_use_href_for_images() -> None:+ # https://github.com/jbsparrow/CyberDropDownloader/issues/1093+ content = """+ <a href="https://celebforum.to/attachments/jc6huqrju9-jpg.321620/" target="_blank"><img alt="JC6HUqrju9" class="bbImage"+ height="200" loading="lazy"+ src="https://celebforum.to/data/attachments/321/321141-2d66067afbf5cd3d546479380c08929d.jpg" style=""+ title="JC6HUqrju9" width="150" /></a>+ """+ crawler = crawler_instances[crawlers.CelebForumCrawler]+ post = _post(content, crawler=crawler)+ expected_result = ["https://celebforum.to/attachments/jc6huqrju9-jpg.321620/"]+ result = list(crawler._images(post))+ count, expected_count = len(result), len(expected_result)+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"+ assert result == expected_resultdef test_get_post_title_thread_w_prefixes() -> None:@@ -588,7 +828,7 @@ def test_get_post_title_thread_w_prefixes() -> None:</div>""")soup = BeautifulSoup(html, "html.parser")- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)assert title == "GunplaMeli"@@ -599,7 +839,7 @@ def test_get_post_title_thread_w_no_prefixes() -> None:</div>"""soup = BeautifulSoup(html, "html.parser")- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)assert title == "Staged/Fake Japanese Candid Videos from Gcolle/Pcolle or FC2"@@ -607,7 +847,7 @@ def test_get_post_title_no_title_found() -> None:html = _html("")soup = BeautifulSoup(html, "html.parser")with pytest.raises(ScrapeError) as exc_info:- xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)assert exc_info.value.status == 429assert exc_info.value.message == "Invalid response from forum. You may have been rate limited"@@ -617,7 +857,7 @@ def test_get_post_title_empty_title_block() -> None:html = _html("""<h1 class="p-title-value"></h1>""")soup = BeautifulSoup(html, "html.parser")with pytest.raises(ScrapeError):- xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)def test_get_post_title_non_english_chars() -> None:@@ -629,7 +869,7 @@ def test_get_post_title_non_english_chars() -> None:</div>""")soup = BeautifulSoup(html, "html.parser")- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"@@ -644,7 +884,7 @@ def test_get_post_title_should_strip_new_lines() -> None:</div>""")soup = BeautifulSoup(html, "html.parser")- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"@@ -693,7 +933,7 @@ class TestCheckPostId:expected_continue_scraping: bool,expected_scrape_this_post: bool,) -> None:- continue_scraping, scrape_this_post = xenforo.check_post_id(+ continue_scraping, scrape_this_post = _forum.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)assert continue_scraping == expected_continue_scraping@@ -703,7 +943,7 @@ class TestCheckPostId:init_post_id = Nonecurrent_post_id = 100scrape_single_forum_post = False- continue_scraping, scrape_this_post = xenforo.check_post_id(+ continue_scraping, scrape_this_post = _forum.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)assert continue_scraping is True@@ -715,23 +955,84 @@ class TestCheckPostId:scrape_single_forum_post = Truewith pytest.raises(AssertionError):- xenforo.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)+ _forum.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)# og_post_id = 23549340POST_TEMPLATE = """-<article class="message message--post js-post js-inlineModContainer" data-author="cyberdrop-dl-patched" data-content="post{id}" id="js-post{id}" itemscope="" itemtype="https://schema.org/Comment" itemid="https://xenforo.com/posts/{id}/">- <meta itemprop="parentItem" itemscope="" itemid="https://xenforo.com/threads/fanfan.33077/" />+<article class="message message--post js-post js-inlineModContainer" data-author="" data-content="post-{id}" id="js-post-{id}" itemscope="" itemtype="https://schema.org/Comment" itemid="https://simpcity.su/posts/{id}/">+ <meta itemprop="parentItem" itemscope="" itemid="https://xenforocomunity.com/threads/fanfan.33077/" />- <span class="u-anchorTarget" id="post{id}"></span>+ <span class="u-anchorTarget" id="post-{id}"></span><div class="message-inner">+ <div class="message-cell message-cell--user">+ <section class="message-user" itemprop="author" itemscope="" itemtype="https://schema.org/Person" itemid="https://xenforocomunity.com/members/mrspike.5160076/">+ <meta itemprop="url" content="https://xenforocomunity.com/members/mrspike.5160076/" />++ <div class="message-avatar">+ <div class="message-avatar-wrapper">+ <a href="/members/mrspike.5160076/" class="avatar avatar--m" data-user-id="5160076" data-xf-init="member-tooltip" id="js-XFUniqueId10">+ <img+ src="http://simp6.jpg5.su/simpo/data/avatars/m/5160/5160076.jpg?1746748029"+ srcset="http://simp6.jpg5.su/simpo/data/avatars/l/5160/5160076.jpg?1746748029 2x"+ alt=""+ class="avatar-u5160076-m"+ width="96"+ height="96"+ loading="lazy"+ itemprop="image"+ />+ </a>+ </div>+ </div>+ <div class="message-userDetails">+ <h4 class="message-name">+ <a href="/members/mrspike.5160076/" class="username" dir="auto" data-user-id="5160076" data-xf-init="member-tooltip" id="js-XFUniqueId11"><span itemprop="name"></span></a>+ </h4>+ <h5 class="userTitle message-userTitle" dir="auto" itemprop="jobTitle">Bathwater Drinker</h5>+ <div class="userBanner banner--simp message-userBanner" itemprop="jobTitle"><span class="userBanner-before"></span><strong>⠀Simp⠀</strong><span class="userBanner-after"></span></div>+ </div>++ <div class="message-userExtras">+ <dl class="pairs pairs--justified">+ <dt>+ <i class="fa--xf far fa-user">+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#user"></use></svg>+ </i>+ </dt>+ <dd>Aug 1, 2024</dd>+ </dl>++ <dl class="pairs pairs--justified">+ <dt>+ <i class="fa--xf far fa-comment">+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#comment"></use></svg>+ </i>+ </dt>+ <dd>38</dd>+ </dl>++ <dl class="pairs pairs--justified">+ <dt>+ <i class="fa--xf far fa-thumbs-up">+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#thumbs-up"></use></svg>+ </i>+ </dt>+ <dd>3,783</dd>+ </dl>+ </div>++ <span class="message-userArrow"></span>+ </section>+ </div>+<div class="message-cell message-cell--main"><div class="message-main js-quickEditTarget"><header class="message-attribution message-attribution--split"><ul class="message-attribution-main listInline"><li class="u-concealed">- <a href="/threads/fanfan.33077/post{id}" rel="nofollow" itemprop="url">+ <a href="/threads/fanfan.33077/post-{id}" rel="nofollow" itemprop="url"><timeclass="u-dt"dir="auto"@@ -739,7 +1040,7 @@ POST_TEMPLATE = """data-timestamp="1749508210"data-date="Jun 9, 2025"data-time="5:30 PM"- data-short="7d"+ data-short="12d"title="Jun 9, 2025 at 5:30 PM"itemprop="datePublished">@@ -748,13 +1049,54 @@ POST_TEMPLATE = """</a></li></ul>++ <ul class="message-attribution-opposite message-attribution-opposite--list">+ <li>+ <a href="/threads/fanfan.33077/post-{id}" class="message-attribution-gadget" data-xf-init="share-tooltip" data-href="/posts/{id}/share" aria-label="Share" rel="nofollow" id="js-XFUniqueId41">+ <i class="fa--xf far fa-share-alt">+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#share-alt"></use></svg>+ </i>+ </a>+ </li>++ <li class="u-hidden js-embedCopy">+ <a+ href="javascript:"+ data-xf-init="copy-to-clipboard"+ data-copy-text='<div class="js-xf-embed" data-url="https://xenforocomunity.com" data-content="post-{id}"></div><script defer src="https://xenforocomunity.com/js/xf/external_embed.js?_v=dc874496"></script>'+ data-success="Embed code HTML copied to clipboard."+ class=""+ >+ <i class="fa--xf far fa-code">+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#code"></use></svg>+ </i>+ </a>+ </li>++ <li>+ <a+ href="/posts/{id}/bookmark"+ class="bookmarkLink message-attribution-gadget bookmarkLink--highlightable"+ title="Add bookmark"+ data-xf-click="bookmark-click"+ data-label=".js-bookmarkText"+ data-sk-bookmarked="addClass:is-bookmarked, titleAttr:sync"+ data-sk-bookmarkremoved="removeClass:is-bookmarked, titleAttr:sync"+ >+ <span class="js-bookmarkText u-srOnly">Add bookmark</span>+ </a>+ </li>++ <li>+ <a href="/threads/fanfan.33077/post-{id}" rel="nofollow">+ #282+ </a>+ </li>+ </ul></header>- <div class="message-attachments">- {message_attachments}- </div><div class="message-content js-messageContent">- <div class="message-userContent lbContainer js-lbContainer" data-lb-id="post{id}" data-lb-caption-desc="cyberdrop-dl-patched · Jun 9, 2025 at 5:30 PM">+ <div class="message-userContent lbContainer js-lbContainer" data-lb-id="post-{id}" data-lb-caption-desc=" · Jun 9, 2025 at 5:30 PM"><article class="message-body js-selectToQuote">{message_body}<div class="js-selectToQuoteEnd"> </div>@@ -762,13 +1104,17 @@ POST_TEMPLATE = """</div><aside class="message-signature">- <div class="bbWrapper">Can't post to Bunkr. Mirrors are always appreciated.</div>+ <div class="bbWrapper">Cant post to Bunkr. Mirrors are always appreciated.</div></aside></div>+ <div class="message-attachments">+ {message_attachments}+ </div>+<footer class="message-footer"><div class="message-microdata" itemprop="interactionStatistic" itemtype="https://schema.org/InteractionCounter" itemscope="">- <meta itemprop="userInteractionCount" content="154" />+ <meta itemprop="userInteractionCount" content="160" /><meta itemprop="interactionType" content="https://schema.org/LikeAction" /></div>@@ -780,7 +1126,7 @@ POST_TEMPLATE = """data-reaction-id="1"data-xf-init="reaction"data-reaction-list="< .js-post | .js-reactionsList"- id="js-XFUniqueId13"+ id="js-XFUniqueId12"><i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="Like" title="Like" /><span class="reaction-text js-reactionText"><bdi>Like</bdi></span>@@ -790,7 +1136,14 @@ POST_TEMPLATE = """Quote</a>- <a href="/threads/fanfan.33077/reply?quote={id}" class="actionBar-action actionBar-action--reply" title="Reply, quoting this message" rel="nofollow" data-xf-click="quote" data-quote-href="/posts/{id}/quote">+ <a+ href="/threads/fanfan.33077/reply?quote={id}"+ class="actionBar-action actionBar-action--reply"+ title="Reply, quoting this message"+ rel="nofollow"+ data-xf-click="quote"+ data-quote-href="/posts/{id}/quote"+ >Reply</a></div>@@ -813,14 +1166,14 @@ POST_TEMPLATE = """</span></li><li>- <span class="reaction reaction--small reaction--91" data-reaction-id="91">- <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PepeClown" title="PepeClown" />+ <span class="reaction reaction--small reaction--50" data-reaction-id="50">+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PeepoWew" title="PeepoWew" /></span></li></ul><span class="u-srOnly">Reactions:</span>- <a class="reactionsBar-link" href="/posts/{id}/reactions" data-xf-click="overlay" data-cache="false" rel="nofollow"><bdi>xigxagxion</bdi>, <bdi>Feistee</bdi>, <bdi>kradpmis</bdi> and 140 others</a>+ <a class="reactionsBar-link" href="/posts/{id}/reactions" data-xf-click="overlay" data-cache="false" rel="nofollow"><bdi>DDDICKS</bdi>, <bdi>cachow</bdi>, <bdi>StubbornOne</bdi> and 145 others</a></div><div class="js-historyTarget message-historyTarget toggleTarget" data-href="trigger-href"></div>