Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: xenforo pass 2 (#1095)

NTFSvolume35167e73b019
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index fd4a8a2f7..95b632c28 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -47,6 +47,7 @@ from .imgur import ImgurCrawler
from .imx_to import ImxToCrawler
from .incestflix import IncestflixCrawler
from .influencer_bitches import InfluencerBitchesCrawler
+from .invision import INVISION_CRAWLERS
from .jpg5 import JPG5Crawler
from .kemono import KemonoCrawler
from .luscious import LusciousCrawler
@@ -90,10 +91,10 @@ from .tokyomotion import TokioMotionCrawler
from .toonily import ToonilyCrawler
from .twitter_images import TwimgCrawler
from .twpornstars import TwPornstarsCrawler
-from .vbulletin import ViperGirlsCrawler # TODO: import all vBulletin crawlers as a set
+from .vbulletin import VBULLETIN_CRAWLERS
from .vipr_dot_im import ViprImCrawler
from .wetransfer import WeTransferCrawler
-from .wordpress import WP_CRAWLERS, WP_GENERIC_CRAWLERS
+from .wordpress import WP_CRAWLERS, WordPressHTMLCrawler, WordPressMediaCrawler
from .xbunkr import XBunkrCrawler
from .xenforo import XF_CRAWLERS, SimpCityCrawler
from .xhamster import XhamsterCrawler
@@ -101,8 +102,8 @@ from .xxxbunker import XXXBunkerCrawler
from .yandex_disk import YandexDiskCrawler
from .youjizz import YouJizzCrawler
-FORUM_CRAWLERS = DISCOURSE_CRAWLERS.union(XF_CRAWLERS)
-GENERIC_CRAWLERS: set[type[Crawler]] = WP_GENERIC_CRAWLERS.union({DiscourseCrawler})
+FORUM_CRAWLERS = XF_CRAWLERS.union(INVISION_CRAWLERS, DISCOURSE_CRAWLERS, VBULLETIN_CRAWLERS)
+GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler}
ALL_CRAWLERS: set[type[Crawler]] = {
crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler
}
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 7fb70bd93..5db029c99 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -1,49 +1,314 @@
+"""Base crawlers to scrape any message board / forum
+
+If the message board has a public API, inherit from MessageBoard (ex: Discourse)
+
+If the message board needs to scrape the actual HTML of page, Inherit for HTMLMessageBoard
+
+"""
+# ruff : noqa: RUF009
+
from __future__ import annotations
+import dataclasses
+import datetime
+import re
from abc import abstractmethod
-from typing import TYPE_CHECKING, ClassVar, final
+from collections import defaultdict
+from typing import TYPE_CHECKING, ClassVar, Protocol, final
+
+from bs4 import BeautifulSoup, Tag
+from cyberdrop_dl.constants import HTTP_REGEX_LINKS
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
+from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError
+from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils.dates import TimeStamp, to_timestamp
+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, is_absolute_http_url, is_blob_or_svg
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator, Iterable, Sequence
+
+ from aiohttp_client_cache.response import AnyResponse
+
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+LINK_TRASH_MAPPING = {".th.": ".", ".md.": ".", "ifr": "watch"}
+
+Selector = css.CssAttributeSelector
+
+
[email protected](frozen=True, slots=True)
+class PostSelectors:
+ article: str # the entire html of the post (comments, attachments, user avatar, signature, etc...)
+ content: str # text, links and images (NO attachments)
+ id: Selector
+ attachments: Selector
+ article_trash: Sequence[str] = ("signature", "footer")
+ content_trash: Sequence[str] = ("blockquote", "fauxBlockLink")
+
+ # Most sites should only need to overwrite the attributes above
+ date: Selector = Selector("time", "datetime")
+ embeds: Selector = Selector("iframe", "src")
+ images: Selector = Selector("img.bbImage", "src")
+ a_tag_w_image: Selector = Selector("a:has(img.bbImage)[href]", "href")
+ lazy_load_embeds: Selector = Selector('[class*=iframe][onclick*="loadMedia(this, \'//"]', "onclick")
+ links: Selector = Selector("a:not(:has(img))", "href")
+ videos: Selector = Selector("video source", "src")
+
-class ForumCrawler(Crawler, is_abc=True):
[email protected](frozen=True, slots=True)
+class MessageBoardSelectors:
+ posts: PostSelectors
+ confirmation_button: Selector
+ next_page: Selector
+ last_page: Selector
+ current_page: Selector
+ title: Selector
+ title_trash: Sequence[str] = ("span",)
+
+
[email protected](frozen=True, slots=True, order=True)
+class ForumPost:
+ id: int
+ date: datetime.datetime | None
+ article: Tag = dataclasses.field(compare=False)
+ content: Tag = dataclasses.field(compare=False)
+
+ @staticmethod
+ def new(article: Tag, selectors: PostSelectors) -> ForumPost:
+ for trash in selectors.article_trash:
+ css.decompose(article, trash)
+ content = css.select_one(article, selectors.content)
+ for trash in selectors.content_trash:
+ css.decompose(article, trash)
+ try:
+ date = datetime.datetime.fromisoformat(css.select_one_get_attr(article, *selectors.date))
+ except Exception:
+ date = None
+
+ id_str = css.get_attr(article, selectors.id.attribute)
+ post_id = int(id_str.rsplit("-", 1)[-1])
+ return ForumPost(post_id, date, article, content)
+
+ @property
+ def timestamp(self) -> TimeStamp | None:
+ if self.date:
+ return to_timestamp(self.date)
+
+
+class ForumPostProtocol(Protocol):
+ # Concrete classes may define their own custom `ForumPost` class (ex: a Pydantic Model from an API response)
+ # Those classes need to satisfy this Protocol to make sure they work with all of `MessageBoard` methods
+ # This is just identify type errors.
+ # Subclass implementation does not need to conform to this if they override the necessary methods
+ @property
+ def id(self) -> int: ...
+ @property
+ def date(self) -> datetime.datetime | None: ...
+ @property
+ def article(self) -> Tag: ...
+ @property
+ def content(self) -> Tag: ...
+ @property
+ def timestamp(self) -> TimeStamp | None: ...
+
+
[email protected](frozen=True, slots=True, order=True)
+class Thread:
+ id: int
+ name: str
+ page: int
+ post_id: int | None
+ url: AbsoluteHttpURL
+
+
+class ThreadProtocol(Protocol):
+ # Concrete classes may define their own custom `Thread` class (ex: discourse defines `Topic` from an API Response)
+ # Those classes need to satisfy this Protocol to make sure they work with all of `MessageBoard` methods
+ # This is just identify type errors.
+ # Subclass implementation does not need to conform to this if they override the necessary methods
+ @property
+ def id(self) -> int: ...
+ @property
+ def name(self) -> str: ...
+ @property
+ def page(self) -> int: ...
+ @property
+ def post_id(self) -> int | None: ...
+ @property
+ def url(self) -> AbsoluteHttpURL: ...
+
+
+# quotes_2 : embed fauxBlockLink
+class MessageBoardCrawler(Crawler, is_abc=True):
+ """Base crawler for every MessageBoard.
+
+ A Message board should have: forums, threads (also known as topics) and posts.
+
+ Concrete classes MUST:
+ - implement `parse_thread`
+ - implement `make_post_url`
+ - implement `thread`
+ - implement `post`
+
+ Concrete classes SHOULD define `ATTACHMENT_HOSTS` if internal images of the site are stored on servers with a different domain
+
+ NOTE: Always use this crawler as base, even if the message board logic does not match perfectly.
+
+ In those cases, override `fetch`,`fetch_thread`, `parse_url` or any other non final method as needed
+
+ This crawler is NOT meant to scrape image boards (like 4chan)
+ """
+
+ THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"
+ ATTACHMENT_URL_PARTS: ClassVar[Sequence[str]] = "attachments", "data", "uploads"
+ ATTACHMENT_HOSTS: ClassVar[Sequence[str]] = ()
SUPPORTS_THREAD_RECURSION: ClassVar[bool] = False
+ LOGIN_USER_COOKIE_NAME: ClassVar[str] = ""
+
+ # True: Login is mandatory. If login fails, the crawler will be disabled
+ # False: Login is optional, but CDL will try to log in anyway. ex: Forums where only some threads require auth
+ # None: Completely skip login check and request. Always try to scrape as is the user is logged in
+ # TODO: move login logic to the base crawler
+ login_required: ClassVar[bool | None] = None
+
+ @classmethod
+ @abstractmethod
+ def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> ThreadProtocol: ...
+ @abstractmethod
+ async def post(self, scrape_item: ScrapeItem, post: ForumPostProtocol) -> None: ...
+
+ @abstractmethod
+ async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None: ...
+
+ async def forum(self, scrape_item: ScrapeItem) -> None:
+ # Subclasses can define custom logic for this method.
+ # They would need to also override `fetch` since the default fetch does not take this method into account
+ raise NotImplementedError
+
+ async def resolve_confirmation_link(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ # Not every forum has confirmation link so overriding this method is optional
+ # Implementation of this method MUST return `None` instead of raising an error
+ raise NotImplementedError
+
+ async def async_startup(self) -> None:
+ await self.login()
+
+ @final
+ async def login(self) -> None:
+ if self.login_required is None:
+ return
+
+ if not self.logged_in:
+ login_url = self.PRIMARY_URL / "login"
+ await self._login(login_url)
+
+ @final
@property
def scrape_single_forum_post(self) -> bool:
return self.manager.config_manager.settings_data.download_options.scrape_single_forum_post
+ @final
@property
def max_thread_depth(self) -> int:
return self.manager.config_manager.settings_data.download_options.maximum_thread_depth
- @classmethod
- @abstractmethod
- def is_attachment(cls, url: AbsoluteHttpURL) -> bool: ...
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ if not self.logged_in and self.login_required is True:
+ return
+ scrape_item.url = self.parse_url(str(scrape_item.url))
+ if self.is_attachment(scrape_item.url):
+ return await self.handle_internal_link(scrape_item)
+ if is_confirmation_link(scrape_item.url):
+ return await self.follow_confirmation_link(scrape_item)
- @abstractmethod
- async def forum(self, scrape_item: ScrapeItem) -> None: ...
+ await self.fetch_thread(scrape_item)
- @abstractmethod
- async def thread(self, scrape_item: ScrapeItem) -> None: ...
+ async def fetch_thread(self, scrape_item: ScrapeItem) -> None:
+ thread_part_index = len(self.PRIMARY_URL.parts)
+ match scrape_item.url.parts[thread_part_index:]:
+ case [thread_part, thread_name_and_id, *_] if thread_part in self.THREAD_PART_NAMES:
+ self.check_thread_recursion(scrape_item)
+ thread = self.parse_thread(scrape_item.url, thread_name_and_id)
+ return await self.thread(scrape_item, thread)
+ case ["goto" | "posts", _, *_]:
+ self.check_thread_recursion(scrape_item)
+ return await self.follow_redirect_w_get(scrape_item)
+ case _:
+ raise ValueError
- @abstractmethod
- async def post(self, scrape_item: ScrapeItem) -> None: ...
+ def is_attachment(self, link: AbsoluteHttpURL | str) -> bool:
+ if not link:
+ return False
+ if isinstance(link, str):
+ link = self.parse_url(link)
+ by_parts = len(link.parts) > 2 and any(p in link.parts for p in self.ATTACHMENT_URL_PARTS)
+ by_host = any(host in link.host for host in self.ATTACHMENT_HOSTS)
+ return by_parts or by_host
+
+ # TODO: Move this to base crawler
+ @final
+ @error_handling_wrapper
+ async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:
+ async with self.request_limiter:
+ response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)
+ assert is_absolute_http_url(response.url)
+ scrape_item.url = response.url
+ self.manager.task_group.create_task(self.run(scrape_item))
+
+ # TODO: Move this to base crawler
+ @final
+ @error_handling_wrapper
+ async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:
+ head = await self.client.get_head(self.DOMAIN, scrape_item.url)
+ redirect = head.get("location")
+ if not redirect:
+ raise ScrapeError(422)
+ scrape_item.url = self.parse_url(redirect)
+ self.manager.task_group.create_task(self.run(scrape_item))
+
+ @final
+ async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:
+ url = await self.resolve_confirmation_link(scrape_item.url)
+ if url: # If there was an error, this will be None
+ scrape_item.url = url
+ # This could end up back in here if the URL goes to another thread
+ return self.handle_external_links(scrape_item)
+
+ @final
+ def check_thread_recursion(self, scrape_item: ScrapeItem) -> None:
+ if self.stop_thread_recursion(scrape_item):
+ parents = f"{len(scrape_item.parent_threads)} parent thread(s)"
+ msg = (
+ f"Skipping nested thread URL with {parents}:"
+ f"URL: {scrape_item.url}\n"
+ f"Parent: {scrape_item.parent}\n"
+ f"Origin: {scrape_item.origin}\n"
+ )
+ raise MaxChildrenError(msg)
+
+ @final
+ def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:
+ if n_parents := len(scrape_item.parent_threads):
+ if n_parents > self.max_thread_depth:
+ return True
+
+ return self.SUPPORTS_THREAD_RECURSION and bool(self.max_thread_depth)
+
+ return False
+ @final
@error_handling_wrapper
async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:
if link == self.PRIMARY_URL:
return
if self.is_attachment(link):
- return await self.handle_internal_link(scrape_item.create_new(link))
- if self.PRIMARY_URL.host in link.host and self.stop_thread_recursion(scrape_item):
- msg = f"Skipping nested thread URL {scrape_item.url} found on {scrape_item.origin}"
- return self.log(msg)
+ return await self.handle_internal_link(scrape_item, link)
+ if self.PRIMARY_URL.host == link.host:
+ self.manager.task_group.create_task(self.run(scrape_item.create_child(link)))
+ return
new_scrape_item = scrape_item.create_child(link)
new_scrape_item.type = None
new_scrape_item.reset_childen()
@@ -51,24 +316,12 @@ class ForumCrawler(Crawler, is_abc=True):
scrape_item.add_children()
@error_handling_wrapper
- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:
- # This logic is for discourse
- # if len(scrape_item.url.parts) < 5 and not scrape_item.url.suffix:
- # return
- filename, ext = self.get_filename_and_ext(scrape_item.url.name)
+ async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:
+ link = link or scrape_item.url
+ filename, ext = self.get_filename_and_ext(link.name)
scrape_item.add_to_parent_title("Attachments")
scrape_item.part_of_album = True
- await self.handle_file(scrape_item.url, scrape_item, filename, ext)
-
- @final
- def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:
- if (
- not self.SUPPORTS_THREAD_RECURSION
- or not self.max_thread_depth
- or (len(scrape_item.parent_threads) > self.max_thread_depth)
- ):
- return True
- return False
+ await self.handle_file(link, scrape_item, filename, ext)
@final
async def write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:
@@ -76,5 +329,388 @@ class ForumCrawler(Crawler, is_abc=True):
return
await self.manager.log_manager.write_last_post_log(last_post_url)
+ # TODO: Move this to the base crawler
+ # TODO: Define an unified workflow for crawlers to perform and check login
+ @final
+ @error_handling_wrapper
+ async def _login(self, login_url: AbsoluteHttpURL) -> None:
+ host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)
+ session_cookie = host_cookies.get(self.LOGIN_USER_COOKIE_NAME)
+ session_cookie = session_cookie.value if session_cookie else None
+ msg = f"No cookies found for {self.FOLDER_DOMAIN}"
+ if not session_cookie and self.login_required:
+ raise LoginError(message=msg)
+
+ _, self.logged_in = await self.check_login_with_request(login_url)
+ if self.logged_in:
+ return
+ if session_cookie:
+ msg = f"Cookies for {self.FOLDER_DOMAIN} are not valid."
+ if self.login_required:
+ raise LoginError(message=msg)
+
+ msg += " Scraping without an account"
+ self.log(msg, 30)
+
+ async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:
+ text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)
+ return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))
+
+
+class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
+ """Base crawler that knows how to scrape the html of every MessageBoard.
+
+ Threads of the MessageBoard MUST be paginated.
+
+ Concrete classes MUST:
+ - define: `SELECTORS`, `POST_URL_PART_NAME` and `PAGE_URL_PART_NAME`
+
+ This crawler delegates images to other crawlers by default
+ Concrete classes MAY handle images themselfs if they know how to. This will improve performance by reducing the number of requests
+
+ To handle images, concrete classes need to:
+ - override `IGNORE_EMBEDED_IMAGES_SRC` to `False`
+ - override `is_thumbnail`
+ - override `thumbnail_to_img`
+
+ Concrete classes SHOULD define `ATTACHMENT_HOSTS` if internal images of the site are stored on servers with a different domain
+ """
+
+ IGNORE_EMBEDED_IMAGES_SRC = True
+ SELECTORS: ClassVar[MessageBoardSelectors]
+ POST_URL_PART_NAME: ClassVar[str]
+ PAGE_URL_PART_NAME: ClassVar[str]
+
+ def __init_subclass__(cls, is_abc: bool = False, **kwargs) -> None:
+ super().__init_subclass__(is_abc=is_abc, **kwargs)
+ if is_abc:
+ return
+ REQUIRED_FIELDS = "SELECTORS", "POST_URL_PART_NAME", "PAGE_URL_PART_NAME"
+ for field_name in REQUIRED_FIELDS:
+ assert getattr(cls, field_name, None), f"Subclass {cls.__name__} must override: {field_name}"
+
+ def __post_init__(self) -> None:
+ self.scraped_threads = set()
+
+ @final
+ async def async_startup(self) -> None:
+ await super().async_startup()
+ self.register_cache_filter(self.PRIMARY_URL, self.check_is_not_last_page)
+
+ async def check_is_not_last_page(self, response: AnyResponse) -> bool:
+ return await check_is_not_last_page(response, self.SELECTORS)
+
+ @classmethod
+ def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool:
+ return False
+
+ @classmethod
+ def thumbnail_to_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ return None
+
+ @classmethod
+ def parse_thread(cls, url: AbsoluteHttpURL, thread_name_and_id: str) -> ThreadProtocol:
+ return parse_thread(url, thread_name_and_id, cls.PAGE_URL_PART_NAME, cls.POST_URL_PART_NAME)
+
+ @classmethod
+ def make_post_url(cls, thread: ThreadProtocol, post_id: int) -> AbsoluteHttpURL:
+ return thread.url / f"{cls.POST_URL_PART_NAME}-{post_id}"
+
+ @error_handling_wrapper
+ async def thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:
+ scrape_item.setup_as_forum("")
+ if thread.url in self.scraped_threads:
+ return
+
+ scrape_item.parent_threads.add(thread.url)
+ if self.scrape_single_forum_post and not thread.post_id:
+ msg = "`--scrape-single-forum-post` is `True`, but the provided URL has no post id"
+ raise ScrapeError("User Error", msg)
+
+ self.scraped_threads.add(thread.url)
+ await self.process_thread(scrape_item, thread)
+
+ async def process_thread(self, scrape_item: ScrapeItem, thread: ThreadProtocol) -> None:
+ title: str = ""
+ last_post_url = thread.url
+ async for soup in self.thread_pager(scrape_item):
+ if not title:
+ try:
+ title = self.create_title(get_post_title(soup, self.SELECTORS), thread_id=thread.id)
+ except ScrapeError as e:
+ self.log_debug("Got an unprocessable soup", 40, exc_info=e)
+ raise
+ scrape_item.add_to_parent_title(title)
+
+ continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)
+ if not continue_scraping:
+ break
+
+ await self.write_last_forum_post(thread.url, last_post_url)
+
+ def process_thread_page(
+ self, scrape_item: ScrapeItem, thread: ThreadProtocol, soup: BeautifulSoup
+ ) -> tuple[bool, AbsoluteHttpURL]:
+ continue_scraping = False
+ post_url = thread.url
+ for article in soup.select(self.SELECTORS.posts.article):
+ current_post = ForumPost.new(article, self.SELECTORS.posts)
+ continue_scraping, scrape_this_post = check_post_id(
+ thread.post_id, current_post.id, self.scrape_single_forum_post
+ )
+ if scrape_this_post:
+ post_url = self.make_post_url(thread, current_post.id)
+ new_scrape_item = scrape_item.create_new(
+ thread.url,
+ possible_datetime=current_post.timestamp,
+ add_parent=post_url,
+ )
+ self.manager.task_group.create_task(self.post(new_scrape_item, current_post))
+ try:
+ scrape_item.add_children()
+ except MaxChildrenError:
+ break
+
+ if not continue_scraping:
+ break
+ return continue_scraping, post_url
+
+ @error_handling_wrapper
+ async def post(self, scrape_item: ScrapeItem, post: ForumPostProtocol) -> None:
+ scrape_item.setup_as_post("")
+ post_title = self.create_separate_post_title(None, str(post.id), post.date)
+ scrape_item.add_to_parent_title(post_title)
+
+ seen: set[str] = set()
+ duplicates: set[str] = set()
+ stats: dict[str, int] = defaultdict(int)
+ for scraper in (self._attachments, self._images, self._videos, self._external_links):
+ for link in scraper(post):
+ duplicates.add(link) if link in seen else seen.add(link)
+ stats[scraper.__name__.removeprefix("_")] += 1
+ await self.process_child(scrape_item, link)
+
+ for scraper in (self._embeds, self._lazy_load_embeds):
+ for link in scraper(post):
+ duplicates.add(link) if link in seen else seen.add(link)
+ stats[scraper.__name__.removeprefix("_")] += 1
+ await self.process_child(scrape_item, link, embeds=True)
+
+ if seen:
+ self.log(f"[{self.FOLDER_DOMAIN}] post #{post.id} stats = {dict(stats)}")
+ if duplicates:
+ self.log_bug_report(f"Found duplicate links. Selectors are too generic: {duplicates}")
+
+ def _external_links(self, post: ForumPostProtocol) -> Iterable[str]:
+ selector = self.SELECTORS.posts.links
+ links = css.iselect(post.content, selector.element)
+ valid_links = (link for link in links if not self.is_username_or_attachment(link))
+ return iter_links(valid_links, selector.attribute)
+
+ def _images(self, post: ForumPostProtocol) -> Iterable[str]:
+ if self.IGNORE_EMBEDED_IMAGES_SRC:
+ selector = self.SELECTORS.posts.a_tag_w_image
+ else:
+ selector = self.SELECTORS.posts.images
+ images = css.iselect(post.content, selector.element)
+ return iter_links(images, selector.attribute)
+
+ def _videos(self, post: ForumPostProtocol) -> Iterable[str]:
+ selector = self.SELECTORS.posts.videos
+ videos = css.iselect(post.content, selector.element)
+ return iter_links(videos, selector.attribute)
+
+ def _attachments(self, post: ForumPostProtocol) -> Iterable[str]:
+ selector = self.SELECTORS.posts.attachments
+ attachments = css.iselect(post.article, selector.element)
+ return iter_links(attachments, selector.attribute)
+
+ def _embeds(self, post: ForumPostProtocol) -> Iterable[str]:
+ selector = self.SELECTORS.posts.embeds
+ embeds = css.iselect(post.content, selector.element)
+ return iter_links(embeds, selector.attribute)
+
+ def _lazy_load_embeds(self, post: ForumPostProtocol) -> Iterable[str]:
+ selector = self.SELECTORS.posts.lazy_load_embeds
+ for lazy_media in css.iselect(post.content, selector.element):
+ yield get_text_between(css.get_attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
+
+ async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
+ async for soup in self._web_pager(scrape_item.url, self.get_next_page):
+ yield soup
+
+ def get_next_page(self, soup: BeautifulSoup) -> str | None:
+ return css.select_one_get_attr_or_none(soup, *self.SELECTORS.next_page)
+
+ @final
+ @error_handling_wrapper
+ async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:
+ link_str_ = pre_process_child(link_str, embeds)
+ if not link_str_:
+ return
+ link = await self.get_absolute_link(link_str_)
+ if not link:
+ return
+ if self.is_thumbnail(link):
+ link = self.thumbnail_to_img(link)
+ if not link:
+ return self.log(f"Skipping thumbnail: {link}")
+ await self.handle_link(scrape_item, link)
+
+ async def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ if isinstance(link, str):
+ absolute_link = self.parse_url(clean_link_str(link))
+ else:
+ absolute_link = link
+ if is_confirmation_link(absolute_link):
+ return await self.resolve_confirmation_link(absolute_link)
+ return absolute_link
+
+ @error_handling_wrapper
+ async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ async with self.request_limiter:
+ soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)
+ selector = self.SELECTORS.confirmation_button
+ confirm_button = soup.select_one(selector.element)
+ if not confirm_button:
+ return
+ link_str: str = css.get_attr(confirm_button, selector.attribute)
+ link_str = link_str.split('" class="link link--internal', 1)[0]
+ new_link = self.parse_url(link_str)
+ return await self.get_absolute_link(new_link)
+
+ async def handle_internal_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL | None = None) -> None:
+ link = link or scrape_item.url
+ slug = link.name or link.parent.name
+ if slug.isdigit():
+ return await self.follow_redirect_w_head(scrape_item.create_new(link))
+
+ await super().handle_internal_link(scrape_item, link)
+
+ def is_username_or_attachment(self, link_obj: Tag) -> bool:
+ if link_obj.select_one(".username"):
+ return True
+ try:
+ if link_str := css.get_attr(link_obj, self.SELECTORS.posts.links.element):
+ return self.is_attachment(link_str)
+ except Exception:
+ pass
+ return False
+
+
+def iter_links(links: Iterable[Tag], attribute: str) -> Iterable[str]:
+ for link_tag in links:
+ try:
+ yield css.get_attr(link_tag, attribute)
+ except Exception:
+ continue
+
+
+def parse_thread(
+ url: AbsoluteHttpURL, thread_name_and_id: str, page_part_name: str, post_part_name: str
+) -> ThreadProtocol:
+ name_index = url.parts.index(thread_name_and_id)
+ name, id_ = parse_thread_name_and_id(thread_name_and_id)
+ page, post_id = get_thread_page_and_post(url, name_index, page_part_name, post_part_name)
+ canonical_url = get_thread_canonical_url(url, name_index)
+ return Thread(id_, name, page, post_id, canonical_url)
+
+
+def parse_thread_name_and_id(thread_name_and_id: str) -> tuple[str, int]:
+ try:
+ name, id_str = thread_name_and_id.rsplit(".", 1)
+ except ValueError:
+ id_str, name = thread_name_and_id.split("-", 1)
+ return name, int(id_str)
+
+
+def get_thread_canonical_url(url: AbsoluteHttpURL, thread_name_index: int) -> AbsoluteHttpURL:
+ new_parts = url.parts[1 : thread_name_index + 1]
+ new_path = "/".join(new_parts)
+ return url.with_path(new_path)
+
+
+def get_thread_page_and_post(
+ url: AbsoluteHttpURL, thread_name_index: int, page_name: str, post_name: str
+) -> tuple[int, int | None]:
+ extra_parts = url.parts[thread_name_index + 1 :]
+ if url.fragment:
+ extra_parts = *extra_parts, url.fragment
+
+ def find_number(search_value: str) -> int | None:
+ for sec in extra_parts:
+ if search_value in sec:
+ return int(sec.rsplit(search_value, 1)[-1].replace("-", "").strip())
+
+ post_id = find_number(post_name)
+ page_number = find_number(page_name) or 1
+ return page_number, post_id
+
+
+async def check_is_not_last_page(response: AnyResponse, selectors: MessageBoardSelectors) -> bool:
+ soup = BeautifulSoup(await response.text(), "html.parser")
+ return not is_last_page(soup, selectors)
+
+
+def is_last_page(soup: BeautifulSoup, selectors: MessageBoardSelectors) -> bool:
+ try:
+ last_page = css.select_one_get_attr(soup, *selectors.last_page)
+ current_page = css.select_one_get_attr(soup, *selectors.current_page)
+ except (AttributeError, IndexError, css.SelectorError):
+ return True
+ return current_page == last_page
+
+
+def get_post_title(soup: BeautifulSoup, selectors: MessageBoardSelectors) -> str:
+ try:
+ title_block = css.select_one(soup, selectors.title.element)
+ for trash in selectors.title_trash:
+ css.decompose(title_block, trash)
+ except (AttributeError, AssertionError, css.SelectorError) as e:
+ raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e
+
+ if title := " ".join(css.get_text(title_block).split()):
+ return title
+ raise ScrapeError(422)
+
+
+def extract_embed_url(embed_str: str) -> str:
+ embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")
+ if match := re.search(HTTP_REGEX_LINKS, embed_str):
+ return match.group(0).replace("www.", "")
+ return embed_str
+
+
+def clean_link_str(link: str) -> str:
+ for old, new in LINK_TRASH_MAPPING.items():
+ link = link.replace(old, new)
+ return link
+
+
+def is_confirmation_link(link: AbsoluteHttpURL) -> bool:
+ return "masked" in link.parts or "link-confirmation" in link.path
+
+
+def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]:
+ """Checks if the program should scrape the current post.
+
+ Returns (continue_scraping, scrape_this_post)"""
+ if init_post_id:
+ if init_post_id > current_post_id:
+ return (True, False)
+ elif init_post_id == current_post_id:
+ return (not scrape_single_forum_post, True)
+ else:
+ return (not scrape_single_forum_post, not scrape_single_forum_post)
+
+ assert not scrape_single_forum_post # We should have raised an exception earlier
+ return True, True
+
+
+def pre_process_child(link_str: str, embeds: bool = False) -> str | None:
+ assert isinstance(link_str, str)
+ if embeds:
+ link_str = extract_embed_url(link_str)
-MessageBoardCrawler = ForumCrawler
+ if link_str and not is_blob_or_svg(link_str):
+ return link_str
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 8bfb124d3..760cbd943 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -8,7 +8,7 @@ from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from functools import partial, wraps
from pathlib import Path
-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, TypeAlias, TypeVar, final
+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NoReturn, ParamSpec, TypeAlias, TypeVar, final
import yarl
from aiolimiter import AsyncLimiter
@@ -18,6 +18,7 @@ from cyberdrop_dl import constants
from cyberdrop_dl.constants import NEW_ISSUE_URL
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
+from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError
from cyberdrop_dl.scraper import filters
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.database.tables.history_table import get_db_path
@@ -99,6 +100,13 @@ def create_task_id(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, C
log(f"Scrape Failed: {UNKNOWN_URL_PATH_MSG}: {scrape_item.url}", 40)
self.manager.progress_manager.scrape_stats_progress.add_failure(UNKNOWN_URL_PATH_MSG)
await self.manager.log_manager.write_scrape_error_log(scrape_item.url, UNKNOWN_URL_PATH_MSG)
+ except MaxChildrenError:
+
+ @error_handling_wrapper
+ async def raise_e(self, scrape_item: ScrapeItem) -> NoReturn:
+ raise
+
+ await raise_e(self, scrape_item)
finally:
self.scraping_progress.remove_task(task_id)
@@ -207,8 +215,10 @@ class Crawler(ABC):
await self.manager.states.RUNNING.wait()
self.waiting_items += 1
scrape_prefix = "Scraping"
- if self.DOMAIN == "generic":
+ if self.IS_FALLBACK_GENERIC:
scrape_prefix += " (unsupported domain)"
+ else:
+ scrape_prefix += f" [{self.FOLDER_DOMAIN}]"
async with self._semaphore:
self.waiting_items -= 1
@@ -246,6 +256,10 @@ class Crawler(ABC):
media_item = MediaItem(url, scrape_item, download_folder, filename, original_filename, debrid_link, ext=ext)
await self.handle_media_item(media_item, m3u8_media)
+ def log_bug_report(self, msg: str, level: int = 30) -> None:
+ msg += f". Please file a bug report at {NEW_ISSUE_URL}"
+ log(msg, level)
+
async def handle_media_item(self, media_item: MediaItem, m3u8_media: M3U8Media | None = None) -> None:
await self.manager.states.RUNNING.wait()
if media_item.datetime and not isinstance(media_item.datetime, int):
@@ -323,20 +337,19 @@ class Crawler(ABC):
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- def get_filename_and_ext(self, filename: str, *args, assume_ext: str | None = None, **kwargs):
+ def get_filename_and_ext(
+ self, filename: str, forum: bool = False, assume_ext: str | None = None
+ ) -> tuple[str, str]:
"""Wrapper around `utils.get_filename_and_ext`.
-
- If `ignore_options.exclude_files_with_no_extension` is `True`, `assume_ext` is not None and the file has no extension,
- The value of `assume_ext` will be used as `ext`
-
- In any other case, it will just call `utils.get_filename_and_ext`
+ Calls it as is.
+ If that fails, appedns `assume_ext` and tries again, but only if the user had exclude_files_with_no_extension = `False`
"""
- filename_as_path = Path(filename)
- if assume_ext and self.allow_no_extension and not filename_as_path.suffix:
- filename_as_path = filename_as_path.with_suffix(assume_ext)
- new_filename, ext = get_filename_and_ext(str(filename_as_path), *args, *kwargs)
- return Path(new_filename).stem, ext
- return get_filename_and_ext(filename, *args, *kwargs)
+ try:
+ return get_filename_and_ext(filename, forum)
+ except NoExtensionError:
+ if assume_ext and self.allow_no_extension:
+ return get_filename_and_ext(filename + assume_ext, forum)
+ raise
def check_album_results(self, url: URL, album_results: dict[str, Any]) -> bool:
"""Checks whether an album has completed given its domain and album id."""
diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.py
index c2ae5fd55..7a0681abc 100644
--- a/cyberdrop_dl/crawlers/discourse/_discourse.py
+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py
@@ -89,6 +89,9 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
async def forum(self, scrape_item: ScrapeItem) -> None:
raise NotImplementedError
+ def parse_thread(self, *args) -> None:
+ raise NotImplementedError
+
async def thread(self, scrape_item: ScrapeItem, topic: Topic) -> None:
await self.topic(scrape_item, topic)
@@ -124,7 +127,7 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
return
async def post(self, scrape_item: ScrapeItem, post: AvailablePost) -> None:
- title = self.create_separate_post_title(post.title, str(post.id), None)
+ title = self.create_separate_post_title(post.title, str(post.id), post.created_at)
scrape_item.setup_as_post(title)
for link in self.extract_links(post):
await self.handle_link(scrape_item, link)
diff --git a/cyberdrop_dl/crawlers/invision/__init__.py b/cyberdrop_dl/crawlers/invision/__init__.py
new file mode 100644
index 000000000..3bbf0ee06
--- /dev/null
+++ b/cyberdrop_dl/crawlers/invision/__init__.py
@@ -0,0 +1,8 @@
+from ._invision import InvisionCrawler
+from .bellazon import BellazonCrawler
+
+INVISION_CRAWLERS: set[type[InvisionCrawler]] = {BellazonCrawler}
+
+
+INVISION_CRAWLERS_MAP = {c.__name__: c for c in INVISION_CRAWLERS}
+__all__ = list(INVISION_CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]
diff --git a/cyberdrop_dl/crawlers/invision/_invision.py b/cyberdrop_dl/crawlers/invision/_invision.py
new file mode 100644
index 000000000..2c6433c4d
--- /dev/null
+++ b/cyberdrop_dl/crawlers/invision/_invision.py
@@ -0,0 +1,52 @@
+"""Generic crawler for any Invision forum
+
+A Invision site has a tag with a `ipsCopyright` class and the text: "Powered By Invision"
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers._forum import HTMLMessageBoardCrawler
+from cyberdrop_dl.crawlers.xenforo.xenforo import DEFAULT_XF_POST_SELECTORS, DEFAULT_XF_SELECTORS, XenforoCrawler
+from cyberdrop_dl.utils import css
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers._forum import MessageBoardSelectors
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+
+
+Selector = css.CssAttributeSelector
+
+
+DEFAULT_INVISION_POST_SELECTORS = dataclasses.replace(
+ DEFAULT_XF_POST_SELECTORS,
+ article="div[class*=ipsComment_content]",
+ content="div.cPost_contentWrap",
+ article_trash=(".message-signature", ".message-footer"),
+ content_trash=("blockquote", "fauxBlockLink"),
+ id=Selector("data-commentid", "data-commentid"),
+ attachments=Selector("a[class*=ipsAttachLink]", "href"),
+ images=Selector("a[class*=ipsAttachLink_image]", "href"),
+ videos=Selector("video.ipsEmbeddedVideo source", "src"),
+)
+
+
+DEFAULT_INVISION_SELECTORS = dataclasses.replace(
+ DEFAULT_XF_SELECTORS,
+ posts=DEFAULT_INVISION_POST_SELECTORS,
+ title=Selector("span.ipsType_break.ipsContained span"),
+ next_page=Selector("li.ipsPagination_next a[href]", "href"),
+)
+
+
+class InvisionCrawler(HTMLMessageBoardCrawler, is_abc=True):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = XenforoCrawler.SUPPORTED_PATHS | {"**NOTE**": "base crawler: Invision"} # type: ignore
+ SUPPORTS_THREAD_RECURSION: ClassVar[bool] = True
+ SELECTORS: ClassVar[MessageBoardSelectors] = DEFAULT_INVISION_SELECTORS
+ POST_URL_PART_NAME: ClassVar[str] = "comment"
+ PAGE_URL_PART_NAME: ClassVar[str] = "page"
+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = False
+ LOGIN_USER_COOKIE_NAME: ClassVar[str] = "UNKNOWN" # TODO: Find out cookie name
+ ATTACHMENT_HOSTS = ()
diff --git a/cyberdrop_dl/crawlers/invision/bellazon.py b/cyberdrop_dl/crawlers/invision/bellazon.py
new file mode 100644
index 000000000..127198373
--- /dev/null
+++ b/cyberdrop_dl/crawlers/invision/bellazon.py
@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import ClassVar
+
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+
+from ._invision import InvisionCrawler
+
+
+class BellazonCrawler(InvisionCrawler):
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.bellazon.com/main/")
+ DOMAIN: ClassVar[str] = "bellazon"
+ FOLDER_DOMAIN: ClassVar[str] = "Bellazon"
+ login_required = False
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index aba54c3e8..d364f9a1a 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -46,7 +46,7 @@ class JPG5Crawler(CheveretoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg5.su")
def __post_init__(self) -> None:
- self.request_limiter = AsyncLimiter(1, 5)
+ self.request_limiter = AsyncLimiter(1, 1)
async def fetch(self, scrape_item: ScrapeItem) -> None:
scrape_item.url = fix_host(scrape_item.url)
diff --git a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py
index 5ff7e8bcf..26bc06d9e 100644
--- a/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py
+++ b/cyberdrop_dl/crawlers/vbulletin/_vbulletin.py
@@ -5,12 +5,14 @@ import itertools
from typing import TYPE_CHECKING, ClassVar
from xml.etree import ElementTree
-from cyberdrop_dl.crawlers.xenforo.xenforo import Thread, XenforoCrawler
+from cyberdrop_dl.crawlers.xenforo.xenforo import XenforoCrawler
from cyberdrop_dl.exceptions import MaxChildrenError, ScrapeError
if TYPE_CHECKING:
+ import datetime
from collections.abc import Iterable
+ from cyberdrop_dl.crawlers._forum import Thread
from cyberdrop_dl.crawlers.crawler import SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
@@ -22,6 +24,7 @@ class Post:
id: int
title: str
xml: ElementTree.Element[str]
+ date: datetime.datetime | None = None
@staticmethod
def new(element: ElementTree.Element[str]) -> Post:
@@ -65,7 +68,7 @@ class vBulletinCrawler(XenforoCrawler, is_abc=True): # noqa: N801
async def async_startup(self) -> None:
if not self.logged_in:
login_url = self.PRIMARY_URL / "login.php"
- await self.login_setup(login_url)
+ await self._login(login_url)
self.register_cache_filter(self.PRIMARY_URL, lambda _: True)
diff --git a/cyberdrop_dl/crawlers/wordpress/__init__.py b/cyberdrop_dl/crawlers/wordpress/__init__.py
index 1e1697c25..5b1bc0159 100644
--- a/cyberdrop_dl/crawlers/wordpress/__init__.py
+++ b/cyberdrop_dl/crawlers/wordpress/__init__.py
@@ -12,4 +12,4 @@ if TYPE_CHECKING:
WP_CRAWLERS: set[type[WordPressBaseCrawler]] = {BestPrettyGirlCrawler}
WP_CRAWLERS_MAP = {c.__name__: c for c in WP_CRAWLERS}
-WP_GENERIC_CRAWLERS = {WordPressMediaCrawler, WordPressHTMLCrawler}
+__all__ = [*WP_CRAWLERS_MAP.keys(), "WordPressMediaCrawler", "WordPressHTMLCrawler"] # type: ignore[reportUnsupportedDunderAll]
diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
index f90cd0af8..21336e108 100644
--- a/cyberdrop_dl/crawlers/wordpress/_wordpress.py
+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
@@ -171,7 +171,8 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
async def handle_post_content(self, scrape_item: ScrapeItem, post: Post) -> None:
for link in self.iter_parse_url(_iter_links(post.content, self.WP_USE_REGEX)):
- await self.handle_link(scrape_item, link)
+ if link:
+ await self.handle_link(scrape_item, link)
def parse_url(self, link: str) -> AbsoluteHttpURL:
# TODO: handle more domains and move it to the base crawler
@@ -334,7 +335,7 @@ def _get_original_quality_link(link: str) -> str:
return link
-def _iter_links(html: HTML, use_regex: bool) -> itertools.chain[str]:
+def _iter_links(html: HTML, use_regex: bool) -> Iterable[str]:
soup = BeautifulSoup(html, "html.parser")
images = css.iget(soup, *css.images)
iframes = css.iget(soup, *css.iframes)
diff --git a/cyberdrop_dl/crawlers/xenforo/__init__.py b/cyberdrop_dl/crawlers/xenforo/__init__.py
index b8715e05a..f69a18668 100644
--- a/cyberdrop_dl/crawlers/xenforo/__init__.py
+++ b/cyberdrop_dl/crawlers/xenforo/__init__.py
@@ -1,5 +1,4 @@
from .allporncomix import AllPornComixCrawler
-from .bellazon import BellazonCrawler
from .celebforum import CelebForumCrawler
from .f95zone import F95ZoneCrawler
from .leakedmodels import LeakedModelsCrawler
@@ -12,7 +11,6 @@ from .xenforo import XenforoCrawler
XF_CRAWLERS: set[type[XenforoCrawler]] = {
AllPornComixCrawler,
- BellazonCrawler,
CelebForumCrawler,
F95ZoneCrawler,
LeakedModelsCrawler,
diff --git a/cyberdrop_dl/crawlers/xenforo/allporncomix.py b/cyberdrop_dl/crawlers/xenforo/allporncomix.py
index 71bb4c0db..6012cc22c 100644
--- a/cyberdrop_dl/crawlers/xenforo/allporncomix.py
+++ b/cyberdrop_dl/crawlers/xenforo/allporncomix.py
@@ -12,3 +12,4 @@ class AllPornComixCrawler(XenforoCrawler):
DOMAIN: ClassVar[str] = "allporncomix"
FOLDER_DOMAIN: ClassVar[str] = "AllPornComix"
login_required = False
+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar = False
diff --git a/cyberdrop_dl/crawlers/xenforo/bellazon.py b/cyberdrop_dl/crawlers/xenforo/bellazon.py
deleted file mode 100644
index 037ac7fab..000000000
--- a/cyberdrop_dl/crawlers/xenforo/bellazon.py
+++ /dev/null
@@ -1,33 +0,0 @@
-from __future__ import annotations
-
-from typing import ClassVar
-
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
-
-_post_selectors = PostSelectors(
- article="div[class*=ipsComment_content]",
- attachments=Selector("a[class*=ipsAttachLink]", "href"),
- content=Selector("div.cPost_contentWrap"),
- id=Selector("data-commentid", "data-commentid"),
- images=Selector("a[class*=ipsAttachLink_image]", "href"),
- videos=Selector("video.ipsEmbeddedVideo source", "src"),
-)
-
-
-# TODO: Bellazon uses Invision, not Xenforo. The selectors are completely different.
-# See: https://github.com/jbsparrow/CyberDropDownloader/pull/1079#issuecomment-2982845352
-class BellazonCrawler(XenforoCrawler):
- PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.bellazon.com/main/")
- DOMAIN: ClassVar[str] = "bellazon"
- FOLDER_DOMAIN: ClassVar[str] = "Bellazon"
- login_required = False
-
- XF_THREAD_URL_PART = "topic"
- XF_POST_URL_PART_NAME = "comment-"
- XF_SELECTORS = XenforoSelectors(
- posts=_post_selectors,
- title=Selector("span.ipsType_break.ipsContained span"),
- next_page=Selector("li.ipsPagination_next a[href]", "href"),
- )
diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.py
index fde678e24..f7b47b374 100644
--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py
+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py
@@ -11,11 +11,13 @@ class CelebForumCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://celebforum.to")
DOMAIN: ClassVar[str] = "celebforum"
FOLDER_DOMAIN: ClassVar[str] = "CelebForum"
+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar = True # images src is always a thumbnail
- def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
- if link.host == self.PRIMARY_URL.host:
+ @classmethod
+ def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool:
+ if link.host == cls.PRIMARY_URL.host:
if all(part in link.parts for part in ["data", "attachments"]): # Thumbnails
- return None
+ return True
if all(part in link.parts for part in ["data", "assets"]): # Placeholder content for insufficient rank
- return None
- return link
+ return True
+ return False
diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.py
index 21a00abfa..df4caf1fe 100644
--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py
+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py
@@ -1,16 +1,12 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils.utilities import error_handling_wrapper
from .xenforo import XenforoCrawler
-if TYPE_CHECKING:
- from yarl import URL
-
-
_confirmation_data = ({"xhr": "1", "download": "1"},)
@@ -21,14 +17,22 @@ class F95ZoneCrawler(XenforoCrawler):
@error_handling_wrapper
async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
- """Override to handle protected link confirmation."""
async with self.request_limiter:
json_resp = await self.client.post_data(self.DOMAIN, link, data=_confirmation_data)
if json_resp["status"] == "ok":
return self.parse_url(json_resp["msg"])
- def filter_link(self, link: AbsoluteHttpURL) -> URL:
- if "thumb" in link.parts:
- return link.with_path(link.path.replace("/thumb/", ""))
- return link
+ @classmethod
+ def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool:
+ return "thumb" in link.parts
+
+ @classmethod
+ def thumbnail_to_img(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ return url.with_path(url.path.replace("/thumb/", ""))
+
+ def parse_url(self, link: str) -> AbsoluteHttpURL:
+ url = super().parse_url(link)
+ if self.is_thumbnail(url):
+ return self.thumbnail_to_img(url)
+ return url
diff --git a/cyberdrop_dl/crawlers/xenforo/simpcity.py b/cyberdrop_dl/crawlers/xenforo/simpcity.py
index 2ccb879d7..0b21d0ae4 100644
--- a/cyberdrop_dl/crawlers/xenforo/simpcity.py
+++ b/cyberdrop_dl/crawlers/xenforo/simpcity.py
@@ -2,6 +2,8 @@ from __future__ import annotations
from typing import ClassVar
+from aiolimiter import AsyncLimiter
+
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from .xenforo import XenforoCrawler
@@ -11,5 +13,10 @@ class SimpCityCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://simpcity.su")
DOMAIN: ClassVar[str] = "simpcity"
FOLDER_DOMAIN: ClassVar[str] = "SimpCity"
- XF_USER_COOKIE_NAME = "dontlikebots_user"
+ LOGIN_USER_COOKIE_NAME = "dontlikebots_user"
login_required = False
+ IGNORE_EMBEDED_IMAGES_SRC = False
+
+ def __post_init__(self) -> None:
+ super().__post_init__()
+ self.request_limiter = AsyncLimiter(1, 10)
diff --git a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py
index fb6d28e0d..4fe5f8c07 100644
--- a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py
+++ b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py
@@ -11,3 +11,4 @@ class SocialMediaGirlsCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://forums.socialmediagirls.com")
DOMAIN: ClassVar[str] = "socialmediagirls"
FOLDER_DOMAIN: ClassVar[str] = "SocialMediaGirls"
+ IGNORE_EMBEDED_IMAGES_SRC = False
diff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.py
index 61c7e342e..b34d59bf5 100644
--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py
+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py
@@ -16,22 +16,22 @@ class TitsInTopsCrawler(XenforoCrawler):
DOMAIN: ClassVar[str] = "titsintops"
FOLDER_DOMAIN: ClassVar[str] = "TitsInTops"
- def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL:
- return AbsoluteHttpURL(
- str(link)
- .replace("index.php%3F", "index.php/")
- .replace("index.php?", "index.php/")
- .replace("index.php/goto", "index.php?goto")
- )
-
- def pre_filter_link(self, link: str) -> str:
- return link.replace("index.php?", "index.php/").replace("index.php%3F", "index.php/")
-
- def is_image_username_or_attachment(self, link_obj: Tag) -> bool:
+ def parse_url(self, link: str) -> AbsoluteHttpURL:
+ return super().parse_url(fix_link(link))
+
+ def is_username_or_attachment(self, link_obj: Tag) -> bool:
text = css.get_text(link_obj)
if "view attachment" in text.lower():
return True
title = css.get_attr_no_error(link_obj, "title")
if title and "permanent link" in title.lower():
return True
- return super().is_image_username_or_attachment(link_obj)
+ return super().is_username_or_attachment(link_obj)
+
+
+def fix_link(link: str) -> str:
+ return (
+ link.replace("index.php%3F", "index.php/")
+ .replace("index.php?", "index.php/")
+ .replace("index.php?goto", "index.php/goto")
+ )
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index ba203eb3a..2dfebdf95 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -1,409 +1,96 @@
-# ruff : noqa: RUF009
+"""Generic crawler for any Xenforo
+
+A Xenforo site has these attributes attached to the main html tag of the site:
+id="XF" This identifies the site as a Xenforo site
+data-cookie-prefix="ogaddgmetaprof_" The full cookies name will be `ogaddgmetaprof_user`
+data-xf="2.3" Version number
+
+
+Xenforo sites have a REST API but the APi is private only. Admins of the site need to grand access user by user
+"""
from __future__ import annotations
import asyncio
-import dataclasses
-import datetime
-import re
from typing import TYPE_CHECKING, ClassVar
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup
-from cyberdrop_dl.constants import HTTP_REGEX_LINKS
-from cyberdrop_dl.crawlers._forum import ForumCrawler
+from cyberdrop_dl.crawlers._forum import HTMLMessageBoardCrawler, MessageBoardSelectors, PostSelectors
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError
+from cyberdrop_dl.exceptions import LoginError, ScrapeError
from cyberdrop_dl.utils import css
-from cyberdrop_dl.utils.dates import to_timestamp
-from cyberdrop_dl.utils.utilities import (
- error_handling_wrapper,
- get_text_between,
- is_absolute_http_url,
- is_blob_or_svg,
-)
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Iterable
-
- from aiohttp_client_cache.response import AnyResponse
+ from collections.abc import Iterable, Sequence
from cyberdrop_dl.crawlers.crawler import SupportedPaths
- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
-
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-LINK_TRASH_MAPPING = {
- ".th.": ".",
- ".md.": ".",
- "ifr": "watch",
-}
Selector = css.CssAttributeSelector
-LAST_PAGE_SELECTOR = Selector("li.pageNav-page a:last-of-type", "href")
-CURRENT_PAGE_SELECTOR = Selector("li.pageNav-page.pageNav-page--current a", "href")
-
-
[email protected](frozen=True, slots=True)
-class PostSelectors:
- article: str = "article.message[id*=post]" # the entire html of the post (comments, attachments, user avatar, signature, etc...)
-
- attachments: Selector = Selector(".message-attachments a[href]", "href")
- content: Selector = Selector(".message-userContent") # text, links and images (NO attachments)
- date: Selector = Selector("time", "datetime")
- embeds: Selector = Selector("iframe", "src")
- id: Selector = Selector(".u-concealed a[href]", "href") # TODO: This needs a better selector
- images: Selector = Selector("img.bbImage", "src")
- lazy_load_embeds: Selector = Selector('[class*=iframe][onclick*="loadMedia(this, \'//"]', "onclick")
- links: Selector = Selector(":any-link", "href")
- videos: Selector = Selector("video source", "src")
-
-
[email protected](frozen=True, slots=True)
-class XenforoSelectors:
- confirmation_button: Selector = Selector("a[class*=button--cta][href]", "href")
- next_page: Selector = Selector("a[class*=pageNav-jump--next][href]", "href")
- posts: PostSelectors = PostSelectors()
- quotes: Selector = Selector("blockquote")
- title_trash: Selector = Selector("span")
- title: Selector = Selector("h1[class*=p-title-value]")
- last_page: Selector = LAST_PAGE_SELECTOR
- current_page: Selector = CURRENT_PAGE_SELECTOR
-
-
[email protected](frozen=True, slots=True, order=True)
-class ForumPost:
- id: int
- date: datetime.datetime
- article: Tag = dataclasses.field(compare=False)
- content: Tag = dataclasses.field(compare=False)
-
- @staticmethod
- def new(article: Tag, selectors: PostSelectors, post_name: str = "post") -> ForumPost:
- content = css.select_one(article, selectors.content.element)
- date = datetime.datetime.fromisoformat(css.select_one_get_attr(article, *selectors.date))
- if selectors.id.element == selectors.id.attribute:
- post_id = int(css.get_attr(article, selectors.id.element))
- else:
- id_str = css.select_one_get_attr(article, *selectors.id)
- post_id = int(id_str.rsplit(post_name, 1)[-1].replace("-", ""))
- return ForumPost(post_id, date, article, content)
-
[email protected](frozen=True, slots=True)
-class Thread:
- id: int
- name: str
- page: int
- post_id: int | None
- url: AbsoluteHttpURL
+DEFAULT_XF_POST_SELECTORS = PostSelectors(
+ article="article.message[id*=post]",
+ content=".message-userContent",
+ article_trash=(".message-signature", ".message-footer"),
+ content_trash=("blockquote", "fauxBlockLink"),
+ id=Selector("article.message[id*=post]", "id"),
+ attachments=Selector(".message-attachments a[href]", "href"),
+)
-DEFAULT_XF_SELECTORS = XenforoSelectors()
-KNOWN_THREAD_PART_NAMES = "thread", "topic", "tema"
-KNOWN_THREAD_PART_NAMES = ",".join(f"{part},{part}s" for part in KNOWN_THREAD_PART_NAMES).split(",")
+DEFAULT_XF_SELECTORS = MessageBoardSelectors(
+ posts=DEFAULT_XF_POST_SELECTORS,
+ confirmation_button=Selector("a[class*=button--cta][href]", "href"),
+ next_page=Selector("a[class*=pageNav-jump--next][href]", "href"),
+ title_trash=("span",),
+ title=Selector("h1[class*=p-title-value]"),
+ last_page=Selector("li.pageNav-page a:last-of-type", "href"),
+ current_page=Selector("li.pageNav-page.pageNav-page--current a", "href"),
+)
def _escape(strings: Iterable[str]) -> str:
return r"\|".join(strings)
-class XenforoCrawler(ForumCrawler, is_abc=True):
- XF_ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"
+class XenforoCrawler(HTMLMessageBoardCrawler, is_abc=True):
+ ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"
+ THREAD_PART_NAMES: ClassVar[Sequence[str]] = "thread", "topic", "tema", "threads", "topics", "temas"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Attachments": f"/({_escape(XF_ATTACHMENT_URL_PARTS)})/...",
+ "Attachments": f"/({_escape(ATTACHMENT_URL_PARTS)})/...",
"Threads": (
- f"/({_escape(KNOWN_THREAD_PART_NAMES)})/<thread_name_and_id>",
+ f"/({_escape(THREAD_PART_NAMES)})/<thread_name_and_id>",
"/posts/<post_id>",
"/goto/<post_id>",
),
"**NOTE**": "base crawler: Xenforo",
}
SUPPORTS_THREAD_RECURSION: ClassVar[bool] = True
- XF_SELECTORS = DEFAULT_XF_SELECTORS
- XF_POST_URL_PART_NAME = "post-"
- XF_PAGE_URL_PART_NAME = "page-"
- XF_THREAD_URL_PART = "threads"
- XF_USER_COOKIE_NAME = "xf_user"
- XF_ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"
+ SELECTORS: ClassVar[MessageBoardSelectors] = DEFAULT_XF_SELECTORS
+ POST_URL_PART_NAME: ClassVar[str] = "post"
+ PAGE_URL_PART_NAME: ClassVar[str] = "page"
+ IGNORE_EMBEDED_IMAGES_SRC: ClassVar[bool] = True
+ LOGIN_USER_COOKIE_NAME: ClassVar[str] = "xf_user"
+ # Attachments hosts should technically be defined on each specific Crawler, but they do no harm here
+ ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"
login_required = True
- def __post_init__(self) -> None:
- self.scraped_threads = set()
-
- async def async_startup(self) -> None:
- if not self.logged_in:
- login_url = self.PRIMARY_URL / "login"
- await self.login_setup(login_url)
-
- self.register_cache_filter(self.PRIMARY_URL, check_is_not_last_page)
-
def get_filename_and_ext(self, filename: str) -> tuple[str, str]:
+ # The `forum` keyword is misleading now. It only works for Xenforo sites, not every forum
+ # TODO: Change `forum` parameter to `xenforo`
return super().get_filename_and_ext(filename, forum=True)
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- if not self.logged_in and self.login_required:
- return
- url = self.pre_filter_link(str(scrape_item.url))
- scrape_item.url = self.parse_url(url)
- if self.is_attachment(scrape_item.url):
- return await self.handle_internal_link(scrape_item)
- if is_confirmation_link(scrape_item.url):
- return await self.follow_confirmation_link(scrape_item)
-
- await self.fetch_thread(scrape_item)
-
- async def fetch_thread(self, scrape_item: ScrapeItem) -> None:
- thread_part_index = len(self.PRIMARY_URL.parts)
- match scrape_item.url.parts[thread_part_index:]:
- case [thread_part, thread_name_and_id, *_] if thread_part in KNOWN_THREAD_PART_NAMES:
- return await self.thread(scrape_item, thread_name_and_id)
- case ["goto" | "posts", _, *_]:
- return await self.follow_redirect_w_get(scrape_item)
-
- raise ValueError
-
- async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:
- url = await self.resolve_confirmation_link(scrape_item.url)
- if url: # If there was an error, this will be None
- scrape_item.url = url
- # This could end up back in here if the URL goes to another thread
- return self.handle_external_links(scrape_item)
-
- @error_handling_wrapper
- async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:
- async with self.request_limiter:
- response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)
- assert is_absolute_http_url(response.url)
- scrape_item.url = response.url
- self.manager.task_group.create_task(self.run(scrape_item))
-
@error_handling_wrapper
- async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:
- head = await self.client.get_head(self.DOMAIN, scrape_item.url)
- redirect = head.get("location")
- if not redirect:
- raise ScrapeError(422)
- scrape_item.url = self.parse_url(redirect)
- self.manager.task_group.create_task(self.run(scrape_item))
-
- async def forum(self, scrape_item: ScrapeItem) -> None:
- raise NotImplementedError
-
- @error_handling_wrapper
- async def thread(self, scrape_item: ScrapeItem, thread_name_and_id: str) -> None:
- scrape_item.setup_as_forum("")
- thread = parse_thread(
- scrape_item.url, thread_name_and_id, self.XF_PAGE_URL_PART_NAME, self.XF_POST_URL_PART_NAME
- )
- if thread.url in self.scraped_threads:
- return
-
- scrape_item.parent_threads.add(thread.url)
- if self.scrape_single_forum_post and not thread.post_id:
- msg = "`--scrape-single-forum-post` is `True`, but the provided URL has no post id"
- raise ScrapeError("User Error", msg)
-
- self.scraped_threads.add(thread.url)
- await self.process_thread(scrape_item, thread)
-
- async def process_thread(self, scrape_item: ScrapeItem, thread: Thread) -> None:
- title: str = ""
- last_post_url = thread.url
- async for soup in self.thread_pager(scrape_item):
- if not title:
- try:
- title = self.create_title(get_post_title(soup, self.XF_SELECTORS), thread_id=thread.id)
- except ScrapeError as e:
- self.log_debug("Got an unprocessable soup", 40, exc_info=e)
- raise
- scrape_item.add_to_parent_title(title)
-
- continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)
- if not continue_scraping:
- break
-
- await self.write_last_forum_post(thread.url, last_post_url)
-
- def process_thread_page(
- self, scrape_item: ScrapeItem, thread: Thread, soup: BeautifulSoup
- ) -> tuple[bool, AbsoluteHttpURL]:
- continue_scraping = False
- post_url = thread.url
- for article in soup.select(self.XF_SELECTORS.posts.article):
- current_post = ForumPost.new(article, self.XF_SELECTORS.posts, self.XF_POST_URL_PART_NAME)
- continue_scraping, scrape_this_post = check_post_id(
- thread.post_id, current_post.id, self.scrape_single_forum_post
- )
- if scrape_this_post:
- post_url = thread.url / f"{self.XF_POST_URL_PART_NAME}{current_post.id}"
- new_scrape_item = scrape_item.create_new(
- thread.url,
- possible_datetime=to_timestamp(current_post.date),
- add_parent=post_url,
- )
- self.manager.task_group.create_task(self.post(new_scrape_item, current_post))
- try:
- scrape_item.add_children()
- except MaxChildrenError:
- break
-
- if not continue_scraping:
- break
- return continue_scraping, post_url
-
- @error_handling_wrapper
- async def post(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- scrape_item.setup_as_post("")
- post_title = self.create_separate_post_title(None, str(post.id), post.date)
- scrape_item.add_to_parent_title(post_title)
- for scraper in (
- self._attachments,
- self._embeds,
- self._images,
- self._links,
- self._videos,
- self._lazy_load_embeds,
- ):
- await scraper(scrape_item, post)
-
- async def _links(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = self.XF_SELECTORS.posts.links
- links = css.iselect(post.content, selector.element)
- valid_links = (link for link in links if not self.is_image_username_or_attachment(link))
- await self.process_children(scrape_item, valid_links, selector.attribute)
-
- async def _images(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = self.XF_SELECTORS.posts.images
- images = css.iselect(post.content, selector.element)
- await self.process_children(scrape_item, images, selector.attribute)
-
- async def _videos(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = self.XF_SELECTORS.posts.videos
- videos = css.iselect(post.content, selector.element)
- await self.process_children(scrape_item, videos, selector.attribute)
-
- async def _embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = self.XF_SELECTORS.posts.embeds
- embeds = css.iselect(post.content, selector.element)
- await self.process_children(scrape_item, embeds, selector.attribute, embeds=True)
-
- async def _attachments(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = self.XF_SELECTORS.posts.attachments
- attachments = css.iselect(post.article, selector.element)
- await self.process_children(scrape_item, attachments, selector.attribute)
-
- async def _lazy_load_embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = self.XF_SELECTORS.posts.lazy_load_embeds
- for lazy_media in css.iselect(post.content, selector.element):
- link_str = get_text_between(css.get_attr(lazy_media, selector.attribute), "loadMedia(this, '", "')")
- await self.process_child(scrape_item, link_str, embeds=True)
-
- async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
- async for soup in self._web_pager(scrape_item.url, self.get_next_page):
- yield soup
-
- def get_next_page(self, soup: BeautifulSoup) -> str | None:
- if next_page := css.select_one_get_attr_or_none(soup, *self.XF_SELECTORS.next_page):
- return self.pre_filter_link(next_page)
-
- @error_handling_wrapper
- async def process_children(
- self, scrape_item: ScrapeItem, links: Iterable[Tag], attribute: str, *, embeds: bool = False
- ) -> None:
- for link_tag in links:
- if link_str := css.get_attr_no_error(link_tag, attribute):
- await self.process_child(scrape_item, link_str, embeds=embeds)
-
- @error_handling_wrapper
- async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:
- link_str_ = pre_process_child(link_str, embeds)
- if not link_str_:
- return
- link = await self.get_absolute_link(link_str_)
- link = self.filter_link(link)
- if not link:
- return
- await self.handle_link(scrape_item, link)
-
- def is_attachment(self, link: AbsoluteHttpURL | str) -> bool:
- if not link:
- return False
- if isinstance(link, str):
- link = self.parse_url(link)
- by_parts = len(link.parts) > 2 and any(p in link.parts for p in self.XF_ATTACHMENT_URL_PARTS)
- by_host = any(host in link.host for host in self.XF_ATTACHMENT_HOSTS)
- return by_parts or by_host
-
- async def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpURL | None:
- if isinstance(link, str):
- absolute_link = self.parse_url(clean_link_str(link))
- else:
- absolute_link = link
- if is_confirmation_link(absolute_link):
- return await self.resolve_confirmation_link(absolute_link)
- return absolute_link
-
- @error_handling_wrapper
- async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)
- selector = self.XF_SELECTORS.confirmation_button
- confirm_button = soup.select_one(selector.element)
- if not confirm_button:
- return
- link_str: str = css.get_attr(confirm_button, selector.attribute)
- link_str = link_str.split('" class="link link--internal', 1)[0]
- new_link = self.parse_url(link_str)
- return await self.get_absolute_link(new_link)
-
- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:
- """Handles internal links."""
- slug = scrape_item.url.name or scrape_item.url.parent.name
- if slug.isdigit():
- return await self.follow_redirect_w_head(scrape_item)
-
- return await super().handle_internal_link(scrape_item)
-
- def is_image_username_or_attachment(self, link_obj: Tag) -> bool:
- if link_obj.select_one("img"):
- return True
- if link_obj.select_one(".username"):
- return True
- if link_str := css.get_attr_no_error(link_obj, self.XF_SELECTORS.posts.links.element):
- return self.is_attachment(link_str)
- return False
-
- def pre_filter_link(self, link: str) -> str:
- return link
-
- def filter_link(self, link: AbsoluteHttpURL | None) -> AbsoluteHttpURL | None:
- return link
-
- @error_handling_wrapper
- async def login_setup(self, login_url: AbsoluteHttpURL) -> None:
- host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)
- session_cookie = host_cookies.get(self.XF_USER_COOKIE_NAME)
- session_cookie = session_cookie.value if session_cookie else None
- msg = f"No cookies found for {self.FOLDER_DOMAIN}"
- if not session_cookie and self.login_required:
- raise LoginError(message=msg)
-
- _, self.logged_in = await self.check_login_with_request(login_url)
-
- if self.logged_in:
- return
- if session_cookie:
- msg = f"Cookies for {self.FOLDER_DOMAIN} are not valid."
- if self.login_required:
- raise LoginError(message=msg)
+ async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:
+ """Logic to login as a Xenforo user
- msg += " Scraping without an account"
- self.log(msg, 30)
+ This was deprecated in v6.5.0 but the code itself it useful for debuggig without cookie extraction.
+ Login functionality may come back in a future version..."""
- @error_handling_wrapper
- async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:
- """Logs in to a forum."""
manual_login = username and password
missing_credentials = not (manual_login or session_cookie)
if missing_credentials:
@@ -411,7 +98,7 @@ class XenforoCrawler(ForumCrawler, is_abc=True):
raise LoginError(message=msg)
if session_cookie:
- cookies = {"xf_user": session_cookie}
+ cookies = {self.LOGIN_USER_COOKIE_NAME: session_cookie}
self.update_cookies(cookies)
credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}
@@ -424,7 +111,7 @@ class XenforoCrawler(ForumCrawler, is_abc=True):
retries: int,
wait_time: int | None = None,
) -> None:
- # Try from cookies
+ # Check first if we have cookies and they are valid
text, logged_in = await self.check_login_with_request(login_url)
if logged_in:
self.logged_in = True
@@ -448,91 +135,6 @@ class XenforoCrawler(ForumCrawler, is_abc=True):
msg = f"Failed to login on {self.FOLDER_DOMAIN} after {retries} attempts"
raise LoginError(message=msg)
- async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:
- text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)
- return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))
-
-
-def parse_thread(url: AbsoluteHttpURL, thread_name_and_id: str, page_part_name: str, post_part_name: str) -> Thread:
- name_index = url.parts.index(thread_name_and_id)
- name, id_ = parse_thread_name_and_id(thread_name_and_id)
- page, post_id = get_thread_page_and_post(url, name_index, page_part_name, post_part_name)
- canonical_url = get_thread_canonical_url(url, name_index)
- return Thread(id_, name, page, post_id, canonical_url)
-
-
-def parse_thread_name_and_id(thread_name_and_id: str) -> tuple[str, int]:
- try:
- name, id_str = thread_name_and_id.rsplit(".", 1)
- except ValueError:
- id_str, name = thread_name_and_id.split("-", 1)
- return name, int(id_str)
-
-
-def get_thread_canonical_url(url: AbsoluteHttpURL, thread_name_index: int) -> AbsoluteHttpURL:
- new_parts = url.parts[1 : thread_name_index + 1]
- new_path = "/".join(new_parts)
- return url.with_path(new_path)
-
-
-def get_thread_page_and_post(
- url: AbsoluteHttpURL, thread_name_index: int, page_name: str, post_name: str
-) -> tuple[int, int | None]:
- extra_parts = url.parts[thread_name_index + 1 :]
- if url.fragment:
- extra_parts = *extra_parts, url.fragment
-
- def find_number(search_value: str) -> int | None:
- for sec in extra_parts:
- if search_value in sec:
- return int(sec.replace(search_value, "").replace("-", "").strip())
-
- post_id = find_number(post_name)
- page_number = find_number(page_name) or 1
- return page_number, post_id
-
-
-async def check_is_not_last_page(response: AnyResponse) -> bool:
- soup = BeautifulSoup(await response.text(), "html.parser")
- return not is_last_page(soup)
-
-
-def is_last_page(soup: BeautifulSoup) -> bool:
- try:
- last_page = css.select_one_get_attr(soup, *LAST_PAGE_SELECTOR)
- current_page = css.select_one_get_attr(soup, *CURRENT_PAGE_SELECTOR)
- except (AttributeError, IndexError, css.SelectorError):
- return True
- return current_page == last_page
-
-
-def get_post_title(soup: BeautifulSoup, selectors: XenforoSelectors) -> str:
- try:
- title_block = css.select_one(soup, selectors.title.element)
- trash = title_block.select(selectors.title_trash.element)
- for element in trash:
- element.decompose()
- except (AttributeError, AssertionError, css.SelectorError) as e:
- raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e
-
- if title := " ".join(css.get_text(title_block).split()):
- return title
- raise ScrapeError(422)
-
-
-def extract_embed_url(embed_str: str) -> str:
- embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")
- if match := re.search(HTTP_REGEX_LINKS, embed_str):
- return match.group(0).replace("www.", "")
- return embed_str
-
-
-# TODO: move to parse_url
-def clean_link_str(link: str) -> str:
- for old, new in LINK_TRASH_MAPPING.items():
- link = link.replace(old, new)
- return link
-
def parse_login_form(resp_text: str) -> dict[str, str]:
soup = BeautifulSoup(resp_text, "html.parser")
@@ -545,32 +147,3 @@ def parse_login_form(resp_text: str) -> dict[str, str]:
if data:
return data
raise ScrapeError(422)
-
-
-def is_confirmation_link(link: AbsoluteHttpURL) -> bool:
- return "masked" in link.parts or "link-confirmation" in link.path
-
-
-def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]:
- """Checks if the program should scrape the current post.
-
- Returns (continue_scraping, scrape_this_post)"""
- if init_post_id:
- if init_post_id > current_post_id:
- return (True, False)
- elif init_post_id == current_post_id:
- return (not scrape_single_forum_post, True)
- else:
- return (not scrape_single_forum_post, not scrape_single_forum_post)
-
- assert not scrape_single_forum_post # We should have raised an exception early
- return True, True
-
-
-def pre_process_child(link_str: str, embeds: bool = False) -> str | None:
- assert isinstance(link_str, str)
- if embeds:
- link_str = extract_embed_url(link_str)
-
- if link_str and not is_blob_or_svg(link_str):
- return link_str
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index 4d63fbeb3..d6c857d6c 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -176,7 +176,7 @@ class Video(XHamsterItem):
mp4_sources: dict[str, Any] = Field({}, validation_alias=AliasPath("sources", "mp4"))
def get_formats(self) -> Generator[Format]:
- yield Format(0, 0, self.mp4_file)
+ yield Format(0, "", self.mp4_file)
for resolution, details in self.mp4_sources.items():
height = int(resolution.removesuffix("p"))
link: str = details["link"] if isinstance(details, dict) else details
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 2f2ce7d68..f7e93dcec 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -112,9 +112,15 @@ def _parse_srcset(srcset: str) -> str:
return [src.split(" ")[0] for src in srcset.split(", ")][-1]
-def iget(tag: Tag, selector: str, attribute: str) -> Generator[str | None]:
+def iget(tag: Tag, selector: str, attribute: str) -> Generator[str]:
for inner_tag in iselect(tag, selector):
- yield get_attr_or_none(inner_tag, attribute)
+ if link := get_attr_or_none(inner_tag, attribute):
+ yield link
+
+
+def decompose(tag: Tag, selector: str) -> None:
+ for inner_tag in tag.select(selector):
+ inner_tag.decompose()
iframes = CssAttributeSelector("iframe", "src")
diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
index de34d54dc..315628705 100644
--- a/cyberdrop_dl/utils/logger.py
+++ b/cyberdrop_dl/utils/logger.py
@@ -5,7 +5,7 @@ import logging
import queue
from logging.handlers import QueueHandler, QueueListener
from pathlib import Path
-from typing import IO, TYPE_CHECKING
+from typing import IO, TYPE_CHECKING, Any
from rich._log_render import LogRender
from rich.console import Console, Group
@@ -215,7 +215,7 @@ class RedactedConsole(Console):
return _redact_message(output)
-def process_log_msg(message: dict | Exception | str) -> str:
+def process_log_msg(message: dict[str, Any] | Exception | str) -> str:
if isinstance(message, dict):
return json.dumps(message, indent=4, ensure_ascii=False)
return str(message)
@@ -227,14 +227,14 @@ def create_rich_log_msg(msg: str, level: int = 10) -> Text:
return rich_level + indent_string(msg)
-def log(message: dict | Exception | str, level: int = 10, **kwargs) -> None:
+def log(message: dict[str, Any] | Exception | str, level: int = 10, prefix: str = "", **kwargs) -> None:
"""Simple logging function."""
- msg = process_log_msg(message)
+ msg = prefix + process_log_msg(message)
logger.log(level, msg, **kwargs)
log_debug(msg, level, **kwargs)
-def log_debug(message: dict | Exception | str, level: int = 10, **kwargs) -> None:
+def log_debug(message: dict[str, Any] | Exception | str, level: int = 10, **kwargs) -> None:
"""Simple logging function."""
if env.DEBUG_VAR:
msg = process_log_msg(message)
diff --git a/tests/crawlers/test_vbulletin.py b/tests/crawlers/test_vbulletin.py
index 7a8fa7eb1..78f4e8915 100644
--- a/tests/crawlers/test_vbulletin.py
+++ b/tests/crawlers/test_vbulletin.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
-from cyberdrop_dl.crawlers.xenforo import xenforo
+from cyberdrop_dl.crawlers import _forum, vbulletin
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
@@ -35,6 +35,6 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
)
def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str, int, int], canonical_url: str) -> None:
url_, canonical_url_ = AbsoluteHttpURL(url), AbsoluteHttpURL(canonical_url)
- result_ = xenforo.Thread(*result, canonical_url_)
- parsed = xenforo.parse_thread(url_, thread_name_and_id, "page", "post")
+ result_ = _forum.Thread(*result, canonical_url_)
+ parsed = vbulletin.vBulletinCrawler.parse_thread(url_, thread_name_and_id)
assert result_ == parsed
diff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.py
index 70e7f2296..aba68d0ad 100644
--- a/tests/crawlers/test_xenforo.py
+++ b/tests/crawlers/test_xenforo.py
@@ -6,6 +6,7 @@ from unittest import mock
import pytest
from bs4 import BeautifulSoup
+from cyberdrop_dl.crawlers import _forum
from cyberdrop_dl.crawlers import xenforo as crawlers
from cyberdrop_dl.crawlers.xenforo import xenforo
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
@@ -47,11 +48,11 @@ def _post(
message_attachments: str = "",
id: int = 12345,
crawler: xenforo.XenforoCrawler | None = None,
-) -> xenforo.ForumPost:
+) -> _forum.ForumPost:
crawler = crawler or TEST_CRAWLER
html = _html(POST_TEMPLATE.format(id=id, message_body=message_body, message_attachments=message_attachments))
- soup = BeautifulSoup(html, "html.parser")
- return xenforo.ForumPost.new(soup, crawler.XF_SELECTORS.posts)
+ article = BeautifulSoup(html, "html.parser").select("article")[0]
+ return _forum.ForumPost.new(article, crawler.SELECTORS.posts)
def _item_call(value: Any) -> mock._Call:
@@ -199,8 +200,8 @@ async def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
)
def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str, int, int], canonical_url: str) -> None:
url_, canonical_url_ = AbsoluteHttpURL(url), AbsoluteHttpURL(canonical_url)
- result_ = xenforo.Thread(*result, canonical_url_)
- parsed = xenforo.parse_thread(url_, thread_name_and_id, "page", "post")
+ result_ = _forum.Thread(*result, canonical_url_)
+ parsed = TEST_CRAWLER.parse_thread(url_, thread_name_and_id)
assert result_ == parsed
@@ -238,7 +239,7 @@ def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str,
],
)
def test_clean_link_url(link: str, out: str) -> None:
- assert xenforo.clean_link_str(link) == out
+ assert _forum.clean_link_str(link) == out
def test_parse_login_form_success() -> None:
@@ -357,7 +358,7 @@ def test_parse_login_form_no_input_form_should_fail() -> None:
],
)
def test_extract_embed_url(input_string: str, expected_output: str) -> None:
- assert xenforo.extract_embed_url(input_string) == expected_output
+ assert _forum.extract_embed_url(input_string) == expected_output
@pytest.mark.xfail # regex can not handle URLs with commans in it (kvs)
@@ -375,40 +376,40 @@ def test_extract_embed_url(input_string: str, expected_output: str) -> None:
],
)
def test_extract_embed_url_complex_path(input_string: str, expected_output: str) -> None:
- assert xenforo.extract_embed_url(input_string) == expected_output
+ assert _forum.extract_embed_url(input_string) == expected_output
def test_extract_embed_url_should_extract_only_one_url() -> None:
input_str = r"first_url \/\/first.com/path second_url \/\/www.second.net/another"
expected_output = "https://first.com/path"
- assert xenforo.extract_embed_url(input_str) == expected_output
+ assert _forum.extract_embed_url(input_str) == expected_output
def test_extract_embed_url_with_escaped_backslashes_and_double_slashes() -> None:
input_str = r"some_path\\to\\file\/\/imagepond.net\/clip.mov"
expected_output = "https://imagepond.net/clip.mov"
- assert xenforo.extract_embed_url(input_str) == expected_output
+ assert _forum.extract_embed_url(input_str) == expected_output
def test_extract_embed_url_absolute_url_no_scheme() -> None:
input_str = r"\/\/jpg5.su"
expected_output = r"https://jpg5.su"
- assert xenforo.extract_embed_url(input_str) == expected_output
+ assert _forum.extract_embed_url(input_str) == expected_output
def test_extract_embed_url_should_not_modify_absolute_urls() -> None:
input_str = r"https://celebforum.to/maps"
expected_output = "https://celebforum.to/maps"
- assert xenforo.extract_embed_url(input_str) == expected_output
+ assert _forum.extract_embed_url(input_str) == expected_output
def test_extract_embed_url_string_ends_with_url() -> None:
input_str = r"some text https://www.reddit.com/"
expected_output = "https://reddit.com/"
- assert xenforo.extract_embed_url(input_str) == expected_output
+ assert _forum.extract_embed_url(input_str) == expected_output
-async def test_lazy_load_embeds() -> None:
+def test_lazy_load_embeds() -> None:
post = _post("""
<div class="generic2wide-iframe-div" onclick="loadMedia(this, '//redgifs.com/ifr/downrightcluelesswirm');">
<span data-s9e-mediaembed="redgifs">
@@ -417,19 +418,35 @@ async def test_lazy_load_embeds() -> None:
</span>
</span>
</div>""")
- expected_output = "//redgifs.com/ifr/downrightcluelesswirm"
- with _amock() as mocked:
- await TEST_CRAWLER._lazy_load_embeds(scrape_item, post)
- mocked.assert_called_once_with(scrape_item, expected_output, embeds=True)
+ result = list(TEST_CRAWLER._lazy_load_embeds(post))
+ assert len(result) == 1
+ assert "//redgifs.com/ifr/downrightcluelesswirm" == result[0]
@pytest.mark.parametrize(
("cls", "post_content", "expected_result"),
[
+ (
+ crawlers.AllPornComixCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>""",
+ ["https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"],
+ ),
(
crawlers.SimpCityCrawler,
"""
- <div class="bbWrapper">
<a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
<img
src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
@@ -461,21 +478,229 @@ async def test_lazy_load_embeds() -> None:
"https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
"https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
],
- )
+ ),
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ ),
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ ),
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ ),
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ ),
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ ),
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ ),
],
)
-async def test_post_images(cls: type[xenforo.XenforoCrawler], post_content: str, expected_result: list[str]) -> None:
+def test_post_images(cls: type[xenforo.XenforoCrawler], post_content: str, expected_result: list[str]) -> None:
crawler = crawler_instances[cls]
post = _post(post_content, crawler=crawler)
- with _amock(crawler=crawler) as mocked:
- await crawler._images(scrape_item, post)
- count, expected_count = mocked.call_count, len(expected_result)
- assert count == expected_count, f"Found {count} links, expected {expected_count} links"
- for result, expected in zip(mocked.call_args_list, expected_result, strict=True):
- assert result.args[1] == expected
+ results = list(crawler._images(post))
+ count, expected_count = len(results), len(expected_result)
+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"
+ assert results == expected_result
-async def test_embeds_can_extract_google_drive_links() -> None:
+def test_embeds_can_extract_google_drive_links() -> None:
# https://github.com/jbsparrow/CyberDropDownloader/issues/775
crawler = crawler_instances[crawlers.SimpCityCrawler]
content = """
@@ -492,14 +717,13 @@ async def test_embeds_can_extract_google_drive_links() -> None:
"""
post = _post(content, crawler=crawler)
expected_result = "//drive.google.com/file/d/1gfDjCbNXgJafY6ILQIrgbnuptSCFbM0J/preview"
- with _amock(crawler=crawler) as mocked:
- await crawler._embeds(scrape_item, post)
- mocked.assert_called_once()
- assert mocked.call_args.args[1] == expected_result
+ results = list(crawler._embeds(post))
+ assert len(results) == 1
+ assert results[0] == expected_result
-async def test_post_smg_extract_attachments() -> None:
- # https://agithub.com/jbsparrow/CyberDropDownloader/issues/1070
+def test_post_smg_extract_attachments() -> None:
+ # https://github.com/jbsparrow/CyberDropDownloader/issues/1070
attachments = """
<h4 class="block-textHeader">Attachments</h4>
<ul class="attachmentList">
@@ -564,12 +788,28 @@ async def test_post_smg_extract_attachments() -> None:
"https://smgmedia2.socialmediagirls.com/forum/2022/04/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg",
"https://smgmedia2.socialmediagirls.com/forum/2022/04/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png",
]
- with _amock(crawler=crawler) as mocked:
- await crawler._attachments(scrape_item, post)
- count, expected_count = mocked.call_count, len(expected_result)
- assert count == expected_count, f"Found {count} links, expected {expected_count} links"
- for result, expected in zip(mocked.call_args_list, expected_result, strict=True):
- assert result.args[1] == expected
+
+ result = list(crawler._attachments(post))
+ count, expected_count = len(result), len(expected_result)
+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"
+ assert result == expected_result
+
+
+def test_post_celebforum_should_use_href_for_images() -> None:
+ # https://github.com/jbsparrow/CyberDropDownloader/issues/1093
+ content = """
+ <a href="https://celebforum.to/attachments/jc6huqrju9-jpg.321620/" target="_blank"><img alt="JC6HUqrju9" class="bbImage"
+ height="200" loading="lazy"
+ src="https://celebforum.to/data/attachments/321/321141-2d66067afbf5cd3d546479380c08929d.jpg" style=""
+ title="JC6HUqrju9" width="150" /></a>
+ """
+ crawler = crawler_instances[crawlers.CelebForumCrawler]
+ post = _post(content, crawler=crawler)
+ expected_result = ["https://celebforum.to/attachments/jc6huqrju9-jpg.321620/"]
+ result = list(crawler._images(post))
+ count, expected_count = len(result), len(expected_result)
+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"
+ assert result == expected_result
def test_get_post_title_thread_w_prefixes() -> None:
@@ -588,7 +828,7 @@ def test_get_post_title_thread_w_prefixes() -> None:
</div>
""")
soup = BeautifulSoup(html, "html.parser")
- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)
assert title == "GunplaMeli"
@@ -599,7 +839,7 @@ def test_get_post_title_thread_w_no_prefixes() -> None:
</div>
"""
soup = BeautifulSoup(html, "html.parser")
- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)
assert title == "Staged/Fake Japanese Candid Videos from Gcolle/Pcolle or FC2"
@@ -607,7 +847,7 @@ def test_get_post_title_no_title_found() -> None:
html = _html("")
soup = BeautifulSoup(html, "html.parser")
with pytest.raises(ScrapeError) as exc_info:
- xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)
assert exc_info.value.status == 429
assert exc_info.value.message == "Invalid response from forum. You may have been rate limited"
@@ -617,7 +857,7 @@ def test_get_post_title_empty_title_block() -> None:
html = _html("""<h1 class="p-title-value"></h1>""")
soup = BeautifulSoup(html, "html.parser")
with pytest.raises(ScrapeError):
- xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)
def test_get_post_title_non_english_chars() -> None:
@@ -629,7 +869,7 @@ def test_get_post_title_non_english_chars() -> None:
</div>
""")
soup = BeautifulSoup(html, "html.parser")
- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)
assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"
@@ -644,7 +884,7 @@ def test_get_post_title_should_strip_new_lines() -> None:
</div>
""")
soup = BeautifulSoup(html, "html.parser")
- title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ title = _forum.get_post_title(soup, xenforo.XenforoCrawler.SELECTORS)
assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"
@@ -693,7 +933,7 @@ class TestCheckPostId:
expected_continue_scraping: bool,
expected_scrape_this_post: bool,
) -> None:
- continue_scraping, scrape_this_post = xenforo.check_post_id(
+ continue_scraping, scrape_this_post = _forum.check_post_id(
init_post_id, current_post_id, scrape_single_forum_post
)
assert continue_scraping == expected_continue_scraping
@@ -703,7 +943,7 @@ class TestCheckPostId:
init_post_id = None
current_post_id = 100
scrape_single_forum_post = False
- continue_scraping, scrape_this_post = xenforo.check_post_id(
+ continue_scraping, scrape_this_post = _forum.check_post_id(
init_post_id, current_post_id, scrape_single_forum_post
)
assert continue_scraping is True
@@ -715,23 +955,84 @@ class TestCheckPostId:
scrape_single_forum_post = True
with pytest.raises(AssertionError):
- xenforo.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)
+ _forum.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)
# og_post_id = 23549340
POST_TEMPLATE = """
-<article class="message message--post js-post js-inlineModContainer" data-author="cyberdrop-dl-patched" data-content="post{id}" id="js-post{id}" itemscope="" itemtype="https://schema.org/Comment" itemid="https://xenforo.com/posts/{id}/">
- <meta itemprop="parentItem" itemscope="" itemid="https://xenforo.com/threads/fanfan.33077/" />
+<article class="message message--post js-post js-inlineModContainer" data-author="" data-content="post-{id}" id="js-post-{id}" itemscope="" itemtype="https://schema.org/Comment" itemid="https://simpcity.su/posts/{id}/">
+ <meta itemprop="parentItem" itemscope="" itemid="https://xenforocomunity.com/threads/fanfan.33077/" />
- <span class="u-anchorTarget" id="post{id}"></span>
+ <span class="u-anchorTarget" id="post-{id}"></span>
<div class="message-inner">
+ <div class="message-cell message-cell--user">
+ <section class="message-user" itemprop="author" itemscope="" itemtype="https://schema.org/Person" itemid="https://xenforocomunity.com/members/mrspike.5160076/">
+ <meta itemprop="url" content="https://xenforocomunity.com/members/mrspike.5160076/" />
+
+ <div class="message-avatar">
+ <div class="message-avatar-wrapper">
+ <a href="/members/mrspike.5160076/" class="avatar avatar--m" data-user-id="5160076" data-xf-init="member-tooltip" id="js-XFUniqueId10">
+ <img
+ src="http://simp6.jpg5.su/simpo/data/avatars/m/5160/5160076.jpg?1746748029"
+ srcset="http://simp6.jpg5.su/simpo/data/avatars/l/5160/5160076.jpg?1746748029 2x"
+ alt=""
+ class="avatar-u5160076-m"
+ width="96"
+ height="96"
+ loading="lazy"
+ itemprop="image"
+ />
+ </a>
+ </div>
+ </div>
+ <div class="message-userDetails">
+ <h4 class="message-name">
+ <a href="/members/mrspike.5160076/" class="username" dir="auto" data-user-id="5160076" data-xf-init="member-tooltip" id="js-XFUniqueId11"><span itemprop="name"></span></a>
+ </h4>
+ <h5 class="userTitle message-userTitle" dir="auto" itemprop="jobTitle">Bathwater Drinker</h5>
+ <div class="userBanner banner--simp message-userBanner" itemprop="jobTitle"><span class="userBanner-before"></span><strong>⠀Simp⠀</strong><span class="userBanner-after"></span></div>
+ </div>
+
+ <div class="message-userExtras">
+ <dl class="pairs pairs--justified">
+ <dt>
+ <i class="fa--xf far fa-user">
+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#user"></use></svg>
+ </i>
+ </dt>
+ <dd>Aug 1, 2024</dd>
+ </dl>
+
+ <dl class="pairs pairs--justified">
+ <dt>
+ <i class="fa--xf far fa-comment">
+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#comment"></use></svg>
+ </i>
+ </dt>
+ <dd>38</dd>
+ </dl>
+
+ <dl class="pairs pairs--justified">
+ <dt>
+ <i class="fa--xf far fa-thumbs-up">
+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#thumbs-up"></use></svg>
+ </i>
+ </dt>
+ <dd>3,783</dd>
+ </dl>
+ </div>
+
+ <span class="message-userArrow"></span>
+ </section>
+ </div>
+
<div class="message-cell message-cell--main">
<div class="message-main js-quickEditTarget">
<header class="message-attribution message-attribution--split">
<ul class="message-attribution-main listInline">
<li class="u-concealed">
- <a href="/threads/fanfan.33077/post{id}" rel="nofollow" itemprop="url">
+ <a href="/threads/fanfan.33077/post-{id}" rel="nofollow" itemprop="url">
<time
class="u-dt"
dir="auto"
@@ -739,7 +1040,7 @@ POST_TEMPLATE = """
data-timestamp="1749508210"
data-date="Jun 9, 2025"
data-time="5:30 PM"
- data-short="7d"
+ data-short="12d"
title="Jun 9, 2025 at 5:30 PM"
itemprop="datePublished"
>
@@ -748,13 +1049,54 @@ POST_TEMPLATE = """
</a>
</li>
</ul>
+
+ <ul class="message-attribution-opposite message-attribution-opposite--list">
+ <li>
+ <a href="/threads/fanfan.33077/post-{id}" class="message-attribution-gadget" data-xf-init="share-tooltip" data-href="/posts/{id}/share" aria-label="Share" rel="nofollow" id="js-XFUniqueId41">
+ <i class="fa--xf far fa-share-alt">
+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#share-alt"></use></svg>
+ </i>
+ </a>
+ </li>
+
+ <li class="u-hidden js-embedCopy">
+ <a
+ href="javascript:"
+ data-xf-init="copy-to-clipboard"
+ data-copy-text='<div class="js-xf-embed" data-url="https://xenforocomunity.com" data-content="post-{id}"></div><script defer src="https://xenforocomunity.com/js/xf/external_embed.js?_v=dc874496"></script>'
+ data-success="Embed code HTML copied to clipboard."
+ class=""
+ >
+ <i class="fa--xf far fa-code">
+ <svg xmlns="http://www.w3.org/2000/svg" role="img" aria-hidden="true"><use href="/data/local/icons/regular.svg?v=1750411220#code"></use></svg>
+ </i>
+ </a>
+ </li>
+
+ <li>
+ <a
+ href="/posts/{id}/bookmark"
+ class="bookmarkLink message-attribution-gadget bookmarkLink--highlightable"
+ title="Add bookmark"
+ data-xf-click="bookmark-click"
+ data-label=".js-bookmarkText"
+ data-sk-bookmarked="addClass:is-bookmarked, titleAttr:sync"
+ data-sk-bookmarkremoved="removeClass:is-bookmarked, titleAttr:sync"
+ >
+ <span class="js-bookmarkText u-srOnly">Add bookmark</span>
+ </a>
+ </li>
+
+ <li>
+ <a href="/threads/fanfan.33077/post-{id}" rel="nofollow">
+ #282
+ </a>
+ </li>
+ </ul>
</header>
- <div class="message-attachments">
- {message_attachments}
- </div>
<div class="message-content js-messageContent">
- <div class="message-userContent lbContainer js-lbContainer" data-lb-id="post{id}" data-lb-caption-desc="cyberdrop-dl-patched · Jun 9, 2025 at 5:30 PM">
+ <div class="message-userContent lbContainer js-lbContainer" data-lb-id="post-{id}" data-lb-caption-desc=" · Jun 9, 2025 at 5:30 PM">
<article class="message-body js-selectToQuote">
{message_body}
<div class="js-selectToQuoteEnd">&nbsp;</div>
@@ -762,13 +1104,17 @@ POST_TEMPLATE = """
</div>
<aside class="message-signature">
- <div class="bbWrapper">Can't post to Bunkr. Mirrors are always appreciated.</div>
+ <div class="bbWrapper">Cant post to Bunkr. Mirrors are always appreciated.</div>
</aside>
</div>
+ <div class="message-attachments">
+ {message_attachments}
+ </div>
+
<footer class="message-footer">
<div class="message-microdata" itemprop="interactionStatistic" itemtype="https://schema.org/InteractionCounter" itemscope="">
- <meta itemprop="userInteractionCount" content="154" />
+ <meta itemprop="userInteractionCount" content="160" />
<meta itemprop="interactionType" content="https://schema.org/LikeAction" />
</div>
@@ -780,7 +1126,7 @@ POST_TEMPLATE = """
data-reaction-id="1"
data-xf-init="reaction"
data-reaction-list="< .js-post | .js-reactionsList"
- id="js-XFUniqueId13"
+ id="js-XFUniqueId12"
>
<i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="Like" title="Like" />
<span class="reaction-text js-reactionText"><bdi>Like</bdi></span>
@@ -790,7 +1136,14 @@ POST_TEMPLATE = """
Quote
</a>
- <a href="/threads/fanfan.33077/reply?quote={id}" class="actionBar-action actionBar-action--reply" title="Reply, quoting this message" rel="nofollow" data-xf-click="quote" data-quote-href="/posts/{id}/quote">
+ <a
+ href="/threads/fanfan.33077/reply?quote={id}"
+ class="actionBar-action actionBar-action--reply"
+ title="Reply, quoting this message"
+ rel="nofollow"
+ data-xf-click="quote"
+ data-quote-href="/posts/{id}/quote"
+ >
Reply
</a>
</div>
@@ -813,14 +1166,14 @@ POST_TEMPLATE = """
</span>
</li>
<li>
- <span class="reaction reaction--small reaction--91" data-reaction-id="91">
- <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PepeClown" title="PepeClown" />
+ <span class="reaction reaction--small reaction--50" data-reaction-id="50">
+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PeepoWew" title="PeepoWew" />
</span>
</li>
</ul>
<span class="u-srOnly">Reactions:</span>
- <a class="reactionsBar-link" href="/posts/{id}/reactions" data-xf-click="overlay" data-cache="false" rel="nofollow"><bdi>xigxagxion</bdi>, <bdi>Feistee</bdi>, <bdi>kradpmis</bdi> and 140 others</a>
+ <a class="reactionsBar-link" href="/posts/{id}/reactions" data-xf-click="overlay" data-cache="false" rel="nofollow"><bdi>DDDICKS</bdi>, <bdi>cachow</bdi>, <bdi>StubbornOne</bdi> and 145 others</a>
</div>
<div class="js-historyTarget message-historyTarget toggleTarget" data-href="trigger-href"></div>