Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: rework xenforo (#1079)
NTFSvolume69819543a249
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex eaf3152e9..8a95d86a6 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -91,18 +91,7 @@ from .twpornstars import TwPornstarsCrawlerfrom .vipr_dot_im import ViprImCrawlerfrom .wetransfer import WeTransferCrawlerfrom .xbunkr import XBunkrCrawler-from .xenforo import (- AllPornComixCrawler,- BellazonCrawler,- CelebForumCrawler,- F95ZoneCrawler,- LeakedModelsCrawler,- NudoStarCrawler,- SimpCityCrawler,- SocialMediaGirlsCrawler,- TitsInTopsCrawler,- XBunkerCrawler,-)+from .xenforo import XF_CRAWLERS, SimpCityCrawlerfrom .xhamster import XhamsterCrawlerfrom .xxxbunker import XXXBunkerCrawlerfrom .yandex_disk import YandexDiskCrawler@@ -110,6 +99,7 @@ from .youjizz import YouJizzCrawlerALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler")}ALL_CRAWLERS = ALL_CRAWLERS - {Crawler}+ALL_CRAWLERS.update(XF_CRAWLERS)DEBUG_CRAWLERS = {SimpCityCrawler, BunkrAlbumsIOCrawler, MegaNzCrawler, FikFapCrawler}CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERSdiff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex bbf7edd9b..67078ba6f 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -199,7 +199,7 @@ class CheveretoCrawler(Crawler, is_abc=True):name_index = url.parts.index(found_part) + 1name = url.parts[name_index]item_id = name.rsplit(".")[-1]- new_parts = url.parts[1:name_index] + (item_id,)+ new_parts = (*url.parts[1:name_index], item_id)new_path = "/" + "/".join(new_parts)return item_id, self.parse_url(new_path, url.origin())diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pynew file mode 100644index 000000000..7409ee832--- /dev/null+++ b/cyberdrop_dl/crawlers/_forum.py@@ -0,0 +1,80 @@+from __future__ import annotations++from abc import abstractmethod+from typing import TYPE_CHECKING, ClassVar, final++from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem+++class ForumCrawler(Crawler, is_abc=True):+ SUPPORTS_THREAD_RECURSION: ClassVar[bool] = False++ @property+ def scrape_single_forum_post(self) -> bool:+ return self.manager.config_manager.settings_data.download_options.scrape_single_forum_post++ @property+ def max_thread_depth(self) -> int:+ return self.manager.config_manager.settings_data.download_options.maximum_thread_depth++ @classmethod+ @abstractmethod+ def is_attachment(cls, url: AbsoluteHttpURL) -> bool: ...++ @abstractmethod+ async def forum(self, scrape_item: ScrapeItem) -> None: ...++ @abstractmethod+ async def thread(self, scrape_item: ScrapeItem) -> None: ...++ @abstractmethod+ async def post(self, scrape_item: ScrapeItem) -> None: ...++ @error_handling_wrapper+ async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:+ if link == self.PRIMARY_URL:+ return+ if self.is_attachment(link):+ return await self.handle_internal_link(scrape_item.create_new(link))+ if self.PRIMARY_URL.host in scrape_item.url.host and self.stop_thread_recursion(scrape_item):+ msg = f"Skipping nested thread URL {scrape_item.url} found on {scrape_item.origin}"+ return self.log(msg)+ new_scrape_item = scrape_item.copy()+ new_scrape_item.type = None+ new_scrape_item.reset_childen()+ self.handle_external_links(new_scrape_item)+ scrape_item.add_children()++ @error_handling_wrapper+ async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:+ # This logic is for discourse+ # if len(scrape_item.url.parts) < 5 and not scrape_item.url.suffix:+ # return+ filename, ext = self.get_filename_and_ext(scrape_item.url.name)+ scrape_item.add_to_parent_title("Attachments")+ scrape_item.part_of_album = True+ await self.handle_file(scrape_item.url, scrape_item, filename, ext)++ @final+ def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:+ if (+ not self.SUPPORTS_THREAD_RECURSION+ or not self.max_thread_depth+ or (len(scrape_item.parent_threads) > self.max_thread_depth)+ ):+ return True+ return False++ @final+ async def write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:+ if not last_post_url or last_post_url == thread_url:+ return+ await self.manager.log_manager.write_last_post_log(last_post_url)+++MessageBoardCrawler = ForumCrawlerdiff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex a9efdb4e6..53bb28eff 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -27,6 +27,7 @@ from cyberdrop_dl.utils.utilities import (get_download_path,get_filename_and_ext,is_absolute_http_url,+ is_blob_or_svg,parse_url,remove_file_id,sort_dict,@@ -73,7 +74,7 @@ def create_task_id(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, Ctry:if not self.SKIP_PRE_CHECK:pre_check_scrape_item(scrape_item)- return await func(scrape_item, **kwargs)+ return await func(scrape_item, **kwargs) # type: ignoreexcept ValueError:log(f"Scrape Failed: {UNKNOWN_URL_PATH_MSG}: {scrape_item.url}", 40)self.manager.progress_manager.scrape_stats_progress.add_failure(UNKNOWN_URL_PATH_MSG)@@ -189,7 +190,7 @@ class Crawler(ABC):if item.url.path_qs not in self.scraped_items:log(f"{scrape_prefix}: {item.url}", 20)self.scraped_items.append(item.url.path_qs)- await create_task_id(self.fetch)(self, item)+ await create_task_id(self.fetch)(self, item) # type: ignoreelse:log(f"Skipping {item.url} as it has already been scraped", 10)@@ -401,10 +402,6 @@ class Crawler(ABC):If provided, it will be used as a filter, to only yield items that has not been downloaded before"""album_results = results or {}- def is_embedded_image(link: str) -> bool:- """Checks if the link is an embedded image URL."""- return link.startswith("data:image") or link.startswith("blob:")-for tag in css.iselect(soup, selector):link_str: str | None = css.get_attr_or_none(tag, attribute)if not link_str:@@ -416,7 +413,7 @@ class Crawler(ABC):thumb_str: str | None = css.get_attr_or_none(t_tag, "src")else:thumb_str = None- thumb = self.parse_url(thumb_str) if thumb_str and not is_embedded_image(thumb_str) else None+ thumb = self.parse_url(thumb_str) if thumb_str and not is_blob_or_svg(thumb_str) else Noneyield thumb, linkdef iter_children(diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 1db26ff9b..bc43aa427 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -93,7 +93,7 @@ class MegaNzCrawler(Crawler):file_key[2] ^ file_key[6],file_key[3] ^ file_key[7],)- iv: tuple[int, ...] = file_key[4:6] + (0, 0)+ iv: tuple[int, ...] = (*file_key[4:6], 0, 0)meta_mac: tuple[int, ...] = file_key[6:8]file = FileTuple(file_id, DecryptData(iv, k, meta_mac))await self.proccess_file(scrape_item, file)diff --git a/cyberdrop_dl/crawlers/xenforo/__init__.py b/cyberdrop_dl/crawlers/xenforo/__init__.pyindex 7b0f60d57..1216fb82f 100644--- a/cyberdrop_dl/crawlers/xenforo/__init__.py+++ b/cyberdrop_dl/crawlers/xenforo/__init__.py@@ -10,16 +10,18 @@ from .titsintops import TitsInTopsCrawlerfrom .xbunker import XBunkerCrawlerfrom .xenforo import XenforoCrawler-__all__ = [- "AllPornComixCrawler",- "BellazonCrawler",- "CelebForumCrawler",- "F95ZoneCrawler",- "LeakedModelsCrawler",- "NudoStarCrawler",- "SimpCityCrawler",- "SocialMediaGirlsCrawler",- "TitsInTopsCrawler",- "XBunkerCrawler",- "XenforoCrawler",-]+XF_CRAWLERS: set[type[XenforoCrawler]] = {+ AllPornComixCrawler,+ BellazonCrawler,+ CelebForumCrawler,+ F95ZoneCrawler,+ LeakedModelsCrawler,+ NudoStarCrawler,+ SimpCityCrawler,+ SocialMediaGirlsCrawler,+ TitsInTopsCrawler,+ XBunkerCrawler,+}++CRAWLERS_MAP = {c.__name__: c for c in XF_CRAWLERS}+__all__ = list(CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]diff --git a/cyberdrop_dl/crawlers/xenforo/allporncomix.py b/cyberdrop_dl/crawlers/xenforo/allporncomix.pyindex 82f938736..71bb4c0db 100644--- a/cyberdrop_dl/crawlers/xenforo/allporncomix.py+++ b/cyberdrop_dl/crawlers/xenforo/allporncomix.py@@ -4,7 +4,7 @@ from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerclass AllPornComixCrawler(XenforoCrawler):@@ -12,10 +12,3 @@ class AllPornComixCrawler(XenforoCrawler):DOMAIN: ClassVar[str] = "allporncomix"FOLDER_DOMAIN: ClassVar[str] = "AllPornComix"login_required = False- post_selectors = PostSelectors(- content=Selector("div[class=bbWrapper]"),- images=Selector("img[class*=bbImage]", "data-src"),- date=Selector("time", "datetime"),- attachments=Selector("section[class=message-attachments] .attachmentList .file .file-preview", "href"),- )- selectors = XenforoSelectors(posts=post_selectors)diff --git a/cyberdrop_dl/crawlers/xenforo/bellazon.py b/cyberdrop_dl/crawlers/xenforo/bellazon.pyindex b83c766b2..037ac7fab 100644--- a/cyberdrop_dl/crawlers/xenforo/bellazon.py+++ b/cyberdrop_dl/crawlers/xenforo/bellazon.py@@ -6,26 +6,28 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+_post_selectors = PostSelectors(+ article="div[class*=ipsComment_content]",+ attachments=Selector("a[class*=ipsAttachLink]", "href"),+ content=Selector("div.cPost_contentWrap"),+ id=Selector("data-commentid", "data-commentid"),+ images=Selector("a[class*=ipsAttachLink_image]", "href"),+ videos=Selector("video.ipsEmbeddedVideo source", "src"),+)++# TODO: Bellazon uses Invision, not Xenforo. The selectors are completely different.+# See: https://github.com/jbsparrow/CyberDropDownloader/pull/1079#issuecomment-2982845352class BellazonCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.bellazon.com/main/")DOMAIN: ClassVar[str] = "bellazon"FOLDER_DOMAIN: ClassVar[str] = "Bellazon"- thread_url_part = "topic"login_required = False- post_selectors = PostSelectors(- element="div[class*=ipsComment_content]",- content=Selector("div[class=cPost_contentWrap]"),- attachments=Selector("a[class*=ipsAttachLink]", "href"),- images=Selector("a[class*=ipsAttachLink_image]", "href"),- videos=Selector("video[class=ipsEmbeddedVideo] source", "src"),- date=Selector("time", "datetime"),- number=Selector("data-commentid", "data-commentid"),- )- selectors = XenforoSelectors(- posts=post_selectors,++ XF_THREAD_URL_PART = "topic"+ XF_POST_URL_PART_NAME = "comment-"+ XF_SELECTORS = XenforoSelectors(+ posts=_post_selectors,title=Selector("span.ipsType_break.ipsContained span"),- next_page=Selector("li[class=ipsPagination_next] a", "href"),- post_name="comment-",+ next_page=Selector("li.ipsPagination_next a[href]", "href"),)- ATTACHMENT_URL_PARTS = "attachments", "uploads"diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.pyindex 95970a8f9..fde678e24 100644--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py@@ -1,26 +1,18 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors--if TYPE_CHECKING:- from yarl import URL+from .xenforo import XenforoCrawlerclass CelebForumCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://celebforum.to")- post_selectors = PostSelectors(- date=Selector("time", "data-time"),- images=Selector("a[class*=js-lbImage]", "href"),- )- selectors = XenforoSelectors(posts=post_selectors)DOMAIN: ClassVar[str] = "celebforum"FOLDER_DOMAIN: ClassVar[str] = "CelebForum"- def filter_link(self, link: URL) -> URL | None:+ def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:if link.host == self.PRIMARY_URL.host:if all(part in link.parts for part in ["data", "attachments"]): # Thumbnailsreturn Nonediff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.pyindex 87823a409..21a00abfa 100644--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py@@ -5,41 +5,30 @@ from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils.utilities import error_handling_wrapper-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerif TYPE_CHECKING:from yarl import URL- from cyberdrop_dl.data_structures.url_objects import ScrapeItem++_confirmation_data = ({"xhr": "1", "download": "1"},)class F95ZoneCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://f95zone.to")- post_selectors = PostSelectors(- date=Selector("time", "data-time"),- number=Selector("a[class=u-concealed]", "href"),- )- selectors = XenforoSelectors(posts=post_selectors)DOMAIN: ClassVar[str] = "f95zone"FOLDER_DOMAIN: ClassVar[str] = "F95Zone"- def is_confirmation_link(self, link: URL) -> bool:- return "masked" in link.parts or super().is_confirmation_link(link)-@error_handling_wrapper- async def handle_confirmation_link(self, link: URL, *, origin: ScrapeItem | None = None) -> URL | None:+ async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:"""Override to handle protected link confirmation."""async with self.request_limiter:- data = ({"xhr": "1", "download": "1"},)- JSON_Resp = await self.client.post_data(self.DOMAIN, link, data=data)+ json_resp = await self.client.post_data(self.DOMAIN, link, data=_confirmation_data)- if JSON_Resp["status"] == "ok":- return self.parse_url(JSON_Resp["msg"])- return None+ if json_resp["status"] == "ok":+ return self.parse_url(json_resp["msg"])- def filter_link(self, link: URL) -> URL:+ def filter_link(self, link: AbsoluteHttpURL) -> URL:if "thumb" in link.parts:- parts = [x for x in link.parts if x not in ("thumb", "/")]- new_path = "/".join(parts)- return link.with_path(new_path)+ return link.with_path(link.path.replace("/thumb/", ""))return linkdiff --git a/cyberdrop_dl/crawlers/xenforo/leakedmodels.py b/cyberdrop_dl/crawlers/xenforo/leakedmodels.pyindex b3ac7ba56..5b1b70900 100644--- a/cyberdrop_dl/crawlers/xenforo/leakedmodels.py+++ b/cyberdrop_dl/crawlers/xenforo/leakedmodels.py@@ -4,14 +4,10 @@ from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerclass LeakedModelsCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://leakedmodels.com/forum/")- post_selectors = PostSelectors(- date=Selector("time", "data-time"),- )- selectors = XenforoSelectors(posts=post_selectors)DOMAIN: ClassVar[str] = "leakedmodels"FOLDER_DOMAIN: ClassVar[str] = "LeakedModels"diff --git a/cyberdrop_dl/crawlers/xenforo/nudostar.py b/cyberdrop_dl/crawlers/xenforo/nudostar.pyindex 10e660e8f..96b394837 100644--- a/cyberdrop_dl/crawlers/xenforo/nudostar.py+++ b/cyberdrop_dl/crawlers/xenforo/nudostar.py@@ -4,15 +4,10 @@ from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerclass NudoStarCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://nudostar.com/forum/")- post_selectors = PostSelectors(- date=Selector("time", "data-time"),- number=Selector("a[class=u-concealed]", "href"),- )- selectors = XenforoSelectors(posts=post_selectors)DOMAIN: ClassVar[str] = "nudostar"FOLDER_DOMAIN: ClassVar[str] = "NudoStar"diff --git a/cyberdrop_dl/crawlers/xenforo/simpcity.py b/cyberdrop_dl/crawlers/xenforo/simpcity.pyindex fc988f704..2ccb879d7 100644--- a/cyberdrop_dl/crawlers/xenforo/simpcity.py+++ b/cyberdrop_dl/crawlers/xenforo/simpcity.py@@ -9,7 +9,7 @@ from .xenforo import XenforoCrawlerclass SimpCityCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://simpcity.su")- login_required = FalseDOMAIN: ClassVar[str] = "simpcity"FOLDER_DOMAIN: ClassVar[str] = "SimpCity"- session_cookie_name = "dontlikebots_user"+ XF_USER_COOKIE_NAME = "dontlikebots_user"+ login_required = Falsediff --git a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.pyindex 9ba210386..fb6d28e0d 100644--- a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py+++ b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py@@ -4,16 +4,10 @@ from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerclass SocialMediaGirlsCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://forums.socialmediagirls.com")DOMAIN: ClassVar[str] = "socialmediagirls"FOLDER_DOMAIN: ClassVar[str] = "SocialMediaGirls"- post_selectors = PostSelectors(- content=Selector("div[class=bbWrapper]"),- images=Selector("img[class*=bbImage]", "data-src"),- date=Selector("time", "data-time"),- )- selectors = XenforoSelectors(posts=post_selectors)diff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.pyindex eb0fe06c2..f7332633c 100644--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py@@ -2,12 +2,10 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, ClassVar-from yarl import URL-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import css-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerif TYPE_CHECKING:from bs4 import Tag@@ -17,13 +15,9 @@ class TitsInTopsCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://titsintops.com/phpBB2")DOMAIN: ClassVar[str] = "titsintops"FOLDER_DOMAIN: ClassVar[str] = "TitsInTops"- post_selectors = PostSelectors(- images=Selector("a[class*=file-preview]", "href"),- )- selectors = XenforoSelectors(posts=post_selectors)- def filter_link(self, link: URL) -> URL:- return URL(+ def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL:+ return AbsoluteHttpURL(str(link).replace("index.php%3F", "index.php/").replace("index.php?", "index.php/")@@ -33,13 +27,11 @@ class TitsInTopsCrawler(XenforoCrawler):def pre_filter_link(self, link: str) -> str:return link.replace("index.php?", "index.php/").replace("index.php%3F", "index.php/")- def is_valid_post_link(self, link_obj: Tag) -> bool:- is_image = link_obj.select_one("img")+ def is_not_image_or_attachment(self, link_obj: Tag) -> bool:text = css.get_text(link_obj)if "view attachment" in text.lower():return False- title: str | None = css.get_attr_or_none(link_obj, "title")+ title = css.get_attr_no_error(link_obj, "title")if title and "permanent link" in title.lower():return False- link_str: str | None = css.get_attr_or_none(link_obj, self.selectors.posts.links.element)- return not (is_image and self.is_attachment(link_str))+ return super().is_not_image_or_attachment(link_obj)diff --git a/cyberdrop_dl/crawlers/xenforo/xbunker.py b/cyberdrop_dl/crawlers/xenforo/xbunker.pyindex a1cbac419..dd4320249 100644--- a/cyberdrop_dl/crawlers/xenforo/xbunker.py+++ b/cyberdrop_dl/crawlers/xenforo/xbunker.py@@ -4,16 +4,10 @@ from typing import ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors+from .xenforo import XenforoCrawlerclass XBunkerCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://xbunker.nu/")DOMAIN: ClassVar[str] = "xbunker"FOLDER_DOMAIN: ClassVar[str] = "XBunker"- post_selectors = PostSelectors(- content=Selector("div[class=bbWrapper]"),- images=Selector("img[class*=bbImage], a[class*=js-lbImage]", "data-src"),- date=Selector("time", "data-time"),- )- selectors = XenforoSelectors(posts=post_selectors)diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex 0954c82c4..5c4b36b42 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -3,133 +3,129 @@from __future__ import annotationsimport asyncio-import contextlib+import dataclassesimport re-from dataclasses import astuple, dataclass-from functools import cached_property, singledispatchmethodfrom typing import TYPE_CHECKING, ClassVarfrom bs4 import BeautifulSoup, Tag-from cyberdrop_dl.crawlers.crawler import Crawler-from cyberdrop_dl.data_structures.url_objects import FORUM, AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.exceptions import InvalidURLError, LoginError, ScrapeError+from cyberdrop_dl.crawlers._forum import ForumCrawler+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeErrorfrom cyberdrop_dl.utils import css-from cyberdrop_dl.utils.logger import log, log_debug+from cyberdrop_dl.utils.dates import TimeStampfrom cyberdrop_dl.utils.utilities import (error_handling_wrapper,get_text_between,is_absolute_http_url,- remove_trailing_slash,+ is_blob_or_svg,)if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Sequence+ from collections.abc import AsyncGenerator, Iterablefrom aiohttp_client_cache.response import AnyResponse- from yarl import URL- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL-from cyberdrop_dl.crawlers.crawler import SupportedPaths+ from cyberdrop_dl.crawlers.crawler import SupportedPaths+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem-HTTP_URL_REGEX_STR = r"https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z]{2,12}\b([-a-zA-Z0-9@:%_+.~#?&/=]*)"-HTTP_URL_REGEX = re.compile(HTTP_URL_REGEX_STR)+HTTP_URL_REGEX = re.compile(r"https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z]{2,12}\b([-a-zA-Z0-9@:%_+.~#?&/=]*)")+LINK_TRASH_MAPPING = {+ ".th.": ".",+ ".md.": ".",+ "ifr": "watch",+}-FINAL_PAGE_SELECTOR = "li.pageNav-page a"-CURRENT_PAGE_SELECTOR = "li.pageNav-page.pageNav-page--current a"+Selector = css.CssAttributeSelector+LAST_PAGE_SELECTOR = Selector("li.pageNav-page a:last-of-type", "href")+CURRENT_PAGE_SELECTOR = Selector("li.pageNav-page.pageNav-page--current a", "href")-@dataclass(frozen=True, slots=True)-class Selector:- element: str- attribute: str = ""- @property- def astuple(self) -> tuple[str, str]:- return astuple(self)[email protected](frozen=True, slots=True)+class PostSelectors:+ article: str = "article.message[id*=post]" # the entire html of the post (comments, attachments, user avatar, signature, etc...)--@dataclass(frozen=True, slots=True)-class PostSelectors(Selector):- element: str = "div[class*=message-main]"- attachments: Selector = Selector("section[class=message-attachments] a ", "href")- content: Selector = Selector("div[class*=message-userContent]")+ attachments: Selector = Selector(".message-attachments a[href]", "href")+ content: Selector = Selector(".message-userContent") # text, links and images (NO attachments)date: Selector = Selector("time", "data-timestamp")- embeds: Selector = Selector("span[data-s9e-mediaembed-iframe]", "data-s9e-mediaembed-iframe")- iframe: Selector = Selector("iframe[class=saint-iframe]", "href")- images: Selector = Selector("img[class*=bbImage]", "src")- links: Selector = Selector("a", "href")- number: Selector = Selector("li[class=u-concealed] a", "href")+ embeds: Selector = Selector("iframe", "src")+ id: Selector = Selector(".u-concealed a[href]", "href") # TODO: This needs a better selector+ images: Selector = Selector("img.bbImage", "src")+ lazy_load_embeds: Selector = Selector('[class*=iframe][onclick*="loadMedia(this, \'//"]', "onclick")+ links: Selector = Selector(":any-link", "href")videos: Selector = Selector("video source", "src")- redgifs: Selector = Selector("div[class*='iframe'][onclick*=\"loadMedia(this, '//redgifs\"]", "onclick")-@dataclass(frozen=True, slots=True)[email protected](frozen=True, slots=True)class XenforoSelectors:- next_page: Selector = Selector("a[class*=pageNav-jump--next]", "href")+ confirmation_button: Selector = Selector("a[class*=button--cta][href]", "href")+ next_page: Selector = Selector("a[class*=pageNav-jump--next][href]", "href")posts: PostSelectors = PostSelectors()- title: Selector = Selector("h1[class=p-title-value]")- title_trash: Selector = Selector("span")quotes: Selector = Selector("blockquote")- post_name: str = "post-"+ title_trash: Selector = Selector("span")+ title: Selector = Selector("h1[class*=p-title-value]")+ last_page: Selector = LAST_PAGE_SELECTOR+ current_page: Selector = CURRENT_PAGE_SELECTOR-@dataclass(frozen=True)[email protected](frozen=True, slots=True, order=True)class ForumPost:- soup: Tag- selectors: PostSelectors- post_name: str = "post-"-- @cached_property- def content(self) -> Tag:- return css.select_one(self.soup, self.selectors.content.element)-- @cached_property- def date(self) -> int | None:- if date_str := css.select_one_get_attr_or_none(self.content, *self.selectors.date.astuple):- return int(date_str)-- @cached_property- def number(self) -> int:- if self.selectors.number.element == self.selectors.number.attribute:- number_str: str = css.get_attr(self.soup, self.selectors.number.element)- return int(number_str)-- number_str = css.select_one_get_attr(self.soup, *self.selectors.number.astuple)- return int(number_str.split("/")[-1].split(self.post_name)[-1])+ id: int+ date: TimeStamp+ article: Tag = dataclasses.field(compare=False)+ content: Tag = dataclasses.field(compare=False)++ @staticmethod+ def new(article: Tag, selectors: PostSelectors, post_name: str = "post") -> ForumPost:+ content = css.select_one(article, selectors.content.element)+ date = TimeStamp(int(css.select_one_get_attr(article, *selectors.date)))+ if selectors.id.element == selectors.id.attribute:+ post_id = int(css.get_attr(article, selectors.id.element))+ else:+ id_str = css.select_one_get_attr(article, *selectors.id)+ post_id = int(id_str.rsplit(post_name, 1)[-1].replace("-", ""))+ return ForumPost(post_id, date, article, content)-@dataclass(frozen=True, slots=True)-class ThreadInfo:[email protected](frozen=True, slots=True)+class Thread:+ id: intname: str- id_: intpage: int- post: int+ post_id: int | Noneurl: AbsoluteHttpURL- complete_url: URL-@dataclass(frozen=True, slots=True)-class ForumThreadPage:- thread: ThreadInfo- posts: Sequence[Tag]+DEFAULT_XF_SELECTORS = XenforoSelectors()+KNOWN_THREAD_PART_NAMES = "thread", "topic", "tema"+KNOWN_THREAD_PART_NAMES = ",".join(f"{part},{part}s" for part in KNOWN_THREAD_PART_NAMES).split(",")++def _escape(strings: Iterable[str]) -> str:+ return r"\|".join(strings)-class XenforoCrawler(Crawler, is_abc=True):++class XenforoCrawler(ForumCrawler, is_abc=True):+ XF_ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Attachments": ("/attachments/...", "/data/..."),- "Threads": ("/threads/<thread_name>", "/posts/<post_id>", "/goto/<post_id>"),+ "Attachments": f"/({_escape(XF_ATTACHMENT_URL_PARTS)})/...",+ "Threads": (+ f"/({_escape(KNOWN_THREAD_PART_NAMES)})/<thread_name_and_id>",+ "/posts/<post_id>",+ "/goto/<post_id>",+ ),+ "**NOTE**": "base crawler: Xenforo",}+ SUPPORTS_THREAD_RECURSION: ClassVar[bool] = True+ XF_SELECTORS = DEFAULT_XF_SELECTORS+ XF_POST_URL_PART_NAME = "post-"+ XF_PAGE_URL_PART_NAME = "page-"+ XF_THREAD_URL_PART = "threads"+ XF_USER_COOKIE_NAME = "xf_user"+ XF_ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"login_required = True- selectors = XenforoSelectors()- POST_NAME = "post-"- PAGE_NAME = "page-"- thread_url_part = "threads"- session_cookie_name = "xf_user"-- ATTACHMENT_URL_PARTS = "attachments", "data"- ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"def __post_init__(self) -> None:self.scraped_threads = set()@@ -139,16 +135,10 @@ class XenforoCrawler(Crawler, is_abc=True):login_url = self.PRIMARY_URL / "login"await self.login_setup(login_url)- async def is_not_last_page(response: AnyResponse) -> bool:- soup = BeautifulSoup(await response.text(), "html.parser")- try:- last_page = int(soup.select(FINAL_PAGE_SELECTOR)[-1].text.split("page-")[-1])- current_page = int(soup.select(CURRENT_PAGE_SELECTOR)[0].text.split("page-")[-1])- except (AttributeError, IndexError):- return False- return current_page != last_page+ self.register_cache_filter(self.PRIMARY_URL, check_is_not_last_page)- self.register_cache_filter(self.PRIMARY_URL, is_not_last_page)+ def get_filename_and_ext(self, filename: str) -> tuple[str, str]:+ return super().get_filename_and_ext(filename, forum=True)async def fetch(self, scrape_item: ScrapeItem) -> None:if not self.logged_in and self.login_required:@@ -157,21 +147,27 @@ class XenforoCrawler(Crawler, is_abc=True):scrape_item.url = self.parse_url(url)if self.is_attachment(scrape_item.url):return await self.handle_internal_link(scrape_item)- if self.thread_url_part in scrape_item.url.parts:- return await self.thread(scrape_item)- if self.is_confirmation_link(scrape_item.url):- url = await self.handle_confirmation_link(scrape_item.url)- if url: # If there was an error, this will be None- # This could end up back in here if the URL goes to another thread- scrape_item.url = url- return self.handle_external_links(scrape_item)- if any(p in scrape_item.url.parts for p in ("goto", "posts")):- return await self.redirect(scrape_item)+ if is_confirmation_link(scrape_item.url):+ return await self.follow_confirmation_link(scrape_item)++ thread_part_index = len(self.PRIMARY_URL.parts)+ match scrape_item.url.parts[thread_part_index:]:+ case [thread_part, thread_name_and_id, *_] if thread_part in KNOWN_THREAD_PART_NAMES:+ return await self.thread(scrape_item, thread_name_and_id)+ case ["goto" | "posts", *_]:+ return await self.follow_redirect_w_get(scrape_item)raise ValueError+ async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:+ url = await self.resolve_confirmation_link(scrape_item.url)+ if url: # If there was an error, this will be None+ scrape_item.url = url+ # This could end up back in here if the URL goes to another thread+ return self.handle_external_links(scrape_item)+@error_handling_wrapper- async def redirect(self, scrape_item: ScrapeItem) -> None:+ async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:async with self.request_limiter:response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)assert is_absolute_http_url(response.url)@@ -179,52 +175,69 @@ class XenforoCrawler(Crawler, is_abc=True):self.manager.task_group.create_task(self.run(scrape_item))@error_handling_wrapper- async def thread(self, scrape_item: ScrapeItem) -> None:- scrape_item.set_type(FORUM, self.manager)- thread = self.get_thread_info(scrape_item.url)- title = None- last_post_url = thread.url+ async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:+ head = await self.client.get_head(self.DOMAIN, scrape_item.url)+ redirect = head.get("location")+ if not redirect:+ raise ScrapeError(422)+ scrape_item.url = self.parse_url(redirect)+ self.manager.task_group.create_task(self.run(scrape_item))++ async def forum(self, scrape_item: ScrapeItem) -> None:+ raise NotImplementedError++ @error_handling_wrapper+ async def thread(self, scrape_item: ScrapeItem, thread_name_and_id: str) -> None:+ scrape_item.setup_as_forum("")+ thread = parse_thread(+ scrape_item.url, thread_name_and_id, self.XF_PAGE_URL_PART_NAME, self.XF_POST_URL_PART_NAME+ )if thread.url in self.scraped_threads:return+scrape_item.parent_threads.add(thread.url)+ if self.scrape_single_forum_post and not thread.post_id:+ msg = "`--scrape-single-forum-post` is `True`, but the provided URL has no post id"+ raise ScrapeError("User Error", msg)++ title: str = ""self.scraped_threads.add(thread.url)+ last_post_url = thread.urlasync for soup in self.thread_pager(scrape_item):if not title:- title_block = css.select_one(soup, self.selectors.title.element)try:- trash = title_block.select(self.selectors.title_trash.element)- except AttributeError as e:- log_debug("Got an unprocessable soup", 40, exc_info=e)- raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e-- for element in trash:- element.decompose()-- title = self.create_title(title_block.get_text().replace("\n", ""), thread_id=thread.id_)+ title = self.create_title(get_post_title(soup, self.XF_SELECTORS), thread_id=thread.id)+ except ScrapeError as e:+ self.log_debug("Got an unprocessable soup", 40, exc_info=e)+ raisescrape_item.add_to_parent_title(title)- posts = soup.select(self.selectors.posts.element)- forum_thread_page = ForumThreadPage(thread, posts)- continue_scraping, last_post_url = self.process_thread_page(scrape_item, forum_thread_page)+ continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)if not continue_scraping:breakawait self.write_last_forum_post(thread.url, last_post_url)- def process_thread_page(self, scrape_item: ScrapeItem, forum_page: ForumThreadPage) -> tuple[bool, URL]:+ def process_thread_page(+ self, scrape_item: ScrapeItem, thread: Thread, soup: BeautifulSoup+ ) -> tuple[bool, AbsoluteHttpURL]:continue_scraping = False- thread = forum_page.threadpost_url = thread.url- for post in forum_page.posts:- current_post = ForumPost(post, selectors=self.selectors.posts, post_name=self.POST_NAME)- continue_scraping, scrape_post = self.check_post_number(thread.post, current_post.number)- date = current_post.date- post_string = f"{self.POST_NAME}{current_post.number}"- post_url = thread.url / post_string- if scrape_post:- new_scrape_item = scrape_item.create_new(thread.url, possible_datetime=date, add_parent=post_url)+ for article in soup.select(self.XF_SELECTORS.posts.article):+ current_post = ForumPost.new(article, self.XF_SELECTORS.posts, self.XF_POST_URL_PART_NAME)+ continue_scraping, scrape_this_post = check_post_id(+ thread.post_id, current_post.id, self.scrape_single_forum_post+ )+ if scrape_this_post:+ post_url = thread.url / f"{self.XF_POST_URL_PART_NAME}{current_post.id}"+ new_scrape_item = scrape_item.create_new(+ thread.url, possible_datetime=current_post.date, add_parent=post_url+ )self.manager.task_group.create_task(self.post(new_scrape_item, current_post))- scrape_item.add_children()+ try:+ scrape_item.add_children()+ except MaxChildrenError:+ breakif not continue_scraping:break@@ -233,228 +246,132 @@ class XenforoCrawler(Crawler, is_abc=True):@error_handling_wrapperasync def post(self, scrape_item: ScrapeItem, post: ForumPost) -> None:scrape_item.setup_as_post("")- post_title = self.create_separate_post_title(None, str(post.number), post.date)+ post_title = self.create_separate_post_title(None, str(post.id), post.date)scrape_item.add_to_parent_title(post_title)- posts_scrapers = [self.attachments, self.embeds, self.images, self.links, self.videos, self.hidden_redgifs]- for scraper in posts_scrapers:+ for scraper in (+ self._attachments,+ self._embeds,+ self._images,+ self._links,+ self._videos,+ self._lazy_load_embeds,+ ):await scraper(scrape_item, post)- async def links(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = post.selectors.links+ async def _links(self, scrape_item: ScrapeItem, post: ForumPost) -> None:+ selector = self.XF_SELECTORS.posts.linkslinks = post.content.select(selector.element)- links = [link for link in links if self.is_valid_post_link(link)]+ links = [link for link in links if self.is_not_image_or_attachment(link)]await self.process_children(scrape_item, links, selector.attribute)- async def images(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = post.selectors.images+ async def _images(self, scrape_item: ScrapeItem, post: ForumPost) -> None:+ selector = self.XF_SELECTORS.posts.imagesimages = post.content.select(selector.element)await self.process_children(scrape_item, images, selector.attribute)- async def videos(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = post.selectors.videos- iframe_selector = post.selectors.iframe+ async def _videos(self, scrape_item: ScrapeItem, post: ForumPost) -> None:+ selector = self.XF_SELECTORS.posts.videosvideos = post.content.select(selector.element)- videos.extend(post.content.select(iframe_selector.element))await self.process_children(scrape_item, videos, selector.attribute)- async def embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = post.selectors.embeds+ async def _embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:+ selector = self.XF_SELECTORS.posts.embedsembeds = post.content.select(selector.element)await self.process_children(scrape_item, embeds, selector.attribute, embeds=True)- async def attachments(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = post.selectors.attachments- attachments = post.content.select(selector.attribute)+ async def _attachments(self, scrape_item: ScrapeItem, post: ForumPost) -> None:+ selector = self.XF_SELECTORS.posts.attachments+ attachments = post.article.select(selector.element)await self.process_children(scrape_item, attachments, selector.attribute)- async def hidden_redgifs(self, scrape_item: ScrapeItem, post: ForumPost) -> None:- selector = post.selectors.redgifs+ async def _lazy_load_embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:+ selector = self.XF_SELECTORS.posts.lazy_load_embedsfor redgif in css.iselect(post.content, selector.element):link_str = get_text_between(css.get_attr(redgif, selector.attribute), "loadMedia(this, '", "')")- await self.process_child(scrape_item, link_str)+ await self.process_child(scrape_item, link_str, embeds=True)async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:- """Generator of forum thread pages."""-- def get_next_page(soup: BeautifulSoup) -> str | None:- select, attr = self.selectors.next_page.astuple- next_page = css.select_one_get_attr_or_none(soup, selector=select, attribute=attr)- if next_page:- return self.pre_filter_link(next_page)-- async for soup in self._web_pager(scrape_item.url, get_next_page):+ async for soup in self._web_pager(scrape_item.url, self.get_next_page):yield soup+ def get_next_page(self, soup: BeautifulSoup) -> str | None:+ if next_page := css.select_one_get_attr_or_none(soup, *self.XF_SELECTORS.next_page):+ return self.pre_filter_link(next_page)++ @error_handling_wrapperasync def process_children(- self, scrape_item: ScrapeItem, links: list[Tag], selector: str, *, embeds: bool = False+ self, scrape_item: ScrapeItem, links: list[Tag], attribute: str, *, embeds: bool = False) -> None:- for link_obj in links:- link_tag = link_obj.get(selector)- if isinstance(link_tag, Tag):- if link_tag.parent and (data_simp := link_tag.parent.get("data-simp")) and "init" in data_simp:- continue-- link_str: str | None = css.get_attr_or_none(link_obj, "href")- elif isinstance(link_tag, str):- link_str = link_tag- else:- continue-- if not link_str:- continue-- assert isinstance(link_str, str)- if embeds:- link_str = self.extract_embed_url(link_str)-- if not link_str or link_str.startswith("data:image/svg"):- continue-- await self.process_child(scrape_item, link_str)+ for link_tag in links:+ if link_str := css.get_attr_no_error(link_tag, attribute):+ await self.process_child(scrape_item, link_str, embeds=embeds)@error_handling_wrapper- async def process_child(self, scrape_item: ScrapeItem, link_str: str) -> None:- link = await self.get_absolute_link(link_str)+ async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:+ link_str_ = pre_process_child(link_str, embeds)+ if not link_str_:+ return+ link = await self.get_absolute_link(link_str_)link = self.filter_link(link)if not link:return- new_scrape_item = scrape_item.create_new(link)- await self.handle_link(new_scrape_item)- scrape_item.add_children()+ await self.handle_link(scrape_item, link)- @singledispatchmethod- def is_attachment(self, link: AbsoluteHttpURL) -> bool:+ def is_attachment(self, link: AbsoluteHttpURL | str) -> bool:if not link:return False- parts = self.ATTACHMENT_URL_PARTS- hosts = self.ATTACHMENT_HOSTS- return any(p in link.parts for p in parts) or any(h in link.host for h in hosts)-- @is_attachment.register- def _(self, link_str: str) -> bool:- if not link_str:- return False- link = self.parse_url(link_str)- return self.is_attachment(link)+ if isinstance(link, str):+ link = self.parse_url(link)+ by_parts = len(link.parts) > 2 and any(p in link.parts for p in self.XF_ATTACHMENT_URL_PARTS)+ by_host = any(host in link.host for host in self.XF_ATTACHMENT_HOSTS)+ return by_parts or by_hostasync def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpURL | None:if isinstance(link, str):- absolute_link = self.str_to_url(link)+ absolute_link = self.parse_url(clean_link_str(link))else:absolute_link = link- if self.is_confirmation_link(absolute_link):- absolute_link = await self.handle_confirmation_link(absolute_link)+ if is_confirmation_link(absolute_link):+ return await self.resolve_confirmation_link(absolute_link)return absolute_link- def str_to_url(self, link: str) -> AbsoluteHttpURL:- link_str = link- text_to_replace = [(".th.", "."), (".md.", "."), ("ifr", "watch")]- for old, new in text_to_replace:- link_str = link_str.replace(old, new)- return self.parse_url(link_str)-- @error_handling_wrapper- async def handle_link(self, scrape_item: ScrapeItem) -> None:- if not scrape_item.url or scrape_item.url == self.PRIMARY_URL:- return- if not scrape_item.url.host:- raise InvalidURLError("url has no host")- if self.is_attachment(scrape_item.url):- return await self.handle_internal_link(scrape_item)- if self.PRIMARY_URL.host in scrape_item.url.host and self.stop_thread_recursion(scrape_item):- origin = scrape_item.parents[0]- return log(f"Skipping nested thread URL {scrape_item.url} found on {origin}", 10)- scrape_item.set_type(None, self.manager)- self.handle_external_links(scrape_item)-- @error_handling_wrapper- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:- """Handles internal links."""- scrape_item.url = remove_trailing_slash(scrape_item.url)-- if scrape_item.url.name.isdigit():- head = await self.client.get_head(self.DOMAIN, scrape_item.url)- redirect = head.get("location")- if not redirect:- raise ScrapeError(422)- scrape_item.url = self.parse_url(redirect)- self.manager.task_group.create_task(self.run(scrape_item))- return-- filename, ext = self.get_filename_and_ext(scrape_item.url.name, forum=True)- scrape_item.add_to_parent_title("Attachments")- scrape_item.part_of_album = True- await self.handle_file(scrape_item.url, scrape_item, filename, ext)-@error_handling_wrapper- async def handle_confirmation_link(self, link: URL) -> AbsoluteHttpURL | None:- """Handles link confirmation."""+ async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:async with self.request_limiter:soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)- confirm_button = soup.select_one("a[class*=button--cta]")+ selector = self.XF_SELECTORS.confirmation_button+ confirm_button = soup.select_one(selector.element)if not confirm_button:return- link_str: str = css.get_attr(confirm_button, "href")+ link_str: str = css.get_attr(confirm_button, selector.attribute)link_str = link_str.split('" class="link link--internal', 1)[0]new_link = self.parse_url(link_str)return await self.get_absolute_link(new_link)- def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:- max_thread_depth = self.manager.config_manager.settings_data.download_options.maximum_thread_depth- if not max_thread_depth:- return True- if len(scrape_item.parent_threads) > max_thread_depth:- return True- return False-- def check_post_number(self, post_number: int, current_post_number: int) -> tuple[bool, bool]:- """Checks if the program should scrape the current post.-- Returns (continue_scraping, scrape_post)"""- scrape_single_forum_post = self.manager.config_manager.settings_data.download_options.scrape_single_forum_post- scrape_post = continue_scraping = True- if scrape_single_forum_post:- if not post_number or post_number == current_post_number:- continue_scraping = False- else:- scrape_post = False-- elif post_number and post_number > current_post_number:- scrape_post = False-- return continue_scraping, scrape_post-- async def write_last_forum_post(self, thread_url: URL, last_post_url: URL | None) -> None:- if not last_post_url or last_post_url == thread_url:- return- await self.manager.log_manager.write_last_post_log(last_post_url)-- def is_valid_post_link(self, link_obj: Tag) -> bool:- is_image = link_obj.select_one("img")- link_str: str = css.get_attr_or_none(link_obj, self.selectors.posts.links.element)- return not (is_image or self.is_attachment(link_str))+ async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:+ """Handles internal links."""+ slug = scrape_item.url.name or scrape_item.url.parent.name+ if slug.isdigit():+ return await self.follow_redirect_w_head(scrape_item)- def is_confirmation_link(self, link: URL) -> bool:- return any(keyword in link.path for keyword in ("link-confirmation",))+ return await super().handle_internal_link(scrape_item)- def extract_embed_url(self, embed_str: str) -> str | None:- embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")- if match := re.search(HTTP_URL_REGEX, embed_str):- return match.group(0).replace("www.", "")- return embed_str+ def is_not_image_or_attachment(self, link_obj: Tag) -> bool:+ is_image = link_obj.select_one("img")+ if link_str := css.get_attr_no_error(link_obj, self.XF_SELECTORS.posts.links.element):+ return not (is_image or self.is_attachment(link_str))+ return not is_imagedef pre_filter_link(self, link: str) -> str:return link- def filter_link(self, link: URL | None) -> URL | None:+ def filter_link(self, link: AbsoluteHttpURL | None) -> AbsoluteHttpURL | None:return link- """ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""-@error_handling_wrapper- async def login_setup(self, login_url: URL) -> None:+ async def login_setup(self, login_url: AbsoluteHttpURL) -> None:host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)- session_cookie = host_cookies.get(self.session_cookie_name)+ session_cookie = host_cookies.get(self.XF_USER_COOKIE_NAME)session_cookie = session_cookie.value if session_cookie else Nonemsg = f"No cookies found for {self.FOLDER_DOMAIN}"if not session_cookie and self.login_required:@@ -470,14 +387,11 @@ class XenforoCrawler(Crawler, is_abc=True):raise LoginError(message=msg)msg += " Scraping without an account"- log(msg, 30)+ self.log(msg, 30)@error_handling_wrapper- async def forum_login(self, login_url: URL, session_cookie: str, username: str, password: str) -> None:+ async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:"""Logs in to a forum."""-- attempt = 0- retries = wait_time = 5manual_login = username and passwordmissing_credentials = not (manual_login or session_cookie)if missing_credentials:@@ -488,73 +402,163 @@ class XenforoCrawler(Crawler, is_abc=True):cookies = {"xf_user": session_cookie}self.update_cookies(cookies)+ credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}+ await self.xf_try_login(login_url, credentials, retries=5)++ async def xf_try_login(+ self,+ login_url: AbsoluteHttpURL,+ credentials: dict[str, str],+ retries: int,+ wait_time: int | None = None,+ ) -> None:+ # Try from cookiestext, logged_in = await self.check_login_with_request(login_url)if logged_in:self.logged_in = Truereturn- credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}-- def prepare_login_data(resp_text) -> dict:- soup = BeautifulSoup(resp_text, "html.parser")- inputs = soup.select("form input")- data: dict = {elem["name"]: elem["value"] for elem in inputs if elem.get("name") and elem.get("value")}- return data | credentials-+ wait_time = wait_time or retries+ attempt = 0while attempt < retries:- with contextlib.suppress(TimeoutError):+ try:attempt += 1await asyncio.sleep(wait_time)- data = prepare_login_data(text)+ data = parse_login_form(text) | credentials_ = await self.client._post_data(self.DOMAIN, login_url / "login", data=data, cache_disabled=True)await asyncio.sleep(wait_time)text, logged_in = await self.check_login_with_request(login_url)if logged_in:self.logged_in = Truereturn-+ except TimeoutError:+ continuemsg = f"Failed to login on {self.FOLDER_DOMAIN} after {retries} attempts"raise LoginError(message=msg)- async def check_login_with_request(self, login_url: URL) -> tuple[str, bool]:+ async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""- def get_thread_info(self, url: AbsoluteHttpURL) -> ThreadInfo:- name_index = url.parts.index(self.thread_url_part) + 1- name, id_ = get_thread_name_and_id(url, name_index)- thread_url = get_thread_canonical_url(url, name_index)- page, post = get_thread_page_and_post(url, name_index, self.PAGE_NAME, self.POST_NAME)- return ThreadInfo(name, id_, page, post, thread_url, url)+def parse_thread(url: AbsoluteHttpURL, thread_name_and_id: str, page_part_name: str, post_part_name: str) -> Thread:+ name_index = url.parts.index(thread_name_and_id)+ name, id_ = parse_thread_name_and_id(thread_name_and_id)+ page, post_id = get_thread_page_and_post(url, name_index, page_part_name, post_part_name)+ canonical_url = get_thread_canonical_url(url, name_index)+ return Thread(id_, name, page, post_id, canonical_url)-def get_thread_name_and_id(url: URL, thread_name_index: int) -> tuple[str, int]:- thread_name, thread_id_str = url.parts[thread_name_index].rsplit(".", 1)- thread_id = int(thread_id_str)- return thread_name, thread_id+def parse_thread_name_and_id(thread_name_and_id: str) -> tuple[str, int]:+ try:+ name, id_str = thread_name_and_id.rsplit(".", 1)+ except ValueError:+ id_str, name = thread_name_and_id.split("-", 1)+ return name, int(id_str)def get_thread_canonical_url(url: AbsoluteHttpURL, thread_name_index: int) -> AbsoluteHttpURL:new_parts = url.parts[1 : thread_name_index + 1]new_path = "/".join(new_parts)- thread_url = url.with_path(new_path).with_fragment(None).with_query(None)- return thread_url+ return url.with_path(new_path)-def get_thread_page_and_post(url: URL, thread_name_index: int, page_name: str, post_name: str) -> tuple[int, int]:- post_or_page_index = thread_name_index + 1- extra_parts = set(url.parts[post_or_page_index:])- sections = {url.fragment} if url.fragment else set()- sections.update(extra_parts)+def get_thread_page_and_post(+ url: AbsoluteHttpURL, thread_name_index: int, page_name: str, post_name: str+) -> tuple[int, int | None]:+ extra_parts = url.parts[thread_name_index + 1 :]+ if url.fragment:+ extra_parts = *extra_parts, url.fragment- def find_number(search_value: str) -> int:- for sec in sections:+ def find_number(search_value: str) -> int | None:+ for sec in extra_parts:if search_value in sec:- return int(sec.replace(search_value, "").strip())- return 0+ return int(sec.replace(search_value, "").replace("-", "").strip())++ post_id = find_number(post_name)+ page_number = find_number(page_name) or 1+ return page_number, post_id+++async def check_is_not_last_page(response: AnyResponse) -> bool:+ soup = BeautifulSoup(await response.text(), "html.parser")+ return not is_last_page(soup)+++def is_last_page(soup: BeautifulSoup) -> bool:+ try:+ last_page = css.select_one_get_attr(soup, *LAST_PAGE_SELECTOR)+ current_page = css.select_one_get_attr(soup, *CURRENT_PAGE_SELECTOR)+ except (AttributeError, IndexError, css.SelectorError):+ return True+ return current_page == last_page+++def get_post_title(soup: BeautifulSoup, selectors: XenforoSelectors) -> str:+ try:+ title_block = css.select_one(soup, selectors.title.element)+ trash = title_block.select(selectors.title_trash.element)+ for element in trash:+ element.decompose()+ except (AttributeError, AssertionError, css.SelectorError) as e:+ raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e++ if title := " ".join(css.get_text(title_block).split()):+ return title+ raise ScrapeError(422)+++def extract_embed_url(embed_str: str) -> str:+ embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")+ if match := re.search(HTTP_URL_REGEX, embed_str):+ return match.group(0).replace("www.", "")+ return embed_str+++# TODO: move to parse_url+def clean_link_str(link: str) -> str:+ for old, new in LINK_TRASH_MAPPING.items():+ link = link.replace(old, new)+ return link+++def parse_login_form(resp_text: str) -> dict[str, str]:+ soup = BeautifulSoup(resp_text, "html.parser")+ inputs = soup.select("form:first-of-type input")+ data = {+ name: value+ for elem in inputs+ if (name := css.get_attr_or_none(elem, "name")) and (value := css.get_attr_or_none(elem, "value"))+ }+ if data:+ return data+ raise ScrapeError(422)+++def is_confirmation_link(link: AbsoluteHttpURL) -> bool:+ return "masked" in link.parts or "link-confirmation" in link.path+++def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]:+ """Checks if the program should scrape the current post.++ Returns (continue_scraping, scrape_this_post)"""+ if init_post_id:+ if init_post_id > current_post_id:+ return (True, False)+ elif init_post_id == current_post_id:+ return (not scrape_single_forum_post, True)+ else:+ return (not scrape_single_forum_post, not scrape_single_forum_post)++ assert not scrape_single_forum_post # We should have raised an exception early+ return True, True+++def pre_process_child(link_str: str, embeds: bool = False) -> str | None:+ assert isinstance(link_str, str)+ if embeds:+ link_str = extract_embed_url(link_str)- post_number = find_number(post_name)- page_number = find_number(page_name)- return page_number, post_number+ if link_str and not is_blob_or_svg(link_str):+ return link_strdiff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 2f7106dcd..f0e54da62 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -642,7 +642,7 @@ class MegaApi:if file["t"] == NodeType.FILE:file = cast("File", file)k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], key[3] ^ key[7])- file["iv"] = key[4:6] + (0, 0)+ file["iv"] = (*key[4:6], 0, 0)file["meta_mac"] = key[6:8]# folderelse:@@ -735,7 +735,7 @@ class MegaApi:elif node["t"] == NodeType.FOLDER:k = key- iv: U32IntSequence = key[4:6] + (0, 0)+ iv: U32IntSequence = (*key[4:6], 0, 0)meta_mac: U32IntTupleArray = key[6:8]attrs = decrypt_attr(base64_url_decode(node["a"]), k)diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex f0ba0e7f4..b32227426 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -1,10 +1,12 @@from __future__ import annotationsimport functools-from typing import TYPE_CHECKING, ParamSpec, TypeVar+from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVarimport bs4.css+from cyberdrop_dl.exceptions import ScrapeError+if TYPE_CHECKING:from collections.abc import Callable, Generator@@ -14,19 +16,34 @@ P = ParamSpec("P")R = TypeVar("R")+class SelectorError(ScrapeError):+ def __init__(self, message: str | None = None) -> None:+ super().__init__(422, message)+++class CssAttributeSelector(NamedTuple):+ element: str+ attribute: str = ""++ def __call__(self, soup: Tag) -> str:+ return select_one_get_attr(soup, self.element, self.attribute)++def not_none(func: Callable[P, R | None]) -> Callable[P, R]:@functools.wraps(func)def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:result = func(*args, **kwargs)- assert result is not None+ if result is None:+ raise SelectorErrorreturn resultreturn wrapper-def select_one(tag: Tag, selector: str) -> Tag:+@not_none+def select_one(tag: Tag, selector: str) -> Tag | None:"""Same as `tag.select_one` but asserts the result is not `None`"""- return not_none(tag.select_one)(selector)+ return tag.select_one(selector)def select_one_get_text(tag: Tag, selector: str, strip: bool = True) -> str:@@ -34,20 +51,34 @@ def select_one_get_text(tag: Tag, selector: str, strip: bool = True) -> str:return get_text(select_one(tag, selector), strip)+# TODO: Rename this to just get_attr+# get_attr_no_error should be get_attr_or_none. `or_none` implies no errordef get_attr_or_none(tag: Tag, attribute: str) -> str | None:- """Same as `tag.get(attribute)` but asserts the result is not multiple strings"""- value = tag.get(attribute)- assert not isinstance(value, list)+ """Same as `tag.get(attribute)` but asserts the result is a single str"""+ if attribute == "src":+ value = tag.get("data-src") or tag.get(attribute)+ else:+ value = tag.get(attribute)+ if isinstance(value, list):+ raise SelectorError(f"Expected a single value for {attribute = !r}, got multiple")return value+def get_attr_no_error(tag: Tag, attribute: str) -> str | None:+ try:+ return get_attr_or_none(tag, attribute)+ except Exception:+ return++def get_text(tag: Tag, strip: bool = True) -> str:return tag.get_text(strip=strip)-def get_attr(tag: Tag, attribute: str) -> str:+@not_none+def get_attr(tag: Tag, attribute: str) -> str | None:"""Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""- return not_none(get_attr_or_none)(tag, attribute)+ return get_attr_or_none(tag, attribute)def select_one_get_attr(tag: Tag, selector: str, attribute: str) -> str:diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 61e8ffe12..80255e0cb 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -56,6 +56,7 @@ R = TypeVar("R")TEXT_EDITORS = "micro", "nano", "vim" # Ordered by preferenceALLOWED_FILEPATH_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"+_BLOB_OR_SVG = ("data:", "blob:", "javascript:")subprocess_get_text = partial(subprocess.run, capture_output=True, text=True, check=False)@@ -82,8 +83,8 @@ class OGProperties(dict[str, str]):def error_handling_wrapper(- func: Callable[..., Coroutine[None, None, R]],-) -> Callable[..., Coroutine[None, None, R | None]]:+ func: Callable[P, Coroutine[None, None, R]],+) -> Callable[P, Coroutine[None, None, R | None]]:"""Wrapper handles errors for url scraping."""@wraps(func)@@ -613,6 +614,10 @@ def get_os_common_name() -> str:return default+def is_blob_or_svg(link: str) -> bool:+ return any(link.startswith(x) for x in _BLOB_OR_SVG)++log_cyan = partial(log_with_color, style="cyan", level=20)log_yellow = partial(log_with_color, style="yellow", level=20)log_green = partial(log_with_color, style="green", level=20)diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.mdindex 27db21293..1de47f7f0 100644--- a/docs/reference/cli-arguments.md+++ b/docs/reference/cli-arguments.md@@ -218,7 +218,7 @@ CLI-only options:browser_cookies:--auto-import, --no-auto-import- --browsers [BROWSERS ...]+ --browser BROWSER--sites [SITES ...]download_options:@@ -314,6 +314,7 @@ sorting:general:--allow-insecure-connections, --no-allow-insecure-connections+ --disable-crawlers [DISABLE_CRAWLERS ...]--enable-generic-crawler, --no-enable-generic-crawler--flaresolverr FLARESOLVERR--max-file-name-length MAX_FILE_NAME_LENGTHdiff --git a/docs/reference/supported-websites.md b/docs/reference/supported-websites.mdindex ab6f0c378..8316ca3c4 100644--- a/docs/reference/supported-websites.md+++ b/docs/reference/supported-websites.md@@ -28,100 +28,104 @@ Supported domains via Real-Debrid include `mega.nz`, `rapidgator`, `google drive## Supported sites-List of sites supported by cyberdrop-dl-patched as of version 6.9.1+List of sites supported by cyberdrop-dl-patched as of version 6.10.1-| Site | Primary URL | Supported Domains | Supported Paths |-|---------------------------|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-| 4chan | https://boards.4chan.org | 4chan | Board: <br> - `/<board>`<br>Thread: <br> - `/thread`<br> |-| 8Muses | https://comics.8muses.com | 8muses | Album: <br> - `/comics/album/...`<br> |-| AcidImg | https://acidimg.cc | acidimg.cc | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/upload/...`<br> |-| AllPornComix | https://forum.allporncomix.com | allporncomix | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| ArchiveBate | https://www.archivebate.store | archivebate | Video: <br> - `/watch/<video_id>`<br> |-| Bellazon | https://www.bellazon.com/main/ | bellazon | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| BestPrettyGirl | https://bestprettygirl.com/ | bestprettygirl.com | Collection: <br> - `category`<br> - `date`<br> - `tag`<br>Gallery: <br> - `/<gallery_name>/`<br> |-| Box | https://www.box.com | app.box.com | File or Folder: <br> - `app.box.com/s?sh=<share_code>`<br> |-| Bunkrr | https://bunkr.site | bunkr<br>bunkrr | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br>Video: <br> - `/v/...`<br> |-| BuzzHeavier | https://buzzheavier.com | buzzheavier.com | Direct Links<br> |-| Catbox | https://catbox.moe | files.catbox.moe<br>litter.catbox.moe | Direct Links<br> |-| CelebForum | https://celebforum.to | celebforum | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| Coomer | https://coomer.su | coomer | Direct Links<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |-| Cyberdrop | https://cyberdrop.me/ | cyberdrop | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br> |-| Cyberfile | https://cyberfile.me/ | cyberfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |-| DirtyShip | https://dirtyship.com | dirtyship | Categorie: <br> - `/category/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/gallery/.../...`<br>Tag: <br> - `/tag/...`<br>Video: <br> - `/<video_name>`<br> |-| DoodStream | https://doodstream.com/ | all3do.com<br>do7go.com<br>dood.re<br>dood.yt<br>doodcdn<br>doodstream<br>doodstream.co<br>vidply.com | Video: <br> - `/e/<video_id>`<br> |-| Dropbox | https://dropbox.com/ | dropbox | File: <br> - `/s/...`<br> - `/scl/fi/...`<br>Folder: <br> - `/scl/fo/...`<br> - `/sh/...`<br><br><br>**NOTES**<br> - Folders will be downloaded as a zip file. See: [--download-dropbox-folders-as-zip](https://script-ware.gitbook.io/cyberdrop-dl/reference/cli-arguments#download-dropbox-folders-as-zip)<br> |-| E-Hentai | https://e-hentai.org/ | e-hentai | Album: <br> - `/g/...`<br>File: <br> - `/s/...`<br> |-| E621 | https://e621.net | e621.net | Pools: <br> - `/pools/...`<br>Post: <br> - `/posts/...`<br>Tags: <br> - `/posts?tags=...`<br> |-| Erome | https://www.erome.com | erome | Album: <br> - `/a/...`<br>Profile: <br> - `/...`<br> |-| F95Zone | https://f95zone.to | f95zone | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| Fapello | https://fapello.su/ | fapello | Individual Post: <br> - `/.../...`<br>Model: <br> - `/...`<br> |-| Fileditch | https://fileditchfiles.me/ | fileditch | Direct Links<br> |-| FilesVC | https://files.vc | files.vc | Direct Links<br> |-| GoFile | https://gofile.io | gofile | Album: <br> - `/d/...`<br> |-| GoogleDrive | https://drive.google.com | docs.google<br>drive.google | Docs: <br> - `/document/d/<file_id>`<br>Files: <br> - `/d/<file_id>`<br>Folders: <br> - `/drive/folders/<folder_id>`<br>Sheets: <br> - `/spreadsheets/d/<file_id>`<br>Slides: <br> - `/presentation/d/<file_id>`<br> |-| HotPic | https://hotpic.cc | 2385290.xyz<br>hotpic | Album: <br> - `/album/...`<br>Image: <br> - `/i/...`<br> |-| Iceyfile | https://iceyfile.com/ | iceyfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |-| ImageBam | https://www.imagebam.com/ | imagebam | Album: <br> - `/view/...`<br>Direct Links<br>Image: <br> - `/view/...`<br> |-| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profiles: <br> - `/...`<br>Video: <br> - `/video/..`<br> |-| ImgBB | https://ibb.co | ibb.co<br>imgbb.co | Album: <br> - `/album/...`<br>Image: <br> - `/...`<br> |-| ImgBox | https://imgbox.com | imgbox | Album: <br> - `/g/...`<br>Direct Links<br>Image: <br> - `/...`<br> |-| Imgur | https://imgur.com/ | imgur | Album: <br> - `/a/...`<br>Direct Links<br>Gallery: <br> - `/gallery/...`<br>Image: <br> - `/...`<br> |-| Imx.to | https://imx.to | imx.to | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/t/...`<br> |-| IncestFlix | https://www.incestflix.com | incestflix | Tag: <br> - `/tag/...`<br>Video: <br> - `/watch/...`<br> |-| InfluencerBitches | https://influencerbitches.com | influencerbitches | Model: <br> - `/model/...`<br> |-| JPG5 | https://jpg5.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profile: <br> - `/<user_name>`<br> |-| Kemono | https://kemono.su | kemono | Direct Links<br>Discord Server: <br> - `/discord/<server_id>`<br>Discord Server Channel: <br> - `/discord/server/...#...`<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |-| LeakedModels | https://leakedmodels.com/forum/ | leakedmodels | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| Luscious | https://members.luscious.net | luscious | Album: <br> - `/albums/...`<br> |-| Mediafire | https://www.mediafire.com/ | mediafire | File: <br> - `/file/...`<br>Folder: <br> - `/folder/...`<br> |-| MixDrop | https://mixdrop.sb | mixdrop<br>mxdrop | File: <br> - `/e/<file_id>`<br> - `/f/<file_id>`<br> |-| Motherless | https://motherless.com | motherless | Group: <br> - `/g/<group_name>`<br> - `/gi/<image>`<br> - `/gv/<video>`<br>Image: <br> - `/...`<br>User: <br> - `/f/...`<br> - `/u/...`<br>Video: <br> - `pending`<br><br><br>**NOTES**<br> - Galleries are NOT supported<br> |-| Nekohouse | https://nekohouse.su | nekohouse | Direct Links<br>Individual Post: <br> - `/<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br> |-| NoodleMagazine | https://noodlemagazine.com | noodlemagazine | Search: <br> - `/video/<search_query`<br>Video: <br> - `/watch/<video_id>`<br> |-| NudoStar | https://nudostar.com/forum/ | nudostar | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| NudoStarTV | https://nudostar.tv/ | nudostar.tv | Model: <br> - `/models/...`<br> |-| OmegaScans | https://omegascans.org | omegascans | Chapter: <br> - `/series/.../...`<br>Direct Links<br>Series: <br> - `/series/...`<br> |-| OneDrive | https://onedrive.com/ | 1drv.ms<br>onedrive.live.com | Access Link: <br> - `https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>`<br>Share Link (anyone can access): <br> - `https://1drv.ms/b/<KEY>`<br> - `https://1drv.ms/f/<KEY>`<br> - `https://1drv.ms/t/<KEY>`<br> - `https://1drv.ms/u/<KEY>`<br> |-| PMVHaven | https://pmvhaven.com | pmvhaven | Category: <br> - `/category/...`<br>Music: <br> - `/music/...`<br>Playlist: <br> - `/playlist/...`<br>Search results: <br> - `/search/...`<br>Star: <br> - `/star/...`<br>Tag: <br> - `/tags/...`<br>Users: <br> - `/profile/...`<br>Video: <br> - `/video/...`<br> |-| PimpAndHost | https://pimpandhost.com/ | pimpandhost | Album: <br> - `/album/...`<br>Image: <br> - `/image/...`<br> |-| PixHost | https://pixhost.to/ | pixhost | Gallery: <br> - `/gallery/...`<br>Image: <br> - `/show/...`<br> |-| PixelDrain | https://pixeldrain.com | pixeldrain | File: <br> - `/u/...`<br>Folder: <br> - `/l/...`<br> |-| Pkmncards | https://pkmncards.com | pkmncards | Card: <br> - `/card/...`<br>Series: <br> - `/series/...`<br>Set: <br> - `/set/...`<br> |-| PornHub | https://www.pornhub.com | pornhub | Album: <br> - `/album/...`<br>Channel: <br> - `/channel/...`<br>Gif: <br> - `/gif/...`<br>Photo: <br> - `/photo/...`<br>Playlist: <br> - `/playlist/...`<br>Profile: <br> - `/model/...`<br> - `/pornstar/...`<br> - `/user/...`<br>Video: <br> - `/embed/<video_id>`<br> - `/view_video.php?viewkey=<video_id>`<br> |-| PornPics | https://pornpics.com | pornpics | Categories: <br> - `/categories/....`<br>Channels: <br> - `/channels/...`<br>Direct Links<br>Gallery: <br> - `/galleries/...`<br>Pornstars: <br> - `/pornstars/...`<br>Search: <br> - `/?q=<query>`<br>Tags: <br> - `/tags/...`<br> |-| Porntrex | https://www.porntrex.com | porntrex | Album: <br> - `/albums/...`<br>Category: <br> - `/categories/...`<br>Model: <br> - `/models/...`<br>Playlist: <br> - `/playlists/...`<br>Search: <br> - `/search/...`<br>Tag: <br> - `/tags/...`<br>User: <br> - `/members/...`<br>Video: <br> - `/video/...`<br> |-| PostImg | https://postimages.org/ | postimg | Album: <br> - `/gallery/...`<br>Direct Links<br>Image: <br> - `/...`<br> |-| RealBooru | https://realbooru.com | realbooru | File: <br> - `?id=...`<br>Tags: <br> - `?tags=...`<br> |-| RedGifs | https://redgifs.com/ | redgifs | User: <br> - `/users/<user>`<br>Video: <br> - `/watch/...`<br> |-| Reddit | https://www.reddit.com/ | redd.it<br>reddit | Direct Links<br>Subreddit:: <br> - `/r/<subreddit>`<br>User: <br> - `/u/<user>`<br> - `/user/<user>`<br> - `/user/<user>/...`<br> |-| Rule34Vault | https://rule34vault.com | rule34vault | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |-| Rule34Video | https://rule34video.com/ | rule34video | Members: <br> - `/members/...`<br>Models: <br> - `/models/...`<br>Search: <br> - `/search/...`<br>Tags: <br> - `/tags/...`<br>Video: <br> - `/video/<video_id>/<video_name>`<br> - `/videos/<video_id>/<video_name>`<br> |-| Rule34XXX | https://rule34.xxx | rule34.xxx | File: <br> - `?id=...`<br>Tag: <br> - `?tags=...`<br> |-| Rule34XYZ | https://rule34.xyz | rule34.xyz | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |-| Saint | https://saint2.su/ | saint | Album: <br> - `/a/...`<br>Direct Links<br>Video: <br> - `/d/...`<br> - `/embed/...`<br> |-| Scrolller | https://scrolller.com | scrolller | Subreddit: <br> - `/r/...`<br> |-| SendNow | https://send.now/ | send.now | Direct Links<br> |-| SendVid | https://sendvid.com/ | sendvid | Direct Links<br>Embeds: <br> - `/embed/...`<br>Videos: <br> - `/...`<br> |-| Sex.com | https://sex.com | sex | Shorts Profiles: <br> - `/shorts/<profile>`<br> |-| SocialMediaGirls | https://forums.socialmediagirls.com | socialmediagirls | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| SpankBang | https://spankbang.com/ | spankbang | Playlist: <br> - `/playlist/<playlist-id>`<br>Video: <br> - `/embed/<video_id>`<br> - `/play/<video_id>`<br> - `/playlist/<playlist-id>-<video_id>`<br> - `/video/<video_id>`<br> |-| Streamable | https://streamable.com | streamable | Video: <br> - `/...`<br> |-| TWPornStars | https://www.twpornstars.com | indiantw.com<br>twanal.com<br>twgaymuscle.com<br>twgays.com<br>twlesbian.com<br>twmilf.com<br>twonfans.com<br>twpornstars.com<br>twteens.com<br>twtiktoks.com | Photo: <br> - `/...`<br> |-| ThisVid | https://thisvid.com | thisvid | Albums: <br> - `/albums/<album_name>`<br>Categories: <br> - `/categories/...`<br>Image: <br> - `/albums/<album_name>/<image_name>`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/?q=...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |-| TikTok | https://tiktok.com/ | tiktok | Photo: <br> - `/@<user>/photo/<photo_id>`<br>User: <br> - `/@<user>`<br>Video: <br> - `/@<user>/video/<video_id>`<br> |-| TitsInTops | https://titsintops.com/phpBB2 | titsintops | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| Tokyomotion | https://www.tokyomotion.net | tokyomotion | Albums: <br> - `/album/<album_id>`<br> - `/user/<user>/albums/`<br>Photo: <br> - `/photo/<photo_id>`<br> - `/user/<user>/favorite/photos`<br>Playlist: <br> - `/user/<user>/favorite/videos`<br>Profiles: <br> - `/user/<user>`<br>Search Results: <br> - `/search?...`<br>Video: <br> - `/video/<video_id>`<br> |-| Toonily | https://toonily.com | toonily | Chapter: <br> - `/webtoon/.../...`<br>Direct Links<br>Webtoon: <br> - `/webtoon/...`<br> |-| TwitterImages | https://twimg.com/ | twimg | Photo: <br> - `/...`<br> |-| Vipr.im | https://vipr.im | vipr.im | Image: <br> - `/...`<br>Thumbnail: <br> - `/th/...`<br> |-| WeTransfer | https://wetransfer.com/ | we.tl<br>wetransfer.com | Direct links: <br> - `download.wetransfer.com/...`<br>Public link: <br> - `wetransfer.com/<file_id>/<security_hash>`<br>Share Link: <br> - `wetransfer.com/<file_id>/<recipient_id>/<security_hash>`<br>Short Link: <br> - `we.tl/<id>`<br> |-| XBunker | https://xbunker.nu/ | xbunker | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |-| XBunkr | https://xbunkr.com | xbunkr | Albums: <br> - `/a/...`<br>Direct Links<br> |-| XXXBunker | https://xxxbunker.com | xxxbunker | Search: <br> - `/search/...`<br>Video: <br> - `/<video_id>`<br> |-| YandexDisk | https://disk.yandex.com.tr/ | disk.yandex<br>yadi.sk | Files: <br> - `/d/<folder_id>/<file_name>`<br>Folder: <br> - `/d/<folder_id>`<br><br><br>**NOTES**<br> - Does NOT support nested folders<br> |-| YouJizz | https://www.youjizz.com/ | youjizz | Video: <br> - `/video/<video_id>/...`<br> - `/video/embed/<video_id>`<br> |-| aShemaleTube | https://www.ashemaletube.com | ashemaletube | Model: <br> - `/creators/...`<br> - `/model/...`<br> - `/pornstars/...`<br>Playlist: <br> - `/playlists/...`<br>User: <br> - `/profiles/...`<br>Video: <br> - `/videos/...`<br> |-| ePorner | https://www.eporner.com/ | eporner | Categories: <br> - `/cat/...`<br>Channels: <br> - `/channel/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/photo/...`<br>Pornstar: <br> - `/pornstar/...`<br>Profile: <br> - `/profile/...`<br>Search: <br> - `/search/...`<br>Video: <br> - `/<video_name>-<video-id>`<br> - `/embed/<video_id>`<br> - `/hd-porn/<video_id>`<br> |-| nHentai | https://nhentai.net/ | nhentai.net | Collections: <br> - `artist`<br> - `character`<br> - `favorites`<br> - `group`<br> - `parody`<br> - `search`<br> - `tag`<br>Gallery: <br> - `/g/<gallery_id>`<br> |-| xHamster | https://xhamster.com/ | xhamster | Creator: <br> - `/creatos/<creator_name>`<br>Gallery: <br> - `photos/gallery/<gallery_name>`<br>User: <br> - `/users/<user_name>`<br>Video: <br> - `/<video_title>`<br> |+| Site | Primary URL | Supported Domains | Supported Paths |+|---------------------------|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|+| 4chan | https://boards.4chan.org | 4chan | Board: <br> - `/<board>`<br>Thread: <br> - `/thread`<br> |+| 8Muses | https://comics.8muses.com | 8muses | Album: <br> - `/comics/album/...`<br> |+| AcidImg | https://acidimg.cc | acidimg.cc | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/upload/...`<br> |+| AllPornComix | https://forum.allporncomix.com | allporncomix | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| ArchiveBate | https://www.archivebate.store | archivebate | Video: <br> - `/watch/<video_id>`<br> |+| Bellazon | https://www.bellazon.com/main/ | bellazon | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| BestPrettyGirl | https://bestprettygirl.com/ | bestprettygirl.com | Collection: <br> - `category`<br> - `date`<br> - `tag`<br>Gallery: <br> - `/<gallery_name>/`<br> |+| Box | https://www.box.com | app.box.com | File or Folder: <br> - `app.box.com/s?sh=<share_code>`<br> |+| Bunkrr | https://bunkr.site | bunkr<br>bunkrr | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br>Video: <br> - `/v/...`<br> |+| BuzzHeavier | https://buzzheavier.com | buzzheavier.com | Direct Links<br> |+| Camwhores.tv | https://www.camwhores.tv | camwhores.tv | Albums: <br> - `/albums/<album_name>`<br>Categories: <br> - `/categories/...`<br>Image: <br> - `/albums/<album_name>/<image_name>`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/?q=...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |+| Catbox | https://catbox.moe | files.catbox.moe<br>litter.catbox.moe | Direct Links<br> |+| CelebForum | https://celebforum.to | celebforum | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| Coomer | https://coomer.su | coomer | Direct Links<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |+| Cyberdrop | https://cyberdrop.me/ | cyberdrop | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br> |+| Cyberfile | https://cyberfile.me/ | cyberfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |+| DirtyShip | https://dirtyship.com | dirtyship | Categorie: <br> - `/category/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/gallery/.../...`<br>Tag: <br> - `/tag/...`<br>Video: <br> - `/<video_name>`<br> |+| DoodStream | https://doodstream.com/ | all3do.com<br>do7go.com<br>dood.re<br>dood.yt<br>doodcdn<br>doodstream<br>doodstream.co<br>vidply.com | Video: <br> - `/e/<video_id>`<br> |+| Dropbox | https://dropbox.com/ | dropbox | File: <br> - `/s/...`<br> - `/scl/fi/...`<br>Folder: <br> - `/scl/fo/...`<br> - `/sh/...`<br><br><br>**NOTES**<br> - Folders will be downloaded as a zip file. See: [--download-dropbox-folders-as-zip](https://script-ware.gitbook.io/cyberdrop-dl/reference/cli-arguments#download-dropbox-folders-as-zip)<br> |+| E-Hentai | https://e-hentai.org/ | e-hentai | Album: <br> - `/g/...`<br>File: <br> - `/s/...`<br> |+| E621 | https://e621.net | e621.net | Pools: <br> - `/pools/...`<br>Post: <br> - `/posts/...`<br>Tags: <br> - `/posts?tags=...`<br> |+| Erome | https://www.erome.com | erome | Album: <br> - `/a/...`<br>Profile: <br> - `/...`<br> |+| F95Zone | https://f95zone.to | f95zone | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| Fapello | https://fapello.su/ | fapello | Individual Post: <br> - `/.../...`<br>Model: <br> - `/...`<br> |+| Fileditch | https://fileditchfiles.me/ | fileditch | Direct Links<br> |+| FilesVC | https://files.vc | files.vc | Direct Links<br> |+| GoFile | https://gofile.io | gofile | Album: <br> - `/d/...`<br> |+| GoogleDrive | https://drive.google.com | docs.google<br>drive.google | Docs: <br> - `/document/d/<file_id>`<br>Files: <br> - `/d/<file_id>`<br>Folders: <br> - `/drive/folders/<folder_id>`<br>Sheets: <br> - `/spreadsheets/d/<file_id>`<br>Slides: <br> - `/presentation/d/<file_id>`<br> |+| Hitomi.la | https://hitomi.la | hitomi.la | Collection: <br> - `/artist/...`<br> - `/character/...`<br> - `/group/...`<br> - `/series/...`<br> - `/tag/...`<br> - `/type/...`<br>Gallery: <br> - `/anime/...`<br> - `/cg/...`<br> - `/doujinshi/...`<br> - `/galleries/...`<br> - `/gamecg/...`<br> - `/imageset/...`<br> - `/manga/...`<br> - `/reader/...`<br>Search: <br> - `/search.html?<query>`<br> |+| HotPic | https://hotpic.cc | 2385290.xyz<br>hotpic | Album: <br> - `/album/...`<br>Image: <br> - `/i/...`<br> |+| Iceyfile | https://iceyfile.com/ | iceyfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |+| ImageBam | https://www.imagebam.com/ | imagebam | Album: <br> - `/view/...`<br>Direct Links<br>Image: <br> - `/view/...`<br> |+| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profiles: <br> - `/...`<br>Video: <br> - `/video/..`<br> |+| ImgBB | https://ibb.co | ibb.co<br>imgbb.co | Album: <br> - `/album/...`<br>Image: <br> - `/...`<br> |+| ImgBox | https://imgbox.com | imgbox | Album: <br> - `/g/...`<br>Direct Links<br>Image: <br> - `/...`<br> |+| Imgur | https://imgur.com/ | imgur | Album: <br> - `/a/...`<br>Direct Links<br>Gallery: <br> - `/gallery/...`<br>Image: <br> - `/...`<br> |+| Imx.to | https://imx.to | imx.to | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/t/...`<br> |+| IncestFlix | https://www.incestflix.com | incestflix | Tag: <br> - `/tag/...`<br>Video: <br> - `/watch/...`<br> |+| InfluencerBitches | https://influencerbitches.com | influencerbitches | Model: <br> - `/model/...`<br> |+| JPG5 | https://jpg5.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profile: <br> - `/<user_name>`<br> |+| Kemono | https://kemono.su | kemono | Direct Links<br>Discord Server: <br> - `/discord/<server_id>`<br>Discord Server Channel: <br> - `/discord/server/...#...`<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |+| LeakedModels | https://leakedmodels.com/forum/ | leakedmodels | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| Luscious | https://members.luscious.net | luscious | Album: <br> - `/albums/...`<br> |+| Mediafire | https://www.mediafire.com/ | mediafire | File: <br> - `/file/...`<br>Folder: <br> - `/folder/...`<br> |+| MissAV | https://missav.ws | missav | Video: <br> - `/...`<br> |+| MixDrop | https://mixdrop.sb | mixdrop<br>mxdrop | File: <br> - `/e/<file_id>`<br> - `/f/<file_id>`<br> |+| Motherless | https://motherless.com | motherless | Group: <br> - `/g/<group_name>`<br> - `/gi/<image>`<br> - `/gv/<video>`<br>Image: <br> - `/...`<br>User: <br> - `/f/...`<br> - `/u/...`<br>Video: <br> - `pending`<br><br><br>**NOTES**<br> - Galleries are NOT supported<br> |+| Nekohouse | https://nekohouse.su | nekohouse | Direct Links<br>Individual Post: <br> - `/<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br> |+| NoodleMagazine | https://noodlemagazine.com | noodlemagazine | Search: <br> - `/video/<search_query`<br>Video: <br> - `/watch/<video_id>`<br> |+| NudoStar | https://nudostar.com/forum/ | nudostar | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| NudoStarTV | https://nudostar.tv/ | nudostar.tv | Model: <br> - `/models/...`<br> |+| OmegaScans | https://omegascans.org | omegascans | Chapter: <br> - `/series/.../...`<br>Direct Links<br>Series: <br> - `/series/...`<br> |+| OneDrive | https://onedrive.com/ | 1drv.ms<br>onedrive.live.com | Access Link: <br> - `https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>`<br>Share Link (anyone can access): <br> - `https://1drv.ms/b/<KEY>`<br> - `https://1drv.ms/f/<KEY>`<br> - `https://1drv.ms/t/<KEY>`<br> - `https://1drv.ms/u/<KEY>`<br> |+| PMVHaven | https://pmvhaven.com | pmvhaven | Category: <br> - `/category/...`<br>Music: <br> - `/music/...`<br>Playlist: <br> - `/playlist/...`<br>Search results: <br> - `/search/...`<br>Star: <br> - `/star/...`<br>Tag: <br> - `/tags/...`<br>Users: <br> - `/profile/...`<br>Video: <br> - `/video/...`<br> |+| PimpAndHost | https://pimpandhost.com/ | pimpandhost | Album: <br> - `/album/...`<br>Image: <br> - `/image/...`<br> |+| PixHost | https://pixhost.to/ | pixhost | Gallery: <br> - `/gallery/...`<br>Image: <br> - `/show/...`<br> |+| PixelDrain | https://pixeldrain.com | pixeldrain | File: <br> - `/u/...`<br>Folder: <br> - `/l/...`<br> |+| Pkmncards | https://pkmncards.com | pkmncards | Card: <br> - `/card/...`<br>Series: <br> - `/series/...`<br>Set: <br> - `/set/...`<br> |+| PornHub | https://www.pornhub.com | pornhub | Album: <br> - `/album/...`<br>Channel: <br> - `/channel/...`<br>Gif: <br> - `/gif/...`<br>Photo: <br> - `/photo/...`<br>Playlist: <br> - `/playlist/...`<br>Profile: <br> - `/model/...`<br> - `/pornstar/...`<br> - `/user/...`<br>Video: <br> - `/embed/<video_id>`<br> - `/view_video.php?viewkey=<video_id>`<br> |+| PornPics | https://pornpics.com | pornpics | Categories: <br> - `/categories/....`<br>Channels: <br> - `/channels/...`<br>Direct Links<br>Gallery: <br> - `/galleries/...`<br>Pornstars: <br> - `/pornstars/...`<br>Search: <br> - `/?q=<query>`<br>Tags: <br> - `/tags/...`<br> |+| Porntrex | https://www.porntrex.com | porntrex | Album: <br> - `/albums/...`<br>Category: <br> - `/categories/...`<br>Model: <br> - `/models/...`<br>Playlist: <br> - `/playlists/...`<br>Search: <br> - `/search/...`<br>Tag: <br> - `/tags/...`<br>User: <br> - `/members/...`<br>Video: <br> - `/video/...`<br> |+| PostImg | https://postimages.org/ | postimg | Album: <br> - `/gallery/...`<br>Direct Links<br>Image: <br> - `/...`<br> |+| RealBooru | https://realbooru.com | realbooru | File: <br> - `?id=...`<br>Tags: <br> - `?tags=...`<br> |+| RedGifs | https://redgifs.com/ | redgifs | User: <br> - `/users/<user>`<br>Video: <br> - `/watch/...`<br> |+| Reddit | https://www.reddit.com/ | redd.it<br>reddit | Direct Links<br>Subreddit:: <br> - `/r/<subreddit>`<br>User: <br> - `/u/<user>`<br> - `/user/<user>`<br> - `/user/<user>/...`<br> |+| Rule34Vault | https://rule34vault.com | rule34vault | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |+| Rule34Video | https://rule34video.com/ | rule34video | Members: <br> - `/members/...`<br>Models: <br> - `/models/...`<br>Search: <br> - `/search/...`<br>Tags: <br> - `/tags/...`<br>Video: <br> - `/video/<video_id>/<video_name>`<br> - `/videos/<video_id>/<video_name>`<br> |+| Rule34XXX | https://rule34.xxx | rule34.xxx | File: <br> - `?id=...`<br>Tag: <br> - `?tags=...`<br> |+| Rule34XYZ | https://rule34.xyz | rule34.xyz | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |+| Saint | https://saint2.su/ | saint | Album: <br> - `/a/...`<br>Direct Links<br>Video: <br> - `/d/...`<br> - `/embed/...`<br> |+| Scrolller | https://scrolller.com | scrolller | Subreddit: <br> - `/r/...`<br> |+| SendNow | https://send.now/ | send.now | Direct Links<br> |+| SendVid | https://sendvid.com/ | sendvid | Direct Links<br>Embeds: <br> - `/embed/...`<br>Videos: <br> - `/...`<br> |+| Sex.com | https://sex.com | sex | Shorts Profiles: <br> - `/shorts/<profile>`<br> |+| SocialMediaGirls | https://forums.socialmediagirls.com | socialmediagirls | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| SpankBang | https://spankbang.com/ | spankbang | Playlist: <br> - `/playlist/<playlist-id>`<br>Video: <br> - `/embed/<video_id>`<br> - `/play/<video_id>`<br> - `/playlist/<playlist-id>-<video_id>`<br> - `/video/<video_id>`<br> |+| Streamable | https://streamable.com | streamable | Video: <br> - `/...`<br> |+| TWPornStars | https://www.twpornstars.com | indiantw.com<br>twanal.com<br>twgaymuscle.com<br>twgays.com<br>twlesbian.com<br>twmilf.com<br>twonfans.com<br>twpornstars.com<br>twteens.com<br>twtiktoks.com | Photo: <br> - `/...`<br> |+| ThisVid | https://thisvid.com | thisvid | Albums: <br> - `/albums/<album_name>`<br>Categories: <br> - `/categories/...`<br>Image: <br> - `/albums/<album_name>/<image_name>`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/?q=...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |+| TikTok | https://tiktok.com/ | tiktok | Photo: <br> - `/@<user>/photo/<photo_id>`<br>User: <br> - `/@<user>`<br>Video: <br> - `/@<user>/video/<video_id>`<br> |+| TitsInTops | https://titsintops.com/phpBB2 | titsintops | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| Tokyomotion | https://www.tokyomotion.net | tokyomotion | Albums: <br> - `/album/<album_id>`<br> - `/user/<user>/albums/`<br>Photo: <br> - `/photo/<photo_id>`<br> - `/user/<user>/favorite/photos`<br>Playlist: <br> - `/user/<user>/favorite/videos`<br>Profiles: <br> - `/user/<user>`<br>Search Results: <br> - `/search?...`<br>Video: <br> - `/video/<video_id>`<br> |+| Toonily | https://toonily.com | toonily | Chapter: <br> - `/webtoon/.../...`<br>Direct Links<br>Webtoon: <br> - `/webtoon/...`<br> |+| TwitterImages | https://twimg.com/ | twimg | Photo: <br> - `/...`<br> |+| Vipr.im | https://vipr.im | vipr.im | Image: <br> - `/...`<br>Thumbnail: <br> - `/th/...`<br> |+| WeTransfer | https://wetransfer.com/ | we.tl<br>wetransfer.com | Direct links: <br> - `download.wetransfer.com/...`<br>Public link: <br> - `wetransfer.com/<file_id>/<security_hash>`<br>Share Link: <br> - `wetransfer.com/<file_id>/<recipient_id>/<security_hash>`<br>Short Link: <br> - `we.tl/<id>`<br> |+| XBunker | https://xbunker.nu/ | xbunker | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| XBunkr | https://xbunkr.com | xbunkr | Albums: <br> - `/a/...`<br>Direct Links<br> |+| XXXBunker | https://xxxbunker.com | xxxbunker | Search: <br> - `/search/...`<br>Video: <br> - `/<video_id>`<br> |+| YandexDisk | https://disk.yandex.com.tr/ | disk.yandex<br>yadi.sk | Files: <br> - `/d/<folder_id>/<file_name>`<br>Folder: <br> - `/d/<folder_id>`<br><br><br>**NOTES**<br> - Does NOT support nested folders<br> |+| YouJizz | https://www.youjizz.com/ | youjizz | Video: <br> - `/video/<video_id>/...`<br> - `/video/embed/<video_id>`<br> |+| aShemaleTube | https://www.ashemaletube.com | ashemaletube | Model: <br> - `/creators/...`<br> - `/model/...`<br> - `/pornstars/...`<br>Playlist: <br> - `/playlists/...`<br>User: <br> - `/profiles/...`<br>Video: <br> - `/videos/...`<br> |+| eFukt | https://efukt.com | efukt.com | Gif: <br> - `/view.gif.php?id=<id>`<br>Homepage: <br> - `/`<br>Photo: <br> - `/pics/....`<br>Series: <br> - `/series/<series_name>`<br>Video: <br> - `/...`<br> |+| ePorner | https://www.eporner.com/ | eporner | Categories: <br> - `/cat/...`<br>Channels: <br> - `/channel/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/photo/...`<br>Pornstar: <br> - `/pornstar/...`<br>Profile: <br> - `/profile/...`<br>Search: <br> - `/search/...`<br>Video: <br> - `/<video_name>-<video-id>`<br> - `/embed/<video_id>`<br> - `/hd-porn/<video_id>`<br> |+| nHentai | https://nhentai.net/ | nhentai.net | Collections: <br> - `artist`<br> - `character`<br> - `favorites`<br> - `group`<br> - `parody`<br> - `search`<br> - `tag`<br>Gallery: <br> - `/g/<gallery_id>`<br> |+| xHamster | https://xhamster.com/ | xhamster | Creator: <br> - `/creatos/<creator_name>`<br>Gallery: <br> - `photos/gallery/<gallery_name>`<br>User: <br> - `/users/<user_name>`<br>Video: <br> - `/videos/<video_title>`<br> |<!-- END_SUPPORTED_SITES-->diff --git a/poetry.lock b/poetry.lockindex 6a08cd2a2..563c46fc7 100644--- a/poetry.lock+++ b/poetry.lock@@ -2297,30 +2297,30 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"][[package]]name = "ruff"-version = "0.11.0"+version = "0.12.0"description = "An extremely fast Python linter and code formatter, written in Rust."optional = falsepython-versions = ">=3.7"groups = ["dev"]files = [- {file = "ruff-0.11.0-py3-none-linux_armv6l.whl", hash = "sha256:dc67e32bc3b29557513eb7eeabb23efdb25753684b913bebb8a0c62495095acb"},- {file = "ruff-0.11.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:38c23fd9bdec4eb437b4c1e3595905a0a8edfccd63a790f818b28c78fe345639"},- {file = "ruff-0.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7c8661b0be91a38bd56db593e9331beaf9064a79028adee2d5f392674bbc5e88"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6c0e8d3d2db7e9f6efd884f44b8dc542d5b6b590fc4bb334fdbc624d93a29a2"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c3156d3f4b42e57247275a0a7e15a851c165a4fc89c5e8fa30ea6da4f7407b8"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:490b1e147c1260545f6d041c4092483e3f6d8eba81dc2875eaebcf9140b53905"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1bc09a7419e09662983b1312f6fa5dab829d6ab5d11f18c3760be7ca521c9329"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcfa478daf61ac8002214eb2ca5f3e9365048506a9d52b11bea3ecea822bb844"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fbb2aed66fe742a6a3a0075ed467a459b7cedc5ae01008340075909d819df1e"},- {file = "ruff-0.11.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92c0c1ff014351c0b0cdfdb1e35fa83b780f1e065667167bb9502d47ca41e6db"},- {file = "ruff-0.11.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e4fd5ff5de5f83e0458a138e8a869c7c5e907541aec32b707f57cf9a5e124445"},- {file = "ruff-0.11.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:96bc89a5c5fd21a04939773f9e0e276308be0935de06845110f43fd5c2e4ead7"},- {file = "ruff-0.11.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a9352b9d767889ec5df1483f94870564e8102d4d7e99da52ebf564b882cdc2c7"},- {file = "ruff-0.11.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:049a191969a10897fe052ef9cc7491b3ef6de79acd7790af7d7897b7a9bfbcb6"},- {file = "ruff-0.11.0-py3-none-win32.whl", hash = "sha256:3191e9116b6b5bbe187447656f0c8526f0d36b6fd89ad78ccaad6bdc2fad7df2"},- {file = "ruff-0.11.0-py3-none-win_amd64.whl", hash = "sha256:c58bfa00e740ca0a6c43d41fb004cd22d165302f360aaa56f7126d544db31a21"},- {file = "ruff-0.11.0-py3-none-win_arm64.whl", hash = "sha256:868364fc23f5aa122b00c6f794211e85f7e78f5dffdf7c590ab90b8c4e69b657"},- {file = "ruff-0.11.0.tar.gz", hash = "sha256:e55c620690a4a7ee6f1cccb256ec2157dc597d109400ae75bbf944fc9d6462e2"},+ {file = "ruff-0.12.0-py3-none-linux_armv6l.whl", hash = "sha256:5652a9ecdb308a1754d96a68827755f28d5dfb416b06f60fd9e13f26191a8848"},+ {file = "ruff-0.12.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:05ed0c914fabc602fc1f3b42c53aa219e5736cb030cdd85640c32dbc73da74a6"},+ {file = "ruff-0.12.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:07a7aa9b69ac3fcfda3c507916d5d1bca10821fe3797d46bad10f2c6de1edda0"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7731c3eec50af71597243bace7ec6104616ca56dda2b99c89935fe926bdcd48"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:952d0630eae628250ab1c70a7fffb641b03e6b4a2d3f3ec6c1d19b4ab6c6c807"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c021f04ea06966b02614d442e94071781c424ab8e02ec7af2f037b4c1e01cc82"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:7d235618283718ee2fe14db07f954f9b2423700919dc688eacf3f8797a11315c"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0758038f81beec8cc52ca22de9685b8ae7f7cc18c013ec2050012862cc9165"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:139b3d28027987b78fc8d6cfb61165447bdf3740e650b7c480744873688808c2"},+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68853e8517b17bba004152aebd9dd77d5213e503a5f2789395b25f26acac0da4"},+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3a9512af224b9ac4757f7010843771da6b2b0935a9e5e76bb407caa901a1a514"},+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b08df3d96db798e5beb488d4df03011874aff919a97dcc2dd8539bb2be5d6a88"},+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6a315992297a7435a66259073681bb0d8647a826b7a6de45c6934b2ca3a9ed51"},+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e55e44e770e061f55a7dbc6e9aed47feea07731d809a3710feda2262d2d4d8a"},+ {file = "ruff-0.12.0-py3-none-win32.whl", hash = "sha256:7162a4c816f8d1555eb195c46ae0bd819834d2a3f18f98cc63819a7b46f474fb"},+ {file = "ruff-0.12.0-py3-none-win_amd64.whl", hash = "sha256:d00b7a157b8fb6d3827b49d3324da34a1e3f93492c1f97b08e222ad7e9b291e0"},+ {file = "ruff-0.12.0-py3-none-win_arm64.whl", hash = "sha256:8cd24580405ad8c1cc64d61725bca091d6b6da7eb3d36f72cc605467069d7e8b"},+ {file = "ruff-0.12.0.tar.gz", hash = "sha256:4d047db3662418d4a848a3fdbfaf17488b34b62f527ed6f10cb8afd78135bc5c"},][[package]]@@ -2927,4 +2927,4 @@ propcache = ">=0.2.1"[metadata]lock-version = "2.1"python-versions = ">=3.11,<4"-content-hash = "933320618925b324324b5a02e96f9c777b935494d59c626e755df35124ec7fe6"+content-hash = "973b07f0a1e83b46a6d40afdb5d95e3689c3e99a00876efe553f7dc9fd201e25"diff --git a/pyproject.toml b/pyproject.tomlindex 2872371af..7f87327fd 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -70,7 +70,7 @@ requires-poetry = ">=2.0"[tool.poetry.group.dev.dependencies]pre-commit = ">=4.0.1,<5"py-markdown-table = "^1.3.0"-ruff = ">=0.11.0"+ruff = ">=0.12.0"snakeviz = "^2.2.2"tox = "^4.24.1"diff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.pynew file mode 100644index 000000000..70e7f2296--- /dev/null+++ b/tests/crawlers/test_xenforo.py@@ -0,0 +1,833 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any+from unittest import mock++import pytest+from bs4 import BeautifulSoup++from cyberdrop_dl.crawlers import xenforo as crawlers+from cyberdrop_dl.crawlers.xenforo import xenforo+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.managers.manager import Manager++if TYPE_CHECKING:+ from collections.abc import AsyncGenerator+ from pathlib import Path+++def _item(url: str) -> ScrapeItem:+ return ScrapeItem(url=AbsoluteHttpURL(url))+++class MockProgress:+ scraping_progress = None+++manager = Manager()+scrape_item = _item("https://xenforo.com/community")+manager.progress_manager = MockProgress() # type: ignore+crawler_instances = {crawler: crawler(manager) for crawler in crawlers.XF_CRAWLERS}+TEST_CRAWLER = crawler_instances[crawlers.CelebForumCrawler]+++def _html(string: str) -> str:+ return f"""+ <html>+ <body>+ {string}+ </body>+ </html>+ """+++def _post(+ message_body: str = "",+ message_attachments: str = "",+ id: int = 12345,+ crawler: xenforo.XenforoCrawler | None = None,+) -> xenforo.ForumPost:+ crawler = crawler or TEST_CRAWLER+ html = _html(POST_TEMPLATE.format(id=id, message_body=message_body, message_attachments=message_attachments))+ soup = BeautifulSoup(html, "html.parser")+ return xenforo.ForumPost.new(soup, crawler.XF_SELECTORS.posts)+++def _item_call(value: Any) -> mock._Call:+ return mock.call(scrape_item, value)+++def _any_item_call(value: Any) -> mock._Call:+ return mock.call(mock.ANY, value)+++def _amock(func: str = "process_child", crawler: xenforo.XenforoCrawler | None = None) -> mock._patch[mock.AsyncMock]:+ crawler = crawler or TEST_CRAWLER+ return mock.patch.object(crawler, func, new_callable=mock.AsyncMock)++[email protected](name="manager")+async def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[Manager]:+ appdata = str(tmp_path)+ downloads = str(tmp_path / "Downloads")+ monkeypatch.chdir(tmp_path)+ manager = Manager(("--appdata-folder", appdata, "-d", downloads))+ manager.startup()+ manager.path_manager.startup()+ manager.log_manager.startup()+ await manager.async_startup()+ yield manager+ await manager.async_db_close()+ await manager.close()+++ ("url", "thread_name_and_id", "result", "canonical_url"),+ [+ [+ "https://simpcity.su/threads/general-support.208041/page-260#post-23934165",+ "general-support.208041",+ (+ 208041,+ "general-support",+ 260,+ 23934165,+ ),+ "https://simpcity.su/threads/general-support.208041",+ ],+ [+ "https://celebforum.to/threads/infos-regelaenderungen.18821/page-3",+ "infos-regelaenderungen.18821",+ (+ 18821,+ "infos-regelaenderungen",+ 3,+ None,+ ),+ "https://celebforum.to/threads/infos-regelaenderungen.18821",+ ],+ [+ "https://www.bellazon.com/main/topic/27120-the-official-victorias-secret-thread",+ "27120-the-official-victorias-secret-thread",+ (+ 27120,+ "the-official-victorias-secret-thread",+ 1,+ None,+ ),+ "https://www.bellazon.com/main/topic/27120-the-official-victorias-secret-thread",+ ],+ [+ "https://forums.socialmediagirls.com/threads/forum-rules.14/post-34",+ "forum-rules.14",+ (+ 14,+ "forum-rules",+ 1,+ 34,+ ),+ "https://forums.socialmediagirls.com/threads/forum-rules.14",+ ],+ [+ "https://forums.socialmediagirls.com/threads/should-we-ban-gofile-76-5-say-no.436901/post-3942103",+ "should-we-ban-gofile-76-5-say-no.436901",+ (+ 436901,+ "should-we-ban-gofile-76-5-say-no",+ 1,+ 3942103,+ ),+ "https://forums.socialmediagirls.com/threads/should-we-ban-gofile-76-5-say-no.436901",+ ],+ [+ "https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930/page-11/#post-2070848",+ "en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930",+ (+ 13930,+ "en-fr-tools-to-download-upload-content-websites-softwares-extensions",+ 11,+ 2070848,+ ),+ "https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930",+ ],+ [+ "https://f95zone.to/threads/mod-uploading-rules-12-02-2018.9236/post-2726083",+ "mod-uploading-rules-12-02-2018.9236",+ (+ 9236,+ "mod-uploading-rules-12-02-2018",+ 1,+ 2726083,+ ),+ "https://f95zone.to/threads/mod-uploading-rules-12-02-2018.9236",+ ],+ [+ "https://650f.bike/threads/central-stand-honda-cb650r-constands-power-evo.3093/page-2",+ "central-stand-honda-cb650r-constands-power-evo.3093",+ (+ 3093,+ "central-stand-honda-cb650r-constands-power-evo",+ 2,+ None,+ ),+ "https://650f.bike/threads/central-stand-honda-cb650r-constands-power-evo.3093",+ ],+ [+ "https://arstechnica.com/civis/threads/the-new-perpetual-photo-accessory-thread.1274775/page-3#post-29155063",+ "the-new-perpetual-photo-accessory-thread.1274775",+ (+ 1274775,+ "the-new-perpetual-photo-accessory-thread",+ 3,+ 29155063,+ ),+ "https://arstechnica.com/civis/threads/the-new-perpetual-photo-accessory-thread.1274775",+ ],+ [+ "https://www.laneros.com/temas/iphone-16-16e-16-plus-16-pro-16-promax.256047/page-64#post-7512404",+ "iphone-16-16e-16-plus-16-pro-16-promax.256047",+ (+ 256047,+ "iphone-16-16e-16-plus-16-pro-16-promax",+ 64,+ 7512404,+ ),+ "https://www.laneros.com/temas/iphone-16-16e-16-plus-16-pro-16-promax.256047",+ ],+ ],+)+def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str, int, int], canonical_url: str) -> None:+ url_, canonical_url_ = AbsoluteHttpURL(url), AbsoluteHttpURL(canonical_url)+ result_ = xenforo.Thread(*result, canonical_url_)+ parsed = xenforo.parse_thread(url_, thread_name_and_id, "page", "post")+ assert result_ == parsed+++ "link, out",+ [+ (+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",+ ),+ (+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.th.jpg",+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",+ ),+ (+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.md.jpg",+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",+ ),+ (+ "https://simp6.jpg5.su/images3/20250612_051526cd1f36d75c763fe4.md.jpg",+ "https://simp6.jpg5.su/images3/20250612_051526cd1f36d75c763fe4.jpg",+ ),+ (+ "https://jpg5.su/img/aDNRwDH",+ "https://jpg5.su/img/aDNRwDH",+ ),+ (+ "https://saint2.cr/embed/M8XwlvJUlW-",+ "https://saint2.cr/embed/M8XwlvJUlW-",+ ),+ (+ "https://saint2.cr/ifr/M8XwlvJUlW-",+ "https://saint2.cr/watch/M8XwlvJUlW-",+ ),+ ],+)+def test_clean_link_url(link: str, out: str) -> None:+ assert xenforo.clean_link_str(link) == out+++def test_parse_login_form_success() -> None:+ html = _html("""+ <form id="loginForm">+ <input type="text" name="username" value="testuser">+ <input type="password" name="password" value="testpass">+ <input type="hidden" name="csrf_token" value="some_token_123">+ <input type="submit" value="Login">+ <input type="text" id="noName" value="shouldBeIgnored">+ <input type="text" name="noValue">+ </form>+ <form id="anotherForm">+ <input type="text" name="anotherField" value="anotherValue">+ </form>+ """)+ expected_data = {+ "username": "testuser",+ "password": "testpass",+ "csrf_token": "some_token_123",+ }+ parsed_data = xenforo.parse_login_form(html)+ assert parsed_data == expected_data+++def test_parse_login_form_no_form_should_fail() -> None:+ with pytest.raises(ScrapeError):+ xenforo.parse_login_form("")+++def test_parse_login_form_inputs_without_name_or_value_should_be_ignored() -> None:+ html = _html("""+ <form>+ <input type="text" value="somevalue">+ <input type="text" name="someName">+ <input type="text" name="validField" value="validValue">+ </form>+ """)+ expected_data = {"validField": "validValue"}+ parsed_data = xenforo.parse_login_form(html)+ assert parsed_data == expected_data+++def test_parse_login_form_no_input_form_should_fail() -> None:+ html = _html("""+ <form>+ <div>Some content</div>+ <p>More content</p>+ </form>+ """)+ with pytest.raises(ScrapeError):+ xenforo.parse_login_form(html)+++ "input_string, expected_output",+ [+ (+ r"some_text_before \/\/simpcity.su/path/to/resource.mp4 some_text_after",+ "https://simpcity.su/path/to/resource.mp4",+ ),+ (+ r"prefix \/\/celebforum.to/search?q=test suffix",+ "https://celebforum.to/search?q=test",+ ),+ (+ r"only_url \/\/sub.domain.co.uk/file.pdf",+ "https://sub.domain.co.uk/file.pdf",+ ),+ (+ r"http://simpcity.su/some_page",+ "http://simpcity.su/some_page",+ ),+ (+ r"text with no slashes http://xenforo.com",+ "http://xenforo.com",+ ),+ (+ r"text with no slashes https://xenforo.com/path",+ "https://xenforo.com/path",+ ),+ # Test cases where no URL is found+ (+ "just some plain text",+ "just some plain text",+ ),+ ("", ""),+ (+ "no\\url\\here",+ "nourlhere",+ ), # check if backslashes are removed+ (+ "//invalid.c",+ "//invalid.c",+ ), # too short domain suffix+ (+ "ftp://simpcity.su/file",+ "ftp://simpcity.su/file",+ ), # unsupported protocol+ (+ "www.simpcity.su",+ "www.simpcity.su",+ ), # missing http/https+ (+ "simpcity.su",+ "simpcity.su",+ ), # missing http/https+ (+ "text with https: //simpcity.su/ (space after colon)",+ "text with https: //simpcity.su/ (space after colon)",+ ),+ (+ "text with unicode \u00a9 characters",+ "text with unicode \u00a9 characters",+ ),+ ],+)+def test_extract_embed_url(input_string: str, expected_output: str) -> None:+ assert xenforo.extract_embed_url(input_string) == expected_output++[email protected] # regex can not handle URLs with commans in it (kvs)+ ("input_string", "expected_output"),+ [+ (+ r"start \/\/media.jp5.net/videos/2023/clip_id-123.mp4?autoplay=true&loop=false#t=10s end",+ "https://media.jp5.net/videos/2023/clip_id-123.mp4?autoplay=true&loop=false#t=10s",+ ),+ (+ r"start \/\/jupiter4.thisvid.com/key=SEtXHaueMU2PByWg4GNMnw,end=1750179436/speed=1.4/buffer=5.0/12702000/12702535/12702535.mp4 other",+ "https://jupiter4.thisvid.com/key=SEtXHaueMU2PByWg4GNMnw,end=1750179436/speed=1.4/buffer=5.0/12702000/12702535/12702535.mp4",+ ),+ ],+)+def test_extract_embed_url_complex_path(input_string: str, expected_output: str) -> None:+ assert xenforo.extract_embed_url(input_string) == expected_output+++def test_extract_embed_url_should_extract_only_one_url() -> None:+ input_str = r"first_url \/\/first.com/path second_url \/\/www.second.net/another"+ expected_output = "https://first.com/path"+ assert xenforo.extract_embed_url(input_str) == expected_output+++def test_extract_embed_url_with_escaped_backslashes_and_double_slashes() -> None:+ input_str = r"some_path\\to\\file\/\/imagepond.net\/clip.mov"+ expected_output = "https://imagepond.net/clip.mov"+ assert xenforo.extract_embed_url(input_str) == expected_output+++def test_extract_embed_url_absolute_url_no_scheme() -> None:+ input_str = r"\/\/jpg5.su"+ expected_output = r"https://jpg5.su"+ assert xenforo.extract_embed_url(input_str) == expected_output+++def test_extract_embed_url_should_not_modify_absolute_urls() -> None:+ input_str = r"https://celebforum.to/maps"+ expected_output = "https://celebforum.to/maps"+ assert xenforo.extract_embed_url(input_str) == expected_output+++def test_extract_embed_url_string_ends_with_url() -> None:+ input_str = r"some text https://www.reddit.com/"+ expected_output = "https://reddit.com/"+ assert xenforo.extract_embed_url(input_str) == expected_output+++async def test_lazy_load_embeds() -> None:+ post = _post("""+ <div class="generic2wide-iframe-div" onclick="loadMedia(this, '//redgifs.com/ifr/downrightcluelesswirm');">+ <span data-s9e-mediaembed="redgifs">+ <span class="iframe-wrapper-redgifs" style="">+ <span class="iframe-wrapper-redgifs-info">Click here to load redgifs media</span>+ </span>+ </span>+ </div>""")+ expected_output = "//redgifs.com/ifr/downrightcluelesswirm"+ with _amock() as mocked:+ await TEST_CRAWLER._lazy_load_embeds(scrape_item, post)+ mocked.assert_called_once_with(scrape_item, expected_output, embeds=True)+++ ("cls", "post_content", "expected_result"),+ [+ (+ crawlers.SimpCityCrawler,+ """+ <div class="bbWrapper">+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">+ <img+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"+ class="bbImage"+ loading="lazy"+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"+ style=""+ width=""+ height=""+ />+ </a>+ </div>""",+ [+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",+ ],+ )+ ],+)+async def test_post_images(cls: type[xenforo.XenforoCrawler], post_content: str, expected_result: list[str]) -> None:+ crawler = crawler_instances[cls]+ post = _post(post_content, crawler=crawler)+ with _amock(crawler=crawler) as mocked:+ await crawler._images(scrape_item, post)+ count, expected_count = mocked.call_count, len(expected_result)+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"+ for result, expected in zip(mocked.call_args_list, expected_result, strict=True):+ assert result.args[1] == expected+++async def test_embeds_can_extract_google_drive_links() -> None:+ # https://github.com/jbsparrow/CyberDropDownloader/issues/775+ crawler = crawler_instances[crawlers.SimpCityCrawler]+ content = """+ <div itemprop="text">+ <div class="bbWrapper">+ Got this video From her.<br />+ <span data-s9e-mediaembed="googledrive">+ <iframe allowfullscreen="" scrolling="no" src="//drive.google.com/file/d/1gfDjCbNXgJafY6ILQIrgbnuptSCFbM0J/preview" loading="eager"></iframe></span>+ </span>+ </div>+ </div>++ <div class="js-selectToQuoteEnd"> </div>+ """+ post = _post(content, crawler=crawler)+ expected_result = "//drive.google.com/file/d/1gfDjCbNXgJafY6ILQIrgbnuptSCFbM0J/preview"+ with _amock(crawler=crawler) as mocked:+ await crawler._embeds(scrape_item, post)+ mocked.assert_called_once()+ assert mocked.call_args.args[1] == expected_result+++async def test_post_smg_extract_attachments() -> None:+ # https://agithub.com/jbsparrow/CyberDropDownloader/issues/1070+ attachments = """+ <h4 class="block-textHeader">Attachments</h4>+ <ul class="attachmentList">+ <li class="file file--linked">+ <a class="u-anchorTarget" id="attachment-3494354"></a>++ <a+ class="file-preview js-lbImage"+ href="https://smgmedia2.socialmediagirls.com/forum/2022/04/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg"+ target="_blank"+ data-fancybox="lb-thread-111099"+ style="cursor: pointer;"+ data-caption='<h4>33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg</h4><p><a href="https:&#x2F;&#x2F;forums.socialmediagirls.com&#x2F;threads&#x2F;loalux.111099&#x2F;#post-1707346" class="js-lightboxCloser">Ixvvxi · Apr 23, 2022 at 10:10 PM</a></p>'+ >+ <img+ src="https://smgmedia2.socialmediagirls.com/forum/2022/04/thumb/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg"+ alt="33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg"+ width="250"+ height="425"+ loading="lazy"+ />+ </a>++ <div class="file-content">+ <div class="file-info">+ <span class="file-name" title="33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg">33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg</span>+ <div class="file-meta">+ 1.3 MB · Views: 0+ </div>+ </div>+ </div>+ </li>++ <li class="file file--linked">+ <a class="u-anchorTarget" id="attachment-3494355"></a>++ <a+ class="file-preview js-lbImage"+ href="https://smgmedia2.socialmediagirls.com/forum/2022/04/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png"+ target="_blank"+ data-fancybox="lb-thread-111099"+ style="cursor: pointer;"+ data-caption='<h4>3663188F-00C6-4C90-AB4D-D8C6E7859286.png</h4><p><a href="https:&#x2F;&#x2F;forums.socialmediagirls.com&#x2F;threads&#x2F;loalux.111099&#x2F;#post-1707346" class="js-lightboxCloser">Ixvvxi · Apr 23, 2022 at 10:10 PM</a></p>'+ >+ <img src="https://smgmedia2.socialmediagirls.com/forum/2022/04/thumb/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png" alt="3663188F-00C6-4C90-AB4D-D8C6E7859286.png" width="364" height="250" loading="lazy" />+ </a>++ <div class="file-content">+ <div class="file-info">+ <span class="file-name" title="3663188F-00C6-4C90-AB4D-D8C6E7859286.png">3663188F-00C6-4C90-AB4D-D8C6E7859286.png</span>+ <div class="file-meta">+ 1.6 MB · Views: 0+ </div>+ </div>+ </div>+ </li>+ </ul>+ """+ crawler = crawler_instances[crawlers.SocialMediaGirlsCrawler]+ post = _post(message_attachments=attachments, crawler=crawler)+ expected_result = [+ "https://smgmedia2.socialmediagirls.com/forum/2022/04/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg",+ "https://smgmedia2.socialmediagirls.com/forum/2022/04/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png",+ ]+ with _amock(crawler=crawler) as mocked:+ await crawler._attachments(scrape_item, post)+ count, expected_count = mocked.call_count, len(expected_result)+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"+ for result, expected in zip(mocked.call_args_list, expected_result, strict=True):+ assert result.args[1] == expected+++def test_get_post_title_thread_w_prefixes() -> None:+ html = _html("""+ <div class="p-title ">+ <h1 class="p-title-value">+ <a href="/forums/requests.7/?prefix_id=2" class="labelLink" rel="nofollow">+ <span class="label label--requests" dir="auto">Request</span></a>+ <span class="label-append"> </span><a href="/forums/requests.7/?prefix_id=8" class="labelLink" rel="nofollow">+ <span class="label label--youtube" dir="auto">Youtube</span></a><span class="label-append"> </span>+ <a href="/forums/requests.7/?prefix_id=3" class="labelLink" rel="nofollow">+ <span class="label label--onlyfans" dir="auto">OnlyFans</span></a>+ <span class="label-append"> </span><a href="/forums/requests.7/?prefix_id=7" class="labelLink" rel="nofollow">+ <span class="label label--insta" dir="auto">Instagram</span></a>+ <span class="label-append"> </span>GunplaMeli</h1>+ </div>+ """)+ soup = BeautifulSoup(html, "html.parser")+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ assert title == "GunplaMeli"+++def test_get_post_title_thread_w_no_prefixes() -> None:+ html = """+ <div class="p-title">+ <h1 class="p-title-value">Staged/Fake Japanese Candid Videos from Gcolle/Pcolle or FC2</h1>+ </div>+ """+ soup = BeautifulSoup(html, "html.parser")+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ assert title == "Staged/Fake Japanese Candid Videos from Gcolle/Pcolle or FC2"+++def test_get_post_title_no_title_found() -> None:+ html = _html("")+ soup = BeautifulSoup(html, "html.parser")+ with pytest.raises(ScrapeError) as exc_info:+ xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)++ assert exc_info.value.status == 429+ assert exc_info.value.message == "Invalid response from forum. You may have been rate limited"+++def test_get_post_title_empty_title_block() -> None:+ html = _html("""<h1 class="p-title-value"></h1>""")+ soup = BeautifulSoup(html, "html.parser")+ with pytest.raises(ScrapeError):+ xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+++def test_get_post_title_non_english_chars() -> None:+ html = _html("""+ <div class="p-title">+ <h1 class="p-title-value">+ ㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia+ </h1>+ </div>+ """)+ soup = BeautifulSoup(html, "html.parser")+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"+++def test_get_post_title_should_strip_new_lines() -> None:+ html = _html("""+ <div class="p-title">+ <h1 class="p-title-value">+ ㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️)+ / りお@Rio /+ rio_hcup_fantia+ </h1>+ </div>+ """)+ soup = BeautifulSoup(html, "html.parser")+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)+ assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"+++def test_is_attachment_should_handle_none() -> None:+ assert TEST_CRAWLER.is_attachment(None) is False # type: ignore+++ "link_str",+ [+ "https://media.imagepond.net/media/golden-hour86caaa9858726a64.jpg",+ "https://simp6.jpg5.su/images3/1752x1172_a682e9c85a276121e82018b4031206b9be2e64efe5bc8004.md.jpg",+ "https://jpg5.su/img/aDDJbdK",+ "https://simp6.jpg5.su/attachment/1752x1172_a682e9c85a276121e82018b4031206b9be2e64efe5bc8004.md.jpg", # No exact 'attachment' part+ "http://celebforum.to/attachments", # missing file slug+ ],+)+def test_is_attachment_string_false(link_str: str) -> None:+ assert TEST_CRAWLER.is_attachment(link_str) is False+++def test_is_attachment_empty_string_should_be_false() -> None:+ assert TEST_CRAWLER.is_attachment("") is False+++class TestCheckPostId:+ @pytest.mark.parametrize(+ "init_post_id, current_post_id, scrape_single_forum_post, expected_continue_scraping, expected_scrape_this_post",+ [+ # init_post_id > current_post_id+ (100, 90, True, True, False),+ (100, 90, False, True, False),+ # init_post_id == current_post_id+ (100, 100, True, False, True),+ (100, 100, False, True, True),+ # init_post_id < current_post_id+ (100, 110, True, False, False),+ (100, 110, False, True, True),+ ],+ )+ def test_init_post_id_was_provided(+ self,+ init_post_id: int,+ current_post_id: int,+ scrape_single_forum_post: bool,+ expected_continue_scraping: bool,+ expected_scrape_this_post: bool,+ ) -> None:+ continue_scraping, scrape_this_post = xenforo.check_post_id(+ init_post_id, current_post_id, scrape_single_forum_post+ )+ assert continue_scraping == expected_continue_scraping+ assert scrape_this_post == expected_scrape_this_post++ def test_no_init_post_id_and_scrape_single_post_false(self) -> None:+ init_post_id = None+ current_post_id = 100+ scrape_single_forum_post = False+ continue_scraping, scrape_this_post = xenforo.check_post_id(+ init_post_id, current_post_id, scrape_single_forum_post+ )+ assert continue_scraping is True+ assert scrape_this_post is True++ def test_no_init_post_id_and_scrape_single_post_true_raises_error(self) -> None:+ init_post_id = None+ current_post_id = 100+ scrape_single_forum_post = True++ with pytest.raises(AssertionError):+ xenforo.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)+++# og_post_id = 23549340+POST_TEMPLATE = """+<article class="message message--post js-post js-inlineModContainer" data-author="cyberdrop-dl-patched" data-content="post{id}" id="js-post{id}" itemscope="" itemtype="https://schema.org/Comment" itemid="https://xenforo.com/posts/{id}/">+ <meta itemprop="parentItem" itemscope="" itemid="https://xenforo.com/threads/fanfan.33077/" />++ <span class="u-anchorTarget" id="post{id}"></span>++ <div class="message-inner">+ <div class="message-cell message-cell--main">+ <div class="message-main js-quickEditTarget">+ <header class="message-attribution message-attribution--split">+ <ul class="message-attribution-main listInline">+ <li class="u-concealed">+ <a href="/threads/fanfan.33077/post{id}" rel="nofollow" itemprop="url">+ <time+ class="u-dt"+ dir="auto"+ datetime="2025-06-09T17:30:10-0500"+ data-timestamp="1749508210"+ data-date="Jun 9, 2025"+ data-time="5:30 PM"+ data-short="7d"+ title="Jun 9, 2025 at 5:30 PM"+ itemprop="datePublished"+ >+ Jun 9, 2025+ </time>+ </a>+ </li>+ </ul>+ </header>+ <div class="message-attachments">+ {message_attachments}+ </div>++ <div class="message-content js-messageContent">+ <div class="message-userContent lbContainer js-lbContainer" data-lb-id="post{id}" data-lb-caption-desc="cyberdrop-dl-patched · Jun 9, 2025 at 5:30 PM">+ <article class="message-body js-selectToQuote">+ {message_body}+ <div class="js-selectToQuoteEnd"> </div>+ </article>+ </div>++ <aside class="message-signature">+ <div class="bbWrapper">Can't post to Bunkr. Mirrors are always appreciated.</div>+ </aside>+ </div>++ <footer class="message-footer">+ <div class="message-microdata" itemprop="interactionStatistic" itemtype="https://schema.org/InteractionCounter" itemscope="">+ <meta itemprop="userInteractionCount" content="154" />+ <meta itemprop="interactionType" content="https://schema.org/LikeAction" />+ </div>++ <div class="message-actionBar actionBar">+ <div class="actionBar-set actionBar-set--external">+ <a+ href="/posts/{id}/react?reaction_id=1"+ class="reaction reaction--small actionBar-action actionBar-action--reaction reaction--imageHidden reaction--1"+ data-reaction-id="1"+ data-xf-init="reaction"+ data-reaction-list="< .js-post | .js-reactionsList"+ id="js-XFUniqueId13"+ >+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="Like" title="Like" />+ <span class="reaction-text js-reactionText"><bdi>Like</bdi></span>+ </a>++ <a href="/threads/fanfan.33077/reply?quote={id}" class="actionBar-action actionBar-action--mq u-jsOnly js-multiQuote" title="Toggle multi-quote" rel="nofollow" data-message-id="{id}" data-mq-action="add">+ Quote+ </a>++ <a href="/threads/fanfan.33077/reply?quote={id}" class="actionBar-action actionBar-action--reply" title="Reply, quoting this message" rel="nofollow" data-xf-click="quote" data-quote-href="/posts/{id}/quote">+ Reply+ </a>+ </div>++ <div class="actionBar-set actionBar-set--internal">+ <a href="/posts/{id}/report" class="actionBar-action actionBar-action--report" data-xf-click="overlay" data-cache="false">Report</a>+ </div>+ </div>++ <div class="reactionsBar js-reactionsList is-active">+ <ul class="reactionSummary">+ <li>+ <span class="reaction reaction--small reaction--1" data-reaction-id="1">+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="Like" title="Like" />+ </span>+ </li>+ <li>+ <span class="reaction reaction--small reaction--33" data-reaction-id="33">+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PeepoLove" title="PeepoLove" />+ </span>+ </li>+ <li>+ <span class="reaction reaction--small reaction--91" data-reaction-id="91">+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PepeClown" title="PepeClown" />+ </span>+ </li>+ </ul>++ <span class="u-srOnly">Reactions:</span>+ <a class="reactionsBar-link" href="/posts/{id}/reactions" data-xf-click="overlay" data-cache="false" rel="nofollow"><bdi>xigxagxion</bdi>, <bdi>Feistee</bdi>, <bdi>kradpmis</bdi> and 140 others</a>+ </div>++ <div class="js-historyTarget message-historyTarget toggleTarget" data-href="trigger-href"></div>+ </footer>+ </div>+ </div>+ </div>+</article>++"""