Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: rework xenforo (#1079)

NTFSvolume69819543a249
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index eaf3152e9..8a95d86a6 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -91,18 +91,7 @@ from .twpornstars import TwPornstarsCrawler
from .vipr_dot_im import ViprImCrawler
from .wetransfer import WeTransferCrawler
from .xbunkr import XBunkrCrawler
-from .xenforo import (
- AllPornComixCrawler,
- BellazonCrawler,
- CelebForumCrawler,
- F95ZoneCrawler,
- LeakedModelsCrawler,
- NudoStarCrawler,
- SimpCityCrawler,
- SocialMediaGirlsCrawler,
- TitsInTopsCrawler,
- XBunkerCrawler,
-)
+from .xenforo import XF_CRAWLERS, SimpCityCrawler
from .xhamster import XhamsterCrawler
from .xxxbunker import XXXBunkerCrawler
from .yandex_disk import YandexDiskCrawler
@@ -110,6 +99,7 @@ from .youjizz import YouJizzCrawler
ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler")}
ALL_CRAWLERS = ALL_CRAWLERS - {Crawler}
+ALL_CRAWLERS.update(XF_CRAWLERS)
DEBUG_CRAWLERS = {SimpCityCrawler, BunkrAlbumsIOCrawler, MegaNzCrawler, FikFapCrawler}
CRAWLERS = ALL_CRAWLERS - DEBUG_CRAWLERS
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index bbf7edd9b..67078ba6f 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -199,7 +199,7 @@ class CheveretoCrawler(Crawler, is_abc=True):
name_index = url.parts.index(found_part) + 1
name = url.parts[name_index]
item_id = name.rsplit(".")[-1]
- new_parts = url.parts[1:name_index] + (item_id,)
+ new_parts = (*url.parts[1:name_index], item_id)
new_path = "/" + "/".join(new_parts)
return item_id, self.parse_url(new_path, url.origin())
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
new file mode 100644
index 000000000..7409ee832
--- /dev/null
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, ClassVar, final
+
+from cyberdrop_dl.crawlers.crawler import Crawler
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+
+
+class ForumCrawler(Crawler, is_abc=True):
+ SUPPORTS_THREAD_RECURSION: ClassVar[bool] = False
+
+ @property
+ def scrape_single_forum_post(self) -> bool:
+ return self.manager.config_manager.settings_data.download_options.scrape_single_forum_post
+
+ @property
+ def max_thread_depth(self) -> int:
+ return self.manager.config_manager.settings_data.download_options.maximum_thread_depth
+
+ @classmethod
+ @abstractmethod
+ def is_attachment(cls, url: AbsoluteHttpURL) -> bool: ...
+
+ @abstractmethod
+ async def forum(self, scrape_item: ScrapeItem) -> None: ...
+
+ @abstractmethod
+ async def thread(self, scrape_item: ScrapeItem) -> None: ...
+
+ @abstractmethod
+ async def post(self, scrape_item: ScrapeItem) -> None: ...
+
+ @error_handling_wrapper
+ async def handle_link(self, scrape_item: ScrapeItem, link: AbsoluteHttpURL) -> None:
+ if link == self.PRIMARY_URL:
+ return
+ if self.is_attachment(link):
+ return await self.handle_internal_link(scrape_item.create_new(link))
+ if self.PRIMARY_URL.host in scrape_item.url.host and self.stop_thread_recursion(scrape_item):
+ msg = f"Skipping nested thread URL {scrape_item.url} found on {scrape_item.origin}"
+ return self.log(msg)
+ new_scrape_item = scrape_item.copy()
+ new_scrape_item.type = None
+ new_scrape_item.reset_childen()
+ self.handle_external_links(new_scrape_item)
+ scrape_item.add_children()
+
+ @error_handling_wrapper
+ async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:
+ # This logic is for discourse
+ # if len(scrape_item.url.parts) < 5 and not scrape_item.url.suffix:
+ # return
+ filename, ext = self.get_filename_and_ext(scrape_item.url.name)
+ scrape_item.add_to_parent_title("Attachments")
+ scrape_item.part_of_album = True
+ await self.handle_file(scrape_item.url, scrape_item, filename, ext)
+
+ @final
+ def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:
+ if (
+ not self.SUPPORTS_THREAD_RECURSION
+ or not self.max_thread_depth
+ or (len(scrape_item.parent_threads) > self.max_thread_depth)
+ ):
+ return True
+ return False
+
+ @final
+ async def write_last_forum_post(self, thread_url: AbsoluteHttpURL, last_post_url: AbsoluteHttpURL | None) -> None:
+ if not last_post_url or last_post_url == thread_url:
+ return
+ await self.manager.log_manager.write_last_post_log(last_post_url)
+
+
+MessageBoardCrawler = ForumCrawler
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index a9efdb4e6..53bb28eff 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -27,6 +27,7 @@ from cyberdrop_dl.utils.utilities import (
get_download_path,
get_filename_and_ext,
is_absolute_http_url,
+ is_blob_or_svg,
parse_url,
remove_file_id,
sort_dict,
@@ -73,7 +74,7 @@ def create_task_id(func: Callable[P, Coroutine[None, None, R]]) -> Callable[P, C
try:
if not self.SKIP_PRE_CHECK:
pre_check_scrape_item(scrape_item)
- return await func(scrape_item, **kwargs)
+ return await func(scrape_item, **kwargs) # type: ignore
except ValueError:
log(f"Scrape Failed: {UNKNOWN_URL_PATH_MSG}: {scrape_item.url}", 40)
self.manager.progress_manager.scrape_stats_progress.add_failure(UNKNOWN_URL_PATH_MSG)
@@ -189,7 +190,7 @@ class Crawler(ABC):
if item.url.path_qs not in self.scraped_items:
log(f"{scrape_prefix}: {item.url}", 20)
self.scraped_items.append(item.url.path_qs)
- await create_task_id(self.fetch)(self, item)
+ await create_task_id(self.fetch)(self, item) # type: ignore
else:
log(f"Skipping {item.url} as it has already been scraped", 10)
@@ -401,10 +402,6 @@ class Crawler(ABC):
If provided, it will be used as a filter, to only yield items that has not been downloaded before"""
album_results = results or {}
- def is_embedded_image(link: str) -> bool:
- """Checks if the link is an embedded image URL."""
- return link.startswith("data:image") or link.startswith("blob:")
-
for tag in css.iselect(soup, selector):
link_str: str | None = css.get_attr_or_none(tag, attribute)
if not link_str:
@@ -416,7 +413,7 @@ class Crawler(ABC):
thumb_str: str | None = css.get_attr_or_none(t_tag, "src")
else:
thumb_str = None
- thumb = self.parse_url(thumb_str) if thumb_str and not is_embedded_image(thumb_str) else None
+ thumb = self.parse_url(thumb_str) if thumb_str and not is_blob_or_svg(thumb_str) else None
yield thumb, link
def iter_children(
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index 1db26ff9b..bc43aa427 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -93,7 +93,7 @@ class MegaNzCrawler(Crawler):
file_key[2] ^ file_key[6],
file_key[3] ^ file_key[7],
)
- iv: tuple[int, ...] = file_key[4:6] + (0, 0)
+ iv: tuple[int, ...] = (*file_key[4:6], 0, 0)
meta_mac: tuple[int, ...] = file_key[6:8]
file = FileTuple(file_id, DecryptData(iv, k, meta_mac))
await self.proccess_file(scrape_item, file)
diff --git a/cyberdrop_dl/crawlers/xenforo/__init__.py b/cyberdrop_dl/crawlers/xenforo/__init__.py
index 7b0f60d57..1216fb82f 100644
--- a/cyberdrop_dl/crawlers/xenforo/__init__.py
+++ b/cyberdrop_dl/crawlers/xenforo/__init__.py
@@ -10,16 +10,18 @@ from .titsintops import TitsInTopsCrawler
from .xbunker import XBunkerCrawler
from .xenforo import XenforoCrawler
-__all__ = [
- "AllPornComixCrawler",
- "BellazonCrawler",
- "CelebForumCrawler",
- "F95ZoneCrawler",
- "LeakedModelsCrawler",
- "NudoStarCrawler",
- "SimpCityCrawler",
- "SocialMediaGirlsCrawler",
- "TitsInTopsCrawler",
- "XBunkerCrawler",
- "XenforoCrawler",
-]
+XF_CRAWLERS: set[type[XenforoCrawler]] = {
+ AllPornComixCrawler,
+ BellazonCrawler,
+ CelebForumCrawler,
+ F95ZoneCrawler,
+ LeakedModelsCrawler,
+ NudoStarCrawler,
+ SimpCityCrawler,
+ SocialMediaGirlsCrawler,
+ TitsInTopsCrawler,
+ XBunkerCrawler,
+}
+
+CRAWLERS_MAP = {c.__name__: c for c in XF_CRAWLERS}
+__all__ = list(CRAWLERS_MAP.keys()) # type: ignore[reportUnsupportedDunderAll]
diff --git a/cyberdrop_dl/crawlers/xenforo/allporncomix.py b/cyberdrop_dl/crawlers/xenforo/allporncomix.py
index 82f938736..71bb4c0db 100644
--- a/cyberdrop_dl/crawlers/xenforo/allporncomix.py
+++ b/cyberdrop_dl/crawlers/xenforo/allporncomix.py
@@ -4,7 +4,7 @@ from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
class AllPornComixCrawler(XenforoCrawler):
@@ -12,10 +12,3 @@ class AllPornComixCrawler(XenforoCrawler):
DOMAIN: ClassVar[str] = "allporncomix"
FOLDER_DOMAIN: ClassVar[str] = "AllPornComix"
login_required = False
- post_selectors = PostSelectors(
- content=Selector("div[class=bbWrapper]"),
- images=Selector("img[class*=bbImage]", "data-src"),
- date=Selector("time", "datetime"),
- attachments=Selector("section[class=message-attachments] .attachmentList .file .file-preview", "href"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
diff --git a/cyberdrop_dl/crawlers/xenforo/bellazon.py b/cyberdrop_dl/crawlers/xenforo/bellazon.py
index b83c766b2..037ac7fab 100644
--- a/cyberdrop_dl/crawlers/xenforo/bellazon.py
+++ b/cyberdrop_dl/crawlers/xenforo/bellazon.py
@@ -6,26 +6,28 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+_post_selectors = PostSelectors(
+ article="div[class*=ipsComment_content]",
+ attachments=Selector("a[class*=ipsAttachLink]", "href"),
+ content=Selector("div.cPost_contentWrap"),
+ id=Selector("data-commentid", "data-commentid"),
+ images=Selector("a[class*=ipsAttachLink_image]", "href"),
+ videos=Selector("video.ipsEmbeddedVideo source", "src"),
+)
+
+# TODO: Bellazon uses Invision, not Xenforo. The selectors are completely different.
+# See: https://github.com/jbsparrow/CyberDropDownloader/pull/1079#issuecomment-2982845352
class BellazonCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://www.bellazon.com/main/")
DOMAIN: ClassVar[str] = "bellazon"
FOLDER_DOMAIN: ClassVar[str] = "Bellazon"
- thread_url_part = "topic"
login_required = False
- post_selectors = PostSelectors(
- element="div[class*=ipsComment_content]",
- content=Selector("div[class=cPost_contentWrap]"),
- attachments=Selector("a[class*=ipsAttachLink]", "href"),
- images=Selector("a[class*=ipsAttachLink_image]", "href"),
- videos=Selector("video[class=ipsEmbeddedVideo] source", "src"),
- date=Selector("time", "datetime"),
- number=Selector("data-commentid", "data-commentid"),
- )
- selectors = XenforoSelectors(
- posts=post_selectors,
+
+ XF_THREAD_URL_PART = "topic"
+ XF_POST_URL_PART_NAME = "comment-"
+ XF_SELECTORS = XenforoSelectors(
+ posts=_post_selectors,
title=Selector("span.ipsType_break.ipsContained span"),
- next_page=Selector("li[class=ipsPagination_next] a", "href"),
- post_name="comment-",
+ next_page=Selector("li.ipsPagination_next a[href]", "href"),
)
- ATTACHMENT_URL_PARTS = "attachments", "uploads"
diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.py
index 95970a8f9..fde678e24 100644
--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py
+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py
@@ -1,26 +1,18 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
-
-if TYPE_CHECKING:
- from yarl import URL
+from .xenforo import XenforoCrawler
class CelebForumCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://celebforum.to")
- post_selectors = PostSelectors(
- date=Selector("time", "data-time"),
- images=Selector("a[class*=js-lbImage]", "href"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
DOMAIN: ClassVar[str] = "celebforum"
FOLDER_DOMAIN: ClassVar[str] = "CelebForum"
- def filter_link(self, link: URL) -> URL | None:
+ def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
if link.host == self.PRIMARY_URL.host:
if all(part in link.parts for part in ["data", "attachments"]): # Thumbnails
return None
diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.py
index 87823a409..21a00abfa 100644
--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py
+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py
@@ -5,41 +5,30 @@ from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils.utilities import error_handling_wrapper
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
if TYPE_CHECKING:
from yarl import URL
- from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+_confirmation_data = ({"xhr": "1", "download": "1"},)
class F95ZoneCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://f95zone.to")
- post_selectors = PostSelectors(
- date=Selector("time", "data-time"),
- number=Selector("a[class=u-concealed]", "href"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
DOMAIN: ClassVar[str] = "f95zone"
FOLDER_DOMAIN: ClassVar[str] = "F95Zone"
- def is_confirmation_link(self, link: URL) -> bool:
- return "masked" in link.parts or super().is_confirmation_link(link)
-
@error_handling_wrapper
- async def handle_confirmation_link(self, link: URL, *, origin: ScrapeItem | None = None) -> URL | None:
+ async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
"""Override to handle protected link confirmation."""
async with self.request_limiter:
- data = ({"xhr": "1", "download": "1"},)
- JSON_Resp = await self.client.post_data(self.DOMAIN, link, data=data)
+ json_resp = await self.client.post_data(self.DOMAIN, link, data=_confirmation_data)
- if JSON_Resp["status"] == "ok":
- return self.parse_url(JSON_Resp["msg"])
- return None
+ if json_resp["status"] == "ok":
+ return self.parse_url(json_resp["msg"])
- def filter_link(self, link: URL) -> URL:
+ def filter_link(self, link: AbsoluteHttpURL) -> URL:
if "thumb" in link.parts:
- parts = [x for x in link.parts if x not in ("thumb", "/")]
- new_path = "/".join(parts)
- return link.with_path(new_path)
+ return link.with_path(link.path.replace("/thumb/", ""))
return link
diff --git a/cyberdrop_dl/crawlers/xenforo/leakedmodels.py b/cyberdrop_dl/crawlers/xenforo/leakedmodels.py
index b3ac7ba56..5b1b70900 100644
--- a/cyberdrop_dl/crawlers/xenforo/leakedmodels.py
+++ b/cyberdrop_dl/crawlers/xenforo/leakedmodels.py
@@ -4,14 +4,10 @@ from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
class LeakedModelsCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://leakedmodels.com/forum/")
- post_selectors = PostSelectors(
- date=Selector("time", "data-time"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
DOMAIN: ClassVar[str] = "leakedmodels"
FOLDER_DOMAIN: ClassVar[str] = "LeakedModels"
diff --git a/cyberdrop_dl/crawlers/xenforo/nudostar.py b/cyberdrop_dl/crawlers/xenforo/nudostar.py
index 10e660e8f..96b394837 100644
--- a/cyberdrop_dl/crawlers/xenforo/nudostar.py
+++ b/cyberdrop_dl/crawlers/xenforo/nudostar.py
@@ -4,15 +4,10 @@ from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
class NudoStarCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://nudostar.com/forum/")
- post_selectors = PostSelectors(
- date=Selector("time", "data-time"),
- number=Selector("a[class=u-concealed]", "href"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
DOMAIN: ClassVar[str] = "nudostar"
FOLDER_DOMAIN: ClassVar[str] = "NudoStar"
diff --git a/cyberdrop_dl/crawlers/xenforo/simpcity.py b/cyberdrop_dl/crawlers/xenforo/simpcity.py
index fc988f704..2ccb879d7 100644
--- a/cyberdrop_dl/crawlers/xenforo/simpcity.py
+++ b/cyberdrop_dl/crawlers/xenforo/simpcity.py
@@ -9,7 +9,7 @@ from .xenforo import XenforoCrawler
class SimpCityCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://simpcity.su")
- login_required = False
DOMAIN: ClassVar[str] = "simpcity"
FOLDER_DOMAIN: ClassVar[str] = "SimpCity"
- session_cookie_name = "dontlikebots_user"
+ XF_USER_COOKIE_NAME = "dontlikebots_user"
+ login_required = False
diff --git a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py
index 9ba210386..fb6d28e0d 100644
--- a/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py
+++ b/cyberdrop_dl/crawlers/xenforo/socialmediagirls.py
@@ -4,16 +4,10 @@ from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
class SocialMediaGirlsCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://forums.socialmediagirls.com")
DOMAIN: ClassVar[str] = "socialmediagirls"
FOLDER_DOMAIN: ClassVar[str] = "SocialMediaGirls"
- post_selectors = PostSelectors(
- content=Selector("div[class=bbWrapper]"),
- images=Selector("img[class*=bbImage]", "data-src"),
- date=Selector("time", "data-time"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
diff --git a/cyberdrop_dl/crawlers/xenforo/titsintops.py b/cyberdrop_dl/crawlers/xenforo/titsintops.py
index eb0fe06c2..f7332633c 100644
--- a/cyberdrop_dl/crawlers/xenforo/titsintops.py
+++ b/cyberdrop_dl/crawlers/xenforo/titsintops.py
@@ -2,12 +2,10 @@ from __future__ import annotations
from typing import TYPE_CHECKING, ClassVar
-from yarl import URL
-
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import css
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
if TYPE_CHECKING:
from bs4 import Tag
@@ -17,13 +15,9 @@ class TitsInTopsCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://titsintops.com/phpBB2")
DOMAIN: ClassVar[str] = "titsintops"
FOLDER_DOMAIN: ClassVar[str] = "TitsInTops"
- post_selectors = PostSelectors(
- images=Selector("a[class*=file-preview]", "href"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
- def filter_link(self, link: URL) -> URL:
- return URL(
+ def filter_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ return AbsoluteHttpURL(
str(link)
.replace("index.php%3F", "index.php/")
.replace("index.php?", "index.php/")
@@ -33,13 +27,11 @@ class TitsInTopsCrawler(XenforoCrawler):
def pre_filter_link(self, link: str) -> str:
return link.replace("index.php?", "index.php/").replace("index.php%3F", "index.php/")
- def is_valid_post_link(self, link_obj: Tag) -> bool:
- is_image = link_obj.select_one("img")
+ def is_not_image_or_attachment(self, link_obj: Tag) -> bool:
text = css.get_text(link_obj)
if "view attachment" in text.lower():
return False
- title: str | None = css.get_attr_or_none(link_obj, "title")
+ title = css.get_attr_no_error(link_obj, "title")
if title and "permanent link" in title.lower():
return False
- link_str: str | None = css.get_attr_or_none(link_obj, self.selectors.posts.links.element)
- return not (is_image and self.is_attachment(link_str))
+ return super().is_not_image_or_attachment(link_obj)
diff --git a/cyberdrop_dl/crawlers/xenforo/xbunker.py b/cyberdrop_dl/crawlers/xenforo/xbunker.py
index a1cbac419..dd4320249 100644
--- a/cyberdrop_dl/crawlers/xenforo/xbunker.py
+++ b/cyberdrop_dl/crawlers/xenforo/xbunker.py
@@ -4,16 +4,10 @@ from typing import ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from .xenforo import PostSelectors, Selector, XenforoCrawler, XenforoSelectors
+from .xenforo import XenforoCrawler
class XBunkerCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://xbunker.nu/")
DOMAIN: ClassVar[str] = "xbunker"
FOLDER_DOMAIN: ClassVar[str] = "XBunker"
- post_selectors = PostSelectors(
- content=Selector("div[class=bbWrapper]"),
- images=Selector("img[class*=bbImage], a[class*=js-lbImage]", "data-src"),
- date=Selector("time", "data-time"),
- )
- selectors = XenforoSelectors(posts=post_selectors)
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index 0954c82c4..5c4b36b42 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -3,133 +3,129 @@
from __future__ import annotations
import asyncio
-import contextlib
+import dataclasses
import re
-from dataclasses import astuple, dataclass
-from functools import cached_property, singledispatchmethod
from typing import TYPE_CHECKING, ClassVar
from bs4 import BeautifulSoup, Tag
-from cyberdrop_dl.crawlers.crawler import Crawler
-from cyberdrop_dl.data_structures.url_objects import FORUM, AbsoluteHttpURL, ScrapeItem
-from cyberdrop_dl.exceptions import InvalidURLError, LoginError, ScrapeError
+from cyberdrop_dl.crawlers._forum import ForumCrawler
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.exceptions import LoginError, MaxChildrenError, ScrapeError
from cyberdrop_dl.utils import css
-from cyberdrop_dl.utils.logger import log, log_debug
+from cyberdrop_dl.utils.dates import TimeStamp
from cyberdrop_dl.utils.utilities import (
error_handling_wrapper,
get_text_between,
is_absolute_http_url,
- remove_trailing_slash,
+ is_blob_or_svg,
)
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Sequence
+ from collections.abc import AsyncGenerator, Iterable
from aiohttp_client_cache.response import AnyResponse
- from yarl import URL
- from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
-from cyberdrop_dl.crawlers.crawler import SupportedPaths
+ from cyberdrop_dl.crawlers.crawler import SupportedPaths
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
-HTTP_URL_REGEX_STR = r"https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z]{2,12}\b([-a-zA-Z0-9@:%_+.~#?&/=]*)"
-HTTP_URL_REGEX = re.compile(HTTP_URL_REGEX_STR)
+HTTP_URL_REGEX = re.compile(r"https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z]{2,12}\b([-a-zA-Z0-9@:%_+.~#?&/=]*)")
+LINK_TRASH_MAPPING = {
+ ".th.": ".",
+ ".md.": ".",
+ "ifr": "watch",
+}
-FINAL_PAGE_SELECTOR = "li.pageNav-page a"
-CURRENT_PAGE_SELECTOR = "li.pageNav-page.pageNav-page--current a"
+Selector = css.CssAttributeSelector
+LAST_PAGE_SELECTOR = Selector("li.pageNav-page a:last-of-type", "href")
+CURRENT_PAGE_SELECTOR = Selector("li.pageNav-page.pageNav-page--current a", "href")
-@dataclass(frozen=True, slots=True)
-class Selector:
- element: str
- attribute: str = ""
- @property
- def astuple(self) -> tuple[str, str]:
- return astuple(self)
[email protected](frozen=True, slots=True)
+class PostSelectors:
+ article: str = "article.message[id*=post]" # the entire html of the post (comments, attachments, user avatar, signature, etc...)
-
-@dataclass(frozen=True, slots=True)
-class PostSelectors(Selector):
- element: str = "div[class*=message-main]"
- attachments: Selector = Selector("section[class=message-attachments] a ", "href")
- content: Selector = Selector("div[class*=message-userContent]")
+ attachments: Selector = Selector(".message-attachments a[href]", "href")
+ content: Selector = Selector(".message-userContent") # text, links and images (NO attachments)
date: Selector = Selector("time", "data-timestamp")
- embeds: Selector = Selector("span[data-s9e-mediaembed-iframe]", "data-s9e-mediaembed-iframe")
- iframe: Selector = Selector("iframe[class=saint-iframe]", "href")
- images: Selector = Selector("img[class*=bbImage]", "src")
- links: Selector = Selector("a", "href")
- number: Selector = Selector("li[class=u-concealed] a", "href")
+ embeds: Selector = Selector("iframe", "src")
+ id: Selector = Selector(".u-concealed a[href]", "href") # TODO: This needs a better selector
+ images: Selector = Selector("img.bbImage", "src")
+ lazy_load_embeds: Selector = Selector('[class*=iframe][onclick*="loadMedia(this, \'//"]', "onclick")
+ links: Selector = Selector(":any-link", "href")
videos: Selector = Selector("video source", "src")
- redgifs: Selector = Selector("div[class*='iframe'][onclick*=\"loadMedia(this, '//redgifs\"]", "onclick")
-@dataclass(frozen=True, slots=True)
[email protected](frozen=True, slots=True)
class XenforoSelectors:
- next_page: Selector = Selector("a[class*=pageNav-jump--next]", "href")
+ confirmation_button: Selector = Selector("a[class*=button--cta][href]", "href")
+ next_page: Selector = Selector("a[class*=pageNav-jump--next][href]", "href")
posts: PostSelectors = PostSelectors()
- title: Selector = Selector("h1[class=p-title-value]")
- title_trash: Selector = Selector("span")
quotes: Selector = Selector("blockquote")
- post_name: str = "post-"
+ title_trash: Selector = Selector("span")
+ title: Selector = Selector("h1[class*=p-title-value]")
+ last_page: Selector = LAST_PAGE_SELECTOR
+ current_page: Selector = CURRENT_PAGE_SELECTOR
-@dataclass(frozen=True)
[email protected](frozen=True, slots=True, order=True)
class ForumPost:
- soup: Tag
- selectors: PostSelectors
- post_name: str = "post-"
-
- @cached_property
- def content(self) -> Tag:
- return css.select_one(self.soup, self.selectors.content.element)
-
- @cached_property
- def date(self) -> int | None:
- if date_str := css.select_one_get_attr_or_none(self.content, *self.selectors.date.astuple):
- return int(date_str)
-
- @cached_property
- def number(self) -> int:
- if self.selectors.number.element == self.selectors.number.attribute:
- number_str: str = css.get_attr(self.soup, self.selectors.number.element)
- return int(number_str)
-
- number_str = css.select_one_get_attr(self.soup, *self.selectors.number.astuple)
- return int(number_str.split("/")[-1].split(self.post_name)[-1])
+ id: int
+ date: TimeStamp
+ article: Tag = dataclasses.field(compare=False)
+ content: Tag = dataclasses.field(compare=False)
+
+ @staticmethod
+ def new(article: Tag, selectors: PostSelectors, post_name: str = "post") -> ForumPost:
+ content = css.select_one(article, selectors.content.element)
+ date = TimeStamp(int(css.select_one_get_attr(article, *selectors.date)))
+ if selectors.id.element == selectors.id.attribute:
+ post_id = int(css.get_attr(article, selectors.id.element))
+ else:
+ id_str = css.select_one_get_attr(article, *selectors.id)
+ post_id = int(id_str.rsplit(post_name, 1)[-1].replace("-", ""))
+ return ForumPost(post_id, date, article, content)
-@dataclass(frozen=True, slots=True)
-class ThreadInfo:
[email protected](frozen=True, slots=True)
+class Thread:
+ id: int
name: str
- id_: int
page: int
- post: int
+ post_id: int | None
url: AbsoluteHttpURL
- complete_url: URL
-@dataclass(frozen=True, slots=True)
-class ForumThreadPage:
- thread: ThreadInfo
- posts: Sequence[Tag]
+DEFAULT_XF_SELECTORS = XenforoSelectors()
+KNOWN_THREAD_PART_NAMES = "thread", "topic", "tema"
+KNOWN_THREAD_PART_NAMES = ",".join(f"{part},{part}s" for part in KNOWN_THREAD_PART_NAMES).split(",")
+
+def _escape(strings: Iterable[str]) -> str:
+ return r"\|".join(strings)
-class XenforoCrawler(Crawler, is_abc=True):
+
+class XenforoCrawler(ForumCrawler, is_abc=True):
+ XF_ATTACHMENT_URL_PARTS = "attachments", "data", "uploads"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Attachments": ("/attachments/...", "/data/..."),
- "Threads": ("/threads/<thread_name>", "/posts/<post_id>", "/goto/<post_id>"),
+ "Attachments": f"/({_escape(XF_ATTACHMENT_URL_PARTS)})/...",
+ "Threads": (
+ f"/({_escape(KNOWN_THREAD_PART_NAMES)})/<thread_name_and_id>",
+ "/posts/<post_id>",
+ "/goto/<post_id>",
+ ),
+ "**NOTE**": "base crawler: Xenforo",
}
+ SUPPORTS_THREAD_RECURSION: ClassVar[bool] = True
+ XF_SELECTORS = DEFAULT_XF_SELECTORS
+ XF_POST_URL_PART_NAME = "post-"
+ XF_PAGE_URL_PART_NAME = "page-"
+ XF_THREAD_URL_PART = "threads"
+ XF_USER_COOKIE_NAME = "xf_user"
+ XF_ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"
login_required = True
- selectors = XenforoSelectors()
- POST_NAME = "post-"
- PAGE_NAME = "page-"
- thread_url_part = "threads"
- session_cookie_name = "xf_user"
-
- ATTACHMENT_URL_PARTS = "attachments", "data"
- ATTACHMENT_HOSTS = "smgmedia", "attachments.f95zone"
def __post_init__(self) -> None:
self.scraped_threads = set()
@@ -139,16 +135,10 @@ class XenforoCrawler(Crawler, is_abc=True):
login_url = self.PRIMARY_URL / "login"
await self.login_setup(login_url)
- async def is_not_last_page(response: AnyResponse) -> bool:
- soup = BeautifulSoup(await response.text(), "html.parser")
- try:
- last_page = int(soup.select(FINAL_PAGE_SELECTOR)[-1].text.split("page-")[-1])
- current_page = int(soup.select(CURRENT_PAGE_SELECTOR)[0].text.split("page-")[-1])
- except (AttributeError, IndexError):
- return False
- return current_page != last_page
+ self.register_cache_filter(self.PRIMARY_URL, check_is_not_last_page)
- self.register_cache_filter(self.PRIMARY_URL, is_not_last_page)
+ def get_filename_and_ext(self, filename: str) -> tuple[str, str]:
+ return super().get_filename_and_ext(filename, forum=True)
async def fetch(self, scrape_item: ScrapeItem) -> None:
if not self.logged_in and self.login_required:
@@ -157,21 +147,27 @@ class XenforoCrawler(Crawler, is_abc=True):
scrape_item.url = self.parse_url(url)
if self.is_attachment(scrape_item.url):
return await self.handle_internal_link(scrape_item)
- if self.thread_url_part in scrape_item.url.parts:
- return await self.thread(scrape_item)
- if self.is_confirmation_link(scrape_item.url):
- url = await self.handle_confirmation_link(scrape_item.url)
- if url: # If there was an error, this will be None
- # This could end up back in here if the URL goes to another thread
- scrape_item.url = url
- return self.handle_external_links(scrape_item)
- if any(p in scrape_item.url.parts for p in ("goto", "posts")):
- return await self.redirect(scrape_item)
+ if is_confirmation_link(scrape_item.url):
+ return await self.follow_confirmation_link(scrape_item)
+
+ thread_part_index = len(self.PRIMARY_URL.parts)
+ match scrape_item.url.parts[thread_part_index:]:
+ case [thread_part, thread_name_and_id, *_] if thread_part in KNOWN_THREAD_PART_NAMES:
+ return await self.thread(scrape_item, thread_name_and_id)
+ case ["goto" | "posts", *_]:
+ return await self.follow_redirect_w_get(scrape_item)
raise ValueError
+ async def follow_confirmation_link(self, scrape_item: ScrapeItem) -> None:
+ url = await self.resolve_confirmation_link(scrape_item.url)
+ if url: # If there was an error, this will be None
+ scrape_item.url = url
+ # This could end up back in here if the URL goes to another thread
+ return self.handle_external_links(scrape_item)
+
@error_handling_wrapper
- async def redirect(self, scrape_item: ScrapeItem) -> None:
+ async def follow_redirect_w_get(self, scrape_item: ScrapeItem) -> None:
async with self.request_limiter:
response, _ = await self.client._get_response_and_soup(self.DOMAIN, scrape_item.url)
assert is_absolute_http_url(response.url)
@@ -179,52 +175,69 @@ class XenforoCrawler(Crawler, is_abc=True):
self.manager.task_group.create_task(self.run(scrape_item))
@error_handling_wrapper
- async def thread(self, scrape_item: ScrapeItem) -> None:
- scrape_item.set_type(FORUM, self.manager)
- thread = self.get_thread_info(scrape_item.url)
- title = None
- last_post_url = thread.url
+ async def follow_redirect_w_head(self, scrape_item: ScrapeItem) -> None:
+ head = await self.client.get_head(self.DOMAIN, scrape_item.url)
+ redirect = head.get("location")
+ if not redirect:
+ raise ScrapeError(422)
+ scrape_item.url = self.parse_url(redirect)
+ self.manager.task_group.create_task(self.run(scrape_item))
+
+ async def forum(self, scrape_item: ScrapeItem) -> None:
+ raise NotImplementedError
+
+ @error_handling_wrapper
+ async def thread(self, scrape_item: ScrapeItem, thread_name_and_id: str) -> None:
+ scrape_item.setup_as_forum("")
+ thread = parse_thread(
+ scrape_item.url, thread_name_and_id, self.XF_PAGE_URL_PART_NAME, self.XF_POST_URL_PART_NAME
+ )
if thread.url in self.scraped_threads:
return
+
scrape_item.parent_threads.add(thread.url)
+ if self.scrape_single_forum_post and not thread.post_id:
+ msg = "`--scrape-single-forum-post` is `True`, but the provided URL has no post id"
+ raise ScrapeError("User Error", msg)
+
+ title: str = ""
self.scraped_threads.add(thread.url)
+ last_post_url = thread.url
async for soup in self.thread_pager(scrape_item):
if not title:
- title_block = css.select_one(soup, self.selectors.title.element)
try:
- trash = title_block.select(self.selectors.title_trash.element)
- except AttributeError as e:
- log_debug("Got an unprocessable soup", 40, exc_info=e)
- raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e
-
- for element in trash:
- element.decompose()
-
- title = self.create_title(title_block.get_text().replace("\n", ""), thread_id=thread.id_)
+ title = self.create_title(get_post_title(soup, self.XF_SELECTORS), thread_id=thread.id)
+ except ScrapeError as e:
+ self.log_debug("Got an unprocessable soup", 40, exc_info=e)
+ raise
scrape_item.add_to_parent_title(title)
- posts = soup.select(self.selectors.posts.element)
- forum_thread_page = ForumThreadPage(thread, posts)
- continue_scraping, last_post_url = self.process_thread_page(scrape_item, forum_thread_page)
+ continue_scraping, last_post_url = self.process_thread_page(scrape_item, thread, soup)
if not continue_scraping:
break
await self.write_last_forum_post(thread.url, last_post_url)
- def process_thread_page(self, scrape_item: ScrapeItem, forum_page: ForumThreadPage) -> tuple[bool, URL]:
+ def process_thread_page(
+ self, scrape_item: ScrapeItem, thread: Thread, soup: BeautifulSoup
+ ) -> tuple[bool, AbsoluteHttpURL]:
continue_scraping = False
- thread = forum_page.thread
post_url = thread.url
- for post in forum_page.posts:
- current_post = ForumPost(post, selectors=self.selectors.posts, post_name=self.POST_NAME)
- continue_scraping, scrape_post = self.check_post_number(thread.post, current_post.number)
- date = current_post.date
- post_string = f"{self.POST_NAME}{current_post.number}"
- post_url = thread.url / post_string
- if scrape_post:
- new_scrape_item = scrape_item.create_new(thread.url, possible_datetime=date, add_parent=post_url)
+ for article in soup.select(self.XF_SELECTORS.posts.article):
+ current_post = ForumPost.new(article, self.XF_SELECTORS.posts, self.XF_POST_URL_PART_NAME)
+ continue_scraping, scrape_this_post = check_post_id(
+ thread.post_id, current_post.id, self.scrape_single_forum_post
+ )
+ if scrape_this_post:
+ post_url = thread.url / f"{self.XF_POST_URL_PART_NAME}{current_post.id}"
+ new_scrape_item = scrape_item.create_new(
+ thread.url, possible_datetime=current_post.date, add_parent=post_url
+ )
self.manager.task_group.create_task(self.post(new_scrape_item, current_post))
- scrape_item.add_children()
+ try:
+ scrape_item.add_children()
+ except MaxChildrenError:
+ break
if not continue_scraping:
break
@@ -233,228 +246,132 @@ class XenforoCrawler(Crawler, is_abc=True):
@error_handling_wrapper
async def post(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
scrape_item.setup_as_post("")
- post_title = self.create_separate_post_title(None, str(post.number), post.date)
+ post_title = self.create_separate_post_title(None, str(post.id), post.date)
scrape_item.add_to_parent_title(post_title)
- posts_scrapers = [self.attachments, self.embeds, self.images, self.links, self.videos, self.hidden_redgifs]
- for scraper in posts_scrapers:
+ for scraper in (
+ self._attachments,
+ self._embeds,
+ self._images,
+ self._links,
+ self._videos,
+ self._lazy_load_embeds,
+ ):
await scraper(scrape_item, post)
- async def links(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = post.selectors.links
+ async def _links(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
+ selector = self.XF_SELECTORS.posts.links
links = post.content.select(selector.element)
- links = [link for link in links if self.is_valid_post_link(link)]
+ links = [link for link in links if self.is_not_image_or_attachment(link)]
await self.process_children(scrape_item, links, selector.attribute)
- async def images(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = post.selectors.images
+ async def _images(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
+ selector = self.XF_SELECTORS.posts.images
images = post.content.select(selector.element)
await self.process_children(scrape_item, images, selector.attribute)
- async def videos(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = post.selectors.videos
- iframe_selector = post.selectors.iframe
+ async def _videos(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
+ selector = self.XF_SELECTORS.posts.videos
videos = post.content.select(selector.element)
- videos.extend(post.content.select(iframe_selector.element))
await self.process_children(scrape_item, videos, selector.attribute)
- async def embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = post.selectors.embeds
+ async def _embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
+ selector = self.XF_SELECTORS.posts.embeds
embeds = post.content.select(selector.element)
await self.process_children(scrape_item, embeds, selector.attribute, embeds=True)
- async def attachments(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = post.selectors.attachments
- attachments = post.content.select(selector.attribute)
+ async def _attachments(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
+ selector = self.XF_SELECTORS.posts.attachments
+ attachments = post.article.select(selector.element)
await self.process_children(scrape_item, attachments, selector.attribute)
- async def hidden_redgifs(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
- selector = post.selectors.redgifs
+ async def _lazy_load_embeds(self, scrape_item: ScrapeItem, post: ForumPost) -> None:
+ selector = self.XF_SELECTORS.posts.lazy_load_embeds
for redgif in css.iselect(post.content, selector.element):
link_str = get_text_between(css.get_attr(redgif, selector.attribute), "loadMedia(this, '", "')")
- await self.process_child(scrape_item, link_str)
+ await self.process_child(scrape_item, link_str, embeds=True)
async def thread_pager(self, scrape_item: ScrapeItem) -> AsyncGenerator[BeautifulSoup]:
- """Generator of forum thread pages."""
-
- def get_next_page(soup: BeautifulSoup) -> str | None:
- select, attr = self.selectors.next_page.astuple
- next_page = css.select_one_get_attr_or_none(soup, selector=select, attribute=attr)
- if next_page:
- return self.pre_filter_link(next_page)
-
- async for soup in self._web_pager(scrape_item.url, get_next_page):
+ async for soup in self._web_pager(scrape_item.url, self.get_next_page):
yield soup
+ def get_next_page(self, soup: BeautifulSoup) -> str | None:
+ if next_page := css.select_one_get_attr_or_none(soup, *self.XF_SELECTORS.next_page):
+ return self.pre_filter_link(next_page)
+
+ @error_handling_wrapper
async def process_children(
- self, scrape_item: ScrapeItem, links: list[Tag], selector: str, *, embeds: bool = False
+ self, scrape_item: ScrapeItem, links: list[Tag], attribute: str, *, embeds: bool = False
) -> None:
- for link_obj in links:
- link_tag = link_obj.get(selector)
- if isinstance(link_tag, Tag):
- if link_tag.parent and (data_simp := link_tag.parent.get("data-simp")) and "init" in data_simp:
- continue
-
- link_str: str | None = css.get_attr_or_none(link_obj, "href")
- elif isinstance(link_tag, str):
- link_str = link_tag
- else:
- continue
-
- if not link_str:
- continue
-
- assert isinstance(link_str, str)
- if embeds:
- link_str = self.extract_embed_url(link_str)
-
- if not link_str or link_str.startswith("data:image/svg"):
- continue
-
- await self.process_child(scrape_item, link_str)
+ for link_tag in links:
+ if link_str := css.get_attr_no_error(link_tag, attribute):
+ await self.process_child(scrape_item, link_str, embeds=embeds)
@error_handling_wrapper
- async def process_child(self, scrape_item: ScrapeItem, link_str: str) -> None:
- link = await self.get_absolute_link(link_str)
+ async def process_child(self, scrape_item: ScrapeItem, link_str: str, *, embeds: bool = False) -> None:
+ link_str_ = pre_process_child(link_str, embeds)
+ if not link_str_:
+ return
+ link = await self.get_absolute_link(link_str_)
link = self.filter_link(link)
if not link:
return
- new_scrape_item = scrape_item.create_new(link)
- await self.handle_link(new_scrape_item)
- scrape_item.add_children()
+ await self.handle_link(scrape_item, link)
- @singledispatchmethod
- def is_attachment(self, link: AbsoluteHttpURL) -> bool:
+ def is_attachment(self, link: AbsoluteHttpURL | str) -> bool:
if not link:
return False
- parts = self.ATTACHMENT_URL_PARTS
- hosts = self.ATTACHMENT_HOSTS
- return any(p in link.parts for p in parts) or any(h in link.host for h in hosts)
-
- @is_attachment.register
- def _(self, link_str: str) -> bool:
- if not link_str:
- return False
- link = self.parse_url(link_str)
- return self.is_attachment(link)
+ if isinstance(link, str):
+ link = self.parse_url(link)
+ by_parts = len(link.parts) > 2 and any(p in link.parts for p in self.XF_ATTACHMENT_URL_PARTS)
+ by_host = any(host in link.host for host in self.XF_ATTACHMENT_HOSTS)
+ return by_parts or by_host
async def get_absolute_link(self, link: str | AbsoluteHttpURL) -> AbsoluteHttpURL | None:
if isinstance(link, str):
- absolute_link = self.str_to_url(link)
+ absolute_link = self.parse_url(clean_link_str(link))
else:
absolute_link = link
- if self.is_confirmation_link(absolute_link):
- absolute_link = await self.handle_confirmation_link(absolute_link)
+ if is_confirmation_link(absolute_link):
+ return await self.resolve_confirmation_link(absolute_link)
return absolute_link
- def str_to_url(self, link: str) -> AbsoluteHttpURL:
- link_str = link
- text_to_replace = [(".th.", "."), (".md.", "."), ("ifr", "watch")]
- for old, new in text_to_replace:
- link_str = link_str.replace(old, new)
- return self.parse_url(link_str)
-
- @error_handling_wrapper
- async def handle_link(self, scrape_item: ScrapeItem) -> None:
- if not scrape_item.url or scrape_item.url == self.PRIMARY_URL:
- return
- if not scrape_item.url.host:
- raise InvalidURLError("url has no host")
- if self.is_attachment(scrape_item.url):
- return await self.handle_internal_link(scrape_item)
- if self.PRIMARY_URL.host in scrape_item.url.host and self.stop_thread_recursion(scrape_item):
- origin = scrape_item.parents[0]
- return log(f"Skipping nested thread URL {scrape_item.url} found on {origin}", 10)
- scrape_item.set_type(None, self.manager)
- self.handle_external_links(scrape_item)
-
- @error_handling_wrapper
- async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:
- """Handles internal links."""
- scrape_item.url = remove_trailing_slash(scrape_item.url)
-
- if scrape_item.url.name.isdigit():
- head = await self.client.get_head(self.DOMAIN, scrape_item.url)
- redirect = head.get("location")
- if not redirect:
- raise ScrapeError(422)
- scrape_item.url = self.parse_url(redirect)
- self.manager.task_group.create_task(self.run(scrape_item))
- return
-
- filename, ext = self.get_filename_and_ext(scrape_item.url.name, forum=True)
- scrape_item.add_to_parent_title("Attachments")
- scrape_item.part_of_album = True
- await self.handle_file(scrape_item.url, scrape_item, filename, ext)
-
@error_handling_wrapper
- async def handle_confirmation_link(self, link: URL) -> AbsoluteHttpURL | None:
- """Handles link confirmation."""
+ async def resolve_confirmation_link(self, link: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
async with self.request_limiter:
soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, link)
- confirm_button = soup.select_one("a[class*=button--cta]")
+ selector = self.XF_SELECTORS.confirmation_button
+ confirm_button = soup.select_one(selector.element)
if not confirm_button:
return
- link_str: str = css.get_attr(confirm_button, "href")
+ link_str: str = css.get_attr(confirm_button, selector.attribute)
link_str = link_str.split('" class="link link--internal', 1)[0]
new_link = self.parse_url(link_str)
return await self.get_absolute_link(new_link)
- def stop_thread_recursion(self, scrape_item: ScrapeItem) -> bool:
- max_thread_depth = self.manager.config_manager.settings_data.download_options.maximum_thread_depth
- if not max_thread_depth:
- return True
- if len(scrape_item.parent_threads) > max_thread_depth:
- return True
- return False
-
- def check_post_number(self, post_number: int, current_post_number: int) -> tuple[bool, bool]:
- """Checks if the program should scrape the current post.
-
- Returns (continue_scraping, scrape_post)"""
- scrape_single_forum_post = self.manager.config_manager.settings_data.download_options.scrape_single_forum_post
- scrape_post = continue_scraping = True
- if scrape_single_forum_post:
- if not post_number or post_number == current_post_number:
- continue_scraping = False
- else:
- scrape_post = False
-
- elif post_number and post_number > current_post_number:
- scrape_post = False
-
- return continue_scraping, scrape_post
-
- async def write_last_forum_post(self, thread_url: URL, last_post_url: URL | None) -> None:
- if not last_post_url or last_post_url == thread_url:
- return
- await self.manager.log_manager.write_last_post_log(last_post_url)
-
- def is_valid_post_link(self, link_obj: Tag) -> bool:
- is_image = link_obj.select_one("img")
- link_str: str = css.get_attr_or_none(link_obj, self.selectors.posts.links.element)
- return not (is_image or self.is_attachment(link_str))
+ async def handle_internal_link(self, scrape_item: ScrapeItem) -> None:
+ """Handles internal links."""
+ slug = scrape_item.url.name or scrape_item.url.parent.name
+ if slug.isdigit():
+ return await self.follow_redirect_w_head(scrape_item)
- def is_confirmation_link(self, link: URL) -> bool:
- return any(keyword in link.path for keyword in ("link-confirmation",))
+ return await super().handle_internal_link(scrape_item)
- def extract_embed_url(self, embed_str: str) -> str | None:
- embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")
- if match := re.search(HTTP_URL_REGEX, embed_str):
- return match.group(0).replace("www.", "")
- return embed_str
+ def is_not_image_or_attachment(self, link_obj: Tag) -> bool:
+ is_image = link_obj.select_one("img")
+ if link_str := css.get_attr_no_error(link_obj, self.XF_SELECTORS.posts.links.element):
+ return not (is_image or self.is_attachment(link_str))
+ return not is_image
def pre_filter_link(self, link: str) -> str:
return link
- def filter_link(self, link: URL | None) -> URL | None:
+ def filter_link(self, link: AbsoluteHttpURL | None) -> AbsoluteHttpURL | None:
return link
- """ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-
@error_handling_wrapper
- async def login_setup(self, login_url: URL) -> None:
+ async def login_setup(self, login_url: AbsoluteHttpURL) -> None:
host_cookies: dict = self.client.client_manager.cookies.filter_cookies(self.PRIMARY_URL)
- session_cookie = host_cookies.get(self.session_cookie_name)
+ session_cookie = host_cookies.get(self.XF_USER_COOKIE_NAME)
session_cookie = session_cookie.value if session_cookie else None
msg = f"No cookies found for {self.FOLDER_DOMAIN}"
if not session_cookie and self.login_required:
@@ -470,14 +387,11 @@ class XenforoCrawler(Crawler, is_abc=True):
raise LoginError(message=msg)
msg += " Scraping without an account"
- log(msg, 30)
+ self.log(msg, 30)
@error_handling_wrapper
- async def forum_login(self, login_url: URL, session_cookie: str, username: str, password: str) -> None:
+ async def xf_login(self, login_url: AbsoluteHttpURL, session_cookie: str, username: str, password: str) -> None:
"""Logs in to a forum."""
-
- attempt = 0
- retries = wait_time = 5
manual_login = username and password
missing_credentials = not (manual_login or session_cookie)
if missing_credentials:
@@ -488,73 +402,163 @@ class XenforoCrawler(Crawler, is_abc=True):
cookies = {"xf_user": session_cookie}
self.update_cookies(cookies)
+ credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}
+ await self.xf_try_login(login_url, credentials, retries=5)
+
+ async def xf_try_login(
+ self,
+ login_url: AbsoluteHttpURL,
+ credentials: dict[str, str],
+ retries: int,
+ wait_time: int | None = None,
+ ) -> None:
+ # Try from cookies
text, logged_in = await self.check_login_with_request(login_url)
if logged_in:
self.logged_in = True
return
- credentials = {"login": username, "password": password, "_xfRedirect": str(self.PRIMARY_URL)}
-
- def prepare_login_data(resp_text) -> dict:
- soup = BeautifulSoup(resp_text, "html.parser")
- inputs = soup.select("form input")
- data: dict = {elem["name"]: elem["value"] for elem in inputs if elem.get("name") and elem.get("value")}
- return data | credentials
-
+ wait_time = wait_time or retries
+ attempt = 0
while attempt < retries:
- with contextlib.suppress(TimeoutError):
+ try:
attempt += 1
await asyncio.sleep(wait_time)
- data = prepare_login_data(text)
+ data = parse_login_form(text) | credentials
_ = await self.client._post_data(self.DOMAIN, login_url / "login", data=data, cache_disabled=True)
await asyncio.sleep(wait_time)
text, logged_in = await self.check_login_with_request(login_url)
if logged_in:
self.logged_in = True
return
-
+ except TimeoutError:
+ continue
msg = f"Failed to login on {self.FOLDER_DOMAIN} after {retries} attempts"
raise LoginError(message=msg)
- async def check_login_with_request(self, login_url: URL) -> tuple[str, bool]:
+ async def check_login_with_request(self, login_url: AbsoluteHttpURL) -> tuple[str, bool]:
text = await self.client.get_text(self.DOMAIN, login_url, cache_disabled=True)
return text, any(p in text for p in ('<span class="p-navgroup-user-linkText">', "You are already logged in."))
- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
- def get_thread_info(self, url: AbsoluteHttpURL) -> ThreadInfo:
- name_index = url.parts.index(self.thread_url_part) + 1
- name, id_ = get_thread_name_and_id(url, name_index)
- thread_url = get_thread_canonical_url(url, name_index)
- page, post = get_thread_page_and_post(url, name_index, self.PAGE_NAME, self.POST_NAME)
- return ThreadInfo(name, id_, page, post, thread_url, url)
+def parse_thread(url: AbsoluteHttpURL, thread_name_and_id: str, page_part_name: str, post_part_name: str) -> Thread:
+ name_index = url.parts.index(thread_name_and_id)
+ name, id_ = parse_thread_name_and_id(thread_name_and_id)
+ page, post_id = get_thread_page_and_post(url, name_index, page_part_name, post_part_name)
+ canonical_url = get_thread_canonical_url(url, name_index)
+ return Thread(id_, name, page, post_id, canonical_url)
-def get_thread_name_and_id(url: URL, thread_name_index: int) -> tuple[str, int]:
- thread_name, thread_id_str = url.parts[thread_name_index].rsplit(".", 1)
- thread_id = int(thread_id_str)
- return thread_name, thread_id
+def parse_thread_name_and_id(thread_name_and_id: str) -> tuple[str, int]:
+ try:
+ name, id_str = thread_name_and_id.rsplit(".", 1)
+ except ValueError:
+ id_str, name = thread_name_and_id.split("-", 1)
+ return name, int(id_str)
def get_thread_canonical_url(url: AbsoluteHttpURL, thread_name_index: int) -> AbsoluteHttpURL:
new_parts = url.parts[1 : thread_name_index + 1]
new_path = "/".join(new_parts)
- thread_url = url.with_path(new_path).with_fragment(None).with_query(None)
- return thread_url
+ return url.with_path(new_path)
-def get_thread_page_and_post(url: URL, thread_name_index: int, page_name: str, post_name: str) -> tuple[int, int]:
- post_or_page_index = thread_name_index + 1
- extra_parts = set(url.parts[post_or_page_index:])
- sections = {url.fragment} if url.fragment else set()
- sections.update(extra_parts)
+def get_thread_page_and_post(
+ url: AbsoluteHttpURL, thread_name_index: int, page_name: str, post_name: str
+) -> tuple[int, int | None]:
+ extra_parts = url.parts[thread_name_index + 1 :]
+ if url.fragment:
+ extra_parts = *extra_parts, url.fragment
- def find_number(search_value: str) -> int:
- for sec in sections:
+ def find_number(search_value: str) -> int | None:
+ for sec in extra_parts:
if search_value in sec:
- return int(sec.replace(search_value, "").strip())
- return 0
+ return int(sec.replace(search_value, "").replace("-", "").strip())
+
+ post_id = find_number(post_name)
+ page_number = find_number(page_name) or 1
+ return page_number, post_id
+
+
+async def check_is_not_last_page(response: AnyResponse) -> bool:
+ soup = BeautifulSoup(await response.text(), "html.parser")
+ return not is_last_page(soup)
+
+
+def is_last_page(soup: BeautifulSoup) -> bool:
+ try:
+ last_page = css.select_one_get_attr(soup, *LAST_PAGE_SELECTOR)
+ current_page = css.select_one_get_attr(soup, *CURRENT_PAGE_SELECTOR)
+ except (AttributeError, IndexError, css.SelectorError):
+ return True
+ return current_page == last_page
+
+
+def get_post_title(soup: BeautifulSoup, selectors: XenforoSelectors) -> str:
+ try:
+ title_block = css.select_one(soup, selectors.title.element)
+ trash = title_block.select(selectors.title_trash.element)
+ for element in trash:
+ element.decompose()
+ except (AttributeError, AssertionError, css.SelectorError) as e:
+ raise ScrapeError(429, message="Invalid response from forum. You may have been rate limited") from e
+
+ if title := " ".join(css.get_text(title_block).split()):
+ return title
+ raise ScrapeError(422)
+
+
+def extract_embed_url(embed_str: str) -> str:
+ embed_str = embed_str.replace(r"\/\/", "https://www.").replace("\\", "")
+ if match := re.search(HTTP_URL_REGEX, embed_str):
+ return match.group(0).replace("www.", "")
+ return embed_str
+
+
+# TODO: move to parse_url
+def clean_link_str(link: str) -> str:
+ for old, new in LINK_TRASH_MAPPING.items():
+ link = link.replace(old, new)
+ return link
+
+
+def parse_login_form(resp_text: str) -> dict[str, str]:
+ soup = BeautifulSoup(resp_text, "html.parser")
+ inputs = soup.select("form:first-of-type input")
+ data = {
+ name: value
+ for elem in inputs
+ if (name := css.get_attr_or_none(elem, "name")) and (value := css.get_attr_or_none(elem, "value"))
+ }
+ if data:
+ return data
+ raise ScrapeError(422)
+
+
+def is_confirmation_link(link: AbsoluteHttpURL) -> bool:
+ return "masked" in link.parts or "link-confirmation" in link.path
+
+
+def check_post_id(init_post_id: int | None, current_post_id: int, scrape_single_forum_post: bool) -> tuple[bool, bool]:
+ """Checks if the program should scrape the current post.
+
+ Returns (continue_scraping, scrape_this_post)"""
+ if init_post_id:
+ if init_post_id > current_post_id:
+ return (True, False)
+ elif init_post_id == current_post_id:
+ return (not scrape_single_forum_post, True)
+ else:
+ return (not scrape_single_forum_post, not scrape_single_forum_post)
+
+ assert not scrape_single_forum_post # We should have raised an exception early
+ return True, True
+
+
+def pre_process_child(link_str: str, embeds: bool = False) -> str | None:
+ assert isinstance(link_str, str)
+ if embeds:
+ link_str = extract_embed_url(link_str)
- post_number = find_number(post_name)
- page_number = find_number(page_name)
- return page_number, post_number
+ if link_str and not is_blob_or_svg(link_str):
+ return link_str
diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.py
index 2f7106dcd..f0e54da62 100644
--- a/cyberdrop_dl/downloader/mega_nz.py
+++ b/cyberdrop_dl/downloader/mega_nz.py
@@ -642,7 +642,7 @@ class MegaApi:
if file["t"] == NodeType.FILE:
file = cast("File", file)
k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], key[3] ^ key[7])
- file["iv"] = key[4:6] + (0, 0)
+ file["iv"] = (*key[4:6], 0, 0)
file["meta_mac"] = key[6:8]
# folder
else:
@@ -735,7 +735,7 @@ class MegaApi:
elif node["t"] == NodeType.FOLDER:
k = key
- iv: U32IntSequence = key[4:6] + (0, 0)
+ iv: U32IntSequence = (*key[4:6], 0, 0)
meta_mac: U32IntTupleArray = key[6:8]
attrs = decrypt_attr(base64_url_decode(node["a"]), k)
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index f0ba0e7f4..b32227426 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -1,10 +1,12 @@
from __future__ import annotations
import functools
-from typing import TYPE_CHECKING, ParamSpec, TypeVar
+from typing import TYPE_CHECKING, NamedTuple, ParamSpec, TypeVar
import bs4.css
+from cyberdrop_dl.exceptions import ScrapeError
+
if TYPE_CHECKING:
from collections.abc import Callable, Generator
@@ -14,19 +16,34 @@ P = ParamSpec("P")
R = TypeVar("R")
+class SelectorError(ScrapeError):
+ def __init__(self, message: str | None = None) -> None:
+ super().__init__(422, message)
+
+
+class CssAttributeSelector(NamedTuple):
+ element: str
+ attribute: str = ""
+
+ def __call__(self, soup: Tag) -> str:
+ return select_one_get_attr(soup, self.element, self.attribute)
+
+
def not_none(func: Callable[P, R | None]) -> Callable[P, R]:
@functools.wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
result = func(*args, **kwargs)
- assert result is not None
+ if result is None:
+ raise SelectorError
return result
return wrapper
-def select_one(tag: Tag, selector: str) -> Tag:
+@not_none
+def select_one(tag: Tag, selector: str) -> Tag | None:
"""Same as `tag.select_one` but asserts the result is not `None`"""
- return not_none(tag.select_one)(selector)
+ return tag.select_one(selector)
def select_one_get_text(tag: Tag, selector: str, strip: bool = True) -> str:
@@ -34,20 +51,34 @@ def select_one_get_text(tag: Tag, selector: str, strip: bool = True) -> str:
return get_text(select_one(tag, selector), strip)
+# TODO: Rename this to just get_attr
+# get_attr_no_error should be get_attr_or_none. `or_none` implies no error
def get_attr_or_none(tag: Tag, attribute: str) -> str | None:
- """Same as `tag.get(attribute)` but asserts the result is not multiple strings"""
- value = tag.get(attribute)
- assert not isinstance(value, list)
+ """Same as `tag.get(attribute)` but asserts the result is a single str"""
+ if attribute == "src":
+ value = tag.get("data-src") or tag.get(attribute)
+ else:
+ value = tag.get(attribute)
+ if isinstance(value, list):
+ raise SelectorError(f"Expected a single value for {attribute = !r}, got multiple")
return value
+def get_attr_no_error(tag: Tag, attribute: str) -> str | None:
+ try:
+ return get_attr_or_none(tag, attribute)
+ except Exception:
+ return
+
+
def get_text(tag: Tag, strip: bool = True) -> str:
return tag.get_text(strip=strip)
-def get_attr(tag: Tag, attribute: str) -> str:
+@not_none
+def get_attr(tag: Tag, attribute: str) -> str | None:
"""Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""
- return not_none(get_attr_or_none)(tag, attribute)
+ return get_attr_or_none(tag, attribute)
def select_one_get_attr(tag: Tag, selector: str, attribute: str) -> str:
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 61e8ffe12..80255e0cb 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -56,6 +56,7 @@ R = TypeVar("R")
TEXT_EDITORS = "micro", "nano", "vim" # Ordered by preference
ALLOWED_FILEPATH_PUNCTUATION = " .-_!#$%'()+,;=@[]^{}~"
+_BLOB_OR_SVG = ("data:", "blob:", "javascript:")
subprocess_get_text = partial(subprocess.run, capture_output=True, text=True, check=False)
@@ -82,8 +83,8 @@ class OGProperties(dict[str, str]):
def error_handling_wrapper(
- func: Callable[..., Coroutine[None, None, R]],
-) -> Callable[..., Coroutine[None, None, R | None]]:
+ func: Callable[P, Coroutine[None, None, R]],
+) -> Callable[P, Coroutine[None, None, R | None]]:
"""Wrapper handles errors for url scraping."""
@wraps(func)
@@ -613,6 +614,10 @@ def get_os_common_name() -> str:
return default
+def is_blob_or_svg(link: str) -> bool:
+ return any(link.startswith(x) for x in _BLOB_OR_SVG)
+
+
log_cyan = partial(log_with_color, style="cyan", level=20)
log_yellow = partial(log_with_color, style="yellow", level=20)
log_green = partial(log_with_color, style="green", level=20)
diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.md
index 27db21293..1de47f7f0 100644
--- a/docs/reference/cli-arguments.md
+++ b/docs/reference/cli-arguments.md
@@ -218,7 +218,7 @@ CLI-only options:
browser_cookies:
--auto-import, --no-auto-import
- --browsers [BROWSERS ...]
+ --browser BROWSER
--sites [SITES ...]
download_options:
@@ -314,6 +314,7 @@ sorting:
general:
--allow-insecure-connections, --no-allow-insecure-connections
+ --disable-crawlers [DISABLE_CRAWLERS ...]
--enable-generic-crawler, --no-enable-generic-crawler
--flaresolverr FLARESOLVERR
--max-file-name-length MAX_FILE_NAME_LENGTH
diff --git a/docs/reference/supported-websites.md b/docs/reference/supported-websites.md
index ab6f0c378..8316ca3c4 100644
--- a/docs/reference/supported-websites.md
+++ b/docs/reference/supported-websites.md
@@ -28,100 +28,104 @@ Supported domains via Real-Debrid include `mega.nz`, `rapidgator`, `google drive
## Supported sites
-List of sites supported by cyberdrop-dl-patched as of version 6.9.1
+List of sites supported by cyberdrop-dl-patched as of version 6.10.1
-| Site | Primary URL | Supported Domains | Supported Paths |
-|---------------------------|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 4chan | https://boards.4chan.org | 4chan | Board: <br> - `/<board>`<br>Thread: <br> - `/thread`<br> |
-| 8Muses | https://comics.8muses.com | 8muses | Album: <br> - `/comics/album/...`<br> |
-| AcidImg | https://acidimg.cc | acidimg.cc | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/upload/...`<br> |
-| AllPornComix | https://forum.allporncomix.com | allporncomix | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| ArchiveBate | https://www.archivebate.store | archivebate | Video: <br> - `/watch/<video_id>`<br> |
-| Bellazon | https://www.bellazon.com/main/ | bellazon | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| BestPrettyGirl | https://bestprettygirl.com/ | bestprettygirl.com | Collection: <br> - `category`<br> - `date`<br> - `tag`<br>Gallery: <br> - `/<gallery_name>/`<br> |
-| Box | https://www.box.com | app.box.com | File or Folder: <br> - `app.box.com/s?sh=<share_code>`<br> |
-| Bunkrr | https://bunkr.site | bunkr<br>bunkrr | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br>Video: <br> - `/v/...`<br> |
-| BuzzHeavier | https://buzzheavier.com | buzzheavier.com | Direct Links<br> |
-| Catbox | https://catbox.moe | files.catbox.moe<br>litter.catbox.moe | Direct Links<br> |
-| CelebForum | https://celebforum.to | celebforum | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| Coomer | https://coomer.su | coomer | Direct Links<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |
-| Cyberdrop | https://cyberdrop.me/ | cyberdrop | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br> |
-| Cyberfile | https://cyberfile.me/ | cyberfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |
-| DirtyShip | https://dirtyship.com | dirtyship | Categorie: <br> - `/category/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/gallery/.../...`<br>Tag: <br> - `/tag/...`<br>Video: <br> - `/<video_name>`<br> |
-| DoodStream | https://doodstream.com/ | all3do.com<br>do7go.com<br>dood.re<br>dood.yt<br>doodcdn<br>doodstream<br>doodstream.co<br>vidply.com | Video: <br> - `/e/<video_id>`<br> |
-| Dropbox | https://dropbox.com/ | dropbox | File: <br> - `/s/...`<br> - `/scl/fi/...`<br>Folder: <br> - `/scl/fo/...`<br> - `/sh/...`<br><br><br>**NOTES**<br> - Folders will be downloaded as a zip file. See: [--download-dropbox-folders-as-zip](https://script-ware.gitbook.io/cyberdrop-dl/reference/cli-arguments#download-dropbox-folders-as-zip)<br> |
-| E-Hentai | https://e-hentai.org/ | e-hentai | Album: <br> - `/g/...`<br>File: <br> - `/s/...`<br> |
-| E621 | https://e621.net | e621.net | Pools: <br> - `/pools/...`<br>Post: <br> - `/posts/...`<br>Tags: <br> - `/posts?tags=...`<br> |
-| Erome | https://www.erome.com | erome | Album: <br> - `/a/...`<br>Profile: <br> - `/...`<br> |
-| F95Zone | https://f95zone.to | f95zone | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| Fapello | https://fapello.su/ | fapello | Individual Post: <br> - `/.../...`<br>Model: <br> - `/...`<br> |
-| Fileditch | https://fileditchfiles.me/ | fileditch | Direct Links<br> |
-| FilesVC | https://files.vc | files.vc | Direct Links<br> |
-| GoFile | https://gofile.io | gofile | Album: <br> - `/d/...`<br> |
-| GoogleDrive | https://drive.google.com | docs.google<br>drive.google | Docs: <br> - `/document/d/<file_id>`<br>Files: <br> - `/d/<file_id>`<br>Folders: <br> - `/drive/folders/<folder_id>`<br>Sheets: <br> - `/spreadsheets/d/<file_id>`<br>Slides: <br> - `/presentation/d/<file_id>`<br> |
-| HotPic | https://hotpic.cc | 2385290.xyz<br>hotpic | Album: <br> - `/album/...`<br>Image: <br> - `/i/...`<br> |
-| Iceyfile | https://iceyfile.com/ | iceyfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |
-| ImageBam | https://www.imagebam.com/ | imagebam | Album: <br> - `/view/...`<br>Direct Links<br>Image: <br> - `/view/...`<br> |
-| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profiles: <br> - `/...`<br>Video: <br> - `/video/..`<br> |
-| ImgBB | https://ibb.co | ibb.co<br>imgbb.co | Album: <br> - `/album/...`<br>Image: <br> - `/...`<br> |
-| ImgBox | https://imgbox.com | imgbox | Album: <br> - `/g/...`<br>Direct Links<br>Image: <br> - `/...`<br> |
-| Imgur | https://imgur.com/ | imgur | Album: <br> - `/a/...`<br>Direct Links<br>Gallery: <br> - `/gallery/...`<br>Image: <br> - `/...`<br> |
-| Imx.to | https://imx.to | imx.to | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/t/...`<br> |
-| IncestFlix | https://www.incestflix.com | incestflix | Tag: <br> - `/tag/...`<br>Video: <br> - `/watch/...`<br> |
-| InfluencerBitches | https://influencerbitches.com | influencerbitches | Model: <br> - `/model/...`<br> |
-| JPG5 | https://jpg5.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profile: <br> - `/<user_name>`<br> |
-| Kemono | https://kemono.su | kemono | Direct Links<br>Discord Server: <br> - `/discord/<server_id>`<br>Discord Server Channel: <br> - `/discord/server/...#...`<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |
-| LeakedModels | https://leakedmodels.com/forum/ | leakedmodels | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| Luscious | https://members.luscious.net | luscious | Album: <br> - `/albums/...`<br> |
-| Mediafire | https://www.mediafire.com/ | mediafire | File: <br> - `/file/...`<br>Folder: <br> - `/folder/...`<br> |
-| MixDrop | https://mixdrop.sb | mixdrop<br>mxdrop | File: <br> - `/e/<file_id>`<br> - `/f/<file_id>`<br> |
-| Motherless | https://motherless.com | motherless | Group: <br> - `/g/<group_name>`<br> - `/gi/<image>`<br> - `/gv/<video>`<br>Image: <br> - `/...`<br>User: <br> - `/f/...`<br> - `/u/...`<br>Video: <br> - `pending`<br><br><br>**NOTES**<br> - Galleries are NOT supported<br> |
-| Nekohouse | https://nekohouse.su | nekohouse | Direct Links<br>Individual Post: <br> - `/<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br> |
-| NoodleMagazine | https://noodlemagazine.com | noodlemagazine | Search: <br> - `/video/<search_query`<br>Video: <br> - `/watch/<video_id>`<br> |
-| NudoStar | https://nudostar.com/forum/ | nudostar | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| NudoStarTV | https://nudostar.tv/ | nudostar.tv | Model: <br> - `/models/...`<br> |
-| OmegaScans | https://omegascans.org | omegascans | Chapter: <br> - `/series/.../...`<br>Direct Links<br>Series: <br> - `/series/...`<br> |
-| OneDrive | https://onedrive.com/ | 1drv.ms<br>onedrive.live.com | Access Link: <br> - `https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>`<br>Share Link (anyone can access): <br> - `https://1drv.ms/b/<KEY>`<br> - `https://1drv.ms/f/<KEY>`<br> - `https://1drv.ms/t/<KEY>`<br> - `https://1drv.ms/u/<KEY>`<br> |
-| PMVHaven | https://pmvhaven.com | pmvhaven | Category: <br> - `/category/...`<br>Music: <br> - `/music/...`<br>Playlist: <br> - `/playlist/...`<br>Search results: <br> - `/search/...`<br>Star: <br> - `/star/...`<br>Tag: <br> - `/tags/...`<br>Users: <br> - `/profile/...`<br>Video: <br> - `/video/...`<br> |
-| PimpAndHost | https://pimpandhost.com/ | pimpandhost | Album: <br> - `/album/...`<br>Image: <br> - `/image/...`<br> |
-| PixHost | https://pixhost.to/ | pixhost | Gallery: <br> - `/gallery/...`<br>Image: <br> - `/show/...`<br> |
-| PixelDrain | https://pixeldrain.com | pixeldrain | File: <br> - `/u/...`<br>Folder: <br> - `/l/...`<br> |
-| Pkmncards | https://pkmncards.com | pkmncards | Card: <br> - `/card/...`<br>Series: <br> - `/series/...`<br>Set: <br> - `/set/...`<br> |
-| PornHub | https://www.pornhub.com | pornhub | Album: <br> - `/album/...`<br>Channel: <br> - `/channel/...`<br>Gif: <br> - `/gif/...`<br>Photo: <br> - `/photo/...`<br>Playlist: <br> - `/playlist/...`<br>Profile: <br> - `/model/...`<br> - `/pornstar/...`<br> - `/user/...`<br>Video: <br> - `/embed/<video_id>`<br> - `/view_video.php?viewkey=<video_id>`<br> |
-| PornPics | https://pornpics.com | pornpics | Categories: <br> - `/categories/....`<br>Channels: <br> - `/channels/...`<br>Direct Links<br>Gallery: <br> - `/galleries/...`<br>Pornstars: <br> - `/pornstars/...`<br>Search: <br> - `/?q=<query>`<br>Tags: <br> - `/tags/...`<br> |
-| Porntrex | https://www.porntrex.com | porntrex | Album: <br> - `/albums/...`<br>Category: <br> - `/categories/...`<br>Model: <br> - `/models/...`<br>Playlist: <br> - `/playlists/...`<br>Search: <br> - `/search/...`<br>Tag: <br> - `/tags/...`<br>User: <br> - `/members/...`<br>Video: <br> - `/video/...`<br> |
-| PostImg | https://postimages.org/ | postimg | Album: <br> - `/gallery/...`<br>Direct Links<br>Image: <br> - `/...`<br> |
-| RealBooru | https://realbooru.com | realbooru | File: <br> - `?id=...`<br>Tags: <br> - `?tags=...`<br> |
-| RedGifs | https://redgifs.com/ | redgifs | User: <br> - `/users/<user>`<br>Video: <br> - `/watch/...`<br> |
-| Reddit | https://www.reddit.com/ | redd.it<br>reddit | Direct Links<br>Subreddit:: <br> - `/r/<subreddit>`<br>User: <br> - `/u/<user>`<br> - `/user/<user>`<br> - `/user/<user>/...`<br> |
-| Rule34Vault | https://rule34vault.com | rule34vault | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |
-| Rule34Video | https://rule34video.com/ | rule34video | Members: <br> - `/members/...`<br>Models: <br> - `/models/...`<br>Search: <br> - `/search/...`<br>Tags: <br> - `/tags/...`<br>Video: <br> - `/video/<video_id>/<video_name>`<br> - `/videos/<video_id>/<video_name>`<br> |
-| Rule34XXX | https://rule34.xxx | rule34.xxx | File: <br> - `?id=...`<br>Tag: <br> - `?tags=...`<br> |
-| Rule34XYZ | https://rule34.xyz | rule34.xyz | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |
-| Saint | https://saint2.su/ | saint | Album: <br> - `/a/...`<br>Direct Links<br>Video: <br> - `/d/...`<br> - `/embed/...`<br> |
-| Scrolller | https://scrolller.com | scrolller | Subreddit: <br> - `/r/...`<br> |
-| SendNow | https://send.now/ | send.now | Direct Links<br> |
-| SendVid | https://sendvid.com/ | sendvid | Direct Links<br>Embeds: <br> - `/embed/...`<br>Videos: <br> - `/...`<br> |
-| Sex.com | https://sex.com | sex | Shorts Profiles: <br> - `/shorts/<profile>`<br> |
-| SocialMediaGirls | https://forums.socialmediagirls.com | socialmediagirls | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| SpankBang | https://spankbang.com/ | spankbang | Playlist: <br> - `/playlist/<playlist-id>`<br>Video: <br> - `/embed/<video_id>`<br> - `/play/<video_id>`<br> - `/playlist/<playlist-id>-<video_id>`<br> - `/video/<video_id>`<br> |
-| Streamable | https://streamable.com | streamable | Video: <br> - `/...`<br> |
-| TWPornStars | https://www.twpornstars.com | indiantw.com<br>twanal.com<br>twgaymuscle.com<br>twgays.com<br>twlesbian.com<br>twmilf.com<br>twonfans.com<br>twpornstars.com<br>twteens.com<br>twtiktoks.com | Photo: <br> - `/...`<br> |
-| ThisVid | https://thisvid.com | thisvid | Albums: <br> - `/albums/<album_name>`<br>Categories: <br> - `/categories/...`<br>Image: <br> - `/albums/<album_name>/<image_name>`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/?q=...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |
-| TikTok | https://tiktok.com/ | tiktok | Photo: <br> - `/@<user>/photo/<photo_id>`<br>User: <br> - `/@<user>`<br>Video: <br> - `/@<user>/video/<video_id>`<br> |
-| TitsInTops | https://titsintops.com/phpBB2 | titsintops | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| Tokyomotion | https://www.tokyomotion.net | tokyomotion | Albums: <br> - `/album/<album_id>`<br> - `/user/<user>/albums/`<br>Photo: <br> - `/photo/<photo_id>`<br> - `/user/<user>/favorite/photos`<br>Playlist: <br> - `/user/<user>/favorite/videos`<br>Profiles: <br> - `/user/<user>`<br>Search Results: <br> - `/search?...`<br>Video: <br> - `/video/<video_id>`<br> |
-| Toonily | https://toonily.com | toonily | Chapter: <br> - `/webtoon/.../...`<br>Direct Links<br>Webtoon: <br> - `/webtoon/...`<br> |
-| TwitterImages | https://twimg.com/ | twimg | Photo: <br> - `/...`<br> |
-| Vipr.im | https://vipr.im | vipr.im | Image: <br> - `/...`<br>Thumbnail: <br> - `/th/...`<br> |
-| WeTransfer | https://wetransfer.com/ | we.tl<br>wetransfer.com | Direct links: <br> - `download.wetransfer.com/...`<br>Public link: <br> - `wetransfer.com/<file_id>/<security_hash>`<br>Share Link: <br> - `wetransfer.com/<file_id>/<recipient_id>/<security_hash>`<br>Short Link: <br> - `we.tl/<id>`<br> |
-| XBunker | https://xbunker.nu/ | xbunker | Attachments: <br> - `/attachments/...`<br> - `/data/...`<br>Threads: <br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br> - `/threads/<thread_name>`<br> |
-| XBunkr | https://xbunkr.com | xbunkr | Albums: <br> - `/a/...`<br>Direct Links<br> |
-| XXXBunker | https://xxxbunker.com | xxxbunker | Search: <br> - `/search/...`<br>Video: <br> - `/<video_id>`<br> |
-| YandexDisk | https://disk.yandex.com.tr/ | disk.yandex<br>yadi.sk | Files: <br> - `/d/<folder_id>/<file_name>`<br>Folder: <br> - `/d/<folder_id>`<br><br><br>**NOTES**<br> - Does NOT support nested folders<br> |
-| YouJizz | https://www.youjizz.com/ | youjizz | Video: <br> - `/video/<video_id>/...`<br> - `/video/embed/<video_id>`<br> |
-| aShemaleTube | https://www.ashemaletube.com | ashemaletube | Model: <br> - `/creators/...`<br> - `/model/...`<br> - `/pornstars/...`<br>Playlist: <br> - `/playlists/...`<br>User: <br> - `/profiles/...`<br>Video: <br> - `/videos/...`<br> |
-| ePorner | https://www.eporner.com/ | eporner | Categories: <br> - `/cat/...`<br>Channels: <br> - `/channel/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/photo/...`<br>Pornstar: <br> - `/pornstar/...`<br>Profile: <br> - `/profile/...`<br>Search: <br> - `/search/...`<br>Video: <br> - `/<video_name>-<video-id>`<br> - `/embed/<video_id>`<br> - `/hd-porn/<video_id>`<br> |
-| nHentai | https://nhentai.net/ | nhentai.net | Collections: <br> - `artist`<br> - `character`<br> - `favorites`<br> - `group`<br> - `parody`<br> - `search`<br> - `tag`<br>Gallery: <br> - `/g/<gallery_id>`<br> |
-| xHamster | https://xhamster.com/ | xhamster | Creator: <br> - `/creatos/<creator_name>`<br>Gallery: <br> - `photos/gallery/<gallery_name>`<br>User: <br> - `/users/<user_name>`<br>Video: <br> - `/<video_title>`<br> |
+| Site | Primary URL | Supported Domains | Supported Paths |
+|---------------------------|---------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 4chan | https://boards.4chan.org | 4chan | Board: <br> - `/<board>`<br>Thread: <br> - `/thread`<br> |
+| 8Muses | https://comics.8muses.com | 8muses | Album: <br> - `/comics/album/...`<br> |
+| AcidImg | https://acidimg.cc | acidimg.cc | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/upload/...`<br> |
+| AllPornComix | https://forum.allporncomix.com | allporncomix | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| ArchiveBate | https://www.archivebate.store | archivebate | Video: <br> - `/watch/<video_id>`<br> |
+| Bellazon | https://www.bellazon.com/main/ | bellazon | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| BestPrettyGirl | https://bestprettygirl.com/ | bestprettygirl.com | Collection: <br> - `category`<br> - `date`<br> - `tag`<br>Gallery: <br> - `/<gallery_name>/`<br> |
+| Box | https://www.box.com | app.box.com | File or Folder: <br> - `app.box.com/s?sh=<share_code>`<br> |
+| Bunkrr | https://bunkr.site | bunkr<br>bunkrr | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br>Video: <br> - `/v/...`<br> |
+| BuzzHeavier | https://buzzheavier.com | buzzheavier.com | Direct Links<br> |
+| Camwhores.tv | https://www.camwhores.tv | camwhores.tv | Albums: <br> - `/albums/<album_name>`<br>Categories: <br> - `/categories/...`<br>Image: <br> - `/albums/<album_name>/<image_name>`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/?q=...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |
+| Catbox | https://catbox.moe | files.catbox.moe<br>litter.catbox.moe | Direct Links<br> |
+| CelebForum | https://celebforum.to | celebforum | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| Coomer | https://coomer.su | coomer | Direct Links<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |
+| Cyberdrop | https://cyberdrop.me/ | cyberdrop | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br> |
+| Cyberfile | https://cyberfile.me/ | cyberfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |
+| DirtyShip | https://dirtyship.com | dirtyship | Categorie: <br> - `/category/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/gallery/.../...`<br>Tag: <br> - `/tag/...`<br>Video: <br> - `/<video_name>`<br> |
+| DoodStream | https://doodstream.com/ | all3do.com<br>do7go.com<br>dood.re<br>dood.yt<br>doodcdn<br>doodstream<br>doodstream.co<br>vidply.com | Video: <br> - `/e/<video_id>`<br> |
+| Dropbox | https://dropbox.com/ | dropbox | File: <br> - `/s/...`<br> - `/scl/fi/...`<br>Folder: <br> - `/scl/fo/...`<br> - `/sh/...`<br><br><br>**NOTES**<br> - Folders will be downloaded as a zip file. See: [--download-dropbox-folders-as-zip](https://script-ware.gitbook.io/cyberdrop-dl/reference/cli-arguments#download-dropbox-folders-as-zip)<br> |
+| E-Hentai | https://e-hentai.org/ | e-hentai | Album: <br> - `/g/...`<br>File: <br> - `/s/...`<br> |
+| E621 | https://e621.net | e621.net | Pools: <br> - `/pools/...`<br>Post: <br> - `/posts/...`<br>Tags: <br> - `/posts?tags=...`<br> |
+| Erome | https://www.erome.com | erome | Album: <br> - `/a/...`<br>Profile: <br> - `/...`<br> |
+| F95Zone | https://f95zone.to | f95zone | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| Fapello | https://fapello.su/ | fapello | Individual Post: <br> - `/.../...`<br>Model: <br> - `/...`<br> |
+| Fileditch | https://fileditchfiles.me/ | fileditch | Direct Links<br> |
+| FilesVC | https://files.vc | files.vc | Direct Links<br> |
+| GoFile | https://gofile.io | gofile | Album: <br> - `/d/...`<br> |
+| GoogleDrive | https://drive.google.com | docs.google<br>drive.google | Docs: <br> - `/document/d/<file_id>`<br>Files: <br> - `/d/<file_id>`<br>Folders: <br> - `/drive/folders/<folder_id>`<br>Sheets: <br> - `/spreadsheets/d/<file_id>`<br>Slides: <br> - `/presentation/d/<file_id>`<br> |
+| Hitomi.la | https://hitomi.la | hitomi.la | Collection: <br> - `/artist/...`<br> - `/character/...`<br> - `/group/...`<br> - `/series/...`<br> - `/tag/...`<br> - `/type/...`<br>Gallery: <br> - `/anime/...`<br> - `/cg/...`<br> - `/doujinshi/...`<br> - `/galleries/...`<br> - `/gamecg/...`<br> - `/imageset/...`<br> - `/manga/...`<br> - `/reader/...`<br>Search: <br> - `/search.html?<query>`<br> |
+| HotPic | https://hotpic.cc | 2385290.xyz<br>hotpic | Album: <br> - `/album/...`<br>Image: <br> - `/i/...`<br> |
+| Iceyfile | https://iceyfile.com/ | iceyfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |
+| ImageBam | https://www.imagebam.com/ | imagebam | Album: <br> - `/view/...`<br>Direct Links<br>Image: <br> - `/view/...`<br> |
+| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profiles: <br> - `/...`<br>Video: <br> - `/video/..`<br> |
+| ImgBB | https://ibb.co | ibb.co<br>imgbb.co | Album: <br> - `/album/...`<br>Image: <br> - `/...`<br> |
+| ImgBox | https://imgbox.com | imgbox | Album: <br> - `/g/...`<br>Direct Links<br>Image: <br> - `/...`<br> |
+| Imgur | https://imgur.com/ | imgur | Album: <br> - `/a/...`<br>Direct Links<br>Gallery: <br> - `/gallery/...`<br>Image: <br> - `/...`<br> |
+| Imx.to | https://imx.to | imx.to | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/t/...`<br> |
+| IncestFlix | https://www.incestflix.com | incestflix | Tag: <br> - `/tag/...`<br>Video: <br> - `/watch/...`<br> |
+| InfluencerBitches | https://influencerbitches.com | influencerbitches | Model: <br> - `/model/...`<br> |
+| JPG5 | https://jpg5.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profile: <br> - `/<user_name>`<br> |
+| Kemono | https://kemono.su | kemono | Direct Links<br>Discord Server: <br> - `/discord/<server_id>`<br>Discord Server Channel: <br> - `/discord/server/...#...`<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |
+| LeakedModels | https://leakedmodels.com/forum/ | leakedmodels | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| Luscious | https://members.luscious.net | luscious | Album: <br> - `/albums/...`<br> |
+| Mediafire | https://www.mediafire.com/ | mediafire | File: <br> - `/file/...`<br>Folder: <br> - `/folder/...`<br> |
+| MissAV | https://missav.ws | missav | Video: <br> - `/...`<br> |
+| MixDrop | https://mixdrop.sb | mixdrop<br>mxdrop | File: <br> - `/e/<file_id>`<br> - `/f/<file_id>`<br> |
+| Motherless | https://motherless.com | motherless | Group: <br> - `/g/<group_name>`<br> - `/gi/<image>`<br> - `/gv/<video>`<br>Image: <br> - `/...`<br>User: <br> - `/f/...`<br> - `/u/...`<br>Video: <br> - `pending`<br><br><br>**NOTES**<br> - Galleries are NOT supported<br> |
+| Nekohouse | https://nekohouse.su | nekohouse | Direct Links<br>Individual Post: <br> - `/<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br> |
+| NoodleMagazine | https://noodlemagazine.com | noodlemagazine | Search: <br> - `/video/<search_query`<br>Video: <br> - `/watch/<video_id>`<br> |
+| NudoStar | https://nudostar.com/forum/ | nudostar | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| NudoStarTV | https://nudostar.tv/ | nudostar.tv | Model: <br> - `/models/...`<br> |
+| OmegaScans | https://omegascans.org | omegascans | Chapter: <br> - `/series/.../...`<br>Direct Links<br>Series: <br> - `/series/...`<br> |
+| OneDrive | https://onedrive.com/ | 1drv.ms<br>onedrive.live.com | Access Link: <br> - `https://onedrive.live.com/?authkey=<KEY>&id=<ID>&cid=<CID>`<br>Share Link (anyone can access): <br> - `https://1drv.ms/b/<KEY>`<br> - `https://1drv.ms/f/<KEY>`<br> - `https://1drv.ms/t/<KEY>`<br> - `https://1drv.ms/u/<KEY>`<br> |
+| PMVHaven | https://pmvhaven.com | pmvhaven | Category: <br> - `/category/...`<br>Music: <br> - `/music/...`<br>Playlist: <br> - `/playlist/...`<br>Search results: <br> - `/search/...`<br>Star: <br> - `/star/...`<br>Tag: <br> - `/tags/...`<br>Users: <br> - `/profile/...`<br>Video: <br> - `/video/...`<br> |
+| PimpAndHost | https://pimpandhost.com/ | pimpandhost | Album: <br> - `/album/...`<br>Image: <br> - `/image/...`<br> |
+| PixHost | https://pixhost.to/ | pixhost | Gallery: <br> - `/gallery/...`<br>Image: <br> - `/show/...`<br> |
+| PixelDrain | https://pixeldrain.com | pixeldrain | File: <br> - `/u/...`<br>Folder: <br> - `/l/...`<br> |
+| Pkmncards | https://pkmncards.com | pkmncards | Card: <br> - `/card/...`<br>Series: <br> - `/series/...`<br>Set: <br> - `/set/...`<br> |
+| PornHub | https://www.pornhub.com | pornhub | Album: <br> - `/album/...`<br>Channel: <br> - `/channel/...`<br>Gif: <br> - `/gif/...`<br>Photo: <br> - `/photo/...`<br>Playlist: <br> - `/playlist/...`<br>Profile: <br> - `/model/...`<br> - `/pornstar/...`<br> - `/user/...`<br>Video: <br> - `/embed/<video_id>`<br> - `/view_video.php?viewkey=<video_id>`<br> |
+| PornPics | https://pornpics.com | pornpics | Categories: <br> - `/categories/....`<br>Channels: <br> - `/channels/...`<br>Direct Links<br>Gallery: <br> - `/galleries/...`<br>Pornstars: <br> - `/pornstars/...`<br>Search: <br> - `/?q=<query>`<br>Tags: <br> - `/tags/...`<br> |
+| Porntrex | https://www.porntrex.com | porntrex | Album: <br> - `/albums/...`<br>Category: <br> - `/categories/...`<br>Model: <br> - `/models/...`<br>Playlist: <br> - `/playlists/...`<br>Search: <br> - `/search/...`<br>Tag: <br> - `/tags/...`<br>User: <br> - `/members/...`<br>Video: <br> - `/video/...`<br> |
+| PostImg | https://postimages.org/ | postimg | Album: <br> - `/gallery/...`<br>Direct Links<br>Image: <br> - `/...`<br> |
+| RealBooru | https://realbooru.com | realbooru | File: <br> - `?id=...`<br>Tags: <br> - `?tags=...`<br> |
+| RedGifs | https://redgifs.com/ | redgifs | User: <br> - `/users/<user>`<br>Video: <br> - `/watch/...`<br> |
+| Reddit | https://www.reddit.com/ | redd.it<br>reddit | Direct Links<br>Subreddit:: <br> - `/r/<subreddit>`<br>User: <br> - `/u/<user>`<br> - `/user/<user>`<br> - `/user/<user>/...`<br> |
+| Rule34Vault | https://rule34vault.com | rule34vault | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |
+| Rule34Video | https://rule34video.com/ | rule34video | Members: <br> - `/members/...`<br>Models: <br> - `/models/...`<br>Search: <br> - `/search/...`<br>Tags: <br> - `/tags/...`<br>Video: <br> - `/video/<video_id>/<video_name>`<br> - `/videos/<video_id>/<video_name>`<br> |
+| Rule34XXX | https://rule34.xxx | rule34.xxx | File: <br> - `?id=...`<br>Tag: <br> - `?tags=...`<br> |
+| Rule34XYZ | https://rule34.xyz | rule34.xyz | Playlist: <br> - `/playlists/view/...`<br>Post: <br> - `/post/...`<br>Tag: <br> - `/...`<br> |
+| Saint | https://saint2.su/ | saint | Album: <br> - `/a/...`<br>Direct Links<br>Video: <br> - `/d/...`<br> - `/embed/...`<br> |
+| Scrolller | https://scrolller.com | scrolller | Subreddit: <br> - `/r/...`<br> |
+| SendNow | https://send.now/ | send.now | Direct Links<br> |
+| SendVid | https://sendvid.com/ | sendvid | Direct Links<br>Embeds: <br> - `/embed/...`<br>Videos: <br> - `/...`<br> |
+| Sex.com | https://sex.com | sex | Shorts Profiles: <br> - `/shorts/<profile>`<br> |
+| SocialMediaGirls | https://forums.socialmediagirls.com | socialmediagirls | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| SpankBang | https://spankbang.com/ | spankbang | Playlist: <br> - `/playlist/<playlist-id>`<br>Video: <br> - `/embed/<video_id>`<br> - `/play/<video_id>`<br> - `/playlist/<playlist-id>-<video_id>`<br> - `/video/<video_id>`<br> |
+| Streamable | https://streamable.com | streamable | Video: <br> - `/...`<br> |
+| TWPornStars | https://www.twpornstars.com | indiantw.com<br>twanal.com<br>twgaymuscle.com<br>twgays.com<br>twlesbian.com<br>twmilf.com<br>twonfans.com<br>twpornstars.com<br>twteens.com<br>twtiktoks.com | Photo: <br> - `/...`<br> |
+| ThisVid | https://thisvid.com | thisvid | Albums: <br> - `/albums/<album_name>`<br>Categories: <br> - `/categories/...`<br>Image: <br> - `/albums/<album_name>/<image_name>`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/?q=...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |
+| TikTok | https://tiktok.com/ | tiktok | Photo: <br> - `/@<user>/photo/<photo_id>`<br>User: <br> - `/@<user>`<br>Video: <br> - `/@<user>/video/<video_id>`<br> |
+| TitsInTops | https://titsintops.com/phpBB2 | titsintops | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| Tokyomotion | https://www.tokyomotion.net | tokyomotion | Albums: <br> - `/album/<album_id>`<br> - `/user/<user>/albums/`<br>Photo: <br> - `/photo/<photo_id>`<br> - `/user/<user>/favorite/photos`<br>Playlist: <br> - `/user/<user>/favorite/videos`<br>Profiles: <br> - `/user/<user>`<br>Search Results: <br> - `/search?...`<br>Video: <br> - `/video/<video_id>`<br> |
+| Toonily | https://toonily.com | toonily | Chapter: <br> - `/webtoon/.../...`<br>Direct Links<br>Webtoon: <br> - `/webtoon/...`<br> |
+| TwitterImages | https://twimg.com/ | twimg | Photo: <br> - `/...`<br> |
+| Vipr.im | https://vipr.im | vipr.im | Image: <br> - `/...`<br>Thumbnail: <br> - `/th/...`<br> |
+| WeTransfer | https://wetransfer.com/ | we.tl<br>wetransfer.com | Direct links: <br> - `download.wetransfer.com/...`<br>Public link: <br> - `wetransfer.com/<file_id>/<security_hash>`<br>Share Link: <br> - `wetransfer.com/<file_id>/<recipient_id>/<security_hash>`<br>Short Link: <br> - `we.tl/<id>`<br> |
+| XBunker | https://xbunker.nu/ | xbunker | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|threads\|topic\|topics\|tema\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| XBunkr | https://xbunkr.com | xbunkr | Albums: <br> - `/a/...`<br>Direct Links<br> |
+| XXXBunker | https://xxxbunker.com | xxxbunker | Search: <br> - `/search/...`<br>Video: <br> - `/<video_id>`<br> |
+| YandexDisk | https://disk.yandex.com.tr/ | disk.yandex<br>yadi.sk | Files: <br> - `/d/<folder_id>/<file_name>`<br>Folder: <br> - `/d/<folder_id>`<br><br><br>**NOTES**<br> - Does NOT support nested folders<br> |
+| YouJizz | https://www.youjizz.com/ | youjizz | Video: <br> - `/video/<video_id>/...`<br> - `/video/embed/<video_id>`<br> |
+| aShemaleTube | https://www.ashemaletube.com | ashemaletube | Model: <br> - `/creators/...`<br> - `/model/...`<br> - `/pornstars/...`<br>Playlist: <br> - `/playlists/...`<br>User: <br> - `/profiles/...`<br>Video: <br> - `/videos/...`<br> |
+| eFukt | https://efukt.com | efukt.com | Gif: <br> - `/view.gif.php?id=<id>`<br>Homepage: <br> - `/`<br>Photo: <br> - `/pics/....`<br>Series: <br> - `/series/<series_name>`<br>Video: <br> - `/...`<br> |
+| ePorner | https://www.eporner.com/ | eporner | Categories: <br> - `/cat/...`<br>Channels: <br> - `/channel/...`<br>Gallery: <br> - `/gallery/...`<br>Photo: <br> - `/photo/...`<br>Pornstar: <br> - `/pornstar/...`<br>Profile: <br> - `/profile/...`<br>Search: <br> - `/search/...`<br>Video: <br> - `/<video_name>-<video-id>`<br> - `/embed/<video_id>`<br> - `/hd-porn/<video_id>`<br> |
+| nHentai | https://nhentai.net/ | nhentai.net | Collections: <br> - `artist`<br> - `character`<br> - `favorites`<br> - `group`<br> - `parody`<br> - `search`<br> - `tag`<br>Gallery: <br> - `/g/<gallery_id>`<br> |
+| xHamster | https://xhamster.com/ | xhamster | Creator: <br> - `/creatos/<creator_name>`<br>Gallery: <br> - `photos/gallery/<gallery_name>`<br>User: <br> - `/users/<user_name>`<br>Video: <br> - `/videos/<video_title>`<br> |
<!-- END_SUPPORTED_SITES-->
diff --git a/poetry.lock b/poetry.lock
index 6a08cd2a2..563c46fc7 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2297,30 +2297,30 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
[[package]]
name = "ruff"
-version = "0.11.0"
+version = "0.12.0"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
groups = ["dev"]
files = [
- {file = "ruff-0.11.0-py3-none-linux_armv6l.whl", hash = "sha256:dc67e32bc3b29557513eb7eeabb23efdb25753684b913bebb8a0c62495095acb"},
- {file = "ruff-0.11.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:38c23fd9bdec4eb437b4c1e3595905a0a8edfccd63a790f818b28c78fe345639"},
- {file = "ruff-0.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7c8661b0be91a38bd56db593e9331beaf9064a79028adee2d5f392674bbc5e88"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6c0e8d3d2db7e9f6efd884f44b8dc542d5b6b590fc4bb334fdbc624d93a29a2"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c3156d3f4b42e57247275a0a7e15a851c165a4fc89c5e8fa30ea6da4f7407b8"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:490b1e147c1260545f6d041c4092483e3f6d8eba81dc2875eaebcf9140b53905"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1bc09a7419e09662983b1312f6fa5dab829d6ab5d11f18c3760be7ca521c9329"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcfa478daf61ac8002214eb2ca5f3e9365048506a9d52b11bea3ecea822bb844"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fbb2aed66fe742a6a3a0075ed467a459b7cedc5ae01008340075909d819df1e"},
- {file = "ruff-0.11.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92c0c1ff014351c0b0cdfdb1e35fa83b780f1e065667167bb9502d47ca41e6db"},
- {file = "ruff-0.11.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e4fd5ff5de5f83e0458a138e8a869c7c5e907541aec32b707f57cf9a5e124445"},
- {file = "ruff-0.11.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:96bc89a5c5fd21a04939773f9e0e276308be0935de06845110f43fd5c2e4ead7"},
- {file = "ruff-0.11.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a9352b9d767889ec5df1483f94870564e8102d4d7e99da52ebf564b882cdc2c7"},
- {file = "ruff-0.11.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:049a191969a10897fe052ef9cc7491b3ef6de79acd7790af7d7897b7a9bfbcb6"},
- {file = "ruff-0.11.0-py3-none-win32.whl", hash = "sha256:3191e9116b6b5bbe187447656f0c8526f0d36b6fd89ad78ccaad6bdc2fad7df2"},
- {file = "ruff-0.11.0-py3-none-win_amd64.whl", hash = "sha256:c58bfa00e740ca0a6c43d41fb004cd22d165302f360aaa56f7126d544db31a21"},
- {file = "ruff-0.11.0-py3-none-win_arm64.whl", hash = "sha256:868364fc23f5aa122b00c6f794211e85f7e78f5dffdf7c590ab90b8c4e69b657"},
- {file = "ruff-0.11.0.tar.gz", hash = "sha256:e55c620690a4a7ee6f1cccb256ec2157dc597d109400ae75bbf944fc9d6462e2"},
+ {file = "ruff-0.12.0-py3-none-linux_armv6l.whl", hash = "sha256:5652a9ecdb308a1754d96a68827755f28d5dfb416b06f60fd9e13f26191a8848"},
+ {file = "ruff-0.12.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:05ed0c914fabc602fc1f3b42c53aa219e5736cb030cdd85640c32dbc73da74a6"},
+ {file = "ruff-0.12.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:07a7aa9b69ac3fcfda3c507916d5d1bca10821fe3797d46bad10f2c6de1edda0"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7731c3eec50af71597243bace7ec6104616ca56dda2b99c89935fe926bdcd48"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:952d0630eae628250ab1c70a7fffb641b03e6b4a2d3f3ec6c1d19b4ab6c6c807"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c021f04ea06966b02614d442e94071781c424ab8e02ec7af2f037b4c1e01cc82"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:7d235618283718ee2fe14db07f954f9b2423700919dc688eacf3f8797a11315c"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0758038f81beec8cc52ca22de9685b8ae7f7cc18c013ec2050012862cc9165"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:139b3d28027987b78fc8d6cfb61165447bdf3740e650b7c480744873688808c2"},
+ {file = "ruff-0.12.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68853e8517b17bba004152aebd9dd77d5213e503a5f2789395b25f26acac0da4"},
+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3a9512af224b9ac4757f7010843771da6b2b0935a9e5e76bb407caa901a1a514"},
+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b08df3d96db798e5beb488d4df03011874aff919a97dcc2dd8539bb2be5d6a88"},
+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6a315992297a7435a66259073681bb0d8647a826b7a6de45c6934b2ca3a9ed51"},
+ {file = "ruff-0.12.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e55e44e770e061f55a7dbc6e9aed47feea07731d809a3710feda2262d2d4d8a"},
+ {file = "ruff-0.12.0-py3-none-win32.whl", hash = "sha256:7162a4c816f8d1555eb195c46ae0bd819834d2a3f18f98cc63819a7b46f474fb"},
+ {file = "ruff-0.12.0-py3-none-win_amd64.whl", hash = "sha256:d00b7a157b8fb6d3827b49d3324da34a1e3f93492c1f97b08e222ad7e9b291e0"},
+ {file = "ruff-0.12.0-py3-none-win_arm64.whl", hash = "sha256:8cd24580405ad8c1cc64d61725bca091d6b6da7eb3d36f72cc605467069d7e8b"},
+ {file = "ruff-0.12.0.tar.gz", hash = "sha256:4d047db3662418d4a848a3fdbfaf17488b34b62f527ed6f10cb8afd78135bc5c"},
]
[[package]]
@@ -2927,4 +2927,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4"
-content-hash = "933320618925b324324b5a02e96f9c777b935494d59c626e755df35124ec7fe6"
+content-hash = "973b07f0a1e83b46a6d40afdb5d95e3689c3e99a00876efe553f7dc9fd201e25"
diff --git a/pyproject.toml b/pyproject.toml
index 2872371af..7f87327fd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,7 +70,7 @@ requires-poetry = ">=2.0"
[tool.poetry.group.dev.dependencies]
pre-commit = ">=4.0.1,<5"
py-markdown-table = "^1.3.0"
-ruff = ">=0.11.0"
+ruff = ">=0.12.0"
snakeviz = "^2.2.2"
tox = "^4.24.1"
diff --git a/tests/crawlers/test_xenforo.py b/tests/crawlers/test_xenforo.py
new file mode 100644
index 000000000..70e7f2296
--- /dev/null
+++ b/tests/crawlers/test_xenforo.py
@@ -0,0 +1,833 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+from unittest import mock
+
+import pytest
+from bs4 import BeautifulSoup
+
+from cyberdrop_dl.crawlers import xenforo as crawlers
+from cyberdrop_dl.crawlers.xenforo import xenforo
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.managers.manager import Manager
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+ from pathlib import Path
+
+
+def _item(url: str) -> ScrapeItem:
+ return ScrapeItem(url=AbsoluteHttpURL(url))
+
+
+class MockProgress:
+ scraping_progress = None
+
+
+manager = Manager()
+scrape_item = _item("https://xenforo.com/community")
+manager.progress_manager = MockProgress() # type: ignore
+crawler_instances = {crawler: crawler(manager) for crawler in crawlers.XF_CRAWLERS}
+TEST_CRAWLER = crawler_instances[crawlers.CelebForumCrawler]
+
+
+def _html(string: str) -> str:
+ return f"""
+ <html>
+ <body>
+ {string}
+ </body>
+ </html>
+ """
+
+
+def _post(
+ message_body: str = "",
+ message_attachments: str = "",
+ id: int = 12345,
+ crawler: xenforo.XenforoCrawler | None = None,
+) -> xenforo.ForumPost:
+ crawler = crawler or TEST_CRAWLER
+ html = _html(POST_TEMPLATE.format(id=id, message_body=message_body, message_attachments=message_attachments))
+ soup = BeautifulSoup(html, "html.parser")
+ return xenforo.ForumPost.new(soup, crawler.XF_SELECTORS.posts)
+
+
+def _item_call(value: Any) -> mock._Call:
+ return mock.call(scrape_item, value)
+
+
+def _any_item_call(value: Any) -> mock._Call:
+ return mock.call(mock.ANY, value)
+
+
+def _amock(func: str = "process_child", crawler: xenforo.XenforoCrawler | None = None) -> mock._patch[mock.AsyncMock]:
+ crawler = crawler or TEST_CRAWLER
+ return mock.patch.object(crawler, func, new_callable=mock.AsyncMock)
+
+
[email protected](name="manager")
+async def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> AsyncGenerator[Manager]:
+ appdata = str(tmp_path)
+ downloads = str(tmp_path / "Downloads")
+ monkeypatch.chdir(tmp_path)
+ manager = Manager(("--appdata-folder", appdata, "-d", downloads))
+ manager.startup()
+ manager.path_manager.startup()
+ manager.log_manager.startup()
+ await manager.async_startup()
+ yield manager
+ await manager.async_db_close()
+ await manager.close()
+
+
+ ("url", "thread_name_and_id", "result", "canonical_url"),
+ [
+ [
+ "https://simpcity.su/threads/general-support.208041/page-260#post-23934165",
+ "general-support.208041",
+ (
+ 208041,
+ "general-support",
+ 260,
+ 23934165,
+ ),
+ "https://simpcity.su/threads/general-support.208041",
+ ],
+ [
+ "https://celebforum.to/threads/infos-regelaenderungen.18821/page-3",
+ "infos-regelaenderungen.18821",
+ (
+ 18821,
+ "infos-regelaenderungen",
+ 3,
+ None,
+ ),
+ "https://celebforum.to/threads/infos-regelaenderungen.18821",
+ ],
+ [
+ "https://www.bellazon.com/main/topic/27120-the-official-victorias-secret-thread",
+ "27120-the-official-victorias-secret-thread",
+ (
+ 27120,
+ "the-official-victorias-secret-thread",
+ 1,
+ None,
+ ),
+ "https://www.bellazon.com/main/topic/27120-the-official-victorias-secret-thread",
+ ],
+ [
+ "https://forums.socialmediagirls.com/threads/forum-rules.14/post-34",
+ "forum-rules.14",
+ (
+ 14,
+ "forum-rules",
+ 1,
+ 34,
+ ),
+ "https://forums.socialmediagirls.com/threads/forum-rules.14",
+ ],
+ [
+ "https://forums.socialmediagirls.com/threads/should-we-ban-gofile-76-5-say-no.436901/post-3942103",
+ "should-we-ban-gofile-76-5-say-no.436901",
+ (
+ 436901,
+ "should-we-ban-gofile-76-5-say-no",
+ 1,
+ 3942103,
+ ),
+ "https://forums.socialmediagirls.com/threads/should-we-ban-gofile-76-5-say-no.436901",
+ ],
+ [
+ "https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930/page-11/#post-2070848",
+ "en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930",
+ (
+ 13930,
+ "en-fr-tools-to-download-upload-content-websites-softwares-extensions",
+ 11,
+ 2070848,
+ ),
+ "https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930",
+ ],
+ [
+ "https://f95zone.to/threads/mod-uploading-rules-12-02-2018.9236/post-2726083",
+ "mod-uploading-rules-12-02-2018.9236",
+ (
+ 9236,
+ "mod-uploading-rules-12-02-2018",
+ 1,
+ 2726083,
+ ),
+ "https://f95zone.to/threads/mod-uploading-rules-12-02-2018.9236",
+ ],
+ [
+ "https://650f.bike/threads/central-stand-honda-cb650r-constands-power-evo.3093/page-2",
+ "central-stand-honda-cb650r-constands-power-evo.3093",
+ (
+ 3093,
+ "central-stand-honda-cb650r-constands-power-evo",
+ 2,
+ None,
+ ),
+ "https://650f.bike/threads/central-stand-honda-cb650r-constands-power-evo.3093",
+ ],
+ [
+ "https://arstechnica.com/civis/threads/the-new-perpetual-photo-accessory-thread.1274775/page-3#post-29155063",
+ "the-new-perpetual-photo-accessory-thread.1274775",
+ (
+ 1274775,
+ "the-new-perpetual-photo-accessory-thread",
+ 3,
+ 29155063,
+ ),
+ "https://arstechnica.com/civis/threads/the-new-perpetual-photo-accessory-thread.1274775",
+ ],
+ [
+ "https://www.laneros.com/temas/iphone-16-16e-16-plus-16-pro-16-promax.256047/page-64#post-7512404",
+ "iphone-16-16e-16-plus-16-pro-16-promax.256047",
+ (
+ 256047,
+ "iphone-16-16e-16-plus-16-pro-16-promax",
+ 64,
+ 7512404,
+ ),
+ "https://www.laneros.com/temas/iphone-16-16e-16-plus-16-pro-16-promax.256047",
+ ],
+ ],
+)
+def test_parse_thread(url: str, thread_name_and_id: str, result: tuple[int, str, int, int], canonical_url: str) -> None:
+ url_, canonical_url_ = AbsoluteHttpURL(url), AbsoluteHttpURL(canonical_url)
+ result_ = xenforo.Thread(*result, canonical_url_)
+ parsed = xenforo.parse_thread(url_, thread_name_and_id, "page", "post")
+ assert result_ == parsed
+
+
+ "link, out",
+ [
+ (
+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",
+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",
+ ),
+ (
+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.th.jpg",
+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",
+ ),
+ (
+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.md.jpg",
+ "https://media.imagepond.net/media/IMG_2153a940fb5680979a52.jpg",
+ ),
+ (
+ "https://simp6.jpg5.su/images3/20250612_051526cd1f36d75c763fe4.md.jpg",
+ "https://simp6.jpg5.su/images3/20250612_051526cd1f36d75c763fe4.jpg",
+ ),
+ (
+ "https://jpg5.su/img/aDNRwDH",
+ "https://jpg5.su/img/aDNRwDH",
+ ),
+ (
+ "https://saint2.cr/embed/M8XwlvJUlW-",
+ "https://saint2.cr/embed/M8XwlvJUlW-",
+ ),
+ (
+ "https://saint2.cr/ifr/M8XwlvJUlW-",
+ "https://saint2.cr/watch/M8XwlvJUlW-",
+ ),
+ ],
+)
+def test_clean_link_url(link: str, out: str) -> None:
+ assert xenforo.clean_link_str(link) == out
+
+
+def test_parse_login_form_success() -> None:
+ html = _html("""
+ <form id="loginForm">
+ <input type="text" name="username" value="testuser">
+ <input type="password" name="password" value="testpass">
+ <input type="hidden" name="csrf_token" value="some_token_123">
+ <input type="submit" value="Login">
+ <input type="text" id="noName" value="shouldBeIgnored">
+ <input type="text" name="noValue">
+ </form>
+ <form id="anotherForm">
+ <input type="text" name="anotherField" value="anotherValue">
+ </form>
+ """)
+ expected_data = {
+ "username": "testuser",
+ "password": "testpass",
+ "csrf_token": "some_token_123",
+ }
+ parsed_data = xenforo.parse_login_form(html)
+ assert parsed_data == expected_data
+
+
+def test_parse_login_form_no_form_should_fail() -> None:
+ with pytest.raises(ScrapeError):
+ xenforo.parse_login_form("")
+
+
+def test_parse_login_form_inputs_without_name_or_value_should_be_ignored() -> None:
+ html = _html("""
+ <form>
+ <input type="text" value="somevalue">
+ <input type="text" name="someName">
+ <input type="text" name="validField" value="validValue">
+ </form>
+ """)
+ expected_data = {"validField": "validValue"}
+ parsed_data = xenforo.parse_login_form(html)
+ assert parsed_data == expected_data
+
+
+def test_parse_login_form_no_input_form_should_fail() -> None:
+ html = _html("""
+ <form>
+ <div>Some content</div>
+ <p>More content</p>
+ </form>
+ """)
+ with pytest.raises(ScrapeError):
+ xenforo.parse_login_form(html)
+
+
+ "input_string, expected_output",
+ [
+ (
+ r"some_text_before \/\/simpcity.su/path/to/resource.mp4 some_text_after",
+ "https://simpcity.su/path/to/resource.mp4",
+ ),
+ (
+ r"prefix \/\/celebforum.to/search?q=test suffix",
+ "https://celebforum.to/search?q=test",
+ ),
+ (
+ r"only_url \/\/sub.domain.co.uk/file.pdf",
+ "https://sub.domain.co.uk/file.pdf",
+ ),
+ (
+ r"http://simpcity.su/some_page",
+ "http://simpcity.su/some_page",
+ ),
+ (
+ r"text with no slashes http://xenforo.com",
+ "http://xenforo.com",
+ ),
+ (
+ r"text with no slashes https://xenforo.com/path",
+ "https://xenforo.com/path",
+ ),
+ # Test cases where no URL is found
+ (
+ "just some plain text",
+ "just some plain text",
+ ),
+ ("", ""),
+ (
+ "no\\url\\here",
+ "nourlhere",
+ ), # check if backslashes are removed
+ (
+ "//invalid.c",
+ "//invalid.c",
+ ), # too short domain suffix
+ (
+ "ftp://simpcity.su/file",
+ "ftp://simpcity.su/file",
+ ), # unsupported protocol
+ (
+ "www.simpcity.su",
+ "www.simpcity.su",
+ ), # missing http/https
+ (
+ "simpcity.su",
+ "simpcity.su",
+ ), # missing http/https
+ (
+ "text with https: //simpcity.su/ (space after colon)",
+ "text with https: //simpcity.su/ (space after colon)",
+ ),
+ (
+ "text with unicode \u00a9 characters",
+ "text with unicode \u00a9 characters",
+ ),
+ ],
+)
+def test_extract_embed_url(input_string: str, expected_output: str) -> None:
+ assert xenforo.extract_embed_url(input_string) == expected_output
+
+
[email protected] # regex can not handle URLs with commans in it (kvs)
+ ("input_string", "expected_output"),
+ [
+ (
+ r"start \/\/media.jp5.net/videos/2023/clip_id-123.mp4?autoplay=true&loop=false#t=10s end",
+ "https://media.jp5.net/videos/2023/clip_id-123.mp4?autoplay=true&loop=false#t=10s",
+ ),
+ (
+ r"start \/\/jupiter4.thisvid.com/key=SEtXHaueMU2PByWg4GNMnw,end=1750179436/speed=1.4/buffer=5.0/12702000/12702535/12702535.mp4 other",
+ "https://jupiter4.thisvid.com/key=SEtXHaueMU2PByWg4GNMnw,end=1750179436/speed=1.4/buffer=5.0/12702000/12702535/12702535.mp4",
+ ),
+ ],
+)
+def test_extract_embed_url_complex_path(input_string: str, expected_output: str) -> None:
+ assert xenforo.extract_embed_url(input_string) == expected_output
+
+
+def test_extract_embed_url_should_extract_only_one_url() -> None:
+ input_str = r"first_url \/\/first.com/path second_url \/\/www.second.net/another"
+ expected_output = "https://first.com/path"
+ assert xenforo.extract_embed_url(input_str) == expected_output
+
+
+def test_extract_embed_url_with_escaped_backslashes_and_double_slashes() -> None:
+ input_str = r"some_path\\to\\file\/\/imagepond.net\/clip.mov"
+ expected_output = "https://imagepond.net/clip.mov"
+ assert xenforo.extract_embed_url(input_str) == expected_output
+
+
+def test_extract_embed_url_absolute_url_no_scheme() -> None:
+ input_str = r"\/\/jpg5.su"
+ expected_output = r"https://jpg5.su"
+ assert xenforo.extract_embed_url(input_str) == expected_output
+
+
+def test_extract_embed_url_should_not_modify_absolute_urls() -> None:
+ input_str = r"https://celebforum.to/maps"
+ expected_output = "https://celebforum.to/maps"
+ assert xenforo.extract_embed_url(input_str) == expected_output
+
+
+def test_extract_embed_url_string_ends_with_url() -> None:
+ input_str = r"some text https://www.reddit.com/"
+ expected_output = "https://reddit.com/"
+ assert xenforo.extract_embed_url(input_str) == expected_output
+
+
+async def test_lazy_load_embeds() -> None:
+ post = _post("""
+ <div class="generic2wide-iframe-div" onclick="loadMedia(this, '//redgifs.com/ifr/downrightcluelesswirm');">
+ <span data-s9e-mediaembed="redgifs">
+ <span class="iframe-wrapper-redgifs" style="">
+ <span class="iframe-wrapper-redgifs-info">Click here to load redgifs media</span>
+ </span>
+ </span>
+ </div>""")
+ expected_output = "//redgifs.com/ifr/downrightcluelesswirm"
+ with _amock() as mocked:
+ await TEST_CRAWLER._lazy_load_embeds(scrape_item, post)
+ mocked.assert_called_once_with(scrape_item, expected_output, embeds=True)
+
+
+ ("cls", "post_content", "expected_result"),
+ [
+ (
+ crawlers.SimpCityCrawler,
+ """
+ <div class="bbWrapper">
+ <a href="https://jpg5.su/img/aqDYT6c" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ title="issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ <a href="https://jpg5.su/img/aqDYZAt" target="_blank" class="link link--external" rel="nofollow ugc noopener">
+ <img
+ src="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ data-url="https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ class="bbImage"
+ loading="lazy"
+ alt="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ title="issfanfan-20250129-0001ef763780833ed714.md.jpg"
+ style=""
+ width=""
+ height=""
+ />
+ </a>
+ </div>""",
+ [
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0002b2b2fa9a5390f521.md.jpg",
+ "https://simp6.jpg5.su/images3/issfanfan-20250129-0001ef763780833ed714.md.jpg",
+ ],
+ )
+ ],
+)
+async def test_post_images(cls: type[xenforo.XenforoCrawler], post_content: str, expected_result: list[str]) -> None:
+ crawler = crawler_instances[cls]
+ post = _post(post_content, crawler=crawler)
+ with _amock(crawler=crawler) as mocked:
+ await crawler._images(scrape_item, post)
+ count, expected_count = mocked.call_count, len(expected_result)
+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"
+ for result, expected in zip(mocked.call_args_list, expected_result, strict=True):
+ assert result.args[1] == expected
+
+
+async def test_embeds_can_extract_google_drive_links() -> None:
+ # https://github.com/jbsparrow/CyberDropDownloader/issues/775
+ crawler = crawler_instances[crawlers.SimpCityCrawler]
+ content = """
+ <div itemprop="text">
+ <div class="bbWrapper">
+ Got this video From her.<br />
+ <span data-s9e-mediaembed="googledrive">
+ <iframe allowfullscreen="" scrolling="no" src="//drive.google.com/file/d/1gfDjCbNXgJafY6ILQIrgbnuptSCFbM0J/preview" loading="eager"></iframe></span>
+ </span>
+ </div>
+ </div>
+
+ <div class="js-selectToQuoteEnd">&nbsp;</div>
+ """
+ post = _post(content, crawler=crawler)
+ expected_result = "//drive.google.com/file/d/1gfDjCbNXgJafY6ILQIrgbnuptSCFbM0J/preview"
+ with _amock(crawler=crawler) as mocked:
+ await crawler._embeds(scrape_item, post)
+ mocked.assert_called_once()
+ assert mocked.call_args.args[1] == expected_result
+
+
+async def test_post_smg_extract_attachments() -> None:
+ # https://agithub.com/jbsparrow/CyberDropDownloader/issues/1070
+ attachments = """
+ <h4 class="block-textHeader">Attachments</h4>
+ <ul class="attachmentList">
+ <li class="file file--linked">
+ <a class="u-anchorTarget" id="attachment-3494354"></a>
+
+ <a
+ class="file-preview js-lbImage"
+ href="https://smgmedia2.socialmediagirls.com/forum/2022/04/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg"
+ target="_blank"
+ data-fancybox="lb-thread-111099"
+ style="cursor: pointer;"
+ data-caption='<h4>33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg</h4><p><a href="https:&amp;#x2F;&amp;#x2F;forums.socialmediagirls.com&amp;#x2F;threads&amp;#x2F;loalux.111099&amp;#x2F;#post-1707346" class="js-lightboxCloser">Ixvvxi · Apr 23, 2022 at 10:10 PM</a></p>'
+ >
+ <img
+ src="https://smgmedia2.socialmediagirls.com/forum/2022/04/thumb/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg"
+ alt="33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg"
+ width="250"
+ height="425"
+ loading="lazy"
+ />
+ </a>
+
+ <div class="file-content">
+ <div class="file-info">
+ <span class="file-name" title="33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg">33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4.jpeg</span>
+ <div class="file-meta">
+ 1.3 MB · Views: 0
+ </div>
+ </div>
+ </div>
+ </li>
+
+ <li class="file file--linked">
+ <a class="u-anchorTarget" id="attachment-3494355"></a>
+
+ <a
+ class="file-preview js-lbImage"
+ href="https://smgmedia2.socialmediagirls.com/forum/2022/04/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png"
+ target="_blank"
+ data-fancybox="lb-thread-111099"
+ style="cursor: pointer;"
+ data-caption='<h4>3663188F-00C6-4C90-AB4D-D8C6E7859286.png</h4><p><a href="https:&amp;#x2F;&amp;#x2F;forums.socialmediagirls.com&amp;#x2F;threads&amp;#x2F;loalux.111099&amp;#x2F;#post-1707346" class="js-lightboxCloser">Ixvvxi · Apr 23, 2022 at 10:10 PM</a></p>'
+ >
+ <img src="https://smgmedia2.socialmediagirls.com/forum/2022/04/thumb/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png" alt="3663188F-00C6-4C90-AB4D-D8C6E7859286.png" width="364" height="250" loading="lazy" />
+ </a>
+
+ <div class="file-content">
+ <div class="file-info">
+ <span class="file-name" title="3663188F-00C6-4C90-AB4D-D8C6E7859286.png">3663188F-00C6-4C90-AB4D-D8C6E7859286.png</span>
+ <div class="file-meta">
+ 1.6 MB · Views: 0
+ </div>
+ </div>
+ </div>
+ </li>
+ </ul>
+ """
+ crawler = crawler_instances[crawlers.SocialMediaGirlsCrawler]
+ post = _post(message_attachments=attachments, crawler=crawler)
+ expected_result = [
+ "https://smgmedia2.socialmediagirls.com/forum/2022/04/33E3EDFF-B0ED-4AE3-8D5D-4D2BC6D7EFD4_3526918.jpeg",
+ "https://smgmedia2.socialmediagirls.com/forum/2022/04/3663188F-00C6-4C90-AB4D-D8C6E7859286_3526919.png",
+ ]
+ with _amock(crawler=crawler) as mocked:
+ await crawler._attachments(scrape_item, post)
+ count, expected_count = mocked.call_count, len(expected_result)
+ assert count == expected_count, f"Found {count} links, expected {expected_count} links"
+ for result, expected in zip(mocked.call_args_list, expected_result, strict=True):
+ assert result.args[1] == expected
+
+
+def test_get_post_title_thread_w_prefixes() -> None:
+ html = _html("""
+ <div class="p-title ">
+ <h1 class="p-title-value">
+ <a href="/forums/requests.7/?prefix_id=2" class="labelLink" rel="nofollow">
+ <span class="label label--requests" dir="auto">Request</span></a>
+ <span class="label-append">&nbsp;</span><a href="/forums/requests.7/?prefix_id=8" class="labelLink" rel="nofollow">
+ <span class="label label--youtube" dir="auto">Youtube</span></a><span class="label-append">&nbsp;</span>
+ <a href="/forums/requests.7/?prefix_id=3" class="labelLink" rel="nofollow">
+ <span class="label label--onlyfans" dir="auto">OnlyFans</span></a>
+ <span class="label-append">&nbsp;</span><a href="/forums/requests.7/?prefix_id=7" class="labelLink" rel="nofollow">
+ <span class="label label--insta" dir="auto">Instagram</span></a>
+ <span class="label-append">&nbsp;</span>GunplaMeli</h1>
+ </div>
+ """)
+ soup = BeautifulSoup(html, "html.parser")
+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ assert title == "GunplaMeli"
+
+
+def test_get_post_title_thread_w_no_prefixes() -> None:
+ html = """
+ <div class="p-title">
+ <h1 class="p-title-value">Staged/Fake Japanese Candid Videos from Gcolle/Pcolle or FC2</h1>
+ </div>
+ """
+ soup = BeautifulSoup(html, "html.parser")
+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ assert title == "Staged/Fake Japanese Candid Videos from Gcolle/Pcolle or FC2"
+
+
+def test_get_post_title_no_title_found() -> None:
+ html = _html("")
+ soup = BeautifulSoup(html, "html.parser")
+ with pytest.raises(ScrapeError) as exc_info:
+ xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+
+ assert exc_info.value.status == 429
+ assert exc_info.value.message == "Invalid response from forum. You may have been rate limited"
+
+
+def test_get_post_title_empty_title_block() -> None:
+ html = _html("""<h1 class="p-title-value"></h1>""")
+ soup = BeautifulSoup(html, "html.parser")
+ with pytest.raises(ScrapeError):
+ xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+
+
+def test_get_post_title_non_english_chars() -> None:
+ html = _html("""
+ <div class="p-title">
+ <h1 class="p-title-value">
+ ㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia
+ </h1>
+ </div>
+ """)
+ soup = BeautifulSoup(html, "html.parser")
+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"
+
+
+def test_get_post_title_should_strip_new_lines() -> None:
+ html = _html("""
+ <div class="p-title">
+ <h1 class="p-title-value">
+ ㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️)
+ / りお@Rio /
+ rio_hcup_fantia
+ </h1>
+ </div>
+ """)
+ soup = BeautifulSoup(html, "html.parser")
+ title = xenforo.get_post_title(soup, xenforo.XenforoCrawler.XF_SELECTORS)
+ assert title == "㊙️Hcupりおの極秘えち任務🙊💗 (りお❤️❤️❤️) / りお@Rio / rio_hcup_fantia"
+
+
+def test_is_attachment_should_handle_none() -> None:
+ assert TEST_CRAWLER.is_attachment(None) is False # type: ignore
+
+
+ "link_str",
+ [
+ "https://media.imagepond.net/media/golden-hour86caaa9858726a64.jpg",
+ "https://simp6.jpg5.su/images3/1752x1172_a682e9c85a276121e82018b4031206b9be2e64efe5bc8004.md.jpg",
+ "https://jpg5.su/img/aDDJbdK",
+ "https://simp6.jpg5.su/attachment/1752x1172_a682e9c85a276121e82018b4031206b9be2e64efe5bc8004.md.jpg", # No exact 'attachment' part
+ "http://celebforum.to/attachments", # missing file slug
+ ],
+)
+def test_is_attachment_string_false(link_str: str) -> None:
+ assert TEST_CRAWLER.is_attachment(link_str) is False
+
+
+def test_is_attachment_empty_string_should_be_false() -> None:
+ assert TEST_CRAWLER.is_attachment("") is False
+
+
+class TestCheckPostId:
+ @pytest.mark.parametrize(
+ "init_post_id, current_post_id, scrape_single_forum_post, expected_continue_scraping, expected_scrape_this_post",
+ [
+ # init_post_id > current_post_id
+ (100, 90, True, True, False),
+ (100, 90, False, True, False),
+ # init_post_id == current_post_id
+ (100, 100, True, False, True),
+ (100, 100, False, True, True),
+ # init_post_id < current_post_id
+ (100, 110, True, False, False),
+ (100, 110, False, True, True),
+ ],
+ )
+ def test_init_post_id_was_provided(
+ self,
+ init_post_id: int,
+ current_post_id: int,
+ scrape_single_forum_post: bool,
+ expected_continue_scraping: bool,
+ expected_scrape_this_post: bool,
+ ) -> None:
+ continue_scraping, scrape_this_post = xenforo.check_post_id(
+ init_post_id, current_post_id, scrape_single_forum_post
+ )
+ assert continue_scraping == expected_continue_scraping
+ assert scrape_this_post == expected_scrape_this_post
+
+ def test_no_init_post_id_and_scrape_single_post_false(self) -> None:
+ init_post_id = None
+ current_post_id = 100
+ scrape_single_forum_post = False
+ continue_scraping, scrape_this_post = xenforo.check_post_id(
+ init_post_id, current_post_id, scrape_single_forum_post
+ )
+ assert continue_scraping is True
+ assert scrape_this_post is True
+
+ def test_no_init_post_id_and_scrape_single_post_true_raises_error(self) -> None:
+ init_post_id = None
+ current_post_id = 100
+ scrape_single_forum_post = True
+
+ with pytest.raises(AssertionError):
+ xenforo.check_post_id(init_post_id, current_post_id, scrape_single_forum_post)
+
+
+# og_post_id = 23549340
+POST_TEMPLATE = """
+<article class="message message--post js-post js-inlineModContainer" data-author="cyberdrop-dl-patched" data-content="post{id}" id="js-post{id}" itemscope="" itemtype="https://schema.org/Comment" itemid="https://xenforo.com/posts/{id}/">
+ <meta itemprop="parentItem" itemscope="" itemid="https://xenforo.com/threads/fanfan.33077/" />
+
+ <span class="u-anchorTarget" id="post{id}"></span>
+
+ <div class="message-inner">
+ <div class="message-cell message-cell--main">
+ <div class="message-main js-quickEditTarget">
+ <header class="message-attribution message-attribution--split">
+ <ul class="message-attribution-main listInline">
+ <li class="u-concealed">
+ <a href="/threads/fanfan.33077/post{id}" rel="nofollow" itemprop="url">
+ <time
+ class="u-dt"
+ dir="auto"
+ datetime="2025-06-09T17:30:10-0500"
+ data-timestamp="1749508210"
+ data-date="Jun 9, 2025"
+ data-time="5:30 PM"
+ data-short="7d"
+ title="Jun 9, 2025 at 5:30 PM"
+ itemprop="datePublished"
+ >
+ Jun 9, 2025
+ </time>
+ </a>
+ </li>
+ </ul>
+ </header>
+ <div class="message-attachments">
+ {message_attachments}
+ </div>
+
+ <div class="message-content js-messageContent">
+ <div class="message-userContent lbContainer js-lbContainer" data-lb-id="post{id}" data-lb-caption-desc="cyberdrop-dl-patched · Jun 9, 2025 at 5:30 PM">
+ <article class="message-body js-selectToQuote">
+ {message_body}
+ <div class="js-selectToQuoteEnd">&nbsp;</div>
+ </article>
+ </div>
+
+ <aside class="message-signature">
+ <div class="bbWrapper">Can't post to Bunkr. Mirrors are always appreciated.</div>
+ </aside>
+ </div>
+
+ <footer class="message-footer">
+ <div class="message-microdata" itemprop="interactionStatistic" itemtype="https://schema.org/InteractionCounter" itemscope="">
+ <meta itemprop="userInteractionCount" content="154" />
+ <meta itemprop="interactionType" content="https://schema.org/LikeAction" />
+ </div>
+
+ <div class="message-actionBar actionBar">
+ <div class="actionBar-set actionBar-set--external">
+ <a
+ href="/posts/{id}/react?reaction_id=1"
+ class="reaction reaction--small actionBar-action actionBar-action--reaction reaction--imageHidden reaction--1"
+ data-reaction-id="1"
+ data-xf-init="reaction"
+ data-reaction-list="< .js-post | .js-reactionsList"
+ id="js-XFUniqueId13"
+ >
+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="Like" title="Like" />
+ <span class="reaction-text js-reactionText"><bdi>Like</bdi></span>
+ </a>
+
+ <a href="/threads/fanfan.33077/reply?quote={id}" class="actionBar-action actionBar-action--mq u-jsOnly js-multiQuote" title="Toggle multi-quote" rel="nofollow" data-message-id="{id}" data-mq-action="add">
+ Quote
+ </a>
+
+ <a href="/threads/fanfan.33077/reply?quote={id}" class="actionBar-action actionBar-action--reply" title="Reply, quoting this message" rel="nofollow" data-xf-click="quote" data-quote-href="/posts/{id}/quote">
+ Reply
+ </a>
+ </div>
+
+ <div class="actionBar-set actionBar-set--internal">
+ <a href="/posts/{id}/report" class="actionBar-action actionBar-action--report" data-xf-click="overlay" data-cache="false">Report</a>
+ </div>
+ </div>
+
+ <div class="reactionsBar js-reactionsList is-active">
+ <ul class="reactionSummary">
+ <li>
+ <span class="reaction reaction--small reaction--1" data-reaction-id="1">
+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="Like" title="Like" />
+ </span>
+ </li>
+ <li>
+ <span class="reaction reaction--small reaction--33" data-reaction-id="33">
+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PeepoLove" title="PeepoLove" />
+ </span>
+ </li>
+ <li>
+ <span class="reaction reaction--small reaction--91" data-reaction-id="91">
+ <i aria-hidden="true"></i><img src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" loading="lazy" class="reaction-sprite js-reaction" alt="PepeClown" title="PepeClown" />
+ </span>
+ </li>
+ </ul>
+
+ <span class="u-srOnly">Reactions:</span>
+ <a class="reactionsBar-link" href="/posts/{id}/reactions" data-xf-click="overlay" data-cache="false" rel="nofollow"><bdi>xigxagxion</bdi>, <bdi>Feistee</bdi>, <bdi>kradpmis</bdi> and 140 others</a>
+ </div>
+
+ <div class="js-historyTarget message-historyTarget toggleTarget" data-href="trigger-href"></div>
+ </footer>
+ </div>
+ </div>
+ </div>
+</article>
+
+"""