Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: make chevereto a generic crawler (#1176)

NTFSvolume67df969fc9ca
diff --git a/cyberdrop_dl/config/global_model.py b/cyberdrop_dl/config/global_model.py
index 835d94c30..1a268e7dd 100755
--- a/cyberdrop_dl/config/global_model.py
+++ b/cyberdrop_dl/config/global_model.py
@@ -105,6 +105,7 @@ class GenericCrawlerInstances(BaseModel):
wordpress_media: ListPydanticURL = []
wordpress_html: ListPydanticURL = []
discourse: ListPydanticURL = []
+ chevereto: ListPydanticURL = []
class GlobalSettings(ConfigModel):
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 80c5ca7f8..e5c9c6eee 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -3,6 +3,7 @@ from __future__ import annotations
from cyberdrop_dl import env
+from ._chevereto import CheveretoCrawler
from .acidimg import AcidImgCrawler
from .archivebate import ArchiveBateCrawler
from .ashemaletube import AShemaleTubeCrawler
@@ -104,7 +105,7 @@ from .yandex_disk import YandexDiskCrawler
from .youjizz import YouJizzCrawler
FORUM_CRAWLERS = XF_CRAWLERS.union(INVISION_CRAWLERS, DISCOURSE_CRAWLERS, VBULLETIN_CRAWLERS)
-GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler}
+GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler, CheveretoCrawler}
ALL_CRAWLERS: set[type[Crawler]] = {
crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler
}
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 5ed25c549..16fe232b3 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -1,211 +1,197 @@
from __future__ import annotations
-from enum import StrEnum, auto
-from functools import partialmethod
+import base64
+import itertools
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError
-from cyberdrop_dl.utils import css
+from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
from bs4 import BeautifulSoup
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
-from cyberdrop_dl.crawlers.crawler import SupportedPaths
-ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"
-ALBUM_TITLE_SELECTOR = "a[data-text=album-name]"
-ITEM_SELECTOR = "a[class='image-container --media']"
-PROFILE_TITLE_SELECTOR = 'meta[property="og:title"]'
-IMAGES_PARTS = "image", "img"
-ALBUM_PARTS = "a", "album"
-VIDEO_PARTS = "video", "videos"
-DIRECT_LINK_PARTS = ("images",)
-VIDEO_SELECTOR = "meta[property='og:video']", "content"
-IMAGE_SELECTOR = "div[id=image-viewer] img", "src"
-ENCRYPTED_IMAGE_SELECTOR = "div.image-viewer-main > img"
-
-class Media(StrEnum):
- ALBUM = auto()
- IMAGE = auto()
- VIDEO = auto()
-
-
-def clean_name(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- new_name = url.name
- for trash in (".md.", ".th.", ".fr."):
- new_name = new_name.replace(trash, ".")
- return url.with_name(new_name)
-
-
-def sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
- init_page = int(url.query.get("page") or 1)
- return url.with_query(sort="date_desc", page=init_page)
-
-
-class CheveretoCrawler(Crawler, is_abc=True):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Album": "/a/...",
- "Image": "/img/...",
- "Profiles": "/...",
- "Video": "/video/..",
+class Selectors:
+ ITEM_DESCRIPTION = "p[class*=description-meta]"
+ ITEM = "a[class='image-container --media']"
+ MAIN_IMAGE = "div#image-viewer img", "src"
+ DATE_SINGLE_ITEM = f"{ITEM_DESCRIPTION}:contains('Uploaded') span"
+ DATE_ALBUM_ITEM = f"{ITEM_DESCRIPTION}:contains('Added to') span"
+ DATE = f"{DATE_SINGLE_ITEM}, {DATE_ALBUM_ITEM}", "title"
+
+
+_SELECTORS = Selectors()
+_DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"
+
+
+class CheveretoCrawler(Crawler, is_generic=True):
+ SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {
+ "Album": (
+ "/a/...",
+ "/album/...",
+ ),
+ "Image": (
+ "/img/...",
+ "/image/...",
+ ),
+ "Profiles": "/<user_name>",
+ "Video": (
+ "/video/..",
+ "/videos/...",
+ ),
"Direct links": "",
}
NEXT_PAGE_SELECTOR: ClassVar[str] = "a[data-pagination=next]"
+ CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = True
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- return await self._fetch_chevereto_defaults(scrape_item)
+ def __init_subclass__(cls, **kwargs) -> None:
+ if not cls.CHEVERETO_SUPPORTS_VIDEO:
+ paths = cls.SUPPORTED_PATHS.copy()
+ _ = paths.pop("Video", None)
+ cls.SUPPORTED_PATHS = paths
+ super().__init_subclass__(**kwargs)
- async def _fetch_chevereto_defaults(self, scrape_item: ScrapeItem) -> None:
- if scrape_item.url.host.removeprefix("www.").count(".") > 1:
- return await self.handle_direct_link(scrape_item)
- if any(part in scrape_item.url.parts for part in ALBUM_PARTS):
- return await self.album(scrape_item)
- if any(part in scrape_item.url.parts for part in IMAGES_PARTS):
- return await self.image(scrape_item)
- if any(part in scrape_item.url.parts for part in VIDEO_PARTS):
- return await self.video(scrape_item)
- if any(part in scrape_item.url.parts for part in DIRECT_LINK_PARTS):
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ if self.is_subdomain(scrape_item.url):
return await self.handle_direct_link(scrape_item)
- await self.profile(scrape_item)
+
+ match scrape_item.url.parts[1:]:
+ case ["a" | "album", album_slug]:
+ return await self.album(scrape_item, _id(album_slug))
+ case ["img" | "image" | "video" | "videos" as part, slug]:
+ item_id = _id(slug)
+ canonical_url = self.PRIMARY_URL / part / item_id
+ scrape_item.url = canonical_url
+ return await self.media(scrape_item)
+ case ["images", _, *_]:
+ return await self.handle_direct_link(scrape_item)
+ case [_]:
+ return await self.profile(scrape_item)
+ case _:
+ raise ValueError
@error_handling_wrapper
async def profile(self, scrape_item: ScrapeItem) -> None:
title: str = ""
- async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):
+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):
if not title:
- title: str = css.select_one_get_attr(soup, PROFILE_TITLE_SELECTOR, "content")
- title = self.create_title(title)
+ title = self.create_title(open_graph.title(soup))
scrape_item.setup_as_profile(title)
-
- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
- # Item may be an image, a video or an album
- # For images, we can download the file from the thumbnail
- assert thumb
- if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):
- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)
- await self.handle_direct_link(new_scrape_item, thumb)
- continue
- # For videos and albums, we have to keep scraping
- self.manager.task_group.create_task(self.run(new_scrape_item))
+ self._process_page(scrape_item, soup)
@error_handling_wrapper
- async def album(self, scrape_item: ScrapeItem) -> None:
- album_id, canonical_url = self.get_canonical_url(scrape_item.url)
+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
results = await self.get_album_results(album_id)
- original_url = scrape_item.url
title: str = ""
- async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):
+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):
if not title:
- if is_password_protected(soup):
+ if _is_password_protected(soup):
soup = await self._get_soup_w_password(scrape_item)
- title: str = css.select_one_get_text(soup, ALBUM_TITLE_SELECTOR)
- title = self.create_title(title, album_id)
+ title = self.create_title(open_graph.title(soup), album_id)
scrape_item.setup_as_album(title, album_id=album_id)
- scrape_item.url = canonical_url
-
- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
- assert thumb
- source = clean_name(thumb)
- if self.check_album_results(source, results):
- continue
- if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):
- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)
- await self.handle_direct_link(new_scrape_item, thumb)
- continue
-
- self.manager.task_group.create_task(self.run(new_scrape_item))
-
- # Sub album URL needs to be the full URL + a 'sub'
- # Using the canonical URL + 'sub' won't work because it redirects to the "homepage" of the album
- sub_album_url = original_url / "sub"
- async for soup in self.web_pager(sort_by_new(sub_album_url), trim=False):
- for _, sub_album in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
- self.manager.task_group.create_task(self.run(sub_album))
+ self._process_page(scrape_item, soup, results)
+
+ sub_album_url = scrape_item.url / "sub"
+ async for soup in self.web_pager(_sort_by_new(sub_album_url), trim=False):
+ for _, sub_album in self.iter_children(scrape_item, soup, _SELECTORS.ITEM):
+ self.create_task(self.run(sub_album))
+
+ def _process_page(
+ self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None
+ ) -> None:
+ for thumb, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ITEM):
+ assert thumb
+ source = _thumbnail_to_src(thumb)
+ if results and self.check_album_results(source, results):
+ continue
+ # For images, we can download the file from the thumbnail, skipping an aditional
+ # cons: we won't get the upload date
+ if image_url := _match_img(new_scrape_item.url):
+ new_scrape_item.url = image_url
+ self.create_task(self.handle_direct_link(new_scrape_item, source))
+ continue
+
+ self.create_task(self.run(new_scrape_item))
async def _get_soup_w_password(self, scrape_item: ScrapeItem) -> BeautifulSoup:
password = scrape_item.pop_query("password")
if not password:
raise PasswordProtectedError
- data = {"content-password": password}
async with self.request_limiter:
- soup = await self.client.post_data_get_soup(self.DOMAIN, sort_by_new(scrape_item.url), data=data)
- if is_password_protected(soup):
+ soup = await self.client.post_data_get_soup(
+ self.DOMAIN, _sort_by_new(scrape_item.url), data={"content-password": password}
+ )
+ if _is_password_protected(soup):
raise PasswordProtectedError(message="Wrong password")
return soup
@error_handling_wrapper
async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
- """Handles a direct link."""
- link = clean_name(url or scrape_item.url)
+ link = _thumbnail_to_src(url or scrape_item.url)
filename, ext = self.get_filename_and_ext(link.name)
await self.handle_file(link, scrape_item, filename, ext)
- async def get_embed_info(self, url: AbsoluteHttpURL) -> tuple[str, AbsoluteHttpURL]:
- embed_url = self.PRIMARY_URL / "oembed" / ""
- embed_url = embed_url.with_query(url=str(url), format="json")
- async with self.request_limiter:
- json_resp: dict[str, str] = await self.client.get_json(self.DOMAIN, embed_url)
-
- link = clean_name(self.parse_url(json_resp["url"]))
- filename = json_resp["title"] + link.suffix
- return filename, link
-
@error_handling_wrapper
- async def _proccess_media_item(self, scrape_item: ScrapeItem, media_type: Media, selector: tuple[str, str]) -> None:
+ async def media(self, scrape_item: ScrapeItem) -> None:
if await self.check_complete_from_referer(scrape_item):
return
- _, canonical_url = self.get_canonical_url(scrape_item.url, media_type)
- if await self.check_complete_from_referer(canonical_url):
- return
-
async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)
try:
- link_str: str = css.select_one_get_attr(soup, *selector)
- if "loading.svg" in link_str:
- link_str = css.select_one_get_attr(soup, ENCRYPTED_IMAGE_SELECTOR, "data-src")
+ link_str = open_graph.get("video", soup) or open_graph.get("image", soup)
+ if not link_str or "loading.svg" in link_str:
+ link_str = css.select_one_get_attr(soup, *_SELECTORS.MAIN_IMAGE)
link = self.parse_url(link_str)
- except AttributeError:
- raise ScrapeError(422, f"Couldn't find {media_type} source") from None
+ except css.SelectorError:
+ raise ScrapeError(422, "Couldn't find media source") from None
- scrape_item.possible_datetime = self.parse_date(get_date_from_soup(soup))
- scrape_item.url = canonical_url
+ date_str = css.select_one_get_attr(soup, *_SELECTORS.DATE)
+ scrape_item.possible_datetime = self.parse_iso_date(date_str)
await self.handle_direct_link(scrape_item, link)
- video = partialmethod(_proccess_media_item, media_type=Media.VIDEO, selector=VIDEO_SELECTOR)
- image = partialmethod(_proccess_media_item, media_type=Media.IMAGE, selector=IMAGE_SELECTOR)
+ def parse_url(
+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
+ ) -> AbsoluteHttpURL:
+ if not link_str.startswith("https") and not link_str.startswith("/"):
+ link_str = _xor_decrypt(link_str, _DECRYPTION_KEY)
+ return super().parse_url(link_str, relative_to, trim=trim)
- def get_canonical_url(self, url: AbsoluteHttpURL, media_type: Media = Media.ALBUM) -> tuple[str, AbsoluteHttpURL]:
- """Returns the id and canonical URL from a given item (album, image or video)."""
- search_parts = ALBUM_PARTS
- if media_type == Media.IMAGE:
- search_parts = IMAGES_PARTS
- elif media_type == Media.VIDEO:
- search_parts = VIDEO_PARTS
- found_part = next(p for p in search_parts if p in url.parts)
- name_index = url.parts.index(found_part) + 1
- name = url.parts[name_index]
- item_id = name.rsplit(".")[-1]
- new_parts = (*url.parts[1:name_index], item_id)
- new_path = "/" + "/".join(new_parts)
- return item_id, self.parse_url(new_path, url.origin())
+def _is_password_protected(soup: BeautifulSoup) -> bool:
+ return "This content is password protected" in soup.text
-def get_date_from_soup(soup: BeautifulSoup) -> str:
- for row in soup.select(ITEM_DESCRIPTION_SELECTOR):
- if any(text in row.get_text().casefold() for text in ("uploaded", "added to")):
- return css.select_one_get_attr(row, "span", "title")
- return ""
+def _id(slug: str) -> str:
+ return slug.rsplit(".")[-1]
-def is_password_protected(soup: BeautifulSoup) -> bool:
- return "This content is password protected" in soup.text
+def _match_img(url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:
+ match url.parts[1:]:
+ case ["img" | "image" as part, image_slug, *_]:
+ return url.origin() / part / _id(image_slug)
+
+
+def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ new_name = url.name
+ for trash in (".md.", ".th.", ".fr."):
+ new_name = new_name.replace(trash, ".")
+ return url.with_name(new_name)
+
+
+def _sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ init_page = int(url.query.get("page") or 1)
+ return url.with_query(sort="date_desc", page=init_page)
+
+
+def _xor_decrypt(encrypted_str: str, key: bytes) -> str:
+ encrypted_data = bytes.fromhex(base64.b64decode(encrypted_str).decode())
+ decrypted_data = bytearray(b_input ^ b_key for b_input, b_key in zip(encrypted_data, itertools.cycle(key)))
+ return decrypted_data.decode("utf-8", errors="ignore")
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 92cd4ac02..6a7209941 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -252,9 +252,13 @@ class Crawler(ABC):
finally:
self.manager.progress_manager.scraping_progress.remove_task(task_id)
+ @staticmethod
+ def is_subdomain(url: AbsoluteHttpURL) -> bool:
+ return url.host.removeprefix("www.").count(".") > 1
+
@classmethod
- def is_subdomain(cls, url: AbsoluteHttpURL, domain_to_compare: str | None = None) -> bool:
- primary_domain = (domain_to_compare or cls.PRIMARY_URL.host).removeprefix("www.")
+ def is_self_subdomain(cls, url: AbsoluteHttpURL) -> bool:
+ primary_domain = cls.PRIMARY_URL.host.removeprefix("www.")
other_domain = url.host.removeprefix("www.")
if primary_domain == other_domain:
return False
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index b6f8274f3..5e81e99e2 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import binascii
import re
from typing import TYPE_CHECKING, ClassVar
@@ -11,11 +10,9 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from ._chevereto import CheveretoCrawler
if TYPE_CHECKING:
- from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPaths
+ from cyberdrop_dl.crawlers.crawler import SupportedDomains
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
-
-_DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"
_OLD_DOMAINS = (
"jpg.homes",
"jpg.church",
@@ -35,47 +32,29 @@ _REPLACE_OLD_HOST_REGEX = re.compile("|".join(_OLD_DOMAINS))
class JPG5Crawler(CheveretoCrawler):
- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Album": "/a/...",
- "Image": "/img/...",
- "Profile": "/<user_name>",
- "Direct links": "",
- }
-
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (*_OLD_DOMAINS, "jpg6.su")
DOMAIN: ClassVar[str] = "jpg5.su"
FOLDER_DOMAIN: ClassVar[str] = "JPG5"
-
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg6.su")
+ CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = False
def __post_init__(self) -> None:
self.request_limiter = AsyncLimiter(1, 1)
async def fetch(self, scrape_item: ScrapeItem) -> None:
- scrape_item.url = fix_host(scrape_item.url)
- return await self._fetch_chevereto_defaults(scrape_item)
-
- async def video(self, scrape_item: ScrapeItem) -> None:
- raise ValueError
-
- def parse_url(
- self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
- ) -> AbsoluteHttpURL:
- if not link_str.startswith("https") and not link_str.startswith("/"):
- link_str = decrypt_xor(link_str, _DECRYPTION_KEY)
- return super().parse_url(link_str, relative_to, trim=trim)
+ scrape_item.url = _fix_host(scrape_item.url)
+ return await super().fetch(scrape_item)
async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:
- """Handles a direct link."""
link = url or scrape_item.url
- link = fix_host(link)
+ link = _fix_host(link)
await super().handle_direct_link(scrape_item, link)
"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-def fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+def _fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
new_host = re.sub(_REPLACE_OLD_HOST_REGEX, r"jpg5.su", url.host)
if new_host.removeprefix("www.") == "jpg5.su":
# replace only if it is matches the second level domain exactly
@@ -84,12 +63,6 @@ def fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
return url.with_host(new_host)
-def decrypt_xor(encrypted_str: str, key: bytes) -> str:
- div = len(key)
- encrypted = bytes.fromhex(binascii.a2b_base64(encrypted_str).decode())
- return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()
-
-
def fix_db_referer(referer: str) -> str:
url = AbsoluteHttpURL(referer)
- return str(fix_host(url))
+ return str(_fix_host(url))
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index 0f26ff0b9..06ef47d28 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -62,7 +62,7 @@ class RedGifsCrawler(Crawler):
case ["watch" | "ifr", gif_id]:
return await self.gif(scrape_item, _id(gif_id))
- if self.is_subdomain(scrape_item.url) and len(scrape_item.url.parts) == 2:
+ if self.is_self_subdomain(scrape_item.url) and len(scrape_item.url.parts) == 2:
scrape_item.url = _canonical_url(scrape_item.url.name)
self.manager.task_group.create_task(self.run(scrape_item))
return
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 15f0f295c..bf7a550ed 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -12,6 +12,7 @@ from yarl import URL
from cyberdrop_dl.constants import BLOCKED_DOMAINS, REGEX_LINKS
from cyberdrop_dl.crawlers import CRAWLERS
+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
from cyberdrop_dl.crawlers.crawler import Crawler, create_crawlers
from cyberdrop_dl.crawlers.discourse import DiscourseCrawler
from cyberdrop_dl.crawlers.generic import GenericCrawler
@@ -451,6 +452,8 @@ def create_generic_crawlers_by_config(generic_crawlers: GenericCrawlerInstances)
new_crawlers.update(create_crawlers(generic_crawlers.wordpress_media, WordPressMediaCrawler))
if generic_crawlers.discourse:
new_crawlers.update(create_crawlers(generic_crawlers.discourse, DiscourseCrawler))
+ if generic_crawlers.chevereto:
+ new_crawlers.update(create_crawlers(generic_crawlers.chevereto, CheveretoCrawler))
return new_crawlers
diff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.md
index 6167867a5..c7b808b45 100644
--- a/docs/reference/cli-arguments.md
+++ b/docs/reference/cli-arguments.md
@@ -337,5 +337,6 @@ generic_crawlers_instances:
--wordpress-media [WORDPRESS_MEDIA ...]
--wordpress-html [WORDPRESS_HTML ...]
--discourse [DISCOURSE ...]
+ --chevereto [CHEVERETO ...]
```
diff --git a/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md b/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md
index cb65efae9..1311fc575 100644
--- a/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md
+++ b/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md
@@ -7,15 +7,23 @@ CDL has some "generic" crawlers disabled by default. Generic crawlers are design
Currently, there are three generic crawlers:
-- `wordpress_media`: This crawler should work on any [WordPress](https://wordpress.com/) site where content primarily consists of images or galleries. The images need to be hosted on the site itself. It requires sites to have a public WordPress REST API.
+- `chevereto`: This works on any site that uses [Chevereto](https://chevereto.com//).
+
+- `discourse`: This works on any forum that uses [Discourse](https://www.discourse.org/).
- `wordpress_html`: This works on any WordPress site. It scrapes the actual HTML of the site, which means it works even on sites that have embedded third-party media like videos or links to hosting sites. It is always slower than `wordpress_media`.
-- `discourse`: This works on any forum that uses [Discourse](https://www.discourse.org/).
+- `wordpress_media`: This crawler should work on any [WordPress](https://wordpress.com/) site where content primarily consists of images or galleries. The images need to be hosted on the site itself. It requires sites to have a public WordPress REST API.
# generic_crawlers_instances
-## `wordpress_media`
+## `chevereto`
+
+| Type | Default |
+| --------------- | ------- |
+| `list[HttpURL]` | `[]` |
+
+## `discourse`
| Type | Default |
| --------------- | ------- |
@@ -27,7 +35,7 @@ Currently, there are three generic crawlers:
| --------------- | ------- |
| `list[HttpURL]` | `[]` |
-## `discourse`
+## `wordpress_media`
| Type | Default |
| --------------- | ------- |
diff --git a/docs/reference/supported-websites.md b/docs/reference/supported-websites.md
index fa4eaf819..9a466e448 100644
--- a/docs/reference/supported-websites.md
+++ b/docs/reference/supported-websites.md
@@ -46,6 +46,7 @@ List of sites supported by cyberdrop-dl-patched as of version 7.2.1
| Camwhores.tv | https://www.camwhores.tv | camwhores.tv | Categories: <br> - `/categories/...`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> |
| Catbox | https://catbox.moe | files.catbox.moe<br>litter.catbox.moe | Direct Links<br> |
| CelebForum | https://celebforum.to | celebforum | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|topic\|tema\|threads\|topics\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
+| Chevereto | ::GENERIC CRAWLER:: | | Album: <br> - `/a/...`<br> - `/album/...`<br>Direct Links<br>Image: <br> - `/image/...`<br> - `/img/...`<br>Profiles: <br> - `/<user_name>`<br>Video: <br> - `/video/..`<br> - `/videos/...`<br> |
| Coomer | https://coomer.su | coomer | Direct Links<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |
| Cyberdrop | https://cyberdrop.me/ | cyberdrop | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br> |
| Cyberfile | https://cyberfile.me/ | cyberfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |
@@ -70,14 +71,14 @@ List of sites supported by cyberdrop-dl-patched as of version 7.2.1
| HotPic | https://hotpic.cc | 2385290.xyz<br>hotpic | Album: <br> - `/album/...`<br>Image: <br> - `/i/...`<br> |
| Iceyfile | https://iceyfile.com/ | iceyfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |
| ImageBam | https://www.imagebam.com/ | imagebam | Gallery: <br> - `/gallery/...`<br>Gallery or Image: <br> - `/view/...`<br>Image: <br> - `/image/...`<br>Thumbnails: <br> - `thumbs<x>.imagebam.com/...`<br> |
-| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profiles: <br> - `/...`<br>Video: <br> - `/video/..`<br> |
+| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br> - `/album/...`<br>Direct Links<br>Image: <br> - `/image/...`<br> - `/img/...`<br>Profiles: <br> - `/<user_name>`<br>Video: <br> - `/video/..`<br> - `/videos/...`<br> |
| ImgBB | https://ibb.co | ibb.co<br>imgbb.co | Album: <br> - `/album/...`<br>Image: <br> - `/...`<br> |
| ImgBox | https://imgbox.com | imgbox | Album: <br> - `/g/...`<br>Direct Links<br>Image: <br> - `/...`<br> |
| Imgur | https://imgur.com/ | imgur | Album: <br> - `/a/...`<br>Direct Links<br>Gallery: <br> - `/gallery/...`<br>Image: <br> - `/...`<br> |
| Imx.to | https://imx.to | imx.to | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/t/...`<br> |
| IncestFlix | https://www.incestflix.com | incestflix | Tag: <br> - `/tag/...`<br>Video: <br> - `/watch/...`<br> |
| InfluencerBitches | https://influencerbitches.com | influencerbitches | Model: <br> - `/model/...`<br> |
-| JPG5 | https://jpg5.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profile: <br> - `/<user_name>`<br> |
+| JPG5 | https://jpg6.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su<br>jpg6.su | Album: <br> - `/a/...`<br> - `/album/...`<br>Direct Links<br>Image: <br> - `/image/...`<br> - `/img/...`<br>Profiles: <br> - `/<user_name>`<br> |
| Kemono | https://kemono.su | kemono | Direct Links<br>Discord Server: <br> - `/discord/<server_id>`<br>Discord Server Channel: <br> - `/discord/server/...#...`<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> |
| LeakedModels | https://leakedmodels.com/forum/ | leakedmodels | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|topic\|tema\|threads\|topics\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |
| LeakedZone | https://leakedzone.com | leakedzone | Model: <br> - `/<model_id>`<br>Video: <br> - `/<model_id>/video/<video_id>`<br> |