Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: make chevereto a generic crawler (#1176)
NTFSvolume67df969fc9ca
diff --git a/cyberdrop_dl/config/global_model.py b/cyberdrop_dl/config/global_model.pyindex 835d94c30..1a268e7dd 100755--- a/cyberdrop_dl/config/global_model.py+++ b/cyberdrop_dl/config/global_model.py@@ -105,6 +105,7 @@ class GenericCrawlerInstances(BaseModel):wordpress_media: ListPydanticURL = []wordpress_html: ListPydanticURL = []discourse: ListPydanticURL = []+ chevereto: ListPydanticURL = []class GlobalSettings(ConfigModel):diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 80c5ca7f8..e5c9c6eee 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -3,6 +3,7 @@ from __future__ import annotationsfrom cyberdrop_dl import env+from ._chevereto import CheveretoCrawlerfrom .acidimg import AcidImgCrawlerfrom .archivebate import ArchiveBateCrawlerfrom .ashemaletube import AShemaleTubeCrawler@@ -104,7 +105,7 @@ from .yandex_disk import YandexDiskCrawlerfrom .youjizz import YouJizzCrawlerFORUM_CRAWLERS = XF_CRAWLERS.union(INVISION_CRAWLERS, DISCOURSE_CRAWLERS, VBULLETIN_CRAWLERS)-GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler}+GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler, CheveretoCrawler}ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler}diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 5ed25c549..16fe232b3 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -1,211 +1,197 @@from __future__ import annotations-from enum import StrEnum, auto-from functools import partialmethod+import base64+import itertoolsfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import PasswordProtectedError, ScrapeError-from cyberdrop_dl.utils import css+from cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapperif TYPE_CHECKING:from bs4 import BeautifulSoupfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.crawlers.crawler import SupportedPaths-ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"-ALBUM_TITLE_SELECTOR = "a[data-text=album-name]"-ITEM_SELECTOR = "a[class='image-container --media']"-PROFILE_TITLE_SELECTOR = 'meta[property="og:title"]'-IMAGES_PARTS = "image", "img"-ALBUM_PARTS = "a", "album"-VIDEO_PARTS = "video", "videos"-DIRECT_LINK_PARTS = ("images",)-VIDEO_SELECTOR = "meta[property='og:video']", "content"-IMAGE_SELECTOR = "div[id=image-viewer] img", "src"-ENCRYPTED_IMAGE_SELECTOR = "div.image-viewer-main > img"--class Media(StrEnum):- ALBUM = auto()- IMAGE = auto()- VIDEO = auto()---def clean_name(url: AbsoluteHttpURL) -> AbsoluteHttpURL:- new_name = url.name- for trash in (".md.", ".th.", ".fr."):- new_name = new_name.replace(trash, ".")- return url.with_name(new_name)---def sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:- init_page = int(url.query.get("page") or 1)- return url.with_query(sort="date_desc", page=init_page)---class CheveretoCrawler(Crawler, is_abc=True):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Album": "/a/...",- "Image": "/img/...",- "Profiles": "/...",- "Video": "/video/..",+class Selectors:+ ITEM_DESCRIPTION = "p[class*=description-meta]"+ ITEM = "a[class='image-container --media']"+ MAIN_IMAGE = "div#image-viewer img", "src"+ DATE_SINGLE_ITEM = f"{ITEM_DESCRIPTION}:contains('Uploaded') span"+ DATE_ALBUM_ITEM = f"{ITEM_DESCRIPTION}:contains('Added to') span"+ DATE = f"{DATE_SINGLE_ITEM}, {DATE_ALBUM_ITEM}", "title"+++_SELECTORS = Selectors()+_DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"+++class CheveretoCrawler(Crawler, is_generic=True):+ SUPPORTED_PATHS: ClassVar[dict[str, str | tuple[str, ...]]] = {+ "Album": (+ "/a/...",+ "/album/...",+ ),+ "Image": (+ "/img/...",+ "/image/...",+ ),+ "Profiles": "/<user_name>",+ "Video": (+ "/video/..",+ "/videos/...",+ ),"Direct links": "",}NEXT_PAGE_SELECTOR: ClassVar[str] = "a[data-pagination=next]"+ CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = True- async def fetch(self, scrape_item: ScrapeItem) -> None:- return await self._fetch_chevereto_defaults(scrape_item)+ def __init_subclass__(cls, **kwargs) -> None:+ if not cls.CHEVERETO_SUPPORTS_VIDEO:+ paths = cls.SUPPORTED_PATHS.copy()+ _ = paths.pop("Video", None)+ cls.SUPPORTED_PATHS = paths+ super().__init_subclass__(**kwargs)- async def _fetch_chevereto_defaults(self, scrape_item: ScrapeItem) -> None:- if scrape_item.url.host.removeprefix("www.").count(".") > 1:- return await self.handle_direct_link(scrape_item)- if any(part in scrape_item.url.parts for part in ALBUM_PARTS):- return await self.album(scrape_item)- if any(part in scrape_item.url.parts for part in IMAGES_PARTS):- return await self.image(scrape_item)- if any(part in scrape_item.url.parts for part in VIDEO_PARTS):- return await self.video(scrape_item)- if any(part in scrape_item.url.parts for part in DIRECT_LINK_PARTS):+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ if self.is_subdomain(scrape_item.url):return await self.handle_direct_link(scrape_item)- await self.profile(scrape_item)++ match scrape_item.url.parts[1:]:+ case ["a" | "album", album_slug]:+ return await self.album(scrape_item, _id(album_slug))+ case ["img" | "image" | "video" | "videos" as part, slug]:+ item_id = _id(slug)+ canonical_url = self.PRIMARY_URL / part / item_id+ scrape_item.url = canonical_url+ return await self.media(scrape_item)+ case ["images", _, *_]:+ return await self.handle_direct_link(scrape_item)+ case [_]:+ return await self.profile(scrape_item)+ case _:+ raise ValueError@error_handling_wrapperasync def profile(self, scrape_item: ScrapeItem) -> None:title: str = ""- async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):if not title:- title: str = css.select_one_get_attr(soup, PROFILE_TITLE_SELECTOR, "content")- title = self.create_title(title)+ title = self.create_title(open_graph.title(soup))scrape_item.setup_as_profile(title)-- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):- # Item may be an image, a video or an album- # For images, we can download the file from the thumbnail- assert thumb- if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)- await self.handle_direct_link(new_scrape_item, thumb)- continue- # For videos and albums, we have to keep scraping- self.manager.task_group.create_task(self.run(new_scrape_item))+ self._process_page(scrape_item, soup)@error_handling_wrapper- async def album(self, scrape_item: ScrapeItem) -> None:- album_id, canonical_url = self.get_canonical_url(scrape_item.url)+ async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:results = await self.get_album_results(album_id)- original_url = scrape_item.urltitle: str = ""- async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):+ async for soup in self.web_pager(_sort_by_new(scrape_item.url), trim=False):if not title:- if is_password_protected(soup):+ if _is_password_protected(soup):soup = await self._get_soup_w_password(scrape_item)- title: str = css.select_one_get_text(soup, ALBUM_TITLE_SELECTOR)- title = self.create_title(title, album_id)+ title = self.create_title(open_graph.title(soup), album_id)scrape_item.setup_as_album(title, album_id=album_id)- scrape_item.url = canonical_url-- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):- assert thumb- source = clean_name(thumb)- if self.check_album_results(source, results):- continue- if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)- await self.handle_direct_link(new_scrape_item, thumb)- continue-- self.manager.task_group.create_task(self.run(new_scrape_item))-- # Sub album URL needs to be the full URL + a 'sub'- # Using the canonical URL + 'sub' won't work because it redirects to the "homepage" of the album- sub_album_url = original_url / "sub"- async for soup in self.web_pager(sort_by_new(sub_album_url), trim=False):- for _, sub_album in self.iter_children(scrape_item, soup, ITEM_SELECTOR):- self.manager.task_group.create_task(self.run(sub_album))+ self._process_page(scrape_item, soup, results)++ sub_album_url = scrape_item.url / "sub"+ async for soup in self.web_pager(_sort_by_new(sub_album_url), trim=False):+ for _, sub_album in self.iter_children(scrape_item, soup, _SELECTORS.ITEM):+ self.create_task(self.run(sub_album))++ def _process_page(+ self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None+ ) -> None:+ for thumb, new_scrape_item in self.iter_children(scrape_item, soup, _SELECTORS.ITEM):+ assert thumb+ source = _thumbnail_to_src(thumb)+ if results and self.check_album_results(source, results):+ continue+ # For images, we can download the file from the thumbnail, skipping an aditional+ # cons: we won't get the upload date+ if image_url := _match_img(new_scrape_item.url):+ new_scrape_item.url = image_url+ self.create_task(self.handle_direct_link(new_scrape_item, source))+ continue++ self.create_task(self.run(new_scrape_item))async def _get_soup_w_password(self, scrape_item: ScrapeItem) -> BeautifulSoup:password = scrape_item.pop_query("password")if not password:raise PasswordProtectedError- data = {"content-password": password}async with self.request_limiter:- soup = await self.client.post_data_get_soup(self.DOMAIN, sort_by_new(scrape_item.url), data=data)- if is_password_protected(soup):+ soup = await self.client.post_data_get_soup(+ self.DOMAIN, _sort_by_new(scrape_item.url), data={"content-password": password}+ )+ if _is_password_protected(soup):raise PasswordProtectedError(message="Wrong password")return soup@error_handling_wrapperasync def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:- """Handles a direct link."""- link = clean_name(url or scrape_item.url)+ link = _thumbnail_to_src(url or scrape_item.url)filename, ext = self.get_filename_and_ext(link.name)await self.handle_file(link, scrape_item, filename, ext)- async def get_embed_info(self, url: AbsoluteHttpURL) -> tuple[str, AbsoluteHttpURL]:- embed_url = self.PRIMARY_URL / "oembed" / ""- embed_url = embed_url.with_query(url=str(url), format="json")- async with self.request_limiter:- json_resp: dict[str, str] = await self.client.get_json(self.DOMAIN, embed_url)-- link = clean_name(self.parse_url(json_resp["url"]))- filename = json_resp["title"] + link.suffix- return filename, link-@error_handling_wrapper- async def _proccess_media_item(self, scrape_item: ScrapeItem, media_type: Media, selector: tuple[str, str]) -> None:+ async def media(self, scrape_item: ScrapeItem) -> None:if await self.check_complete_from_referer(scrape_item):return- _, canonical_url = self.get_canonical_url(scrape_item.url, media_type)- if await self.check_complete_from_referer(canonical_url):- return-async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.DOMAIN, scrape_item.url)+ soup = await self.client.get_soup(self.DOMAIN, scrape_item.url)try:- link_str: str = css.select_one_get_attr(soup, *selector)- if "loading.svg" in link_str:- link_str = css.select_one_get_attr(soup, ENCRYPTED_IMAGE_SELECTOR, "data-src")+ link_str = open_graph.get("video", soup) or open_graph.get("image", soup)+ if not link_str or "loading.svg" in link_str:+ link_str = css.select_one_get_attr(soup, *_SELECTORS.MAIN_IMAGE)link = self.parse_url(link_str)- except AttributeError:- raise ScrapeError(422, f"Couldn't find {media_type} source") from None+ except css.SelectorError:+ raise ScrapeError(422, "Couldn't find media source") from None- scrape_item.possible_datetime = self.parse_date(get_date_from_soup(soup))- scrape_item.url = canonical_url+ date_str = css.select_one_get_attr(soup, *_SELECTORS.DATE)+ scrape_item.possible_datetime = self.parse_iso_date(date_str)await self.handle_direct_link(scrape_item, link)- video = partialmethod(_proccess_media_item, media_type=Media.VIDEO, selector=VIDEO_SELECTOR)- image = partialmethod(_proccess_media_item, media_type=Media.IMAGE, selector=IMAGE_SELECTOR)+ def parse_url(+ self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True+ ) -> AbsoluteHttpURL:+ if not link_str.startswith("https") and not link_str.startswith("/"):+ link_str = _xor_decrypt(link_str, _DECRYPTION_KEY)+ return super().parse_url(link_str, relative_to, trim=trim)- def get_canonical_url(self, url: AbsoluteHttpURL, media_type: Media = Media.ALBUM) -> tuple[str, AbsoluteHttpURL]:- """Returns the id and canonical URL from a given item (album, image or video)."""- search_parts = ALBUM_PARTS- if media_type == Media.IMAGE:- search_parts = IMAGES_PARTS- elif media_type == Media.VIDEO:- search_parts = VIDEO_PARTS- found_part = next(p for p in search_parts if p in url.parts)- name_index = url.parts.index(found_part) + 1- name = url.parts[name_index]- item_id = name.rsplit(".")[-1]- new_parts = (*url.parts[1:name_index], item_id)- new_path = "/" + "/".join(new_parts)- return item_id, self.parse_url(new_path, url.origin())+def _is_password_protected(soup: BeautifulSoup) -> bool:+ return "This content is password protected" in soup.text-def get_date_from_soup(soup: BeautifulSoup) -> str:- for row in soup.select(ITEM_DESCRIPTION_SELECTOR):- if any(text in row.get_text().casefold() for text in ("uploaded", "added to")):- return css.select_one_get_attr(row, "span", "title")- return ""+def _id(slug: str) -> str:+ return slug.rsplit(".")[-1]-def is_password_protected(soup: BeautifulSoup) -> bool:- return "This content is password protected" in soup.text+def _match_img(url: AbsoluteHttpURL) -> AbsoluteHttpURL | None:+ match url.parts[1:]:+ case ["img" | "image" as part, image_slug, *_]:+ return url.origin() / part / _id(image_slug)+++def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ new_name = url.name+ for trash in (".md.", ".th.", ".fr."):+ new_name = new_name.replace(trash, ".")+ return url.with_name(new_name)+++def _sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ init_page = int(url.query.get("page") or 1)+ return url.with_query(sort="date_desc", page=init_page)+++def _xor_decrypt(encrypted_str: str, key: bytes) -> str:+ encrypted_data = bytes.fromhex(base64.b64decode(encrypted_str).decode())+ decrypted_data = bytearray(b_input ^ b_key for b_input, b_key in zip(encrypted_data, itertools.cycle(key)))+ return decrypted_data.decode("utf-8", errors="ignore")diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 92cd4ac02..6a7209941 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -252,9 +252,13 @@ class Crawler(ABC):finally:self.manager.progress_manager.scraping_progress.remove_task(task_id)+ @staticmethod+ def is_subdomain(url: AbsoluteHttpURL) -> bool:+ return url.host.removeprefix("www.").count(".") > 1+@classmethod- def is_subdomain(cls, url: AbsoluteHttpURL, domain_to_compare: str | None = None) -> bool:- primary_domain = (domain_to_compare or cls.PRIMARY_URL.host).removeprefix("www.")+ def is_self_subdomain(cls, url: AbsoluteHttpURL) -> bool:+ primary_domain = cls.PRIMARY_URL.host.removeprefix("www.")other_domain = url.host.removeprefix("www.")if primary_domain == other_domain:return Falsediff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex b6f8274f3..5e81e99e2 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -1,6 +1,5 @@from __future__ import annotations-import binasciiimport refrom typing import TYPE_CHECKING, ClassVar@@ -11,11 +10,9 @@ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom ._chevereto import CheveretoCrawlerif TYPE_CHECKING:- from cyberdrop_dl.crawlers.crawler import SupportedDomains, SupportedPaths+ from cyberdrop_dl.crawlers.crawler import SupportedDomainsfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem--_DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"_OLD_DOMAINS = ("jpg.homes","jpg.church",@@ -35,47 +32,29 @@ _REPLACE_OLD_HOST_REGEX = re.compile("|".join(_OLD_DOMAINS))class JPG5Crawler(CheveretoCrawler):- SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Album": "/a/...",- "Image": "/img/...",- "Profile": "/<user_name>",- "Direct links": "",- }-SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = (*_OLD_DOMAINS, "jpg6.su")DOMAIN: ClassVar[str] = "jpg5.su"FOLDER_DOMAIN: ClassVar[str] = "JPG5"-PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://jpg6.su")+ CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = Falsedef __post_init__(self) -> None:self.request_limiter = AsyncLimiter(1, 1)async def fetch(self, scrape_item: ScrapeItem) -> None:- scrape_item.url = fix_host(scrape_item.url)- return await self._fetch_chevereto_defaults(scrape_item)-- async def video(self, scrape_item: ScrapeItem) -> None:- raise ValueError-- def parse_url(- self, link_str: str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True- ) -> AbsoluteHttpURL:- if not link_str.startswith("https") and not link_str.startswith("/"):- link_str = decrypt_xor(link_str, _DECRYPTION_KEY)- return super().parse_url(link_str, relative_to, trim=trim)+ scrape_item.url = _fix_host(scrape_item.url)+ return await super().fetch(scrape_item)async def handle_direct_link(self, scrape_item: ScrapeItem, url: AbsoluteHttpURL | None = None) -> None:- """Handles a direct link."""link = url or scrape_item.url- link = fix_host(link)+ link = _fix_host(link)await super().handle_direct_link(scrape_item, link)"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""-def fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+def _fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:new_host = re.sub(_REPLACE_OLD_HOST_REGEX, r"jpg5.su", url.host)if new_host.removeprefix("www.") == "jpg5.su":# replace only if it is matches the second level domain exactly@@ -84,12 +63,6 @@ def fix_host(url: AbsoluteHttpURL) -> AbsoluteHttpURL:return url.with_host(new_host)-def decrypt_xor(encrypted_str: str, key: bytes) -> str:- div = len(key)- encrypted = bytes.fromhex(binascii.a2b_base64(encrypted_str).decode())- return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()--def fix_db_referer(referer: str) -> str:url = AbsoluteHttpURL(referer)- return str(fix_host(url))+ return str(_fix_host(url))diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.pyindex 0f26ff0b9..06ef47d28 100644--- a/cyberdrop_dl/crawlers/redgifs.py+++ b/cyberdrop_dl/crawlers/redgifs.py@@ -62,7 +62,7 @@ class RedGifsCrawler(Crawler):case ["watch" | "ifr", gif_id]:return await self.gif(scrape_item, _id(gif_id))- if self.is_subdomain(scrape_item.url) and len(scrape_item.url.parts) == 2:+ if self.is_self_subdomain(scrape_item.url) and len(scrape_item.url.parts) == 2:scrape_item.url = _canonical_url(scrape_item.url.name)self.manager.task_group.create_task(self.run(scrape_item))returndiff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 15f0f295c..bf7a550ed 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -12,6 +12,7 @@ from yarl import URLfrom cyberdrop_dl.constants import BLOCKED_DOMAINS, REGEX_LINKSfrom cyberdrop_dl.crawlers import CRAWLERS+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawlerfrom cyberdrop_dl.crawlers.crawler import Crawler, create_crawlersfrom cyberdrop_dl.crawlers.discourse import DiscourseCrawlerfrom cyberdrop_dl.crawlers.generic import GenericCrawler@@ -451,6 +452,8 @@ def create_generic_crawlers_by_config(generic_crawlers: GenericCrawlerInstances)new_crawlers.update(create_crawlers(generic_crawlers.wordpress_media, WordPressMediaCrawler))if generic_crawlers.discourse:new_crawlers.update(create_crawlers(generic_crawlers.discourse, DiscourseCrawler))+ if generic_crawlers.chevereto:+ new_crawlers.update(create_crawlers(generic_crawlers.chevereto, CheveretoCrawler))return new_crawlersdiff --git a/docs/reference/cli-arguments.md b/docs/reference/cli-arguments.mdindex 6167867a5..c7b808b45 100644--- a/docs/reference/cli-arguments.md+++ b/docs/reference/cli-arguments.md@@ -337,5 +337,6 @@ generic_crawlers_instances:--wordpress-media [WORDPRESS_MEDIA ...]--wordpress-html [WORDPRESS_HTML ...]--discourse [DISCOURSE ...]+ --chevereto [CHEVERETO ...]```diff --git a/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md b/docs/reference/configuration-options/global-settings/generic_crawlers_instances.mdindex cb65efae9..1311fc575 100644--- a/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md+++ b/docs/reference/configuration-options/global-settings/generic_crawlers_instances.md@@ -7,15 +7,23 @@ CDL has some "generic" crawlers disabled by default. Generic crawlers are designCurrently, there are three generic crawlers:-- `wordpress_media`: This crawler should work on any [WordPress](https://wordpress.com/) site where content primarily consists of images or galleries. The images need to be hosted on the site itself. It requires sites to have a public WordPress REST API.+- `chevereto`: This works on any site that uses [Chevereto](https://chevereto.com//).++- `discourse`: This works on any forum that uses [Discourse](https://www.discourse.org/).- `wordpress_html`: This works on any WordPress site. It scrapes the actual HTML of the site, which means it works even on sites that have embedded third-party media like videos or links to hosting sites. It is always slower than `wordpress_media`.-- `discourse`: This works on any forum that uses [Discourse](https://www.discourse.org/).+- `wordpress_media`: This crawler should work on any [WordPress](https://wordpress.com/) site where content primarily consists of images or galleries. The images need to be hosted on the site itself. It requires sites to have a public WordPress REST API.# generic_crawlers_instances-## `wordpress_media`+## `chevereto`++| Type | Default |+| --------------- | ------- |+| `list[HttpURL]` | `[]` |++## `discourse`| Type | Default || --------------- | ------- |@@ -27,7 +35,7 @@ Currently, there are three generic crawlers:| --------------- | ------- || `list[HttpURL]` | `[]` |-## `discourse`+## `wordpress_media`| Type | Default || --------------- | ------- |diff --git a/docs/reference/supported-websites.md b/docs/reference/supported-websites.mdindex fa4eaf819..9a466e448 100644--- a/docs/reference/supported-websites.md+++ b/docs/reference/supported-websites.md@@ -46,6 +46,7 @@ List of sites supported by cyberdrop-dl-patched as of version 7.2.1| Camwhores.tv | https://www.camwhores.tv | camwhores.tv | Categories: <br> - `/categories/...`<br>Members: <br> - `/members/<member_id>`<br>Search: <br> - `/search/...`<br>Tags: <br> - `/tags/...`<br>Videos: <br> - `/videos/...`<br> || Catbox | https://catbox.moe | files.catbox.moe<br>litter.catbox.moe | Direct Links<br> || CelebForum | https://celebforum.to | celebforum | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|topic\|tema\|threads\|topics\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> |+| Chevereto | ::GENERIC CRAWLER:: | | Album: <br> - `/a/...`<br> - `/album/...`<br>Direct Links<br>Image: <br> - `/image/...`<br> - `/img/...`<br>Profiles: <br> - `/<user_name>`<br>Video: <br> - `/video/..`<br> - `/videos/...`<br> || Coomer | https://coomer.su | coomer | Direct Links<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> || Cyberdrop | https://cyberdrop.me/ | cyberdrop | Album: <br> - `/a/...`<br>Direct Links<br>File: <br> - `/f/...`<br> || Cyberfile | https://cyberfile.me/ | cyberfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> |@@ -70,14 +71,14 @@ List of sites supported by cyberdrop-dl-patched as of version 7.2.1| HotPic | https://hotpic.cc | 2385290.xyz<br>hotpic | Album: <br> - `/album/...`<br>Image: <br> - `/i/...`<br> || Iceyfile | https://iceyfile.com/ | iceyfile | Files: <br> - `/...`<br>Folder: <br> - `/folder/...`<br>Shared folder: <br> - `/shared/...`<br> || ImageBam | https://www.imagebam.com/ | imagebam | Gallery: <br> - `/gallery/...`<br>Gallery or Image: <br> - `/view/...`<br>Image: <br> - `/image/...`<br>Thumbnails: <br> - `thumbs<x>.imagebam.com/...`<br> |-| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profiles: <br> - `/...`<br>Video: <br> - `/video/..`<br> |+| ImagePond | https://imagepond.net | imagepond.net | Album: <br> - `/a/...`<br> - `/album/...`<br>Direct Links<br>Image: <br> - `/image/...`<br> - `/img/...`<br>Profiles: <br> - `/<user_name>`<br>Video: <br> - `/video/..`<br> - `/videos/...`<br> || ImgBB | https://ibb.co | ibb.co<br>imgbb.co | Album: <br> - `/album/...`<br>Image: <br> - `/...`<br> || ImgBox | https://imgbox.com | imgbox | Album: <br> - `/g/...`<br>Direct Links<br>Image: <br> - `/...`<br> || Imgur | https://imgur.com/ | imgur | Album: <br> - `/a/...`<br>Direct Links<br>Gallery: <br> - `/gallery/...`<br>Image: <br> - `/...`<br> || Imx.to | https://imx.to | imx.to | Image: <br> - `/i/...`<br>Thumbnail: <br> - `/t/...`<br> || IncestFlix | https://www.incestflix.com | incestflix | Tag: <br> - `/tag/...`<br>Video: <br> - `/watch/...`<br> || InfluencerBitches | https://influencerbitches.com | influencerbitches | Model: <br> - `/model/...`<br> |-| JPG5 | https://jpg5.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su | Album: <br> - `/a/...`<br>Direct Links<br>Image: <br> - `/img/...`<br>Profile: <br> - `/<user_name>`<br> |+| JPG5 | https://jpg6.su | host.church<br>jpeg.pet<br>jpg.church<br>jpg.fish<br>jpg.fishing<br>jpg.homes<br>jpg.pet<br>jpg1.su<br>jpg2.su<br>jpg3.su<br>jpg4.su<br>jpg5.su<br>jpg6.su | Album: <br> - `/a/...`<br> - `/album/...`<br>Direct Links<br>Image: <br> - `/image/...`<br> - `/img/...`<br>Profiles: <br> - `/<user_name>`<br> || Kemono | https://kemono.su | kemono | Direct Links<br>Discord Server: <br> - `/discord/<server_id>`<br>Discord Server Channel: <br> - `/discord/server/...#...`<br>Favorites: <br> - `/favorites/<user>`<br>Individual Post: <br> - `<service>/<user>/post/...`<br>Model: <br> - `/<service>/user/<user>`<br>Search: <br> - `/search?...`<br> || LeakedModels | https://leakedmodels.com/forum/ | leakedmodels | Attachments: <br> - `/(attachments\|data\|uploads)/...`<br>Threads: <br> - `/(thread\|topic\|tema\|threads\|topics\|temas)/<thread_name_and_id>`<br> - `/goto/<post_id>`<br> - `/posts/<post_id>`<br><br><br>**NOTES**<br> - base crawler: Xenforo<br> || LeakedZone | https://leakedzone.com | leakedzone | Model: <br> - `/<model_id>`<br>Video: <br> - `/<model_id>/video/<video_id>`<br> |