Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: switch to inheritance for chevereto sites (#898)
NTFSvolumeda7724101dae
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex b978caeb5..2e7f85f18 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -11,7 +11,6 @@ from .bestprettygirl import BestPrettyGirlCrawlerfrom .bunkrr import BunkrrCrawlerfrom .bunkrr_albums_io import BunkrAlbumsIOCrawlerfrom .catbox import CatboxCrawler-from .chevereto import CheveretoCrawlerfrom .coomer import CoomerCrawlerfrom .crawler import Crawlerfrom .cyberdrop import CyberdropCrawler@@ -33,9 +32,11 @@ from .google_drive import GoogleDriveCrawlerfrom .hotpic import HotPicCrawlerfrom .iceyfile import IceyFileCrawlerfrom .imagebam import ImageBamCrawler+from .imagepond import ImagePondCrawlerfrom .imgbb import ImgBBCrawlerfrom .imgbox import ImgBoxCrawlerfrom .imgur import ImgurCrawler+from .jpg5 import JPG5Crawlerfrom .kemono import KemonoCrawlerfrom .luscious import LusciousCrawlerfrom .mediafire import MediaFireCrawlerdiff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pynew file mode 100644index 000000000..4e95cb4e6--- /dev/null+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -0,0 +1,199 @@+from __future__ import annotations++from enum import StrEnum, auto+from functools import partialmethod+from typing import TYPE_CHECKING++from bs4 import BeautifulSoup++from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeError+from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ from yarl import URL++ from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem+++ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"+ALBUM_TITLE_SELECTOR = "a[data-text=album-name]"+ITEM_SELECTOR = "a[class='image-container --media']"+PROFILE_TITLE_SELECTOR = 'meta[property="og:title"]'+IMAGES_PARTS = "image", "img"+ALBUM_PARTS = "a", "album"+VIDEO_PARTS = "video", "videos"+DIRECT_LINK_PARTS = ("images",)+PASSWORD_PROTECTED = "This content is password protected"+VIDEO_SELECTOR = "meta[property='og:video']", "content"+IMAGE_SELECTOR = "div[id=image-viewer] img", "src"+++class Media(StrEnum):+ ALBUM = auto()+ IMAGE = auto()+ VIDEO = auto()+++def clean_name(url: URL) -> URL:+ return url.with_name(url.name.replace(".md.", ".").replace(".th.", "."))+++def sort_by_new(url: URL) -> URL:+ init_page = int(url.query.get("page") or 1)+ return url.with_query(sort="date_desc", page=init_page)+++class CheveretoCrawler(Crawler):+ next_page_selector = "a[data-pagination=next]"++ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""++ @create_task_id+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ """Determines where to send the scrape item based on the url."""+ return await self._fetch_chevereto_defaults(scrape_item)++ async def _fetch_chevereto_defaults(self, scrape_item: ScrapeItem) -> None:+ if scrape_item.url.host.count(".") > 1: # type: ignore+ return await self.handle_direct_link(scrape_item)+ if any(part in scrape_item.url.parts for part in ALBUM_PARTS):+ return await self.album(scrape_item)+ if any(part in scrape_item.url.parts for part in IMAGES_PARTS):+ return await self.image(scrape_item)+ if any(part in scrape_item.url.parts for part in VIDEO_PARTS):+ return await self.video(scrape_item)+ if any(part in scrape_item.url.parts for part in DIRECT_LINK_PARTS):+ return await self.handle_direct_link(scrape_item)+ await self.profile(scrape_item)++ @error_handling_wrapper+ async def profile(self, scrape_item: ScrapeItem) -> None:+ """Scrapes an user profile."""+ title: str = ""+ async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):+ if not title:+ title: str = soup.select_one(PROFILE_TITLE_SELECTOR)["content"] # type: ignore+ title = self.create_title(title)+ scrape_item.setup_as_profile(title)++ for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ # Item may be an image, a video or an album+ # For images, we can download the file from the thumbnail+ if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):+ _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)+ await self.handle_direct_link(new_scrape_item, thumb)+ continue+ # For videos and albums, we have to keep scraping+ self.manager.task_group.create_task(self.run(new_scrape_item))++ @error_handling_wrapper+ async def album(self, scrape_item: ScrapeItem) -> None:+ """Scrapes an album."""+ album_id, canonical_url = self.get_canonical_url(scrape_item.url)+ results = await self.get_album_results(album_id)+ original_url = scrape_item.url+ title: str = ""+ async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):+ if not title:+ await self.check_password_protected(soup, scrape_item.url)+ title: str = soup.select_one(ALBUM_TITLE_SELECTOR).text # type: ignore+ title = self.create_title(title, album_id)+ scrape_item.setup_as_album(title, album_id=album_id)+ scrape_item.url = canonical_url++ for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ assert thumb+ if self.check_album_results(thumb, results):+ continue+ _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)+ await self.handle_direct_link(new_scrape_item, thumb)++ # Sub album URL needs to be the full URL + a 'sub'+ # Using the canonical URL + 'sub' won't work because it redirects to the "homepage" of the album+ sub_album_url = original_url / "sub"+ async for soup in self.web_pager(sort_by_new(sub_album_url), trim=False):+ for _, sub_album in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ self.manager.task_group.create_task(self.run(sub_album))++ async def check_password_protected(self, soup: BeautifulSoup, url: URL) -> None:+ password = url.query.get("password", "")+ url = url.with_query(None)++ if PASSWORD_PROTECTED in soup.text and password:+ data = {"content-password": password}+ async with self.request_limiter:+ html = await self.client.post_data(self.domain, url, data=data, raw=True)+ soup = BeautifulSoup(html, "html.parser")++ if PASSWORD_PROTECTED in soup.text:+ raise PasswordProtectedError(message="Wrong password" if password else None)++ @error_handling_wrapper+ async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:+ """Handles a direct link."""+ link = clean_name(url or scrape_item.url)+ filename, ext = self.get_filename_and_ext(link.name)+ await self.handle_file(link, scrape_item, filename, ext)++ async def get_embed_info(self, url: URL) -> tuple[str, URL]:+ embed_url = self.primary_base_domain / "oembed" / ""+ embed_url = embed_url.with_query(url=str(url), format="json")+ async with self.request_limiter:+ json_resp: dict[str, str] = await self.client.get_json(self.domain, embed_url)++ link = clean_name(self.parse_url(json_resp["url"]))+ filename = json_resp["title"] + link.suffix+ return filename, link++ @error_handling_wrapper+ async def _proccess_media_item(self, scrape_item: ScrapeItem, type: Media, selector: tuple[str, str]) -> None:+ """Scrapes a media item."""+ if await self.check_complete_from_referer(scrape_item):+ return++ _, canonical_url = self.get_canonical_url(scrape_item.url, type)+ if await self.check_complete_from_referer(canonical_url):+ return++ async with self.request_limiter:+ soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)++ try:+ link_str: str = soup.select_one(selector[0])[selector[1]] # type: ignore+ link = self.parse_url(link_str)++ except AttributeError:+ raise ScrapeError(422, f"Couldn't find {type} source") from None++ scrape_item.possible_datetime = self.parse_date(get_date_from_soup(soup))+ scrape_item.url = canonical_url+ await self.handle_direct_link(scrape_item, link)++ video = partialmethod(_proccess_media_item, url_type=Media.VIDEO, selector=VIDEO_SELECTOR)+ image = partialmethod(_proccess_media_item, url_type=Media.IMAGE, selector=IMAGE_SELECTOR)++ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""++ def get_canonical_url(self, url: URL, url_type: Media = Media.ALBUM) -> tuple[str, URL]:+ """Returns the id and canonical URL from a given item (album, image or video)."""+ search_parts = ALBUM_PARTS+ if url_type == Media.IMAGE:+ search_parts = IMAGES_PARTS+ elif url_type == Media.VIDEO:+ search_parts = VIDEO_PARTS++ found_part = next(p for p in search_parts if p in url.parts)+ name_index = url.parts.index(found_part) + 1+ name = url.parts[name_index]+ item_id = name.rsplit(".")[-1]+ new_parts = url.parts[1:name_index] + (item_id,)+ new_path = "/" + "/".join(new_parts)+ return item_id, self.parse_url(new_path, url.origin())+++def get_date_from_soup(soup: BeautifulSoup) -> str:+ for row in soup.select(ITEM_DESCRIPTION_SELECTOR):+ if any(text in row.text.casefold() for text in ("uploaded", "added to")):+ return row.select_one("span")["title"] # type: ignore+ return ""diff --git a/cyberdrop_dl/crawlers/chevereto.py b/cyberdrop_dl/crawlers/chevereto.pydeleted file mode 100644index 730608ff6..000000000--- a/cyberdrop_dl/crawlers/chevereto.py+++ /dev/null@@ -1,273 +0,0 @@-from __future__ import annotations--import calendar-import datetime-import re-from typing import TYPE_CHECKING, ClassVar, Literal, TypeAlias--from aiolimiter import AsyncLimiter-from bs4 import BeautifulSoup-from yarl import URL--from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeError-from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id-from cyberdrop_dl.utils.utilities import error_handling_wrapper--if TYPE_CHECKING:- from cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem--CDN_PATTERNS = {- "jpg5.su": r"^(?:https?:\/\/?)((jpg.church\/images)|(simp..jpg.church)|(jpg.fish\/images)|(simp..jpg.fish)|(jpg.fishing\/images)|(simp..jpg.fishing)|(simp..host.church)|(simp..jpg..su))(\/.*)",- "imagepond.net": r"^(?:https?:\/\/)?(media.imagepond.net\/.*)",- "img.kiwi": r"^(?:https?:\/\/)?img\.kiwi\/images\/.*",-}--JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")--CDN_POSSIBILITIES = re.compile("|".join(CDN_PATTERNS.values()))-JS_SELECTOR = "script[data-cfasync='false']:contains('image_viewer_full_fix')"-JS_CONTENT_START = "document.addEventListener('DOMContentLoaded', function(event)"-ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"---UrlType: TypeAlias = Literal["album", "image", "video"]---class CheveretoCrawler(Crawler):- JPG5_DOMAINS: ClassVar[list[str]] = [- "jpg5.su",- "jpg.homes",- "jpg.church",- "jpg.fish",- "jpg.fishing",- "jpg.pet",- "jpeg.pet",- "jpg1.su",- "jpg2.su",- "jpg3.su",- "jpg4.su",- "host.church",- ]-- PRIMARY_BASE_DOMAINS: ClassVar[dict[str, URL]] = {- "jpg5.su": URL("https://jpg5.su"),- "imagepond.net": URL("https://imagepond.net"),- "img.kiwi": URL("https://img.kiwi"),- }-- FOLDER_DOMAINS: ClassVar[dict[str, str]] = {- "jpg5.su": "JPG5",- "imagepond.net": "ImagePond",- "img.kiwi": "ImgKiwi",- }-- SUPPORTED_SITES: ClassVar[dict[str, list]] = {- "jpg5.su": JPG5_DOMAINS,- "imagepond.net": ["imagepond.net"],- "img.kiwi": ["img.kiwi"],- }-- def __init__(self, manager: Manager, site: str) -> None:- super().__init__(manager, site, self.FOLDER_DOMAINS.get(site, "Chevereto"))- self.primary_base_domain = self.PRIMARY_BASE_DOMAINS.get(site, URL(f"https://{site}")) # type: ignore- self.next_page_selector = "a[data-pagination=next]" # type: ignore- self.album_title_selector = "a[data-text=album-name]"- self.item_selector = "a[class='image-container --media']"- self.profile_title_selector = 'meta[property="og:title"]'-- self.images_parts = "image", "img"- self.album_parts = "a", "album"- self.video_parts = "video", "videos"- self.direct_link_parts = ("images",)- if site == "jpg5.su":- self.request_limiter = AsyncLimiter(1, 5)-- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""-- @create_task_id- async def fetch(self, scrape_item: ScrapeItem) -> None:- """Determines where to send the scrape item based on the url."""- if is_direct_link(scrape_item.url):- return await self.handle_direct_link(scrape_item)-- scrape_item.url = scrape_item.url.with_host(self.primary_base_domain.host) # type: ignore- if any(part in scrape_item.url.parts for part in self.album_parts):- return await self.album(scrape_item)- if any(part in scrape_item.url.parts for part in self.images_parts):- return await self.image(scrape_item)- if any(part in scrape_item.url.parts for part in self.video_parts):- return await self.video(scrape_item)- if any(part in scrape_item.url.parts for part in self.direct_link_parts):- return await self.handle_direct_link(scrape_item)- await self.profile(scrape_item)-- @error_handling_wrapper- async def profile(self, scrape_item: ScrapeItem) -> None:- """Scrapes an user profile."""- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)-- title_tag = soup.select_one(self.profile_title_selector)- title_text: str = title_tag.get("content") # type: ignore- title = self.create_title(title_text)- scrape_item.setup_as_profile(title)-- async for soup in self.web_pager(get_sort_by_new_url(scrape_item.url), trim=False):- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, self.item_selector):- # Item may be an image, a video or an album- # For images, we can download the file from the thumbnail- if any(p in new_scrape_item.url.parts for p in self.images_parts):- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item, url_type="image")- await self.handle_direct_link(new_scrape_item, thumb)- continue- # For videos and albums, we have to keep scraping- self.manager.task_group.create_task(self.run(new_scrape_item))-- @error_handling_wrapper- async def album(self, scrape_item: ScrapeItem) -> None:- """Scrapes an album."""- album_id, canonical_url = self.get_canonical_url(scrape_item)- results = await self.get_album_results(album_id)- password = scrape_item.url.query.get("password", "")- scrape_item.url = scrape_item.url.with_query(None)- original_url = scrape_item.url-- async with self.request_limiter:- sub_albums = scrape_item.url / "sub"- soup: BeautifulSoup = await self.client.get_soup(self.domain, sub_albums)-- scrape_item.url = canonical_url-- if "This content is password protected" in soup.text and password:- data = {"content-password": password}- async with self.request_limiter:- html = await self.client.post_data(self.domain, scrape_item.url, data=data, raw=True)- soup = BeautifulSoup(html, "html.parser")-- if "This content is password protected" in soup.text:- raise PasswordProtectedError(message="Wrong password" if password else None)-- title_tag = soup.select_one(self.album_title_selector)- title_text: str = title_tag.get_text() # type: ignore- title = self.create_title(title_text, album_id)- scrape_item.setup_as_album(title, album_id=album_id)-- async for soup in self.web_pager(get_sort_by_new_url(scrape_item.url), trim=False):- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, self.item_selector):- assert thumb- if not self.check_album_results(thumb, results):- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item, url_type="image")- await self.handle_direct_link(new_scrape_item, thumb)-- # Sub album URL needs to be the full URL + a 'sub'- # Using the canonical URL + 'sub' won't work because it redirects to the "homepage" of the album- sub_album_url = original_url / "sub"- async for soup in self.web_pager(get_sort_by_new_url(sub_album_url), trim=False):- for _, sub_album in self.iter_children(scrape_item, soup, self.item_selector):- self.manager.task_group.create_task(self.run(sub_album))-- async def video(self, scrape_item: ScrapeItem) -> None:- """Scrapes a video."""- selector = "meta[property='og:video']", "content"- await self._proccess_media_item(scrape_item, "video", selector)-- async def image(self, scrape_item: ScrapeItem) -> None:- """Scrapes an image."""- selector = "div[id=image-viewer] img", "src"- await self._proccess_media_item(scrape_item, "image", selector)-- @error_handling_wrapper- async def _proccess_media_item(self, scrape_item: ScrapeItem, url_type: UrlType, selector: tuple[str, str]) -> None:- """Scrapes a media item."""- if await self.check_complete_from_referer(scrape_item):- return-- _, canonical_url = self.get_canonical_url(scrape_item, url_type=url_type)- if await self.check_complete_from_referer(canonical_url):- return-- if self.domain == "jpg5.su":- _, link = await self.get_embed_info(scrape_item.url)-- else:- async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)-- try:- link_str: str = soup.select_one(selector[0]).get(selector[1]) # type: ignore- link = self.parse_url(link_str)-- except AttributeError:- raise ScrapeError(422, f"Couldn't find {url_type} source") from None-- add_date_from_soup(scrape_item, soup)-- scrape_item.url = canonical_url- await self.handle_direct_link(scrape_item, link)-- @error_handling_wrapper- async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:- """Handles a direct link."""- link = url or scrape_item.url- link = link.with_name(link.name.replace(".md.", ".").replace(".th.", "."))- link = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))- filename, ext = self.get_filename_and_ext(link.name)- await self.handle_file(link, scrape_item, filename, ext)-- async def get_embed_info(self, url: URL) -> tuple[str, URL]:- embed_url = self.primary_base_domain / "oembed" / ""- embed_url = embed_url.with_query(url=str(url), format="json")- async with self.request_limiter:- json_resp: dict[str, str] = await self.client.get_json(self.domain, embed_url)-- link_str = json_resp["url"].replace(".md.", ".").replace(".th.", ".")- link = self.parse_url(link_str)- filename = json_resp["title"] + link.suffix- return filename, link-- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""-- def get_canonical_url(self, scrape_item: ScrapeItem, url_type: UrlType = "album") -> tuple[str, URL]:- """Returns the id and canonical URL from a given item (album, image or video)."""- search_parts = self.album_parts- if url_type == "image":- search_parts = self.images_parts- elif url_type == "video":- search_parts = self.video_parts-- found_part = next(part for part in search_parts if part in scrape_item.url.parts)- name_index = scrape_item.url.parts.index(found_part) + 1- name = scrape_item.url.parts[name_index]- _id = name.rsplit(".")[-1]- new_parts = scrape_item.url.parts[1:name_index] + (_id,)- new_path = "/" + "/".join(new_parts)- return _id, self.parse_url(new_path, scrape_item.url.with_path("/"))---def get_sort_by_new_url(url: URL) -> URL:- return url.with_query({"sort": "date_desc", "page": 1})---def parse_datetime(date: str) -> int:- """Parses a datetime string into a unix timestamp."""- parsed_date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")- return calendar.timegm(parsed_date.timetuple())---def is_direct_link(url: URL) -> bool:- """Determines if the url is a direct link or not."""- return bool(CDN_POSSIBILITIES.match(str(url)))---def add_date_from_soup(scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:- desc_rows = soup.select(ITEM_DESCRIPTION_SELECTOR)- date_str: str | None = None- for row in desc_rows:- if any(text in row.text.casefold() for text in ("uploaded", "added to")):- date_str = row.select_one("span").get("title") # type: ignore- break-- if date_str:- date = parse_datetime(date_str)- scrape_item.possible_datetime = datediff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.pynew file mode 100644index 000000000..88e606106--- /dev/null+++ b/cyberdrop_dl/crawlers/imagepond.py@@ -0,0 +1,21 @@+from __future__ import annotations++from typing import TYPE_CHECKING++from yarl import URL++from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler++if TYPE_CHECKING:+ from cyberdrop_dl.managers.manager import Manager++PRIMARY_BASE_DOMAIN = URL("https://imagepond.net")+++class ImagePondCrawler(CheveretoCrawler):+ primary_base_domain = PRIMARY_BASE_DOMAIN++ def __init__(self, manager: Manager) -> None:+ super().__init__(manager, "imagepond.net", "ImagePond")++ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pynew file mode 100644index 000000000..a891cf614--- /dev/null+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -0,0 +1,73 @@+from __future__ import annotations++import re+from typing import TYPE_CHECKING++from aiolimiter import AsyncLimiter+from yarl import URL++from cyberdrop_dl.crawlers.crawler import create_task_id+from cyberdrop_dl.utils.utilities import error_handling_wrapper++from ._chevereto import CheveretoCrawler++if TYPE_CHECKING:+ from cyberdrop_dl.managers.manager import Manager+ from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem++PRIMARY_BASE_DOMAIN = URL("https://jpg5.su")+JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")+JPG5_DOMAINS = [+ "jpg5.su",+ "jpg.homes",+ "jpg.church",+ "jpg.fish",+ "jpg.fishing",+ "jpg.pet",+ "jpeg.pet",+ "jpg1.su",+ "jpg2.su",+ "jpg3.su",+ "jpg4.su",+ "host.church",+]+++class JPG5Crawler(CheveretoCrawler):+ primary_base_domain = PRIMARY_BASE_DOMAIN+ SUPPORTED_SITES = {"jpg5.su": JPG5_DOMAINS} # noqa: RUF012++ def __init__(self, manager: Manager, _) -> None:+ super().__init__(manager, "jpg5.su", "JPG5")+ self.request_limiter = AsyncLimiter(1, 5)++ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""++ @create_task_id+ async def fetch(self, scrape_item: ScrapeItem):+ scrape_item.url = scrape_item.url.with_host("jpg5.su")+ return await self._fetch_chevereto_defaults(scrape_item)++ async def video(self, scrape_item: ScrapeItem) -> None:+ """Scrapes a video."""+ raise ValueError++ @error_handling_wrapper+ async def _proccess_media_item(self, scrape_item: ScrapeItem, url_type, *_) -> None:+ """Scrapes a media item."""+ if await self.check_complete_from_referer(scrape_item):+ return++ _, canonical_url = self.get_canonical_url(scrape_item.url, url_type)+ if await self.check_complete_from_referer(canonical_url):+ return++ _, link = await self.get_embed_info(scrape_item.url)+ scrape_item.url = canonical_url+ await self.handle_direct_link(scrape_item, link)++ async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:+ """Handles a direct link."""+ link = url or scrape_item.url+ link = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))+ super().handle_direct_link(scrape_item, link)