Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: switch to inheritance for chevereto sites (#898)

NTFSvolumeda7724101dae
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index b978caeb5..2e7f85f18 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -11,7 +11,6 @@ from .bestprettygirl import BestPrettyGirlCrawler
from .bunkrr import BunkrrCrawler
from .bunkrr_albums_io import BunkrAlbumsIOCrawler
from .catbox import CatboxCrawler
-from .chevereto import CheveretoCrawler
from .coomer import CoomerCrawler
from .crawler import Crawler
from .cyberdrop import CyberdropCrawler
@@ -33,9 +32,11 @@ from .google_drive import GoogleDriveCrawler
from .hotpic import HotPicCrawler
from .iceyfile import IceyFileCrawler
from .imagebam import ImageBamCrawler
+from .imagepond import ImagePondCrawler
from .imgbb import ImgBBCrawler
from .imgbox import ImgBoxCrawler
from .imgur import ImgurCrawler
+from .jpg5 import JPG5Crawler
from .kemono import KemonoCrawler
from .luscious import LusciousCrawler
from .mediafire import MediaFireCrawler
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
new file mode 100644
index 000000000..4e95cb4e6
--- /dev/null
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+from enum import StrEnum, auto
+from functools import partialmethod
+from typing import TYPE_CHECKING
+
+from bs4 import BeautifulSoup
+
+from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeError
+from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from yarl import URL
+
+ from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
+
+
+ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"
+ALBUM_TITLE_SELECTOR = "a[data-text=album-name]"
+ITEM_SELECTOR = "a[class='image-container --media']"
+PROFILE_TITLE_SELECTOR = 'meta[property="og:title"]'
+IMAGES_PARTS = "image", "img"
+ALBUM_PARTS = "a", "album"
+VIDEO_PARTS = "video", "videos"
+DIRECT_LINK_PARTS = ("images",)
+PASSWORD_PROTECTED = "This content is password protected"
+VIDEO_SELECTOR = "meta[property='og:video']", "content"
+IMAGE_SELECTOR = "div[id=image-viewer] img", "src"
+
+
+class Media(StrEnum):
+ ALBUM = auto()
+ IMAGE = auto()
+ VIDEO = auto()
+
+
+def clean_name(url: URL) -> URL:
+ return url.with_name(url.name.replace(".md.", ".").replace(".th.", "."))
+
+
+def sort_by_new(url: URL) -> URL:
+ init_page = int(url.query.get("page") or 1)
+ return url.with_query(sort="date_desc", page=init_page)
+
+
+class CheveretoCrawler(Crawler):
+ next_page_selector = "a[data-pagination=next]"
+
+ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
+
+ @create_task_id
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ """Determines where to send the scrape item based on the url."""
+ return await self._fetch_chevereto_defaults(scrape_item)
+
+ async def _fetch_chevereto_defaults(self, scrape_item: ScrapeItem) -> None:
+ if scrape_item.url.host.count(".") > 1: # type: ignore
+ return await self.handle_direct_link(scrape_item)
+ if any(part in scrape_item.url.parts for part in ALBUM_PARTS):
+ return await self.album(scrape_item)
+ if any(part in scrape_item.url.parts for part in IMAGES_PARTS):
+ return await self.image(scrape_item)
+ if any(part in scrape_item.url.parts for part in VIDEO_PARTS):
+ return await self.video(scrape_item)
+ if any(part in scrape_item.url.parts for part in DIRECT_LINK_PARTS):
+ return await self.handle_direct_link(scrape_item)
+ await self.profile(scrape_item)
+
+ @error_handling_wrapper
+ async def profile(self, scrape_item: ScrapeItem) -> None:
+ """Scrapes an user profile."""
+ title: str = ""
+ async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):
+ if not title:
+ title: str = soup.select_one(PROFILE_TITLE_SELECTOR)["content"] # type: ignore
+ title = self.create_title(title)
+ scrape_item.setup_as_profile(title)
+
+ for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ # Item may be an image, a video or an album
+ # For images, we can download the file from the thumbnail
+ if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):
+ _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)
+ await self.handle_direct_link(new_scrape_item, thumb)
+ continue
+ # For videos and albums, we have to keep scraping
+ self.manager.task_group.create_task(self.run(new_scrape_item))
+
+ @error_handling_wrapper
+ async def album(self, scrape_item: ScrapeItem) -> None:
+ """Scrapes an album."""
+ album_id, canonical_url = self.get_canonical_url(scrape_item.url)
+ results = await self.get_album_results(album_id)
+ original_url = scrape_item.url
+ title: str = ""
+ async for soup in self.web_pager(sort_by_new(scrape_item.url), trim=False):
+ if not title:
+ await self.check_password_protected(soup, scrape_item.url)
+ title: str = soup.select_one(ALBUM_TITLE_SELECTOR).text # type: ignore
+ title = self.create_title(title, album_id)
+ scrape_item.setup_as_album(title, album_id=album_id)
+ scrape_item.url = canonical_url
+
+ for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ assert thumb
+ if self.check_album_results(thumb, results):
+ continue
+ _, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)
+ await self.handle_direct_link(new_scrape_item, thumb)
+
+ # Sub album URL needs to be the full URL + a 'sub'
+ # Using the canonical URL + 'sub' won't work because it redirects to the "homepage" of the album
+ sub_album_url = original_url / "sub"
+ async for soup in self.web_pager(sort_by_new(sub_album_url), trim=False):
+ for _, sub_album in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ self.manager.task_group.create_task(self.run(sub_album))
+
+ async def check_password_protected(self, soup: BeautifulSoup, url: URL) -> None:
+ password = url.query.get("password", "")
+ url = url.with_query(None)
+
+ if PASSWORD_PROTECTED in soup.text and password:
+ data = {"content-password": password}
+ async with self.request_limiter:
+ html = await self.client.post_data(self.domain, url, data=data, raw=True)
+ soup = BeautifulSoup(html, "html.parser")
+
+ if PASSWORD_PROTECTED in soup.text:
+ raise PasswordProtectedError(message="Wrong password" if password else None)
+
+ @error_handling_wrapper
+ async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:
+ """Handles a direct link."""
+ link = clean_name(url or scrape_item.url)
+ filename, ext = self.get_filename_and_ext(link.name)
+ await self.handle_file(link, scrape_item, filename, ext)
+
+ async def get_embed_info(self, url: URL) -> tuple[str, URL]:
+ embed_url = self.primary_base_domain / "oembed" / ""
+ embed_url = embed_url.with_query(url=str(url), format="json")
+ async with self.request_limiter:
+ json_resp: dict[str, str] = await self.client.get_json(self.domain, embed_url)
+
+ link = clean_name(self.parse_url(json_resp["url"]))
+ filename = json_resp["title"] + link.suffix
+ return filename, link
+
+ @error_handling_wrapper
+ async def _proccess_media_item(self, scrape_item: ScrapeItem, type: Media, selector: tuple[str, str]) -> None:
+ """Scrapes a media item."""
+ if await self.check_complete_from_referer(scrape_item):
+ return
+
+ _, canonical_url = self.get_canonical_url(scrape_item.url, type)
+ if await self.check_complete_from_referer(canonical_url):
+ return
+
+ async with self.request_limiter:
+ soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
+
+ try:
+ link_str: str = soup.select_one(selector[0])[selector[1]] # type: ignore
+ link = self.parse_url(link_str)
+
+ except AttributeError:
+ raise ScrapeError(422, f"Couldn't find {type} source") from None
+
+ scrape_item.possible_datetime = self.parse_date(get_date_from_soup(soup))
+ scrape_item.url = canonical_url
+ await self.handle_direct_link(scrape_item, link)
+
+ video = partialmethod(_proccess_media_item, url_type=Media.VIDEO, selector=VIDEO_SELECTOR)
+ image = partialmethod(_proccess_media_item, url_type=Media.IMAGE, selector=IMAGE_SELECTOR)
+
+ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
+
+ def get_canonical_url(self, url: URL, url_type: Media = Media.ALBUM) -> tuple[str, URL]:
+ """Returns the id and canonical URL from a given item (album, image or video)."""
+ search_parts = ALBUM_PARTS
+ if url_type == Media.IMAGE:
+ search_parts = IMAGES_PARTS
+ elif url_type == Media.VIDEO:
+ search_parts = VIDEO_PARTS
+
+ found_part = next(p for p in search_parts if p in url.parts)
+ name_index = url.parts.index(found_part) + 1
+ name = url.parts[name_index]
+ item_id = name.rsplit(".")[-1]
+ new_parts = url.parts[1:name_index] + (item_id,)
+ new_path = "/" + "/".join(new_parts)
+ return item_id, self.parse_url(new_path, url.origin())
+
+
+def get_date_from_soup(soup: BeautifulSoup) -> str:
+ for row in soup.select(ITEM_DESCRIPTION_SELECTOR):
+ if any(text in row.text.casefold() for text in ("uploaded", "added to")):
+ return row.select_one("span")["title"] # type: ignore
+ return ""
diff --git a/cyberdrop_dl/crawlers/chevereto.py b/cyberdrop_dl/crawlers/chevereto.py
deleted file mode 100644
index 730608ff6..000000000
--- a/cyberdrop_dl/crawlers/chevereto.py
+++ /dev/null
@@ -1,273 +0,0 @@
-from __future__ import annotations
-
-import calendar
-import datetime
-import re
-from typing import TYPE_CHECKING, ClassVar, Literal, TypeAlias
-
-from aiolimiter import AsyncLimiter
-from bs4 import BeautifulSoup
-from yarl import URL
-
-from cyberdrop_dl.clients.errors import PasswordProtectedError, ScrapeError
-from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
-
-if TYPE_CHECKING:
- from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
-
-CDN_PATTERNS = {
- "jpg5.su": r"^(?:https?:\/\/?)((jpg.church\/images)|(simp..jpg.church)|(jpg.fish\/images)|(simp..jpg.fish)|(jpg.fishing\/images)|(simp..jpg.fishing)|(simp..host.church)|(simp..jpg..su))(\/.*)",
- "imagepond.net": r"^(?:https?:\/\/)?(media.imagepond.net\/.*)",
- "img.kiwi": r"^(?:https?:\/\/)?img\.kiwi\/images\/.*",
-}
-
-JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")
-
-CDN_POSSIBILITIES = re.compile("|".join(CDN_PATTERNS.values()))
-JS_SELECTOR = "script[data-cfasync='false']:contains('image_viewer_full_fix')"
-JS_CONTENT_START = "document.addEventListener('DOMContentLoaded', function(event)"
-ITEM_DESCRIPTION_SELECTOR = "p[class*=description-meta]"
-
-
-UrlType: TypeAlias = Literal["album", "image", "video"]
-
-
-class CheveretoCrawler(Crawler):
- JPG5_DOMAINS: ClassVar[list[str]] = [
- "jpg5.su",
- "jpg.homes",
- "jpg.church",
- "jpg.fish",
- "jpg.fishing",
- "jpg.pet",
- "jpeg.pet",
- "jpg1.su",
- "jpg2.su",
- "jpg3.su",
- "jpg4.su",
- "host.church",
- ]
-
- PRIMARY_BASE_DOMAINS: ClassVar[dict[str, URL]] = {
- "jpg5.su": URL("https://jpg5.su"),
- "imagepond.net": URL("https://imagepond.net"),
- "img.kiwi": URL("https://img.kiwi"),
- }
-
- FOLDER_DOMAINS: ClassVar[dict[str, str]] = {
- "jpg5.su": "JPG5",
- "imagepond.net": "ImagePond",
- "img.kiwi": "ImgKiwi",
- }
-
- SUPPORTED_SITES: ClassVar[dict[str, list]] = {
- "jpg5.su": JPG5_DOMAINS,
- "imagepond.net": ["imagepond.net"],
- "img.kiwi": ["img.kiwi"],
- }
-
- def __init__(self, manager: Manager, site: str) -> None:
- super().__init__(manager, site, self.FOLDER_DOMAINS.get(site, "Chevereto"))
- self.primary_base_domain = self.PRIMARY_BASE_DOMAINS.get(site, URL(f"https://{site}")) # type: ignore
- self.next_page_selector = "a[data-pagination=next]" # type: ignore
- self.album_title_selector = "a[data-text=album-name]"
- self.item_selector = "a[class='image-container --media']"
- self.profile_title_selector = 'meta[property="og:title"]'
-
- self.images_parts = "image", "img"
- self.album_parts = "a", "album"
- self.video_parts = "video", "videos"
- self.direct_link_parts = ("images",)
- if site == "jpg5.su":
- self.request_limiter = AsyncLimiter(1, 5)
-
- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-
- @create_task_id
- async def fetch(self, scrape_item: ScrapeItem) -> None:
- """Determines where to send the scrape item based on the url."""
- if is_direct_link(scrape_item.url):
- return await self.handle_direct_link(scrape_item)
-
- scrape_item.url = scrape_item.url.with_host(self.primary_base_domain.host) # type: ignore
- if any(part in scrape_item.url.parts for part in self.album_parts):
- return await self.album(scrape_item)
- if any(part in scrape_item.url.parts for part in self.images_parts):
- return await self.image(scrape_item)
- if any(part in scrape_item.url.parts for part in self.video_parts):
- return await self.video(scrape_item)
- if any(part in scrape_item.url.parts for part in self.direct_link_parts):
- return await self.handle_direct_link(scrape_item)
- await self.profile(scrape_item)
-
- @error_handling_wrapper
- async def profile(self, scrape_item: ScrapeItem) -> None:
- """Scrapes an user profile."""
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
-
- title_tag = soup.select_one(self.profile_title_selector)
- title_text: str = title_tag.get("content") # type: ignore
- title = self.create_title(title_text)
- scrape_item.setup_as_profile(title)
-
- async for soup in self.web_pager(get_sort_by_new_url(scrape_item.url), trim=False):
- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, self.item_selector):
- # Item may be an image, a video or an album
- # For images, we can download the file from the thumbnail
- if any(p in new_scrape_item.url.parts for p in self.images_parts):
- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item, url_type="image")
- await self.handle_direct_link(new_scrape_item, thumb)
- continue
- # For videos and albums, we have to keep scraping
- self.manager.task_group.create_task(self.run(new_scrape_item))
-
- @error_handling_wrapper
- async def album(self, scrape_item: ScrapeItem) -> None:
- """Scrapes an album."""
- album_id, canonical_url = self.get_canonical_url(scrape_item)
- results = await self.get_album_results(album_id)
- password = scrape_item.url.query.get("password", "")
- scrape_item.url = scrape_item.url.with_query(None)
- original_url = scrape_item.url
-
- async with self.request_limiter:
- sub_albums = scrape_item.url / "sub"
- soup: BeautifulSoup = await self.client.get_soup(self.domain, sub_albums)
-
- scrape_item.url = canonical_url
-
- if "This content is password protected" in soup.text and password:
- data = {"content-password": password}
- async with self.request_limiter:
- html = await self.client.post_data(self.domain, scrape_item.url, data=data, raw=True)
- soup = BeautifulSoup(html, "html.parser")
-
- if "This content is password protected" in soup.text:
- raise PasswordProtectedError(message="Wrong password" if password else None)
-
- title_tag = soup.select_one(self.album_title_selector)
- title_text: str = title_tag.get_text() # type: ignore
- title = self.create_title(title_text, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
-
- async for soup in self.web_pager(get_sort_by_new_url(scrape_item.url), trim=False):
- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, self.item_selector):
- assert thumb
- if not self.check_album_results(thumb, results):
- _, new_scrape_item.url = self.get_canonical_url(new_scrape_item, url_type="image")
- await self.handle_direct_link(new_scrape_item, thumb)
-
- # Sub album URL needs to be the full URL + a 'sub'
- # Using the canonical URL + 'sub' won't work because it redirects to the "homepage" of the album
- sub_album_url = original_url / "sub"
- async for soup in self.web_pager(get_sort_by_new_url(sub_album_url), trim=False):
- for _, sub_album in self.iter_children(scrape_item, soup, self.item_selector):
- self.manager.task_group.create_task(self.run(sub_album))
-
- async def video(self, scrape_item: ScrapeItem) -> None:
- """Scrapes a video."""
- selector = "meta[property='og:video']", "content"
- await self._proccess_media_item(scrape_item, "video", selector)
-
- async def image(self, scrape_item: ScrapeItem) -> None:
- """Scrapes an image."""
- selector = "div[id=image-viewer] img", "src"
- await self._proccess_media_item(scrape_item, "image", selector)
-
- @error_handling_wrapper
- async def _proccess_media_item(self, scrape_item: ScrapeItem, url_type: UrlType, selector: tuple[str, str]) -> None:
- """Scrapes a media item."""
- if await self.check_complete_from_referer(scrape_item):
- return
-
- _, canonical_url = self.get_canonical_url(scrape_item, url_type=url_type)
- if await self.check_complete_from_referer(canonical_url):
- return
-
- if self.domain == "jpg5.su":
- _, link = await self.get_embed_info(scrape_item.url)
-
- else:
- async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.domain, scrape_item.url)
-
- try:
- link_str: str = soup.select_one(selector[0]).get(selector[1]) # type: ignore
- link = self.parse_url(link_str)
-
- except AttributeError:
- raise ScrapeError(422, f"Couldn't find {url_type} source") from None
-
- add_date_from_soup(scrape_item, soup)
-
- scrape_item.url = canonical_url
- await self.handle_direct_link(scrape_item, link)
-
- @error_handling_wrapper
- async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:
- """Handles a direct link."""
- link = url or scrape_item.url
- link = link.with_name(link.name.replace(".md.", ".").replace(".th.", "."))
- link = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))
- filename, ext = self.get_filename_and_ext(link.name)
- await self.handle_file(link, scrape_item, filename, ext)
-
- async def get_embed_info(self, url: URL) -> tuple[str, URL]:
- embed_url = self.primary_base_domain / "oembed" / ""
- embed_url = embed_url.with_query(url=str(url), format="json")
- async with self.request_limiter:
- json_resp: dict[str, str] = await self.client.get_json(self.domain, embed_url)
-
- link_str = json_resp["url"].replace(".md.", ".").replace(".th.", ".")
- link = self.parse_url(link_str)
- filename = json_resp["title"] + link.suffix
- return filename, link
-
- """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-
- def get_canonical_url(self, scrape_item: ScrapeItem, url_type: UrlType = "album") -> tuple[str, URL]:
- """Returns the id and canonical URL from a given item (album, image or video)."""
- search_parts = self.album_parts
- if url_type == "image":
- search_parts = self.images_parts
- elif url_type == "video":
- search_parts = self.video_parts
-
- found_part = next(part for part in search_parts if part in scrape_item.url.parts)
- name_index = scrape_item.url.parts.index(found_part) + 1
- name = scrape_item.url.parts[name_index]
- _id = name.rsplit(".")[-1]
- new_parts = scrape_item.url.parts[1:name_index] + (_id,)
- new_path = "/" + "/".join(new_parts)
- return _id, self.parse_url(new_path, scrape_item.url.with_path("/"))
-
-
-def get_sort_by_new_url(url: URL) -> URL:
- return url.with_query({"sort": "date_desc", "page": 1})
-
-
-def parse_datetime(date: str) -> int:
- """Parses a datetime string into a unix timestamp."""
- parsed_date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
- return calendar.timegm(parsed_date.timetuple())
-
-
-def is_direct_link(url: URL) -> bool:
- """Determines if the url is a direct link or not."""
- return bool(CDN_POSSIBILITIES.match(str(url)))
-
-
-def add_date_from_soup(scrape_item: ScrapeItem, soup: BeautifulSoup) -> None:
- desc_rows = soup.select(ITEM_DESCRIPTION_SELECTOR)
- date_str: str | None = None
- for row in desc_rows:
- if any(text in row.text.casefold() for text in ("uploaded", "added to")):
- date_str = row.select_one("span").get("title") # type: ignore
- break
-
- if date_str:
- date = parse_datetime(date_str)
- scrape_item.possible_datetime = date
diff --git a/cyberdrop_dl/crawlers/imagepond.py b/cyberdrop_dl/crawlers/imagepond.py
new file mode 100644
index 000000000..88e606106
--- /dev/null
+++ b/cyberdrop_dl/crawlers/imagepond.py
@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from yarl import URL
+
+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.managers.manager import Manager
+
+PRIMARY_BASE_DOMAIN = URL("https://imagepond.net")
+
+
+class ImagePondCrawler(CheveretoCrawler):
+ primary_base_domain = PRIMARY_BASE_DOMAIN
+
+ def __init__(self, manager: Manager) -> None:
+ super().__init__(manager, "imagepond.net", "ImagePond")
+
+ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
new file mode 100644
index 000000000..a891cf614
--- /dev/null
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from aiolimiter import AsyncLimiter
+from yarl import URL
+
+from cyberdrop_dl.crawlers.crawler import create_task_id
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+from ._chevereto import CheveretoCrawler
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.managers.manager import Manager
+ from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
+
+PRIMARY_BASE_DOMAIN = URL("https://jpg5.su")
+JPG5_REPLACE_HOST_REGEX = re.compile(r"(jpg\.fish/)|(jpg\.fishing/)|(jpg\.church/)")
+JPG5_DOMAINS = [
+ "jpg5.su",
+ "jpg.homes",
+ "jpg.church",
+ "jpg.fish",
+ "jpg.fishing",
+ "jpg.pet",
+ "jpeg.pet",
+ "jpg1.su",
+ "jpg2.su",
+ "jpg3.su",
+ "jpg4.su",
+ "host.church",
+]
+
+
+class JPG5Crawler(CheveretoCrawler):
+ primary_base_domain = PRIMARY_BASE_DOMAIN
+ SUPPORTED_SITES = {"jpg5.su": JPG5_DOMAINS} # noqa: RUF012
+
+ def __init__(self, manager: Manager, _) -> None:
+ super().__init__(manager, "jpg5.su", "JPG5")
+ self.request_limiter = AsyncLimiter(1, 5)
+
+ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
+
+ @create_task_id
+ async def fetch(self, scrape_item: ScrapeItem):
+ scrape_item.url = scrape_item.url.with_host("jpg5.su")
+ return await self._fetch_chevereto_defaults(scrape_item)
+
+ async def video(self, scrape_item: ScrapeItem) -> None:
+ """Scrapes a video."""
+ raise ValueError
+
+ @error_handling_wrapper
+ async def _proccess_media_item(self, scrape_item: ScrapeItem, url_type, *_) -> None:
+ """Scrapes a media item."""
+ if await self.check_complete_from_referer(scrape_item):
+ return
+
+ _, canonical_url = self.get_canonical_url(scrape_item.url, url_type)
+ if await self.check_complete_from_referer(canonical_url):
+ return
+
+ _, link = await self.get_embed_info(scrape_item.url)
+ scrape_item.url = canonical_url
+ await self.handle_direct_link(scrape_item, link)
+
+ async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:
+ """Handles a direct link."""
+ link = url or scrape_item.url
+ link = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))
+ super().handle_direct_link(scrape_item, link)