Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: handling encrypted image urls (jpg5) (#1018)

Barbarella666666693f987bb0b25
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 368b22b17..b89f9ef69 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -27,6 +27,7 @@ DIRECT_LINK_PARTS = ("images",)
PASSWORD_PROTECTED = "This content is password protected"
VIDEO_SELECTOR = "meta[property='og:video']", "content"
IMAGE_SELECTOR = "div[id=image-viewer] img", "src"
+ENCRYPTED_IMAGE_SELECTOR = "div.image-viewer-main > img"
class Media(StrEnum):
@@ -77,12 +78,12 @@ class CheveretoCrawler(Crawler):
title = self.create_title(title)
scrape_item.setup_as_profile(title)
- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
# Item may be an image, a video or an album
# For images, we can download the file from the thumbnail
if any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):
_, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)
- await self.handle_direct_link(new_scrape_item, thumb)
+ await self.image(new_scrape_item)
continue
# For videos and albums, we have to keep scraping
self.manager.task_group.create_task(self.run(new_scrape_item))
@@ -161,6 +162,8 @@ class CheveretoCrawler(Crawler):
try:
link_str: str = soup.select_one(selector[0])[selector[1]] # type: ignore
+ if "loading.svg" in link_str:
+ link_str = soup.select_one(ENCRYPTED_IMAGE_SELECTOR)["data-src"] # type: ignore
link = self.parse_url(link_str)
except AttributeError:
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index 09e5b50b0..68bd74eb9 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import binascii
import re
from typing import TYPE_CHECKING
@@ -7,7 +8,6 @@ from aiolimiter import AsyncLimiter
from yarl import URL
from cyberdrop_dl.crawlers.crawler import create_task_id
-from cyberdrop_dl.utils.utilities import error_handling_wrapper
from ._chevereto import CheveretoCrawler
@@ -32,6 +32,8 @@ JPG5_DOMAINS = [
"host.church",
]
+DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"
+
class JPG5Crawler(CheveretoCrawler):
primary_base_domain = PRIMARY_BASE_DOMAIN
@@ -52,22 +54,22 @@ class JPG5Crawler(CheveretoCrawler):
"""Scrapes a video."""
raise ValueError
- @error_handling_wrapper
- async def _proccess_media_item(self, scrape_item: ScrapeItem, url_type, *_) -> None:
- """Scrapes a media item."""
- if await self.check_complete_from_referer(scrape_item):
- return
-
- _, canonical_url = self.get_canonical_url(scrape_item.url, url_type)
- if await self.check_complete_from_referer(canonical_url):
- return
-
- _, link = await self.get_embed_info(scrape_item.url)
- scrape_item.url = canonical_url
- await self.handle_direct_link(scrape_item, link)
+ def parse_url(self, link_str: str, relative_to: URL | None = None, *, trim: bool = True) -> URL:
+ if not link_str.startswith("https") and not link_str.startswith("/"):
+ link_str = decrypt_xor(link_str, DECRYPTION_KEY)
+ return super().parse_url(link_str, relative_to, trim=trim)
async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:
"""Handles a direct link."""
link = url or scrape_item.url
link = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))
await super().handle_direct_link(scrape_item, link)
+
+
+"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
+
+
+def decrypt_xor(encrypted, key):
+ div = len(key)
+ encrypted = bytes.fromhex(binascii.a2b_base64(encrypted).decode())
+ return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()