Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: handling encrypted image urls (jpg5) (#1018)
Barbarella666666693f987bb0b25
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 368b22b17..b89f9ef69 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -27,6 +27,7 @@ DIRECT_LINK_PARTS = ("images",)PASSWORD_PROTECTED = "This content is password protected"VIDEO_SELECTOR = "meta[property='og:video']", "content"IMAGE_SELECTOR = "div[id=image-viewer] img", "src"+ENCRYPTED_IMAGE_SELECTOR = "div.image-viewer-main > img"class Media(StrEnum):@@ -77,12 +78,12 @@ class CheveretoCrawler(Crawler):title = self.create_title(title)scrape_item.setup_as_profile(title)- for thumb, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):# Item may be an image, a video or an album# For images, we can download the file from the thumbnailif any(part in new_scrape_item.url.parts for part in IMAGES_PARTS):_, new_scrape_item.url = self.get_canonical_url(new_scrape_item.url, Media.IMAGE)- await self.handle_direct_link(new_scrape_item, thumb)+ await self.image(new_scrape_item)continue# For videos and albums, we have to keep scrapingself.manager.task_group.create_task(self.run(new_scrape_item))@@ -161,6 +162,8 @@ class CheveretoCrawler(Crawler):try:link_str: str = soup.select_one(selector[0])[selector[1]] # type: ignore+ if "loading.svg" in link_str:+ link_str = soup.select_one(ENCRYPTED_IMAGE_SELECTOR)["data-src"] # type: ignorelink = self.parse_url(link_str)except AttributeError:diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex 09e5b50b0..68bd74eb9 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -1,5 +1,6 @@from __future__ import annotations+import binasciiimport refrom typing import TYPE_CHECKING@@ -7,7 +8,6 @@ from aiolimiter import AsyncLimiterfrom yarl import URLfrom cyberdrop_dl.crawlers.crawler import create_task_id-from cyberdrop_dl.utils.utilities import error_handling_wrapperfrom ._chevereto import CheveretoCrawler@@ -32,6 +32,8 @@ JPG5_DOMAINS = ["host.church",]+DECRYPTION_KEY = b"seltilovessimpcity@simpcityhatesscrapers"+class JPG5Crawler(CheveretoCrawler):primary_base_domain = PRIMARY_BASE_DOMAIN@@ -52,22 +54,22 @@ class JPG5Crawler(CheveretoCrawler):"""Scrapes a video."""raise ValueError- @error_handling_wrapper- async def _proccess_media_item(self, scrape_item: ScrapeItem, url_type, *_) -> None:- """Scrapes a media item."""- if await self.check_complete_from_referer(scrape_item):- return-- _, canonical_url = self.get_canonical_url(scrape_item.url, url_type)- if await self.check_complete_from_referer(canonical_url):- return-- _, link = await self.get_embed_info(scrape_item.url)- scrape_item.url = canonical_url- await self.handle_direct_link(scrape_item, link)+ def parse_url(self, link_str: str, relative_to: URL | None = None, *, trim: bool = True) -> URL:+ if not link_str.startswith("https") and not link_str.startswith("/"):+ link_str = decrypt_xor(link_str, DECRYPTION_KEY)+ return super().parse_url(link_str, relative_to, trim=trim)async def handle_direct_link(self, scrape_item: ScrapeItem, url: URL | None = None) -> None:"""Handles a direct link."""link = url or scrape_item.urllink = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))await super().handle_direct_link(scrape_item, link)+++"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""+++def decrypt_xor(encrypted, key):+ div = len(key)+ encrypted = bytes.fromhex(binascii.a2b_base64(encrypted).decode())+ return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()