Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: always use impersonation (chevereto) (#1356)
Barbarella666666601db4f764dee
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 0be522f85..f4817b387 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -4,7 +4,7 @@ import base64from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signaturefrom cyberdrop_dl.exceptions import PasswordProtectedErrorfrom cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, xor_decrypt@@ -54,7 +54,8 @@ class CheveretoCrawler(Crawler, is_generic=True):"Direct links": "",}NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE- CHEVERETO_SUPPORTS_VIDEO = True+ DEFAULT_TRIM_URLS: ClassVar[bool] = False+ CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = Truedef __init_subclass__(cls, **kwargs) -> None:if not cls.CHEVERETO_SUPPORTS_VIDEO:@@ -78,8 +79,15 @@ class CheveretoCrawler(Crawler, is_generic=True):case _:raise ValueError+ @copy_signature(Crawler.request_soup)+ async def request_soup(self, url: AbsoluteHttpURL, *args, impersonate: bool = False, **kwargs) -> BeautifulSoup:+ # chevereto redirects are URL encoded and aiohttp always reencodes them by default, leading to an infinite redirect loop, so we use cURL+ # We may be able to use aiohttp in v4+ # See: https://github.com/jbsparrow/CyberDropDownloader/pull/1356#issuecomment-3349190328+ return await super().request_soup(url, *args, impersonate=True, **kwargs)+@classmethod- def transform_url(cls, url: AbsoluteHttpURL):+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:url = super().transform_url(url)match url.parts[1:]:case ["a" | "album" as part, album_slug, "sub"]:@@ -221,4 +229,6 @@ def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:def _sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:init_page = int(url.query.get("page") or 1)+ if url.name:+ url = url / ""return url.with_query(sort="date_desc", page=init_page)diff --git a/cyberdrop_dl/crawlers/imglike.py b/cyberdrop_dl/crawlers/imglike.pyindex 32936c0f4..385283b35 100644--- a/cyberdrop_dl/crawlers/imglike.py+++ b/cyberdrop_dl/crawlers/imglike.py@@ -1,13 +1,9 @@from __future__ import annotations-from typing import TYPE_CHECKING, ClassVar+from typing import ClassVarfrom cyberdrop_dl.crawlers._chevereto import CheveretoCrawler-from cyberdrop_dl.crawlers.crawler import Crawler-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature--if TYPE_CHECKING:- from bs4 import BeautifulSoup+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLPRIMARY_URL = AbsoluteHttpURL("https://imglike.com")@@ -16,8 +12,3 @@ class ImgLikeCrawler(CheveretoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URLDOMAIN: ClassVar[str] = "imglike.com"FOLDER_DOMAIN: ClassVar[str] = "ImgLike"-- @copy_signature(Crawler.request_soup)- async def request_soup(self, url: AbsoluteHttpURL, *args, impersonate: bool = False, **kwargs) -> BeautifulSoup:- impersonate = impersonate or "image" in url.parts- return await super().request_soup(url, *args, impersonate=impersonate, **kwargs)diff --git a/tests/crawlers/test_cases/imagepond.py b/tests/crawlers/test_cases/imagepond.pynew file mode 100644index 000000000..9f7b04ecd--- /dev/null+++ b/tests/crawlers/test_cases/imagepond.py@@ -0,0 +1,13 @@+DOMAIN = "imagepond.net"+TEST_CASES = [+ (+ "https://www.imagepond.net/image/zJfHAH",+ [+ {+ "url": "https://media.imagepond.net/media/Palm-Desert-Resuscitation-Education-YourCPRMD.com8c5d221f74fbb880.png",+ "filename": "Palm-Desert-Resuscitation-Education-YourCPRMD.com8c5d221f74fbb880.png",+ "referer": "https://www.imagepond.net/image/zJfHAH",+ }+ ],+ ),+]diff --git a/tests/crawlers/test_cases/imglike.py b/tests/crawlers/test_cases/imglike.pyindex 4cd5c3d36..ac0fca450 100644--- a/tests/crawlers/test_cases/imglike.py+++ b/tests/crawlers/test_cases/imglike.py@@ -65,4 +65,19 @@ TEST_CASES = [},],),+ (+ # Fails with aiohttp and works with curl-cffi+ "https://imglike.com/album/D%C3%A9sir%C3%A9e-Nick-Boob-Slip-at-unesco-benefiz-gala%2C-2001.vihF",+ [+ {+ "url": "https://imglike.com/images/2020/12/03/Desiree-Nick-Boob-Slip-at-unesco-benefiz-gala-2001-1.jpg",+ "filename": "Desiree-Nick-Boob-Slip-at-unesco-benefiz-gala-2001-1.jpg",+ "referer": "https://imglike.com/image/rq5gV",+ "album_id": "vihF",+ "download_folder": r"re:Désirée Nick Boob Slip at unesco-benefiz gala, 2001 \(ImgLike\)",+ "datetime": None,+ },+ ],+ 10,+ ),]diff --git a/tests/crawlers/test_cases/jpg5.py b/tests/crawlers/test_cases/jpg5.pyindex ca6ea2a1f..df20d9fe7 100644--- a/tests/crawlers/test_cases/jpg5.py+++ b/tests/crawlers/test_cases/jpg5.py@@ -101,4 +101,14 @@ TEST_CASES = [],1065,),+ (+ "https://jpg5.su/img/NK9wL8h",+ [+ {+ "url": "https://simp6.selti-delivery.ru/images3/Palm-Desert-Resuscitation-Education-YourCPRMD.com4fc75181ec3d573e.png",+ "filename": "Palm-Desert-Resuscitation-Education-YourCPRMD.com4fc75181ec3d573e.png",+ "referer": "https://jpg6.su/img/NK9wL8h",+ },+ ],+ ),]