Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: always use impersonation (chevereto) (#1356)

Barbarella666666601db4f764dee
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 0be522f85..f4817b387 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -4,7 +4,7 @@ import base64
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature
from cyberdrop_dl.exceptions import PasswordProtectedError
from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.utilities import error_handling_wrapper, xor_decrypt
@@ -54,7 +54,8 @@ class CheveretoCrawler(Crawler, is_generic=True):
"Direct links": "",
}
NEXT_PAGE_SELECTOR = Selector.NEXT_PAGE
- CHEVERETO_SUPPORTS_VIDEO = True
+ DEFAULT_TRIM_URLS: ClassVar[bool] = False
+ CHEVERETO_SUPPORTS_VIDEO: ClassVar[bool] = True
def __init_subclass__(cls, **kwargs) -> None:
if not cls.CHEVERETO_SUPPORTS_VIDEO:
@@ -78,8 +79,15 @@ class CheveretoCrawler(Crawler, is_generic=True):
case _:
raise ValueError
+ @copy_signature(Crawler.request_soup)
+ async def request_soup(self, url: AbsoluteHttpURL, *args, impersonate: bool = False, **kwargs) -> BeautifulSoup:
+ # chevereto redirects are URL encoded and aiohttp always reencodes them by default, leading to an infinite redirect loop, so we use cURL
+ # We may be able to use aiohttp in v4
+ # See: https://github.com/jbsparrow/CyberDropDownloader/pull/1356#issuecomment-3349190328
+ return await super().request_soup(url, *args, impersonate=True, **kwargs)
+
@classmethod
- def transform_url(cls, url: AbsoluteHttpURL):
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
url = super().transform_url(url)
match url.parts[1:]:
case ["a" | "album" as part, album_slug, "sub"]:
@@ -221,4 +229,6 @@ def _thumbnail_to_src(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
def _sort_by_new(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
init_page = int(url.query.get("page") or 1)
+ if url.name:
+ url = url / ""
return url.with_query(sort="date_desc", page=init_page)
diff --git a/cyberdrop_dl/crawlers/imglike.py b/cyberdrop_dl/crawlers/imglike.py
index 32936c0f4..385283b35 100644
--- a/cyberdrop_dl/crawlers/imglike.py
+++ b/cyberdrop_dl/crawlers/imglike.py
@@ -1,13 +1,9 @@
from __future__ import annotations
-from typing import TYPE_CHECKING, ClassVar
+from typing import ClassVar
from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
-from cyberdrop_dl.crawlers.crawler import Crawler
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, copy_signature
-
-if TYPE_CHECKING:
- from bs4 import BeautifulSoup
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
PRIMARY_URL = AbsoluteHttpURL("https://imglike.com")
@@ -16,8 +12,3 @@ class ImgLikeCrawler(CheveretoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = PRIMARY_URL
DOMAIN: ClassVar[str] = "imglike.com"
FOLDER_DOMAIN: ClassVar[str] = "ImgLike"
-
- @copy_signature(Crawler.request_soup)
- async def request_soup(self, url: AbsoluteHttpURL, *args, impersonate: bool = False, **kwargs) -> BeautifulSoup:
- impersonate = impersonate or "image" in url.parts
- return await super().request_soup(url, *args, impersonate=impersonate, **kwargs)
diff --git a/tests/crawlers/test_cases/imagepond.py b/tests/crawlers/test_cases/imagepond.py
new file mode 100644
index 000000000..9f7b04ecd
--- /dev/null
+++ b/tests/crawlers/test_cases/imagepond.py
@@ -0,0 +1,13 @@
+DOMAIN = "imagepond.net"
+TEST_CASES = [
+ (
+ "https://www.imagepond.net/image/zJfHAH",
+ [
+ {
+ "url": "https://media.imagepond.net/media/Palm-Desert-Resuscitation-Education-YourCPRMD.com8c5d221f74fbb880.png",
+ "filename": "Palm-Desert-Resuscitation-Education-YourCPRMD.com8c5d221f74fbb880.png",
+ "referer": "https://www.imagepond.net/image/zJfHAH",
+ }
+ ],
+ ),
+]
diff --git a/tests/crawlers/test_cases/imglike.py b/tests/crawlers/test_cases/imglike.py
index 4cd5c3d36..ac0fca450 100644
--- a/tests/crawlers/test_cases/imglike.py
+++ b/tests/crawlers/test_cases/imglike.py
@@ -65,4 +65,19 @@ TEST_CASES = [
},
],
),
+ (
+ # Fails with aiohttp and works with curl-cffi
+ "https://imglike.com/album/D%C3%A9sir%C3%A9e-Nick-Boob-Slip-at-unesco-benefiz-gala%2C-2001.vihF",
+ [
+ {
+ "url": "https://imglike.com/images/2020/12/03/Desiree-Nick-Boob-Slip-at-unesco-benefiz-gala-2001-1.jpg",
+ "filename": "Desiree-Nick-Boob-Slip-at-unesco-benefiz-gala-2001-1.jpg",
+ "referer": "https://imglike.com/image/rq5gV",
+ "album_id": "vihF",
+ "download_folder": r"re:Désirée Nick Boob Slip at unesco-benefiz gala, 2001 \(ImgLike\)",
+ "datetime": None,
+ },
+ ],
+ 10,
+ ),
]
diff --git a/tests/crawlers/test_cases/jpg5.py b/tests/crawlers/test_cases/jpg5.py
index ca6ea2a1f..df20d9fe7 100644
--- a/tests/crawlers/test_cases/jpg5.py
+++ b/tests/crawlers/test_cases/jpg5.py
@@ -101,4 +101,14 @@ TEST_CASES = [
],
1065,
),
+ (
+ "https://jpg5.su/img/NK9wL8h",
+ [
+ {
+ "url": "https://simp6.selti-delivery.ru/images3/Palm-Desert-Resuscitation-Education-YourCPRMD.com4fc75181ec3d573e.png",
+ "filename": "Palm-Desert-Resuscitation-Education-YourCPRMD.com4fc75181ec3d573e.png",
+ "referer": "https://jpg6.su/img/NK9wL8h",
+ },
+ ],
+ ),
]