Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: scraping user albums (chevereto) (#1357)

Barbarella6666666cd4edfedfb4d
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index f4817b387..3f1bf8f2e 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
class Selector:
ITEM_DESCRIPTION = "p[class*=description-meta]"
- ITEM = "a[class='image-container --media']"
+ ITEM = "a.image-container"
NEXT_PAGE = "a[data-pagination=next]"
DATE_SINGLE_ITEM = f"{ITEM_DESCRIPTION}:-soup-contains('Uploaded') span"
@@ -74,6 +74,8 @@ class CheveretoCrawler(Crawler, is_generic=True):
return await self.media(scrape_item)
case ["images", _, *_]:
return await self.direct_file(scrape_item)
+ case [_, "albums"]:
+ return await self.profile(scrape_item)
case [_]:
return await self.profile(scrape_item)
case _:
@@ -148,17 +150,18 @@ class CheveretoCrawler(Crawler, is_generic=True):
self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None
) -> None:
for thumb, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):
- assert thumb
- source = _thumbnail_to_src(thumb)
- if results and self.check_album_results(source, results):
- continue
-
- # for images, we can download the file from the thumbnail, skipping an additional request per img
- # cons: we won't get the upload date
if image_url := _match_img(new_scrape_item.url):
new_scrape_item.url = image_url
- self.create_task(self.direct_file(new_scrape_item, source))
- continue
+
+ if thumb:
+ # for images, we can download the file from the thumbnail, skipping an additional request per img
+ # cons: we won't get the upload date
+ source = _thumbnail_to_src(thumb)
+ if results and self.check_album_results(source, results):
+ continue
+
+ self.create_task(self.direct_file(new_scrape_item, source))
+ continue
self.create_task(self.run(new_scrape_item))
diff --git a/tests/crawlers/test_cases/imglike.py b/tests/crawlers/test_cases/imglike.py
index ac0fca450..064a2fa53 100644
--- a/tests/crawlers/test_cases/imglike.py
+++ b/tests/crawlers/test_cases/imglike.py
@@ -65,6 +65,17 @@ TEST_CASES = [
},
],
),
+ (
+ "https://imglike.com/bonton18/albums",
+ [
+ {
+ "url": "https://imglike.com/images/2022/10/03/01__4_.jpg",
+ "filename": "01__4_.jpg",
+ "referer": "https://imglike.com/image/L1iUm",
+ "album_id": "iNlR",
+ }
+ ],
+ ),
(
# Fails with aiohttp and works with curl-cffi
"https://imglike.com/album/D%C3%A9sir%C3%A9e-Nick-Boob-Slip-at-unesco-benefiz-gala%2C-2001.vihF",