Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: scraping user albums (chevereto) (#1357)
Barbarella6666666cd4edfedfb4d
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex f4817b387..3f1bf8f2e 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -17,7 +17,7 @@ if TYPE_CHECKING:class Selector:ITEM_DESCRIPTION = "p[class*=description-meta]"- ITEM = "a[class='image-container --media']"+ ITEM = "a.image-container"NEXT_PAGE = "a[data-pagination=next]"DATE_SINGLE_ITEM = f"{ITEM_DESCRIPTION}:-soup-contains('Uploaded') span"@@ -74,6 +74,8 @@ class CheveretoCrawler(Crawler, is_generic=True):return await self.media(scrape_item)case ["images", _, *_]:return await self.direct_file(scrape_item)+ case [_, "albums"]:+ return await self.profile(scrape_item)case [_]:return await self.profile(scrape_item)case _:@@ -148,17 +150,18 @@ class CheveretoCrawler(Crawler, is_generic=True):self, scrape_item: ScrapeItem, soup: BeautifulSoup, results: dict[str, int] | None = None) -> None:for thumb, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ITEM):- assert thumb- source = _thumbnail_to_src(thumb)- if results and self.check_album_results(source, results):- continue-- # for images, we can download the file from the thumbnail, skipping an additional request per img- # cons: we won't get the upload dateif image_url := _match_img(new_scrape_item.url):new_scrape_item.url = image_url- self.create_task(self.direct_file(new_scrape_item, source))- continue++ if thumb:+ # for images, we can download the file from the thumbnail, skipping an additional request per img+ # cons: we won't get the upload date+ source = _thumbnail_to_src(thumb)+ if results and self.check_album_results(source, results):+ continue++ self.create_task(self.direct_file(new_scrape_item, source))+ continueself.create_task(self.run(new_scrape_item))diff --git a/tests/crawlers/test_cases/imglike.py b/tests/crawlers/test_cases/imglike.pyindex ac0fca450..064a2fa53 100644--- a/tests/crawlers/test_cases/imglike.py+++ b/tests/crawlers/test_cases/imglike.py@@ -65,6 +65,17 @@ TEST_CASES = [},],),+ (+ "https://imglike.com/bonton18/albums",+ [+ {+ "url": "https://imglike.com/images/2022/10/03/01__4_.jpg",+ "filename": "01__4_.jpg",+ "referer": "https://imglike.com/image/L1iUm",+ "album_id": "iNlR",+ }+ ],+ ),(# Fails with aiohttp and works with curl-cffi"https://imglike.com/album/D%C3%A9sir%C3%A9e-Nick-Boob-Slip-at-unesco-benefiz-gala%2C-2001.vihF",