Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: handle paginated albums (bunkr) (#1269)
NTFSvolume62a3f701bb29
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 22263b489..06a95c5f4 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport base64+import itertoolsimport jsonimport mathimport re@@ -9,18 +10,20 @@ from collections import defaultdictfrom dataclasses import dataclassfrom itertools import cyclefrom pathlib import Path-from typing import TYPE_CHECKING, ClassVar, NamedTuple+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuplefrom aiohttp import ClientConnectorErrorfrom cyberdrop_dl.constants import FILE_FORMATS-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_idfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import DDOSGuardError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url, with_suffix_encodedif TYPE_CHECKING:+ from collections.abc import AsyncGenerator+from bs4 import BeautifulSoup, Tagfrom cyberdrop_dl.data_structures.url_objects import ScrapeItem@@ -72,6 +75,7 @@ class Selectors:IMAGE_PREVIEW = "img.max-h-full.w-auto.object-cover.relative"VIDEO = "video > source"JS_SLUG = "script:contains('jsSlug')"+ NEXT_PAGE = "nav.pagination a[href]:contains('»')"_SELECTORS = Selectors()@@ -148,27 +152,47 @@ class BunkrrCrawler(Crawler):@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem) -> None:- soup: BeautifulSoup = await self.get_soup_lenient(scrape_item.url)album_id = scrape_item.url.parts[2]- title = css.select_one_get_text(soup, "title").rsplit(" | Bunkr")[0].strip()- title = self.create_title(title, album_id)- scrape_item.setup_as_album(title, album_id=album_id)+ title: str = ""results = await self.get_album_results(album_id)- for tag in soup.select(_SELECTORS.ALBUM_ITEM):- item = AlbumItem.from_tag(tag)- link = self.parse_url(item.path_qs, relative_to=scrape_item.url.origin())- new_scrape_item = scrape_item.create_child(- link, possible_datetime=self.parse_date(item.date, "%H:%M:%S %d/%m/%Y")- )- await self.process_album_item(new_scrape_item, item, results)- scrape_item.add_children()+ async for soup in self.web_pager(scrape_item.url):+ if not title:+ title = css.page_title(soup, "bunkr")+ title = self.create_title(title, album_id)+ scrape_item.setup_as_album(title, album_id=album_id)++ for tag in soup.select(_SELECTORS.ALBUM_ITEM):+ item = AlbumItem.from_tag(tag)+ link = self.parse_url(item.path_qs, relative_to=scrape_item.url.origin())+ new_scrape_item = scrape_item.create_child(+ link, possible_datetime=self.parse_date(item.date, "%H:%M:%S %d/%m/%Y")+ )+ self.create_task(self._process_album_item_task(new_scrape_item, item, results))+ scrape_item.add_children()++ async def web_pager(+ self,+ url: AbsoluteHttpURL,+ next_page_selector: str | None = None,+ *,+ cffi: bool = False,+ **kwargs: Any,+ ) -> AsyncGenerator[BeautifulSoup]:+ page_url = url+ init_page = int(url.query.get("page") or 1)+ for page in itertools.count(init_page):+ soup = await self.get_soup_lenient(page_url.with_query(page=page))+ yield soup+ has_next_page = soup.select_one(_SELECTORS.NEXT_PAGE)+ if not has_next_page:+ break@error_handling_wrapper- async def process_album_item(self, scrape_item: ScrapeItem, item: AlbumItem, results: dict) -> None:+ async def _process_album_item(self, scrape_item: ScrapeItem, item: AlbumItem, results: dict) -> None:link = item.srcif link.suffix.lower() not in VIDEO_AND_IMAGE_EXTS or "no-image" in link.name or self.deep_scrape(link):- self.manager.task_group.create_task(self.run(scrape_item))+ self.create_task(self.run(scrape_item))returnif self.check_album_results(link, results):@@ -186,6 +210,8 @@ class BunkrrCrawler(Crawler):await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)+ _process_album_item_task = auto_task_id(_process_album_item)+@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem) -> None:link: AbsoluteHttpURL | None = Nonediff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 66df8cb9c..c9a3e0f2b 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -125,7 +125,7 @@ def decompose(tag: Tag, selector: str) -> None:def sanitize_page_title(title: str, domain: str) -> str:- sld = domain.rsplit(".", 1)[0]+ sld = domain.rsplit(".", 1)[0].casefold()def clean(string: str, char: str):if char in string:diff --git a/tests/crawlers/test_cases/bunkr.py b/tests/crawlers/test_cases/bunkr.pynew file mode 100644index 000000000..2378c9dd1--- /dev/null+++ b/tests/crawlers/test_cases/bunkr.py@@ -0,0 +1,34 @@+DOMAIN = "bunkrr"+TEST_CASES = [+ (+ # paginated+ "https://bunkr.cr/a/5aZU25Cb",+ [+ {+ "url": "https://mlk-bk.cdn.gigachad-cdn.ru/01101506-9a96-4379-a226-c27f128923f6.jpg?n=xl_g_elin_maddie150.jpg",+ "filename": "xl_g_elin_maddie150.jpg",+ "original_filename": "01101506-9a96-4379-a226-c27f128923f6.jpg",+ "referer": "https://bunkr.site/f/1EhT44cSv6Dlq",+ "download_folder": r"re:abbywinters - Elin & Maddie \(Girl - Girl Extra Large\)",+ "album_id": "5aZU25Cb",+ "datetime": 1755391253,+ }+ ],+ 258,+ ),+ (+ "https://bunkr.cr/a/TQAgjP8m",+ [+ {+ "url": "re:2020-01-09---Fake-Porn-Star-Orders-Pizza-NAKED-Prank----6dh068JH.mp4",+ "filename": "2020-01-09 - Fake Porn Star Orders Pizza NAKED Prank 🍕.mp4",+ "original_filename": "2020-01-09---Fake-Porn-Star-Orders-Pizza-NAKED-Prank----6dh068JH.mp4",+ "referer": "https://bunkr.site/f/UAvbs2GPhWOGc",+ "download_folder": r"re:NerdballerTV - Videos \(2018-2023\) \[Complete\]",+ "album_id": "TQAgjP8m",+ "datetime": 1709776012,+ }+ ],+ 220,+ ),+]