Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: handle paginated albums (bunkr) (#1269)

NTFSvolume62a3f701bb29
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 22263b489..06a95c5f4 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import base64
+import itertools
import json
import math
import re
@@ -9,18 +10,20 @@ from collections import defaultdict
from dataclasses import dataclass
from itertools import cycle
from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, NamedTuple
+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple
from aiohttp import ClientConnectorError
from cyberdrop_dl.constants import FILE_FORMATS
-from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_id
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import DDOSGuardError, NoExtensionError, ScrapeError
from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_text_between, parse_url, with_suffix_encoded
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
+
from bs4 import BeautifulSoup, Tag
from cyberdrop_dl.data_structures.url_objects import ScrapeItem
@@ -72,6 +75,7 @@ class Selectors:
IMAGE_PREVIEW = "img.max-h-full.w-auto.object-cover.relative"
VIDEO = "video > source"
JS_SLUG = "script:contains('jsSlug')"
+ NEXT_PAGE = "nav.pagination a[href]:contains('»')"
_SELECTORS = Selectors()
@@ -148,27 +152,47 @@ class BunkrrCrawler(Crawler):
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem) -> None:
- soup: BeautifulSoup = await self.get_soup_lenient(scrape_item.url)
album_id = scrape_item.url.parts[2]
- title = css.select_one_get_text(soup, "title").rsplit(" | Bunkr")[0].strip()
- title = self.create_title(title, album_id)
- scrape_item.setup_as_album(title, album_id=album_id)
+ title: str = ""
results = await self.get_album_results(album_id)
- for tag in soup.select(_SELECTORS.ALBUM_ITEM):
- item = AlbumItem.from_tag(tag)
- link = self.parse_url(item.path_qs, relative_to=scrape_item.url.origin())
- new_scrape_item = scrape_item.create_child(
- link, possible_datetime=self.parse_date(item.date, "%H:%M:%S %d/%m/%Y")
- )
- await self.process_album_item(new_scrape_item, item, results)
- scrape_item.add_children()
+ async for soup in self.web_pager(scrape_item.url):
+ if not title:
+ title = css.page_title(soup, "bunkr")
+ title = self.create_title(title, album_id)
+ scrape_item.setup_as_album(title, album_id=album_id)
+
+ for tag in soup.select(_SELECTORS.ALBUM_ITEM):
+ item = AlbumItem.from_tag(tag)
+ link = self.parse_url(item.path_qs, relative_to=scrape_item.url.origin())
+ new_scrape_item = scrape_item.create_child(
+ link, possible_datetime=self.parse_date(item.date, "%H:%M:%S %d/%m/%Y")
+ )
+ self.create_task(self._process_album_item_task(new_scrape_item, item, results))
+ scrape_item.add_children()
+
+ async def web_pager(
+ self,
+ url: AbsoluteHttpURL,
+ next_page_selector: str | None = None,
+ *,
+ cffi: bool = False,
+ **kwargs: Any,
+ ) -> AsyncGenerator[BeautifulSoup]:
+ page_url = url
+ init_page = int(url.query.get("page") or 1)
+ for page in itertools.count(init_page):
+ soup = await self.get_soup_lenient(page_url.with_query(page=page))
+ yield soup
+ has_next_page = soup.select_one(_SELECTORS.NEXT_PAGE)
+ if not has_next_page:
+ break
@error_handling_wrapper
- async def process_album_item(self, scrape_item: ScrapeItem, item: AlbumItem, results: dict) -> None:
+ async def _process_album_item(self, scrape_item: ScrapeItem, item: AlbumItem, results: dict) -> None:
link = item.src
if link.suffix.lower() not in VIDEO_AND_IMAGE_EXTS or "no-image" in link.name or self.deep_scrape(link):
- self.manager.task_group.create_task(self.run(scrape_item))
+ self.create_task(self.run(scrape_item))
return
if self.check_album_results(link, results):
@@ -186,6 +210,8 @@ class BunkrrCrawler(Crawler):
await self.handle_file(link, scrape_item, filename, ext, custom_filename=custom_filename)
+ _process_album_item_task = auto_task_id(_process_album_item)
+
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem) -> None:
link: AbsoluteHttpURL | None = None
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 66df8cb9c..c9a3e0f2b 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -125,7 +125,7 @@ def decompose(tag: Tag, selector: str) -> None:
def sanitize_page_title(title: str, domain: str) -> str:
- sld = domain.rsplit(".", 1)[0]
+ sld = domain.rsplit(".", 1)[0].casefold()
def clean(string: str, char: str):
if char in string:
diff --git a/tests/crawlers/test_cases/bunkr.py b/tests/crawlers/test_cases/bunkr.py
new file mode 100644
index 000000000..2378c9dd1
--- /dev/null
+++ b/tests/crawlers/test_cases/bunkr.py
@@ -0,0 +1,34 @@
+DOMAIN = "bunkrr"
+TEST_CASES = [
+ (
+ # paginated
+ "https://bunkr.cr/a/5aZU25Cb",
+ [
+ {
+ "url": "https://mlk-bk.cdn.gigachad-cdn.ru/01101506-9a96-4379-a226-c27f128923f6.jpg?n=xl_g_elin_maddie150.jpg",
+ "filename": "xl_g_elin_maddie150.jpg",
+ "original_filename": "01101506-9a96-4379-a226-c27f128923f6.jpg",
+ "referer": "https://bunkr.site/f/1EhT44cSv6Dlq",
+ "download_folder": r"re:abbywinters - Elin & Maddie \(Girl - Girl Extra Large\)",
+ "album_id": "5aZU25Cb",
+ "datetime": 1755391253,
+ }
+ ],
+ 258,
+ ),
+ (
+ "https://bunkr.cr/a/TQAgjP8m",
+ [
+ {
+ "url": "re:2020-01-09---Fake-Porn-Star-Orders-Pizza-NAKED-Prank----6dh068JH.mp4",
+ "filename": "2020-01-09 - Fake Porn Star Orders Pizza NAKED Prank 🍕.mp4",
+ "original_filename": "2020-01-09---Fake-Porn-Star-Orders-Pizza-NAKED-Prank----6dh068JH.mp4",
+ "referer": "https://bunkr.site/f/UAvbs2GPhWOGc",
+ "download_folder": r"re:NerdballerTV - Videos \(2018-2023\) \[Complete\]",
+ "album_id": "TQAgjP8m",
+ "datetime": 1709776012,
+ }
+ ],
+ 220,
+ ),
+]