Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: get a good host before following redirects (bunkr) (#1731)

NTFSvolume8d39b9fd384d
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 883d1d79c..752e4bf68 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import base64
import dataclasses
import json
@@ -8,6 +9,7 @@ from collections.abc import Generator
from typing import TYPE_CHECKING, Any, ClassVar
from aiohttp import ClientConnectorError
+from typing_extensions import override
from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths, auto_task_id
@@ -114,6 +116,7 @@ class BunkrrCrawler(Crawler):
_known_good_host: ClassVar[str | None] = None
def __post_init__(self) -> None:
+ self._redirect_lock: asyncio.Lock = asyncio.Lock()
self._parse_album_files = _make_album_parser()
async def fetch(self, scrape_item: ScrapeItem) -> None:
@@ -137,6 +140,15 @@ class BunkrrCrawler(Crawler):
case _:
raise ValueError
+ @override
+ async def _get_redirect_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ if not self._known_good_host:
+ async with self._redirect_lock:
+ if not self._known_good_host:
+ _ = await self._request_soup_lenient(url)
+ assert self._known_good_host
+ return await super()._get_redirect_url(url.with_host(self._known_good_host))
+
@error_handling_wrapper
async def album(self, scrape_item: ScrapeItem, album_id: str) -> None:
soup = await self._request_soup_lenient(scrape_item.url.with_query(advanced=1))
diff --git a/tests/crawlers/test_cases/bunkr.py b/tests/crawlers/test_cases/bunkr.py
index 94f0ab023..35f4f9a75 100644
--- a/tests/crawlers/test_cases/bunkr.py
+++ b/tests/crawlers/test_cases/bunkr.py
@@ -1,5 +1,17 @@
DOMAIN = "bunkr"
TEST_CASES = [
+ (
+ "https://bunkr.black/v/bla4XTB1v8aG6",
+ [
+ {
+ "url": "re:/AZ-Haul-119-tqLy8ZWy.mp4?n=AZ+Haul+119.mp4",
+ "filename": "AZ Haul 119.mp4",
+ "original_filename": "AZ Haul 119.mp4",
+ "referer": "https://bunkr.site/f/bla4XTB1v8aG6",
+ }
+ ],
+ 1,
+ ),
(
"https://bunkr.ax/v/rFicV4QnhSHBE",
[