Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: get a good host before following redirects (bunkr) (#1731)
NTFSvolume8d39b9fd384d
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 883d1d79c..752e4bf68 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport base64import dataclassesimport json@@ -8,6 +9,7 @@ from collections.abc import Generatorfrom typing import TYPE_CHECKING, Any, ClassVarfrom aiohttp import ClientConnectorError+from typing_extensions import overridefrom cyberdrop_dl.constants import FileExtfrom cyberdrop_dl.crawlers.crawler import Crawler, RateLimit, SupportedPaths, auto_task_id@@ -114,6 +116,7 @@ class BunkrrCrawler(Crawler):_known_good_host: ClassVar[str | None] = Nonedef __post_init__(self) -> None:+ self._redirect_lock: asyncio.Lock = asyncio.Lock()self._parse_album_files = _make_album_parser()async def fetch(self, scrape_item: ScrapeItem) -> None:@@ -137,6 +140,15 @@ class BunkrrCrawler(Crawler):case _:raise ValueError+ @override+ async def _get_redirect_url(self, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ if not self._known_good_host:+ async with self._redirect_lock:+ if not self._known_good_host:+ _ = await self._request_soup_lenient(url)+ assert self._known_good_host+ return await super()._get_redirect_url(url.with_host(self._known_good_host))+@error_handling_wrapperasync def album(self, scrape_item: ScrapeItem, album_id: str) -> None:soup = await self._request_soup_lenient(scrape_item.url.with_query(advanced=1))diff --git a/tests/crawlers/test_cases/bunkr.py b/tests/crawlers/test_cases/bunkr.pyindex 94f0ab023..35f4f9a75 100644--- a/tests/crawlers/test_cases/bunkr.py+++ b/tests/crawlers/test_cases/bunkr.py@@ -1,5 +1,17 @@DOMAIN = "bunkr"TEST_CASES = [+ (+ "https://bunkr.black/v/bla4XTB1v8aG6",+ [+ {+ "url": "re:/AZ-Haul-119-tqLy8ZWy.mp4?n=AZ+Haul+119.mp4",+ "filename": "AZ Haul 119.mp4",+ "original_filename": "AZ Haul 119.mp4",+ "referer": "https://bunkr.site/f/bla4XTB1v8aG6",+ }+ ],+ 1,+ ),("https://bunkr.ax/v/rFicV4QnhSHBE",[