Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add buzzheavier support (#942)
NTFSvolumec7d0df54d0ae
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex 4c24d4ffc..d7d77c8a7 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -14,8 +14,8 @@ from pathlib import Pathfrom typing import TYPE_CHECKING, Any, ParamSpec, TypeVarimport aiohttp-from aiohttp_client_cache import CachedSessionfrom aiohttp_client_cache.response import AnyResponse+from aiohttp_client_cache.session import CachedSessionfrom bs4 import BeautifulSoupimport cyberdrop_dl.utils.constants as constants@@ -337,7 +337,13 @@ class ScraperClient:@limiterasync def get_head(- self, domain: str, url: URL, headers: dict | None = None, *, cache_disabled: bool = False, **request_params: Any+ self,+ domain: str,+ url: URL,+ headers: dict[str, str] | None = None,+ *,+ cache_disabled: bool = False,+ **request_params: Any,) -> CIMultiDictProxy[str]:"""Makes a GET request to an URL and returns its headersdiff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex a98ef630d..93eb57308 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -11,6 +11,7 @@ from .bestprettygirl import BestPrettyGirlCrawlerfrom .box_dot_com import BoxDotComCrawlerfrom .bunkrr import BunkrrCrawlerfrom .bunkrr_albums_io import BunkrAlbumsIOCrawler+from .buzzheavier import BuzzHeavierCrawlerfrom .catbox import CatboxCrawlerfrom .coomer import CoomerCrawlerfrom .crawler import Crawlerdiff --git a/cyberdrop_dl/crawlers/buzzheavier.py b/cyberdrop_dl/crawlers/buzzheavier.pynew file mode 100644index 000000000..18e6e63f4--- /dev/null+++ b/cyberdrop_dl/crawlers/buzzheavier.py@@ -0,0 +1,45 @@+from __future__ import annotations++from typing import TYPE_CHECKING++from yarl import URL++from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headers++if TYPE_CHECKING:+ from cyberdrop_dl.managers.manager import Manager+ from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem+++class BuzzHeavierCrawler(Crawler):+ primary_base_domain = URL("https://buzzheavier.com")++ def __init__(self, manager: Manager) -> None:+ super().__init__(manager, "buzzheavier.com", "BuzzHeavier")++ """~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""++ @create_task_id+ async def fetch(self, scrape_item: ScrapeItem) -> None:+ """Determines where to send the scrape item based on the url."""+ return await self.file(scrape_item)++ @error_handling_wrapper+ async def file(self, scrape_item: ScrapeItem) -> None:+ """Scrapes a file."""++ if await self.check_complete_from_referer(scrape_item):+ return++ url = scrape_item.url / "download"+ headers = {"HX-Current-URL": str(scrape_item.url), "HX-Request": "true"}+ async with self.request_limiter:+ headers = await self.client.get_head(self.domain, url, headers=headers)+ redirect = headers["hx-redirect"]+ filename = get_filename_from_headers(headers)++ assert filename+ link = self.parse_url(redirect)+ filename, ext = self.get_filename_and_ext(filename, assume_ext=".zip")+ await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=link)