Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: always use impersonation (Celebforum) (#1623)
NTFSvolumed6ab271ec8cd
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 3bb2306d5..f2cf7c0b5 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -39,7 +39,7 @@ _CHROME_ANDROID_USER_AGENT: str = ()_FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()-_USE_IMPERSONATION: set[str] = {"vsco"}+_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}class DownloadClient:diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 52737a2a3..307dd0d4f 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -125,13 +125,20 @@ class Crawler(ABC):_RATE_LIMIT: ClassVar[RateLimit] = 25, 1_DOWNLOAD_SLOTS: ClassVar[int | None] = None_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False+ _IMPERSONATE: ClassVar[str | bool | None] = Nonecreate_db_path = staticmethod(DBPathBuilder.path)@copy_signature(ScraperClient._request)@contextlib.asynccontextmanager- async def request(self, *args, **kwargs) -> AsyncGenerator[AbstractResponse]:- async with self.client._limiter(self.DOMAIN), self.client._request(*args, **kwargs) as resp:+ async def request(self, *args, impersonate: str | bool | None = None, **kwargs) -> AsyncGenerator[AbstractResponse]:+ if impersonate is None:+ impersonate = self._IMPERSONATE++ async with (+ self.client._limiter(self.DOMAIN),+ self.client._request(*args, impersonate=impersonate, **kwargs) as resp,+ ):yield resp@copy_signature(ScraperClient._request)diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.pyindex f7b47b374..8a2c1a0be 100644--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py@@ -1,17 +1,22 @@from __future__ import annotations-from typing import ClassVar+from typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom .xenforo import XenforoCrawler+if TYPE_CHECKING:+ from cyberdrop_dl.crawlers.crawler import RateLimit+class CelebForumCrawler(XenforoCrawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://celebforum.to")DOMAIN: ClassVar[str] = "celebforum"FOLDER_DOMAIN: ClassVar[str] = "CelebForum"IGNORE_EMBEDED_IMAGES_SRC: ClassVar = True # images src is always a thumbnail+ _IMPERSONATE: ClassVar[str | bool | None] = True+ _RATE_LIMIT: ClassVar[RateLimit] = 3, 10@classmethoddef is_thumbnail(cls, link: AbsoluteHttpURL) -> bool: