Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: always use impersonation (Celebforum) (#1623)

NTFSvolumed6ab271ec8cd
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 3bb2306d5..f2cf7c0b5 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -39,7 +39,7 @@ _CHROME_ANDROID_USER_AGENT: str = (
)
_FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks
_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()
-_USE_IMPERSONATION: set[str] = {"vsco"}
+_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}
class DownloadClient:
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 52737a2a3..307dd0d4f 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -125,13 +125,20 @@ class Crawler(ABC):
_RATE_LIMIT: ClassVar[RateLimit] = 25, 1
_DOWNLOAD_SLOTS: ClassVar[int | None] = None
_USE_DOWNLOAD_SERVERS_LOCKS: ClassVar[bool] = False
+ _IMPERSONATE: ClassVar[str | bool | None] = None
create_db_path = staticmethod(DBPathBuilder.path)
@copy_signature(ScraperClient._request)
@contextlib.asynccontextmanager
- async def request(self, *args, **kwargs) -> AsyncGenerator[AbstractResponse]:
- async with self.client._limiter(self.DOMAIN), self.client._request(*args, **kwargs) as resp:
+ async def request(self, *args, impersonate: str | bool | None = None, **kwargs) -> AsyncGenerator[AbstractResponse]:
+ if impersonate is None:
+ impersonate = self._IMPERSONATE
+
+ async with (
+ self.client._limiter(self.DOMAIN),
+ self.client._request(*args, impersonate=impersonate, **kwargs) as resp,
+ ):
yield resp
@copy_signature(ScraperClient._request)
diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.py
index f7b47b374..8a2c1a0be 100644
--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py
+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py
@@ -1,17 +1,22 @@
from __future__ import annotations
-from typing import ClassVar
+from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from .xenforo import XenforoCrawler
+if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers.crawler import RateLimit
+
class CelebForumCrawler(XenforoCrawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://celebforum.to")
DOMAIN: ClassVar[str] = "celebforum"
FOLDER_DOMAIN: ClassVar[str] = "CelebForum"
IGNORE_EMBEDED_IMAGES_SRC: ClassVar = True # images src is always a thumbnail
+ _IMPERSONATE: ClassVar[str | bool | None] = True
+ _RATE_LIMIT: ClassVar[RateLimit] = 3, 10
@classmethod
def is_thumbnail(cls, link: AbsoluteHttpURL) -> bool: