Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: let crawlers set download headers (#145)

NTFSvolumef7e22e6e9c74
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index a09bc3de5..c0eb17c12 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -14,7 +14,6 @@ import aiofiles
from cyberdrop_dl import aio, constants, storage
from cyberdrop_dl.clients.response import AbstractResponse
from cyberdrop_dl.constants import FileExt
-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadError
from cyberdrop_dl.utils import dates
@@ -25,7 +24,7 @@ if TYPE_CHECKING:
import aiohttp
- from cyberdrop_dl.data_structures.url_objects import MediaItem
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem
from cyberdrop_dl.managers.client_manager import ClientManager
from cyberdrop_dl.managers.manager import Manager
@@ -35,9 +34,6 @@ logger = logging.getLogger(__name__)
_CONTENT_TYPES_OVERRIDES: dict[str, str] = {"text/vnd.trolltech.linguist": "video/MP2T"}
_SLOW_DOWNLOAD_PERIOD: int = 10 # seconds
-_CHROME_ANDROID_USER_AGENT: str = (
- "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36"
-)
_FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks
_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()
_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}
@@ -65,36 +61,8 @@ class DownloadClient:
with self.client_manager.request_context(domain):
yield
- def _get_download_headers(self, domain: str, referer: AbsoluteHttpURL) -> dict[str, str]:
- download_headers = {
- "User-Agent": self.manager.config.global_settings.general.user_agent,
- "Referer": str(referer),
- }
- auth_data = self.manager.config.auth
- if domain == "pixeldrain" and auth_data.pixeldrain.api_key:
- download_headers["Authorization"] = self.manager.client_manager.basic_auth(
- "Cyberdrop-DL", auth_data.pixeldrain.api_key
- )
- elif domain == "gofile":
- gofile_cookies = self.client_manager.cookies.filter_cookies(AbsoluteHttpURL("https://gofile.io"))
- api_key = gofile_cookies.get("accountToken", "")
- if api_key:
- download_headers["Authorization"] = f"Bearer {api_key.value}" # type: ignore
- elif domain == "odnoklassniki":
- # TODO: Add "headers" attribute to MediaItem to use custom headers for downloads
- download_headers |= {
- "Accept-Language": "en-gb, en;q=0.8",
- "User-Agent": _CHROME_ANDROID_USER_AGENT,
- "Referer": "https://m.ok.ru/",
- "Origin": "https://m.ok.ru",
- }
- elif domain == "megacloud":
- download_headers["Referer"] = "https://megacloud.blog/"
- return download_headers
-
async def _download(self, domain: str, media_item: MediaItem) -> bool:
"""Downloads a file."""
- download_headers = self._get_download_headers(domain, media_item.referer)
downloaded_filename = await self.manager.database.history.get_downloaded_filename(domain, media_item)
download_dir = self.get_download_dir(media_item)
if media_item.is_segment:
@@ -105,14 +73,14 @@ class DownloadClient:
resume_point = 0
if self._supports_ranges and media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):
resume_point = size
- download_headers["Range"] = f"bytes={size}-"
+ media_item.headers["Range"] = f"bytes={size}-"
await asyncio.sleep(self.manager.config.global_settings.rate_limiting_options.total_delay)
- def process_response(resp: aiohttp.ClientResponse | AbstractResponse):
+ def process_response(resp: aiohttp.ClientResponse | AbstractResponse[Any]):
return self._process_response(media_item, domain, resume_point, resp)
- return await self._request_download(media_item, download_headers, process_response)
+ return await self._request_download(media_item, process_response)
async def _process_response(
self,
@@ -197,7 +165,6 @@ class DownloadClient:
async def _request_download(
self,
media_item: MediaItem,
- download_headers: dict[str, str],
process_response: Callable[[aiohttp.ClientResponse | AbstractResponse], Coroutine[None, None, bool]],
) -> bool:
download_url = media_item.debrid_link or media_item.url
@@ -208,7 +175,7 @@ class DownloadClient:
while True:
resp = None
try:
- async with self.__request_context(download_url, media_item.domain, download_headers) as resp:
+ async with self.__request_context(download_url, media_item.domain, media_item.headers) as resp:
return await process_response(resp)
except (DownloadError, DDOSGuardError):
if resp is None:
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 999a15302..799dcbf5b 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -443,8 +443,15 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
media_item.metadata = metadata
if referer:
media_item.referer = referer
+ media_item.headers.update(self._prepare_headers(scrape_item))
await self.handle_media_item(media_item, m3u8)
+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
+ return {
+ "User-Agent": self.manager.config.global_settings.general.user_agent,
+ "Referer": str(scrape_item.url),
+ }
+
@final
async def _download(self, media_item: MediaItem, m3u8: m3u8.Rendition | None) -> None:
try:
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index e7d1584d4..40c4198e2 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -113,6 +113,9 @@ class GoFileCrawler(Crawler):
async def __async_post_init__(self) -> None:
await self._get_credentials(_API_ENTRYPOINT)
+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
+ return super()._prepare_headers(scrape_item) | {"Authorization": f"Bearer {self._api_key}"}
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["d", content_id]:
diff --git a/cyberdrop_dl/crawlers/megacloud.py b/cyberdrop_dl/crawlers/megacloud.py
index b6996c0d9..1b8bca48e 100644
--- a/cyberdrop_dl/crawlers/megacloud.py
+++ b/cyberdrop_dl/crawlers/megacloud.py
@@ -40,6 +40,9 @@ class MegaCloudCrawler(Crawler):
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL
DOMAIN: ClassVar[str] = "megacloud"
+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
+ return super()._prepare_headers(scrape_item) | {"Referer": "https://megacloud.blog/"}
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["embed-2", "v3", _, _]:
diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.py
index 097118839..3cd3b40d8 100644
--- a/cyberdrop_dl/crawlers/mixdrop.py
+++ b/cyberdrop_dl/crawlers/mixdrop.py
@@ -26,6 +26,9 @@ class MixDropCrawler(Crawler):
DOMAIN: ClassVar[str] = "mixdrop"
FOLDER_DOMAIN: ClassVar[str] = "MixDrop"
+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
+ return super()._prepare_headers(scrape_item) | {"Referer": "https://m1xdrop.click/"}
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["f" | "e", file_id]:
diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.py
index c908079f6..034702b12 100644
--- a/cyberdrop_dl/crawlers/odnoklassniki.py
+++ b/cyberdrop_dl/crawlers/odnoklassniki.py
@@ -17,6 +17,9 @@ if TYPE_CHECKING:
_find_video_ids = re.compile("/video/(\\d+)").finditer
+_CHROME_ANDROID_USER_AGENT: str = (
+ "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36"
+)
_HEADERS = {
"Accept-Language": "en-gb, en;q=0.8",
"Referer": "https://ok.ru/",
@@ -24,7 +27,7 @@ _HEADERS = {
}
_MOBILE_HEADERS = _HEADERS | {
- "User-Agent": "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36",
+ "User-Agent": _CHROME_ANDROID_USER_AGENT,
"Referer": "https://m.ok.ru/",
"Origin": "https://m.ok.ru",
}
@@ -59,6 +62,9 @@ class OdnoklassnikiCrawler(Crawler):
DOMAIN = "odnoklassniki"
FOLDER_DOMAIN = "ok.ru"
+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
+ return super()._prepare_headers(scrape_item) | _MOBILE_HEADERS
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["video", id_]:
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index e4fe3735c..e8b797189 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -119,6 +119,9 @@ class PixelDrainCrawler(Crawler):
msg = f"{json_resp['message']} ({json_resp['value']})"
raise ScrapeError(resp.status, msg)
+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:
+ return super()._prepare_headers(scrape_item) | self._headers
+
async def _api_request(self, api_url: AbsoluteHttpURL) -> str:
return await self.request_text(api_url, headers=self._headers)
diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.py
index 135cad846..8084bfe1b 100644
--- a/cyberdrop_dl/data_structures/url_objects.py
+++ b/cyberdrop_dl/data_structures/url_objects.py
@@ -167,6 +167,7 @@ class MediaItem:
id: tuple[str, ...] = field(init=False)
base64_id: str = field(init=False)
+ headers: dict[str, Any] = field(default_factory=dict)
def __post_init__(self) -> None:
self.id = self.domain, self.db_path
diff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.py
index d50d45da2..9117af0db 100644
--- a/cyberdrop_dl/logs.py
+++ b/cyberdrop_dl/logs.py
@@ -284,7 +284,7 @@ def setup_file_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[N
),
),
):
- logger.info(f"Debug log file: '{debug_log_file}'")
+ logger.info(f"Debug log file: {debug_log_file}")
token = MAIN_LOG_FILE.set(file)
try:
yield
diff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.py
index 2103e7364..7c0bd78ef 100644
--- a/cyberdrop_dl/managers/client_manager.py
+++ b/cyberdrop_dl/managers/client_manager.py
@@ -420,8 +420,7 @@ class ClientManager:
properties = await probe(media_item.path)
else:
- headers = self.download_client._get_download_headers(media_item.domain, media_item.referer)
- properties = await probe(media_item.url, headers=headers)
+ properties = await probe(media_item.url, headers=media_item.headers)
if properties.format.duration:
return properties.format.duration