Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: let crawlers set download headers (#145)
NTFSvolumef7e22e6e9c74
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex a09bc3de5..c0eb17c12 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -14,7 +14,6 @@ import aiofilesfrom cyberdrop_dl import aio, constants, storagefrom cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.constants import FileExt-from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadErrorfrom cyberdrop_dl.utils import dates@@ -25,7 +24,7 @@ if TYPE_CHECKING:import aiohttp- from cyberdrop_dl.data_structures.url_objects import MediaItem+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.managers.client_manager import ClientManagerfrom cyberdrop_dl.managers.manager import Manager@@ -35,9 +34,6 @@ logger = logging.getLogger(__name__)_CONTENT_TYPES_OVERRIDES: dict[str, str] = {"text/vnd.trolltech.linguist": "video/MP2T"}_SLOW_DOWNLOAD_PERIOD: int = 10 # seconds-_CHROME_ANDROID_USER_AGENT: str = (- "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36"-)_FREE_SPACE_CHECK_PERIOD: int = 5 # Check every 5 chunks_NULL_CONTEXT: contextlib.nullcontext[None] = contextlib.nullcontext()_USE_IMPERSONATION: set[str] = {"vsco", "celebforum"}@@ -65,36 +61,8 @@ class DownloadClient:with self.client_manager.request_context(domain):yield- def _get_download_headers(self, domain: str, referer: AbsoluteHttpURL) -> dict[str, str]:- download_headers = {- "User-Agent": self.manager.config.global_settings.general.user_agent,- "Referer": str(referer),- }- auth_data = self.manager.config.auth- if domain == "pixeldrain" and auth_data.pixeldrain.api_key:- download_headers["Authorization"] = self.manager.client_manager.basic_auth(- "Cyberdrop-DL", auth_data.pixeldrain.api_key- )- elif domain == "gofile":- gofile_cookies = self.client_manager.cookies.filter_cookies(AbsoluteHttpURL("https://gofile.io"))- api_key = gofile_cookies.get("accountToken", "")- if api_key:- download_headers["Authorization"] = f"Bearer {api_key.value}" # type: ignore- elif domain == "odnoklassniki":- # TODO: Add "headers" attribute to MediaItem to use custom headers for downloads- download_headers |= {- "Accept-Language": "en-gb, en;q=0.8",- "User-Agent": _CHROME_ANDROID_USER_AGENT,- "Referer": "https://m.ok.ru/",- "Origin": "https://m.ok.ru",- }- elif domain == "megacloud":- download_headers["Referer"] = "https://megacloud.blog/"- return download_headers-async def _download(self, domain: str, media_item: MediaItem) -> bool:"""Downloads a file."""- download_headers = self._get_download_headers(domain, media_item.referer)downloaded_filename = await self.manager.database.history.get_downloaded_filename(domain, media_item)download_dir = self.get_download_dir(media_item)if media_item.is_segment:@@ -105,14 +73,14 @@ class DownloadClient:resume_point = 0if self._supports_ranges and media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):resume_point = size- download_headers["Range"] = f"bytes={size}-"+ media_item.headers["Range"] = f"bytes={size}-"await asyncio.sleep(self.manager.config.global_settings.rate_limiting_options.total_delay)- def process_response(resp: aiohttp.ClientResponse | AbstractResponse):+ def process_response(resp: aiohttp.ClientResponse | AbstractResponse[Any]):return self._process_response(media_item, domain, resume_point, resp)- return await self._request_download(media_item, download_headers, process_response)+ return await self._request_download(media_item, process_response)async def _process_response(self,@@ -197,7 +165,6 @@ class DownloadClient:async def _request_download(self,media_item: MediaItem,- download_headers: dict[str, str],process_response: Callable[[aiohttp.ClientResponse | AbstractResponse], Coroutine[None, None, bool]],) -> bool:download_url = media_item.debrid_link or media_item.url@@ -208,7 +175,7 @@ class DownloadClient:while True:resp = Nonetry:- async with self.__request_context(download_url, media_item.domain, download_headers) as resp:+ async with self.__request_context(download_url, media_item.domain, media_item.headers) as resp:return await process_response(resp)except (DownloadError, DDOSGuardError):if resp is None:diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 999a15302..799dcbf5b 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -443,8 +443,15 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):media_item.metadata = metadataif referer:media_item.referer = referer+ media_item.headers.update(self._prepare_headers(scrape_item))await self.handle_media_item(media_item, m3u8)+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:+ return {+ "User-Agent": self.manager.config.global_settings.general.user_agent,+ "Referer": str(scrape_item.url),+ }+@finalasync def _download(self, media_item: MediaItem, m3u8: m3u8.Rendition | None) -> None:try:diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex e7d1584d4..40c4198e2 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -113,6 +113,9 @@ class GoFileCrawler(Crawler):async def __async_post_init__(self) -> None:await self._get_credentials(_API_ENTRYPOINT)+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:+ return super()._prepare_headers(scrape_item) | {"Authorization": f"Bearer {self._api_key}"}+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["d", content_id]:diff --git a/cyberdrop_dl/crawlers/megacloud.py b/cyberdrop_dl/crawlers/megacloud.pyindex b6996c0d9..1b8bca48e 100644--- a/cyberdrop_dl/crawlers/megacloud.py+++ b/cyberdrop_dl/crawlers/megacloud.py@@ -40,6 +40,9 @@ class MegaCloudCrawler(Crawler):PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URLDOMAIN: ClassVar[str] = "megacloud"+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:+ return super()._prepare_headers(scrape_item) | {"Referer": "https://megacloud.blog/"}+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["embed-2", "v3", _, _]:diff --git a/cyberdrop_dl/crawlers/mixdrop.py b/cyberdrop_dl/crawlers/mixdrop.pyindex 097118839..3cd3b40d8 100644--- a/cyberdrop_dl/crawlers/mixdrop.py+++ b/cyberdrop_dl/crawlers/mixdrop.py@@ -26,6 +26,9 @@ class MixDropCrawler(Crawler):DOMAIN: ClassVar[str] = "mixdrop"FOLDER_DOMAIN: ClassVar[str] = "MixDrop"+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:+ return super()._prepare_headers(scrape_item) | {"Referer": "https://m1xdrop.click/"}+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["f" | "e", file_id]:diff --git a/cyberdrop_dl/crawlers/odnoklassniki.py b/cyberdrop_dl/crawlers/odnoklassniki.pyindex c908079f6..034702b12 100644--- a/cyberdrop_dl/crawlers/odnoklassniki.py+++ b/cyberdrop_dl/crawlers/odnoklassniki.py@@ -17,6 +17,9 @@ if TYPE_CHECKING:_find_video_ids = re.compile("/video/(\\d+)").finditer+_CHROME_ANDROID_USER_AGENT: str = (+ "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36"+)_HEADERS = {"Accept-Language": "en-gb, en;q=0.8","Referer": "https://ok.ru/",@@ -24,7 +27,7 @@ _HEADERS = {}_MOBILE_HEADERS = _HEADERS | {- "User-Agent": "Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.180 Mobile Safari/537.36",+ "User-Agent": _CHROME_ANDROID_USER_AGENT,"Referer": "https://m.ok.ru/","Origin": "https://m.ok.ru",}@@ -59,6 +62,9 @@ class OdnoklassnikiCrawler(Crawler):DOMAIN = "odnoklassniki"FOLDER_DOMAIN = "ok.ru"+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:+ return super()._prepare_headers(scrape_item) | _MOBILE_HEADERS+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["video", id_]:diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.pyindex e4fe3735c..e8b797189 100644--- a/cyberdrop_dl/crawlers/pixeldrain.py+++ b/cyberdrop_dl/crawlers/pixeldrain.py@@ -119,6 +119,9 @@ class PixelDrainCrawler(Crawler):msg = f"{json_resp['message']} ({json_resp['value']})"raise ScrapeError(resp.status, msg)+ def _prepare_headers(self, scrape_item: ScrapeItem) -> dict[str, str]:+ return super()._prepare_headers(scrape_item) | self._headers+async def _api_request(self, api_url: AbsoluteHttpURL) -> str:return await self.request_text(api_url, headers=self._headers)diff --git a/cyberdrop_dl/data_structures/url_objects.py b/cyberdrop_dl/data_structures/url_objects.pyindex 135cad846..8084bfe1b 100644--- a/cyberdrop_dl/data_structures/url_objects.py+++ b/cyberdrop_dl/data_structures/url_objects.py@@ -167,6 +167,7 @@ class MediaItem:id: tuple[str, ...] = field(init=False)base64_id: str = field(init=False)+ headers: dict[str, Any] = field(default_factory=dict)def __post_init__(self) -> None:self.id = self.domain, self.db_pathdiff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.pyindex d50d45da2..9117af0db 100644--- a/cyberdrop_dl/logs.py+++ b/cyberdrop_dl/logs.py@@ -284,7 +284,7 @@ def setup_file_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[N),),):- logger.info(f"Debug log file: '{debug_log_file}'")+ logger.info(f"Debug log file: {debug_log_file}")token = MAIN_LOG_FILE.set(file)try:yielddiff --git a/cyberdrop_dl/managers/client_manager.py b/cyberdrop_dl/managers/client_manager.pyindex 2103e7364..7c0bd78ef 100644--- a/cyberdrop_dl/managers/client_manager.py+++ b/cyberdrop_dl/managers/client_manager.py@@ -420,8 +420,7 @@ class ClientManager:properties = await probe(media_item.path)else:- headers = self.download_client._get_download_headers(media_item.domain, media_item.referer)- properties = await probe(media_item.url, headers=headers)+ properties = await probe(media_item.url, headers=media_item.headers)if properties.format.duration:return properties.format.duration