Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: crawlers http calls (#992)
NTFSvolumeaa75dd9b79df
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.pyindex d7d77c8a7..07d7ffd96 100644--- a/cyberdrop_dl/clients/scraper_client.py+++ b/cyberdrop_dl/clients/scraper_client.py@@ -152,22 +152,23 @@ class ScraperClient:raise@limiter- async def get_soup_cffi(+ async def _get_response_and_soup_cffi(self,domain: str,url: URL,headers: dict[str, str] | None = None,impersonate: BrowserTarget | None = "chrome",- **request_params: Any,- ) -> BeautifulSoup:+ request_params: dict[str, Any] | None = None,+ ) -> tuple[CurlResponse, BeautifulSoup]:"""Makes a GET request using curl-cffi and creates a soup.:param domain: The crawler's domain (for rate limiting):param url: The URL to fetch.:param headers: Optional headers to include in the request. Will be added to session's default headers.:param impersonate: Optional browser target to impersonate. Defaults to `chrome`.- :param **request_params: Additional keyword arguments to pass to `curl_session.get` (e.g., `timeout`).+ :param request_params: Additional keyword arguments to pass to `curl_session.get` (e.g., `timeout`)."""+ request_params = request_params or {}headers = self._headers | (headers or {})response: CurlResponse = await self._curl_session.get(str(url), impersonate=impersonate, headers=headers, **request_params@@ -180,6 +181,11 @@ class ScraperClient:if self._save_pages_html:await self.write_soup_to_disk(url, response, soup)self.client_manager.cookies.update_cookies(self._curl_session.cookies, url)+ return response, soup++ @copy_signature(_get_response_and_soup_cffi)+ async def get_soup_cffi(self, *args, **kwargs) -> BeautifulSoup:+ _, soup = await self._get_response_and_soup_cffi(*args, **kwargs)return soup@limiter@@ -189,9 +195,9 @@ class ScraperClient:url: URL,headers: dict[str, str] | None = None,impersonate: BrowserTarget | None = "chrome",- data: dict[str, Any] | None = None,- json: dict[str, Any] | None = None,- **request_params: Any,+ data: Any = None,+ json: Any = None,+ request_params: dict[str, Any] | None = None,) -> CurlResponse:"""Makes a POST request using curl-cffi@@ -201,8 +207,9 @@ class ScraperClient::param impersonate: Optional browser target to impersonate. Defaults to `chrome`.:param data: Data to include in the requests. Will be sent as is (FormData).:param json: JSON data to include in the request. This will be serialized into a JSON string, and the `Content-Type` header will be set to `application/json`.- :param **request_params: Additional keyword arguments to pass to `curl_session.post` (e.g., `timeout`).+ :param request_params: Additional keyword arguments to pass to `curl_session.post` (e.g., `timeout`)."""+ request_params = request_params or {}headers = self._headers | (headers or {})response: CurlResponse = await self._curl_session.post(str(url), data=data, json=json, impersonate=impersonate, headers=headers, **request_params@@ -212,10 +219,13 @@ class ScraperClient:# ~~~~~~~~~~~~~ AIOHTTP ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~- async def _resilient_get(self, url: URL, **request_params: Any) -> tuple[AnyResponse, BeautifulSoup | None]:+ async def _resilient_get(+ self, url: URL, headers: dict[str, str], request_params: dict[str, Any] | None = None+ ) -> tuple[AnyResponse, BeautifulSoup | None]:"""Makes a GET request and automatically retries it with flaresolverr (if needed)"""+ request_params = request_params or {}for retry in range(2):- response = await self._session.get(url, **request_params)+ response = await self._session.get(url, headers=headers, **request_params)async with self.write_soup_on_error(url, response):try:await self.client_manager.check_http_status(response)@@ -242,9 +252,9 @@ class ScraperClient:domain: str,url: URL,headers: dict[str, str] | None = None,+ request_params: dict[str, Any] | None = None,*,cache_disabled: bool = False,- **request_params: Any,) -> tuple[AnyResponse, BeautifulSoup]:"""Makes a GET request using aiohttp and creates a soup.@@ -252,9 +262,10 @@ class ScraperClient::param domain: The crawler's domain (for rate limiting):param url: The URL to fetch.:param headers: Optional headers to include in the request. Will be added to session's default headers.+ :param request_params: Additional keyword arguments to pass to `session.get` (e.g., `timeout`).:param cache_disabled: Whether to disable caching for this request. Defaults to `False`.- :param **request_params: Additional keyword arguments to pass to `session.get` (e.g., `timeout`)."""+ request_params = request_params or {}headers = self._headers | (headers or {})async with cache_control_manager(self._session, disabled=cache_disabled):response, soup_or_none = await self._resilient_get(url, headers=headers, **request_params)@@ -278,10 +289,11 @@ class ScraperClient:domain: str,url: URL,headers: dict[str, str] | None = None,+ request_params: dict[str, Any] | None = None,*,cache_disabled: bool = False,- **request_params: Any,- ) -> dict[str, Any]:+ ) -> Any:+ request_params = request_params or {}headers = self._headers | (headers or {})async with cache_control_manager(self._session, disabled=cache_disabled):response = await self._session.get(url, headers=headers, **request_params)@@ -301,11 +313,11 @@ class ScraperClient:domain: str,url: URL,headers: dict[str, str] | None = None,- data: dict[str, Any] | None = None,- json: dict[str, Any] | None = None,+ data: Any = None,+ json: Any = None,+ request_params: dict[str, Any] | None = None,*,cache_disabled: bool = False,- **request_params: Any,) -> aiohttp.ClientResponse:"""Makes a post request using aiohtttp@@ -314,9 +326,10 @@ class ScraperClient::param headers: Optional headers to include in the request. Will be added to session's default headers.:param data: Data to include in the requests. Will be sent as is (FormData).:param json: JSON data to include in the request. This will be serialized into a JSON string, and the `Content-Type` header will be set to `application/json`.+ :param request_params: Additional keyword arguments to pass to `session.post` (e.g., `timeout`).:param cache_disabled: Whether to disable caching for this request. Defaults to `False`.- :param **request_params: Additional keyword arguments to pass to `session.post` (e.g., `timeout`)."""+ request_params = request_params or {}headers = self._headers | {"Accept-Encoding": "identity"} | (headers or {})async with cache_control_manager(self._session, disabled=cache_disabled):response = await self._session.post(url, headers=headers, data=data, json=json, **request_params)@@ -331,7 +344,7 @@ class ScraperClient:return await response.read()@copy_signature(_post_data)- async def post_data(self, *args: Any, **kwargs: Any) -> dict[str, Any]:+ async def post_data(self, *args: Any, **kwargs: Any) -> Any:content = await self.post_data_raw(*args, **kwargs)return json_loads(content)@@ -341,9 +354,9 @@ class ScraperClient:domain: str,url: URL,headers: dict[str, str] | None = None,+ request_params: dict[str, Any] | None = None,*,cache_disabled: bool = False,- **request_params: Any,) -> CIMultiDictProxy[str]:"""Makes a GET request to an URL and returns its headers@@ -351,9 +364,10 @@ class ScraperClient::param domain: The crawler's domain (for rate limiting):param url: The URL to fetch.:param headers: Optional headers to include in the request. Will be added to session's default headers.+ :param request_params: Additional keyword arguments to pass to `session.head` (e.g., `timeout`).:param cache_disabled: Whether to disable caching for this request. Defaults to `False`.- :param **request_params: Additional keyword arguments to pass to `session.head` (e.g., `timeout`)."""+ request_params = request_params or {}headers = self._headers | {"Accept-Encoding": "identity"} | (headers or {})async with cache_control_manager(self._session, disabled=cache_disabled):response = await self._session.head(url, headers=headers, **request_params)@@ -411,12 +425,12 @@ async def response_to_soup(response: AnyResponse | CurlResponse) -> BeautifulSoureturn BeautifulSoup(content, "html.parser")-async def response_to_json(response: AnyResponse) -> dict[str, Any]:+async def response_to_json(response: AnyResponse) -> Any:content_type: str = response.headers["Content-Type"].lower()if "text/plain" in content_type:return json_loads(await response.text())elif "json" in content_type:- return await response.json() or {}+ return await response.json()else:raise InvalidContentTypeError(message=f"Received {content_type}, was expecting JSON")diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.pyindex 8bfe0364c..55df64960 100644--- a/cyberdrop_dl/crawlers/_chevereto.py+++ b/cyberdrop_dl/crawlers/_chevereto.py@@ -123,7 +123,7 @@ class CheveretoCrawler(Crawler):if PASSWORD_PROTECTED in soup.text and password:data = {"content-password": password}async with self.request_limiter:- html = await self.client.post_data(self.domain, url, data=data, raw=True)+ html = await self.client.post_data_raw(self.domain, url, data=data)soup = BeautifulSoup(html, "html.parser")if PASSWORD_PROTECTED in soup.text:diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.pyindex 651e6c291..195bea04c 100644--- a/cyberdrop_dl/crawlers/bunkrr.py+++ b/cyberdrop_dl/crawlers/bunkrr.py@@ -183,7 +183,9 @@ class BunkrrCrawler(Crawler):link: URL | None = Nonesoup: BeautifulSoup | None = Noneif is_stream_redirect(scrape_item.url):- soup, scrape_item.url = await self.client.get_soup_and_return_url(self.domain, scrape_item.url)+ response, soup = await self.client._get_response_and_soup(self.domain, scrape_item.url)+ scrape_item.url = response.url+ del responsedatabase_url = scrape_item.url.with_host(self.DATABASE_PRIMARY_HOST)if await self.check_complete_from_referer(database_url):@@ -285,7 +287,7 @@ class BunkrrCrawler(Crawler):data = json.dumps(data_dict)async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers_inc=headers)+ json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers=headers)api_response = ApiResponse(**json_resp)link_str = decrypt_api_response(api_response)diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.pyindex c92fa1579..da8693037 100644--- a/cyberdrop_dl/crawlers/cyberdrop.py+++ b/cyberdrop_dl/crawlers/cyberdrop.py@@ -99,5 +99,5 @@ class CyberdropCrawler(Crawler):return urlif url.host.count(".") > 1 or "e" in url.parts:return self.primary_base_domain / "f" / url.name- _, streaming_url = await self.client.get_soup_and_return_url(self.domain, url)- return streaming_url+ response, _ = await self.client._get_response_and_soup(self.domain, url)+ return response.urldiff --git a/cyberdrop_dl/crawlers/cyberfile.py b/cyberdrop_dl/crawlers/cyberfile.pyindex 694d3c824..fcd8b96e8 100644--- a/cyberdrop_dl/crawlers/cyberfile.py+++ b/cyberdrop_dl/crawlers/cyberfile.py@@ -141,10 +141,10 @@ class CyberfileCrawler(Crawler):if not form:raise PasswordProtectedError("Unable to parse Password Protected File details")- password_post_url = form["action"]+ password_post_url = self.parse_url(form["action"]) # type: ignoredata = {"filePassword": password, "submitme": 1}async with self.request_limiter:- resp_bytes = await self.client.post_data(self.domain, password_post_url, data=data, raw=True)+ resp_bytes = await self.client.post_data_raw(self.domain, password_post_url, data=data)soup = BeautifulSoup(resp_bytes, "html.parser")if is_password_protected(soup):diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.pyindex 62ed85fbf..2c0344107 100644--- a/cyberdrop_dl/crawlers/dropbox.py+++ b/cyberdrop_dl/crawlers/dropbox.py@@ -11,6 +11,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headersif TYPE_CHECKING:+ from collections.abc import Mapping+from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem@@ -27,9 +29,10 @@ class DropboxCrawler(Crawler):@create_task_idasync def fetch(self, scrape_item: ScrapeItem) -> None:"""Determines where to send the scrape item based on the url."""- scrape_item.url = await self.get_share_url(scrape_item)- if not scrape_item.url:+ url = await self.get_share_url(scrape_item)+ if not url:return+ scrape_item.url = urlif any(p in scrape_item.url.path for p in ("/scl/fi/", "/scl/fo/")):return await self.file(scrape_item)@@ -64,14 +67,14 @@ class DropboxCrawler(Crawler):async def get_folder_name(self, url: URL) -> str | None:url = await self.get_redict_url(url)async with self.request_limiter:- headers: dict = await self.client.get_head(self.domain, url)+ headers = await self.client.get_head(self.domain, url)if not ("Content-Disposition" in headers and not is_html(headers)):raise ScrapeError(422)return get_filename_from_headers(headers)async def get_redict_url(self, url: URL) -> URL:async with self.request_limiter:- headers: dict = await self.client.get_head(self.domain, url)+ headers = await self.client.get_head(self.domain, url)location = headers.get("location")if not location:raise ScrapeError(400)@@ -138,6 +141,6 @@ def get_item_info(url: URL) -> DropboxItem:return DropboxItem(file_id, folder_tokens, url, rlkey, filename)-def is_html(headers: dict) -> bool:+def is_html(headers: Mapping[str, str]) -> bool:content_type: str = headers.get("Content-Type", "").lower()return any(s in content_type for s in ("html", "text"))diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.pyindex 5896244cd..d1bebe137 100644--- a/cyberdrop_dl/crawlers/e621.py+++ b/cyberdrop_dl/crawlers/e621.py@@ -46,7 +46,7 @@ class E621Crawler(Crawler):for page in itertools.count(initial_page):url = url.with_query(tags=scrape_item.url.query["tags"], page=page)async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.domain, url, headers_inc=self.custom_headers)+ json_resp: dict = await self.client.get_json(self.domain, url, headers=self.custom_headers)posts: list[dict] = json_resp.get("posts", [])if not posts:@@ -79,7 +79,7 @@ class E621Crawler(Crawler):pool_id = scrape_item.url.nameurl = self.primary_base_domain / f"pools/{pool_id}.json"async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.domain, url, headers_inc=self.custom_headers)+ json_resp: dict = await self.client.get_json(self.domain, url, headers=self.custom_headers)posts = json_resp.get("post_ids", [])title: str = json_resp.get("name", "Unknown Pool").replace("_", " ")@@ -97,7 +97,7 @@ class E621Crawler(Crawler):post_id = scrape_item.url.nameurl = self.primary_base_domain / f"posts/{post_id}.json"async with self.request_limiter:- json_resp: dict = await self.client.get_json(self.domain, url, headers_inc=self.custom_headers)+ json_resp: dict = await self.client.get_json(self.domain, url, headers=self.custom_headers)try:file_url = json_resp["post"]["file"]["url"]diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.pyindex 06cb6d2aa..cc977ee59 100644--- a/cyberdrop_dl/crawlers/generic.py+++ b/cyberdrop_dl/crawlers/generic.py@@ -87,7 +87,7 @@ class GenericCrawler(Crawler):async def get_content_type(self, url: URL) -> str:async with self.request_limiter:- headers: dict = await self.client.get_head(self.domain, url)+ headers = await self.client.get_head(self.domain, url)content_type: str = headers.get("Content-Type", "")if not content_type:raise ScrapeError(422)diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.pyindex 750e2b48e..0c7d72cfd 100644--- a/cyberdrop_dl/crawlers/gofile.py+++ b/cyberdrop_dl/crawlers/gofile.py@@ -156,7 +156,7 @@ class GoFileCrawler(Crawler):async def _update_website_token(self) -> None:async with self.request_limiter:- text = await self.client.get_text(self.domain, GLOBAL_JS_URL, origin=GLOBAL_JS_URL)+ text = await self.client.get_text(self.domain, GLOBAL_JS_URL)match = re.search(WT_REGEX, str(text))if not match:raise ScrapeError(401, "Couldn't generate GoFile websiteToken", origin=GLOBAL_JS_URL)diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.pyindex 8a62dd9a6..23a2b0a82 100644--- a/cyberdrop_dl/crawlers/google_drive.py+++ b/cyberdrop_dl/crawlers/google_drive.py@@ -43,6 +43,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_idfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headersif TYPE_CHECKING:+ from collections.abc import Mapping+from bs4 import BeautifulSoupfrom cyberdrop_dl.managers.manager import Manager@@ -138,7 +140,7 @@ class GoogleDriveCrawler(Crawler):filename, ext = self.get_filename_and_ext(filename)await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)- async def get_file_url_and_headers(self, url: URL, file_id: str) -> tuple[URL, dict]:+ async def get_file_url_and_headers(self, url: URL, file_id: str) -> tuple[URL, Mapping[str, str]]:soup = last_error = Nonecurrent_url: URL | None = urltry_file_open_url = True@@ -151,7 +153,9 @@ class GoogleDriveCrawler(Crawler):try:async with self.request_limiter:- soup, headers = await self.client.get_soup(self.domain, current_url, with_response_headers=True)+ response, soup = await self.client._get_response_and_soup(self.domain, current_url)+ headers = response.headers+except DownloadError as e:last_error = eif e.status == 500 and try_file_open_url:@@ -176,9 +180,9 @@ class GoogleDriveCrawler(Crawler):raise ScrapeError(422)- async def add_headers(self, url: URL) -> tuple[URL, dict]:+ async def add_headers(self, url: URL) -> tuple[URL, Mapping[str, str]]:async with self.request_limiter:- headers: dict = await self.client.get_head(self.domain, url)+ headers = await self.client.get_head(self.domain, url)location = headers.get("location")if location:link = self.parse_url(location)@@ -264,11 +268,11 @@ def get_download_url(item_id: str, folder: bool = False) -> URL:# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-def are_valid_headers(headers: dict):+def are_valid_headers(headers: Mapping[str, str]):return "Content-Disposition" in headers and not is_html(headers)-def is_html(headers: dict) -> bool:+def is_html(headers: Mapping[str, str]) -> bool:content_type: str = headers.get("Content-Type", "").lower()return any(s in content_type for s in ("html", "text"))diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.pyindex 3e74e38d7..591ba92b0 100644--- a/cyberdrop_dl/crawlers/imgur.py+++ b/cyberdrop_dl/crawlers/imgur.py@@ -56,7 +56,7 @@ class ImgurCrawler(Crawler):async with self.request_limiter:await self.check_imgur_credits(scrape_item)api_url = API_ENTRYPOINT / "album" / album_id- json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)+ json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers=self.headers)title_part = json_resp["data"].get("title", album_id)title = self.create_title(title_part, album_id)@@ -64,7 +64,7 @@ class ImgurCrawler(Crawler):async with self.request_limiter:api_url = API_ENTRYPOINT / "album" / album_id / "images"- json_resp = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)+ json_resp = await self.client.get_json(self.domain, api_url, headers=self.headers)for image in json_resp["data"]:await self.process_image(scrape_item, image)@@ -80,7 +80,7 @@ class ImgurCrawler(Crawler):async with self.request_limiter:await self.check_imgur_credits(scrape_item)api_url = API_ENTRYPOINT / "image" / image_id- json_resp = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)+ json_resp = await self.client.get_json(self.domain, api_url, headers=self.headers)await self.process_image(scrape_item, json_resp["data"])@@ -108,7 +108,7 @@ class ImgurCrawler(Crawler):async def check_imgur_credits(self, _=None) -> None:"""Checks the remaining credits."""credits_url = API_ENTRYPOINT / "credits"- json_resp = await self.client.get_json(self.domain, credits_url, headers_inc=self.headers)+ json_resp = await self.client.get_json(self.domain, credits_url, headers=self.headers)self.imgur_client_remaining = json_resp["data"]["ClientRemaining"]if self.imgur_client_remaining < 100:raise ScrapeError(429, "Imgur API rate limit reached")diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.pyindex a891cf614..d9fc17e4d 100644--- a/cyberdrop_dl/crawlers/jpg5.py+++ b/cyberdrop_dl/crawlers/jpg5.py@@ -70,4 +70,4 @@ class JPG5Crawler(CheveretoCrawler):"""Handles a direct link."""link = url or scrape_item.urllink = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))- super().handle_direct_link(scrape_item, link)+ await super().handle_direct_link(scrape_item, link)diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex 308b89cb2..4387eb7e5 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -356,7 +356,7 @@ class KemonoCrawler(Crawler):query_url = api_url.with_query(type="post" if is_post else "artist")async with self.request_limiter:- json_resp = await self.client.get_json(self.domain, query_url)+ json_resp: list[dict] = await self.client.get_json(self.domain, query_url)self.update_cookies({"session": ""})diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.pyindex 8da0261ea..f010cb02d 100644--- a/cyberdrop_dl/crawlers/luscious.py+++ b/cyberdrop_dl/crawlers/luscious.py@@ -123,6 +123,6 @@ class LusciousCrawler(Crawler):api_url = GRAPHQL_URL.with_query(operationName=query_name)headers = {"Content-Type": "application/json"}async with self.request_limiter:- json_resp = await self.client.post_data(self.domain, api_url, data=query, headers_inc=headers)+ json_resp = await self.client.post_data(self.domain, api_url, data=query, headers=headers)log_debug(json_resp)return json_respdiff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.pyindex 01906ff61..9b157a252 100644--- a/cyberdrop_dl/crawlers/onedrive.py+++ b/cyberdrop_dl/crawlers/onedrive.py@@ -130,7 +130,7 @@ class OneDriveCrawler(Crawler):@error_handling_wrapperasync def share_link(self, scrape_item: ScrapeItem) -> None:async with self.request_limiter:- headers: dict = await self.client.get_head(self.domain, scrape_item.url)+ headers = await self.client.get_head(self.domain, scrape_item.url)location = headers.get("location")if not location:raise ScrapeError(400)@@ -210,7 +210,7 @@ class OneDriveCrawler(Crawler):async def make_api_request(self, api_url: URL) -> dict[str, Any]:headers = {"Content-Type": "application/json"} | self.auth_headersasync with self.request_limiter:- json_resp: dict = await self.client.get_json(self.domain, api_url, headers_inc=headers)+ json_resp: dict = await self.client.get_json(self.domain, api_url, headers=headers)return json_resp@@ -220,9 +220,7 @@ class OneDriveCrawler(Crawler):data = {"appId": APP_UUID}data_json = json.dumps(data)async with self.request_limiter:- json_resp: dict = await self.client.post_data(- self.domain, badger_url, headers_inc=new_headers, data=data_json- )+ json_resp: dict = await self.client.post_data(self.domain, badger_url, headers=new_headers, data=data_json)badger_token: str = json_resp["token"]badger_token_expires: str = json_resp["expiryTimeUtc"]diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.pyindex f5f87df96..e4d0526da 100644--- a/cyberdrop_dl/crawlers/pmvhaven.py+++ b/cyberdrop_dl/crawlers/pmvhaven.py@@ -78,7 +78,7 @@ class PMVHavenCrawler(Crawler):add_headers = {"Content-Type": "text/plain;charset=UTF-8"}add_data = json.dumps({"profile": username, "mode": "GetUser"})async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.domain, api_url, data=add_data, headers_inc=add_headers)+ json_resp: dict = await self.client.post_data(self.domain, api_url, data=add_data, headers=add_headers)user_info: dict[str, dict] = json_resp["data"]for playlist in user_info["playlists"]:@@ -223,7 +223,7 @@ class PMVHavenCrawler(Crawler):if is_profile:data = json.dumps(data)async with self.request_limiter:- json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers_inc=add_headers)+ json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers=add_headers)has_videos = bool(json_resp[check_key])if not has_videos:diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.pyindex 4d05f4119..c697aafc7 100644--- a/cyberdrop_dl/crawlers/reddit.py+++ b/cyberdrop_dl/crawlers/reddit.py@@ -187,11 +187,11 @@ class RedditCrawler(Crawler):await self.handle_file(url, scrape_item, filename, ext)async def get_final_location(self, url) -> URL:- headers: dict[str, str] = await self.client.get_head(self.domain, url)+ headers = await self.client.get_head(self.domain, url)content_type = headers.get("Content-Type", "")if any(s in content_type.lower() for s in ("html", "text")):- _, url = await self.client.get_soup_and_return_url(self.domain, url)- return url+ response, _ = await self.client._get_response_and_soup(self.domain, url)+ return response.urllocation = headers.get("location")if not location:raise ScrapeError(422)diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.pyindex 3d4898ebe..e9d2963bb 100644--- a/cyberdrop_dl/crawlers/redgifs.py+++ b/cyberdrop_dl/crawlers/redgifs.py@@ -63,7 +63,7 @@ class RedGifsCrawler(Crawler):async with self.request_limiter:api_url = API_ENTRYPOINT / "v2/users" / user_id / "search"api_url = api_url.with_query(order="new", page=page)- json_resp = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)+ json_resp = await self.client.get_json(self.domain, api_url, headers=self.headers)gifs = json_resp["gifs"]yield json_respif total_gifs is None:@@ -78,7 +78,7 @@ class RedGifsCrawler(Crawler):post_id = scrape_item.url.parts[-1].split(".")[0]async with self.request_limiter:api_url = API_ENTRYPOINT / "v2/gifs" / post_id- json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)+ json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers=self.headers)title_part: str = json_resp["gif"].get("title") or "Loose Files"title = self.create_title(title_part)diff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.pyindex 753f802b7..938a4b2f8 100644--- a/cyberdrop_dl/crawlers/send_now.py+++ b/cyberdrop_dl/crawlers/send_now.py@@ -32,10 +32,11 @@ class SendNowCrawler(Crawler):async def file(self, scrape_item: ScrapeItem) -> None:file_id = scrape_item.url.namedata = {"op": "download2", "id": file_id, "rand": "", "referer": "", "method_free": "", "method_premium": ""}+ params = {"allow_redirects": False, "stream": True}async with self.request_limiter:response: CurlResponse = await self.client.post_data_cffi(- self.domain, scrape_item.url, data=data, allow_redirects=False, stream=True+ self.domain, scrape_item.url, data=data, request_params=params)debrid_link = self.parse_url(response.headers.get("location"))diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.pyindex a9f11eb59..4df17f2a4 100644--- a/cyberdrop_dl/crawlers/sex_dot_com.py+++ b/cyberdrop_dl/crawlers/sex_dot_com.py@@ -58,8 +58,7 @@ class SexDotComCrawler(Crawler):breakpage += 1- @error_handling_wrapper- async def get_media(self, scrape_item: ScrapeItem) -> dict:+ async def get_media(self, scrape_item: ScrapeItem) -> dict[str, Any]:"""Gets media from its relative URL."""relative_url = "/".join(scrape_item.url.parts[3:])data_url = API_URL / "media" / "getMedia"@@ -70,22 +69,22 @@ class SexDotComCrawler(Crawler):@error_handling_wrapperasync def handle_media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:- item = item or await self.get_media(scrape_item)- relative_url = item["relativeUrl"]+ real_item = item or await self.get_media(scrape_item)+ relative_url = real_item["relativeUrl"]canonical_url = URL("https://sex.com/en/shorts") / relative_urlif await self.check_complete_from_referer(canonical_url):return- fileType: str = item.get("fileType") or item["mediaType"]+ fileType: str = real_item.get("fileType") or real_item["mediaType"]if fileType.startswith("image"):- media_url = URL(item["fullPath"]).with_query(optimizer="image", width=1200)- filename, ext = f"{item['pictureUid']}.jpg", "jpg"+ media_url = URL(real_item["fullPath"]).with_query(optimizer="image", width=1200)+ filename, ext = f"{real_item['pictureUid']}.jpg", "jpg"elif fileType.startswith("video"):- media_url = URL(item["sources"][0]["fullPath"])+ media_url = URL(real_item["sources"][0]["fullPath"])filename, ext = self.get_filename_and_ext(media_url.name)- scrape_item.possible_datetime = self.parse_datetime(item["createdAt"])+ scrape_item.possible_datetime = self.parse_datetime(real_item["createdAt"])scrape_item.url = canonical_urlawait self.handle_file(media_url, scrape_item, filename, ext)scrape_item.add_children()diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.pyindex 45947c460..22c0897de 100644--- a/cyberdrop_dl/crawlers/wetransfer.py+++ b/cyberdrop_dl/crawlers/wetransfer.py@@ -46,7 +46,7 @@ class WeTransferCrawler(Crawler):headers = {"Content-Type": "application/json"}async with self.request_limiter:json_resp: dict = await self.client.post_data(- self.domain, file_info.download_url, data=file_info.json, headers_inc=headers+ self.domain, file_info.download_url, data=file_info.json, headers=headers)link_str: str = json_resp.get("direct_link") # type: ignore@@ -64,7 +64,7 @@ class WeTransferCrawler(Crawler):async def get_final_url(self, scrape_item: ScrapeItem) -> URL:async with self.request_limiter:- headers: dict = await self.client.get_head(self.domain, scrape_item.url)+ headers = await self.client.get_head(self.domain, scrape_item.url)location: str = headers.get("location") # type: ignorereturn self.parse_url(location)diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.pyindex 4ca337f94..a4756884c 100644--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py@@ -22,7 +22,7 @@ class CelebForumCrawler(XenforoCrawler):def __init__(self, manager: Manager) -> None:super().__init__(manager, self.domain, "CelebForum")- def filter_link(self, link):+ def filter_link(self, link: URL) -> URL | None:if link.host == self.primary_base_domain.host:if all(part in link.parts for part in ["data", "attachments"]): # Thumbnailsreturn Nonediff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.pyindex 2d4e42f61..f84c7864e 100644--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py@@ -33,7 +33,7 @@ class F95ZoneCrawler(XenforoCrawler):"""Override to handle protected link confirmation."""async with self.request_limiter:data = ({"xhr": "1", "download": "1"},)- JSON_Resp = await self.client.post_data(self.domain, link, data=data, origin=origin)+ JSON_Resp = await self.client.post_data(self.domain, link, data=data)if JSON_Resp["status"] == "ok":return self.parse_url(JSON_Resp["msg"])diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.pyindex d95f7fd53..bd1f69e2a 100644--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py@@ -153,9 +153,10 @@ class XenforoCrawler(Crawler):if self.thread_url_part in scrape_item.url.parts:return await self.thread(scrape_item)if self.is_confirmation_link(scrape_item.url):- scrape_item.url = await self.handle_confirmation_link(scrape_item.url)- if scrape_item.url: # If there was an error, this will be None+ url = await self.handle_confirmation_link(scrape_item.url)+ if url: # If there was an error, this will be None# This could end up back in here if the URL goes to another thread+ scrape_item.url = urlreturn self.handle_external_links(scrape_item)if any(p in scrape_item.url.parts for p in ("goto", "posts")):return await self.redirect(scrape_item)@@ -384,7 +385,7 @@ class XenforoCrawler(Crawler):async def handle_confirmation_link(self, link: URL, *, origin: ScrapeItem | None = None) -> URL | None:"""Handles link confirmation."""async with self.request_limiter:- soup: BeautifulSoup = await self.client.get_soup(self.domain, link, origin=origin)+ soup: BeautifulSoup = await self.client.get_soup(self.domain, link)confirm_button = soup.select_one("a[class*=button--cta]")if not confirm_button:return@@ -500,9 +501,7 @@ class XenforoCrawler(Crawler):attempt += 1await asyncio.sleep(wait_time)data = prepare_login_data(text)- await self.client.post_data(- self.domain, login_url / "login", data=data, req_resp=False, cache_disabled=True- )+ _ = await self.client._post_data(self.domain, login_url / "login", data=data, cache_disabled=True)await asyncio.sleep(wait_time)text, logged_in = await self.check_login_with_request(login_url)if logged_in:diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.pyindex e6bb091ef..ed48d56f9 100644--- a/cyberdrop_dl/crawlers/yandex_disk.py+++ b/cyberdrop_dl/crawlers/yandex_disk.py@@ -97,7 +97,7 @@ class YandexDiskCrawler(Crawler):}async with self.request_limiter:json_resp: dict[str, Any] = await self.client.post_data(- self.domain, DOWNLOAD_API_ENTRYPOINT, data=file.post_data, headers_inc=headers+ self.domain, DOWNLOAD_API_ENTRYPOINT, data=file.post_data, headers=headers)new_sk = json_resp.get("new_sk")