Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: crawlers http calls (#992)

NTFSvolumeaa75dd9b79df
diff --git a/cyberdrop_dl/clients/scraper_client.py b/cyberdrop_dl/clients/scraper_client.py
index d7d77c8a7..07d7ffd96 100644
--- a/cyberdrop_dl/clients/scraper_client.py
+++ b/cyberdrop_dl/clients/scraper_client.py
@@ -152,22 +152,23 @@ class ScraperClient:
raise
@limiter
- async def get_soup_cffi(
+ async def _get_response_and_soup_cffi(
self,
domain: str,
url: URL,
headers: dict[str, str] | None = None,
impersonate: BrowserTarget | None = "chrome",
- **request_params: Any,
- ) -> BeautifulSoup:
+ request_params: dict[str, Any] | None = None,
+ ) -> tuple[CurlResponse, BeautifulSoup]:
"""Makes a GET request using curl-cffi and creates a soup.
:param domain: The crawler's domain (for rate limiting)
:param url: The URL to fetch.
:param headers: Optional headers to include in the request. Will be added to session's default headers.
:param impersonate: Optional browser target to impersonate. Defaults to `chrome`.
- :param **request_params: Additional keyword arguments to pass to `curl_session.get` (e.g., `timeout`).
+ :param request_params: Additional keyword arguments to pass to `curl_session.get` (e.g., `timeout`).
"""
+ request_params = request_params or {}
headers = self._headers | (headers or {})
response: CurlResponse = await self._curl_session.get(
str(url), impersonate=impersonate, headers=headers, **request_params
@@ -180,6 +181,11 @@ class ScraperClient:
if self._save_pages_html:
await self.write_soup_to_disk(url, response, soup)
self.client_manager.cookies.update_cookies(self._curl_session.cookies, url)
+ return response, soup
+
+ @copy_signature(_get_response_and_soup_cffi)
+ async def get_soup_cffi(self, *args, **kwargs) -> BeautifulSoup:
+ _, soup = await self._get_response_and_soup_cffi(*args, **kwargs)
return soup
@limiter
@@ -189,9 +195,9 @@ class ScraperClient:
url: URL,
headers: dict[str, str] | None = None,
impersonate: BrowserTarget | None = "chrome",
- data: dict[str, Any] | None = None,
- json: dict[str, Any] | None = None,
- **request_params: Any,
+ data: Any = None,
+ json: Any = None,
+ request_params: dict[str, Any] | None = None,
) -> CurlResponse:
"""Makes a POST request using curl-cffi
@@ -201,8 +207,9 @@ class ScraperClient:
:param impersonate: Optional browser target to impersonate. Defaults to `chrome`.
:param data: Data to include in the requests. Will be sent as is (FormData).
:param json: JSON data to include in the request. This will be serialized into a JSON string, and the `Content-Type` header will be set to `application/json`.
- :param **request_params: Additional keyword arguments to pass to `curl_session.post` (e.g., `timeout`).
+ :param request_params: Additional keyword arguments to pass to `curl_session.post` (e.g., `timeout`).
"""
+ request_params = request_params or {}
headers = self._headers | (headers or {})
response: CurlResponse = await self._curl_session.post(
str(url), data=data, json=json, impersonate=impersonate, headers=headers, **request_params
@@ -212,10 +219,13 @@ class ScraperClient:
# ~~~~~~~~~~~~~ AIOHTTP ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- async def _resilient_get(self, url: URL, **request_params: Any) -> tuple[AnyResponse, BeautifulSoup | None]:
+ async def _resilient_get(
+ self, url: URL, headers: dict[str, str], request_params: dict[str, Any] | None = None
+ ) -> tuple[AnyResponse, BeautifulSoup | None]:
"""Makes a GET request and automatically retries it with flaresolverr (if needed)"""
+ request_params = request_params or {}
for retry in range(2):
- response = await self._session.get(url, **request_params)
+ response = await self._session.get(url, headers=headers, **request_params)
async with self.write_soup_on_error(url, response):
try:
await self.client_manager.check_http_status(response)
@@ -242,9 +252,9 @@ class ScraperClient:
domain: str,
url: URL,
headers: dict[str, str] | None = None,
+ request_params: dict[str, Any] | None = None,
*,
cache_disabled: bool = False,
- **request_params: Any,
) -> tuple[AnyResponse, BeautifulSoup]:
"""
Makes a GET request using aiohttp and creates a soup.
@@ -252,9 +262,10 @@ class ScraperClient:
:param domain: The crawler's domain (for rate limiting)
:param url: The URL to fetch.
:param headers: Optional headers to include in the request. Will be added to session's default headers.
+ :param request_params: Additional keyword arguments to pass to `session.get` (e.g., `timeout`).
:param cache_disabled: Whether to disable caching for this request. Defaults to `False`.
- :param **request_params: Additional keyword arguments to pass to `session.get` (e.g., `timeout`).
"""
+ request_params = request_params or {}
headers = self._headers | (headers or {})
async with cache_control_manager(self._session, disabled=cache_disabled):
response, soup_or_none = await self._resilient_get(url, headers=headers, **request_params)
@@ -278,10 +289,11 @@ class ScraperClient:
domain: str,
url: URL,
headers: dict[str, str] | None = None,
+ request_params: dict[str, Any] | None = None,
*,
cache_disabled: bool = False,
- **request_params: Any,
- ) -> dict[str, Any]:
+ ) -> Any:
+ request_params = request_params or {}
headers = self._headers | (headers or {})
async with cache_control_manager(self._session, disabled=cache_disabled):
response = await self._session.get(url, headers=headers, **request_params)
@@ -301,11 +313,11 @@ class ScraperClient:
domain: str,
url: URL,
headers: dict[str, str] | None = None,
- data: dict[str, Any] | None = None,
- json: dict[str, Any] | None = None,
+ data: Any = None,
+ json: Any = None,
+ request_params: dict[str, Any] | None = None,
*,
cache_disabled: bool = False,
- **request_params: Any,
) -> aiohttp.ClientResponse:
"""Makes a post request using aiohtttp
@@ -314,9 +326,10 @@ class ScraperClient:
:param headers: Optional headers to include in the request. Will be added to session's default headers.
:param data: Data to include in the requests. Will be sent as is (FormData).
:param json: JSON data to include in the request. This will be serialized into a JSON string, and the `Content-Type` header will be set to `application/json`.
+ :param request_params: Additional keyword arguments to pass to `session.post` (e.g., `timeout`).
:param cache_disabled: Whether to disable caching for this request. Defaults to `False`.
- :param **request_params: Additional keyword arguments to pass to `session.post` (e.g., `timeout`).
"""
+ request_params = request_params or {}
headers = self._headers | {"Accept-Encoding": "identity"} | (headers or {})
async with cache_control_manager(self._session, disabled=cache_disabled):
response = await self._session.post(url, headers=headers, data=data, json=json, **request_params)
@@ -331,7 +344,7 @@ class ScraperClient:
return await response.read()
@copy_signature(_post_data)
- async def post_data(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+ async def post_data(self, *args: Any, **kwargs: Any) -> Any:
content = await self.post_data_raw(*args, **kwargs)
return json_loads(content)
@@ -341,9 +354,9 @@ class ScraperClient:
domain: str,
url: URL,
headers: dict[str, str] | None = None,
+ request_params: dict[str, Any] | None = None,
*,
cache_disabled: bool = False,
- **request_params: Any,
) -> CIMultiDictProxy[str]:
"""
Makes a GET request to an URL and returns its headers
@@ -351,9 +364,10 @@ class ScraperClient:
:param domain: The crawler's domain (for rate limiting)
:param url: The URL to fetch.
:param headers: Optional headers to include in the request. Will be added to session's default headers.
+ :param request_params: Additional keyword arguments to pass to `session.head` (e.g., `timeout`).
:param cache_disabled: Whether to disable caching for this request. Defaults to `False`.
- :param **request_params: Additional keyword arguments to pass to `session.head` (e.g., `timeout`).
"""
+ request_params = request_params or {}
headers = self._headers | {"Accept-Encoding": "identity"} | (headers or {})
async with cache_control_manager(self._session, disabled=cache_disabled):
response = await self._session.head(url, headers=headers, **request_params)
@@ -411,12 +425,12 @@ async def response_to_soup(response: AnyResponse | CurlResponse) -> BeautifulSou
return BeautifulSoup(content, "html.parser")
-async def response_to_json(response: AnyResponse) -> dict[str, Any]:
+async def response_to_json(response: AnyResponse) -> Any:
content_type: str = response.headers["Content-Type"].lower()
if "text/plain" in content_type:
return json_loads(await response.text())
elif "json" in content_type:
- return await response.json() or {}
+ return await response.json()
else:
raise InvalidContentTypeError(message=f"Received {content_type}, was expecting JSON")
diff --git a/cyberdrop_dl/crawlers/_chevereto.py b/cyberdrop_dl/crawlers/_chevereto.py
index 8bfe0364c..55df64960 100644
--- a/cyberdrop_dl/crawlers/_chevereto.py
+++ b/cyberdrop_dl/crawlers/_chevereto.py
@@ -123,7 +123,7 @@ class CheveretoCrawler(Crawler):
if PASSWORD_PROTECTED in soup.text and password:
data = {"content-password": password}
async with self.request_limiter:
- html = await self.client.post_data(self.domain, url, data=data, raw=True)
+ html = await self.client.post_data_raw(self.domain, url, data=data)
soup = BeautifulSoup(html, "html.parser")
if PASSWORD_PROTECTED in soup.text:
diff --git a/cyberdrop_dl/crawlers/bunkrr.py b/cyberdrop_dl/crawlers/bunkrr.py
index 651e6c291..195bea04c 100644
--- a/cyberdrop_dl/crawlers/bunkrr.py
+++ b/cyberdrop_dl/crawlers/bunkrr.py
@@ -183,7 +183,9 @@ class BunkrrCrawler(Crawler):
link: URL | None = None
soup: BeautifulSoup | None = None
if is_stream_redirect(scrape_item.url):
- soup, scrape_item.url = await self.client.get_soup_and_return_url(self.domain, scrape_item.url)
+ response, soup = await self.client._get_response_and_soup(self.domain, scrape_item.url)
+ scrape_item.url = response.url
+ del response
database_url = scrape_item.url.with_host(self.DATABASE_PRIMARY_HOST)
if await self.check_complete_from_referer(database_url):
@@ -285,7 +287,7 @@ class BunkrrCrawler(Crawler):
data = json.dumps(data_dict)
async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers_inc=headers)
+ json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers=headers)
api_response = ApiResponse(**json_resp)
link_str = decrypt_api_response(api_response)
diff --git a/cyberdrop_dl/crawlers/cyberdrop.py b/cyberdrop_dl/crawlers/cyberdrop.py
index c92fa1579..da8693037 100644
--- a/cyberdrop_dl/crawlers/cyberdrop.py
+++ b/cyberdrop_dl/crawlers/cyberdrop.py
@@ -99,5 +99,5 @@ class CyberdropCrawler(Crawler):
return url
if url.host.count(".") > 1 or "e" in url.parts:
return self.primary_base_domain / "f" / url.name
- _, streaming_url = await self.client.get_soup_and_return_url(self.domain, url)
- return streaming_url
+ response, _ = await self.client._get_response_and_soup(self.domain, url)
+ return response.url
diff --git a/cyberdrop_dl/crawlers/cyberfile.py b/cyberdrop_dl/crawlers/cyberfile.py
index 694d3c824..fcd8b96e8 100644
--- a/cyberdrop_dl/crawlers/cyberfile.py
+++ b/cyberdrop_dl/crawlers/cyberfile.py
@@ -141,10 +141,10 @@ class CyberfileCrawler(Crawler):
if not form:
raise PasswordProtectedError("Unable to parse Password Protected File details")
- password_post_url = form["action"]
+ password_post_url = self.parse_url(form["action"]) # type: ignore
data = {"filePassword": password, "submitme": 1}
async with self.request_limiter:
- resp_bytes = await self.client.post_data(self.domain, password_post_url, data=data, raw=True)
+ resp_bytes = await self.client.post_data_raw(self.domain, password_post_url, data=data)
soup = BeautifulSoup(resp_bytes, "html.parser")
if is_password_protected(soup):
diff --git a/cyberdrop_dl/crawlers/dropbox.py b/cyberdrop_dl/crawlers/dropbox.py
index 62ed85fbf..2c0344107 100644
--- a/cyberdrop_dl/crawlers/dropbox.py
+++ b/cyberdrop_dl/crawlers/dropbox.py
@@ -11,6 +11,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headers
if TYPE_CHECKING:
+ from collections.abc import Mapping
+
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.utils.data_enums_classes.url_objects import ScrapeItem
@@ -27,9 +29,10 @@ class DropboxCrawler(Crawler):
@create_task_id
async def fetch(self, scrape_item: ScrapeItem) -> None:
"""Determines where to send the scrape item based on the url."""
- scrape_item.url = await self.get_share_url(scrape_item)
- if not scrape_item.url:
+ url = await self.get_share_url(scrape_item)
+ if not url:
return
+ scrape_item.url = url
if any(p in scrape_item.url.path for p in ("/scl/fi/", "/scl/fo/")):
return await self.file(scrape_item)
@@ -64,14 +67,14 @@ class DropboxCrawler(Crawler):
async def get_folder_name(self, url: URL) -> str | None:
url = await self.get_redict_url(url)
async with self.request_limiter:
- headers: dict = await self.client.get_head(self.domain, url)
+ headers = await self.client.get_head(self.domain, url)
if not ("Content-Disposition" in headers and not is_html(headers)):
raise ScrapeError(422)
return get_filename_from_headers(headers)
async def get_redict_url(self, url: URL) -> URL:
async with self.request_limiter:
- headers: dict = await self.client.get_head(self.domain, url)
+ headers = await self.client.get_head(self.domain, url)
location = headers.get("location")
if not location:
raise ScrapeError(400)
@@ -138,6 +141,6 @@ def get_item_info(url: URL) -> DropboxItem:
return DropboxItem(file_id, folder_tokens, url, rlkey, filename)
-def is_html(headers: dict) -> bool:
+def is_html(headers: Mapping[str, str]) -> bool:
content_type: str = headers.get("Content-Type", "").lower()
return any(s in content_type for s in ("html", "text"))
diff --git a/cyberdrop_dl/crawlers/e621.py b/cyberdrop_dl/crawlers/e621.py
index 5896244cd..d1bebe137 100644
--- a/cyberdrop_dl/crawlers/e621.py
+++ b/cyberdrop_dl/crawlers/e621.py
@@ -46,7 +46,7 @@ class E621Crawler(Crawler):
for page in itertools.count(initial_page):
url = url.with_query(tags=scrape_item.url.query["tags"], page=page)
async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.domain, url, headers_inc=self.custom_headers)
+ json_resp: dict = await self.client.get_json(self.domain, url, headers=self.custom_headers)
posts: list[dict] = json_resp.get("posts", [])
if not posts:
@@ -79,7 +79,7 @@ class E621Crawler(Crawler):
pool_id = scrape_item.url.name
url = self.primary_base_domain / f"pools/{pool_id}.json"
async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.domain, url, headers_inc=self.custom_headers)
+ json_resp: dict = await self.client.get_json(self.domain, url, headers=self.custom_headers)
posts = json_resp.get("post_ids", [])
title: str = json_resp.get("name", "Unknown Pool").replace("_", " ")
@@ -97,7 +97,7 @@ class E621Crawler(Crawler):
post_id = scrape_item.url.name
url = self.primary_base_domain / f"posts/{post_id}.json"
async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.domain, url, headers_inc=self.custom_headers)
+ json_resp: dict = await self.client.get_json(self.domain, url, headers=self.custom_headers)
try:
file_url = json_resp["post"]["file"]["url"]
diff --git a/cyberdrop_dl/crawlers/generic.py b/cyberdrop_dl/crawlers/generic.py
index 06cb6d2aa..cc977ee59 100644
--- a/cyberdrop_dl/crawlers/generic.py
+++ b/cyberdrop_dl/crawlers/generic.py
@@ -87,7 +87,7 @@ class GenericCrawler(Crawler):
async def get_content_type(self, url: URL) -> str:
async with self.request_limiter:
- headers: dict = await self.client.get_head(self.domain, url)
+ headers = await self.client.get_head(self.domain, url)
content_type: str = headers.get("Content-Type", "")
if not content_type:
raise ScrapeError(422)
diff --git a/cyberdrop_dl/crawlers/gofile.py b/cyberdrop_dl/crawlers/gofile.py
index 750e2b48e..0c7d72cfd 100644
--- a/cyberdrop_dl/crawlers/gofile.py
+++ b/cyberdrop_dl/crawlers/gofile.py
@@ -156,7 +156,7 @@ class GoFileCrawler(Crawler):
async def _update_website_token(self) -> None:
async with self.request_limiter:
- text = await self.client.get_text(self.domain, GLOBAL_JS_URL, origin=GLOBAL_JS_URL)
+ text = await self.client.get_text(self.domain, GLOBAL_JS_URL)
match = re.search(WT_REGEX, str(text))
if not match:
raise ScrapeError(401, "Couldn't generate GoFile websiteToken", origin=GLOBAL_JS_URL)
diff --git a/cyberdrop_dl/crawlers/google_drive.py b/cyberdrop_dl/crawlers/google_drive.py
index 8a62dd9a6..23a2b0a82 100644
--- a/cyberdrop_dl/crawlers/google_drive.py
+++ b/cyberdrop_dl/crawlers/google_drive.py
@@ -43,6 +43,8 @@ from cyberdrop_dl.crawlers.crawler import Crawler, create_task_id
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_filename_from_headers
if TYPE_CHECKING:
+ from collections.abc import Mapping
+
from bs4 import BeautifulSoup
from cyberdrop_dl.managers.manager import Manager
@@ -138,7 +140,7 @@ class GoogleDriveCrawler(Crawler):
filename, ext = self.get_filename_and_ext(filename)
await self.handle_file(canonical_url, scrape_item, filename, ext, debrid_link=link)
- async def get_file_url_and_headers(self, url: URL, file_id: str) -> tuple[URL, dict]:
+ async def get_file_url_and_headers(self, url: URL, file_id: str) -> tuple[URL, Mapping[str, str]]:
soup = last_error = None
current_url: URL | None = url
try_file_open_url = True
@@ -151,7 +153,9 @@ class GoogleDriveCrawler(Crawler):
try:
async with self.request_limiter:
- soup, headers = await self.client.get_soup(self.domain, current_url, with_response_headers=True)
+ response, soup = await self.client._get_response_and_soup(self.domain, current_url)
+ headers = response.headers
+
except DownloadError as e:
last_error = e
if e.status == 500 and try_file_open_url:
@@ -176,9 +180,9 @@ class GoogleDriveCrawler(Crawler):
raise ScrapeError(422)
- async def add_headers(self, url: URL) -> tuple[URL, dict]:
+ async def add_headers(self, url: URL) -> tuple[URL, Mapping[str, str]]:
async with self.request_limiter:
- headers: dict = await self.client.get_head(self.domain, url)
+ headers = await self.client.get_head(self.domain, url)
location = headers.get("location")
if location:
link = self.parse_url(location)
@@ -264,11 +268,11 @@ def get_download_url(item_id: str, folder: bool = False) -> URL:
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-def are_valid_headers(headers: dict):
+def are_valid_headers(headers: Mapping[str, str]):
return "Content-Disposition" in headers and not is_html(headers)
-def is_html(headers: dict) -> bool:
+def is_html(headers: Mapping[str, str]) -> bool:
content_type: str = headers.get("Content-Type", "").lower()
return any(s in content_type for s in ("html", "text"))
diff --git a/cyberdrop_dl/crawlers/imgur.py b/cyberdrop_dl/crawlers/imgur.py
index 3e74e38d7..591ba92b0 100644
--- a/cyberdrop_dl/crawlers/imgur.py
+++ b/cyberdrop_dl/crawlers/imgur.py
@@ -56,7 +56,7 @@ class ImgurCrawler(Crawler):
async with self.request_limiter:
await self.check_imgur_credits(scrape_item)
api_url = API_ENTRYPOINT / "album" / album_id
- json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)
+ json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers=self.headers)
title_part = json_resp["data"].get("title", album_id)
title = self.create_title(title_part, album_id)
@@ -64,7 +64,7 @@ class ImgurCrawler(Crawler):
async with self.request_limiter:
api_url = API_ENTRYPOINT / "album" / album_id / "images"
- json_resp = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)
+ json_resp = await self.client.get_json(self.domain, api_url, headers=self.headers)
for image in json_resp["data"]:
await self.process_image(scrape_item, image)
@@ -80,7 +80,7 @@ class ImgurCrawler(Crawler):
async with self.request_limiter:
await self.check_imgur_credits(scrape_item)
api_url = API_ENTRYPOINT / "image" / image_id
- json_resp = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)
+ json_resp = await self.client.get_json(self.domain, api_url, headers=self.headers)
await self.process_image(scrape_item, json_resp["data"])
@@ -108,7 +108,7 @@ class ImgurCrawler(Crawler):
async def check_imgur_credits(self, _=None) -> None:
"""Checks the remaining credits."""
credits_url = API_ENTRYPOINT / "credits"
- json_resp = await self.client.get_json(self.domain, credits_url, headers_inc=self.headers)
+ json_resp = await self.client.get_json(self.domain, credits_url, headers=self.headers)
self.imgur_client_remaining = json_resp["data"]["ClientRemaining"]
if self.imgur_client_remaining < 100:
raise ScrapeError(429, "Imgur API rate limit reached")
diff --git a/cyberdrop_dl/crawlers/jpg5.py b/cyberdrop_dl/crawlers/jpg5.py
index a891cf614..d9fc17e4d 100644
--- a/cyberdrop_dl/crawlers/jpg5.py
+++ b/cyberdrop_dl/crawlers/jpg5.py
@@ -70,4 +70,4 @@ class JPG5Crawler(CheveretoCrawler):
"""Handles a direct link."""
link = url or scrape_item.url
link = self.parse_url(re.sub(JPG5_REPLACE_HOST_REGEX, r"host.church/", str(link)))
- super().handle_direct_link(scrape_item, link)
+ await super().handle_direct_link(scrape_item, link)
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.py
index 308b89cb2..4387eb7e5 100644
--- a/cyberdrop_dl/crawlers/kemono.py
+++ b/cyberdrop_dl/crawlers/kemono.py
@@ -356,7 +356,7 @@ class KemonoCrawler(Crawler):
query_url = api_url.with_query(type="post" if is_post else "artist")
async with self.request_limiter:
- json_resp = await self.client.get_json(self.domain, query_url)
+ json_resp: list[dict] = await self.client.get_json(self.domain, query_url)
self.update_cookies({"session": ""})
diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.py
index 8da0261ea..f010cb02d 100644
--- a/cyberdrop_dl/crawlers/luscious.py
+++ b/cyberdrop_dl/crawlers/luscious.py
@@ -123,6 +123,6 @@ class LusciousCrawler(Crawler):
api_url = GRAPHQL_URL.with_query(operationName=query_name)
headers = {"Content-Type": "application/json"}
async with self.request_limiter:
- json_resp = await self.client.post_data(self.domain, api_url, data=query, headers_inc=headers)
+ json_resp = await self.client.post_data(self.domain, api_url, data=query, headers=headers)
log_debug(json_resp)
return json_resp
diff --git a/cyberdrop_dl/crawlers/onedrive.py b/cyberdrop_dl/crawlers/onedrive.py
index 01906ff61..9b157a252 100644
--- a/cyberdrop_dl/crawlers/onedrive.py
+++ b/cyberdrop_dl/crawlers/onedrive.py
@@ -130,7 +130,7 @@ class OneDriveCrawler(Crawler):
@error_handling_wrapper
async def share_link(self, scrape_item: ScrapeItem) -> None:
async with self.request_limiter:
- headers: dict = await self.client.get_head(self.domain, scrape_item.url)
+ headers = await self.client.get_head(self.domain, scrape_item.url)
location = headers.get("location")
if not location:
raise ScrapeError(400)
@@ -210,7 +210,7 @@ class OneDriveCrawler(Crawler):
async def make_api_request(self, api_url: URL) -> dict[str, Any]:
headers = {"Content-Type": "application/json"} | self.auth_headers
async with self.request_limiter:
- json_resp: dict = await self.client.get_json(self.domain, api_url, headers_inc=headers)
+ json_resp: dict = await self.client.get_json(self.domain, api_url, headers=headers)
return json_resp
@@ -220,9 +220,7 @@ class OneDriveCrawler(Crawler):
data = {"appId": APP_UUID}
data_json = json.dumps(data)
async with self.request_limiter:
- json_resp: dict = await self.client.post_data(
- self.domain, badger_url, headers_inc=new_headers, data=data_json
- )
+ json_resp: dict = await self.client.post_data(self.domain, badger_url, headers=new_headers, data=data_json)
badger_token: str = json_resp["token"]
badger_token_expires: str = json_resp["expiryTimeUtc"]
diff --git a/cyberdrop_dl/crawlers/pmvhaven.py b/cyberdrop_dl/crawlers/pmvhaven.py
index f5f87df96..e4d0526da 100644
--- a/cyberdrop_dl/crawlers/pmvhaven.py
+++ b/cyberdrop_dl/crawlers/pmvhaven.py
@@ -78,7 +78,7 @@ class PMVHavenCrawler(Crawler):
add_headers = {"Content-Type": "text/plain;charset=UTF-8"}
add_data = json.dumps({"profile": username, "mode": "GetUser"})
async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.domain, api_url, data=add_data, headers_inc=add_headers)
+ json_resp: dict = await self.client.post_data(self.domain, api_url, data=add_data, headers=add_headers)
user_info: dict[str, dict] = json_resp["data"]
for playlist in user_info["playlists"]:
@@ -223,7 +223,7 @@ class PMVHavenCrawler(Crawler):
if is_profile:
data = json.dumps(data)
async with self.request_limiter:
- json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers_inc=add_headers)
+ json_resp: dict = await self.client.post_data(self.domain, api_url, data=data, headers=add_headers)
has_videos = bool(json_resp[check_key])
if not has_videos:
diff --git a/cyberdrop_dl/crawlers/reddit.py b/cyberdrop_dl/crawlers/reddit.py
index 4d05f4119..c697aafc7 100644
--- a/cyberdrop_dl/crawlers/reddit.py
+++ b/cyberdrop_dl/crawlers/reddit.py
@@ -187,11 +187,11 @@ class RedditCrawler(Crawler):
await self.handle_file(url, scrape_item, filename, ext)
async def get_final_location(self, url) -> URL:
- headers: dict[str, str] = await self.client.get_head(self.domain, url)
+ headers = await self.client.get_head(self.domain, url)
content_type = headers.get("Content-Type", "")
if any(s in content_type.lower() for s in ("html", "text")):
- _, url = await self.client.get_soup_and_return_url(self.domain, url)
- return url
+ response, _ = await self.client._get_response_and_soup(self.domain, url)
+ return response.url
location = headers.get("location")
if not location:
raise ScrapeError(422)
diff --git a/cyberdrop_dl/crawlers/redgifs.py b/cyberdrop_dl/crawlers/redgifs.py
index 3d4898ebe..e9d2963bb 100644
--- a/cyberdrop_dl/crawlers/redgifs.py
+++ b/cyberdrop_dl/crawlers/redgifs.py
@@ -63,7 +63,7 @@ class RedGifsCrawler(Crawler):
async with self.request_limiter:
api_url = API_ENTRYPOINT / "v2/users" / user_id / "search"
api_url = api_url.with_query(order="new", page=page)
- json_resp = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)
+ json_resp = await self.client.get_json(self.domain, api_url, headers=self.headers)
gifs = json_resp["gifs"]
yield json_resp
if total_gifs is None:
@@ -78,7 +78,7 @@ class RedGifsCrawler(Crawler):
post_id = scrape_item.url.parts[-1].split(".")[0]
async with self.request_limiter:
api_url = API_ENTRYPOINT / "v2/gifs" / post_id
- json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers_inc=self.headers)
+ json_resp: dict[str, dict] = await self.client.get_json(self.domain, api_url, headers=self.headers)
title_part: str = json_resp["gif"].get("title") or "Loose Files"
title = self.create_title(title_part)
diff --git a/cyberdrop_dl/crawlers/send_now.py b/cyberdrop_dl/crawlers/send_now.py
index 753f802b7..938a4b2f8 100644
--- a/cyberdrop_dl/crawlers/send_now.py
+++ b/cyberdrop_dl/crawlers/send_now.py
@@ -32,10 +32,11 @@ class SendNowCrawler(Crawler):
async def file(self, scrape_item: ScrapeItem) -> None:
file_id = scrape_item.url.name
data = {"op": "download2", "id": file_id, "rand": "", "referer": "", "method_free": "", "method_premium": ""}
+ params = {"allow_redirects": False, "stream": True}
async with self.request_limiter:
response: CurlResponse = await self.client.post_data_cffi(
- self.domain, scrape_item.url, data=data, allow_redirects=False, stream=True
+ self.domain, scrape_item.url, data=data, request_params=params
)
debrid_link = self.parse_url(response.headers.get("location"))
diff --git a/cyberdrop_dl/crawlers/sex_dot_com.py b/cyberdrop_dl/crawlers/sex_dot_com.py
index a9f11eb59..4df17f2a4 100644
--- a/cyberdrop_dl/crawlers/sex_dot_com.py
+++ b/cyberdrop_dl/crawlers/sex_dot_com.py
@@ -58,8 +58,7 @@ class SexDotComCrawler(Crawler):
break
page += 1
- @error_handling_wrapper
- async def get_media(self, scrape_item: ScrapeItem) -> dict:
+ async def get_media(self, scrape_item: ScrapeItem) -> dict[str, Any]:
"""Gets media from its relative URL."""
relative_url = "/".join(scrape_item.url.parts[3:])
data_url = API_URL / "media" / "getMedia"
@@ -70,22 +69,22 @@ class SexDotComCrawler(Crawler):
@error_handling_wrapper
async def handle_media(self, scrape_item: ScrapeItem, item: dict[str, Any] | None) -> None:
- item = item or await self.get_media(scrape_item)
- relative_url = item["relativeUrl"]
+ real_item = item or await self.get_media(scrape_item)
+ relative_url = real_item["relativeUrl"]
canonical_url = URL("https://sex.com/en/shorts") / relative_url
if await self.check_complete_from_referer(canonical_url):
return
- fileType: str = item.get("fileType") or item["mediaType"]
+ fileType: str = real_item.get("fileType") or real_item["mediaType"]
if fileType.startswith("image"):
- media_url = URL(item["fullPath"]).with_query(optimizer="image", width=1200)
- filename, ext = f"{item['pictureUid']}.jpg", "jpg"
+ media_url = URL(real_item["fullPath"]).with_query(optimizer="image", width=1200)
+ filename, ext = f"{real_item['pictureUid']}.jpg", "jpg"
elif fileType.startswith("video"):
- media_url = URL(item["sources"][0]["fullPath"])
+ media_url = URL(real_item["sources"][0]["fullPath"])
filename, ext = self.get_filename_and_ext(media_url.name)
- scrape_item.possible_datetime = self.parse_datetime(item["createdAt"])
+ scrape_item.possible_datetime = self.parse_datetime(real_item["createdAt"])
scrape_item.url = canonical_url
await self.handle_file(media_url, scrape_item, filename, ext)
scrape_item.add_children()
diff --git a/cyberdrop_dl/crawlers/wetransfer.py b/cyberdrop_dl/crawlers/wetransfer.py
index 45947c460..22c0897de 100644
--- a/cyberdrop_dl/crawlers/wetransfer.py
+++ b/cyberdrop_dl/crawlers/wetransfer.py
@@ -46,7 +46,7 @@ class WeTransferCrawler(Crawler):
headers = {"Content-Type": "application/json"}
async with self.request_limiter:
json_resp: dict = await self.client.post_data(
- self.domain, file_info.download_url, data=file_info.json, headers_inc=headers
+ self.domain, file_info.download_url, data=file_info.json, headers=headers
)
link_str: str = json_resp.get("direct_link") # type: ignore
@@ -64,7 +64,7 @@ class WeTransferCrawler(Crawler):
async def get_final_url(self, scrape_item: ScrapeItem) -> URL:
async with self.request_limiter:
- headers: dict = await self.client.get_head(self.domain, scrape_item.url)
+ headers = await self.client.get_head(self.domain, scrape_item.url)
location: str = headers.get("location") # type: ignore
return self.parse_url(location)
diff --git a/cyberdrop_dl/crawlers/xenforo/celebforum.py b/cyberdrop_dl/crawlers/xenforo/celebforum.py
index 4ca337f94..a4756884c 100644
--- a/cyberdrop_dl/crawlers/xenforo/celebforum.py
+++ b/cyberdrop_dl/crawlers/xenforo/celebforum.py
@@ -22,7 +22,7 @@ class CelebForumCrawler(XenforoCrawler):
def __init__(self, manager: Manager) -> None:
super().__init__(manager, self.domain, "CelebForum")
- def filter_link(self, link):
+ def filter_link(self, link: URL) -> URL | None:
if link.host == self.primary_base_domain.host:
if all(part in link.parts for part in ["data", "attachments"]): # Thumbnails
return None
diff --git a/cyberdrop_dl/crawlers/xenforo/f95zone.py b/cyberdrop_dl/crawlers/xenforo/f95zone.py
index 2d4e42f61..f84c7864e 100644
--- a/cyberdrop_dl/crawlers/xenforo/f95zone.py
+++ b/cyberdrop_dl/crawlers/xenforo/f95zone.py
@@ -33,7 +33,7 @@ class F95ZoneCrawler(XenforoCrawler):
"""Override to handle protected link confirmation."""
async with self.request_limiter:
data = ({"xhr": "1", "download": "1"},)
- JSON_Resp = await self.client.post_data(self.domain, link, data=data, origin=origin)
+ JSON_Resp = await self.client.post_data(self.domain, link, data=data)
if JSON_Resp["status"] == "ok":
return self.parse_url(JSON_Resp["msg"])
diff --git a/cyberdrop_dl/crawlers/xenforo/xenforo.py b/cyberdrop_dl/crawlers/xenforo/xenforo.py
index d95f7fd53..bd1f69e2a 100644
--- a/cyberdrop_dl/crawlers/xenforo/xenforo.py
+++ b/cyberdrop_dl/crawlers/xenforo/xenforo.py
@@ -153,9 +153,10 @@ class XenforoCrawler(Crawler):
if self.thread_url_part in scrape_item.url.parts:
return await self.thread(scrape_item)
if self.is_confirmation_link(scrape_item.url):
- scrape_item.url = await self.handle_confirmation_link(scrape_item.url)
- if scrape_item.url: # If there was an error, this will be None
+ url = await self.handle_confirmation_link(scrape_item.url)
+ if url: # If there was an error, this will be None
# This could end up back in here if the URL goes to another thread
+ scrape_item.url = url
return self.handle_external_links(scrape_item)
if any(p in scrape_item.url.parts for p in ("goto", "posts")):
return await self.redirect(scrape_item)
@@ -384,7 +385,7 @@ class XenforoCrawler(Crawler):
async def handle_confirmation_link(self, link: URL, *, origin: ScrapeItem | None = None) -> URL | None:
"""Handles link confirmation."""
async with self.request_limiter:
- soup: BeautifulSoup = await self.client.get_soup(self.domain, link, origin=origin)
+ soup: BeautifulSoup = await self.client.get_soup(self.domain, link)
confirm_button = soup.select_one("a[class*=button--cta]")
if not confirm_button:
return
@@ -500,9 +501,7 @@ class XenforoCrawler(Crawler):
attempt += 1
await asyncio.sleep(wait_time)
data = prepare_login_data(text)
- await self.client.post_data(
- self.domain, login_url / "login", data=data, req_resp=False, cache_disabled=True
- )
+ _ = await self.client._post_data(self.domain, login_url / "login", data=data, cache_disabled=True)
await asyncio.sleep(wait_time)
text, logged_in = await self.check_login_with_request(login_url)
if logged_in:
diff --git a/cyberdrop_dl/crawlers/yandex_disk.py b/cyberdrop_dl/crawlers/yandex_disk.py
index e6bb091ef..ed48d56f9 100644
--- a/cyberdrop_dl/crawlers/yandex_disk.py
+++ b/cyberdrop_dl/crawlers/yandex_disk.py
@@ -97,7 +97,7 @@ class YandexDiskCrawler(Crawler):
}
async with self.request_limiter:
json_resp: dict[str, Any] = await self.client.post_data(
- self.domain, DOWNLOAD_API_ENTRYPOINT, data=file.post_data, headers_inc=headers
+ self.domain, DOWNLOAD_API_ENTRYPOINT, data=file.post_data, headers=headers
)
new_sk = json_resp.get("new_sk")